import streamlit as st

from torchtyping import TensorType as TT
from transformer_lens.hook_points import HookPoint
from .environment import get_action_preds
from .visualizations import plot_action_preds, plot_single_residual_stream_contributions, plot_single_residual_stream_contributions_comparison
from .analysis import get_residual_decomp

def show_ablation(dt, logit_dir, original_cache):

    with st.expander("Ablation Experiment"):
        st.write("ablation experiment here")
        # list out the components of the transformer

        # make a streamlit form for choosing a component to ablate
        n_layers = dt.n_layers
        n_heads = dt.n_heads

        columns = st.columns(4)
        with columns[0]:
            layer = st.selectbox("Layer", list(range(n_layers)))
        with columns[1]:
            component = st.selectbox("Component", ["MLP", "HEAD"], index=1)
        with columns[2]:
            if component == "HEAD":
                head = st.selectbox("Head", list(range(n_heads)))
        with columns[3]:
            ablate_to_mean = st.checkbox("Ablate to mean", value=True)

        if component == "HEAD":
            ablation_func = get_ablation_function(ablate_to_mean, head)
            dt.transformer.blocks[layer].attn.hook_z.add_hook(ablation_func)
        elif component == "MLP":
            ablation_func = get_ablation_function(
                ablate_to_mean, layer, component="MLP")
            dt.transformer.blocks[layer].hook_mlp_out.add_hook(ablation_func)

        action_preds, x, cache, tokens = get_action_preds(dt)

        if st.checkbox("show action predictions"):
            plot_action_preds(action_preds)
        if st.checkbox("show counterfactual residual contributions"):
            original_residual_decomp = get_residual_decomp(
                dt, original_cache, logit_dir)
            ablation_residual_decomp = get_residual_decomp(
                dt, cache, logit_dir)
            plot_single_residual_stream_contributions_comparison(
                original_residual_decomp, ablation_residual_decomp)

    # then, render a single residual stream contribution with the ablation

def get_ablation_function(ablate_to_mean, head_to_ablate, component="HEAD"):

    def head_ablation_hook(
        value: TT["batch", "pos", "head_index", "d_head"],  # noqa: F821
        hook: HookPoint
    ) -> TT["batch", "pos", "head_index", "d_head"]:  # noqa: F821
        print(f"Shape of the value tensor: {value.shape}")

        if ablate_to_mean:
            value[:, :, head_to_ablate, :] = value[:, :,
                                                   head_to_ablate, :].mean(dim=2, keepdim=True)
        else:
            value[:, :, head_to_ablate, :] = 0.
        return value

    def mlp_ablation_hook(
        value: TT["batch", "pos", "d_model"],  # noqa: F821
        hook: HookPoint
    ) -> TT["batch", "pos", "d_model"]:  # noqa: F821
        print(f"Shape of the value tensor: {value.shape}")

        if ablate_to_mean:
            value[:, :-1, :] = value[:, :-1, :].mean(dim=2, keepdim=True)
        else:
            value[:, :-1, :] = 0  # ablate all but the last token
        return value

    if component == "HEAD":
        return head_ablation_hook
    elif component == "MLP":
        return mlp_ablation_hook


import json
import os

file_path = os.path.join('bananafashion/input_data/womendata.json')
with open(file_path) as f:
    women_new_arrival = json.load(f)

file_path = os.path.join('bananafashion/input_data/homedata.json')
with open(file_path) as f:
    home = json.load(f)

file_path = os.path.join('bananafashion/input_data/workwear.json')
with open(file_path) as f:
    workwear = json.load(f)
#print(women_new_arrival)

# k = data['productCategoryFacetedSearch']['productCategory']['childCategories'][0]['childProducts']
# items = []
# for item in k:
#     items.append(item['businessCatalogItemId'])
# #print(items)

# k2 =  data['productCategoryFacetedSearch']['productCategory']['childCategories'][0]['childProducts']
# items_2  = []
# for item in k2:
#     items_2.append(item['parentStyleId'])
# #print(items_2)


import re
from copy import deepcopy
from tqdm import tqdm

# split sentences into lines without meta
def get_lines(sent, ans_sent):
    meta_lines = re.search('^(#.*\n)*', sent).group()
    no_meta_sent = sent[len(meta_lines):]
    lines = no_meta_sent.split('\n')
    meta_lines_ans = re.search('^(#.*\n)*', ans_sent).group()
    no_meta_ans_sent = ans_sent[len(meta_lines_ans):]
    ans_lines = no_meta_ans_sent.split('\n')
    return lines, ans_lines, meta_lines

# match predicted token indices to one gold token index
def match_pred_tokens_to_gold(lines, ans_lines):
    ind_matches = {}
    i = 0
    num_of_skipped = 0
    for j in range(len(ans_lines)):
        # если равны, добавляем индекс-сопоставление, идем дальше
        try:
            if ans_lines[j].split('\t')[1] == lines[i].split('\t')[1]:
                ind_matches[ans_lines[j].split('\t')[0]] = [lines[i].split('\t')[0]]
                i += 1
                continue
        except:
            print('\n'.join(ans_lines))
            print('\n'.join(lines))
        # если не равны, но начинается с того же символа или если это [UNK], пропускаем
        # ЧТО ЭТО И ЗАЧЕМ?
        # if not ans_lines[j].split('\t')[1].startswith(lines[i].split('\t')[1]) and lines[i].split('\t')[1] != '[UNK]':
        #     num_of_skipped += 1
        #     print(ans_lines[j], lines[i])
        #     continue
        # в остальных случаях full_token - текущий токен (ответы), откусываем от него длину текущего токена (пред)
        full_token = ans_lines[j].split('\t')[1]
        full_token = full_token[len(lines[i].split('\t')[1]):]
        indices = [lines[i].split('\t')[0]]
        for line in lines[i+1:]:  # идем дальше по (пред) линиям - пока начала совпадают - пропускаем
            # !NEW! Added full_token != ''
            if full_token.startswith(line.split('\t')[1]) or line.split('\t')[1] == '[UNK]' and full_token != '':
                if line.split('\t')[1] != '[UNK]':
                    full_token = full_token[len(line.split('\t')[1]):]
                else:
                    full_token = full_token[1:]  # assumption that [UNK] is always only 1 symbol in length
                i += 1
                indices.append(line.split('\t')[0])
            else:
                break
        ind_matches[ans_lines[j].split('\t')[0]] = indices
        i += 1
    return ind_matches, num_of_skipped

# collect all tokens matched to one original-token in one list and in groups
def get_additional_lists(ind_matches):
    all_misplaced = []
    misplaced_groups = []
    for key, value in ind_matches.items():
        if len(value) > 1:
            all_misplaced.extend(value)
            misplaced_groups.append(value)
    return all_misplaced, misplaced_groups

# create our predict with fixed tokenization (only enh deps will be used from here)
def match_ud_tokenization(lines, all_misplaced, misplaced_groups):
    group_ind = 0
    new_lines = []
    i = 0
    while i < len(lines):
        if str(i+1) in all_misplaced:
            begin = misplaced_groups[group_ind][0]
            end = misplaced_groups[group_ind][-1]
            full_token = ''
            head_candidates = []
            for line in lines[int(begin)-1:int(end)]:
                i += 1
                dep_nums = [dep.split(':')[0] for dep in line.split('\t')[8].split('|')]
                for num in dep_nums:
                    if num not in misplaced_groups[group_ind] and line.split('\t')[8] not in head_candidates:
                        head_candidates.append(line.split('\t')[8])
                full_token += line.split('\t')[1]
            format_line = lines[int(end)-1]
            parts = format_line.split('\t')
            parts[1] = full_token
            parts[2] = full_token
            if len(head_candidates) == 0:
                head_candidates.append('0:X')
                print('no head candidates')
            parts[8] = head_candidates[0]  # simply chooses first candidate - place for improvement
            new_line = '\t'.join(parts)
            new_lines.append(new_line)
            group_ind += 1
        else:
            new_lines.append(lines[i])
            i += 1
    return new_lines

def fix_indices(new_lines):
    fixed_lines = []
    for i in range(len(new_lines)):
        parts = new_lines[i].split('\t')
        parts[0] = str(i+1)
        fixed_lines.append('\t'.join(parts))
    return fixed_lines

# fix head numbers in enh deps
def fix_dep_numbers(fixed_lines, ind_matches, num_of_skipped):
    new_deps_lines = []
    for line in fixed_lines:
        parts = line.split('\t')
        deps = parts[8].split('|')
        new_deps = []
        for dep in deps:
            num = dep.split(':')[0]
            new_num = num  # !!! NEW NUM MUST BE ALWAYS DEFINED - PROBLEM!!!
            if int(num) < 0:
                new_num = '0'
            for orig_ind in ind_matches:
                if num == '0' or int(num) < 1:
                    new_num = '0'
                    break
                if num in ind_matches[orig_ind]:
                    new_num = str(int(orig_ind) - num_of_skipped)
                    break
            new_deps.append(new_num + ':' + ':'.join(dep.split(':')[1:]))
        new_full_dep = '|'.join(list(set(new_deps)))  # SET: if they start to duplicate - collapse into one
        parts[8] = new_full_dep
        new_deps_lines.append('\t'.join(parts))
    return '\n'.join(new_deps_lines)

def match_if_original_shorter(lines, ans_lines):
    new_lines = deepcopy(lines)
    count_added = 0
    ind_matches = {}
    for i in range(len(lines)):
        try:
            if lines[i].split('\t')[1] == ans_lines[i + count_added].split('\t')[1] or lines[i].split('\t')[1] == '[UNK]':
                ind_matches[ans_lines[i + count_added].split('\t')[0]] = [lines[i].split('\t')[0]]
                continue
            if not ans_lines[i + count_added].split('\t')[1].startswith(lines[i].split('\t')[1]):
                j = 0
                for ans_line in ans_lines[i + count_added + 1:]:
                    if ans_line.split('\t')[1] == lines[i].split('\t')[1]:
                        parts = ans_lines[i + count_added +j].split('\t')
                        parts[8] = '0:X'
                        new_lines.insert(i, '\t'.join(parts))
                        count_added += 1
                        ind_matches[ans_lines[i + count_added].split('\t')[0]] = [parts[0]]
                        break
                    j += 1
            else:
                # print(lines)
                print('Something is wrong!')
        except:
            print('\n'.join(lines))
            exit()
    return new_lines, ind_matches

def align_tokenization(ans_sents, sents):
    sents_fixed_tokenization = []
    for i in tqdm(range(len(sents))):
        lines, ans_lines, meta_lines = get_lines(sents[i], ans_sents[i])
        # less in original
        ind_matches, num_skipped = match_pred_tokens_to_gold(lines, ans_lines)
        all_misplaced, misplaced_groups = get_additional_lists(ind_matches)
        new_lines = match_ud_tokenization(lines, all_misplaced, misplaced_groups)
        fixed_lines = fix_indices(new_lines)
        ud_sent = fix_dep_numbers(fixed_lines, ind_matches, num_skipped)
        # more in original
        new_lines, ind_matches = match_if_original_shorter(ud_sent.split('\n'), ans_lines)
        fixed_lines = fix_indices(new_lines)
        ud_sent = fix_dep_numbers(fixed_lines, ind_matches, 0)
        sents_fixed_tokenization.append(meta_lines + ud_sent)
    return sents_fixed_tokenization

def insert_enhanced_in_original(ans_file, predict_file, output_file):
    with open(ans_file, 'r', encoding='utf-8') as f:
        ans_text = f.read()
    with open(predict_file, 'r', encoding='utf-8') as f:
        text = f.read()
    ans_sents = ans_text.split('\n\n')[:-1]
    sents = text.split('\n\n')[:-1]
    new_sents = align_tokenization(ans_sents, sents)
    print('Alignment is done!\nInserting into original...')
    ans_with_pred_deps = []
    for i in tqdm(range(len(new_sents))):
        lines, ans_lines, meta = get_lines(new_sents[i], ans_sents[i])
        if len(lines) != len(ans_lines):
            print('Tokenization align error!')
        for j in range(len(lines)):
            parts = ans_lines[j].split('\t')
            parts[8] = lines[j].split('\t')[8]
            ans_lines[j] = '\t'.join(parts)
        ans_with_pred_deps.append(meta + '\n'.join(ans_lines))
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write('\n\n'.join(ans_with_pred_deps) + '\n\n')


import os
import cv2
import tensorflow as tf
import numpy as np
from DataGenerator import ImageDataGenerator
from DHN import *

batch_size = 100
hash_K = 48
checkpoint_path = os.path.join(os.getcwd(), "tmp\\dhn")

def toBinaryString(binary_like_values):
    numOfImage, bit_length = binary_like_values.shape
    list_string_binary = []
    for i in range(numOfImage):
        str = ''
        for j in range(bit_length):
            str += '0' if binary_like_values[i][j] <= 0 else '1'
        list_string_binary.append(str)
    return list_string_binary

def toString(binary_like_values):
    numOfImage, bit_length = binary_like_values.shape
    list_string_binary = []
    for i in range(numOfImage):
        st = ''
        for j in range(bit_length):
            st += str(binary_like_values[i][j]) + ' '
        list_string_binary.append(st)
    return list_string_binary

def get_hanming(binarystr1, binarystr2):
    L = len(binarystr1)
    dis = 0
    #print(len(binarystr1))
    #print(len(binarystr2))
    for i in range(L):
        if binarystr1[i] != binarystr2[i]:
            dis += 1
    return dis

def get_manhattan(originfeature1, originfeature2):
    if (len(originfeature1) != len(originfeature2)):
        print("Differnet Dimensions!")
        return
    dis = 0
    for i in range(len(originfeature1)):
        dis += abs(originfeature1[i]-originfeature2[i])
    return dis

def file_read(path):
    file_in = open(path, 'r')
    lines = file_in.read()
    ls = lines.split('\n')
    names = []
    features = []
    origin_features = []
    for line in ls:
        tmp = line.split('\t')
        if len(line) < 3: continue
        # print(tmp)
        temp = tmp[0].split(' ')
        ori = []
        for digstr in temp:
            if len(digstr) == 0: continue
            ori.append((float)(digstr))
        origin_features.append(ori)
        # print(origin_features[-1])
        # print(len(origin_features[-1]))
        features.append(tmp[1])
        names.append(tmp[2])
    return names, features, origin_features

def getSecond(x):
    return x[1]

def test_accuracy_dhn(names, features, originfeature):
    tot = 0
    less10 = 0
    for i in range(len(features)):
        cnt = 0
        acc = 0
        index_tmp = []
        print(i)
        for j in range(len(features)):
            #if (get_manhattan(originfeature[i], originfeature[j]) < 50.):
            index_tmp.append( (j, get_manhattan(originfeature[i], originfeature[j]), get_hanming(features[i], features[j])) )

        # print(len(index))

        index_tmp.sort(key=getSecond)
        index = []
        for j in range(len(index_tmp)):
            if j >= 100: break
            index.append( (index_tmp[j][0], index_tmp[j][2]) )
        index.sort(key=getSecond)

        for j in range(len(index)):
            if j >= 10: break
            cnt += 1
            # print(names[index[j][0]])
            if (names[index[j][0]].split('_')[0] == names[i].split('_')[0]):
                acc += 1
        if (cnt < 10):
            less10 += 1
        tot += (float)(acc/cnt)
        print((float)(acc/cnt))
    print(tot/len(features))
    print(less10)

def getHash(path):
    x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3], name='x')
    model = DeepHashingNet(x, 1, hash_K, [])
    fch = model.fch
    print("Reading checkpoints...")
    ckpt = tf.train.get_checkpoint_state(checkpoint_path)
    saver = tf.train.Saver(tf.all_variables())

    sess = tf.InteractiveSession()

    query_generator = ImageDataGenerator(path,
                                         horizontal_flip=False, shuffle=False)

    file_res = open('query_dhn.txt', 'w')
    # sys.stdout = file_res
    if ckpt and ckpt.model_checkpoint_path:
        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
        print(ckpt_name)
        global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
        saver.restore(sess, os.path.join(checkpoint_path, ckpt_name))
        print('Loading success, global_step is %s' % global_step)

        k = 0
        for i in range(query_generator.dataSize // batch_size):
            print(str(i) + "th batch in " + str(query_generator.dataSize // batch_size))
            images, label = query_generator.next_batch(batch_size)
            eval_sess = sess.run(fch, feed_dict={x: images})
            # print(eval_sess)
            w_res = toBinaryString(eval_sess)
            wzy = toString(eval_sess)

            for j in range(batch_size):
                file_res.write(wzy[j] + '\t' + w_res[j] + '\t' + str(query_generator .images[k]) + '\n')
                k+=1

    file_res.close()
    sess.close()

def showimage(name, showname = 'wzy'):
    if os.path.exists(os.path.join('./data/test', name)):
        img = cv2.imread(os.path.join('./data/test', name))
    elif os.path.exists(os.path.join('./data/train', name)):
        img = cv2.imread(os.path.join('./data/train', name))
    elif os.path.exists(os.path.join('./data/query', name)):
        img = cv2.imread(os.path.join('./data/query', name))
    cv2.imshow(showname, img)
    cv2.waitKey()

def query_dhn(names_dataset, features_dataset, originfeature_dataset, names_query, features_query, originfeature_query):
    file_out = open("answer_dhn.txt", "w")
    for i in range(2):
        index_tmp = []
        print(i)
        showimage(names_query[i], str(i))
        for j in range(len(features_dataset)):
            index_tmp.append( (j, get_manhattan(originfeature_query[i], originfeature_dataset[j]), get_hanming(features_query[i], features_dataset[j])) )

        index_tmp.sort(key=getSecond)
        index = []
        for j in range(len(index_tmp)):
            if j >= 100: break
            index.append( (index_tmp[j][0], index_tmp[j][2]) )
        index.sort(key=getSecond)

        file_out.write(names_query[i] + ':')
        index.sort(key=getSecond)
        for j in range(len(features_dataset)):
            if j >= 10: continue
            showimage(names_dataset[index[j][0]], str(i)+" "+str(j))
            file_out.write(" " + names_dataset[index[j][0]])
        file_out.write("\n")
    file_out.close()

names_dhn, features_dhn, origin_features  = file_read(r"C:\Users\ycdfw\Desktop\DHN_0.1\result_dhn.txt")
#getHash("data/query")
names_query_dhn, features_query_dhn, origin_features_query = file_read(r"C:\Users\ycdfw\Desktop\DHN_0.1\query_dhn.txt")
#test_accuracy_dhn(names_dhn, features_dhn, origin_features)
query_dhn(names_dhn, features_dhn, origin_features, names_query_dhn, features_query_dhn, origin_features_query)


import os
import sys
import cv2

def images2vedio(in_dir, out_dir, out_name, fps=2):
  im_names = os.listdir(in_dir)
  im_names.sort()
  im_paths = [in_dir + im_name.strip() for im_name in im_names]

  out_path = out_dir + out_name
  # error: 'module' object has no attribute VideoWriter_fourcc
  # fourcc = cv2.VideoWriter_fourcc(*'avi')	# error
  fourcc = cv2.cv.CV_FOURCC(*'XVID')
  out = cv2.VideoWriter(out_path, fourcc, fps, (640,480))

  for im_path in im_paths:
  	im = cv2.imread(im_path)
  	out.write(im)
  	if (cv2.waitKey(1) & 0xFF) == ord('q'):
  		break
  out.release()

if __name__ == '__main__':

	in_dir = "/home/ddk/malong/dataset/person.torso/demo/vision/mude.images.pose1/"
	out_name = "demo.avi"
	out_dir = "/home/ddk/malong/dataset/person.torso/demo/vision/mude.images.pose1/"

	images2vedio(in_dir, out_dir, out_name)


import itertools

def part1(numbers):
    for x, y in itertools.combinations(numbers, 2):
        if x + y == 2020:
            return x * y

def part2(numbers):
    for x, y, z in itertools.combinations(numbers, 3):
        if x + y + z == 2020:
            return x * y * z

def main(inp):
    with open(inp) as f:
        lines = f.read().splitlines()
    numbers = list(map(int, lines))
    res1 = part1(numbers)
    res2 = part2(numbers)
    return res1, res2

if __name__ == '__main__':
    a, b = main('../input/input01.txt')
    print('day01.1:', a)
    print('day01.2:', b)


# -*- coding: utf-8 -*-
#!/usr/bin/env python

import matplotlib.pyplot as plt
import os.path
import os
import argparse
import json

parser = argparse.ArgumentParser(description='Train report build')
parser.add_argument('-a', '--history', help='History file name',default=None)
args = parser.parse_args()

history_list = []
model_cont = 1

def search_history(*args):
    print()
    for item in args:
        for p, _, files in os.walk(os.path.abspath(item)):
            for file in files:
                if(file == 'train_history.json'):
                    with open(os.path.join(p, file), 'r') as fp:
                        train_history = {
                            "history" : json.load(fp),
                            "label" : str(p)
                        }

                        history_list.append(train_history)

def smooth(scalars, weight):
    last = scalars[0]
    smoothed = list()
    for point in scalars:
        smoothed_val = last * weight + (1 - weight) * point
        smoothed.append(smoothed_val)
        last = smoothed_val

    return smoothed

search_history("/home/luismalta/Projects/DeepRL/logs/Dueling_DQN")

# with open('/home/luismalta/Projects/DeepRL/logs/Dueling_DQN/log_01-06-2019_22:28/train_history.json') as fp:
#     train_history = {
#         "history" : json.load(fp),
#         "label" : 'DUELING DQN MODELO 7'
#     }
#     history_list.append(train_history)
#
# with open('/home/luismalta/Projects/DeepRL/logs/Dueling_DQN/log_04-07-2019_00:51/train_history.json') as fp:
#     train_history = {
#         "history" : json.load(fp),
#         "label" : 'DUELING DQN MODELO 4'
#     }
#     history_list.append(train_history)
#
# with open('/home/luismalta/Projects/DeepRL/logs/Dueling_DQN/log_10-07-2019_23:44/train_history.json') as fp:
#     train_history = {
#         "history" : json.load(fp),
#         "label" : 'DUELING DQN MODELO 8'
#     }
#     history_list.append(train_history)

plt.rcParams['figure.figsize']= (15,5)
plt.rcParams.update({'font.size': 12})
plt.rcParams.update({'font.weight':0})
plt.get_cmap()

for history in history_list:
    reward = smooth(history["history"]["episode_reward"],0.995)
    #plt.plot(reward, label=history["label"][-21:])
    plt.plot(reward, label="DUELING DQN MODELO " + str(model_cont))
    model_cont += 1

plt.legend()
plt.grid()
plt.title(u'Recompensas por epsódios', loc='left', fontsize=16, fontweight=10, color='black')
plt.xlabel(u"Episódios")
plt.ylabel("Recompensa")
plt.show()


# "The __init__.py files are required to make Python treat the
# directories as containing packages"
# -- http://docs.python.org/tutorial/modules.html

from ConfigParser import ConfigParser

from sqlalchemy.engine.url import make_url

from traincheck import TrainingRecordsRd

TRAINING_SECTION = 'training'

def from_config(ini, create_engine):
    cp = ConfigParser()
    cp.readfp(ini.open(), str(ini))
    u = make_url(cp.get(TRAINING_SECTION, 'url'))
    redcapdb = (None if u.drivername == 'sqlite' else 'redcap')

    # Address connection timeouts using pool_recycle
    # ref http://docs.sqlalchemy.org/en/rel_1_0/dialects/mysql.html#connection-timeouts  # noqa
    trainingdb = create_engine(u, pool_recycle=3600)
    account = lambda: trainingdb.connect(), u.database, redcapdb

    tr = TrainingRecordsRd(account)

    return tr.__getitem__


import cv2 as cv
import numpy as np
import pickle

# Load the camera calibration parameters from the .pkl files
calibration_data = pickle.load(open("calibration.pkl", "rb"))
cameraMatrix, dist = calibration_data

# Load an image for pose estimation
image_path = 'images/img21.png'  # Replace with the path to your image
img = cv.imread(image_path)

# Perform pose estimation
gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

# Define the chessboard size used for calibration
chessboard_size = (7, 7)

# Find chessboard corners
ret, corners = cv.findChessboardCorners(gray, chessboard_size, None)

if ret:
    # Refine corner locations
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
    corners = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)

    # SolvePnP for pose estimation
    objp = np.zeros((chessboard_size[0] * chessboard_size[1], 3), np.float32)
    objp[:, :2] = np.mgrid[0:chessboard_size[0], 0:chessboard_size[1]].T.reshape(-1, 2)
    objp *= 20  # Assuming square size in mm

    _, rvecs, tvecs, inliers = cv.solvePnPRansac(objp, corners, cameraMatrix, dist)

    # Define cube vertices in the object coordinate system
    cube_size = 50
    cube_points = np.float32([
        [0, 0, 0],
        [cube_size, 0, 0],
        [cube_size, cube_size, 0],
        [0, cube_size, 0],
        [0, 0, -cube_size],
        [cube_size, 0, -cube_size],
        [cube_size, cube_size, -cube_size],
        [0, cube_size, -cube_size]
    ])

    # Project cube points into the image plane
    cube_img_points, _ = cv.projectPoints(cube_points, rvecs, tvecs, cameraMatrix, dist)

    # Draw cube edges on the image
    img = cv.drawFrameAxes(img, cameraMatrix, dist, rvecs, tvecs, cube_size)
    img = cv.drawContours(img, [cube_img_points.astype(int).reshape(-1, 2)], -1, (0, 255, 0), 3)

    # Display the image with the rendered cube
    cv.imshow('Rendered Cube', img)
    cv.waitKey(0)
    cv.destroyAllWindows()
else:
    print("Chessboard corners not found in the image.")


# Corner Detections

import cv2
import numpy as np

# Load image then grayscale
image = cv2.imread('images/chess.jpg')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# The cornerHarris function requires the array datatype to be float32
gray = np.float32(gray)

harris_corners = cv2.cornerHarris(gray, 3, 3, 0.05)

#We use dilation of the corner points to enlarge them\
kernel = np.ones((7,7),np.uint8)
harris_corners = cv2.dilate(harris_corners, kernel, iterations = 2)

# Threshold for an optimal value, it may vary depending on the image.
image[harris_corners > 0.025 * harris_corners.max() ] = [255, 127, 127]

cv2.imshow('Harris Corners', image)
cv2.waitKey(0)

# We specific the top 50 corners
corners = cv2.goodFeaturesToTrack(gray, 100, 0.01, 150)

for corner in corners:
    x, y = corner[0]
    x = int(x)
    y = int(y)
    cv2.rectangle(image, (x - 10, y - 10), (x + 10, y + 10), (0, 255, 0), 2)

cv2.imshow("Corners Found", image)
cv2.waitKey()

cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 24 01:57:39 2020

@author: agarcia20
"""

import yfinance as yf
import math

stock = yf.Ticker("XOM")                                                                 # Get ticker
expiration = stock.options[0]                                                            # Get next expiration date
opt = stock.option_chain(expiration)                                                     # Get the option chain

options = opt.calls                                                                      # Get options
for idx_left_body, left_body in options.iterrows():                                      # Iterate over all options
    left_wings = options[options['strike'] < left_body['strike']]                        # Get left wings

    for idx_left, left in left_wings.iterrows():                                         # Iterate over left wings
        right_bodies = options[options['strike'] > left_body['strike']]                  # Get right bodies

        for idx_right_body, right_body in right_bodies.iterrows():                       # Iterate over right bodies
            right_wings = options[options['strike'] > right_body['strike']]              # Get right wings

            for idx_right, right in right_wings.iterrows():                              # Iterate over right wings

                left_count = round(right['strike'] - right_body['strike'], 2) * 1000     # Left raw wing/body count
                right_count = round(left_body['strike'] - left['strike'], 2) * 1000      # Right raw wing/body count

                if (left_count == int(left_count) and right_count == int(right_count)):  # If there is a integer number of contracts try to minimize the quantity
                    gcd = math.gcd(int(left_count), int(right_count))                    # Greated Common Divisor between the number of contracts

                    left_count /= gcd
                    right_count /= gcd

                    if left_count <= 10 and right_count <= 10:                           # Filter all the condors with less than 10 contracts
                        print("Buy %d Contract (Strike %.2f) - Sell %d Contract (Strike %.2f) - Sell %d Contract (Strike %.2f) - Buy %d Contract (Strike %.2f)" %
                            (left_count, left['strike'], left_count, left_body['strike'], right_count, right_body['strike'], right_count, right['strike']))


import serial
import sys

with open('Tester_clean.hex', 'r') as f:
    archivo = f.read().split('\n')

archivo.pop()
print(archivo)
print(type(archivo))
# puerto = serial.Serial('COM4', 9600,  rtscts = True) # rtscts = False PARA PROTEUS !!!!!!!!!!!
puerto = serial.Serial('COM1', 9600,  rtscts = False) # rtscts = False PARA PROTEUS !!!!!!!!!!!

for linea in archivo:
    while 1:
        puerto.write(linea.encode())

        respuesta = puerto.readline().decode()

        if(respuesta == 'OK\n'):
            print('Linea enviada correctamente...')
            break

        elif(respuesta == 'BAD\n'):
            print('Checksum invalido, reenviando...')

print('Grabado finalizado...')


from recipes.decorators import catch
import pytest

class TestCatch:

    @pytest.mark.parametrize(
        'action, kls, kws',
        [('warn', UserWarning, {}),
         ('raise', ValueError, {}),
         ('raise', TypeError, dict(alternate=TypeError))]
    )
    def test_decor(self, action, kls, kws):

        @catch(action=action, **kws)
        def foo():
            raise ValueError('Nope!')

        ctx = pytest.warns if issubclass(kls, Warning) else pytest.raises
        with ctx(kls):
            foo()

    @pytest.mark.parametrize(
        'action, kls, kws',
        [('warn', UserWarning, {}),
         ('raise', ValueError, {}),
         ('raise', TypeError, dict(alternate=TypeError))]
    )
    def test_context(self, action, kls, kws):
        ctx = pytest.warns if issubclass(kls, Warning) else pytest.raises
        with ctx(kls):
            with catch(action=action, **kws):
                raise ValueError('Nope!')

    @pytest.mark.parametrize('action', ('ignore', 'warn', 'raise'))
    def test_context_no_error(self, action):
        with catch(action=action):
            pass


import cherrypy
import json
import splunk.appserver.mrsparkle.controllers as controllers
from splunk.appserver.mrsparkle.lib import jsonresponse
from splunk.appserver.mrsparkle.lib.decorators import expose_page
from splunk.appserver.mrsparkle.lib.routes import route
from splunk.rest import simpleRequest
import sys

from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "bin"]))
from custom_search_builder.base                    import CustomSearchBuilderBase
from custom_search_builder.exceptions              import *
from custom_search_builder.make_correlation_search import makeCorrelationSearch
from custom_search_builder.make_lookup_generating_search import makeLookupGeneratingSearch

sys.path.append(make_splunkhome_path(["etc", "apps", "Splunk_SA_CIM", "lib"]))
from cim_models import DataModels

import logging
logger = logging.getLogger('splunk.appserver.SA-ThreatIntelligence.controllers.CustomSearchBuilder')

class CustomSearchBuilder(controllers.BaseController):

    @route('/:get_data_models=get_data_models')
    @expose_page(must_login=True, methods=['GET'])
    def getDataModelsAndObjects(self, **kwargs):

        # Get the session key
        sessionKey = cherrypy.session.get('sessionKey')

        # This will contain all of the information about the data-models and the associated objects
        data_models_info = []

        # Get the list of data-models
        for data_model in DataModels.getDatamodelList(sessionKey):

            try:
                data_models_info.append( {
                                          'name' : data_model,
                                          'objects' : DataModels.getDatamodelObjectList(data_model, sessionKey)
                                          } )
            except:
                pass

        return self.render_json(data_models_info)

    @route('/:get_available_fields=get_available_fields')
    @expose_page(must_login=True, methods=['GET'])
    def getAvailableFieldsFromSpec(self, search_spec, **kwargs):
        # Get the session key
        sessionKey = cherrypy.session.get('sessionKey')

        # Parse the JSON
        search_spec_parsed = json.loads(search_spec)

        # Get instance of CustomSearchBuilderBase
        csb = CustomSearchBuilderBase(sessionKey, logger)

        try:
            available_fields = csb.getAvailableFields(search_spec_parsed)

        except (InvalidResultFilter, InvalidAggregate, InvalidDatamodelObject, InvalidInputlookup) as e:
            return self.render_json( {
                                      'success': False,
                                      'message': 'Search specification is invalid: '  + str(e),
                                      })

        except Exception as e:
            return self.render_json( {
                                      'success': False,
                                      'message': 'Search specification could not be converted: ' + str(e)
                                      })

        return self.render_json( {
                                  'success'   : True,
                                  'available_fields' : available_fields,
                                  'message'    : 'Search specification converted successfully'
                                  })

    @route('/:make_search_from_spec=make_search_from_spec')
    @expose_page(must_login=True, methods=['GET'])
    def makeSearchFromSpec(self, routine, search_spec, **kwargs):

        # Get the session key
        sessionKey = cherrypy.session.get('sessionKey')

        # Parse the JSON
        search_spec_parsed = json.loads(search_spec)

        # Make the correlation search string
        try:
            if routine == 'makeCorrelationSearch':
              raw_search, parses = makeCorrelationSearch(search_spec_parsed, sessionKey=sessionKey, logger=logger)
            elif routine == 'makeLookupGeneratingSearch':
              raw_search, parses = makeLookupGeneratingSearch(search_spec_parsed, sessionKey=sessionKey, logger=logger)
            elif routine == 'raw':
              ## Get instance of CustomSearchBuilderBase
              ## This validates sessionKey and logger
              rsb = CustomSearchBuilderBase(sessionKey, logger)
              rs = search_spec_parsed.get('search', {})
              if rs.get('inputlookup', False):
                raw_search = rsb.makeInputlookup(rs)
              else:
                raw_search = rsb.makeRaw(rs, modelJson=None)
              response, contents = simpleRequest("search/parser", sessionKey=sessionKey, getargs={'q': raw_search, 'parse_only': 't', 'output_mode': 'json'})
              parses = (response.status == 200)
            else:
              return self.render_json( {
                                      'success': False,
                                      'message': 'Routine not supported: ' + str(routine),
                                      })
        except (InvalidResultFilter, InvalidAggregate, InvalidDatamodelObject, InvalidInputlookup) as e:
            return self.render_json( {
                                      'success': False,
                                      'message': 'Search specification is invalid: '  + str(e),
                                      })

        except Exception as e:
            return self.render_json( {
                                      'success': False,
                                      'message': 'Search specification could not be converted: ' + str(e)
                                      })

        return self.render_json( {
                                  'success'    : True,
                                  'raw_search' : raw_search,
                                  'parses'     : parses,
                                  'message'    : 'Search specification converted successfully'
                                  })


#!/usr/bin/env python2

import logging
import subprocess
import os.path
import shlex
import importlib.resources
import clamsig
import csv

class ClamSigVis:

    '''
    Use the tooling
    '''

    # Use the default directory
    _freshclam_dir = None

    # Use the default path by default
    _bin_path = None

    #
    _clamftypes = {0: "any file",
                   1: "Portable Executable, both 32 - and 64 - bit ",
                   2: "OLE2 containers, including specific macros.Primarily used by MS Office and MSI installation files",
                   3: "HTML(normalized)",
                   4: "Mail file",
                   5: "Graphics",
                   6: "ELF",
                   7: "ASCII text file(normalized)",
                   8: "Unused",
                   9: "Mach-O files",
                   10: "PDF files",
                   11: "Flash files",
                   12: "Java class files"}

    def __init__(self, signature_name=None, **kwargs):

        self.logger = logging.getLogger("ClamSigVis")

        self.freshclam_dir = kwargs.get("freshclam_dir", self._freshclam_dir)
        self.bin_path = kwargs.get("bin_path", self._bin_path)

        self.clamftypes = kwargs.get("clamftypes", self._clamftypes)
        self.clamflevels = kwargs.get("clamflevels", self.read_flevels())

        self.signature_search_name = signature_name

        self.sig_text_raw = self.get_sig_data()
        self.sig_decoded = self.get_decoded_sig()

    def read_flevels(self):

        flevels = dict()

        with importlib.resources.path(clamsig, "flevel.csv") as flevel_csv:
            with open(flevel_csv, "r") as flevel_csv_fobj:
                reader = csv.DictReader(flevel_csv_fobj)

                for row in reader:
                    flevels[int(row["FLEVEL"])] = row

        return flevels

    def get_sig_data(self):

        '''
        Use sigtool to get the signature string
        :return:
        '''

        get_sig_cmd = list()

        if self.bin_path is not None:
            sigtool_bin = os.path.join(self.bin_path, "sigtool")

        get_sig_cmd.append("sigtool")

        if self.freshclam_dir is not None:

            defs_path = os.path.join(self.freshclam_dir, "defs")

            get_sig_cmd.append("-d {}".format(defs_path.abspath))

        get_sig_cmd.append("--find-sigs {}".format(self.signature_search_name))

        try:
            read_sig_result = subprocess.run(" ".join(get_sig_cmd),
                                             executable="/bin/bash",
                                             shell=True,
                                             timeout=120,
                                             capture_output=True)
        except Exception as sig_read_error:
            self.logger.error("Unable to Read Signature")
            self.logger.info(sig_read_error)
            signature_raw_text = None
        else:
            # Let's Parse the Results

            signature_raw_text = read_sig_result.stdout.decode("utf-8")

            self.logger.debug("Signature Raw Text:\n{}".format(signature_raw_text))

            total_signatures = len(signature_raw_text.splitlines())

            self.logger.debug("Found {} Signature for search term: {}".format(total_signatures, self.signature_search_name))

            if total_signatures == 0:
                raise FileNotFoundError("I was unable to find a signature.")
            elif total_signatures > 1:
                raise ValueError("I found too many signatures!")

        return signature_raw_text

    def get_decoded_sig(self):

        '''
        Decode the Signature and Return the Data Back
        :return:
        '''

        decode_sig_cmd = ['''echo -e "{}" |'''.format(self.sig_text_raw)]

        if self.bin_path is not None:
            sigtool_bin = os.path.join(self.bin_path, "sigtool")

        decode_sig_cmd.append("sigtool --decode-sigs")

        try:
            decode_sig_result = subprocess.run(" ".join(decode_sig_cmd),
                                               executable="/bin/bash",
                                               shell=True,
                                               timeout=120,
                                               capture_output=True)
        except Exception as sig_read_error:
            self.logger.error("Unable to Read Signature")
            # Change back to info later
            self.logger.error(sig_read_error)
            signature_decode = None
        else:
            # Let's Parse the Results

            signature_decode = decode_sig_result.stdout.decode("utf-8")

            self.logger.debug("Signature Decoded Text:\n{}".format(signature_decode))

        return signature_decode


from sentence_transformers import SentenceTransformer
from collections import defaultdict
from graph_model import dialogue_graph
import pickle
import numpy as np
import sys
import torch
from graph_model.cluster_selection.cluster_selection import PredictNextCluster, prepare_input

from graph_model import dialogue_graph
from graph_model import dataset
from graph_model import clustering
from graph_model import embedders
from graph_model import dialogue_sim

sys.modules["dialogue_graph"] = dialogue_graph
sys.modules["dataset"] = dataset
sys.modules["clustering"] = clustering
sys.modules["embedders"] = embedders
sys.modules["dialogue_sim"] = dialogue_sim

class Utterance:
    def __init__(self, string, embedding):
        self.string = string
        self.embedding = embedding

class GraphModel:
    def __init__(self):
        self.cluster_prediction = PredictNextCluster(403, 512, 11)
        self.cluster_prediction.load_state_dict(torch.load("graph_model/cluster_selection/cluster_selection.pt", map_location=torch.device('cpu')))
        self.cluster_prediction.eval()
        self.model = SentenceTransformer("all-MiniLM-L6-v2")
        with open("graph_model/dialogue_sim/graphs/one_serv_10_taxi.pkl", 'rb') as file:
            self.graph = pickle.load(file)
        with open("graph_model/dialogue_sim/data/train_dials.pkl", "rb") as file:
            self.train_dials = pickle.load(file)
        self.transitions = self.train_dials[0].transitions

        self.cluster_to_utts = defaultdict(list)
        for dial in self.train_dials:
            for i in range(len(dial.utterances)):
                self.cluster_to_utts[dial.second_stage_clusters[i]].append(Utterance(dial.utterances[i], dial.lm_embeddings[i]))

        self.cluster_to_embs = {}
        for i, utts in self.cluster_to_utts.items():
            embs = []
            for utt in utts:
                embs.append(utt.embedding)
            self.cluster_to_embs[i] = np.array(embs)

    def get_closest(self, target, utt_embs, k = 5):
        similarity = (utt_embs @ target.reshape(-1, 1)).ravel()
        closest = np.argsort(similarity)[:-k:-1]
        return closest

    def get_next_cluster(self, cur_cluster):
        return np.argmax(self.transitions[cur_cluster])

    def __call__(self, text):
        embedding = self.model.encode(text)
        one_stage_cluster = self.graph.one_stage_clustering._subclusters["USER"].predict_cluster(embedding[0]).id
        second_stage_cluster = self.graph.cluster_kmeans_labels[0][one_stage_cluster]

        prepared_input = prepare_input(embedding[0], self.transitions[second_stage_cluster], self.graph.cluster_embeddings[one_stage_cluster] )
        #next_cluster = self.get_next_cluster(second_stage_cluster)
        next_cluster = torch.argmax(self.cluster_prediction(prepared_input.reshape(1, -1)), 1).item()
        closest_idxs = self.get_closest(embedding, self.cluster_to_embs[next_cluster])
        closest_str = []
        for i in closest_idxs:
            closest_str.append(self.cluster_to_utts[next_cluster][i].string)
        return [closest_str[0]], next_cluster


import os
import json
import numpy as np

def convert(JSON_FOLDER, NPY_PATH):
    # Initialize an empty list to accumulate the pose data
    pose_data_list = []

    # Loop over all JSON files in the folder
    for JSON_FILE in sorted(os.listdir(JSON_FOLDER)):
        # Check if the file is a JSON file
        if JSON_FILE.endswith('.json'):
            # Construct the full path to the JSON file
            JSON_PATH = os.path.join(JSON_FOLDER, JSON_FILE)

            # Load the JSON data
            with open(JSON_PATH, 'r') as f:
                data = json.load(f)

            # Extract the pose data from the JSON data
            pose_data = np.array(data['people'][0]['pose_keypoints_2d']).reshape((-1, 3))

            # Add the pose data to the list
            pose_data_list.append(pose_data)

    # Convert the list of pose data arrays to a single NumPy array
    pose_data_array = np.stack(pose_data_list, axis=0)

    # Save the pose data as an NPY file
    np.save(NPY_PATH, pose_data_array)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Convert OpenPose JSON files to NPY file")
    parser.add_argument("--json_dir", type=str, required=True)
    parser.add_argument("--output_file", type=str, default="keypoints.npy")
    args = parser.parse_args()

    JSON_FOLDER = args.json_dir
    NPY_PATH = os.path.join(JSON_FOLDER, "..", args.output_file)

    convert(JSON_FOLDER, NPY_PATH)


#!/usr/bin/env python

''' zeromq server example.

    One front facing server and three workers are instantiated.
    Front facing server accepts connections from multiple clients and distributes computation requests among workers.
    Also it collects computed results from workers and send the result back to the original client.

    Demonstrables:
        Multiple client connections.
        Work distribution among a pool of workers. '''

# Author - Kasun Herath <kasunh01 at gmail.com>
# Source - https://github.com/kasun/zeromq-client-server.git

import threading

import zmq

from face_pb2 import  ProcessStatus
import face_pb2 as pb2

import subprocess

import os

class Server(object):
    ''' Front facing server. '''

    def __init__(self):
        self.zmq_context = zmq.Context()

    def start(self):
        ''' Main execution.
            Instantiate workers, Accept client connections,
            distribute computation requests among workers and route computed results back to clients. '''

        # Front facing socket to accept client connections.
        socket_front = self.zmq_context.socket(zmq.ROUTER)
        socket_front.bind('tcp://*:5000')

        # Backend socket to distribute work.
        socket_back = self.zmq_context.socket(zmq.DEALER)
        socket_back.bind('inproc://backend')

        # Start three workers.
        for i in range(1,2):
            worker = Worker(self.zmq_context, i)
            worker.start()

        # Use built in queue device to distribute requests among workers.
        # What queue device does internally is,
        #   1. Read a client's socket ID and request.
        #   2. Send socket ID and request to a worker.
        #   3. Read a client's socket ID and result from a worker.
        #   4. Route result back to the client using socket ID.
        zmq.device(zmq.QUEUE, socket_front, socket_back)

class Worker(threading.Thread):
    ''' Workers accept computation requests from front facing server.
        Does computations and return results back to server. '''

    def __init__(self, zmq_context, _id):
        threading.Thread.__init__(self)
        self.zmq_context = zmq_context
        self.worker_id = _id

    def run(self):
        ''' Main execution. '''
        # Socket to communicate with front facing server.
        print("\n")
        print("Start ZMQ listening server...")
        print("worker id : "+ str(self.worker_id))
        print("\n")

        socket = self.zmq_context.socket(zmq.DEALER)
        socket.connect('inproc://backend')

        processStatus =  ProcessStatus()

        while True:
            # main process
            print("Receive Frame from backend")
            msg = socket.recv()
            # emptyString = socket.recv()
            topic = socket.recv()
            messsage = socket.recv()

            print(msg)
            # print(emptyString)
            print(topic)
            print(messsage)

            process = pb2.ProcessStatus()
            process.ParseFromString(messsage)
            print(process)

if __name__ == '__main__':
    # server = Server().start()
    # list_files = subprocess.run(["ps", "aux"],[ "grep", "nginx"])
    # print("The exit code was: %d" % list_files.returncode)

    # Kill process via ssh

    # execute_command = []
    # str_command1 = 'ssh'
    # str_command2 = 'root@192.168.0.221 -p 2235'
    # # str_command3 = 'kill ' + str(7536)
    # str_command3 = 'ls'

    # execute_command.append(str_command1)
    # execute_command.append(str_command2)
    # execute_command.append(str_command3)

    # proc1 = subprocess.Popen(execute_command, stdout=subprocess.PIPE)
    # proc1 = subprocess.Popen(['ssh','root@192.168.0.221','-p 2235'], stdout=subprocess.PIPE)

    # proc2 = subprocess.Popen(['ls'], stdin=proc1.ssstdout,
    # stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # proc1.stdout.close() # Allow proc1 to receive a SIGPIPE if proc2 exits.
    # out, err = proc1.communicate()
    # print('out: {0}'.format(out))
    # print('err: {0}'.format(err))
    command = 'ssh root@192.168.0.221 -p 2235 kill ' + str(1132)
    os.system(command)


# Databricks notebook source
from azure.storage.blob import BlobServiceClient
import os
import pandas as pd
from pyspark.sql.types import StructField, StructType, StringType

connection_string = "<connection_string>"
service = BlobServiceClient.from_connection_string(conn_str=connection_string)

# COMMAND ----------

v_container_name = "folder_name_on_blob_service"
#local_path = 'c:\\work\\'
#local_file_name = '.txt'
#var_filename_from_blob = 'GLIM_Delicias_Order_66.csv'

# COMMAND ----------

container_client = service.get_container_client(v_container_name)

# COMMAND ----------

# criando a lista de arquivos NJM_GLIM e GLIMS
ct = 0
ct2 = 0
ct_glims = 0
list_njm = []
list_glim = []
def insere_lista(nome_lista, item):
    nome_lista.append(str(item.name))

for item_blob in container_client.list_blobs():
    # find a specific string in the names of all items on a blob
    if 'NJM_GLIM20' in str(item_blob.name).upper():
        insere_lista(list_njm, item_blob)
        ct += 1
        if ct < 5:
            print(str(item_blob.name))
    elif 'GLIM20' in str(item_blob.name).upper() and 'delicia' not in str(item_blob.name).lower():
        insere_lista(list_glim, item_blob)
        ct2 += 1
        if ct2 < 5:
            print(str(item_blob.name))

print('# ct NJM ---> ' + str(ct))
print('# ct2 Glim -> ' + str(ct2))

# COMMAND ----------

# delete to Glim
ct = 0
ct2 = 0
for arq_no_blob in container_client.list_blobs():
    if str(arq_no_blob.name) in list_glim:
        ct += 1
        blobFile = service.get_blob_client(v_container_name, arq_no_blob.name)
        blobFile.delete_blob()
    elif str(arq_no_blob.name) in list_njm:
        ct2 += 1
        blobFile = service.get_blob_client(v_container_name, arq_no_blob.name)
        blobFile.delete_blob()
print('# Excluidos NJM  -> ' + str(ct))
print('# Excluidos Glim -> ' + str(ct2))


import os
from setuptools import setup, find_packages

version = "0.1.0"

requirements_file = 'requirements.txt'
with open(requirements_file) as f:
  install_requires = f.read().splitlines()

setup(name="deepmeme",
      packages=find_packages(),
      version=version,
      description="Package for accessing the clustering tools",
      classifiers=[
          'Development Status :: Beta',
          'Environment :: Web/Desktop Environment',
          'Intended Audience :: Developers',
          'Intended Audience :: Science/Research',
          'Natural Language :: English',
          'Programming Language :: Python :: 3',
      ],
      author='Albert Chung',
      author_email='albertswchung@gmail.com',
      license='',
      python_requires='>=3',
      install_requires=install_requires,
)


import os
import time
import traceback

from pyChaining import Chains
from progress_utils_xh import ProgressPinger
from kafka import KafkaAdminClient
from kafka.admin import NewTopic

if __name__ == '__main__':
    print(os.environ, flush=True)
    brokers = os.getenv("hosts")
    print("Broker from env: ", brokers, flush=True)

    topics = []
    with open("topic_list.txt") as f:
        for line in f.readlines():
            topic = line.strip("\n")
            print(f"Add topic[{line}]", flush=True)
            topics.append(line.rstrip("\n"))
    topics = Chains.of(topics).filter(lambda x: x!="").list()
    i = 0
    while True:
        print(f"retry for {i} time{'s' if i > 1 else ''}", flush=True)
        i += 1
        try:
            print(brokers, flush=True)
            admin = KafkaAdminClient(
                bootstrap_servers=brokers,
                client_id="test-client"
            )
            existingTopic = admin.list_topics()
            print("Existing topic ",existingTopic, flush=True)
            topic_list = Chains.of(topics).\
                filter(lambda x: x not in existingTopic).\
                map(lambda x: NewTopic(x, num_partitions=1, replication_factor=1)).\
                list()

            if len(topic_list) == 0:
                print("Complete creating all topics", flush=True)
                break
            else:
                print(f"{len(topic_list)} topics to create", flush=True)
                rs = admin.create_topics(new_topics=topic_list)
                print(rs, flush=True)
        except Exception as ex:
            print("err", flush=True)
            print(ex, flush=True)
            traceback.print_exc()
            pass
        time.sleep(2)
    print("Done", flush=True)


import json
from collections import Counter
import timeit

data_file = open('data.json')
data = json.load(data_file)

starttime = timeit.default_timer()

status_list = []
piece_list = []
start_time_list = []

for item in data:
    piece_list.append(item["piece_id"])
    status_list.append(item["status"])
    start_time_list.append(item["start_time"])

unique_p_id = []

# get only the unique piece ids
for item in piece_list:
    if item not in unique_p_id:
        unique_p_id.append(item)
print(len(unique_p_id))
# create a dictionary of just unique ids
data_dict = {key: [] for key in unique_p_id}

# store status list and start time list with the key as the piece_id
for i in range(len(piece_list)):
    data_dict[piece_list[i]].append([status_list[i], start_time_list[i]])

# get just the paths
paths = list()

# append the keys to the paths
for i, values in enumerate(data_dict.items()):
    paths.append(values[1])  # status, time

paths = sorted(paths)  # sort on the time

path_list = []

for k in paths:
    temp = []
    if not k[0]:
        del k[0]
    k.sort(key=lambda x: x[1])
    for i, vals in enumerate(k):
        if vals:
            temp.append(vals[0])

    path_list.append(tuple(temp))

    # if key in path_dict:
    #     path_dict[key] += 1
    # else:
    #     path_dict[key] = path_dict.setdefault(key, 1)

# print(path_dict)
# print(counter)
most_common_path, occurrences = Counter(path_list).most_common(1)[0][0], Counter(path_list).most_common(1)[0][1]
print(f"The most common path a piece takes is: {most_common_path}")
print(f"It occurs {occurrences} times.")
print(timeit.default_timer() - starttime)
data_file.close()


import re
from django.conf import settings
from django.contrib.auth import load_backend
from django.core.exceptions import ImproperlyConfigured, PermissionDenied

SESSION_KEY = '_auth_user_id'
BACKEND_SESSION_KEY = '_auth_user_backend'
HASH_SESSION_KEY = '_auth_user_hash'

def get_user(request):
    from django.contrib.auth.models import AnonymousUser
    try:
        user_id = request.session[SESSION_KEY]
        backend_path = request.session[BACKEND_SESSION_KEY]
        assert backend_path in settings.AUTHENTICATION_BACKENDS
        backend = load_backend(backend_path)
        user = backend.get_user(user_id) or AnonymousUser()
    except (KeyError, AssertionError):
        user = AnonymousUser()
    return user


from seminars.app import app
from seminars import db
from seminars.categories import categories

from flask import render_template, request
import datetime

from lmfdb.utils import (
    SearchArray, TextBox, SelectBox, YesNoBox,
    to_dict, search_wrap,
)

class SemSearchArray(SearchArray):
    noun = "seminar"
    plural_noun = "seminars"
    def __init__(self):
        category = SelectBox(
            name="category",
            label="Category",
            options=[("", "")] + categories)
        keywords = TextBox(
            name="keywords",
            label="Keywords")
        speaker = TextBox(
            name="speaker",
            label="Speaker")
        affiliation = TextBox(
            name="affiliation",
            label="Affiliation")
        institution = TextBox(
            name="institution",
            label="Institution")
        title = TextBox(
            name="title",
            label="Title")
        online = SelectBox(
            name="online",
            label="Online",
            options=[("", "only"),
                     ("all", "and offline"),
                     ("exclude", "exclude")])
        when = SelectBox(
            name="when",
            label="Occuring in",
            options=[("", ""),
                     ("future", "the future"),
                     ("past", "the past")])
        date = TextBox( # should have date widget?
            name="date",
            label="Date")
        video = YesNoBox(
            name="video",
            label="Has video")
        avail = SelectBox(
            name="access",
            label="Access",
            options=[("", ""),
                     ("open", "open only")])
        count = TextBox(
            name="count",
            label="Results to display",
            example=50)
        self.browse_array = [[category, keywords], [speaker, affiliation], [title, institution], [when, date], [online], [video, avail], [count]]

@app.route("/")
def index():
    print("starting index")
    info = to_dict(request.args, search_array=SemSearchArray())
    if len(request.args) > 0:
        return search(info)
    today = datetime.datetime.today().weekday() # account for time zone....
    days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
    if today not in [5, 6]: # weekday
        days = days[today:] + days[:today]
        days[0] = "Today"
        if today !=  4:
            days[1] = "Tomorrow"

    return render_template(
        'browse.html',
        title="Math Seminars",
        info=info,
        categories=categories,
        days=days,
        bread=None)

@app.route("/<category>")
def by_category(category):
    # raise error if not existing category?
    return search({"category":category})

@search_wrap(template="seminar_search_results.html",
             table=db.seminars,
             title="Seminar Search Results",
             err_title="Seminar Search Input Error",
             bread=lambda:[("Search results", " ")])
def seminar_search(info, query):
    # For now, just ignore the info and return all results
    pass


# -*- coding: utf-8 -*-
# --------------------------------- #
#       Author: Anemily Sippola     #
#       Python Version: 3.10.4       #
#       Created on: 2023-01-31      #
# --------------------------------- #

"""
Essentially just wraps an actual model and outputs the loss and predictions
"""

import torch

from .custom_nn_module import CustomNNModule

class SingleClassClassification(CustomNNModule):

	def __init__(self, *args, **kwargs):

		self.number_of_classes = kwargs.pop("number_of_classes")

		self.prediction_logits_key = kwargs.pop("prediction_logits_key")
		self.true_labels_key = kwargs.pop("true_labels_key")

		self.loss_values_key = kwargs.pop("loss_values_key", None)
		self.class_predictions_key = kwargs.pop("class_predictions_key")
		self.class_percentages_key = kwargs.pop("class_percentages_key", None)

		#call to custom NN module
		super(SingleClassClassification, self).__init__(*args, **kwargs)

		self.model = None
		self.loss_fn = None
		self.predict_class = None
		self.output_dimension = None

	def set_model(self, model):

		self.model = model

	def set_loss_fn(self, fn):

		self.loss_fn = fn

	def set_predict_class_fn(self, fn):

		self.predict_class = fn

	def to_device(self):
		self.model.to_device()

	def forward(self, state_object):

		state_object = self.model(state_object)

		prediction_logits = state_object[self.prediction_logits_key]

		with torch.no_grad():

			state_object[self.class_predictions_key] = self.predict_class(self, prediction_logits)

			if (self.class_percentages_key is not None) and (self.class_percentages_key not in state_object):

				state_object[self.class_percentages_key] = torch.nn.functional.softmax(prediction_logits, dim=1)

		if (self.loss_values_key is not None) and (self.true_labels_key in state_object):

			true_labels = state_object[self.true_labels_key]
			true_labels = true_labels.to(prediction_logits.device)

			state_object[self.loss_values_key] = self.loss_fn(prediction_logits, true_labels)

		return state_object

	def _max_value_prediction(self, predictions):

		predictions = predictions.detach()

		return (predictions.max(1)[1]).detach()

	def get_param_dict(self):

		param_dict={
			"model_dict":self.model.get_param_dict()
		}
		return param_dict

	def load_from_dict(self, model_dict):

		self.model.load_from_dict(**model_dict)


from flask import Blueprint, redirect, request, jsonify
from app.models import db, User, Post, Comment, FriendRequest
from flask_login import current_user
from app.forms import FriendForm
from sqlalchemy import or_, and_

friend_routes = Blueprint('requests', __name__, url_prefix="/requests")

@friend_routes.route('')
def all_friends():
  friends = User.query.join(FriendRequest,
    (FriendRequest.recipient_id == User.id) | (FriendRequest.sender_id == User.id)).filter(
    (FriendRequest.sender_id == current_user.id) | (FriendRequest.recipient_id == current_user.id)
    ).filter(FriendRequest.status == 'accepted').filter(User.id != current_user.id).all()

  return {'all_friends': [friend.to_dict() for friend in friends]}

@friend_routes.route('/profile/<userId>')
def get_one(userId):
  one_request = FriendRequest.query.filter((FriendRequest.recipient_id == userId) | (FriendRequest.sender_id == userId)).filter(
    (FriendRequest.recipient_id == current_user.id) | (FriendRequest.sender_id == current_user.id)).first()

  if one_request:
    return one_request.to_dict()
  else: return {}

@friend_routes.route("", methods=["POST"])
def create_friend_request():
  new_request = FriendForm()
  new_request['csrf_token'].data = request.cookies['csrf_token']

  recipient_id = new_request.data["recipient_id"]
  status = new_request.data["status"]

  friend_request = FriendRequest(
    sender_id=current_user.id,
    recipient_id=recipient_id,
    status=status
  )
  friend = User.query.get(recipient_id)
  db.session.add(friend_request)
  db.session.commit()
  return {'one_request': friend_request.to_dict()}

@friend_routes.route('/<requestId>', methods=["PUT"])
def accept_request(requestId):
  request = FriendRequest.query.get(requestId)
  updated_request = FriendForm()

  # updated_request['csrf_token'].data = request.cookies['csrf_token']
  sender_id = updated_request.data["sender_id"]
  recipient_id = updated_request.data["recipient_id"]
  status = updated_request.data["status"]

  request.sender_id = sender_id
  request.recipient_id = recipient_id
  request.status = status

  sender = User.query.get(sender_id)
  recipient = User.query.get(recipient_id)

  db.session.commit()
  return sender.to_dict()

@friend_routes.route('/<requestId>/<userId>', methods=["DELETE"])
def delete_request(requestId, userId):
  request = FriendRequest.query.get(requestId)
  sender = User.query.get(request.sender_id)
  recipient = User.query.get(request.recipient_id)
  friend = User.query.get(userId)

  db.session.delete(request)
  db.session.commit()
  return "Successfully deleted"


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 31 15:35:55 2023

@author: chggo
"""

import matplotlib.pyplot as plt
import matplotlib.colors as mcolors

import cv2
import numpy as np
import os
import math

class ProcessImages():
    def __init__(self,dumpImg,resultPath,gamma=2.5):
        self.dumpImg=dumpImg
        self.resultPath=resultPath
        self.gamma=gamma

    def pickImages(self):
        subList=[]
        idx=np.round(np.linspace(0, len(self.dumpImg) - 1, 10)).astype(int)  # pick equally spaced 10 images
        for i in range(len(self.dumpImg)):
              if i in idx:
                img=self.dumpImg[i]
                img=img[92:292,92:292]  #crop
                img=cv2.resize(img,(96,96))
                img=self.gammaCorrection(img,self.gamma)
                cv2.putText(img=img, text='frame-'+str(i), org=(10,12), fontFace=cv2.FONT_HERSHEY_DUPLEX, fontScale=0.3,color=0xBDF7,thickness=1)

                subList.append(img)
        Img=(cv2.hconcat(subList))

        path=os.path.join(self.resultPath,'img.png')
        #cv2.imwrite(path,Img)
        plt.imsave(path,Img,cmap='jet',vmin=0,vmax=65535)
        return(idx)

    def gammaCorrection(self,src, gamma):
        inv_gamma = 1 / gamma
        table = ((np.arange(0, 65536) / 65535) ** inv_gamma) * 65535
        table = table.astype(np.uint16)

        return table[src]

    def makeList(self):
        # create list for all graphs which are saved or to be saved
        imageNameList=[]

        L=['maxIn.png','img.png','Plot1.png','Plot2.png']
        for i in L:
            imageNameList.append(os.path.join(self.resultPath,i))
        return(imageNameList)

    def readPng(self):
        Im=[]
        imageNameList= self.makeList()
        for j in imageNameList:
            im=cv2.imread(j,cv2.IMREAD_ANYCOLOR)
            Im.append(im)
        return(Im,imageNameList)

    def mergePng(self,count):
        Im,imageNameList=self.readPng()

        graph=cv2.vconcat(Im)
        path=os.path.join(self.resultPath,'final_'+str(count)+'.png')
        cv2.imwrite(path,graph)
        self.remove(imageNameList)

        return(Im)

    def remove(self,imageNameList):
        for i in imageNameList:
           os.remove(i)


# iris cnn cp

from sklearn.datasets import load_iris
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dropout, Dense
from tensorflow.keras.layers import Conv2D, Flatten, MaxPool2D
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import EarlyStopping, TensorBoard

import matplotlib.pyplot as plt
import numpy as np

# 데이터
iris = load_iris()
x = iris.data
y = iris.target

print(x.shape) #(150, 4)
print(y.shape) #(150,)

# 데이터 전처리
scaler = StandardScaler()
scaler.fit(x)
x = scaler.transform(x)
x = x.reshape(150, 2, 2, 1)

x_train, x_test, y_train, y_test = train_test_split(
    x, y, shuffle=True, train_size=0.7
)

y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

x_predict = x_train[:10]
y_real = y_train[:10]

# 모델링
model = Sequential()
model.add(Conv2D(100, (2,2), padding='same', input_shape=(x_train.shape[1],2,1)))
model.add(Conv2D(300, (1,1)))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(20))
model.add(Dense(3, activation='softmax'))

model.summary()

# 3. 컴파일, 훈련
modelpath = './model/cifar10-{epoch:02d}-{val_loss:.4f}.hdf5' #현재 모델 경로(study에 model폴더)
#파일명 : epoch:02니깐 2자리 정수 - val_loss .4니깐 소수 4째자리 표기
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
es = EarlyStopping(monitor='val_loss', patience=5, mode='auto')
cp = ModelCheckpoint(filepath=modelpath, monitor='val_loss',
        save_best_only=True, mode='auto')

model.compile(loss='categorical_crossentropy', optimizer='adam',
                metrics=['acc'])

hist = model.fit(x_train, y_train, epochs=30, batch_size=32,
            verbose=1, validation_split=0.5, callbacks=[es,cp])

# 4. 평가, 예측
result = model.evaluate(x_test, y_test, batch_size=32)
print("loss : ", result[0])
print("acc : ", result[1])

# 시각화
import matplotlib.pyplot as plt

plt.figure(figsize=(10,6)) #단위 무엇인지 찾아볼것!

plt.subplot(2,1,1) #(2행 1열에서 1번째 그림)
plt.plot(hist.history['loss'], marker='.', c='red', label='loss') # x값과, y값이 들어가야 합니다
#x는 안넣어도 순서대로 list 형식으로 저장되서 안 넣어줬습니다

plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')
plt.grid()
plt.title('loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(loc='upper right') #우측 상단에 legend(label 2개 loss랑 val_loss) 표시

plt.subplot(2,1,2) #(2행 1열에서 2번째 그림)
plt.plot(hist.history['acc'], marker='.', c='red')
plt.plot(hist.history['val_acc'], marker='.', c='blue')
plt.grid()

plt.title('acc')
plt.ylabel('acc')
plt.xlabel('epoch')
plt.legend(['acc', 'val_acc'])

plt.show()

'''
loss :  0.18260696530342102
acc :  0.9111111164093018
'''


#!/usr/bin/env python
import glob
import os
import pandas as pd
import json
import nltk
from nltk.tokenize import MWETokenizer

from utils import int2str_label, get_rel_stance_label, get_pronouns, get_fulltext, read_quote_json, mv_files

PRONOUNS = get_pronouns()

# Create MWE Tokenizer for special words
tk = MWETokenizer()
tk.add_mwe(('peer', 'review'))
tk.add_mwe(('non', 'peer', 'review'))
tk.add_mwe(('climate', 'scientist'))
tk.add_mwe(('climate', 'scientist'))
tk.add_mwe(('medium', 'outlet'))
tk.add_mwe(('nobel', 'prize', 'win'))
tk.add_mwe(('nobel', 'win'))
tk.add_mwe(('nobel', 'peace', 'prize'))
tk.add_mwe(('nobel', 'laureate'))
tk.add_mwe(('nobel', 'laureates'))
tk.add_mwe(('prize', 'win'))
tk.add_mwe(('ocasio', 'cortez'))

def get_sorted_indices(q_dict):
    """Helper function to `get_context()`.
    Sorts indices of every Quote component ((main) subj., (main) verb, (main) neg.) within a Quote obj."""
    context_dict_ = {}
    for context_type in q_dict:
        context_dict_[context_type] = sorted([int(x) for x in q_dict[context_type]])
    return context_dict_

def get_inorder_text(context_indices,idx2text,idx2lemma,idx2coref):
    """Helper function to `get_context()`."""
    text_dict_ = {}
    for context_type in context_indices:
        text_dict_[context_type] = [idx2coref[str(x)] if idx2text[str(x)].lower() in PRONOUNS
                                    and idx2coref[str(x)] is not None
                                    else idx2text[str(x)]
                                    for x in context_indices[context_type]]
        text_dict_[context_type+'_lemmas'] = [idx2coref[str(x)] if idx2lemma[str(x)].lower() in PRONOUNS
                                    and idx2coref[str(x)] is not None
                                    else idx2lemma[str(x)]
                                    for x in context_indices[context_type]]
    return text_dict_

def get_context(src_guid,src_sent_no,src_q_no,q_dir):
    j = read_quote_json(src_guid,q_dir)
    q_dict = j['quote_tags'][src_sent_no]['quotes'][src_q_no]
    context_indices = get_sorted_indices(q_dict)
    idx2text = j['quote_tags'][src_sent_no]['idx2text']
    idx2lemma = j['quote_tags'][src_sent_no]['idx2lemma']
    idx2coref = j['coref_tags']
    text_dict = get_inorder_text(context_indices,idx2text,idx2lemma,idx2coref)
    return text_dict

def get_verb_tense(verb_text):
    return nltk.pos_tag([verb_text])[0][1]

def get_verb_mods(verb_list):
    """Given a list of verbs ('v_lemmas' in a Quote obj), groups into other verbs, adverbs, modals."""
    out = nltk.pos_tag(' '.join(verb_list).split())
    return {pos: [x[0] for x in out if x[1][:2] == pos] for pos in ['RB','MD','VB']}

with open('implicatives.txt','r') as f:
    IMPLICATIVES = set(f.read().splitlines())
print('Using implicatives set:',IMPLICATIVES)

def has_neg(context_dict):
    """Determines whether a comp. clause has been negated in its subject or verb, with a negative or implicative."""
    has_subj_neg = len(context_dict['main_neg_s']) > 0
    has_verb_neg = len(context_dict['main_neg_v']) > 0
    has_verb_impl = set(context_dict['v_lemmas']).intersection(IMPLICATIVES)
    has_verb_NEG = has_verb_neg or has_verb_impl
    if has_subj_neg and has_verb_NEG:
        return 'subj_verb_neg'
    elif has_subj_neg:
        return 'subj_neg'
    elif has_verb_neg:
        return 'verb_neg'
    elif has_verb_impl:
        return 'verb_impl'
    else:
        return 'no_neg'

if __name__=="__main__":

    articles_df = pd.read_pickle('../1_data_scraping/output/filtered_dedup_combined_df_2000_1_1_to_2020_4_12.pkl')
    src_guid2attrs = dict()
    for attr in ['stance','domain','date','is_AP']:
        src_guid2attrs[attr] = dict(zip(articles_df['guid'],articles_df[attr]))

    def get_src_attrs(src_guid):
        """Function for getting attributes of source article given article guid"""
        return {attr: src_guid2attrs[attr][src_guid] for attr in src_guid2attrs}

    orig = pd.read_csv('../2_data_processing/output/keyword_filtered_comp_clauses.tsv',
                      sep='\t',header=0,index_col=0)
    orig.reset_index(drop=True,inplace=True)

    print("Reading in batched BERT predictions...")
    PRED_DIR = "../3_cc_stance/2_Stance_model/model_preds"
    pred_files = glob.glob(os.path.join(PRED_DIR,'batch_*_pred.tsv'))
    print('\tFound {} prediction files.'.format(len(pred_files)))
    all_preds = [pd.read_csv(pred_file,sep='\t',header=0) for pred_file in pred_files]
    all_preds = pd.concat([x for x in all_preds],ignore_index=True)
    print('\tRead in {} predictions.'.format(len(all_preds)))

    print("Processing predictions...")
    all_preds['quote_text'] = orig['quote_text'].copy()
    all_preds['src_guid'] = orig['guid'].copy()
    all_preds['src_sent_no'] = orig['sent_no'].copy().apply(lambda x: str(int(x)))
    all_preds['src_quote_no'] = orig['quote_no'].copy().apply(lambda x: str(int(x)))
    all_preds['quote_guid'] = all_preds['src_guid'].apply(lambda x: x+"_") + \
                              all_preds['src_sent_no'].apply(lambda x: str(int(x))+"_") + \
                              all_preds['src_quote_no']
    all_preds['predicted_label'] = all_preds['predicted'].apply(int2str_label)
    all_preds['max_prob'] = all_preds[['0', '1','2']].max(axis=1)

    print("Creating main analysis dataframe...")
    src_guid_,src_sent_no_,src_q_no_,med_stance_,pub_date_,domain_,is_AP_,\
    quote_,cc_abs_stance_,cc_rel_stance_,neg_type_,subjs_,main_subj_lemma_,\
    verb_lemma_,verb_tense_,verb_advs_,verb_mds_,verb_other_ = [],[],[],[],[],[],[],[],[],\
                                                               [],[],[],[],[],[],[],[],[]
    for pred_item_ix,row in all_preds.iterrows():
        quote_.append(row['quote_text'])
        pred_label = row['predicted']
        src_guid = row['src_guid']
        src_sent_no = row['src_sent_no']
        src_q_no = int(row['src_quote_no'])

        src_media_attrs = get_src_attrs(src_guid)
        stance,domain,pub_date,is_AP = src_media_attrs['stance'],src_media_attrs['domain'],\
                                       src_media_attrs['date'],src_media_attrs['is_AP']
        rel_label = get_rel_stance_label(pred_label,stance)

        med_stance_.append(stance)
        domain_.append(domain)
        pub_date_.append(pub_date)
        is_AP_.append(is_AP)
        cc_abs_stance_.append(pred_label)
        cc_rel_stance_.append(rel_label)
        src_guid_.append(src_guid)
        src_sent_no_.append(src_sent_no)
        src_q_no_.append(src_q_no)

        context = get_context(src_guid,src_sent_no,src_q_no,
                              '../2_data_processing/url_quotes/')
        main_v_text = context['main_v'][0]
        verb_tense_.append(get_verb_tense(main_v_text))

        main_v_lemma = context['main_v_lemmas'][0]
        all_v = context['v_lemmas']
        other_v = all_v.copy()
        other_v.remove(main_v_lemma)
        verb_mods = get_verb_mods(other_v)
        verb_advs_.append(json.dumps(verb_mods['RB']))
        verb_mds_.append(json.dumps(verb_mods['MD']))
        verb_other_.append(json.dumps(verb_mods['VB']))

        main_s_lemma = context['main_s_lemmas']
        if len(main_s_lemma) > 0:
            main_subj_lemma_.append(main_s_lemma[0])
        else:
            main_subj_lemma_.append(None)
        subjs_.append(json.dumps(context['s_lemmas']))

        # Append particles
        if len(context['v_prt']) > 0:
            good_v_prts = list(set(context['v_prt']).intersection({'out'}))
            if len(good_v_prts) > 0:
                main_v_lemma += '_'+good_v_prts[0]

        verb_lemma_.append(main_v_lemma)

        neg_type = has_neg(context)
        neg_type_.append(neg_type)

        if pred_item_ix % 1000 == 0:
            print('Processed row:',pred_item_ix)

    main_df = pd.DataFrame({
        "guid":src_guid_,"sent_no":src_sent_no_,"quote_no":src_q_no_,
        "outlet_stance":med_stance_,"domain":domain_,"is_AP":is_AP_,"pub_date":pub_date_,
        "neg_type":neg_type_,"main_s_lemma":main_subj_lemma_,"s_lemmas":subjs_,
        "main_v_lemma":verb_lemma_,"tense":verb_tense_,"advs":verb_advs_,"modals":verb_mds_,"other_verbs":verb_other_,
        "quote":quote_,"rel_quote_stance":cc_rel_stance_,"abs_quote_stance":cc_abs_stance_
    })
    main_df['joined_s_lemmas'] = main_df['s_lemmas'].apply(lambda x: ' '.join(json.loads(x)).strip().replace(' - ',' ').lower())
    main_df['mwe_tok_s_lemmas'] = main_df['joined_s_lemmas'].apply(lambda x: json.dumps(tk.tokenize(x.split())))

    output_dir = 'test_output_dir'
    if not os.path.exists(output_dir):
        os.mkdir(output_dir)

    main_df.to_pickle(os.path.join(output_dir,'quote_analysis_df.pkl'))
    print("Saving to {}...".format(os.path.join(output_dir,'quote_analysis_df.pkl')))
    print("Done!")


from sympy import Matrix
from sympy.solvers import solve
from fractions import Fraction
from Molecules import Molecule
import math
import re

# modos: redox(ácido y básico), ácido-base, desplazamiento, dismutación
# cuando falta algo de algún lado?
# cargas?
# si hay más de una opción? cómo elegir

class Reaction:

    def __init__(self, reacts, prods, mode=1):
        self.reacts = reacts
        self.prods = prods
        self.mode = mode
        self.neto = 0           # agregar o no e-

    def starter(self):
        atoms = []
        charges = []

        info_r = {}

        for elem in self.reacts:
            molecule = Molecule(elem)
            molecule.get_symbol()
            charges.append(molecule.charge)
            atoms += molecule.get_elements()
            info_r[elem] = dict(zip(molecule.get_elements(), molecule.get_indexes()))

        info_p = {}

        for elem in self.prods:
            molecule = Molecule(elem)
            molecule.get_symbol()
            charges.append(molecule.charge)
            atoms += molecule.get_elements()
            info_p[elem] = dict(zip(molecule.get_elements(), molecule.get_indexes()))

        return (info_r, info_p), list(set(atoms)), charges

    def fix_charges(self, charges):
        my_charges = []
        for cha in charges:
            if type(cha) is str:
                x = re.split("(?<=\d)(?=\D)|(?<=\D)(?=\d)", cha)
                if len(x) == 1:
                    x.insert(0, "1")
                if not x[0].isnumeric():
                    x[0], x[1] = x[1], x[0]
                if x[1] == "+":
                    my_charges.append(int(x[0]))
                else:
                    my_charges.append(-int(x[0]))

            else:
                my_charges.append(0)

        return my_charges

    def set_matrix(self):
        columns = []
        info, atoms, charges = self.starter()
        new_charges = self.fix_charges(charges)
        for side in info:
            for molec in side:
                v = []
                for atom in atoms:
                    v.append(int(side[molec].get(atom, 0)))
                columns.append(Matrix(v))

        return Matrix.hstack(*columns), atoms, new_charges

    def get_coeffs(self):
        mat, atoms, charges = self.set_matrix()
        zeros = Matrix.zeros(len(atoms), 1)
        first, taus = mat.gauss_jordan_solve(zeros)
        dot_p = first.dot(Matrix(charges))
        unos = dict.fromkeys(taus[:-1], 1)
        try:
            ultimo = solve(dot_p.subs(unos))[0]
        except IndexError:
            ultimo = 0

        if self.mode == 2 or ultimo <= 0:
            intento = first.subs(dict.fromkeys(taus, 1)).tolist()
            self.neto = Matrix(charges).dot(Matrix(intento))
            return [float(abs(e[0])) for e in intento]

        unos[taus[-1]] = ultimo
        s = first.subs(unos).tolist()
        s_joined = [float(abs(e[0])) for e in s]
        return s_joined

    def normalizer(self):
        coffs = self.get_coeffs()
        denoms = [Fraction(x).limit_denominator().denominator for x in coffs]

        for d in denoms:
            coffs = [d*values for values in coffs]
            self.neto = self.neto*d
        coffs = [int(f) for f in coffs]
        gcd = math.gcd(*coffs)
        if gcd == 0:
            gcd = 1
        coffs = [int(e/gcd) for e in coffs]

        self.neto = int(self.neto/gcd)

        return coffs

class Balanced(Reaction):

    def __init__(self, reacts, prods, mode=1):
        super().__init__(reacts, prods, mode)

    def reaction_printer(self):
        coefficients = self.normalizer()
        s = ""
        for i in range(len(self.reacts)):
            m = Molecule(self.reacts[i])
            s += str(coefficients.pop(0)) + " " + m.get_symbol()
            if i != len(self.reacts) - 1:
                s += " + "

        # s += str(self.neto)
        if self.neto < 0:
            s += " + " + str(abs(self.neto)) + " e-"

        s += " → "

        for j in range(len(self.prods)):
            m = Molecule(self.prods[j])
            s += str(coefficients.pop(0)) + " " + m.get_symbol()
            if j != len(self.prods) - 1:
                s += " + "

        if self.neto > 0:
            s += " + " + str(self.neto) + " e-"

        # s += self.molar_mass_printer()
        return s, self.molar_mass_printer()

    def molar_mass_printer(self):
        s = []
        # s += "

Masas molares:
"
        for molecule in set(self.reacts + self.prods):
            m = Molecule(molecule)
            s.append(m.get_symbol() + " → " + str(m.get_molar_mass()))
        return s


from typing import NamedTuple

class Segment(NamedTuple):
    start: int
    end: int

    def fully_contains(self, other: 'Segment'):
        return self.start <= other.start and self.end >= other.end

    def overlaps(self, other: 'Segment'):
        return not (self.start > other.end or self.end < other.start)

def parse_line(l) -> tuple[Segment, Segment]:
    c1, c2 = l.split(',')
    s1, s2 = [Segment(*map(int, c.split('-'))) for c in (c1, c2)]
    return s1, s2

def part1():
    text = open('input.txt').read()
    spairs = [parse_line(l) for l in text.split('\n')]
    total = 0
    for fs, ss in spairs:
        if fs.fully_contains(ss) or ss.fully_contains(fs):
            total += 1
    print(total)

def part2():
    text = open('input.txt').read()
    spairs = [parse_line(l) for l in text.split('\n')]
    total = 0
    for fs, ss in spairs:
        if fs.overlaps(ss) or ss.overlaps(fs):
            total += 1
    print(total)

def main():
    part1()
    part2()

if __name__ == '__main__':
    main()


#Final version
from itertools import combinations as combi

def solution(n,k):
    #The cap on number of each key type (kcap) in circulation is given by n-(k-1)
    kcap = n-(k-1)

    #lets make a matrix giving all posible ways of sampling kcap from a vector of length n.
    subsets = list(combi(list(range(0,n)),kcap))

    #initialize an empty matrix to hold the key distribution
    scheme = [[] for x in range(0,n)]

    #in 'subsets' row indices are key type names and elements are rabbit names
    #so we just need to do the old switcharoo
    for r in range(0,len(subsets)):
        subsrow = subsets[r]
        for c in range(0,len(subsrow)):
            rabbit = subsets[r][c]
            keytype = r
            scheme[rabbit].append(keytype)

    return scheme


#!/usr/bin/python
import re
import sys
import pefile
import struct
import binascii
import subprocess

class Symbol(object):
	def __init__(self, sec, offset, name):
		self.sec = sec
		self.offset = offset
		self.name = name

class Section(object):
	def __init__(self, offset, length, name, kind):
		self.offset = offset
		self.length = length
		self.name = name
		self.kind = kind
		self.symbols = []

pefilename = sys.argv[1]
symbolfilename = sys.argv[2]
outputfilename = sys.argv[3]

image_base = None
sections = []
symbols = []

pe = pefile.PE(pefilename)
#print(pe.dump_info())
image_base = pe.OPTIONAL_HEADER.ImageBase
for section in pe.sections:
	offset = section.VirtualAddress
	length = section.SizeOfRawData
	name = section.Name.strip('\x00')
	kind = section.Characteristics
	if section.Characteristics & pefile.SECTION_CHARACTERISTICS['IMAGE_SCN_CNT_CODE']:
		kind = "CODE"
	else:
		kind = "DATA"
	sections.append(Section(offset, length, name, kind))

section = Section(0x0, sections[0].offset, "HEADER", "DATA")
sections.insert(1, section)

file = open(symbolfilename, "r")
for line in file.readlines():

	### Section
	### Does not work as IDA output is broken. Get from the PE directly instead.
#	m = re.match("^\s*([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)H\s+([^s]*)\s+(CODE|DATA)\s*$", line)
#	if m:
#		sec = m.group(1)
#		offset = m.group(2)
#		length = m.group(3)
#		name = m.group(4)
#		kind = m.group(5)
#
#		sec = int(sec, 16)
#		offset = int(offset, 16)
#		length = int(length, 16)
#

	### Symbol
	m = re.match("^\s*([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+([^\s]+)\s*$", line)
	if m:
		sec = m.group(1)
		offset = m.group(2)
		name = m.group(3)

		sec = int(sec, 16)
		sec = sections[sec - 1]

		offset, = struct.unpack('>q', binascii.unhexlify(offset))
		if offset < 0:
			offset = (sec.offset + sec.length + offset) - sec.offset

		symbol = Symbol(sec, offset, name)
		sec.symbols.append(symbol)

file.close()

for section in sections[:]:
	print(section.name)
	if section.name == 'HEADER' or \
	   section.name == '.reloc' or \
	   section.name == 'text' or \
	   section.name == '':
	   sections.remove(section)

last_address = 0x0
file = open("/tmp/gdbtab.s", "w")

sorted_sections = sorted(sections, key = lambda k: k.offset)
for section in sorted_sections:
	print("last_address: 0x{0:x}".format(last_address))
	print("offset: 0x{0:x} length: 0x{1:x} name: {2!s} kind: {3!s}".format(section.offset, section.length, section.name, section.kind))
	if not section.symbols:
		continue

	section_offset = section.offset - sorted_sections[0].offset
	section_space = section_offset - last_address
	if section_space > 0:
		file.write('.space {},0x90\n'.format(section_space))

	if section.name != ".rdata":
		file.write('{}\n'.format(section.name))

	last_address = section_offset

	sorted_symbols = sorted(section.symbols, key = lambda k: k.offset)
	for symbol in sorted_symbols:
		symbol_offset = section_offset + symbol.offset
		symbol_space = symbol_offset - last_address
		print("symbol: 0x{0:x} offset: 0x{1:x} name: {2!s}".format(symbol_offset, symbol.offset, symbol.name))
		if symbol_space > 0:
			file.write('.space {},0x90\n'.format(symbol_space))
		file.write('.global \"{}\"\n'.format(symbol.name))
		file.write(' \"{}\":\n'.format(symbol.name))
		last_address = section_offset + symbol.offset

file.write('.p2align 12'.format(symbol.name))
file.close()

### Mac
popen = subprocess.Popen(("/usr/bin/as", "-arch", "x86_64", "/tmp/gdbtab.s", "-o", "/tmp/gdbtab.o"), stdout=subprocess.PIPE)
popen.wait()
output = popen.stdout.read()
print(output)

# seg1addr and image_base are the same thing
popen = subprocess.Popen(("ld", "-arch", "x86_64", "-macosx_version_min", "10.10", "-preload", "-segalign", "0x20", "-pie", "-seg1addr", "0x240", "-image_base", '0x{:x}'.format(image_base), "-o", outputfilename, "/tmp/gdbtab.o"), stdout=subprocess.PIPE)
popen.wait()
output = popen.stdout.read()
print(output)

### Mac (untested)
#popen = subprocess.Popen(("gcc", "-Wl,-image_base,0x{:x}".format(image_base), "-dynamiclib", "-o",  outputfilename, "/tmp/gdbtab.s"), stdout=subprocess.PIPE)

### Linux (untested)
#popen = subprocess.Popen(("gcc", "-Wl,-N,-Ttext," + image_base, "-nostdlib", "-o",  outputfilename, "/tmp/gdbtab.s"), stdout=subprocess.PIPE)


from aws_cdk import core, aws_codedeploy as codedeploy, aws_lambda as lambda_

from datetime import datetime

class LambdaStack(core.Stack):
    def __init__(self, app: core.App, id: str, **kwargs):
        super().__init__(app, id, **kwargs)

        # [ Lambda: Code ]
        #
        # This represents the code to be supplied by the pipeline.

        self.lambda_code = lambda_.Code.from_cfn_parameters()

        # [ Lambda: Function ]
        #
        # Creates the Lambda.

        func = lambda_.Function(self, "Lambda",
                                code=self.lambda_code,
                                handler="index.handler",
                                runtime=lambda_.Runtime.PYTHON_3_7,
                                )

        # [ Lambda: Version ]
        #
        # Adds versions to the Lambda with date of code execution.

        version = func.add_version(datetime.now().isoformat())

        # [ Lambda: Alias ]
        #
        # Adds aliasing to the Lambda to allow for blue-green deployment.

        alias = lambda_.Alias(self, "LambdaAlias",
                              alias_name="Prod", version=version)

        # [ CodeDeploy: Deployment Group ]
        #
        # Creates the group for Lambda(s) for the Aliases.

        codedeploy.LambdaDeploymentGroup(self, "DeploymentGroup",
                                         alias=alias,
                                         deployment_config=
                                         codedeploy.LambdaDeploymentConfig.LINEAR_10_PERCENT_EVERY_1_MINUTE
                                         )