#! /usr/bin/env python
import os
import argparse
import datetime
import torch
import torchtext.data as data
import torchtext.datasets as datasets
import model
import train
import mydatasets
import numpy as np
np.random.seed(3)
torch.manual_seed(3) # Keeping it fixed

parser = argparse.ArgumentParser(description='classification_model text classificer')
# learning
parser.add_argument('--lr', type=float, default=0.001, help='initial learning rate [default: 0.001]')
parser.add_argument('--epochs', type=int, default=256, help='number of epochs for train [default: 256]')
parser.add_argument('--batch-size', type=int, default=64, help='batch size for training [default: 64]')
parser.add_argument('--log-interval',  type=int, default=1,   help='how many steps to wait before logging training status [default: 1]')
parser.add_argument('--test-interval', type=int, default=100, help='how many steps to wait before testing [default: 100]')
parser.add_argument('--save-interval', type=int, default=500, help='how many steps to wait before saving [default:500]')
parser.add_argument('--save-dir', type=str, default='snapshot', help='where to save the snapshot')
parser.add_argument('--early-stop', type=int, default=1000, help='iteration numbers to stop without performance increasing')
parser.add_argument('--save-best', type=bool, default=True, help='whether to save when get best performance')
# data
parser.add_argument('--shuffle', action='store_true', default=False, help='shuffle the data every epoch')
parser.add_argument('--dataset-dir', default='data', help = 'dataset directory path which contains negetive/positive/neutral files')
parser.add_argument('--max-length', type=int, default=600, help='The maximum number of characters in sequence.')
parser.add_argument('--min-freq', type=int, default=20, help='The minimum frequency of a character to be a vocab member')
# model
parser.add_argument('--dropout', type=float, default=0.5, help='the probability for dropout [default: 0.5]')
parser.add_argument('--max-norm', type=float, default=3.0, help='l2 constraint of parameters [default: 3.0]')
parser.add_argument('--embed-dim', type=int, default=128, help='number of embedding dimension [default: 128]')
parser.add_argument('--kernel-num', type=int, default=100, help='number of each kind of kernel')
parser.add_argument('--kernel-sizes', type=str, default='3,4,5', help='comma-separated kernel size to use for convolution')
parser.add_argument('--static', action='store_true', default=False, help='fix the embedding')
#rnn model
parser.add_argument('--rnn', action = 'store_true', default = False, help = 'activate char rnn')
parser.add_argument('--hidden_dim', type = int, default = 100, help = 'Hidden Dimensions of the RNN.')
parser.add_argument('--rnn-layers', type = int, default = 1, help = "Number of stacked RNN layers")
# device
parser.add_argument('--device', type=int, default=-1, help='device to use for iterate data, -1 mean cpu [default: -1]')
parser.add_argument('--no-cuda', action='store_true', default=False, help='disable the gpu')
# option
parser.add_argument('--snapshot', type=str, default=None, help='filename of model snapshot [default: None]')
parser.add_argument('--predict', type=str, default=None, help='predict the sentence given')
parser.add_argument('--test', action='store_true', default=False, help='train or test')

args = parser.parse_args()

tokenizer = lambda sent: list(sent[::-1])

# load data
print("\nLoading data...")
text_field = data.Field(lower=True, include_lengths = True, batch_first = True,
                        tokenize = tokenizer, fix_length = args.max_length)
label_field = data.Field(sequential=False, batch_first = True)
train_iter, dev_iter = mydatasets.load_twitter_dataset_vocab(text_field, label_field,
                                                             args.dataset_dir, args.min_freq,
                                                             args.batch_size, args.rnn)

args.embed_num = len(text_field.vocab)
print("Size of Vocab :", args.embed_num)
args.class_num = len(label_field.vocab) - 1
args.cuda = (not args.no_cuda) and torch.cuda.is_available(); del args.no_cuda
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
args.save_dir = os.path.join(args.save_dir, datetime.datetime.now()
                                                    .strftime('%Y-%m-%d_%H-%M-%S'))

print("\nParameters:")
for attr, value in sorted(args.__dict__.items()):
    print("\t{}={}".format(attr.upper(), value))

# model

if args.snapshot is not None:
    print('\nLoading model from {}...'.format(args.snapshot))
    classification_model = torch.load(args.snapshot)
elif args.rnn:
    classification_model = model.RNNClassifier(args.embed_num, args.embed_num,
                                               args.hidden_dim, args.rnn_layers,
                                               args.class_num)
else:
    classification_model = model.CharCNN(args.embed_num, args.embed_dim,
                                         args.class_num, args.kernel_num,
                                         args.kernel_sizes, args.dropout)

if args.cuda:
    torch.cuda.set_device(args.device)
    classification_model = classification_model.cuda()

# train or predict
if args.predict is not None:
    label = predict.predict(args.predict, classification_model, text_field, label_field, args.cuda)
    print('\n[Text]  {}\n[Label] {}\n'.format(args.predict, label))
elif args.test:
    train.eval(dev_iter, classification_model, args)
    #Print index of labels
    for i in range(args.class_num):
        print(str(i) +" --> " + label_field.vocab.itos[i+1])
else:
    print()
    try:
        train.train(train_iter, dev_iter, classification_model, args)
    except KeyboardInterrupt:
        print('\n' + '-' * 89)
        print('Exiting from training early')


import gspread
from oauth2client.service_account import ServiceAccountCredentials

# python-app@my-project-202123.iam.gserviceaccount.com

# use creds to create a client to interact with the Google Drive API
scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']
creds = ServiceAccountCredentials.from_json_keyfile_name('Project_278c0e241b49.json', scope)
client = gspread.authorize(creds)

# Find a workbook by name and open the first sheet
# Make sure you use the right name here.
sheet = client.open("Floating_Button_Results").sheet1

# Extract and print all of the values
# list_of_hashes = sheets[0].get_all_records()
sheet.update_cell(1,1,'qqqqqq')
print(sheet.get_all_values())


import os
from collections import (
    namedtuple,
    OrderedDict
)
import pysam
from edlib import align

gff_record = namedtuple('GFF_record', 'seqid source type start end score strand phase attributes')
comparison = namedtuple('Comparison', 'putative_event treatment_events control_events')

SCAN_SOFT_CLIP_REGION = 15

def get_tabix_file(path, preset='gff'):
    """Return a TabixFile instance for a file at `path`."""
    if not os.path.exists("%s.gz.tbi" % path):
        pysam.tabix_index(path, preset=preset, keep_original=True)
    tabix_file = pysam.TabixFile("%s.gz" % path)
    return tabix_file

def to_gff_attributes(s):
    """Convert a GFF attribute string to an OrderedDict."""
    attr_d = OrderedDict()
    attributes = s.split(';')
    for attribute in attributes:
        k, v = attribute.split('=')
        if ',' in v:
            v = v.split(',')
        attr_d[k] = v
    return attr_d

def to_gff_record(r):
    """Convert a string representation of a GFF record to a named tuple."""
    fields = r.split('\t')
    fields[3], fields[4] = int(fields[3]), int(fields[4])
    fields[8] = to_gff_attributes(fields[8])
    return gff_record(*fields)

def gff_record_to_string(gff_record):
    """Convert a `GFF_record` namedtuple to a string representation."""
    fields = list(gff_record)
    fields[3], fields[4] = str(fields[3]), str(fields[4])
    attributes = []
    for k, v in fields[8].items():
        if isinstance(v, list):
            v = ",".join(v)
        attributes.append("%s=%s" % (k, v))
    fields[8] = ";".join(attributes)
    return "%s\n" % "\t".join(fields)

def write_gff_records(records, path):
    """Write GFF_record in records iterable to `path`."""
    with open(path, 'w') as out:
        for r in records:
            out.write(gff_record_to_string(r))

def fetch_records(tabixfile, region):
    """Wrap TabixFile.fetch in try/except clause to ignore region errors."""
    try:
        return tabixfile.fetch(reference=region.seqid,
                               start=int(region.start) - SCAN_SOFT_CLIP_REGION,
                               end=int(region.end) + SCAN_SOFT_CLIP_REGION)

    except ValueError:
        # Happens if region is not present in tabix file, that's fine and expected
        return []

def fill_comparison(putative, controls, treatment):
    """Convert tabix record to a Comparison named tuple."""
    for putative_event in putative.fetch():
        putative_record = to_gff_record(putative_event)
        putative_record_complements = [to_gff_record(r) for r in fetch_records(treatment, putative_record)]
        control_records = []
        for control in controls:
            for r in fetch_records(control, putative_record):
                control_records.append(to_gff_record(r))
        yield (comparison(putative_record, putative_record_complements, control_records))

def annotate_with_overlapping_insertions(putative_record, control_records):
    """If overlapping insertion of the same type is found list them in the overlaps attribute."""
    putative_insert_reference = putative_record.attributes.get('insert_reference_name')
    overlapping_ids = []
    for control_record in control_records:
        if control_record.source == 'findcluster' and control_record.attributes.get('insert_reference_name') == putative_insert_reference:
            overlapping_ids.append(control_record.attributes.get('ID'))
    putative_record.attributes['overlaps'] = ",".join(overlapping_ids)
    return putative_record

def filter_putative_insertions(putative, treatment, controls, min_length, output_discarded_records=True):
    """Remove insertions that are based on clipped sequences which are also present in the control."""
    for (putative_record, putative_complements, control_records) in fill_comparison(putative, controls, treatment):
        valid_record = True
        clips = []
        control_clips = []
        putative_record = annotate_with_overlapping_insertions(putative_record, control_records)
        softclip_clusters = putative_record.attributes.get('softclip_clusters')
        if not softclip_clusters:
            yield putative_record
            continue
        for putative_complement in putative_complements:
            if putative_complement.type in ('3p_clip', '5p_clip') and putative_complement.attributes['ID'] in softclip_clusters:
                # We only verify clipped sequences that are part of the insertion to verify
                clips.append(putative_complement)
        for control_record in control_records:
            if control_record.type in ('3p_clip', '5p_clip'):
                control_clips.append(control_record)
        for c in control_clips:
            for t in clips:
                if c.type == t.type:
                    if abs(c.start - t.start) < SCAN_SOFT_CLIP_REGION or abs(c.end - t.end) < SCAN_SOFT_CLIP_REGION:
                        c_seq = c.attributes.get('consensus')
                        t_seq = t.attributes.get('consensus')
                        if sequences_match(seq1=c_seq, seq2=t_seq, compare=c.type, min_length=min_length):
                            valid_record = False
                            break
            if valid_record is False:
                break
        if valid_record:
            yield putative_record
        elif output_discarded_records:
            putative_record.attributes['FAIL'] = 'clip_seq_matches_%s' % c.attributes.get('ID')
            yield putative_record

def sequences_match(seq1, seq2, compare='5p_clip', min_length=4):
    """
    Compare 2 sequences and determine whether they are likely the same.

    >>> s1, s2 = 'AAAAAAAAAA', 'AAAAAAAAATGC'
    >>> sequences_match(s1, s2, compare='3p_clip')
    True
    >>> sequences_match('TTGAAAAAAA', s2, compare='3p_clip')
    False
    >>> sequences_match(s1[::-1], s2[::-1], compare='5p_clip')
    True
    >>> sequences_match('A', 'A')
    False
    """
    if compare == '3p_clip':
        seq1 = seq1[:10]
        seq2 = seq2[:10]
    else:
        seq1 = seq1[-10:]
        seq2 = seq2[-10:]
    if len(seq1) > min_length and len(seq2) > min_length:
        if len(seq1) >= 10 and len(seq2) >= 10:
            if align(seq1, seq2)['editDistance'] < 2:
                return True
        elif compare == '3p_clip':
            return seq1.startswith(seq2) or seq2.startswith(seq1)
        else:
            return seq1.endswith(seq2) or seq2.endswith(seq1)
    return False

def confirm_insertions(putative_insertions_path, all_treatments_path, all_controls_paths, output_path, min_length=4, output_discarded_records=True):
    """Confirm putative insertions by absence of softclipping patterns in a somatic control file."""
    putative = get_tabix_file(putative_insertions_path)
    treatment = get_tabix_file(all_treatments_path)
    controls = [get_tabix_file(p) for p in all_controls_paths if not p == all_treatments_path]
    gff_record_iterator = filter_putative_insertions(putative=putative,
                                                     treatment=treatment,
                                                     controls=controls,
                                                     min_length=min_length,
                                                     output_discarded_records=output_discarded_records)
    write_gff_records(gff_record_iterator, path=output_path)


# python -m flask --app server run --port 5000 --host="0.0.0.0" --cert="adhoc"
from flask import Flask, Response, send_from_directory, request
from flask_cors import CORS
import base64
import time
from recognition import recognize
import os
import io
import recognition
app = Flask(__name__)
CORS(app)

@app.route('/static/<path:path>')
def index(path):
    return send_from_directory('static', path)

@app.get("/api/v1/words")
def get_words():
    return {"words": recognition.words}

@app.post("/api/v1/words")
def post_word():
    recognition.words = request.json["words"]
    return {"success": True}

@app.post("/api/v1/recognize-speech")
def recognizeSpeech():
    t0 = time.time()
    id = int(request.json["id"])
    audio_file = f"audios/temp_{id}.wav"

    continued = request.json["continued"]

    wav_file = open(audio_file, "wb")
    decode_string = base64.b64decode(request.json["content"].split(",")[1])
    wav_file.write(decode_string)
    wav_file.close()
    # file = io.BytesIO(decode_string)
    text, word = recognize(audio_file, continued, id)
    os.remove(audio_file)
    return {"word": word, "text": text + " ", "time": time.time()-t0}

app.run(port=5000, host="0.0.0.0", ssl_context=(
    './certificates/cert.pem', './certificates/key.pem'))


"""Show data labels"""
import gin

from explain.actions.utils import get_parse_filter_text

@gin.configurable
def show_labels_operation(conversation, parse_text, i, n_features_to_show=float("+inf"), **kwargs):
    """Generates text that shows labels."""
    y_values = conversation.temp_dataset.contents['y']
    intro_text = get_parse_filter_text(conversation)

    if len(y_values) == 0:
        return "There are no instances in the data that meet this description.", 0
    if len(y_values) == 1:
        label = y_values.item()
        label_text = conversation.get_class_name_from_label(label)
        return_string = f"{intro_text} the label is {label_text}."
    else:
        return_string = f"{intro_text} the labels are:

"
        for index, label in zip(list(y_values.index), y_values):
            label_text = conversation.get_class_name_from_label(label)
            return_string += f"id {index} is labeled {label_text}
"

    return return_string, 1


from sklearn.impute import KNNImputer
from utils import *
import matplotlib.pyplot as plt
import scipy.spatial.distance as scidist
import pandas as pd
import numpy as np

#used this function to test out different distance metrics
def cdist_single_scidist(Xa, Xb, missing_values):
    return scidist.jaccard(Xa.T, Xb.T)

 #‘braycurtis’, ‘canberra’, ‘chebyshev’, ‘cityblock’, ‘correlation’, ‘cosine’, ‘dice’, ‘euclidean’, ‘hamming’, ‘jaccard’, ‘jensenshannon’, ‘kulsinski’, ‘mahalanobis’, ‘matching’, ‘minkowski’, ‘rogerstanimoto’, ‘russellrao’, ‘seuclidean’, ‘sokalmichener’, ‘sokalsneath’, ‘sqeuclidean’, ‘wminkowski’, ‘yule
def sparse_matrix_evaluate_item(data, matrix, threshold=0.5):
    """ Given the sparse matrix represent, return the accuracy of the prediction on data.

    :param data: A dictionary {user_id: list, question_id: list, is_correct: list}
    :param matrix: 2D matrix
    :param threshold: float
    :return: float
    """
    total_prediction = 0
    total_accurate = 0
    for i in range(len(data["is_correct"])):
        cur_user_id = data["user_id"][i]
        cur_question_id = data["question_id"][i]
        if matrix[cur_question_id, cur_user_id] >= threshold and data["is_correct"][i]:
            total_accurate += 1
        if matrix[cur_question_id, cur_user_id] < threshold and not data["is_correct"][i]:
            total_accurate += 1
        total_prediction += 1
    return total_accurate / float(total_prediction)

def knn_impute_by_user(matrix, valid_data, k):
    """ Fill in the missing values using k-Nearest Neighbors based on
    student similarity. Return the accuracy on valid_data.

    See https://scikit-learn.org/stable/modules/generated/sklearn.
    impute.KNNImputer.html for details.

    :param matrix: 2D sparse matrix
    :param valid_data: A dictionary {user_id: list, question_id: list,
    is_correct: list}
    :param k: int
    :return: float
    """
    nbrs = KNNImputer(n_neighbors=k)
    # We use NaN-Euclidean distance measure.
    mat = nbrs.fit_transform(matrix)
    acc = sparse_matrix_evaluate(valid_data, mat)
    print("Validation Accuracy: {}".format(acc))
    return acc

def knn_impute_by_item(matrix, valid_data, k):
    """ Fill in the missing values using k-Nearest Neighbors based on
    question similarity. Return the accuracy on valid_data.

    :param matrix: 2D sparse matrix
    :param valid_data: A dictionary {user_id: list, question_id: list,
    is_correct: list}
    :param k: int
    :return: float
    """
    #the underlying assumption of this function is that if  question A has the same correct and incorrect answers from other users as question B,
    #(the same students who got question A correct would also get question B correct), question A's correctness for a specific user would match that of that users correctness on question B

    nbrs = KNNImputer(n_neighbors=k)

    #in the implementation above, they take the sparse matrix as-is (where students are rows and questions are columns) to predict based on students. If we want to predict based on questions, we should transpose the matrix
    #transpose the matrix so questions are now the samples and users are the features

    matrix_t = matrix.transpose()
    mat = nbrs.fit_transform(matrix_t)
    acc = sparse_matrix_evaluate_item(valid_data, mat)
    print("Validation Accuracy: {}".format(acc))
    #####################################################################
    # TODO:                                                             #
    # Implement the function as described in the docstring.             #
    #####################################################################

    #####################################################################
    #                       END OF YOUR CODE                            #
    #####################################################################
    return acc

def main():
    sparse_matrix = load_train_sparse("./data").toarray()
    val_data = load_valid_csv("./data")
    test_data = load_public_test_csv("./data")
    train_data = load_train_csv("./data")

    print("Sparse matrix:")
    print(sparse_matrix)
    print("Shape of sparse matrix:")
    print(sparse_matrix.shape)

    ks = range(1,30)

    #compute validation accuracy for each k and plot as a function of k
    accuracy = []
    for k in ks:
      acc = knn_impute_by_user(sparse_matrix, val_data, k)
      accuracy.append(acc)

    print(accuracy)
    #plot
    plt.plot(ks, accuracy)
    #plt.title("K-NN alidation accuracy (impute user) as a function of k")
    plt.xlabel('k')
    plt.ylabel('Validation Accuracy')
    plt.show()

    #choose the k with the highest accuracy
    k_star = ks[np.argmax(accuracy)]

    #report the test accuracy using k_star

    test_accuracy = knn_impute_by_user(sparse_matrix, test_data, k_star)

    print("the test accuracy on K* for the impute by user model is: " + str(test_accuracy))

    ##now complete a) and b) for the version "impute by item"

    accuracy_item = []
    for k in ks:
      acc_item = knn_impute_by_item(sparse_matrix, val_data, k)
      accuracy_item.append(acc_item)

    #plot
    plt.plot(ks, accuracy_item)
    #plt.title("K-NN validation accuracy (impute item) as a function of k")
    plt.xlabel('k')
    plt.ylabel('Validation Accuracy')
    plt.show()

    #choose the k with the highest accuracy

    k_star_item = ks[np.argmax(accuracy_item)]

    #report the test accuracy using k_star

    test_accuracy_item = knn_impute_by_item(sparse_matrix, test_data, k_star_item)

    print("the test accuracy on K* for the impute by item model is: " + str(test_accuracy_item))

    #####################################################################
    # TODO:                                                             #
    # Compute the validation accuracy for each k. Then pick k* with     #
    # the best performance and report the test accuracy with the        #
    # chosen k*.                                                        #
    #####################################################################
    pass
    #####################################################################
    #                       END OF YOUR CODE                            #
    #####################################################################

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8; -*-
#

import re

from HTMLParser import HTMLParser

from lxml import etree
from StringIO import StringIO
from lxml.html import fromstring

class TravianParser:

    def __init__(self):

        self.res = re.compile('([0-9]+)/([0-9]+)	')
        self.build = re.compile('build.php\?id=([0-9]+)')
        self.profile = re.compile('spieler.php\?uid=([0-9]+)')
        self.building = re.compile('([a-zA-Z ]+) level ([0-9]+)')
        self.movements = re.compile("([0-9]+)([a-zA-Z.]+)in([0-9\:]+)Hrs")
        self.coords = re.compile("\(([0-9\-]+)\|([0-9\-]+)\)")

        self.parser = etree.HTMLParser()

    def parse(self, html):
        #return etree.parse(StringIO(html), self.parser)
        return fromstring(html)

    def getRallyPoint(self, doc):

        persons = {}

        my = {}
        renforcement = {}
        prisoners = {}

        for table in doc.cssselect("table.troop_details"):

            village, owner = [a.text_content() for a in table.cssselect("thead td a")]
            tr = [tr for tr in table.cssselect('tbody.units tr')]
            names = [img.get("title") for img in tr[0].cssselect('td img')]
            units = [td.text_content() for td in tr[1].iter(tag="td")]

            troops = {}

            for name, number in zip(names, units):
                if name not in troops:
                    troops[name] = int(number)

            if owner == "Own troops":
                my[village] = troops
            elif "Troops imprisoned from" in owner:
                owner = owner.replace("Troops imprisoned from ", "").strip()
                if not owner in prisoners:
                    prisoners[owner] = {}

                prisoners[owner][village] = troops
            else:
                owner = owner.replace("'s troops", "").strip()
                if not owner in renforcement:
                    renforcement[owner] = {}

                renforcement[owner][village] = troops

        return [my, renforcement, prisoners]

    # spieler.php?uid=xxx ONLY !
    def getVillages(self, doc):
        villages = []

        table = doc.get_element_by_id("villages")
        for tr in table.iter(tag="tr"):
            name, coords, population = "", "", ""
            for td in tr.iter(tag="td"):
                if "class" in td.attrib:
                    cl = td.get("class")

                    if cl == "nam":
                        name = td.text_content()

                    elif cl == "hab":
                        population = td.text_content()
                    elif cl == "aligned_coords":
                        text = td.text_content().replace("\n", "").replace("\t", "")
                        coords = self.coords.findall(text)[0]

            if name and population and coords:
                villages.append([name, population, coords])

        return villages

    # ALL PAGES !
    def getProfileUid(self, doc):
        uid = 0

        div = doc.get_element_by_id("side_navi")
        for a in div.iter(tag="a"):
            href = a.get("href")

            if self.profile.match(href):
                uid = self.profile.findall(href)[0]

        return uid

    ### ALL PAGES !
    def getRes(self, doc):

        res = []

        div_res = doc.get_element_by_id("resWrap")
        for td in div_res.iter(tag='td'):
            title = td.get('title')
            id = td.get('id')
            text = td.text_content()

            if id in ["l1", "l2", "l3", "l4"]:
                current, capacity = text.split("/")
                res.append([id, int(title), int(current), int(capacity)])
                #self.Village.resources.update(id, production=int(title), value=int(current), capacity=int(capacity))

        return res

    ### dorf1.php ONLY !
    def getTroops(self, doc):
        troops = []
        try:
            div_troops = doc.get_element_by_id("troops")
            for tr in div_troops.iter(tag="tr"):
                unit_name, unit_number = "", ""
                for td in tr.iter(tag="td"):
                    if "class" in td.attrib:
                        cl = td.get("class")
                        if cl == "num":
                            unit_number = td.text_content()
                        elif cl == "un":
                            unit_name = td.text_content()

                    if unit_name and unit_number:
                        troops.append([unit_name, unit_number])

        except:
            return []
        else:
            return troops

    ### dorf1.php ONLY !
    def getFields(self, doc):
        fields = []

        div = doc.get_element_by_id("rx")
        for area in div.iter(tag='area'):
            title = area.get("title")
            href = area.get("href")
            farm, level, id = "", 0, 0

            if self.building.match(title):
                farm, level = self.building.findall(title)[0]
            if self.build.match(href):
                id = self.build.findall(href)[0]

            if farm and level and id:
                fields.append([id, farm, level])

        return fields

    ### dorf1.php ONLY !
    def getMovements(self, doc):

        try:
            table = doc.get_element_by_id("movements")
        except KeyError:
            return []
        else:
            move = []
            for tr in table.iter(tag="tr"):
                href, type, time, number = "", "", "", 0

                for link in tr.iter(tag="a"):
                    if "href" in link.attrib:
                        href = link.get("href")

                for td in tr.iter(tag="td"):
                    text = str(td.text_content()).encode('ascii', 'ignore')
                    if len(text) > 2:
                        if self.movements.match(text):
                            number, type, time = self.movements.findall(text)[0]
                            if type == "Reinf.":
                                type = "reinforcing"
                            move.append([type, number, time, href])

            return move

    ### dorf2.php ONLY !
    def getVillageBuildings(self, doc):
        buildings = []

        map = doc.get_element_by_id("map2")
        for area in map.iter(tag='area'):
            title = area.get('title')
            href = area.get('href')
            build_id, name, level, = "", "", 0

            if title and href:
                if self.build.match(href):
                    build_id = self.build.findall(href)[0]

                if self.building.match(title):
                    name, level = self.building.findall(title)[0]

                buildings.append([build_id, name, level])
                #self.Village.updateBuilding(build_id, name=name, level=level)

        return buildings

    ### ALL PAGES !
    def getVillageName(self, doc):
        name = ""

        for h1 in doc.iter(tag='h1'):
            title = h1.text_content()
            if title:
                name = title

        return name

    ### ALL PAGES !
    def getServerTime(self, doc):
        server_time = ""
        calculated = ""
        div_time = doc.get_element_by_id("ltimeWrap")
        tmp = div_time.text_content()

        for line in tmp.split("\n"):
            line = line.strip()
            if line is not "":
                if "Server time" in line:
                    server_time = line
                elif "am" in line or "pm" in line:
                    server_time += " %s" % line
                elif "Calculated" in line:
                    calculated = line

        return [server_time, calculated]


from bs4 import BeautifulSoup
import urllib.request, urllib.error, urllib.parse, time

def getAndProcessArtistsPage(artists):
    for currentArtist in artists.split("*"):
        currentArtist = currentArtist.lower().replace(" ", "")
        firstLetter = currentArtist[:1]
        generate_url = 'http://azlyrics.com/' + firstLetter + '/' + currentArtist + ".html"
        alternativeProcessing(generate_url, currentArtist, save=True)

def generating(artist, title, save):
    artist = artist.lower().replace(" ", "")
    title = title.lower().replace(" ", "")
    generate_url = 'http://azlyrics.com/lyrics/' + artist + '/' + title + '.html'
    processSong(generate_url, artist, title, save)

def alternativeProcessing(generate_url, artist, save):
    print("generating url: " + generate_url)
    response = urllib.request.urlopen(generate_url)
    read_artist_page = response.read()
    soup = BeautifulSoup(read_artist_page).find_all("a")
    i = 0
    for x in soup:
        i = i + 1
        completeLink = x.extract()
        stringToCompare = '<a href="../lyrics/' + artist
        if completeLink.decode().startswith(stringToCompare):
            sep = '/'
            currentSongName = completeLink.decode().split(sep, 3)[3]
            sep = '.html'
            currentSongName = currentSongName.split(sep, 1)[0]
            print("processing " + artist + "'s song " + currentSongName + "...")
            generating(artist, currentSongName, save)

def processSong(generate_url, artist, title, save):
    response = urllib.request.urlopen(generate_url)
    read_lyrics = response.read()
    soup = BeautifulSoup(read_lyrics)
    lyrics = soup.find_all("div", attrs={"class": None, "id": None})
    lyrics = [x.getText() for x in lyrics]
    printing(artist, title, save, lyrics)

def printing(artist, title, save, lyrics):
    for x in lyrics:
        print(x)
    if save:
        saving(artist, title, lyrics)
    else:
        pass

def saving(artist, title, lyrics):
    f = open(artist + '_' + title + '.txt', 'w')
    f.write("\n".join(lyrics).strip())
    f.close()
    time.sleep(15) #in order to avoid being disconnected from the server


import os
from tempfile import TemporaryFile
from xlwt import Workbook
from .sheet import Sheet

class Excel(object):
    def __init__(self):
        self.book = Workbook(encoding='utf-8')
        self.sheets = []

    def add_sheet(self, sheet):
        if not isinstance(sheet, Sheet):
            raise Exception('sheet need be instance Sheet')

        self.sheets.append(sheet)
        sheet._set_book(self.book)

    def save(self, file_name, dir=''):
        if not '.xls' in file_name:
            file_name += '.xls'

        file = dir + file_name
        self.create_dirs_if_not_existed(file)

        self.book.save(TemporaryFile())
        self.book.save(file)

    @staticmethod
    def create_dirs_if_not_existed(filename):
        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError as exc:
                raise Exception()


import json

def read_data_json(filename):
    with open(filename, 'r') as f:
        return json.load(f)

new_data = {}
sentences = []
equ_dict = {}
for elem in read_data_json("../../data/train23k_processed.json"):
    sentence = elem['text'].strip().split(' ')
    equation = elem['target_norm_post_template'][2:]  # .strip().split(' ')
    for equ_e in equation:
        if equ_e not in equ_dict:
            equ_dict[equ_e] = 1
    sentences.append(sentence)
    for elem in sentence:
        new_data[elem] = new_data.get(elem, 0) + 1
token_list = ['PAD_token', 'UNK_token', 'END_token']
ext_list = ['PAD_token', 'END_token']
for k, v in sorted(new_data.items()):
    token_list.append(k)

for equ_k in sorted(equ_dict.keys()):
    ext_list.append(equ_k)
print("encode_len:", len(token_list), "decode_len:", len(ext_list))
print("de:", ext_list)


import os
import argparse
import tensorflow as tf
from datetime import datetime
from object_detection import model_lib
from object_detection import model_hparams
from object_detection.protos import pipeline_pb2

def tensorflow_shutup(verbose=False):
    """
    Make Tensorflow less verbose
    """
    try:
        # noinspection PyPackageRequirements
        if not verbose:
            #import os
            from tensorflow.compat.v1 import logging
            logging.set_verbosity(logging.ERROR)
            os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

        # Monkey patching deprecation utils to shut it up! Maybe good idea to disable this once after upgrade
        # noinspection PyUnusedLocal
        def deprecated(date, instructions, warn_once=True):
            def deprecated_wrapper(func):
                return func
            return deprecated_wrapper

        from tensorflow.python.util import deprecation
        deprecation.deprecated = deprecated

    except ImportError:
        pass

def find_latest_checkpoint(log_dir, modelId):
    from glob import glob

    log_dir = os.path.normpath(log_dir)
    runs = glob(f"{log_dir}/{modelId}*")

    if not runs:
        return None

    runs.sort()
    latest_run = runs[-1]

    checkpoint = tf.train.get_checkpoint_state(latest_run)
    return checkpoint.model_checkpoint_path if checkpoint else None

parser = argparse.ArgumentParser(description="Train model from pipeline configs")
parser.add_argument('-o', '--out_dir', required=True,
                    help="Path to output model directory where event and checkpoint files will be written.")
parser.add_argument('-p', '--pipeline_config', required=True, help="Path to pipeline config file.")
parser.add_argument('--num_train_steps', required=False, default=None, type=int,
                    help="Number of training steps (overrides pipeline).")
parser.add_argument('--max_checkpoints', required=False, default=None, type=int,
                    help="Maximum checkpoints to save. Default: no limit")
parser.add_argument('--checkpoint_steps', required=False, default=1000, type=int,
                    help="Save checkpoint at every x steps. Default: every 1000 steps")
parser.add_argument('--sample_1_of_n_eval_examples', required=False, default=1, type=int,
                    help="Will sample one of every n eval input examples, where n is provided.")
parser.add_argument('-l', '--logs', required=False, help="Path to logs directory")
parser.add_argument('-v', '--verbose', required=False, action='store_true')

args = parser.parse_args()

tensorflow_shutup(args.verbose)

timestamp = datetime.now().strftime("-%Y%m%d-%H%M%S-%f")
modelId = os.path.basename(os.path.normpath(args.out_dir))
out_dir = args.out_dir+timestamp
log_dir = args.logs if args.logs else out_dir

config_proto = tf.ConfigProto()
config_proto.gpu_options.allow_growth = True
config = tf.estimator.RunConfig(model_dir=out_dir, session_config=config_proto, log_step_count_steps=1,
                                save_checkpoints_steps=args.checkpoint_steps, keep_checkpoint_max=args.max_checkpoints)

pipeline_overrides = None
latest_checkpoint = find_latest_checkpoint(log_dir, modelId)
if latest_checkpoint:
    pipeline_overrides = pipeline_pb2.TrainEvalPipelineConfig()
    pipeline_overrides.train_config.fine_tune_checkpoint = latest_checkpoint

train_and_eval_dict = model_lib.create_estimator_and_inputs(
    run_config=config,
    pipeline_config_path=args.pipeline_config,
    config_override=pipeline_overrides,
    train_steps=args.num_train_steps,
    sample_1_of_n_eval_examples=args.sample_1_of_n_eval_examples,
    hparams=model_hparams.create_hparams(None),
    save_final_config=True)

estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
eval_input_fns = train_and_eval_dict['eval_input_fns']
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
predict_input_fn = train_and_eval_dict['predict_input_fn']
train_steps = train_and_eval_dict['train_steps']

train_spec, eval_specs = model_lib.create_train_and_eval_specs(
        train_input_fn,
        eval_input_fns,
        eval_on_train_input_fn,
        predict_input_fn,
        train_steps,
        eval_on_train_data=False)

tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])

# Log that training completed
with open(os.path.join(log_dir, "completed.log"), 'a') as f:
    f.write(f"{modelId},{timestamp}\n")


import json
import time

import requests
import re
from bs4 import BeautifulSoup

import Firm
import Person
from saveLog import saveLog

import warnings
warnings.filterwarnings("ignore")

def crawleOneFilm(IMDBID : str):
    import Film

    url = "https://www.imdb.com/title/" + IMDBID  # 拼接得到URL
    header = {
        "accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "user-agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
        "sec-ch-us" : '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"'
    }
    try:
        resp = requests.get(url = url , headers = header)
    except requests.exceptions.ConnectionError as e:
        time.sleep(10)
        resp = requests.get(url=url, headers=header)
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text , "html.parser")

    try:
        filmName = soup.find("h1" , attrs={"data-testid":"hero-title-block__title"}).text
    except AttributeError as e:
        saveLog("IMDB ID: " + IMDBID + "找不到电影名字")
        return None
    print(" 电影名字: " + filmName)

    film = Film.Film(filmName , IMDBID)

    # release date
    date = findReleaseDate(soup)
    film.setReleaseDate(date)

    # duration
    duration = findDuration(soup)
    film.setDuration(duration)

    # type(genres)
    genres = findGenres(soup)
    film.setFilmType(genres)

    # boxoffice
    boxoffice = findBoxoffice(soup , IMDBID)
    film.setGrossBoxoffice(boxoffice)

    # actor director writer producer productionFirm distributeFirm
    fullCredit = getFullCredits(IMDBID)
    film.actorList = fullCredit['actor']
    film.directList = fullCredit['director']

    # 人员的票房
    _act_box = 0.0
    for i in film.actorList:
        _act_box += i.totalActorFilmBox
    film.totalActorBox = _act_box

    _dir_box = 0
    for i in film.directList:
        _dir_box += i.totalDirectFilmBox
    film.totalDirectBox = _dir_box

    # 公司的票房
    fullComp = getFullComp(IMDBID)
    film.productionFirm = fullComp['production']

    # 公司的票房
    _prod_firm_box = 0
    for i in film.productionFirm:
        _prod_firm_box += i.totalFilmBox
    film.totalProducFirmBox = _prod_firm_box

    return film

def findReleaseDate(soup) -> str:
    try:
        return soup.find("section" , attrs={"data-testid":"Details"}).find("li" , attrs = {"data-testid":"title-details-releasedate"}).find("div").text.replace("," , " ")
    except AttributeError as e:
        saveLog(e.args[0])
        return ""

def findDuration(soup) -> int:
    try:
        raw_dura = soup.find("section" , attrs = {"data-testid":"TechSpecs"}).find("li" , attrs = {"data-testid":"title-techspec_runtime"}).find("div").text
        return procRawDuartion(raw_dura)
    except AttributeError as e:
        saveLog(e.args)
        return -1

def procRawDuartion(string) -> int:
    '''

    :param string:
    :return: -1 表示有错误
    '''
    try:
        li = re.split(r"hours|hour" , string.replace(" " , "").replace("minutes" , "").replace("minutes" , ""))
        if li[1] == "":
            li[1] = "0"
        return eval(li[0]) * 60 + eval(li[1])
    except IndexError as e:
        saveLog(e.args)
        return -1

def findGenres(soup) -> list:
    try:
        result = []
        for i in soup.find("div", attrs={"data-testid": "genres"}).find_all("a"):
            result.append(i.text)
        return result
    except AttributeError as e:
        saveLog(e.args)
        return [""]

def findBoxoffice(soup , IMDBID) -> float:
    try:
        raw_box = soup.find("section" , attrs = {"data-testid":"BoxOffice"}).find("li" , attrs = {"data-testid":"title-boxoffice-cumulativeworldwidegross"}).find("ul").text
        return procBox(raw_box , IMDBID)
    except AttributeError as e:
        saveLog(e.args[0])
        return 0.0

def procBox(string , IMDBID) -> float:
    # moneyType = 1 if string[0] == '$' else -1
    if string[0] != "$":
        saveLog(IMDBID + "不是美元")
    return float(string[1:].replace("," , ""))

def getFullCredits(IMDBID) -> dict:
    with open("./ing_data/person.json") as f:
        person_json = json.load(f)

    fullList = {
        'actor': [],
        'director': []
    }
    url = "https://www.imdb.com/title/"+IMDBID+"/fullcredits"
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    }
    try:
        resp = requests.get(url=url, headers=header)
    except requests.exceptions.ConnectionError as e:
        time.sleep(10)
        resp = requests.get(url=url, headers=header)
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text)
    try:
        title_l = soup.find("div", attrs={'id': 'fullcredits_content'}).find_all("h4")
        table_l = soup.find("div", attrs={'id': 'fullcredits_content'}).find_all("table")
    except AttributeError as e:
        saveLog(" IMDBID : " + IMDBID + " , Error : " + e.args[0])
        return fullList

    if len(title_l) != len(table_l):
        saveLog(" IMDBID: " + IMDBID + ", 在获取fullcredit时发生两个list长度不相等的问题\n")
        return fullList

    actor_list = []
    directer_list = []

    for i , u in enumerate(title_l):
        text = u.attrs.get("id")
        if text == "cast":
            c_list = table_l[i].find_all("tr" , class_ = ["odd","even"])
            for j,v in enumerate(c_list):
                if j >=5: # 只获取前五个演员的票房
                    break
                try:
                    name = v.find_all("td")[1].text.strip()
                    ID = re.findall('nm\d+' , v.find_all("a")[1].attrs.get("href"))[0]
                    print("  演员 名字 : " + name)
                    if ID not in person_json.keys():
                        person = Person.Person(name, ID , True) # 传参为True，表示这个人没有记录，需要爬取
                        temp = {
                            "actor": person.totalActorFilmBox,
                            "director": person.totalDirectFilmBox
                        }
                        person_json[ID] = temp
                        with open("./ing_data/person.json" , "w") as f:
                            json.dump(person_json , f)
                    else:
                        person = Person.Person(name , ID , False)
                        person.totalActorFilmBox = person_json[ID]["actor"]
                        person.totalDirectFilmBox = person_json[ID]["director"]
                    actor_list.append(person)
                except AttributeError as e:
                    saveLog(" IMDBID : " + IMDBID + " , Error : " + e.args[0])
                    continue
                except ConnectionError as e:
                    saveLog(" IMDBID : " + IMDBID + " , Error : " + e.args[0])
                    continue

        elif text == "director":
            d_tr_list = table_l[i].find_all("tr")
            for j, v in enumerate(d_tr_list):
                try:
                    name = v.find("a").text.strip()
                    ID = re.findall(r"nm\d+", v.find("a").attrs.get("href"))[0]
                    print("  导演 名字 : " + name)
                    if ID not in person_json.keys():
                        person = Person.Person(name, ID)
                        temp = {
                            "actor": person.totalActorFilmBox,
                            "director" : person.totalDirectFilmBox
                        }
                        person_json[ID] = temp
                        with open("./ing_data/person.json", "w") as f:
                            json.dump(person_json, f)
                    else:
                        person = Person.Person(name, ID, False)
                        person.totalActorFilmBox = person_json[ID]["actor"]
                        person.totalDirectFilmBox = person_json[ID]["director"]
                    directer_list.append(person)
                except AttributeError as e:
                    saveLog(" IMDBID : " + IMDBID + " , Error : " + e.args[0])
                    continue
                except ConnectionError as e:
                    saveLog(" IMDBID : " + IMDBID + " , Error : " + e.args[0])
                    continue

    fullList['actor'] = actor_list
    fullList['director'] = directer_list

    return fullList

def getFullComp(IMDBID) -> dict:
    with open("./ing_data/firm.json") as f:
        current_firm = json.load(f)

    url = "https://www.imdb.com/title/"+IMDBID+"/companycredits"
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    }
    try:
        resp = requests.get(url=url, headers=header)
    except requests.exceptions.ConnectionError as e:
        time.sleep(10)
        resp = requests.get(url=url, headers=header)

    resp.raise_for_status()

    soup = BeautifulSoup(resp.text)

    try:
        soup_content = soup.find("div" , attrs = {"id":"company_credits_content"})
    except AttributeError as e:
        saveLog(" IMDBID : " + IMDBID + " 时找不到company_credits_content, Error : " + e.args[0])
        return {'production': []}

    soup_h4 = soup_content.find_all("h4" , recursive= False)
    soup_ul = soup_content.find_all("ul" , recursive= False)

    prod_comp = []

    for i , v in enumerate(soup_h4):
        if v.attrs.get("id") == "production":
            a_s = soup_ul[i].find_all("a")
            for j in a_s:
                firm_name = j.text
                firm_id = re.findall(r"/company/co[0-9]+" , j.attrs.get("href"))[0]
                firm_id = firm_id.replace("/company/" , "")
                print("  公司 名字 : " + firm_name)
                if firm_id in current_firm.keys():
                    firm_temp = Firm.Firm(firm_name , firm_id , False)
                    firm_temp.totalFilmBox = current_firm[firm_id]
                else:
                    firm_temp = Firm.Firm(firm_name, firm_id)
                    current_firm[firm_id] = firm_temp.totalFilmBox
                    with open('./ing_data/firm.json', 'w') as file:
                        json.dump(current_firm, file)
                prod_comp.append(firm_temp)

    fullComp = {
        'production': prod_comp
    }
    return fullComp

# info =  crawleOneFilm("tt7362036")
# print(info)
# i = 1


import tkinter as tk
from tkinter import filedialog
import cv2
import pandas as pd
from PIL import Image, ImageTk
import os
from tkinter import filedialog, ttk  # ttk is imported for Treeview

class VideoPlayer:
    def __init__(self, window, window_title):
        self.window = window
        self.window.title(window_title)

        # Set window to full-screen mode
        self.is_full_screen = True
        self.window.attributes('-fullscreen', self.is_full_screen)

        # Bind the ESC key to the toggle_full_screen method
        self.window.bind('<Escape>', self.toggle_full_screen)

        # Bind mouse click event on the main window
        self.window.bind("<Button-1>", self.on_window_click)

        # Main frame to hold the video and log side by side
        self.main_frame = tk.Frame(window)
        self.main_frame.pack(fill=tk.BOTH, expand=True)

        # Frame for the Treeview
        self.log_frame = tk.Frame(self.main_frame)
        self.log_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)

        # Treeview for displaying log data
        self.log_view = ttk.Treeview(self.log_frame, columns=("Index", "Event", "Frame"), show="headings")
        self.log_view.column("Index", width=40, anchor=tk.CENTER)
        self.log_view.column("Event", anchor=tk.CENTER)
        self.log_view.column("Frame", anchor=tk.CENTER)
        self.log_view.heading("Index", text="Index")
        self.log_view.heading("Event", text="Event")
        self.log_view.heading("Frame", text="Frame")
        self.log_view.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)

        # Adding vertical scrollbar for the Treeview
        self.tree_scroll = tk.Scrollbar(self.log_frame, orient="vertical", command=self.log_view.yview)
        self.tree_scroll.pack(side='right', fill='y')

        # Configure the Treeview to use the scrollbar
        self.log_view.configure(yscrollcommand=self.tree_scroll.set)

        # Canvas for video playback
        self.canvas = tk.Canvas(self.main_frame, width=1600, height=900)
        self.canvas.pack(side=tk.RIGHT, fill=tk.BOTH, expand=True)

        # Frame number display
        self.frame_number = tk.Entry(window)
        self.frame_number.pack()

        # Bind Enter key in the frame number entry to jump_to_frame method
        self.frame_number.bind("<Return>", self.jump_to_frame)

        # Jump to frame button
        self.btn_jump = tk.Button(window, text="Jump to Frame", command=self.jump_to_frame)
        self.btn_jump.pack()

        # Slider for frame navigation
        self.slider = tk.Scale(window, from_=0, to=100, orient=tk.HORIZONTAL, length=1900, command=self.slider_used)
        self.slider.pack(fill=tk.X)

        # Event selection using radio buttons
        self.event_var = tk.StringVar()
        self.event_options = {
            "0: Pinprick": "0",
            "1: 0.07g": "1",
            "2: 0.4g": "2",
            "3: 2g": "3",
            "4: Cold water": "4",
            "5: Room temp": "5",
            "6: Hot water": "6"
        }
        self.radio_frame = tk.Frame(window)
        self.radio_frame.pack()
        for (text, value) in self.event_options.items():
            tk.Radiobutton(self.radio_frame, text=text, variable=self.event_var, value=value).pack(side=tk.LEFT)

        # Log button
        self.btn_log = tk.Button(window, text="Log Event", command=self.log_event)
        self.btn_log.pack()

        # Load video button
        self.btn_load = tk.Button(window, text="Load Video", command=self.load_video)
        self.btn_load.pack()

        # Frame navigation buttons
        self.btn_backward_1800 = tk.Button(window, text="Backward 1800 frames", command=lambda: self.jump_frames(-1800))
        self.btn_backward_1800.pack(side=tk.LEFT)

        self.btn_backward_900 = tk.Button(window, text="Backward 900 frames", command=lambda: self.jump_frames(-900))
        self.btn_backward_900.pack(side=tk.LEFT)

        self.btn_backward_100 = tk.Button(window, text="Backward 100 frames", command=lambda: self.jump_frames(-100))
        self.btn_backward_100.pack(side=tk.LEFT)

        self.btn_backward_50 = tk.Button(window, text="Backward 50 frames", command=lambda: self.jump_frames(-50))
        self.btn_backward_50.pack(side=tk.LEFT)

        self.btn_backward_10 = tk.Button(window, text="Backward 10 frames", command=lambda: self.jump_frames(-10))
        self.btn_backward_10.pack(side=tk.LEFT)

        self.btn_backward_1 = tk.Button(window, text="Backward 1 frames", command=lambda: self.jump_frames(-1))
        self.btn_backward_1.pack(side=tk.LEFT)

        self.btn_forward_1 = tk.Button(window, text="Forward 1 frames", command=lambda: self.jump_frames(1))
        self.btn_forward_1.pack(side=tk.LEFT)

        self.btn_forward_10 = tk.Button(window, text="Forward 10 frames", command=lambda: self.jump_frames(10))
        self.btn_forward_10.pack(side=tk.LEFT)

        self.btn_forward_50 = tk.Button(window, text="Forward 50 frames", command=lambda: self.jump_frames(50))
        self.btn_forward_50.pack(side=tk.LEFT)

        self.btn_forward_100 = tk.Button(window, text="Forward 100 frames", command=lambda: self.jump_frames(100))
        self.btn_forward_100.pack(side=tk.LEFT)

        self.btn_forward_900 = tk.Button(window, text="Forward 900 frames", command=lambda: self.jump_frames(900))
        self.btn_forward_900.pack(side=tk.LEFT)

        self.btn_forward_1800 = tk.Button(window, text="Forward 1800 frames", command=lambda: self.jump_frames(1800))
        self.btn_forward_1800.pack(side=tk.LEFT)

        self.video_source = None
        self.vid = None
        self.log_data = []

    def load_video(self):
        self.video_source = filedialog.askopenfilename()
        if self.video_source:
            self.vid = cv2.VideoCapture(self.video_source)
            total_frames = int(self.vid.get(cv2.CAP_PROP_FRAME_COUNT))
            self.slider.configure(to=total_frames - 1)
            self.update()

    def slider_used(self, event):
        if self.vid:
            frame_no = int(self.slider.get())
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.update_frame_number(frame_no)
            self.update()

    def jump_frames(self, frame_count):
        if self.vid:
            frame_no = int(self.frame_number.get() or 0) + frame_count
            frame_no = max(0, min(frame_no, self.vid.get(cv2.CAP_PROP_FRAME_COUNT) - 1))
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.slider.set(frame_no)
            self.update_frame_number(frame_no)
            self.update()

    def log_event(self):
        if self.vid and self.event_var.get():
            current_frame = int(self.vid.get(cv2.CAP_PROP_POS_FRAMES))
            event_text = [key for key, value in self.event_options.items() if value == self.event_var.get()][0]
            event_index = len(self.log_data) + 1  # Index for the new event
            self.log_data.append([event_index, event_text, current_frame])
            print(f"Logged Event: {event_text}, Frame: {current_frame}")

            # Insert log data into Treeview with index
            new_entry = self.log_view.insert('', 'end', values=(event_index, event_text, current_frame))

            # Ensure the new entry is visible
            self.log_view.see(new_entry)

            df = pd.DataFrame(self.log_data, columns=['Index', 'Event', 'Frame'])
            df.to_excel(os.path.splitext(self.video_source)[0] + "_log.xlsx", index=False)

    def update_frame_number(self, frame_no):
        self.frame_number.delete(0, tk.END)
        self.frame_number.insert(0, str(frame_no))

    def update(self):
        if self.vid:
            ret, frame = self.vid.read()
            if ret:
                frame = self.resize_frame(frame, 1600, 900)
                self.photo = ImageTk.PhotoImage(image=Image.fromarray(frame))
                self.canvas.create_image(0, 0, image=self.photo, anchor=tk.NW)

    def resize_frame(self, frame, width, height):
        (h, w) = frame.shape[:2]
        r = width / float(w)
        dim = (width, int(h * r))
        resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
        return cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)

    def __del__(self):
        if self.vid:
            self.vid.release()

    def toggle_full_screen(self, event=None):
        self.is_full_screen = not self.is_full_screen
        self.window.attributes('-fullscreen', self.is_full_screen)
        if not self.is_full_screen:
            self.window.state('zoomed')  # This maximizes the window

    def on_window_click(self, event):
        """Handle mouse click events on the window to remove focus from Entry widget."""
        # Check if the click is outside the Entry widget
        if event.widget != self.frame_number:
            self.window.focus_set()

    def jump_to_frame(self, event=None):
        # event parameter added to handle the Enter key event
        if self.vid:
            frame_no = int(self.frame_number.get())
            frame_no = max(0, min(frame_no, self.vid.get(cv2.CAP_PROP_FRAME_COUNT) - 1))
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.slider.set(frame_no)
            self.update()
            self.window.focus_set()

# Create a window and pass it to the video player
root = tk.Tk()
VideoPlayer(root, "Tkinter and OpenCV with Event Logger")
root.mainloop()


import numpy as np

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
from scipy.special import comb

matplotlib.rcParams['mathtext.fontset'] = 'stix'
matplotlib.rcParams['font.family'] = 'STIXGeneral'
plt.rcParams["figure.figsize"] = [5,3]

# Import Files
init_state = "UpZero"
Larray = np.arange(4, 11, 2)
T = 1
J = [0.0001, 0.001, 0.01, 0.1, 1.0]
V = 3
hz = 16
alpha = [0.50, 3.00]
kick = 0.00 #0.01
num_J = len(J)
num_alpha = len(alpha)
n_points = num_J * num_alpha
numL = len(Larray)
iter2 = 20480
steps = 10000

m = 0

filename = np.empty(( numL, num_J, num_alpha), dtype=object)
files = np.empty(( numL, num_J, num_alpha), dtype=object)
data = np.empty(( numL, num_J, num_alpha), dtype=object)

sigmaz = np.empty(( numL, num_J, num_alpha), dtype=object)
Z = np.empty(( numL, num_J, num_alpha), dtype=object)

for i in range(num_J):
    for j in range(num_alpha):
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])
            n_iter = iter2 * 2 ** (1 - L / 2)
            #dim_Sz0 = comb(L, L / 2)

#            filename[q, i, j] = 'data/eigen/quasienergies_Swap_LR_Sz0_nspin%d_period%.2f_n_disorder%d_Jxy%.5f_Vzz%.2f_hz%.2f_alpha%.3f_kick%.2f.txt' % (
#                L, T, n_iter, J[i], V, hz, alpha[j], kick)
            filename[ q, i, j] = "data/dynamics/sigmaz_Swap_LR_%s_nspin%d_steps%d_period%.2f_n_disorder%d_J%.5f_V%.2f_hz%.2f_kick%.3f_alpha%.2f.txt" % (
                init_state, L, steps, T, n_iter, J[i], V, hz, kick, alpha[j])

            print(filename[ q, i, j])
            files[ q, i, j] = np.genfromtxt(filename[ q, i, j], skip_header=8)
            print(files[ q, i, j])
            data[ q, i, j] = files[ q, i, j]
            print(data[ q, i, j][0, :])

for i in range(num_J):
    for j in range(num_alpha):
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])
            n_iter = iter2 * 2 ** (1 - L / 2)
            #dim_Sz0 = comb(L, L / 2)
            sigmaz[ q, i, j] = data[ q, i, j][:, 1:L+1]
            Z[ q, i, j] = (np.sign(sigmaz[ q, i,j][0,0::2] - sigmaz[q,i,j][0,1::2]) * (sigmaz[q,i,j][:,0::2] - sigmaz[q,i,j][:,1::2])).sum( axis=1 ) / L
            print(Z[ q,i,j][0:4],"\n")

X = np.empty((numL, num_J, num_alpha), dtype=object)
Y = np.empty((numL, num_J, num_alpha), dtype=object)
signs = np.empty((steps,))
signs[::2] = +1
signs[1::2] = -1

plt.rcParams["axes.prop_cycle"] = plt.cycler("color", plt.cm.rainbow(np.linspace(0,1,numL)))

import matplotlib.cm as cm
from matplotlib.colors import to_hex

cmL = cm.get_cmap('turbo', numL)

for i in range(num_J):
    for j in range(num_alpha):
        m += 1
        plt.figure(m)
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])

            X[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)-1] - 0.5) + 0.5
            Y[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)] - 0.5) + 0.5
            print(sigmaz[q,i,j][0,:])
            print(sigmaz[q,i,j][0,2*(L//4)], sigmaz[q,i,j][0,L-1])
            string = '$L = %d$' % L
            plt.plot(range(1,steps+1), X[q,i,j], label=string, color=to_hex(cmL(q)))
            plt.plot(range(1,steps+1), Y[q,i,j], label=string, linewidth=0.8, linestyle='--', color=to_hex(cmL(q)))

        plt.xlabel('$t$', fontsize=12)
        plt.ylabel('$(-1)^t(\langle\overline{S_k^z}(t)\\rangle-0.5)+0.5$', fontsize=12)
        plt.xticks(fontsize=10)
        plt.yticks(fontsize=10)
        plt.xscale('log')
        plt.ylim([-0.1,1.1])
        plt.legend(loc='lower center', bbox_to_anchor=(0.5, -0.45), fancybox=True, shadow=True, ncol=numL)

        txt = 'figures/sigmaz_avg_Dynamics_%s_Comparison_wrt_L_J%.5f_alpha%.2f_kick%.3f.pdf' % (init_state, J[i], alpha[j], kick)
        print(txt)
        plt.savefig(txt, dpi=600, bbox_inches='tight')
        print("\n")

#end = 20
#for i in range(num_J):
#    for j in range(num_alpha):
#        m += 1
#        plt.figure(m)
#        for q in range(numL):
#            L = Larray[q]
#            print([J[i], alpha[j], L])
#
#            #X[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)] - 0.5) + 0.5
#            #Y[q, i, j] = signs * (sigmaz[q,i,j][:,L-1] - 0.5) + 0.5
#            X[q, i, j] = sigmaz[q,i,j][:,2*(L//4)]
#            Y[q, i, j] = sigmaz[q,i,j][:,L-1]
#            print(sigmaz[q,i,j][0,:])
#            print(sigmaz[q,i,j][0,2*(L//4)], sigmaz[q,i,j][0,L-1])
#            string = '$L = %d$' % L
#            plt.plot(range(1,end+1), X[q,i,j][0:end], label=string, color=to_hex(cmL(q)))
#            #plt.plot(range(1,end+1), Y[q,i,j][0:end-1], label=string, linewidth=0.8, linestyle='--', color=to_hex(cmL(q)))
#
#
#        plt.xlabel('$t$', fontsize=12)
#        plt.ylabel('$(-1)^t(\overline{\sigma_k^z}(t)-0.5)+0.5$', fontsize=12)
#        plt.xticks(fontsize=10)
#        plt.yticks(fontsize=10)
#        plt.xscale('log')
#        plt.ylim([-0.1,1.1])
#        plt.legend(loc='lower center', bbox_to_anchor=(0.5, -0.45), fancybox=True, shadow=True, ncol=numL)
#
#        txt = 'figures/sigmaz_PARTIAL_avg_Dynamics_%s_Comparison_wrt_L_J%.5f_alpha%.2f_kick%.3f.pdf' % (init_state, J[i], alpha[j], kick)
#        print(txt)
#        plt.savefig(txt, dpi=600, bbox_inches='tight')
#        print("\n")


# 计算变量之间的互信息
"""
怎么计算故障之间的互信息啊？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？？

"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cap

data = pd.read_csv('TE_Train_data.csv')  # 导入原始数据
data.iloc[:, :52] = cap.normalize(data.iloc[:, :52])  # 特征归一化

MI = []  # 储存每个特征的互信息
for j in range(52):
    a = np.log(np.var(data.iloc[:, j]))

    b = 0
    for i in range(22):
        data_x = data.loc[data['V52'] == i].iloc[:, j]  # 第i个类别的第j个特征
        b += data_x.shape[0] / data.shape[0] * np.log(np.var(data_x))

    MI.append(1/2 * (a - b))

plt.bar(range(len(MI)), MI)
plt.xlabel('Variables')
plt.ylabel('Mutual Information')
plt.show()


import numpy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras import backend as K
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn import preprocessing
from sklearn.cross_validation import train_test_split
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt
import keras.layers.advanced_activations
import tensorflow as tf;
from tensorflow.python.tools import freeze_graph, optimize_for_inference_lib

MODEL_NAME = 'regression_model'

# load dataset
columns = ['dayOfWeek', 'hour', 'operationType', 'fileType', 'parentFolder', 'fileSize',
           'predessorFile1', 'predessorFile2', 'predessorFile3', 'predessorFile4', 'filename',
           'successorFile1', 'successorFile2', 'successorFile3', 'successorFile4']

dataframe = pandas.read_csv("/home/anuradha/PycharmProjects/data/fyp/final/test-after-filtered.csv", header=None, names=columns)
# dataset = dataframe.values
dataset = dataframe.values

dataframe.drop(dataframe.columns[[0, 1,5]], axis=1, inplace=True)

print(dataframe.describe())

# split into input (X) and output (Y) variables
X = dataset[:, 0:9]
Y = dataset[:, 8]

# fix random seed for reproducibility
seed = 7

# evaluate model with standardized dataset
numpy.random.seed(seed)

scaler = Normalizer().fit(X)
normalizedX = scaler.transform(X)

# normalizedX = preprocessing.scale(X)

# summarize transformed data
# numpy.set_printoptions(precision=3)
print(normalizedX[0:7,:])

# create model
model = Sequential()
model.add(Dense(9, input_dim=9, kernel_initializer='normal', activation="relu", use_bias= True))
model.add(Dense(1, kernel_initializer='normal', activation="relu"))

model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy'])

X_train, X_test, y_train, y_test = train_test_split(normalizedX, Y)

model.fit(X_train, y_train, batch_size=20, epochs=2000, verbose=2)
score = model.evaluate(X_test, y_test, batch_size=1, verbose=2)

print("score : ",score)
model.save("models/regression_model.h5")

prediction = model.predict(X_test)
print (prediction)
prediction = prediction.astype(int)

plt.show()

print(prediction[:10])
print (y_test[:10])

def export_model(saver, model, input_node_names, output_node_name):
    tf.train.write_graph(K.get_session().graph_def, 'out', \
                         MODEL_NAME + '_graph.pbtxt')

    saver.save(K.get_session(), 'out/' + MODEL_NAME + '.chkp')

    freeze_graph.freeze_graph('out/' + MODEL_NAME + '_graph.pbtxt', None, \
                              False, 'out/' + MODEL_NAME + '.chkp', output_node_name, \
                              "save/restore_all", "save/Const:0", \
                              'out/frozen_' + MODEL_NAME + '.pb', True, "")

    input_graph_def = tf.GraphDef()
    with tf.gfile.Open('out/frozen_' + MODEL_NAME + '.pb', "rb") as f:
        input_graph_def.ParseFromString(f.read())

    output_graph_def = optimize_for_inference_lib.optimize_for_inference(
        input_graph_def, input_node_names, [output_node_name],
        tf.float32.as_datatype_enum)

    with tf.gfile.FastGFile('out/opt_' + MODEL_NAME + '.pb', "wb") as f:
        f.write(output_graph_def.SerializeToString())

    print("graph saved!")

    return


import tensorflow as tf
from .yolov4 import YOLO, decode, filter_boxes
from .utils import load_config, load_weights

def convert_darknet_to_tf(model_files, weights="yolov4-tiny_best.weights", input_size=416, score_threshold=.2, tiny=False, framework="tf"):
    strides, anchors, num_class, xyscale = load_config(f"{model_files}obj.names", tiny)
    input_layer = tf.keras.layers.Input([input_size, input_size, 3])
    feature_maps = YOLO(input_layer, num_class, tiny)
    bbox_tensors = []
    prob_tensors = []
    if tiny:
        for i, fm in enumerate(feature_maps):
            if i == 0:
                output_tensors = decode(fm, input_size // 16, num_class, strides, anchors, i, xyscale, framework)
            else:
                output_tensors = decode(fm, input_size // 32, num_class, strides, anchors, i, xyscale, framework)
            bbox_tensors.append(output_tensors[0])
            prob_tensors.append(output_tensors[1])
    else:
        for i, fm in enumerate(feature_maps):
            if i == 0:
                output_tensors = decode(fm, input_size // 8, num_class, strides, anchors, i, xyscale, framework)
            elif i == 1:
                output_tensors = decode(fm, input_size // 16, num_class, strides, anchors, i, xyscale, framework)
            else:
                output_tensors = decode(fm, input_size // 32, num_class, strides, anchors, i, xyscale, framework)
            bbox_tensors.append(output_tensors[0])
            prob_tensors.append(output_tensors[1])
    pred_bbox = tf.concat(bbox_tensors, axis=1)
    pred_prob = tf.concat(prob_tensors, axis=1)
    if framework == "tflite":
        pred = (pred_bbox, pred_prob)
    else:
        boxes, pred_conf = filter_boxes(
            pred_bbox, pred_prob, score_threshold=score_threshold,
            input_shape=tf.constant([input_size, input_size]))
        pred = tf.concat([boxes, pred_conf], axis=-1)
    model = tf.keras.Model(input_layer, pred)
    load_weights(model, f"{model_files}{weights}", tiny)
    model.summary()
    if framework == "tf":
        output = f"{model_files}{weights.replace('.weights', '_tf')}"
    elif framework == "tflite":
        output = f"{model_files}{weights.replace('.weights', '_tflite')}"
    else:
        output = f"{model_files}{weights.replace('.weights', '_trt')}"
    model.save(output)


# Author: Allen Anker
# Created by Allen Anker on 20/09/2018
"""
Given an array of integers, find if the array contains any duplicates.

Your function should return true if any value appears at least twice in the array,
and it should return false if every element is distinct.
"""
from collections import defaultdict

class Solution:
    def contains_duplicate(self, nums):
        """
        :type nums: List[int]
        :rtype: bool
        """
        seen = defaultdict()
        for i in nums:
            if seen.get(i):
                return True
            else:
                seen[i] = 1
        return False

solution = Solution()
nums = [1, 2, 3]
print(solution.contains_duplicate(nums))


import os
import random
import torch
import numpy as np
import pandas as pd
from types import SimpleNamespace
import time
from utils import PairDataset
from torch_geometric.loader import DataLoader
from models import GNNPair

random.seed(42); np.random.seed(42)

class LRDataset(PairDataset):
    def __init__(self, pair_info_df, single_data_pool, reduce_form):
        super().__init__(pair_info_df, single_data_pool, reduce_form)

    def __getitem__(self, idx):
        row = self.info.iloc[idx, ]
        ligand_file = row['ligand_file']
        receptor_file = row['receptor_file']

        ligand_data = self.single_data_pool[ligand_file]
        receptor_data = self.single_data_pool[receptor_file]

        # emb
        ligand_data.emb = torch.tensor(
            np.load(os.path.join(self.emb_path, f'{ligand_file}.npy')), dtype=torch.float32)
        receptor_data.emb = torch.tensor(
            np.load(os.path.join(self.emb_path, f'{receptor_file}.npy')), dtype=torch.float32)
        if self.reduce_func is not None:
            ligand_data.emb = self.reduce_func(ligand_data.emb, dim=0, keepdim=True)
            receptor_data.emb = self.reduce_func(receptor_data.emb, dim=0, keepdim=True)

        return ligand_data, receptor_data, row['ligand'], row['receptor']

if __name__ == '__main__':
    species = "Human"
    args = {
        'device': 'cuda',
        'batch_size': 80
    }
    args = SimpleNamespace(**args)

    # dataloader
    df_potential_lr = pd.read_csv(r'./data/df/df_potential_lr.csv')
    df_ligand = df_potential_lr.query('lr_type == "ligand"').reset_index(drop=True)[['gene', 'name']]
    df_ligand.columns = ['ligand', 'ligand_file']
    df_receptor = df_potential_lr.query('lr_type == "receptor"').reset_index(drop=True)[['gene', 'name']]
    df_receptor.columns = ['receptor', 'receptor_file']
    df_comb = df_ligand.merge(df_receptor, how='cross')

    single_data_pool = {data['name']: data for data in torch.load('./data/graph/human_v4_6_alpha.pt')}

    comb_data = LRDataset(df_comb, single_data_pool, 'sum')
    dataloader = DataLoader(comb_data, batch_size=args.batch_size)
    print('total combinations:', len(dataloader.dataset))

    # model related
    model = GNNPair(input_dim=128, hidden_dim=128, num_layers=1, reduce_form='sum')
    weight_path = r'./result/human_v4_6_alpha_GNNPair/4_acc0.900_roc0.965_prc0.959_recall0.893.pt'
    model.load_state_dict(torch.load(weight_path))
    print('load weight:', weight_path)
    model.to(args.device)
    model.eval()
    print(model)

    # get result
    ligand_ls = []
    receptor_ls = []
    pred_ls = []
    result_file_path = f'../network_filtering/lr_database_raw.csv'
    for batch_idx, (ligand, receptor, l_gene, r_gene) in enumerate(dataloader):
        ligand = ligand.to(args.device)
        receptor = receptor.to(args.device)

        ligand_ls += l_gene
        receptor_ls += r_gene
        with torch.no_grad():
            pred = model(ligand, receptor).reshape(-1).detach().cpu().numpy().tolist()
            pred_ls += pred

        condition1 = len(ligand_ls) >= 20000
        condition2 = batch_idx == len(dataloader)-1  # final batch
        if condition1 or condition2:
            if os.path.exists(result_file_path):
                df0 = pd.read_csv(result_file_path)
            else:
                df0 = pd.DataFrame(columns=['ligand', 'receptor', 'pred'])
            df = pd.DataFrame({'ligand': ligand_ls, 'receptor': receptor_ls, 'pred': pred_ls})
            df = pd.concat((df0, df), ignore_index=True)
            df.to_csv(result_file_path, index=False)

            print(time.strftime('%H:%M:%S'), 'finished num: ', len(df))
            ligand_ls = []
            receptor_ls = []
            pred_ls = []


#
# aggregate_video.py
#
# Aggregate results and render output video for a video that's already been run through MD
#

#%% Constants

import os
import json
from detection.process_video import ProcessVideoOptions
from detection.video_utils import frames_to_video, video_to_frames
from visualization import visualize_detector_output

video_file = os.path.expanduser('~/1_fps_20211216_101100.mp4')
results_file_raw = os.path.expanduser('~/1_fps_20211216_101100.mp4.json')
allowed_categories = ['1']
max_box_size = 0.15
min_box_size = 0.03

results_file = results_file_raw.replace('.json','_animals_only.json')

#%% Processing

with open(results_file_raw,'r') as f:

    n_detections = 0
    n_valid_detections = 0
    min_valid_confidence = 1.0

    d = json.load(f)
    # im = d['images'][0]

    for im in d['images']:

        valid_detections = []
        max_detection_conf = 0

        for det in im['detections']:
            n_detections += 1
            det_size = None
            if 'bbox' in det:
                bbox = det['bbox']
                det_size = bbox[2] * bbox[3]
            if det['category'] in allowed_categories and \
                ((det_size is not None) and (det_size < max_box_size) and \
                 (det_size > min_box_size)):
                if det['conf'] > max_detection_conf:
                    max_detection_conf = det['conf']
                if det['conf'] < min_valid_confidence:
                    min_valid_confidence = det['conf']
                valid_detections.append(det)
                n_valid_detections += 1

        # ...for each detection

        im['detections'] = valid_detections

        # This is no longer included in output files by default
        if 'max_detection_conf' in im:
            im['max_detection_conf'] = max_detection_conf

    print('Kept {} of {} detections (min conf {})'.format(
        n_valid_detections,n_detections,min_valid_confidence))

    with open(results_file,'w') as f:
        json.dump(d,f,indent=2)

base_dir = os.path.expanduser('~/frame_processing')
min_confidence = 0.001

assert os.path.isfile(video_file) and os.path.isfile(results_file)
os.makedirs(base_dir,exist_ok=True)

## Split into frames

options = ProcessVideoOptions()
options.input_video_file = video_file
options.output_json_file = results_file
options.output_video_file = options.input_video_file + '.detections.mp4'

frame_output_folder = os.path.join(
    base_dir, os.path.basename(options.input_video_file) + '_frames')
if os.path.isdir(frame_output_folder):
    print('Frame output folder exists, skipping frame extraction')
else:
    os.makedirs(frame_output_folder, exist_ok=True)

    frame_filenames, Fs = video_to_frames(
        options.input_video_file, frame_output_folder)

## Render output video

### Render detections to images

rendering_output_dir = os.path.join(
    base_dir, os.path.basename(options.input_video_file) + '_detections')
os.makedirs(rendering_output_dir,exist_ok=True)
detected_frame_files = visualize_detector_output.visualize_detector_output(
    detector_output_path=options.output_json_file,
    out_dir=rendering_output_dir,
    images_dir=frame_output_folder,
    confidence=min_confidence)

### Combine into a video

Fs = 20
frames_to_video(detected_frame_files, Fs, options.output_video_file)


from os.path import join

from django.test import TestCase
from django.conf import settings
from django.db import connection
from django.contrib.auth import get_user_model

from ..forms import CommunitiesFilterForm

User = get_user_model()

class TestCommunitiesFilterForm(TestCase):

    COMMUNITIES_NUM = 200

    def setUp(self):
        super().setUp()

        communities_ids = range(self.COMMUNITIES_NUM)

        with open(join(settings.BASE_DIR, '..', 'schema.sql'), encoding='utf-8') as fd:
            sql = fd.read()
        connection.cursor().execute(sql)

        sql = r'INSERT INTO "community" ("vkid","deactivated","type","name","description","status","site","members") VALUES'
        sql += ','.join(
            "({0:d},FALSE,1,'NAME_{0:d}','DESCRIPTION_{0:d}','STATUS_{0:d}','SITE_{0:d}',{1:d})".format(cid,
                                                                                cid * 1000 + 1)
            for cid in communities_ids
        )
        sql += ';'
        connection.cursor().execute(sql)

        sql = (r'INSERT INTO "profile" ("id","sex_vkid","age_range_id","country_vkid","app_id") '
               r'VALUES (1,1,1,1,1)')
        connection.cursor().execute(sql)

        sql = r'INSERT INTO "audience" ("profile_id","community_vkid","count") VALUES'
        sql += ','.join("(1,{0:d},{1:d})".format(cid, cid * 1000 + 1) for cid in communities_ids)
        sql += ';'
        connection.cursor().execute(sql)

    def test_correct_data(self):
        data = {
            'min_members': 0,
            'max_members': 1000000,

            'min_audience': 0,
            'max_audience': 1000000,

            'min_audience_perc': 0,
            'max_audience_perc': 100,

            'sex': [1],
            'age_ranges': [1],
            'countries': [1],
            'apps': [1],

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertTrue(form.is_valid())

    def test_constrains(self):
        data = {
            'min_members': -1,
            'max_members': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience': -1,
            'max_audience': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': -1,
            'max_audience_perc': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': 0,
            'max_audience_perc': 101,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

    def test_ranges(self):
        data = {
            'min_members': 10,
            'max_members': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience': 10,
            'max_audience': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': 10,
            'max_audience_perc': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())


import torch
import torch.nn as nn

# https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3
class Extractor(nn.Module):
    def __init__(self):
        super(Extractor, self).__init__()
        self.padding = (1, 0, 0)
        self.kernel_size = (3, 3, 3)
        self.dilation = (1, 1, 1)
        self.stride = (1, 1, 1)
        self.activation = nn.LeakyReLU()

        self.bn1 = nn.BatchNorm3d(4, affine=True)
        self.conv1 = nn.Conv3d(in_channels=4,
                               out_channels=4,
                               kernel_size=self.kernel_size,
                               padding=self.padding,
                               stride=self.stride)
        self.conv2 = nn.Conv3d(in_channels=4,
                               out_channels=8,
                               kernel_size=self.kernel_size,
                               padding=(2, 2, 2))
        self.pool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=(1, 2, 2))

        self.bn2 = nn.BatchNorm3d(8, affine=True)
        self.conv3 = nn.Conv3d(in_channels=8,
                               out_channels=8,
                               padding=self.padding,
                               kernel_size=self.kernel_size,
                               stride=self.stride)
        self.conv4 = nn.Conv3d(in_channels=8,
                               out_channels=16,
                               kernel_size=self.kernel_size,
                               padding=(2, 2, 2))

        self.bn3 = nn.BatchNorm3d(16, affine=True)
        self.conv5 = nn.Conv3d(in_channels=16,
                               out_channels=16,
                               kernel_size=self.kernel_size,
                               padding=self.padding,
                               stride=self.stride)
        self.conv6 = nn.Conv3d(in_channels=16,
                               out_channels=32,
                               kernel_size=self.kernel_size,
                               padding=(2, 2, 2))

    def forward(self, x):
        x = self.bn1(x)
        y = self.conv1(x)
        y = self.activation(y)
        y = self.conv2(y)
        y = self.pool(y)

        y = self.bn2(y)
        y = self.conv3(y)
        y = self.activation(y)
        y = self.conv4(y)
        y = self.activation(y)
        y = self.pool(y)

        y = self.bn3(y)
        y = self.conv5(y)
        y = self.activation(y)
        y = self.conv6(y)
        y = self.activation(y)
        y = self.pool(y)

        return y.view((x.shape[0], -1, x.shape[2]))

# m = Extractor()
# x = torch.rand((32, 4, 12, 24, 72))
# y = m(x)
# print(y.size())


from django.urls import path,include
from rest_framework.authtoken.views import ObtainAuthToken

from rest_framework_simplejwt.views import (
    TokenObtainPairView,
    TokenRefreshView,
)

from . import views
from rest_framework.routers import DefaultRouter

router = DefaultRouter()

app_name = 'api'
urlpatterns = [

    #Function views
    #path('', name='home'),
    # path('parts/',views.allparts,name='allparts'),
    # path('parts/<str:pk>/',views.partdetail,name='partdetail'),
    # path('suppliers/',views.allsuppliers,name='allsuppliers'),
    # path('suppliers/<int:pk>/',views.supplierdetail,name='supplierdetail'),

    #Class API views

    path('',include(router.urls)),
    path('djangotoken/',ObtainAuthToken.as_view(),name='authview'),   # Django Token
    path('jwtoken/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
    path('jwtoken/refresh/', TokenRefreshView.as_view(), name='token_refresh'),

    path('parts/',views.AllpartsAPIView.as_view(),name='allparts'),
    path('parts/<str:pk>/',views.PartdetailAPIView.as_view(),name='partdetail'),
    path('suppliers/',views.AllsuppliersAPIView.as_view(),name='allsuppliers'),
    path('suppliers/<int:id>/',views.SupplierdetailAPIView.as_view(),name='supplierdetail'),  #custom Class-based views require actual fiield name here, not PK

    #Generic Views

    # path('parts/',views.PartsListAPIView.as_view(),name='allparts'),
    # path('parts/<str:pk>/',views.RetrieveUpdateDeletePart.as_view(),name='partdetail'),
    # path('suppliers/',views.SuppliersListAPIView.as_view(),name='allsuppliers'),
    # path('suppliers/<int:pk>/',views.RetrieveUpdateDeleteSupplier.as_view(),name='supplierdetail'),

]

 ######  Routers


from flask import Flask, render_template, url_for, request, redirect
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
import random

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
functionDatabase = SQLAlchemy(app)

create_post = False
title = 'Šaralica'
func_search = ''

class FunctionSite(functionDatabase.Model):
    id = functionDatabase.Column(functionDatabase.Integer, primary_key=True)
    name = functionDatabase.Column(functionDatabase.String(30), nullable=False, default=' ')
    description = functionDatabase.Column(functionDatabase.String(600), nullable=False, default=' ')
    language = functionDatabase.Column(functionDatabase.String(12), nullable=False, default=' ')
    date_created = functionDatabase.Column(functionDatabase.DateTime, default=datetime.utcnow)
    bookmark = functionDatabase.Column(functionDatabase.Boolean, default=False)

    def __repr__(self):
        return '<Site %r>' % self.id

@app.route('/', methods=['POST', 'GET'])
def index():
    functions = FunctionSite.query.order_by(FunctionSite.name).all()
    card_generated = False
    bookmarked = []

    for function in functions:
        if function.bookmark:
            if not card_generated:
                card_generated = True
            bookmarked.append(function)

    marked = random.choice(bookmarked)

    if request.method == 'POST':
        global func_search
        func_search = request.form['search']
        try:
            return redirect('/pretraga')
        except:
            return 'Error: Pretraga neuspješna :('

    else:
        return render_template('index.html', create_post=create_post, title=title, functions=functions, generated=card_generated, marked=marked)

@app.route('/otvaranje')
def open():
    global create_post, title
    create_post = True
    title = 'Izrada'
    return redirect('/')
@app.route('/uređivanje/<int:id>', methods=['GET', 'POST'])
def open_edit(id):
    function_to_edit = FunctionSite.query.get_or_404(id)

    if request.method == 'POST':
        function_to_edit.description = request.form['description']
        function_to_edit.name = request.form['name']
        function_to_edit.language = request.form['language']

        try:
            functionDatabase.session.commit()
            return redirect('/')

        except:
            return 'Error: Uređivanje neuspješno :('

    else:
        return render_template('update.html', function=function_to_edit)

@app.route('/zatvaranje')
def close():
    global create_post, title
    create_post = False
    title = 'Šaralica'
    return redirect('/')

@app.route('/izrada', methods=['POST', 'GET'])
def create():
    if request.method == 'POST':
        func_description = request.form['description']
        func_name = request.form['name']
        func_language = request.form['language']
        new_func = FunctionSite(description=func_description, name=func_name, language=func_language)

        try:
            functionDatabase.session.add(new_func)
            functionDatabase.session.commit()
            return redirect('/')
        except:
            return 'Error: Izrada neuspješna :('

    else:
        return render_template('index.html', functions=functions)

@app.route('/pretraga')
def search():
    global func_search
    global isFound
    isFound = False
    functions = FunctionSite.query.order_by(FunctionSite.name).all()

    for function in functions:
        if (func_search in function.name or func_search in function.language) or (function.name in func_search or function.language in func_search):
            isFound = True

    return render_template('search.html', search=func_search, functions=functions, isFound=isFound)

@app.route('/brisanje/<int:id>')
def delete(id):
    function_to_delete = FunctionSite.query.get_or_404(id)

    try:
        functionDatabase.session.delete(function_to_delete)
        functionDatabase.session.commit()
        return redirect('/')
    except:
        return 'Error: Brisanje neuspješno :('

@app.route('/označivanje/<int:id>')
def bookmark(id):
    function_to_mark = FunctionSite.query.get_or_404(id)

    try:
        if function_to_mark.bookmark:
            function_to_mark.bookmark = False
        else:
            function_to_mark.bookmark = True
        functionDatabase.session.commit()
        return redirect('/')
    except:
        return 'Error: Označivanje neuspješno :('

@app.route('/definicija/<int:id>', methods=['GET', 'POST'])
def function(id):
    function = FunctionSite.query.get_or_404(id)
    return render_template('function.html', function=function)

if __name__ == "__main__":
    app.run(debug=True)


import os
import sys

HERE = os.path.dirname(os.path.realpath(__file__))
BUILD_DIR = os.path.join(HERE, "../../")
CORE_DIR = os.path.join(BUILD_DIR, "pybmix/core/")
sys.path.insert(0, os.path.realpath(BUILD_DIR))
sys.path.insert(0, os.path.realpath(CORE_DIR))

import numpy as np
import matplotlib.pyplot as plt
from pybmix.core.mixing import DirichletProcessMixing
from pybmix.core.hierarchy import PythonHierarchy
from pybmix.core.mixture_model import MixtureModel
from pybmix.estimators.density_estimator import DensityEstimator

np.random.seed(2021)

def sample_from_mixture(weights, means, sds, n_data):
    n_comp = len(weights)
    clus_alloc = np.random.choice(np.arange(n_comp), p=[0.5, 0.5], size=n_data)
    return np.random.normal(loc=means[clus_alloc], scale=sds[clus_alloc])

y = sample_from_mixture(np.array([0.5, 0.5]), np.array([-3, 3]), np.array([1, 1]), 200)

plt.hist(y, bins=20)
plt.show()

mixing = DirichletProcessMixing(total_mass=5)  # DP mixing

hierarchy = PythonHierarchy("LapNIG_Hierarchy")

mixture = MixtureModel(mixing, hierarchy)

niter = 110
nburn = 10
mixture.run_mcmc(y, algorithm="Neal8", niter=niter, nburn=nburn)

grid = np.linspace(-6, 6, 500)
dens_est = DensityEstimator(mixture)
densities = dens_est.estimate_density(grid)

plt.hist(y, density=True, bins=20)
plt.plot(grid, np.mean(densities, axis=0), lw=3, label="predictive density")
plt.legend()
plt.show()
idxs = [int((niter - nburn) * 0.1), int((niter - nburn) * 0.2), int((niter - nburn) * 0.9)]

for idx in idxs:
    plt.plot(grid, densities[idx, :], "--", label="iteration: {0}".format(idx))
    plt.legend()
    plt.show()


# -*- coding: utf-8 -*-
#

from .ansible.inventory import BaseInventory
from assets.utils import get_assets_by_id_list, get_system_user_by_id

__all__ = [
    'JMSInventory'
]

class JMSInventory(BaseInventory):
    """
    JMS Inventory is the manager with jumpserver assets, so you can
    write you own manager, construct you inventory
    """
    def __init__(self, assets, run_as_admin=False, run_as=None, become_info=None):
        """
        :param host_id_list: ["test1", ]
        :param run_as_admin: True 是否使用管理用户去执行, 每台服务器的管理用户可能不同
        :param run_as: 是否统一使用某个系统用户去执行
        :param become_info: 是否become成某个用户去执行
        """
        self.assets = assets
        self.using_admin = run_as_admin
        self.run_as = run_as
        self.become_info = become_info

        host_list = []

        for asset in assets:
            info = self.convert_to_ansible(asset, run_as_admin=run_as_admin)
            host_list.append(info)

        if run_as:
            run_user_info = self.get_run_user_info()
            for host in host_list:
                host.update(run_user_info)

        if become_info:
            for host in host_list:
                host.update(become_info)
        super().__init__(host_list=host_list)

    def convert_to_ansible(self, asset, run_as_admin=False):
        info = {
            'id': asset.id,
            'hostname': asset.hostname,
            'ip': asset.ip,
            'port': asset.port,
            'vars': dict(),
            'groups': [],
        }
        if asset.domain and asset.domain.has_gateway():
            info["vars"].update(self.make_proxy_command(asset))
        if run_as_admin:
            info.update(asset.get_auth_info())
        for node in asset.nodes.all():
            info["groups"].append(node.value)
        for label in asset.labels.all():
            info["vars"].update({
                label.name: label.value
            })
            info["groups"].append("{}:{}".format(label.name, label.value))
        if asset.domain:
            info["vars"].update({
                "domain": asset.domain.name,
            })
            info["groups"].append("domain_"+asset.domain.name)
        return info

    def get_run_user_info(self):
        system_user = self.run_as
        if not system_user:
            return {}
        else:
            return system_user._to_secret_json()

    @staticmethod
    def make_proxy_command(asset):
        gateway = asset.domain.random_gateway()
        proxy_command_list = [
            "ssh", "-p", str(gateway.port),
            "-o", "StrictHostKeyChecking=no",
            "{}@{}".format(gateway.username, gateway.ip),
            "-W", "%h:%p", "-q",
        ]

        if gateway.password:
            proxy_command_list.insert(
                0, "sshpass -p '{}'".format(gateway.password)
            )
        if gateway.private_key:
            proxy_command_list.append("-i {}".format(gateway.private_key_file))

        proxy_command = "'-o ProxyCommand={}'".format(
            " ".join(proxy_command_list)
        )
        return {"ansible_ssh_common_args": proxy_command}


from django.urls import path
from django.conf.urls.static import static

from sgdtaxi import settings
from .views import index, PropietarioAPi, getDepartamentos, VehiculoApi, PropietarioUpdateAPi, \
    VehiculoUpdateApi, VinculacionApi, VinculacionDocumentosApi

urlpatterns = [
    path('', index, name='index'),
    path('api/propietario/', PropietarioAPi.as_view()),
    path('api/propietario/<int:pk>/', PropietarioAPi.as_view()),
    path('api/propietario/actualizar/', PropietarioUpdateAPi.as_view()),
    path('api/vehiculo/', VehiculoApi.as_view()),
    path('api/vehiculo/actualizar/', VehiculoUpdateApi.as_view()),
    path('api/vehiculo/<int:doc>/', VehiculoApi.as_view()),
    path('api/vehiculo/<str:placa>/', VehiculoUpdateApi.as_view()),
    path('api/vinculacion/', VinculacionApi.as_view()),
    path('api/vinculacion/<int:pk>/', VinculacionApi.as_view()),
    path('api/vinculacion/documento/', VinculacionDocumentosApi.as_view()),
    path('departamentos/', getDepartamentos),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [("meupet", "0008_auto_20150628_1614")]

    operations = [
        migrations.AddField(
            model_name="pet",
            name="sex",
            field=models.CharField(blank=True, choices=[("FE", "Fêmea"), ("MA", "Macho")], max_length=2),
            preserve_default=True,
        )
    ]


 #!/usr/bin/python

from flask import Flask, jsonify, request, make_response
from uuid import uuid4
from werkzeug.exceptions import HTTPException
from argparse import ArgumentParser

import uidHandler
import indexHandler

app = Flask(__name__)
app.config['TRAP_HTTP_EXCEPTIONS']=True
storedObjects = {}

@app.route('/api/objects/', methods=['GET', 'POST'])
def handleIndexRoute():
    """Sends GET and POST requests to the index route to their respective handler methods.
    Rejects other request types and throws a 405 error.

    Returns:
        Json formatted http response.
    """
    if request.method == 'GET':
        return indexHandler.getUidList(storedObjects)
    elif request.method == 'POST':
        return indexHandler.postObject(storedObjects)

@app.route('/api/objects/<string:uid>', methods=['GET', 'PUT', 'DELETE'])
def handleUidRoute(uid):
    """Sends GET, PUT, and DELETE request types to any route with a supplied Id string.
    Rejects other request types and throws a 405 error.

    Args:
        uid: String suffix to the api route.

    Returns:
        Json formatted http response.
    """
    if request.method == 'GET':
        return uidHandler.getSpecificUid(uid, storedObjects)
    elif request.method == 'PUT':
        return uidHandler.putObject(uid, storedObjects)
    elif request.method == 'DELETE':
        return uidHandler.deleteObject(uid, storedObjects)

@app.errorhandler(HTTPException)
def returnError(error):
    """Constructs and returns an error response in JSON.
    Handles all HTTP exceptions as long as TRAP_HTTP_EXCEPTIONS is set to True.

    Args:
        error: flask error type, specifying a description, code, and name.

    Returns:
        Json formatted HTTP response containing the request's type, URL, and an error message.
    """

    errorObject = {
        "verb" : request.method,
        "url" : request.url,
        "message" : str(error.code) +  ': ' + error.name
    }

    return make_response(jsonify(errorObject), error.code)

if __name__ == '__main__':
    parser = ArgumentParser("Launch the API:  python server.py")
    parser.add_argument('-p', '--port',
        type=int,
        help="The port to listen on. Defaults to 5000.",
        default=5000)
    parser.add_argument('-d', '--debug',
        help="Launch the app in debug mode. The server will only be accessible from the local \
            host for security. Defaults to False.",
        default=False,
        action='store_true')
    args = parser.parse_args()

    if args.debug:
        app.run(debug=args.debug, port=args.port)
    else:
        app.run(host="0.0.0.0", port=args.port)