from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
from threading import *
import logging

class myHandler(BaseHTTPRequestHandler):
    #Handler for the GET requests
    def _set_headers(self):
        self.send_response(200)
        self.send_header('Content-type', 'text/html')
        self.end_headers()

    def do_GET(self):
        if self.path == '/get_storage_tp':
            storage_tp = self.server.mon.get_storage_tp()
            self.wfile.write(json.dumps(storage_tp))
        elif self.path == '/date':
            self.wfile.write("Hello World !")
        else:
            self.wfile.write("Endpoint not supported!")

    def do_POST(self):
        length = self.headers.getheader('content-length')
        data = self.rfile.read(int(length))
        logging.info('Received data:%s', data)
        if self.path == '/newpod':
            self.server.mon.set_monitoring_pods(data)

class Listener:
    def __init__(self,port,mon):
        self._port=port
        self.server = HTTPServer(('', port), myHandler)
        self.server.mon = mon

    def start(self):
        logging.info("Listener is running on port %d", self._port)
        self.server.serve_forever()


import face_recognition
from fer import FER
import cv2
import numpy as np

cap = cv2.VideoCapture(0)
detector = FER(mtcnn=True)
font = cv2.FONT_HERSHEY_DUPLEX

# Check if the webcam is opened correctly
if not cap.isOpened():
    raise IOError("Cannot open webcam")

navaneeth_image = face_recognition.load_image_file("navaneeth.jpg")
navaneeth_face_encoding = face_recognition.face_encodings(navaneeth_image)[0]

# Load a second sample picture and learn how to recognize it.
arjun_image = face_recognition.load_image_file("arjun.jpg")
arjun_face_encoding = face_recognition.face_encodings(arjun_image)[0]

known_face_encodings = [
    navaneeth_face_encoding,
    arjun_face_encoding
]
known_face_names = [
    "navaneeth",
    "arjun"
]

face_locations = []
face_encodings = []
face_names = []
process_this_frame = True

while True:
	ret, frame = cap.read()
	small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
	# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
	rgb_frame = small_frame[:, :, ::-1]

	if process_this_frame:
		# Find all the faces and face encodings in the current frame of video
		face_locations = face_recognition.face_locations(rgb_frame)
		face_encodings = face_recognition.face_encodings(rgb_frame, face_locations)

		face_names = []
		for face_encoding in face_encodings:
			# See if the face is a match for the known face(s)
			matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
			name = "Unknown"

			# Or instead, use the known face with the smallest distance to the new face
			face_distances = face_recognition.face_distance(known_face_encodings, face_encoding)
			best_match_index = np.argmin(face_distances)
			if matches[best_match_index]:
				name = known_face_names[best_match_index]

			face_names.append(name)

	process_this_frame = not process_this_frame

	# Display the results
	for (top, right, bottom, left), name in zip(face_locations, face_names):

		# face_frame = frame[(bottom*4)+35:(top*4)-45,(left*4)-25:(right*4)+35]
		# Scale back up face locations since the frame we detected in was scaled to 1/4 size
		top *= 4
		right *= 4
		bottom *= 4
		left *= 4

		# Draw a box around the face
		cv2.rectangle(frame, (left-25, top-45), (right+35, bottom+35), (0, 0, 255), 2)

		try:
			face_frame = frame[top-45:bottom+35,left-25:right+35]
			face = cv2.resize(face_frame, None, fx=0.25, fy=0.25, interpolation=cv2.INTER_AREA)
			emotion, score = detector.top_emotion(face)
			print(emotion,"\t",score)
			cv2.putText(frame, name+":"+emotion, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1)
		except:
			# print("No Face")
			cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1)

	# Display the resulting image
	cv2.imshow('Video', frame)

	# Hit 'q' on the keyboard to quit!
	if cv2.waitKey(1) & 0xFF == ord('q'):
		break

# Release handle to the webcam
cap.release()
cv2.destroyAllWindows()


#!/usr/bin/env python
"""
Taken from -- kupfer      A convenient access and command tool

Copyright 2007--2010 Ulrik Sverdrup <ulrik.sverdrup@gmail.com>

This source/ is a standalone program that can be used together
with luancher.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import os
import sys
import xml.etree.cElementTree as ElementTree

NEEDED_KEYS= set(("title", "artist", "album", "track-number", "location", ))
UNICODE_KEYS = set(("title", "artist", "album"))

def _tounicode(bstr):
	# the XML parser returns `str' only when it's ascii, but we want
	# unicode objects all the time
	if isinstance(bstr, unicode):
		return bstr
	return bstr.decode("ascii")

def _lookup_string(string, strmap):
	"""Look up @string in the string map,
	and return the copy in the map.

	If not found, update the map with the string.
	"""
	string = string or ""
	try:
		return strmap[string]
	except KeyError:
		strmap[string] = string
		return string

def get_rhythmbox_songs(dbfile, typ="song", keys=NEEDED_KEYS):
	"""Return a list of info dictionaries for all songs
	in a Rhythmbox library database file, with dictionary
	keys as given in @keys.
	"""
	rhythmbox_dbfile = os.path.expanduser(dbfile)

	lSongs = []
	strmap = {}

	# Parse with iterparse; we get the elements when
	# they are finished, and can remove them directly after use.

	for event, entry in ElementTree.iterparse(rhythmbox_dbfile):
		if not (entry.tag == ("entry") and entry.get("type") == typ):
			continue
		info = {}
		for child in entry.getchildren():
			if child.tag in keys:
				if child.tag in UNICODE_KEYS:
					childtext = _tounicode(child.text)
				else:
					childtext = child.text
				tag = _lookup_string(child.tag, strmap)
				text = _lookup_string(childtext, strmap)
				info[tag] = text
		lSongs.append(info)
		entry.clear()
	return lSongs

def sort_album(album):
	"""Sort album in track order"""
	def get_track_number(rec):
		try:
			tnr = int(rec["track-number"])
		except (KeyError, ValueError):
			tnr = 0
		return tnr
	album.sort(key=get_track_number)

def sort_album_order(songs):
	"""Sort songs in order by album then by track number

	>>> songs = [
	... {"title": "a", "album": "B", "track-number": "2"},
	... {"title": "b", "album": "A", "track-number": "1"},
	... {"title": "c", "album": "B", "track-number": "1"},
	... ]
	>>> sort_album_order(songs)
	>>> [s["title"] for s in songs]
	['b', 'c', 'a']
	"""
	def get_album_order(rec):
		try:
			tnr = int(rec["track-number"])
		except (KeyError, ValueError):
			tnr = 0
		return (rec["album"], tnr)
	songs.sort(key=get_album_order)

def parse_rhythmbox_albums(songs):
	albums = {}
	for song in songs:
		song_artist = song["artist"]
		if not song_artist:
			continue
		song_album = song["album"]
		if not song_album:
			continue
		album = albums.get(song_album, [])
		album.append(song)
		albums[song_album] = album
	# sort album in track order
	for album in albums:
		sort_album(albums[album])
	return albums

def parse_rhythmbox_artists(songs):
	artists = {}
	for song in songs:
		song_artist = song["artist"]
		if not song_artist:
			continue
		artist = artists.get(song_artist, [])
		artist.append(song)
		artists[song_artist] = artist
	# sort in album + track order
	for artist in artists:
		sort_album_order(artists[artist])
	return artists

def out(msg, fil):
	fil.write(msg.encode("UTF-8"))

def usecache(cachefile, datafile):
	try:
		cstat = os.stat(cachefile)
		dstat = os.stat(datafile)
		return cstat.st_mtime >= dstat.st_mtime
	except OSError:
		return False

if __name__ == '__main__':
	rhcache = "./rhcache"
	fil = os.path.expanduser("~/.local/share/rhythmbox/rhythmdb.xml")

	if not usecache(rhcache, fil):
		sys.stderr.write("Parsing '%s'\n" % fil)
		songs = get_rhythmbox_songs(fil)
		outfile = open("./rhcache", "w")
		for song in songs:
			out(
u"""Bookmark {
  url = "%s",
  name = [==[%s]==],
  description = [=[From %s by %s]=],
  icon_name ="audio-x-generic",
}
""" % (song["location"], song["title"], song["album"], song["artist"]),
			fil=outfile)
		outfile.close()
	sys.stdout.write(open(rhcache).read())
	sys.exit(0)


from semeval2014_dataset import *
import numpy as np
from gensim import corpora
from gensim import models
from gensim.matutils import softcossim
from tqdm import tqdm
from copy import deepcopy

np.random.seed(0)

dataset = SimpleDataset('../../datas/ABSA14_Restaurants_Train.xml', '../../datas/Restaurants_Test_Data_phaseB.xml',1.0)

def find_best_threshold(multi_cat_sentence_pred_label, multi_cat_sentence_gtruth_label):
    threshold = 0.19
    best_res = [0.0, 0.0, 0.0, 0.0]
    while threshold < 0.5:
        TP = 0
        FP = 0
        FN = 0
        for i in range(len(multi_cat_sentence_gtruth_label)):
            pred_labels = deepcopy(multi_cat_sentence_pred_label[i])
            gtruth_labels = deepcopy(multi_cat_sentence_gtruth_label[i])
            for idx in range(len(pred_labels)):
                if pred_labels[idx] > threshold:
                    pred_labels[idx] = 1
                else:
                    pred_labels[idx] = 0
            for idx in range(len(pred_labels)):
                if pred_labels[idx] == 1:
                    if idx in gtruth_labels:
                        TP += 1
                    else:
                        FP += 1
                else:
                    if idx in gtruth_labels:
                        FN += 1
        try:
            precision = TP / (TP + FP)
            recall = TP / (TP + FN)
            f1 = 2 * (precision * recall) / (precision + recall)
        except ZeroDivisionError:
            f1 = 0.0
        if f1 > best_res[0]:
            best_res = [f1, precision, recall, threshold]
        threshold += 0.0001
        threshold = round(threshold, 4)
    return best_res

def softmax(x):
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0)

category_label_num = {
            'service': 0,
            'food': 1,
            'price': 2,
            'ambience': 3,
            'anecdotes/miscellaneous': 4
}
category_num_label = {
            0: 'service',
            1: 'food',
            2: 'price',
            3: 'ambience',
            4: 'anecdotes/miscellaneous'
}
categories = [
            'service',
            'food',
            'price',
            'ambience',
            'anecdotes/miscellaneous'
]

test_sentences = []
test_sentences_with_label = []
for item in dataset.test_data:
    test_sentences.append(item[0])
    test_sentences_with_label.append(item)

category_seed_words = np.load('../../w2vec_candid_dict_2014.npy').item()
# category_seed_words = np.load('../../tempseed_pruned.npy').item()
list_category_seed_words = []
for key in category_seed_words.keys():
    list_category_seed_words.append(category_seed_words[key])

w2v_model = models.KeyedVectors.load_word2vec_format('../../yelp_W2V_300_orig.bin', binary=True)
dictionary = corpora.Dictionary(test_sentences)
corpus = [dictionary.doc2bow(document) for document in test_sentences]
similarity_matrix = w2v_model.similarity_matrix(dictionary)

Y = [dictionary.doc2bow(sentence) for sentence in test_sentences]

def cos_sim(a, b):
    dot_product = np.dot(a, b)
    norm_a = np.linalg.norm(a)
    norm_b = np.linalg.norm(b)
    return dot_product / (norm_a * norm_b)

def average_of_sentence(sentence):
    sum_of_words = np.zeros(300)
    for word in sentence:
        try:
            sum_of_words += w2v_model[word]
        except KeyError:
            continue
    return sum_of_words / len(sentence)

def initialize_clusters(points, k):
    random_points = []
    while True:
        random_num = np.random.randint(0, len(points))
        random_points.append(average_of_sentence(points[random_num]))
        if len(random_points) == k:
            break
    return random_points

def get_distances(center, points):
    """Returns the distance the centroid is from each data point in points."""
    distances = []
    for point in points:

        distances.append(1 - cos_sim(center, point))
    return distances

def get_category_seed_similarity(sentence, seeds, similarity_matrix):
    result = 0
    length = len(seeds)
    for word in seeds.keys():
        if word not in seeds[word]:
            seeds[word].append(word)
        seedsentence_d2b = dictionary.doc2bow(seeds[word])
        result += softcossim(sentence, seedsentence_d2b, similarity_matrix)
    return result / length

k = 17
maxiter = 23
centroids = initialize_clusters(test_sentences, k)

classes = len(Y) * [0]

for i in tqdm(range(maxiter)):
    distances = []
    loss = 0
    for j in range(len(test_sentences)):
        centroid_distances = get_distances(average_of_sentence(test_sentences[j]), centroids)
        distances.append(centroid_distances)
    for j in range(len(distances)):
        try:
            classes[j] = int(np.argmin(distances[j]))
            loss += distances[j][classes[j]]
        except:
            print(distances[j])
            exit()
    # classes = np.argmin(np.array(distances), axis=1)
    for c in range(k):
        if len([j for j in range(len(test_sentences)) if classes[j] == c]) < 2:
            continue
        centroids[c] = np.mean([average_of_sentence(test_sentences[j])
                                for j in range(len(test_sentences))
                                if classes[j] == c], 0)
    print(loss)

TP = 0
FP = 0
FN = 0

classes_pred_label = []

for c in range(len(centroids)):
    cluster_sentences = [test_sentences[j] for j in range(len(test_sentences)) if classes[j] == c]
    cluster_sentences_labels = [test_sentences_with_label[j][1]
                                for j in range(len(test_sentences_with_label))
                                if classes[j] == c]
    pred_labels = len(categories) * [0.0]
    for sentence in cluster_sentences:
        for cat in categories:
            seeds = category_seed_words[cat]
            pred_labels[category_label_num[cat]] += get_category_seed_similarity(dictionary.doc2bow(sentence), seeds, similarity_matrix)
    pred_labels = [pred_labels[j] / len(cluster_sentences) for j in range(len(pred_labels))]
    # pred_labels[int(np.argmax(pred_labels))] = 1
    # for j in range(len(pred_labels)):
    #     if j != np.argmax(pred_labels):
    #         pred_labels[j] = 0
    classes_pred_label.append(pred_labels)
print(classes)
threshold = 0.204
rho = 0.8

predicted_labels = []
gtruth_labels = []

for i, sentence in enumerate(test_sentences):
    sentence_class_label = classes_pred_label[classes[i]]
    gtruth_label = test_sentences_with_label[i][1]

    sentence_seed_word_label = len(categories) * [0.0]
    for cat in categories:
        seeds = category_seed_words[cat]
        sentence_seed_word_label[category_label_num[cat]] += get_category_seed_similarity(dictionary.doc2bow(sentence), seeds, similarity_matrix)
    sentence_pred_label = (1 - rho) * softmax(np.array(sentence_class_label)) + rho * softmax(np.array(sentence_seed_word_label))
    #print(sentence_pred_label)
    #if i > 10:
    #    exit()
    predicted_labels.append(deepcopy(sentence_pred_label))
    gtruth_labels.append(deepcopy(gtruth_label))
    for j in range(len(sentence_pred_label)):
        if sentence_pred_label[j] > threshold:
            sentence_pred_label[j] = 1
        else:
            sentence_pred_label[j] = 0

    for idx in range(len(sentence_pred_label)):
        if sentence_pred_label[idx] == 1:
            if idx in gtruth_label:
                TP += 1
            else:
                FP += 1
        else:
            if idx in gtruth_label:
                FN += 1

print(find_best_threshold(predicted_labels, gtruth_labels))

precision = TP / (TP + FP)
recall = TP / (TP + FN)
print(2 * (precision * recall) / (precision + recall))
print(precision)
print(recall)
print()
print(TP)
print(FP)


# uncompyle6 version 3.6.7
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.8.2 (tags/v3.8.2:7b3ab59, Feb 25 2020, 23:03:10) [MSC v.1916 64 bit (AMD64)]
# Embedded file name: build/bdist.linux-x86_64/egg/healthvaultlib/objects/vocabularyparameters.py
# Compiled at: 2016-01-03 04:45:03
from lxml import etree

class VocabularyParameters:

    def __init__(self, keys):
        self.vocabulary_keys = keys
        self.fixed_culture = False

    def write_xml(self):
        params = etree.Element('vocabulary-parameters')
        for i in self.vocabulary_keys:
            params.append(i.write_xml())

        fixed_culture = etree.Element('fixed-culture')
        fixed_culture.text = 'false'
        if self.fixed_culture:
            fixed_culture.text = 'true'
        params.append(fixed_culture)
        return params


#!/usr/bin/env python
# -*- coding: utf-8

from pprint import pformat

import pytest
from k8s.models.common import ObjectMeta
from k8s.models.pod import ContainerPort, Container, LocalObjectReference, Probe, HTTPGetAction, TCPSocketAction, PodTemplateSpec, PodSpec
from k8s.models.replication_controller import ReplicationController, ReplicationControllerSpec
from util import get_vcr

vcr = get_vcr(__file__)

NAME = "my-name"
NAMESPACE = "my-namespace"

@pytest.mark.usefixtures("logger", "k8s_config")
class TestRc(object):
    @vcr.use_cassette()
    def teardown(self):
        for name, namespace in ((NAME, "default"), (NAME, NAMESPACE)):
            try:
                ReplicationController.delete(name, namespace)
            except:
                pass

    def test_create_blank_rc(self):
        object_meta = ObjectMeta(name=NAME, namespace=NAMESPACE, labels={"test": "true"})
        rc = ReplicationController(metadata=object_meta)
        assert rc.metadata.name == NAME
        assert rc.as_dict()[u"metadata"][u"name"] == NAME

    @vcr.use_cassette()
    def test_lifecycle(self, logger):
        object_meta = ObjectMeta(name=NAME, namespace=NAMESPACE, labels={"test": "true"})
        container_port = ContainerPort(name="http5000", containerPort=5000)
        http = HTTPGetAction(path="/", port="http5000")
        liveness = Probe(httpGet=http)
        tcp = TCPSocketAction(port=5000)
        readiness = Probe(tcpSocket=tcp)
        container = Container(
            name="container",
            image="dummy_image",
            ports=[container_port],
            livenessProbe=liveness,
            readinessProbe=readiness
        )
        image_pull_secret = LocalObjectReference(name="image_pull_secret")
        pod_spec = PodSpec(containers=[container], imagePullSecrets=[image_pull_secret], serviceAccountName="default")
        pod_template_spec = PodTemplateSpec(metadata=object_meta, spec=pod_spec)
        rc_spec = ReplicationControllerSpec(replicas=2, selector={"test": "true"}, template=pod_template_spec)
        first = ReplicationController(metadata=object_meta, spec=rc_spec)
        logger.debug(pformat(first.as_dict()))
        first.save()

        second = ReplicationController.get(NAME, NAMESPACE)
        assert first.metadata.name == second.metadata.name
        assert first.metadata.namespace == second.metadata.namespace


from typing import List, Optional
class SlashChoice:
    def __init__(self, name: str, value: str):
        self.name = name
        self.value = value
    def __dict__(self):
        return {
            'name': self.name,
            'value': self.value
        }
class SlashOption:
    def __init__(self, name: str, description: str, type_: int, required: bool, choices: Optional[List[SlashChoice]]):
        self.name = name
        self.description = description
        self.type = type_
        self.required = required
        if self.type == 3:
            # We should have choices if type is 3
            try:
                self.choices = choices
            except:
                raise Exception("You need choices for this option")
        else:
            self.choices = None
    def __dict__(self):
        return_val = {
            "name": self.name,
            "description": self.description,
            "type": self.type,
            "required": self.required
        }
        if self.choices:
            return_val[self.choices] = dict(self.choices)
        return return_val


import os
from os import path
import json
from datetime import date

import numpy as np
import matplotlib.pyplot as plt

dataset_dir = "../fakenewsnet_dataset/politifact/fake"
dataset_dir = path.abspath( path.join(path.dirname(__file__), dataset_dir))

DATE_MAX = date(2095, 12, 31)
DATE_MIN = date(1995, 2, 5)

# convert month string to number
def month_str_to_num(month_string):
    month_dict = {
        "Jan": 1,
        "Feb": 2,
        "Mar": 3,
        "Apr": 4,
        "May": 5,
        "Jun": 6,
        "Jul": 7,
        "Aug": 8,
        "Sep": 9,
        "Oct": 10,
        "Nov": 11,
        "Dec": 12
    }

    if month_string in month_dict:
        return month_dict[month_string]
    else:
        return -1

def parse_tweet_date(date_string):

    # example of date string: "Fri Dec 08 17:08:28 +0000 2017"
    y = int(date_string.split(" ")[5])
    m = month_str_to_num(date_string.split(" ")[1])
    assert m >= 1
    d = int(date_string.split(" ")[2])

    return date(y, m, d)

def autolabel(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{:.3f}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')

# [STATISTICS] results
NEWS_COUNT = 0
LIFESPAN_PER_CASCADE = [] # lifespan of all cascades
LIFESPAN_PER_CASCADE_RT = [] # lifespan of cascades having Replies / Quotes / Retweets
LIFESPAN_PER_NEWS = []

CASCADE_SIZE = []

# iterate all news
for item in os.listdir(dataset_dir):

    news_dir_path = f"{dataset_dir}/{item}"

    if not path.isdir(news_dir_path):
        continue

    # check existence of news article
    if not path.exists(f"{news_dir_path}/news content.json"):
        continue

    # check existence of cascade file
    if not path.exists(f"{news_dir_path}/cascade.json"):
        continue

    NEWS_COUNT += 1
    print(f"{NEWS_COUNT}: {item}")

    # if NEWS_COUNT > 5:
    #     break

    news_post_count = 0

    with open(f"{news_dir_path}/cascade.json", "r") as cascade_file:
        forest = json.loads(cascade_file.read())

        news_first_date = DATE_MAX
        news_last_date = DATE_MIN

        for cascade in forest:

            cascade_first_date = DATE_MAX
            cascade_last_date = DATE_MIN

            post_count = 0

            # collect IDs from source tweet, replies, quotes using BFS
            i = 0
            tweet_ids = []
            bfs_queue = [cascade]
            while i < len(bfs_queue):
                traversed = bfs_queue[i]
                tweet_ids.append(traversed['id'])

                with open(f"{news_dir_path}/tweets/{traversed['id']}.json", "r") as tweet_file:
                    tweet = json.loads(tweet_file.read())
                    created_at = parse_tweet_date(tweet['created_at'])

                    # update last / first date of cascade / news
                    news_first_date = min(created_at, news_first_date)
                    news_last_date = max(created_at, news_last_date)
                    cascade_first_date = min(created_at, cascade_first_date)
                    cascade_last_date = max(created_at, cascade_last_date)

                if len(traversed['quoted_by']) > 0:
                    bfs_queue += traversed['quoted_by']

                if len(traversed['replied_by']) > 0:
                    bfs_queue += traversed['replied_by']

                i += 1

            # length of tweet_ids is #(source tweet + replies + quites)
            post_count += len(tweet_ids)

            for id in tweet_ids:
                with open(f"{news_dir_path}/retweets/{id}.json", "r") as retweet_list_file:
                    retweet_list = json.loads(retweet_list_file.read())

                    post_count += len(retweet_list['retweets'])
                    for retweet in retweet_list['retweets']:
                        created_at = parse_tweet_date(retweet['created_at'])

                        # update last / first date of cascade / news
                        news_first_date = min(created_at, news_first_date)
                        news_last_date = max(created_at, news_last_date)
                        cascade_first_date = min(created_at, cascade_first_date)
                        cascade_last_date = max(created_at, cascade_last_date)

            cascade_lifespan = (cascade_last_date - cascade_first_date).days # calculated in days

            if post_count > 1:
                LIFESPAN_PER_CASCADE_RT.append(cascade_lifespan)

            LIFESPAN_PER_CASCADE.append(cascade_lifespan)
            CASCADE_SIZE.append(post_count)

            if post_count == 1 and cascade_lifespan > 0:
                print(f"lifespan: {cascade_lifespan} days, start: {cascade_first_date}, end: {cascade_last_date}")

        news_lifespan = (news_last_date - news_first_date).days # calculated in days
        LIFESPAN_PER_NEWS.append(news_lifespan)

#------------------------------------------PRINT RESULT------------------------------------------------

# [RESULT] lifespan per news
LIFESPAN_PER_NEWS = np.array(LIFESPAN_PER_NEWS)
print(f"[lifespan per news] min: {LIFESPAN_PER_NEWS.min()}, max: {LIFESPAN_PER_NEWS.max()}, average: {LIFESPAN_PER_NEWS.mean()}, median: {np.median(LIFESPAN_PER_NEWS)}")

bin_frequency = {
    '1 day': 0 ,
    '1 week': 0,
    '1 month': 0,
    'half year': 0,
    '1 year': 0,
    '> 1 year': 0
}

for n in LIFESPAN_PER_NEWS:
    if n <= 1:
        bin_frequency['1 day'] += 1
    elif n > 1 and n <= 7:
        bin_frequency['1 week'] += 1
    elif n > 7 and n <= 30:
        bin_frequency['1 month'] += 1
    elif n > 30 and n <= 180:
        bin_frequency['half year'] += 1
    elif n > 180 and n <= 365:
        bin_frequency['1 year'] += 1
    else:
        bin_frequency['> 1 year'] += 1

time_lengths = bin_frequency.keys()
news_number = np.array([bin_frequency[n] for n in time_lengths], dtype=np.float)
news_number = news_number / len(LIFESPAN_PER_NEWS)

fig, ax = plt.subplots()
bars = ax.bar(time_lengths, news_number, width=0.8)
autolabel(bars)
ax.set_ylabel('News Number'), ax.set_xlabel('Lifespan')
ax.set_title('Lifespan of News')
plt.show()

# [RESULT] lifespan per cascade
LIFESPAN_PER_CASCADE = np.array(LIFESPAN_PER_CASCADE)
print(f"[lifespan per cascade] min: {LIFESPAN_PER_CASCADE.min()}, max: {LIFESPAN_PER_CASCADE.max()}, average: {LIFESPAN_PER_CASCADE.mean()}, median: {np.median(LIFESPAN_PER_CASCADE)}")

bin_frequency = {
    '1 day': 0 ,
    '3 days': 0,
    '1 week': 0,
    'half month': 0,
    '1 month': 0,
    '> 1 month': 0
}

for n in LIFESPAN_PER_CASCADE:
    if n <= 1:
        bin_frequency['1 day'] += 1
    elif n > 1 and n <= 3:
        bin_frequency['3 days'] += 1
    elif n > 3 and n <= 7:
        bin_frequency['1 week'] += 1
    elif n > 7 and n <= 14:
        bin_frequency['half month'] += 1
    elif n > 14  and n <= 30:
        bin_frequency['1 month'] += 1
    else:
        bin_frequency['> 1 month'] += 1

time_lengths = bin_frequency.keys()
news_number = np.array([bin_frequency[n] for n in time_lengths], dtype=np.float)
news_number = news_number / len(LIFESPAN_PER_CASCADE)
fig, ax = plt.subplots()
bars = ax.bar(time_lengths, news_number, width=0.8)
autolabel(bars)
ax.set_ylabel('Cascade Number'), ax.set_xlabel('Lifespan')
ax.set_title('Lifespan of Cascade')
plt.show()

# [RESULT] lifespan per cascade
LIFESPAN_PER_CASCADE_RT = np.array(LIFESPAN_PER_CASCADE_RT)
print(f"[lifespan per cascade (>= 2 posts)] min: {LIFESPAN_PER_CASCADE_RT.min()}, max: {LIFESPAN_PER_CASCADE_RT.max()}, average: {LIFESPAN_PER_CASCADE_RT.mean()}, median: {np.median(LIFESPAN_PER_CASCADE_RT)}")

bin_frequency = {
    '1 day': 0 ,
    '3 days': 0,
    '1 week': 0,
    'half month': 0,
    '1 month': 0,
    '> 1 month': 0
}

for n in LIFESPAN_PER_CASCADE_RT:
    if n <= 1:
        bin_frequency['1 day'] += 1
    elif n > 1 and n <= 3:
        bin_frequency['3 days'] += 1
    elif n > 3 and n <= 7:
        bin_frequency['1 week'] += 1
    elif n > 7 and n <= 14:
        bin_frequency['half month'] += 1
    elif n > 14  and n <= 30:
        bin_frequency['1 month'] += 1
    else:
        bin_frequency['> 1 month'] += 1

time_lengths = bin_frequency.keys()
news_number = np.array([bin_frequency[n] for n in time_lengths], dtype=np.float)
news_number = news_number / len(LIFESPAN_PER_CASCADE_RT)

fig, ax = plt.subplots()
bars = ax.bar(time_lengths, news_number, width=0.8)
autolabel(bars)
ax.set_ylabel('Cascade Number'), ax.set_xlabel('Lifespan')
ax.set_title('Lifespan of Cascade(#post > 1)')
plt.show()

# [RESULT] cascade lifespan vs cascade size
plt.scatter(CASCADE_SIZE, LIFESPAN_PER_CASCADE, alpha=0.2)
plt.xlabel("Cascade Size")
plt.ylabel("Cascade Lifespan")
plt.title("Cascade: Size vs. Lifespan")
plt.show()


#!/usr/bin/python3
""" github api"""

import sys
import requests

if __name__ == "__main__":
    dict1 = {}
    flag = 0
    req = requests.get('https://api.github.com/user',
                       auth=(sys.argv[1], sys.argv[2]))
    str1 = req.text
    str2 = str1[1:-2].split(',')
    for i in str2:
        str3 = i.split(':')
        if str3[0] == '"id"':
            print(str3[1])
            flag = 1
    if flag == 0:
        print("None")


#!/usr/bin/env python3
# Copyright 2019 Robert Bosch GmbH
# Copyright 2021 Christophe Bedard
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Entrypoint/script to process events from a converted file to build a ROS model."""

import argparse
import os
import sys
import time

from tracetools_analysis.loading import load_file
from tracetools_analysis.processor import Processor
from tracetools_analysis.processor.ros2 import Ros2Handler

from . import time_diff_to_str

def add_args(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        'input_path',
        help='the path to a converted file to import and process, '
        'or the path to a trace directory to convert and process')
    parser.add_argument(
        '-f', '--force-conversion', dest='force_conversion',
        action='store_true', default=False,
        help='re-convert trace directory even if converted file is found')
    command_group = parser.add_mutually_exclusive_group()
    command_group.add_argument(
        '-s', '--hide-results', dest='hide_results',
        action='store_true', default=False,
        help='hide/suppress results from being printed')
    command_group.add_argument(
        '-c', '--convert-only', dest='convert_only',
        action='store_true', default=False,
        help=(
            'only do the first step of converting the file, without processing it '
            '(this should not be necessary, since conversion is done automatically and is mostly '
            'just an implementation detail)'
        ))

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description='Process ROS 2 trace data and output model data.')
    add_args(parser)
    return parser.parse_args()

def process(
    input_path: str,
    force_conversion: bool = False,
    hide_results: bool = False,
    convert_only: bool = False,
) -> int:
    """
    Process ROS 2 trace data and output model data.

    The trace data will be automatically converted into
    an internal intermediate representation if needed.

    :param input_path: the path to a converted file or trace directory
    :param force_conversion: whether to re-creating converted file even if it is found
    :param hide_results: whether to hide results and not print them
    :param convert_only: whether to only convert the file into our internal intermediate
        representation, without processing it. This should usually not be necessary since
        conversion is done automatically only when needed or when explicitly requested with
        force_conversion; conversion is mostly just an implementation detail
    """
    input_path = os.path.expanduser(input_path)
    if not os.path.exists(input_path):
        print(f'input path does not exist: {input_path}', file=sys.stderr)
        return 1

    start_time = time.time()

    events = load_file(input_path, do_convert_if_needed=True, force_conversion=force_conversion)

    # Return now if we only need to convert the file
    if convert_only:
        return 0

    processor = Processor(Ros2Handler())
    processor.process(events)

    time_diff = time.time() - start_time
    if not hide_results:
        processor.print_data()
    print(f'processed {len(events)} events in {time_diff_to_str(time_diff)}')
    return 0

def main():
    args = parse_args()

    process(
        args.input_path,
        args.force_conversion,
        args.hide_results,
        args.convert_only,
    )


import torch
import torch.nn.functional as F
import models.networks as networks
import util.util as util
import pickle
from models.networks.txt_enc import EncoderText
from models.networks.perturbation import PerturbationNet
import random

def l2norm(x, norm_dim=1):
    norm = torch.pow(x, 2).sum(dim=norm_dim, keepdim=True).sqrt()
    x = torch.div(x, norm)
    return x

class OpenEditModel(torch.nn.Module):

    def __init__(self, opt):
        super().__init__()
        self.opt = opt
        self.FloatTensor = torch.cuda.FloatTensor
        self.ByteTensor = torch.cuda.ByteTensor
        self.perturbation = opt.perturbation

        self.netG, self.netD, self.netE = self.initialize_networks(opt)

        self.generator = opt.netG

        if self.perturbation:
            self.netP = PerturbationNet(opt)
            self.netP.cuda()
        if self.opt.manipulation:
            self.vocab = pickle.load(open(opt.vocab_path, 'rb'))
            self.txt_enc = EncoderText(len(self.vocab), 300, 1024, 1)
            self.txt_enc.load_state_dict(torch.load(opt.vse_enc_path, map_location='cpu')['model'][1])
            self.txt_enc.eval().cuda()

        # set loss functions
        if self.perturbation:
            self.criterionPix = torch.nn.L1Loss()
            self.criterionVGG = networks.VGGLoss(self.opt.gpu)
        elif opt.isTrain:
            self.loss_l1pix = opt.l1pix_loss
            self.loss_gan = not opt.no_disc
            self.loss_ganfeat = not opt.no_ganFeat_loss
            self.loss_vgg = not opt.no_vgg_loss

            if self.loss_l1pix:
                self.criterionPix = torch.nn.L1Loss()
            if self.loss_gan:
                self.criterionGAN = networks.GANLoss(
                    opt.gan_mode, tensor=self.FloatTensor, opt=self.opt)
            if self.loss_ganfeat:
                self.criterionFeat = torch.nn.L1Loss()
            if self.loss_vgg:
                self.criterionVGG = networks.VGGLoss(self.opt.gpu)

    def forward(self, data, mode, ori_cap=None, new_cap=None, alpha=1, global_edit=False):
        if not data['image'].is_cuda:
            data['image'] = data['image'].cuda()
        if mode == 'generator':
            g_loss, generated = self.compute_generator_loss(data)
            return g_loss, generated
        elif mode == 'discriminator':
            d_loss = self.compute_discriminator_loss(data)
            return d_loss
        elif mode == 'inference':
            with torch.no_grad():
                fake_image, _ = self.generate_fake(data['image'])
            return fake_image
        elif mode == 'manipulate':
            with torch.no_grad():
                fake_image = self.manipulate(data['image'], ori_cap, new_cap, alpha, global_edit=global_edit)
            return fake_image
        elif mode == 'optimize':
            g_loss, generated = self.optimizeP(data, ori_cap, new_cap, alpha, global_edit=global_edit)
            return g_loss, generated
        else:
            raise ValueError("|mode| is invalid")

    def create_P_optimizers(self, opt):
        P_params = list(self.netP.parameters())

        beta1, beta2 = 0, 0.9
        P_lr = opt.lr

        optimizer_P = torch.optim.Adam(P_params, lr=P_lr, betas=(beta1, beta2))

        return optimizer_P

    def create_optimizers(self, opt):
        G_params = list(self.netG.parameters())
        if opt.isTrain and self.loss_gan:
            D_params = list(self.netD.parameters())

        if opt.no_TTUR:
            beta1, beta2 = opt.beta1, opt.beta2
            G_lr, D_lr = opt.lr, opt.lr
        else:
            beta1, beta2 = 0, 0.9
            G_lr, D_lr = opt.lr / 2, opt.lr * 2

        optimizer_G = torch.optim.Adam(G_params, lr=G_lr, betas=(beta1, beta2))
        if self.loss_gan and opt.isTrain:
            optimizer_D = torch.optim.Adam(D_params, lr=D_lr, betas=(beta1, beta2))
        else:
            optimizer_D = None

        return optimizer_G, optimizer_D

    def save(self, epoch):
        util.save_network(self.netG, 'G', epoch, self.opt)
        if self.loss_gan:
            util.save_network(self.netD, 'D', epoch, self.opt)

    def initialize_networks(self, opt):
        print(opt.isTrain)
        netG = networks.define_G(opt)
        netD = networks.define_D(opt) if opt.isTrain and not opt.no_disc else None
        netE = networks.define_E(opt)

        if not opt.isTrain or opt.continue_train or opt.manipulation:
            netG = util.load_network(netG, 'G', opt.which_epoch, opt)
            if opt.isTrain or opt.needs_D:
                netD = util.load_network(netD, 'D', opt.which_epoch, opt)
                print('network D loaded')

        return netG, netD, netE

    def compute_generator_loss(self, data):
        G_losses = {}

        fake_image, spatial_embedding = self.generate_fake(data['image'])

        if self.loss_l1pix:
            G_losses['Pix'] = self.criterionPix(fake_image, data['image'])
        if self.loss_gan:
            pred_fake, pred_real = self.discriminate(fake_image, data)

        if self.loss_ganfeat:
            actual_num_D = 0
            num_D = len(pred_fake)
            GAN_Feat_loss = self.FloatTensor(1).fill_(0)
            for i in range(num_D):  # for each discriminator
                # last output is the final prediction, so we exclude it
                num_intermediate_outputs = len(pred_fake[i]) - 1
                if num_intermediate_outputs == 0:
                    continue
                else:
                    actual_num_D += 1
                for j in range(num_intermediate_outputs):  # for each layer output
                    unweighted_loss = self.criterionFeat(
                        pred_fake[i][j], pred_real[i][j].detach())
                    GAN_Feat_loss += unweighted_loss * self.opt.lambda_feat
            G_losses['GAN_Feat'] = GAN_Feat_loss / actual_num_D

        if self.loss_gan:
            G_losses['GAN'] = self.criterionGAN(pred_fake, True,
                                                for_discriminator=False)

        if self.loss_vgg:
            G_losses['VGG'] = self.criterionVGG(fake_image, data['image']) \
                * self.opt.lambda_vgg

        return G_losses, fake_image

    def compute_discriminator_loss(self, data):
        D_losses = {}
        with torch.no_grad():
            fake_image, spatial_embedding = self.generate_fake(data['image'])

        pred_fake, pred_real = self.discriminate(fake_image, data)

        D_losses['D_Fake'] = self.criterionGAN(pred_fake, False,
                                               for_discriminator=True)
        D_losses['D_real'] = self.criterionGAN(pred_real, True,
                                               for_discriminator=True)

        return D_losses

    def generate_fake(self, real_image, return_skip = False, return_edge = False):

        with torch.no_grad():
            spatial_embedding, edge = self.netE(real_image)
        fake_image = self.netG(spatial_embedding, edge)
        if return_edge:
            return fake_image, spatial_embedding, edge
        else:
            return fake_image, spatial_embedding

    def generate_fake_withP(self, real_image):
        with torch.no_grad():
            spatial_embedding, edge = self.netE(real_image)
        p, P_reg = self.netP()
        fake_image = self.netG(spatial_embedding, edge, perturbation=True, p=p)

        return fake_image, P_reg

    def discriminate(self, fake_image, data, edge=None):

        fake_and_real_img = torch.cat([fake_image, data['image']], dim=0)

        discriminator_out = self.netD(fake_and_real_img)

        pred_fake, pred_real = self.divide_pred(discriminator_out)

        return pred_fake, pred_real

    def manipulate(self, real_image, ori_txt, new_txt, alpha, global_edit=False):

        spatial_embedding, edge = self.netE(real_image, norm=True)

        with torch.no_grad():
            ori_txt = self.txt_enc(ori_txt, [ori_txt.shape[1]])
            new_txt = self.txt_enc(new_txt, [new_txt.shape[1]])

        proj = spatial_embedding * ori_txt.unsqueeze(2).unsqueeze(3).repeat(1,1,spatial_embedding.shape[2],spatial_embedding.shape[3])
        proj_s = proj.sum(1, keepdim=True)
        proj = proj_s.repeat(1,1024,1,1)
        # proj = F.sigmoid(proj)

        if global_edit:
            proj[:] = 1 # for global attributes, don't need to do grounding

        spatial_embedding = spatial_embedding - alpha * proj * ori_txt.unsqueeze(2).unsqueeze(3).repeat(1,1,spatial_embedding.shape[2],spatial_embedding.shape[3])
        spatial_embedding = spatial_embedding + alpha * proj * new_txt.unsqueeze(2).unsqueeze(3).repeat(1,1,spatial_embedding.shape[2],spatial_embedding.shape[3])

        spatial_embedding = l2norm(spatial_embedding)

        if self.perturbation:
            p, P_reg = self.netP()
            fake_image = self.netG(spatial_embedding, edge, perturbation=True, p=p)
            return fake_image
        else:
            fake_image = self.netG(spatial_embedding, edge)
            return fake_image

    def optimizeP(self, data, ori_cap, new_cap, alpha, global_edit=False):

        P_losses = {}

        fake_image, P_reg = self.generate_fake_withP(data['image'])

        manip_image = self.manipulate(data['image'], ori_cap, new_cap, alpha, global_edit=global_edit)
        cycle_image = self.manipulate(manip_image, new_cap, ori_cap, alpha, global_edit=global_edit)

        P_losses['Pix'] = self.criterionPix(fake_image, data['image'])
        P_losses['VGG'] = self.criterionVGG(fake_image, data['image']) * self.opt.lambda_vgg

        P_losses['cyclePix'] = self.criterionPix(cycle_image, data['image'])
        P_losses['cycleVGG'] = self.criterionVGG(cycle_image, data['image']) * self.opt.lambda_vgg

        P_losses['Reg'] = P_reg

        return P_losses, fake_image


import serial
from time import sleep
import R2Protocol

#print(R2Protocol.encode(b"NUC", b"MOTOR", b"", b"M10001M20001"))
try:
	s = serial.Serial("COM8", 9600, timeout=2)
	print("Connected")
except serial.SerialException:
	print("No serial port found")
	exit()

try:
	str = R2Protocol.encode(b"NUC", b"FLAP", b"T", b"C")
	print(str)
	s.write(str)
	print("sent")
	sleep(1)
	line = s.readline()
	print(line)
except serial.SerialTimeoutException:
	print("Serial port write timed out 1!")


"""cnblog URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,re_path
from blog import views
from django.views.static import serve
from cnblog import settings

urlpatterns = [
    path('admin/', admin.site.urls),
    path('login/', views.login),
    path('get_validCode_img/', views.get_validCode_img),
    path('index/', views.index),
    path('logout/', views.logout),
    re_path('^$', views.index),      # 用户无输入，默认跳转到index页面
    path('register/', views.register),

    path('upload/', views.upload),

    # 点赞
    path('digg/', views.digg),

    # 评论
    path('comment/', views.comment),

    # 评论树
    path('get_comment_tree/', views.get_comment_tree),

    # media配制（记住）  用于用户访问media静态文件夹下的文件，static文件夹用户也可以访问，但是不用我们配制，因为django已经为我们配制好了，但是此次的media需要我们手动配置
    # 该正则表达式表示：只要以media开头的路径即可
    re_path(r'media/(?P<path>.*)$',serve,{'document_root':settings.MEDIA_ROOT}),

    # 个人站点的访问（如soft） 如http://127.0.0.1:8000/soft
    re_path('^(?P<username>\w+)/$', views.home_site),     # home_site(request,username='soft')

    # 个人页面下的跳转
    re_path('^(?P<username>\w+)/(?P<condition>tag|category|archive)/(?P<param>.*)/$', views.home_site),
    # home_site接收的参数：home_site(request,username='soft',condition='tag',param='Python...')

    # 个人站点下的文章详情页
    re_path('^(?P<username>\w+)/articles/(?P<article_id>\d+)$', views.article_detail),

    # 后台管理
    re_path('cn_backend/$',views.cn_backend),
    re_path('cn_backend/add_articles/$',views.add_articles),

]

# \w+：包含数字和字母，但不包含特殊符号
# .*包含特殊符号
# ?P<username>：有名分组，该username是用户访问时输入的如soft等名字，其会作为参数传递给home_site视图函数

# (?P<name>)：分别起别名
# (?P=name)：引用别名为name分组匹配到的字符串

# (?P<key1>)：给匹配规则起的别名，不代表内容，后面的.+才是代表匹配规则的内容部分
# (?P=key1)：使用别名为key1匹配好的分组字符串


import os
import numpy as np
# from utils.read_data import plot_signal
import matplotlib.pyplot as plt

def plot_train_test_curves(train_loss_data, test_loss_data, plot_path, fold_tag=1):
    if not os.path.exists(plot_path):
        os.makedirs(plot_path)

    clip = min(len(train_loss_data), len(test_loss_data))
    x_ax = np.arange(1, clip+1)
    fig = plt.figure()
    plt.plot(x_ax, train_loss_data[:clip], label="train_loss")
    plt.plot(x_ax, test_loss_data[:clip], label="test_loss")
    plt.title('Train-Test Loss')
    plt.ylabel('Loss')
    plt.xlabel('Num Epoch')
    plt.legend(loc='best')
    plt.show()
    fig.savefig(plot_path+f'/loss_fold_{fold_tag}.png', dpi=fig.dpi)

# def plot_test_output(test_op, plot_path):
#     if not os.path.exists(plot_path):
#         os.makedirs(plot_path)
#
#     x_ax = np.arange(1, len(test_op)+1)
#     fig = plt.figure()
#     plt.plot(x_ax, test_op, label="predicted_HR")
#     plt.ylabel('HR')
#     plt.xlabel('Time')
#     plt.show()
#     fig.savefig(plot_path + f'/Test_output.png', dpi=fig.dpi)
#
#
# def plot_rmse(data, plot_path, fold=0):
#     if not os.path.exists(plot_path):
#         os.makedirs(plot_path)
#
#     x_ax = np.arange(1, len(data)+1)
#     fig = plt.figure()
#     plt.plot(x_ax, data, label="predicted_HR_RMSE")
#     plt.ylabel('RMSE_HR')
#     plt.xlabel('Time')
#     plt.show()
#     fig.savefig(plot_path + f'/RMSE_fold{fold}.png', dpi=fig.dpi)

if __name__ == '__main__':
    plot_signal('data/data_preprocessed', 's22_trial05')


from typing import Callable, Optional, List
from ..base.pyg_types import Color, Point
from ..utils.text import TextLineView
from ..utils.clipboard import PyperClipboard
from ..utils.misc import FuncListView, binary_approx
from .pyg_ctl import PygCtl
from .entryline import get_pos_in_kern, EntryLine, clipboard_types
import pygame

class EntryBox(PygCtl):
    cursor_timer_event_id = pygame.USEREVENT + 1
    # cursor threshold, the fraction of character width, height
    #  that represents border between 2 char positions
    cursor_thresh = (.5, 0.0)
    cursor = EntryLine.cursor

    @classmethod
    def init_timer(cls, evt_code: int = None):
        if evt_code is not None:
            cls.cursor_timer_event_id = evt_code
        pygame.time.set_timer(cls.cursor_timer_event_id, 500)
        cls.clipboard = PyperClipboard()  # PygClipboard()

    def __init__(self, fnt: pygame.font.FontType, pos: Point, size: Point, colors: List[Color],
                 pre_chg: Optional[Callable[["EntryBox", pygame.event.EventType], bool]] = None,
                 post_chg: Optional[Callable[["EntryBox", pygame.event.EventType], None]] = None,
                 enter: Optional[Callable[["EntryBox"], None]] = None,
                 default_text: str = "",
                 censor: Optional[Callable[[str], str]] = None):
        super().__init__()
        self.line_sep = "\n"
        self.colors = colors
        self.highlight_colors: List[Color] = [(255, 255, 255), (51, 143, 255)]
        self.pos = pos
        self.size = size
        self.pre_chg = pre_chg
        self.post_chg = post_chg
        self.enter = enter
        self.censor = censor
        self.is_selected: bool = False
        self.highlight: bool = False
        self.highlight_pos: List[int] = [0, 0]
        self.txt = TextLineView(default_text, self.line_sep)
        self.ch_pos: List[int] = [0, 0]
        self.ch_off: int = 0  # x offset in pixels
        self.cur_key = None
        self.coll_rect: pygame.rect.RectType = pygame.rect.Rect(self.pos, self.size)
        self.prev_rect = self.coll_rect
        self.fnt = fnt
        self.cursor_state: bool = True
        self.cursor_col: Optional[int] = None
        self.line_h: int = fnt.get_linesize()
        self.old_cursor: Optional[tuple] = None

    def set_size(self, size):
        self.size = size
        self.coll_rect = pygame.rect.Rect(self.pos, self.size)

    def set_pos(self, pos):
        self.pos = pos
        self.coll_rect = pygame.rect.Rect(self.pos, self.size)

    def set_pos_size(self, pos, size):
        self.pos = pos
        self.size = size
        self.coll_rect = pygame.rect.Rect(self.pos, self.size)

    def on_evt(self, app, evt, pos):
        if evt.type == pygame.MOUSEBUTTONDOWN:
            self.highlight = True
            self.is_selected = True
            pos = [pos[0] - self.pos[0], pos[1] - self.pos[1]]
            if pos[0] < 0:
                pos[0] = 0
            if pos[1] < 0:
                pos[1] = 0
            cursor_thresh_x, cursor_thresh_y = EntryBox.cursor_thresh
            ch_row = max(0, min(int(pos[1] // self.line_h + cursor_thresh_y), len(self.txt.lines) - 1))
            draw_txt = self.txt.get_draw_row(ch_row)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)

            def get_size_text(x):
                return self.fnt.size(draw_txt[:x])[0]

            lst_v = FuncListView(get_size_text, len(draw_txt))
            # since lst_v[0] == 0 and pos[0] >= 0 is True, result->ch_pos >= 1
            ch_col = binary_approx(lst_v, pos[0])
            if ch_col < len(lst_v):
                off_x = lst_v[ch_col - 1]
                ch_w = lst_v[ch_col] - off_x
                if pos[0] - off_x <= cursor_thresh_x * ch_w:
                    ch_col -= 1
            self.ch_pos = [ch_col, ch_row]
            self.highlight_pos = [ch_col, ch_row]
            return True
        return False

    def on_pre_chg(self, evt):
        return self.pre_chg is None or self.pre_chg(self, evt)

    def on_mouse_enter(self, app):
        if pygame.mouse.get_cursor() != self.cursor:
            self.old_cursor = pygame.mouse.get_cursor()
            pygame.mouse.set_cursor(*self.cursor)
        return super().on_mouse_enter(app)

    def on_mouse_exit(self, app):
        if self.old_cursor is not None:
            pygame.mouse.set_cursor(*self.old_cursor)
            self.old_cursor = None
        return super().on_mouse_exit(app)

    def on_evt_global(self, app, evt):
        if evt.type == pygame.MOUSEBUTTONDOWN:
            if self.coll_rect.collidepoint(evt.pos[0], evt.pos[1]):
                return False
            if self.highlight or self.is_selected:
                self.highlight = False
                self.is_selected = False
        elif evt.type == pygame.MOUSEBUTTONUP:
            if self.highlight:
                self.highlight = False
        elif evt.type == pygame.MOUSEMOTION and self.highlight:
            pos = evt.pos
            pos = [pos[0] - self.pos[0], pos[1] - self.pos[1]]
            if pos[0] < 0:
                pos[0] = 0
            if pos[1] < 0:
                pos[1] = 0
            cursor_thresh_x, cursor_thresh_y = EntryBox.cursor_thresh
            ch_row = max(0, min(int(pos[1] // self.line_h + cursor_thresh_y), len(self.txt.lines) - 1))
            draw_txt = self.txt.get_draw_row(ch_row)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)

            def get_size_text(x):
                return self.fnt.size(draw_txt[:x])[0]

            lst_v = FuncListView(get_size_text, len(draw_txt))
            # since lst_v[0] == 0 and pos[0] >= 0 is True, result->ch_pos >= 1
            ch_col = binary_approx(lst_v, pos[0])
            if ch_col < len(lst_v):
                off_x = lst_v[ch_col - 1]
                ch_w = lst_v[ch_col] - off_x
                if pos[0] - off_x <= cursor_thresh_x * ch_w:
                    ch_col -= 1
            self.ch_pos = [ch_col, ch_row]
            return True
        elif evt.type == pygame.KEYDOWN and self.is_selected:
            is_chg = False
            ch_pos = self.txt.row_col_to_pos(*self.ch_pos)
            col_chg = True
            if evt.key == pygame.K_BACKSPACE:
                if ch_pos == 0 and self.ch_pos == self.highlight_pos or not self.on_pre_chg(evt):
                    return False
                elif evt.mod & pygame.KMOD_CTRL > 0:
                    start = EntryLine.prev_word(self.txt.string, ch_pos)
                    off = ch_pos - start
                    highlight_pos = self.txt.row_col_to_pos(*self.highlight_pos)
                    self.txt.delete(start, ch_pos)
                    if highlight_pos >= ch_pos:
                        highlight_pos -= off
                    elif highlight_pos > ch_pos - off:
                        highlight_pos = ch_pos
                    ch_pos = start
                    self.ch_pos = self.txt.pos_to_row_col(ch_pos)
                    self.highlight_pos = self.txt.pos_to_row_col(highlight_pos)
                    is_chg = True
                else:
                    if self.ch_pos != self.highlight_pos:
                        start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                        self.txt.delete(start, end)
                        self.ch_pos = self.txt.pos_to_row_col(start)
                        is_chg = True
                    else:
                        if ch_pos > 0:
                            self.txt.delete(ch_pos - 1, ch_pos)
                            self.ch_pos = self.txt.pos_to_row_col(ch_pos - 1)
                            is_chg = True
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_RETURN and evt.mod & pygame.KMOD_CTRL:
                if self.enter is not None:
                    self.enter(self)
            elif evt.key == pygame.K_DELETE:
                if (ch_pos >= len(self.txt.string) and self.ch_pos == self.highlight_pos) or not self.on_pre_chg(evt):
                    return False
                elif evt.mod & pygame.KMOD_CTRL > 0:
                    end = EntryLine.next_word(self.txt.string, ch_pos)
                    off = end - ch_pos
                    highlight_pos = self.txt.row_col_to_pos(*self.highlight_pos)
                    self.txt.delete(ch_pos, end)
                    if highlight_pos > ch_pos:
                        self.highlight_pos = self.txt.pos_to_row_col(max(highlight_pos - off, ch_pos))
                    is_chg = True
                else:
                    if self.ch_pos != self.highlight_pos:
                        start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                        self.txt.delete(start, end)
                        self.ch_pos = self.txt.pos_to_row_col(start)
                        is_chg = True
                    else:
                        if ch_pos < len(self.txt.string):
                            self.txt.delete(ch_pos, ch_pos + 1)
                            is_chg = True
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_HOME:
                if evt.mod & pygame.KMOD_CTRL > 0:
                    self.ch_pos[1] = 0
                self.ch_pos[0] = 0
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_END:
                if evt.mod & pygame.KMOD_CTRL > 0:
                    self.ch_pos[1] = len(self.txt.lines) - 1
                self.ch_pos[0] = self.txt.get_draw_row_len(self.ch_pos[1])
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_RETURN:
                # if self.enter is not None:
                # self.enter(self)
                self.txt.replace(self.line_sep, *sorted([ch_pos, self.txt.t_row_col_to_pos(self.highlight_pos)]))
                self.ch_pos[0] = 0
                self.ch_pos[1] += 1
                is_chg = True
                self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_LEFT:
                if evt.mod & pygame.KMOD_CTRL > 0:
                    self.ch_pos = self.txt.pos_to_row_col(EntryLine.prev_word(self.txt.string, ch_pos))
                elif ch_pos > 0:
                    if evt.mod & pygame.KMOD_SHIFT == 0 and self.ch_pos != self.highlight_pos:
                        start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                        self.ch_pos = self.txt.pos_to_row_col(start)
                    else:
                        self.ch_pos = self.txt.pos_to_row_col(ch_pos - 1)
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_RIGHT:
                if evt.mod & pygame.KMOD_CTRL > 0:
                    self.ch_pos = self.txt.pos_to_row_col(EntryLine.next_word(self.txt.string, ch_pos))
                elif ch_pos < len(self.txt.string):
                    if evt.mod & pygame.KMOD_SHIFT == 0 and self.ch_pos != self.highlight_pos:
                        start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                        self.ch_pos = self.txt.pos_to_row_col(end)
                    else:
                        self.ch_pos = self.txt.pos_to_row_col(ch_pos + 1)
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
            elif evt.key == pygame.K_UP:
                if self.ch_pos[1] > 0:
                    self.ch_pos[1] -= 1
                if self.cursor_col is not None:
                    self.ch_pos[0] = self.cursor_col
                else:
                    self.cursor_col = self.ch_pos[0]
                if self.ch_pos[0] > self.txt.get_draw_row_len(self.ch_pos[1]):
                    self.ch_pos[0] = self.txt.get_draw_row_len(self.ch_pos[1])
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
                col_chg = False
            elif evt.key == pygame.K_DOWN:
                if self.ch_pos[1] + 1 < len(self.txt.lines):
                    self.ch_pos[1] += 1
                if self.cursor_col is not None:
                    self.ch_pos[0] = self.cursor_col
                else:
                    self.cursor_col = self.ch_pos[0]
                if self.ch_pos[0] > self.txt.get_draw_row_len(self.ch_pos[1]):
                    self.ch_pos[0] = self.txt.get_draw_row_len(self.ch_pos[1])
                if evt.mod & pygame.KMOD_SHIFT == 0:
                    self.highlight_pos = list(self.ch_pos)
                col_chg = False
            elif evt.mod & pygame.KMOD_CTRL > 0:
                start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                if evt.key == pygame.K_v:
                    data = None
                    for Item in clipboard_types:
                        data = self.clipboard.get(Item)
                        if data is not None:
                            break
                    if data is None or not self.on_pre_chg(evt):
                        return False
                    self.txt.replace(data, start, end)
                    self.ch_pos = self.txt.pos_to_row_col(start + len(data))
                    self.highlight_pos = list(self.ch_pos)
                    is_chg = True
                elif evt.key == pygame.K_c:
                    self.clipboard.put(pygame.SCRAP_TEXT, self.txt.get_str(start, end))
                elif evt.key == pygame.K_x:
                    if start == end or not self.on_pre_chg(evt):
                        return False
                    self.clipboard.put(pygame.SCRAP_TEXT, self.txt.get_str(start, end))
                    self.txt.delete(start, end)
                    self.ch_pos = self.txt.pos_to_row_col(start)
                    self.highlight_pos = self.txt.pos_to_row_col(start)
                    is_chg = True
            elif len(evt.unicode) > 0:
                if not self.on_pre_chg(evt):
                    return False
                start, end = sorted((ch_pos, self.txt.row_col_to_pos(*self.highlight_pos)))
                self.txt.replace(evt.unicode, start, end)
                self.ch_pos = self.txt.pos_to_row_col(start + 1)
                self.highlight_pos = list(self.ch_pos)
                is_chg = True
            if col_chg:
                self.cursor_col = None
            if is_chg and self.post_chg is not None:
                self.post_chg(self, evt)
            return True
        elif evt.type == EntryBox.cursor_timer_event_id:
            if self.is_selected:
                self.cursor_state = not self.cursor_state
                return True
        return False

    def draw(self, app):
        start, end = map(self.txt.pos_to_row_col,
                         sorted(map(self.txt.t_row_col_to_pos, [self.ch_pos, self.highlight_pos])))
        rtn = [None] * len(self.txt.lines)
        if len(self.colors) > 1:
            rtn.append(app.surf.fill(self.colors[1], self.coll_rect))
        for c in range(0, start[1]):
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            img = self.fnt.render(draw_txt, True, self.colors[0])
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        if start == end:
            c = start[1]
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            img = self.fnt.render(draw_txt, True, self.colors[0])
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        elif start[1] == end[1]:
            c = start[1]
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            begin_w = get_pos_in_kern(self.fnt, draw_txt, start[0])
            end_w = get_pos_in_kern(self.fnt, draw_txt, end[0])
            img = pygame.Surface(self.fnt.size(draw_txt), pygame.SRCALPHA)
            img.blit(self.fnt.render(draw_txt[:start[0]], True, self.colors[0]), (0, 0))
            img.blit(self.fnt.render(draw_txt[start[0]:end[0]], True, *self.highlight_colors), (begin_w, 0))
            img.blit(self.fnt.render(draw_txt[end[0]:], True, self.colors[0]), (end_w, 0))
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        else:
            c = start[1]
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            begin_w = get_pos_in_kern(self.fnt, draw_txt, start[0])
            img = pygame.Surface(self.fnt.size(draw_txt), pygame.SRCALPHA)
            img.blit(self.fnt.render(draw_txt[:start[0]], True, self.colors[0]), (0, 0))
            img.blit(self.fnt.render(draw_txt[start[0]:], True, *self.highlight_colors), (begin_w, 0))
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
            c = end[1]
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            end_w = get_pos_in_kern(self.fnt, draw_txt, end[0])
            img = pygame.Surface(self.fnt.size(draw_txt), pygame.SRCALPHA)
            img.blit(self.fnt.render(draw_txt[:end[0]], True, *self.highlight_colors), (0, 0))
            img.blit(self.fnt.render(draw_txt[end[0]:], True, self.colors[0]), (end_w, 0))
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        for c in range(start[1] + 1, end[1]):
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            img = self.fnt.render(draw_txt, True, *self.highlight_colors)
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        for c in range(end[1] + 1, len(self.txt.lines)):
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            img = self.fnt.render(draw_txt, True, self.colors[0])
            rtn[c] = app.surf.blit(img, (self.pos[0], self.pos[1] + self.line_h * c))
        if self.cursor_state:
            c = self.ch_pos[1]
            draw_txt = self.txt.get_draw_row(c)
            if self.censor is not None:
                draw_txt = self.censor(draw_txt)
            cursor_pos = (
                self.fnt.size(draw_txt[:self.ch_pos[0]])[0] + self.pos[0] - 1,
                self.pos[1] + self.line_h * self.ch_pos[1])
            rtn.append(app.surf.fill(self.colors[0], pygame.Rect(cursor_pos, (2, self.line_h))))
        self.prev_rect = rtn[-1].unionall(rtn[:-1])
        return [self.prev_rect]

    def pre_draw(self, app):
        if self.prev_rect is not None:
            if len(self.colors) >= 2:
                return [self.prev_rect]
            else:
                return [app.draw_background_rect(self.prev_rect)]
        return []

    def collide_pt(self, pt):
        return self.coll_rect.collidepoint(pt[0], pt[1])


'''
Created on Dec 13, 2013

@author: eocampo

FTP File from Mainframe.
Check predecessor job completion
Rename and place in working dir
Invoke wkf

'''

__version__ = '20140103'

import sys
import os
import procjobs.procsched as psc
import utils.fileutils   as fu
import utils.strutils    as su
import utils.filetransf  as ft
import procdata.procinfa as pi

from apps.infbaseapp       import _InfaBaseApp

# Mandatory to define self.cmdStep
# method _getNextRunDate is sensitive to schedule changes !

# Schedules
SCH_FREQ = 'Wkly'

cur_dayr   = su.getTodayDtStr('%Y%m%d')

class VehPOLIWkly(_InfaBaseApp):
    exitOnError = True

    def __init__(self):
        super(VehPOLIWkly,self).__init__()
        self.landDir    = 'SrcFiles/vehicle'
        self.incFileSet = []    # Incoming Files. Contains full path name.
        self.incFiles   = []
        self.workFiles  = []    # Files that were moved to the working dir (ideally same than incSetFile).
        self.trigFiles  = []      # Incoming Trigger File.

        self.RowCnt     = -1
        self.srcFile    = ('PO230D15.dat',)   # File that Informatica expects. Alphabetical.
        self.ib.fileName = r"'P.PO230D15.VPODATA(0)'"
        self.checkNextRunFlg  = False
        self.runWkfFlowFlg    = False

        self.fileDate   = ''
        self.FILE_SET_LEN = 1

        self.ts        =  su.getTimeSTamp()
        # Allowable commands for this application. Make sure to Set
        self.cmdStep = { 'A' : self.getLock          ,
                         'B' : self.getFtpFiles      ,  # Sets self.incFileSet
                         'C' : self.getIncSetFiles   ,
                         'D' : self.copyFilesWorkDir ,
                         'E' : self.archFilesTS      ,
                         'F' : self.chkNextRunFlg    ,
                         'G' : self.getTrigFiles ,
                         'H' : self.verTrigFileDate  ,
                         'I' : self.procVehPOLIWkly  ,
                        }

        # Infa Environmental variables/
        self.infaEnvVar   = {
                'PMCMD'            : 'mg.pmcmd'           ,
                'INFA_USER'        : 'self.ib.rep_user'   ,
                'INFA_XPWD'        : 'self.ib.rep_xpwd'   ,
                'DOMAIN'           : 'self.ib.dom_name'   ,
                'INT_SERV'         : 'self.ib.IS'         ,
                'INFA_SHARE'       : 'self.ib.shareDir'   ,
                'INFA_APP_CFG'     : 'self.ib.cfgDir'     ,
                'INFA_APP_LCK'     : 'self.ib.lckDir'     ,
                'INFA_APP_CTL'     : 'self.ib.ctlDir'     ,

               }

        # FTP is expecting the following env variables, which should not be in a config file.
        #UX cp Cortera_Trx_Extract_20130429.txt "'P.PO230D15.VPODATA(0)'"
        os.environ['FILE'     ] =  (r"\'P.PO230D15.VPODATA(0)\'")    #
        os.environ['RXFILE'   ] =  ('None')    #
        # Sets a flag, to check for next run.

    def chkNextRunFlg(self) :
        self.checkNextRunFlg = True
        return 0

    def getFtpFiles(self):
        return ft.getn('Vehicle',self.log)

    # Wrapper Method
    def copyFilesWorkDir(self):
        for i in range (len(self.srcFile)):
            self.incFiles.append('%s' % self.incFileSet[i][0])

        return self.cpSrcToTgtFiles()

    def archFilesTS(self): return self.archGenFiles(self.incFiles, self.ts,True)

    def _wkfVehPOWkly(self):
        self.ib.fld = 'Vehicle'
        self.ib.wkf = 'wkf_vpo_line_item_weekly'
        rc = pi.runWkflWait(self.ib,self.log)
        if rc != 0 :
            self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
        else :
            self.log.info('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
        return rc

    def procVehPOLIWkly(self):
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        prev_dayr = self._getCtlFile()
        self.log.debug('Ctl File is %s self.checkNextRunFlg is %s prev_dayr=%s' %  (ctlFile,self.checkNextRunFlg,prev_dayr))

        if self.checkNextRunFlg is True:

            if prev_dayr is None or prev_dayr.strip() == '':
                self.log.error("Could not find control file or No Data")
                return -1

            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log)
            if rc != 0 :
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc

        # Run workflows
        if self._wkfVehPOWkly()       != 0 : return 1

        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile,cur_dayr)
        if rc == 0 :
            if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s'     % (prev_dayr,cur_dayr, ctlFile))
            else                   : self.log.info('Overwriting Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
        else       :
            self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
            return rc

        for i in range (len(self.srcFile)):
            t   = '%s/%s' % (self.ib.workDir,self.srcFile[i])
            r = fu.delFile(t)
            self.log.info('Removing %s rc = %s ' % (t,r))
        return rc

def main(Args):
    a = VehPOLIWkly()
    rc = a.main(Args)
    return rc

if __name__ == '__main__':
    from setwinenv import setEnvVars   # Remove in UX
    setEnvVars()                       # Remove in UX
    rc=  main(sys.argv)


'''
 Copyright (c) 2017 HERE Europe B.V.
 See the LICENSE file in the root of this project for license details.
'''

from jinja2 import Environment, PackageLoader

from generator.tree.nodes.archive import Archive
from generator.tree.nodes.trivial import Structure, Enumeration, Constant, Namespace
from generator.tree.nodes.resources import ResourceBase, BoundResource, Archive as \
    ArchiveResource, Vector, Multivector, Instance, RawData
from generator.tree.syntax_tree import SyntaxTree
from generator.tree.traversal import DfsTraversal
from .RaiseExtension import RaiseExtension

class BaseGenerator:
    def __init__(self, template):
        self._template = template

    def _supported_nodes(self):
        raise RuntimeError("Derived generators must reimplement _supported_nodes")

    def _populate_environment(self):
        raise RuntimeError("Derived generators must reimplement _populate_filters")

    def render(self, tree):
        env = Environment(loader=PackageLoader('generator', 'templates'), lstrip_blocks=True,
                          trim_blocks=True, autoescape=False, extensions=[RaiseExtension])
        env.filters['is_archive'] = lambda n: isinstance(n, Archive)
        env.filters['is_instance'] = lambda n: isinstance(n, Instance)
        env.filters['is_raw_data'] = lambda n: isinstance(n, RawData)
        env.filters['is_archive_resource'] = lambda n: isinstance(n, ArchiveResource)
        env.filters['is_structure'] = lambda n: isinstance(n, Structure)
        env.filters['is_enumeration'] = lambda n: isinstance(n, Enumeration)
        env.filters['is_constant'] = lambda n: isinstance(n, Constant)
        env.filters['is_namespace'] = lambda n: isinstance(n, Namespace)
        env.filters['is_resource'] = lambda n: isinstance(n, ResourceBase)
        env.filters['is_bound_resource'] = lambda n: isinstance(n, BoundResource)
        env.filters['is_vector'] = lambda n: isinstance(n, Vector)
        env.filters['is_multivector'] = lambda n: isinstance(n, Multivector)
        env.filters['is_multivector_index'] = lambda n: (isinstance(n, Structure) and
            "_builtin.multivector" in SyntaxTree.namespace_path(n))
        env.filters['namespaces'] = SyntaxTree.namespaces
        self._populate_environment(env)
        template = env.get_template(self._template)

        nodes = [n for n, _ in DfsTraversal(tree).dependency_order() if
                 any([isinstance(n, t) for t in self._supported_nodes()])]
        return template.render(nodes=nodes, tree=tree)


import numpy as np
import scipy.linalg as l

data = np.loadtxt("train.txt")
mean = np.mean(data, axis=0)
data - mean[np.newaxis,:]
scatter = np.dot(data.transpose(),data)
eigval,eigvec = l.eigh(scatter)
final = np.split(eigvec,10)
finalk = final[-1]
reddim = np.dot(data,finalk.transpose())


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from . import __version__

from collections import defaultdict
from argparse import ArgumentParser
from sys import argv

from progressbar import progressbar

from .main import generate_protein_sequences
from .peptides import extract_peptides, collapse_peptide_sources
from .decoys import generate_decoys

from varcode.reference import genome_for_reference_name

from varcode.cli import variant_collection_from_args, add_variant_args

def add_sources_to_argument_parser(parser):
    sources_group = parser.add_argument_group("Protein sequence sources")
    sources_group.add_argument(
        "--upstream-reading-frames",
        default=False,
        action="store_true",
        help="Include upstream reading frames from 5' UTR start codons")

    sources_group.add_argument(
        "--downstream-reading-frames",
        default=False,
        action="store_true",
        help="Include downstream reading frames from use of alternative start codons")

    sources_group.add_argument(
        "--skip-exons",
        default=False,
        action="store_true",
        help="Include sequences by skipping exons in a coding sequence")

    sources_group.add_argument(
        "--gene-name",
        default=None,
        help="Only use sequences originating from genes with this name")
    return parser

def add_peptide_params_to_argument_parser(parser):
    peptide_group = parser.add_argument_group("Peptides")
    peptide_group.add_argument("--extract-peptides", default=False, action="store_true")
    peptide_group.add_argument(
        "--min-peptide-length",
        default=7,
        help="Shortest peptide to include in output")
    peptide_group.add_argument(
        "--max-peptide-length",
        default=15,
        help="Longest peptide to include in output")
    return parser

def add_decoy_args(parser):
    decoy_group = parser.add_argument_group("Decoys")
    decoy_group.add_argument(
        "--num-decoys-per-hit",
        default=1,
        type=int,
        help="Number of decoys to generate per hit")
    decoy_group.add_argument(
        "--random-seed",
        default=0,
        type=int,
        help="Random seed to make scrambling of sequences deterministic")
    return parser

def create_argument_parser():
    parser = ArgumentParser("MS-MHC")
    parser.add_argument("--output", required=True, help="Name of output FASTA file")
    add_sources_to_argument_parser(parser)
    add_peptide_params_to_argument_parser(parser)
    add_decoy_args(parser)
    add_variant_args(parser)
    return parser

parser = create_argument_parser()

def run(args_list=None):
    if args_list is None:
        args_list = argv[1:]
    args = parser.parse_args(args_list)
    print("MS-MHC version %s" % __version__)
    reference_genome = genome_for_reference_name(args.genome if args.genome else "grch37")
    print("Using reference genome %s" % reference_genome)
    if args.vcf or args.maf or args.variant or args.json_variants:
        variants = variant_collection_from_args(args)
    else:
        variants = []

    hits = generate_protein_sequences(
        genome=reference_genome,
        variants=variants,
        upstream_reading_frames=args.upstream_reading_frames,
        downstream_reading_frames=args.downstream_reading_frames,
        skip_exons=args.skip_exons,
        min_peptide_length=args.min_peptide_length,
        restrict_sources_to_gene_name=args.gene_name)

    if args.extract_peptides:
        print("Extracting %dmer-%dmer peptides from generated sequences" % (
            args.min_peptide_length,
            args.max_peptide_length))
        sequence_dict = extract_peptides(
            hits,
            min_length=args.min_peptide_length,
            max_length=args.max_peptide_length)
    else:
        # make sure we don't have repeated protein sequences
        sequence_dict = defaultdict(list)
        for sequence_obj in hits:
            sequence_dict[sequence_obj.amino_acids].append(sequence_obj)

    hits = collapse_peptide_sources(sequence_dict)

    decoys = generate_decoys(
        hits,
        n_decoys=len(hits) * args.num_decoys_per_hit,
        random_seed=args.random_seed)

    combined_sequences = hits + decoys
    print("Writing %d FASTA records (%d hits, %d decoys)" % (
        len(combined_sequences),
        len(hits),
        len(decoys)))

    with open(args.output, "w") as f:
        for seq in progressbar(combined_sequences):
            seq.write_to_fasta_file(f)
    print("Done.")


#!/usr/bin/python3.8
import os
import http.server
import socketserver
import json

from urllib.request import Request, urlopen

HTTP_PORT = int(os.environ.get("PORT"))
API_SERVER_URL = os.environ.get("API_SERVER_URL")

class RequestHandler(http.server.SimpleHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()

        html = """
            <html>
                <body>
                    <h1>{message}</h1>
                </body>
            </html>
        """

        if API_SERVER_URL is not None:
            url = f"{API_SERVER_URL}/person"
            response = urlopen(Request(url))
            person = json.loads(response.read().decode())
            message = f"Hello from {person['name']} {person['surname']}"
        else:
            message = "No connection to API Server. API_SERVER_URL not defined!"

        self.wfile.write(bytes(html.format(message=message), "utf-8"))

        return

socketserver.TCPServer(("", HTTP_PORT), RequestHandler).serve_forever()


import requests
from bs4 import BeautifulSoup as bs4
from urllib.parse import urljoin
from pprint import pprint
from snip.stream import DefaultFormatter
from os import path, makedirs
from snip import jfileutil
import progressbar
import re
from glob import glob

import selenium_login

def slugify(string):
    return re.sub("[\/\\\:\*\"\'\<\>\|\?]", "-", string)

def makeExtension(req):
    if re.search('\....$', req.url):
        return ""
    extensions = {
        "image/jpeg": ".jpg",
        "text/html": ".html"
    }
    content_type = req.headers.get("Content-Type")
    match = extensions.get(content_type.split(";")[0])
    if match is None:
        print("Unknown content type", content_type)
        return ""
    return match

def fuzzyFileExists(filename):

    (fdirs, fname) = path.split(filename)
    searchstr = slugify(fdirs) + path.sep + slugify(fname) + "*"

    return (len(glob(searchstr)) > 0)

def save(request, filename):
    (fdirs, fname) = path.split(filename)
    makedirs(fdirs, exist_ok=True)
    final_filename = fdirs + path.sep + slugify(fname) + makeExtension(request)
    if not path.isfile(final_filename):
        with open(final_filename, 'wb') as fd:
            for chunk in request.iter_content(chunk_size=128):
                fd.write(chunk)
    return final_filename

def fetch(url, prev=None, soup=True):
    if prev:
        url = urljoin(prev, url)

    req = session.get(url, cookies=myCookies)
    if soup:
        soup = bs4(req.text, features="html.parser")
        return (req, soup)
    else:
        return req

# metadata_fields_interesting = [
#     "authors",
#     "book_title",
#     "description",
#     "images",
#     "subjects"
# ]

formatting = {
    "base_folder": "{book_title:60.60} ({authorstr})",
    "ebook": "{book_title} ({authorstr}).mobi",
    "metadata": "{book_title}_{authorstr}",
    "cover_file": "{book_title:12.12} (Cover)",
    "chapter_file": "{book_title:12.12} - {index:03} - {title}"
}

myCookies = dict()

session = requests.Session()

url_index = "https://www.safaribooksonline.com/library/view/gnu-emacs-pocket/1565924967/"

def update_meta(api_json):
    cmeta = dict()
    cmeta.update(api_json)
    try:
        obj_authors = api_json.get("authors")
        cmeta['authorstr'] = obj_authors[0].get('name') + (" et al" if len(obj_authors) != 1 else "")
    except TypeError:
        print("Bad metadata")
        print(api_json)
        init_cookies()
        raise
    return cmeta

def saveMeta(url_index):

    if url_index[0] == "/":
        url_index = "https://learning.oreilly.com" + url_index

    metad = {
        "main": {},
        "chapters": [],
        "file_cover": ""
    }
    (req_index, soup_index) = fetch(url_index)
    soup_toc = soup_index.find("ol", class_="detail-toc").findAll("li")

    index = 0
    json_file = None
    bar = progressbar.bar.ProgressBar(redirect_stdout=True)
    for url_api_json in bar(set(li['data-content-uri'] for li in soup_toc)):
        index += 1
        print(url_api_json)

        req_api = fetch(url_api_json, prev=url_index, soup=False)
        cmeta = update_meta(req_api.json())
        metad['chapters'].append(cmeta)
        if not json_file:
            formatter_api = DefaultFormatter(**cmeta)
            json_file = slugify(formatter_api.format(formatting['metadata']))
        if not metad['main']:
            metad['main'] = cmeta
            print(json_file)
            try:
                metad = jfileutil.load(json_file)
                bar.finish()
                break
            except FileNotFoundError:
                continue

        if cmeta['images']:
            if not metad['file_cover']:
                metad['file_cover'] = cmeta['images'][0]

    jfileutil.save(metad, json_file)
    return metad

def downloadBook(metad):
    index = 0
    for cmeta in progressbar.progressbar(metad['chapters'], redirect_stdout=True):
        index += 1

        print(index, cmeta.get("title"))
        formatter_api = DefaultFormatter(**cmeta)
        base_folder = slugify(formatter_api.format(formatting['base_folder']))

        url_content = cmeta.get("content")
        chapter_file = formatter_api.format(path.join(base_folder, formatting['chapter_file']), index=index)
        if fuzzyFileExists(chapter_file):
            continue
        else:
            req_content = fetch(url_content, soup=False)

            (fdirs, fname) = path.split(chapter_file)
            fdirs = slugify(fdirs)
            makedirs(fdirs, exist_ok=True)
            final_filename = fdirs + path.sep + slugify(fname) + makeExtension(req_content)
            with open(final_filename, mode="w", encoding="utf8") as html_out:
                soup2 = bs4(req_content.text, features="html.parser")

                header = soup2.find(re.compile(r'h[0-5]'))
                if header:
                    header['id'] = url_content.split('/')[-1]
                # header['class'] = "sbo-rt-content safari-container"

                html_out.write(str(soup2))

            for short_image_path in cmeta.get("images"):
                print(short_image_path)
                req_image = fetch(short_image_path, prev=cmeta.get("asset_base_url"), soup=False)
                image_filename = path.join(base_folder, short_image_path)
                save(req_image, image_filename)

    # cover_file = formatter_api.format(path.join(base_folder, formatting['cover_file']))
    # metad['main']['file_cover'] = save(session.get(metad['main'].get("cover"), cookies=myCookies), cover_file)

def bookPostprocessing(metad):
    import subprocess
    mmeta = metad['main']

    formatter_api = DefaultFormatter(**mmeta)
    base_folder = slugify(formatter_api.format(formatting['base_folder']))
    big_html_file = "{}/big_html.htm".format(base_folder)

    # HTML concat

    with open(big_html_file, "w", encoding='utf-8') as big:
        for htmlfile in sorted(glob(path.join(base_folder, "*.html"))):
            soup_piece = bs4(open(htmlfile, encoding='utf-8'), "html.parser")
            big.write(str(soup_piece))

    # HTML Postprocessing

    globals().update(vars())

    soup_big = bs4(open(big_html_file, encoding='utf-8'), "html.parser")

    globals().update(vars())

    # broken_internal_links = [a for a in [s.find("a")['href'] for s in soup_big.findAll("span", class_="ExternalRef")] if a.find("/") == -1]
    # Fix broken internal links
    internal_links = [s.find("a") for s in soup_big.findAll("span", class_="ExternalRef")]
    broken_internal_links = [a for a in internal_links if a['href'].find("/") == -1]
    for a in broken_internal_links:
        a['href'] = "#" + a['href']

    broken_hash_links = soup_big.findAll("a", href=re.compile("[^/]+?#"))
    for a in broken_hash_links:
        a['href'] = re.sub("[^/]+?#", "#", a['href'])

    broken_image_srcs = soup_big.findAll("img", src=re.compile("\.\./.+"))
    for img in broken_image_srcs:
        img['src'] = re.sub("\.\./", "", img['src'])

    with open(big_html_file, "w", encoding='utf-8') as big:
        big.write(str(soup_big))

    # Book generation

    program = "C:/Program Files/Calibre2/ebook-convert.exe"

    output_mobi_filename = slugify(formatter_api.format(formatting['ebook']))
    input_output = [
        big_html_file,
        output_mobi_filename
    ]

    cover = []

    # cover = [
    #     "--cover",
    #     path.join(base_folder, metad['file_cover'])
    # ]

    metadata = [
        "--authors", " & ".join([a['name'] for a in mmeta['authors']]),
        "--tags", ", ".join([a['name'] for a in mmeta['subjects']]),
        "--title", mmeta['book_title'],
    ]

    fixes = [
        # '--sr1-search', '"href=\"[0-9]+_.+?#"', '--sr1-replace', '"href=\"#"',  # Re-connect seperate files
        # "--level1-toc", "h2", "--level2-toc", "h3",  # Raise the TOC
    ]

    # To-do: Robust re-linking of internal refs

    popenargs = [program, *input_output, *cover, *metadata, *fixes]
    pprint(popenargs)
    import sys
    conv_proc = subprocess.run(popenargs, stdout=sys.stdout, stderr=sys.stderr)

    # subprocess.run(["tar", "caf", formatting['base_folder'] + ".zip", formatting['base_folder']])
    #
    # Emacs_Big.html Emacs.mobi
    # --level1-toc h2 --level2-toc h3
    # --cover GNU\ Emacs\ Pocket\ Reference\ \(Debra\ Cameron\ et\ al\)/GNU\ Emacs\ Po\ \(Cover\).jpg
    # --sr1-search "href=\"[0-9]+_.+?#" --sr1-replace "href=\"#"

def init_cookies():
    global myCookies
    myCookies = jfileutil.load("cookies", default=dict())

    while cookiesAreBad() or not myCookies:
        pprint(myCookies)

        myCookies = selenium_login.login(
            "http://libproxy.utdallas.edu/login?url=https://www.safaribooksonline.com/library/view/temporary-access/",
            lambda browser_instance: (browser_instance.current_url == "https://learning.oreilly.com/home/")
        ).get("cookies")

        pprint(myCookies)
        # browser_instance.quit()

    print("Successfully logged in.")
    pprint(myCookies)
    jfileutil.save(myCookies, "cookies")

def cookiesAreBad():
    req = fetch("https://learning.oreilly.com/api/v1/queue-simple/", soup=False)
    return req.json() == {"detail": "Authentication credentials were not provided."}

def main():
    from sys import argv
    init_cookies()

    if argv[1:]:
        for arg in argv[1:]:
            metad = saveMeta(arg)
            downloadBook(metad)
            bookPostprocessing(metad)
    else:
        while True:
            metad = saveMeta(input("Book index url: > "))
            downloadBook(metad)
            bookPostprocessing(metad)

main()
# url_page1_rel = soup_index.find("span", class_="start-reading").find("a")['href']
# url_page1 = urljoin(url_index, url_page1_rel)
# req_page1 = session.get(url_page1, cookies=myCookies)

# bs_page = bs4(req_page1.text, features="html.parser").find("div", id="sbo-rt-content")


import pymongo as m
from bson.objectid import ObjectId

import config

client = m.MongoClient(config.mongo_hostname, config.mongo_port)
db = client[config.mongo_db]
queries = db[config.mongo_collection]

def get_tags_list(tags):
    return [ tag.strip() for tag in tags.strip().split(",") \
            if len(tag.strip()) > 0]

def insert_query(title, sql, tags, desc, who):
    _tags = get_tags_list(tags)
    queries.insert({
            "title": title,
            "sql": sql,
            "tags": _tags,
            "desc": desc,
            "who": who
        })

def update_query(id, title, sql, tags, desc, who):
    _tags = get_tags_list(tags)
    queries.update({
        "_id": ObjectId(id)
        },
        { "$set" : {
            "title": title,
            "sql": sql,
            "tags": _tags,
            "desc": desc,
            "who": who
            }}, upsert=False)

def delete_query(id):
    queries.remove({"_id": ObjectId(id)})

def get_queries():
    return list(queries.find())

def get_query_details(id):
    return queries.find_one(ObjectId(id))