import networkx as nx
import matplotlib.pyplot as plt
from collections import defaultdict
import greedy
import dwave.inspector
# Size of the grid
size = (3,4)

lagrangian_objective = 1
# lagrangien structure
lagrangian_struct = 5
lagrangian_struct_start = lagrangian_struct
lagrangian_struct_end = lagrangian_struct
lagrangian_struct_path = lagrangian_struct

#convertion tuple coordonné en int index
def coord_to_index(coord):
    return coord[0] + coord[1]*size[0]

#convertion int index en tuple coordonné
def index_to_coord(index):
    return (index % size[0], index // size[0])

#affiche coordonné avec la position minimal sur x et maximal sur y, ça donne juste un graphe en longueur
def unique_edge(left_node_index, right_node_index): #prend un tuple en entré
    return (min(left_node_index, right_node_index), max(left_node_index, right_node_index)) #retourne un tuple

#n'est pas utilisé, remet le bord inférieur toujours a gauche de la liaison de G2 (composé de deux tuples)
def unique_edge_edge(edge1, edge2):
    if (edge1[0] < edge2[0]):
        return (edge1, edge2)
    elif (edge1[0] == edge2[0]):
        if (edge1[1] < edge2[1]):
            return (edge1, edge2)
        else:
            return (edge2, edge1)
    else:
        return (edge2, edge1)

#convertion coord en index vrai que si x : min et y : max
assert (unique_edge(3,2)) == index_to_coord(coord_to_index(unique_edge(3,2)))

# Start and end node
start_point = (1,1)
end_point = (2,3)

start_point_index = coord_to_index(start_point) #4
end_point_index = coord_to_index(end_point)#11

for j in range(0, size[1]):      #for(int j = 0, j < 4, i++)
    for i in range(0, size[0]):  #for(int i = 0, i < 3, i++)
        if (i, j) == start_point:
            print("S", end="")
        elif (i, j) == end_point:
            print("E", end="")
        else:
            print(".", end="")
    print("")

#fonction appliquée a G2
def plot_graph(graph, filename="graph_plot.png", path=[]):
    pos = nx.spring_layout(graph)
    nx.draw_networkx_nodes(graph, pos, node_color='grey')   #color tous les points en gris
    #ne sert a rien car seul les points de G1 sont caractérisés par un index
    #nx.draw_networkx_nodes(graph, pos, nodelist=[node for node in graph.nodes if node == coord_to_index(start_point[0], start_point[1])], node_color='red')
    #nx.draw_networkx_nodes(graph, pos, nodelist=[node for node in graph.nodes if node == coord_to_index(end_point[0], end_point[1])], node_color='black')

    #dessine en rouge la liste de point tel que la coordonné x du point ou y soit egale au coordonné du point start
    # start_point_index est le point de G1 qui commence, relié par 4 liaison devenu des points dans G2, il y a donc 4 points ayant la coordonnée de départ
    nx.draw_networkx_nodes(graph, pos, nodelist=[node for node in G2.nodes if node[0] == start_point_index or node[1] == start_point_index], node_color='r')
    #dessine en cyan la liste de point tel que la coordonné x du point ou y soit egale au coordonné du point end
    #de la même manière, il y a deux points ayant les coordonnées d'arrivé.
    nx.draw_networkx_nodes(graph, pos, nodelist=[node for node in G2.nodes if node[0] == end_point_index or node[1] == end_point_index], node_color='c')

    if len(path) > 0:
        nx.draw_networkx_nodes(graph, pos, nodelist=path, node_color='g') #tous les points du chemin sont colorié en vert
    nx.draw_networkx_edges(graph, pos) #dessine les liaisons

    #nx.draw_networkx_labels(graph, pos)

    plt.show()  #affiche le graphe
    #plt.savefig(filename)

# Initial graph
G1 = nx.Graph()
#les points de G1 ne servent a rien dans le programme
G1.add_nodes_from([(coord_to_index((i, j)), {"x":i,"y":j}) for i in range (0, size[0]) for j in range(0, size[1])])

#tous les points sont reliés sur l'axe x
for i in range(0, size[0]-1):
    G1.add_edges_from([unique_edge(coord_to_index((i, j)), coord_to_index((i+1, j))) for j in range(0, size[1])])

#tous les points sont reliés sur l'axe y
for j in range(0, size[1]-1):
    G1.add_edges_from([unique_edge(coord_to_index((i, j)), coord_to_index((i, j+1))) for i in range(0, size[0])])

G2 = nx.Graph()
#les chemins de G1 deviennent les points de G2
G2.add_nodes_from([unique_edge(u, v) for (u, v) in G1.edges])
#si deux coté de G1 sont relié par un point, alors on créé une liaison de G2 reliant les deux liaisons de G1 (devenu des points de G2)
for (u, v) in G1.edges:
    for (u_i, v_i) in G1.edges:
        if ((u in [u_i, v_i] or v in [u_i, v_i])):  #concordance d'une extremité entre deux liaisons
            G2.add_edge(unique_edge(u, v), unique_edge(u_i, v_i))

plot_graph(G2)

Q = defaultdict(int)
nodes_from_start = [node for node in G2.nodes if node[0] == start_point_index or node[1] == start_point_index]
nodes_from_end = [node for node in G2.nodes if node[0] == end_point_index or node[1] == end_point_index]

# QUBO trouvée : SUM(xii) - SUM(x)

# ajout des noeuds dans le graphe
print('-' * 60 + "TOUTES LES VALEURS BINAIRES DU GRAPHE" + '-' * 60)
#Constraint (1)
for node in G2.nodes:
    Q[(node, node)] = 1 * lagrangian_objective
    print("les noeuds ", (node, node), " ont comme valeur ", Q[(node, node)])

# contrainte sur le départ
print('-' * 60 + "TOUTES LES VALEURS DE DEPART DU GRAPHE" + '-' * 60)
# CONSTRAINT (2)
for node in nodes_from_start:
    Q[(node, node)] += -1 * lagrangian_struct_start
    print("les noeuds ", (node, node), " ont comme valeur ", Q[(node, node)])
    for node2 in nodes_from_start:
        if (node != node2):
            Q[(node, node2)] += 1 * lagrangian_struct_start
            print("les noeuds ", (node, node2), " ont comme valeur ", Q[(node, node2)])

# contrainte sur l'arrivée
print('-' * 60 + "TOUTES LES VALEURS D'ARRIVE DU GRAPHE" + '-' * 60)
# CONSTRAINT (2)
for node in nodes_from_end:
    Q[(node, node)] += -1 * lagrangian_struct_end
    print("les noeuds ", (node, node), " ont comme valeur ", Q[(node, node)])
    for node2 in nodes_from_end:
        if (node != node2):
            Q[(node, node2)] += 1 * lagrangian_struct_end
            print("les noeuds ", (node, node2), " ont comme valeur ", Q[(node, node2)])

print('-' * 60 + "TOUS LES POINTS ENTRANTS" + '-' * 60)
for node in G2.nodes:
    # arcs dits entrants
    #point voisin dont leur coordonnée partage une coordonnée entrante node[0] du point G2 qui correspond a l'extrémité de la liaison de G1 par lequel on entre
    entring_node = [enode for enode in G2.adj[node] if enode[0] == node[0] or enode[1] == node[0]]
    print ("Les points entrants de ", node," sont ", entring_node)

print('-' * 60 + "TOUS LES POINTS SORTANTS" + '-' * 60)
for node in G2.nodes:
    #point voisin dont leur coordonnée partage une coordonnée sortante node[0] du point G2 qui correspond au point de la liaison de G1 sortant
    exiting_node = [enode for enode in G2.adj[node] if enode[0] == node[1] or enode[1] == node[1]]
    print ("Les points sorants de ", node," sont ", exiting_node)

    ## contrainte sur les arcs de passage, représentés par des noeuds dans G2
    # On vérifie que le nombre total de voisins est bien égal à la somme des sortants et entrants
    #assert G2.degree[node] == len(entring_node) + len(exiting_node)

print('-' * 60 + "VALEURS DES POINTS ENTRANTS" + '-' * 60)
for node in G2.nodes:
    # Maintenant on définit les contraintes :
    # Sum_{entrants}(xi) + Sum_{sortants}(xi) + 2*Sum_{entrants,j>i}(xi*xj) + 2*Sum_{sortants,j>i}(xi*xj) - 2*Sum_{entrants}(xi)*Sum_{sortants}(xi)
    # Somme des xi entrants
    # CONSTRAINT (2)
    for enode in entring_node:
        Q[(enode, enode)] += 1 * lagrangian_struct_path
for enode in entring_node:
    print("valeur points entrant :", (enode, enode), Q[(enode, enode)], "pour la valeur des noeurs entrants : ", entring_node)

print('-' * 60 + "VALEURS DES POINTS SORTANTS" + '-' * 60)
for node in G2.nodes:
    # Somme des xi sortants
    # CONSTRAINT (2)
    for enode in exiting_node:
        Q[(enode, enode)] += 1 * lagrangian_struct_path
for enode in exiting_node:
    print("valeur points sortants :", (enode, enode), Q[(enode, enode)])

print('-' * 60 + "VALEURS DES ARCS ENTRANTS" + '-' * 60)
for node in G2.nodes:
    # 2*Sum_{entrants,j>i}(xi*xj)
    # CONSTRAINT (2)
    for enode in entring_node:
        for enode2 in entring_node: # Avec la double boucle on compte deux fois, donc on divise le coeff par 2
            if (enode != enode2):
                Q[(enode, enode2)] += 1 * lagrangian_struct_path
for enode in entring_node:
    for enode2 in entring_node:
        if (enode != enode2):
            print("valeur arcs entrant :", (enode, enode2), Q[(enode, enode2)])
print('-' * 60 + "VALEURS DES ARCS SORTANTS" + '-' * 60)
for node in G2.nodes:
    # 2*Sum_{sortants,j>i}(xi*xj)
    # CONSTRAINT (2)
    for enode in exiting_node:
        for enode2 in exiting_node: # Avec la double boucle on compte deux fois, donc on divise le coeff par 2
            if (enode != enode2):
                Q[(enode, enode2)] += 1 * lagrangian_struct_path
for enode in exiting_node:
        for enode2 in exiting_node: # Avec la double boucle on compte deux fois, donc on divise le coeff par 2
            if (enode != enode2):
                print("valeur arcs sortant :", (enode, enode2), Q[(enode, enode2)])
print('-' * 60 + "VALEURS DES ARCS ENTRANTS ET SORTANTS D'UN MÊME POINT" + '-' * 60)
for node in G2.nodes:
    # -2*Sum_{entrants}(xi)*Sum_{sortants}(xi)
    # CONSTRAINT (2)
    for enode in entring_node:
        for enode2 in exiting_node:
            Q[(enode, enode2)] += -2 * lagrangian_struct_path
for enode in entring_node:
        for enode2 in exiting_node:
            print("valeur des points entrants et sortants d'une liaison :", (enode, enode2), Q[(enode, enode2)])

for node in G2.nodes:
    ## Contrainte sur le nombre d'arcs sortants
    for i in range(0, len(exiting_node)-1):
        for j in range(i+1, len(exiting_node)):
            Q[(exiting_node[i], exiting_node[j])] += 1 * lagrangian_struct_path

#MANQUE CONSTRAINT (3)

for i in range(0, len(exiting_node)-1):
        for j in range(i+1, len(exiting_node)):
            print (Q[(exiting_node[i], exiting_node[j])])

print(Q)
#plot_graph(G2)

solver = greedy.SteepestDescentSolver()
sampleset = solver.sample_qubo(Q, num_reads=100)

print(sampleset)

# On affiche les trois premiers chemins
print(sampleset.variables)
for i in range(0, 3):
    sample = sampleset.record.sample[i]
    print("sample:")
    print(sample)
    path = [sampleset.variables[i] for i in range(0, len(sample)) if sample[i] == 1]
    print("path: " + str(path))
    plot_graph(G2, path=path)

    graph = [[0 for x in range(size[1])] for y in range(size[0]) ]
    for node in path:
        coord0 = index_to_coord(node[0])
        coord1 = index_to_coord(node[1])
        graph[coord0[0]][coord0[1]] = 1
        graph[coord1[0]][coord1[1]] = 1

    for j in range(0, size[1]):
        for i in range(0, size[0]):
            if (i, j) == start_point:
                print("S", end="")
            elif (i, j) == end_point:
                print("E", end="")
            elif graph[i][j] == 1:
                print("X", end="")
            else:
                print(".", end="")
        print("")


from __future__ import absolute_import

import datetime
import uuid

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.translation import ugettext

from . import importer_worker
from podcasts.models import Podcast, PodcastEpisode

class AssetImportRequest(models.Model):
    created = models.DateTimeField(auto_now_add=True)

    podcast = models.ForeignKey(Podcast, null=True, blank=True)
    episode = models.ForeignKey(PodcastEpisode, null=True, blank=True)

    expiration = models.DateTimeField()
    audio_source_url = models.URLField(blank=True, max_length=500)
    image_source_url = models.URLField(blank=True, max_length=500)
    access_token = models.CharField(max_length=128)

    resolved = models.BooleanField(default=False)

    failed = models.BooleanField(default=False)
    failure_message = models.TextField(null=True, blank=True)

    @classmethod
    def create(cls, *args, **kwargs):
        if 'access_token' not in kwargs:
            kwargs['access_token'] = AssetImportRequest.get_token()
        return cls(*args, **kwargs)

    @classmethod
    def get_token(cls):
        return str(uuid.uuid4())

    def get_json_payload(self):
        return {
            'id': self.id,
            'item_type': 'image' if self.image_source_url else 'audio',
            'entity':
                {'id': str(self.episode.id), 'name': self.episode.title, 'type': 'episode'} if
                self.episode else
                {'id': str(self.podcast.id), 'name': self.podcast.name, 'type': 'podcast'},
            'entity_type': 'episode' if self.episode else 'podcast',
            'resolved': self.resolved,
            'failed': self.failed,
            'failure_message': self.failure_message,
        }

    def resolve(self, new_url):
        if self.resolved:
            raise Exception(ugettext('Attempting to double-resolve import request'))

        if self.expiration < datetime.datetime.now():
            raise Exception(ugettext('Attempting to resolve expired import request'))

        if self.podcast:
            if self.audio_source_url:
                raise Exception(ugettext('Invalid import request'))

            orig_url = self.podcast.cover_image
            self.podcast.cover_image = new_url
            self.podcast.save()

            eps = self.podcast.podcastepisode_set.filter(image_url=orig_url)
            for ep in eps:
                ep.image_url = new_url
                ep.save()

        elif self.episode:
            if self.audio_source_url:
                self.episode.audio_url = new_url
            else:
                self.episode.image_url = new_url

            if (settings.S3_BUCKET in self.episode.audio_url and
                settings.S3_BUCKET in self.episode.image_url):
                self.episode.awaiting_import = False
            self.episode.save()

        else:
            raise Exception(ugettext('Invalid import request for neither podcast nor ep'))

        self.resolved = True
        self.save()

    def get_payload(self):
        source = self.audio_source_url or self.image_source_url
        clean_source = source
        if '#' in clean_source:  # Strip off hashes
            clean_source = clean_source[:clean_source.index('#')]
        if '?' in clean_source:  # Strip off query params
            clean_source = clean_source[:clean_source.index('?')]

        if self.podcast:
            key = 'podcasts/covers/'
        else:
            key = 'podcasts/%s/%s/' % (
                str(self.episode.podcast.id),
                'image' if self.image_source_url else 'audio')
        key = '%s%s/%s' % (key, str(uuid.uuid4()), clean_source[clean_source.rindex('/') + 1:])

        pod = self.podcast or self.episode.podcast

        return {
            'type': 'import_asset',
            'token': self.access_token,
            'id': self.id,
            'url': source,
            'bucket': settings.S3_BUCKET,
            'key': key,
        }

    def trace_podcast(self):
        return self.podcast or self.episode.podcast

    def asset_type(self):
        return 'image' if self.image_source_url else 'audio'

    def status(self):
        if self.resolved:
            return 'resolved'
        elif self.failed:
            return 'failed'
        else:
            return 'unresolved'

    def __str__(self):
        return self.audio_source_url or self.image_source_url

class Collaborator(models.Model):
    created = models.DateTimeField(auto_now_add=True)

    podcast = models.ForeignKey(Podcast, null=True, blank=True, related_name='collaborators')
    collaborator = models.ForeignKey(User, related_name='collaborated_podcasts')

    def __str__(self):
        return '%s - %s' % (self.podcast.name, self.collaborator.email)


import json

def read_json(file):
    jf=open(file)
    read_jf=jf.read()
    jf.close()
    json_data=json.loads(read_jf)
    return json_data

def write_json(file,data):
    jw=open(file,"w")
    write_jf = json.dumps(data, indent=4)
    jw.write(write_jf)

def delete_data(file,id):
    print(file,id)
    json_data=read_json(file)
    is_deleted=False
    for data in json_data['Task_list']:
        if(data['id']==id):
            json_data['Task_list'].remove(data)
            is_deleted=True
    if (is_deleted):
        for no in range(len(json_data['Task_list'])):
            json_data['Task_list'][no]['id']=no+1
    write_json(file,json_data)

def update_data(file,id):
    print(file,id)
    json_data=read_json(file)
    for data in json_data['Task_list']:
        if(data['id']==id):
            data['task_details'] = 'Done'
    write_json(file,json_data)


import itertools
import ete3
from msa_functions import get_msa_from_file, count_substitutions
from utils import compute_entropy, lists_diff
from definitions import *
from ete3 import Tree
import ete3.coretype.tree
import numpy as np

def get_newick_tree(tree):
	"""
	:param tree: newick tree string or txt file containing one tree
	:return:	tree: a string of the tree in ete3.Tree format
	"""
	if type(tree) == str:
		if os.path.exists(tree):
			with open(tree, 'r') as tree_fpr:
				tree = tree_fpr.read().strip()
		tree = Tree(tree, format=1)
	return tree

def rescale_tree_branch_lengths(tree, factor):
	"""
	:param tree: newick tree string or txt file containing one tree
	:param factor: the factor by which to multiply all branch lengths in tree
	:return:	reformatted_tree: a string of the scaled tree in Newick format
	"""
	tree = get_newick_tree(tree)
	tree_root = tree.get_tree_root()
	for node in tree_root.iter_descendants(): # the root dist is 1.0, we don't want it
		node.dist = node.dist * factor
	return tree.write(format=1, dist_formatter="%.10f")

def reroot_tree(tree, outgroup_name):
	tree = get_newick_tree(tree)
	tree.set_outgroup(tree & outgroup_name)
	return tree

def scale_tree_to_length(tree, target_dist, outgroup_name=None, scaling_method="tbl"):
	"""
	:param tree:  newick tree string or txt file containing one tree OR ete3.Tree object
	:param target_dist: numeric, the desired total tree distance
	:param outgroup_name: the name of a tree node if tree needs to be rooted,
	                      otherwise would calculate the distance from the inferred root (acoorsding to the newick order)
	:param scaling_method: "height" for longest distance from root to leaf, "tbl" for total branch lengths
	:return: a newick string of the rescaled tree
	"""
	t = get_newick_tree(tree)

	if outgroup_name: # re-root tree
		t = reroot_tree(t, outgroup_name)
	root = t.get_tree_root()

	if scaling_method.lower() == "tbl":
		dist = get_total_branch_lengths(root)
	elif scaling_method.lower() == "height":
		dist = get_tree_height(root)

	scaling_factor = target_dist / dist
	rescaled_tree = rescale_tree_branch_lengths(tree, scaling_factor)

	return rescaled_tree

def get_tree_height(tree_root):
	"""
	:param tree_root: ete3 node; because we traverse only its descendants
	(its dist is 1.0, we don't want it)
	:return: longest distance from root to leaf
	"""
	# the two configurations are the same - I compared! (Shiran)
	# current_length = 0
	# for leaf in tree_root:
	# 	current_length = max(current_length, tree_root.get_distance(leaf))
	return tree_root.get_farthest_leaf()[1]

def get_frac_of_cherries(tree):
	"""
	McKenzie, Andy, and Mike Steel. "Distributions of cherries for two models of trees."
	 Mathematical biosciences 164.1 (2000): 81-92.
	:param tree:
	:return:
	"""
	tree = get_newick_tree(tree)
	tree_root = tree.get_tree_root()
	leaves = list(tree_root.iter_leaves())
	cherries_cnt = 0
	for leaf1, leaf2 in itertools.combinations(leaves, 2):
		if leaf1.up is leaf2.up:
			cherries_cnt += 1
	return 2*cherries_cnt/len(leaves)

def get_leaves_branches(tree):
	"""
	:param tree:
	:return: a list of pendant edges lengths, i.e., the brnches that lead to leaves
	"""
	tree = get_newick_tree(tree)
	tree_root = tree.get_tree_root()
	leaves_bl = []
	for node in tree_root.iter_leaves():
		leaves_bl.append(node.dist)
	return leaves_bl

def get_sackin_index(tree):
	"""
	:param tree: ete3 tree; not rerooted!
	:return: sackin_index
	formula Sackin: https://academic.oup.com/sysbio/article/21/2/225/1713481
	"""
	node_depth_dict = {}
	sackin_index_lst = []
	for node in tree.traverse(strategy="levelorder"):
		if node.is_root():
			node_depth_dict[node] = 1
		else:
			node_depth_dict[node] = node_depth_dict[node.up] + 1

		if node.is_leaf():
			sackin_index_lst.append(node_depth_dict[node])

	return np.mean(sackin_index_lst)

def get_colless_index(tree):
	"""
	:param tree: ete3 tree; not rerooted!
	:return: colless_index
	formula Colless: https://onlinelibrary.wiley.com/doi/epdf/10.1111/j.1558-5646.1992.tb01171.x
	"""
	ndes_dict = {}
	colless_index = 0
	ntaxa = 0
	for node in tree.traverse(strategy="postorder"):
		if node.is_leaf():
			ndes_dict[node] = 1
			ntaxa += 1
		else:
			ndes_dict[node] = ndes_dict[node.children[0]] + ndes_dict[node.children[1]]
			colless_index += abs(ndes_dict[node.children[0]] - ndes_dict[node.children[1]])

	return colless_index/float((ntaxa-1)*(ntaxa-2)/2)

def get_stemminess_indexes(tree):
	"""
		:param tree: ete3 tree; not rerooting!
		:return: cumulative stemminess index Fiala and Sokal 1985
				 noncumulative stemminess index Rohlf 1990
		formula cumulative stemminess: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1558-5646.1985.tb00398.x
		formula noncumulative stemminess: https://onlinelibrary.wiley.com/doi/epdf/10.1111/j.1558-5646.1990.tb03855.x
		"""
	subtree_blsum_dict = {}
	nodes_height_dict = {}
	stem85_index_lst = []
	stem90_index_lst = []
	for node in tree.traverse(strategy="postorder"):
		if node.is_leaf():
			subtree_blsum_dict[node] = 0
			nodes_height_dict[node] = 0
		elif node.is_root():
			continue
		else:
			subtree_blsum_dict[node] = subtree_blsum_dict[node.children[0]] + subtree_blsum_dict[node.children[1]] + \
			                           node.children[0].dist + node.children[1].dist
			nodes_height_dict[node] = max(nodes_height_dict[node.children[0]] + node.children[0].dist,
			                              nodes_height_dict[node.children[1]] + node.children[1].dist)
			stem85_index_lst.append(node.dist/(subtree_blsum_dict[node] + node.dist))
			stem90_index_lst.append(node.dist/(nodes_height_dict[node]) + node.dist)

	return np.mean(stem85_index_lst), np.mean(stem90_index_lst)

def pendant_to_internal_bl_ratio(tree):
	"""
	:param tree:
	:return: sort of stemminess index, i.e., how long are the inner branches compared to the external ones
	"""
	tree = get_newick_tree(tree)
	pendant_bl_sum = np.sum(get_leaves_branches(tree))
	internal_bl_sum = get_total_branch_lengths(tree) - pendant_bl_sum

	if pendant_bl_sum == 0:
		return 0
	if internal_bl_sum == 0:
		return 100000
	return pendant_bl_sum/float(internal_bl_sum)

def get_branch_lengths(tree):
	"""
	:param tree: Tree node or tree file or newick tree string;
	:return: total branch lengths
	"""
	# TBL
	tree = get_newick_tree(tree)
	tree_root = tree.get_tree_root()
	branches = []
	for node in tree_root.iter_descendants(): # the root dist is 1.0, we don't want it
		branches.append(node.dist)
	return branches

def get_branch_lengths_estimates(tree):
	"""
	:param tree: Tree node or tree file or newick tree string;
	:return:
	"""
	# TBL
	branches = get_branch_lengths(tree)
	entropy = compute_entropy(branches)

	return max(branches), min(branches), np.mean(branches), np.std(branches), entropy

def get_diameters_estimates(tree_filepath, actual_bl=True):
	"""
	if not actual_bl - function changes the tree! send only filepath
	:param tree_filepath: tree file or newick tree string;
	:param actual_bl: True to sum actual dists, False for num of branches
	:return: min, max, mean, and std of tree diameters
	"""
	# tree = copy.deepcopy(get_newick_tree(tree)) # do not deepcopy! when trees are large it exceeds recursion depth
	if not actual_bl:
		assert isinstance(tree_filepath, str)
	tree = get_newick_tree(tree_filepath)
	tree_root = tree.get_tree_root()
	if not actual_bl:
		for node in tree_root.iter_descendants():
			node.dist = 1.0
	tree_diams = []
	leaves = list(tree_root.iter_leaves())
	for leaf1, leaf2 in itertools.combinations(leaves, 2):
		tree_diams.append(leaf1.get_distance(leaf2))
	entropy = compute_entropy(tree_diams)

	return max(tree_diams), min(tree_diams), np.mean(tree_diams), np.std(tree_diams), entropy

def get_distance_from_tips(tree):
	"""
	:param tree: Tree node or tree file or newick tree string;
	:return: mean and std of terminal branches
	"""
	tree = get_newick_tree(tree)
	tree_root = tree.get_tree_root()
	terminal_branches = []
	for leaf in tree_root.iter_leaves():
		terminal_branches.append(leaf.dist)
	return np.mean(terminal_branches), np.std(terminal_branches)

def get_total_branch_lengths(tree):
	"""
	:param tree: Tree node or tree file or newick tree string;
	:return: total branch lengths
	"""
	tree = get_newick_tree(tree) #if it's a file
	branches = get_branch_lengths(tree)
	return sum(branches)

def rename_ids(tree_str, conversion_dict):
	"""
	:param tree_str:  Newick format tree string
	:param conversion_dict: {current_id: new_id}
	:return: an updated tree string
	"""
	for sp in conversion_dict:
		tree_str = re.sub(sp + ":", conversion_dict[sp] + ":", tree_str)
	return tree_str

def calc_branch_length(tree):
	branch_lengths = get_branch_lengths(tree)
	N = len(branch_lengths)
	total_BL = sum(branch_lengths)
	mean_BL = total_BL/N

	return total_BL, mean_BL

def assign_simulated_seqs_to_nodes(tree, msa_file, ancestral_seqs_file):
	"""
	:param tree: tree file or ete3 tree
	:param msa_file: sequences names should match tree leaves
	:param ancestral_seqs_file: sequences names should match tree inner nodes + root
	:return: ete3 tree, with feature "seq" inside each node
	To extract this value, use: node.seq
	"""
	tree = get_newick_tree(tree)
	msa = get_msa_from_file(msa_file)

	for rec in msa:
		rec_name = rec.id
		(tree & rec_name).add_feature("seq", rec._seq)
	for line in open(ancestral_seqs_file):
		rec = line.strip().split("\t")
		(tree & rec[0]).add_feature("seq", rec[1])

	return tree

def assign_simulated_seqs_to_indelible_tree(sim_dir, simulated_msa_filename):
	"""
	:param sim_dir: the directory in which indelible was simulated
	:param simulated_msa_filename: the name of the simulated msa file (to extract the simulated msa + ancestral seqs)
	:return: ete3 tree, with feature "seq" inside each node
	To extract this value, use: node.seq
	"""
	sim_msa_file = sim_dir + simulated_msa_filename
	trees_file = sim_dir + "trees.txt"
	ancestral_seqs_file = sim_dir + simulated_msa_filename[:-4] + "_ANCESTRAL.phy"
	tree = indelible_executions.get_indelible_tree_from_treesfile(trees_file)

	return assign_simulated_seqs_to_nodes(tree, sim_msa_file, ancestral_seqs_file)

def copy_nodes_names_from_similar_topology(ref_topology, tree):
	tree = get_newick_tree(tree)
	ref_topology = get_newick_tree(ref_topology)
	for node in tree.traverse("postorder"):
		if node.is_leaf():
			continue
		children = node.children
		node_in_ref = (ref_topology & children[0].name).up

		# assert that nodes in ref and in tree have the same children
		ref_children = node_in_ref.children
		assert ".".join(sorted([child.name for child in children])) == ".".join(sorted([child.name for child in ref_children]))

		node.name = node_in_ref.name
	return tree

def get_cumulative_num_of_substitutions(tree_with_seqs):
	"""
	:param tree_with_seqs: ete3 tree in which every node is assigned with a 'seq' feature
	:return: cumulative number of sequences along the tree (sum over average number of substitutions per branch)
	 over nucleotides and codons substitutions
	"""
	cum_differences_nuc = 0
	cum_differences_codon = 0

	for node in tree_with_seqs.traverse("postorder"):
		if node.name != "ROOT":
			seq1 = node.seq
			seq2 = node.up.seq
			cum_differences_nuc += count_substitutions(seq1, seq2, is_codons=False)
			try:
				cum_differences_codon += count_substitutions(seq1, seq2, is_codons=True)
			except AssertionError:
				cum_differences_codon = -1
		# bl_codons = node.dist

	return cum_differences_nuc, cum_differences_codon

def convert_hexadec_branches_to_decimal(tree_str):
	reformatted_number = re.compile(r'\d+[\.\d+]*e\-\d+') # reformat small numbers
	for number in reformatted_number.findall(tree_str):
		reformatted_num = str("{0:.10f}".format(float(number)))
		tree_str = tree_str.replace(number, reformatted_num, 1)

	return tree_str

def get_internal_and_external_leaves_relative_to_subroot(tree_root, subroot):
	all_leaves = tree_root.get_leaves()
	subtree_leaves = subroot.get_leaves()
	other_leaves = lists_diff(all_leaves, subtree_leaves)
	return subtree_leaves, other_leaves

def prune_tree_at_node(tree, node_as_subroot):
	assert(isinstance(node_as_subroot, ete3.TreeNode))
	subtree_leaves, other_leaves = get_internal_and_external_leaves_relative_to_subroot(tree, node_as_subroot)
	t1 = tree.prune(subtree_leaves, preserve_branch_length=True)
	t2 = tree.prune(other_leaves, preserve_branch_length=True)
	return t1, t2

def get_largest_branch(tree):
	tree = get_newick_tree(tree)
	tree_nodes = list(tree.traverse("levelorder"))
	max_bl_node = max(tree_nodes, key=lambda node: node.dist)
	return max_bl_node


import numpy as np
import matplotlib.pyplot as plt

class Rocket:
    def __init__(self):
        self.mass = 17500.              # kg
        self.thrust_max = 20.*17500        # N
        self.thrust = 0.            # N
        self.mass_min = 10000.          # kg
        self.height = 0.            # m
        self.gravity = 9.81         # m/s^2
        self.velocity = 0.
        self.specific_impulse = 330.  # s
        self.drag = np.zeros([4, 1])              # N
        self.alpha = 1./self.gravity/self.specific_impulse
        self.c = 1./self.alpha
        self.time = 0.
        self.dt = 1.              # time step, 60 Hz
        self.is_sing = False
        self.sing_tol = 1e-6
        self.state = np.zeros([4, 1])

    def rocket_state(self, t, s):
        # rocket state equations
        # s = [h, v, m]
        # integrate height over timestep..
        snew = np.array([0., 0., 0., 0.])

        # get thrust
        thrust = self.get_thrust(s)
        snew[0] = s[1]
        d = self.drag[0]
        snew[1] = thrust/s[2] - d/s[2] - self.gravity
        snew[2] = -self.alpha*thrust
        snew[3] = thrust
        return snew

    def rk4(self, t, s):
        # RK4 Algorithm:
        # y_(n+1) = y_n + 1/6*(k_1 + 2*k_2 + 2*k_3 + k_4)*h
        # t_(n+1) = t_n + h
        # k_1 = f(t_n, y_n)
        # k_2 = f(t_n + h/2, y_n + h*k_1/2)
        # k_3 = f(t_n + h/2, y_n + h*k_2/2)
        # k_4 = f(t_n + h, y_n + h*k_3)

        k1 = self.rocket_state(t, s)
        k2 = self.rocket_state(t + self.dt*0.5, s + self.dt*k1*0.5)
        k3 = self.rocket_state(t + self.dt*0.5, s + self.dt*k2*0.5)
        k4 = self.rocket_state(t + self.dt, s + self.dt*k3)
        self.time = t + self.dt
        return s + 1./6.*(k1 + 2.*k2 + 2.*k3 + k4)*self.dt

    # is the singularity thrust to be calculated?
    def check_singularity(self):
        self.is_sing = self.sing_tol >= \
                        self.drag[0] + self.state[2]*self.gravity - \
                        self.alpha*self.drag[0]*self.state[1] - \
                        self.state[1]*self.drag[1]

    def get_thrust(self, s):
        #self.check_singularity(s)
        if self.is_sing:
            # update drag model.
            #drag = self.calculate_drag(s)
            d = self.drag[0]
            dv = self.drag[1]
            ddv = self.drag[2]
            ddh = self.drag[3]
            ddvdh = 0.  # update from drag terms with updated model
            p1 = s[2]/(d + 2.*self.c*dv+self.c**2*ddv)
            p2 = -self.gravity*(d+self.c*dv) #+self.c*(self.c-s[1])*ddh-s[1]*self.c**2*ddvdh
            #thrust = d + s[2]*self.gravity + p1*p2
            thrust = d + s[2]*self.gravity + s[2]/(d+2*self.c*dv+self.c**2*ddv)*-self.gravity*(d+self.c*dv)
            thrust = float(thrust)  # remove warning
        else:
            thrust = self.thrust_max
        if thrust > self.thrust_max:
            raise Exception("THRUST CANNOT BE BIGGER THAN THRUST_MAX!!!")
        return thrust

    def calculate_drag(self):
        # simple drag model that should be updated for complexity--this mirrors the model used in ECE6570
        drag =  np.array([0., 0., 0., 0.])
        drag[0] = 0.5*self.state[1]**2
        drag[1] = self.state[1]
        drag[2] = 1.
        drag[3] = self.state[0]
        return drag

    def set_state(self):
        self.state[0] = self.height
        self.state[1] = self.velocity
        self.state[2] = self.mass
        self.state[3] = self.thrust

    def set_state_params(self):
        self.height = self.state[0]
        self.velocity = self.state[1]
        self.mass = self.state[2]
        self.thrust = self.state[3]

    # schedule updates to state, drag, etc.
    def update(self):
        # update drag and singularity calculations first
        self.drag = self.calculate_drag()
        # check singularity
        self.check_singularity()
        self.height, self.velocity, self.mass, dummy = self.rk4(self.time, np.array([self.height, self.velocity, self.mass, self.thrust]))
        self.thrust = self.get_thrust(self.state) # uncumbered by rk4 algorithm
        # update state params
        self.set_state()

def main():
    rocket = Rocket()
    rocket.set_state() # initial parameters set to state
    tsav = []
    thsav = []
    hsav = []
    sing = []
    while rocket.mass >= rocket.mass_min:
        # print(rocket.mass)
        rocket.update()
        # print(rocket.mass)
        tsav.append(rocket.time)
        thsav.append(rocket.thrust)
        hsav.append(rocket.height)
        sing.append(rocket.is_sing)
    np.array(tsav)
    np.array(thsav)
    np.array(hsav)
    np.array(sing)
    plt.figure("Thrust v time")
    plt.plot(tsav, thsav)
    plt.show()
    plt.figure("height v time")
    plt.plot(tsav, hsav)
    plt.show()

if __name__ == '__main__':
    main()


import datetime
from dataclasses import asdict
from dataclasses import dataclass
from dataclasses import field
from enum import Enum
from typing import List
from typing import Union

try:
    from typing import Literal
except ImportError:
    from typing_extensions import Literal

from pglet import BarChart
from pglet import Checkbox
from pglet import Dialog
from pglet import Form
from pglet import SpinButton
from pglet import Stack
from pglet import Text
from pglet.barchart import Point
from pydantic import BaseModel
from pydantic import EmailStr
from pydantic import Field

try:
    from replit import db
except ImportError:
    db = dict()

class Content:

    def introduction(self):
        """
        # Easy web development for Python developers

        Web technologies are available everywhere, and they seem like a good way to create a graphical user interface
        for your Python application, either on the desktop or on the web. Unfortunately, most Python coders like
        Python, and would like to avoid using a lot of other languages (HTML, Javascript and CSS) just to get a UI
        created.

        Pglet ("pagelet") is a web server that is written in Go and uses Fluent UI React components. Actual applications
        ("clients") provide the content and react to events using a proprietary protocol.

        None of the above is visible to a Python developer, since the server comes nicely bundled in the client install.
        All you need is:

        `pip install pglet`

        and you are off to creating a web-enabled UI in pure Python (3.7+).

        As a taste of what pglet-python code looks like, this is the code for the box on the right:

        [code]
        ```
        pglet.page().add(main_view)
        ```

        As another example, this site has also been created with pglet-python, with no HTML and no CSS in the code.

        For more details, supported controls and a tutorial, see the [pglet Python docs](https://pglet.io/docs/).

        # Easy forms

        One of the most-repeated type of UI is some kind of form for entering and updating information. While creating
        forms is easy in pglet, it is nevertheless a task that provides little programming joy. The next pages
        show how to create forms using a Form control that eats annotated Python classes, for example, Python's
        [dataclasses](https://docs.python.org/3/library/dataclasses.html).

        # Easy forms with validation

        Typically you also need to somehow validate user input before you can use it for anything: you need to check
        that necessary values have been provided, check that numbers are numbers, check dates etc.

        You can avoid this repetitive code by giving the Form control a [pydantic](https://pydantic-docs.helpmanual.io/)
        object. Validation defined on the object is performed before the data is returned to you. In some cases,
        you can use the exact same data definition with your APIs (e.g. FastAPI) or when writing to a document or SQL
        data store.
        """

        from pglet import Image, Stack, Text

        main_view = Stack(
            horizontal_align="center",
            padding=20, gap=20,
            controls=[
                Image(width="50%", src="https://www.python.org/static/img/python-logo@2x.png"),
                Image(width="30%", src="https://pglet.io/img/logo_dark.svg"),
                Text(value="pydantic", bold=True, size="xxLarge"),
            ],
        )

        return main_view

    def data_first_forms(self):
        """
        Using the Form control, form definition focuses on the data you need out of it, so we define a class with
        necessary information as attributes, information types as annotations, and default values as assignements.

        Python dataclasses are convenient for this, as we do not need to spend time creating the `__init__` and other
        boilerplate.

        To create the form, all we need to do is give the class definition to the Form control:

        [code]

        You can see the resulting form on the right. Change values, and click "OK" to see the data you would get.

        Form control understands the following data types explicitly, others will be by default be represented with a
        basic text box on the form:
        - str (see later for how you can decide between single line and multiline controls for text)
        - int
        - float
        - bool
        - datetime
        - date (current DatePicker timezone restrictions prevent using it here)
        - time
        - Decimal (current SpinButton restrictions prevent using it here)
        """

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            birthdate: datetime.datetime = "2000-01-01"
            address: str = "Some Street 1, Some Town, Some Country"
            age: int = 33
            happy_today: bool = True
            email: str = "some@email.com"

        def show_data_on_submit(event):
            event.control.page.add(Dialog(
                open=True,
                title="Submitted data",
                blocking=True,
                controls=[
                    Text(value=str(event.control.value.__dict__))
                ]
            ))

        form = Form(
            value=DataclassDataModel,
            width=500,
            on_submit=show_data_on_submit
        )

        return form

    data_first_forms.display_name = "Data-first forms"

    def selecting_values(self):
        """
        Python enums are supported for selecting from a specific set of values.

        [code]
        """
        class ContactOption(str, Enum):
            EMAIL = 'email'
            PHONE = 'phone'

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            ok_to_contact: bool = True
            contact_option: ContactOption = ContactOption.EMAIL

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def more_values(self):
        """
        If there are more than 3 values, we switch to a dropdown. The threshold is configurable with the
        `threshold_for_dropdown` Form attribute.
        """
        class ContactOption(str, Enum):
            EMAIL = 'email'
            PHONE = 'phone'
            MESSAGE = 'message'
            DOVE = 'dove'

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            ok_to_contact: bool = True
            contact_option: ContactOption = ContactOption.EMAIL

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def selecting_multiple_values(self):
        """
        Annotating a field with a **list** of enums allows for multiple selection.

        [code]
        """

        class ContactOption(str, Enum):
            EMAIL = "email"
            PHONE = "phone"
            PIGEON = "pigeon"
            SMOKE_SIGNALS = "smoke signals"

        @dataclass
        class DataclassDataModel:
            contact_option: List[ContactOption] = field(
                default_factory=lambda: [ContactOption.EMAIL]
            )

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def lists_of_fields(self):
        """
        Field with a list annotation of other type than an enum is turned into a list control.
        """

        @dataclass
        class DataclassDataModel:
            alphabet: List[str] = field(
                default_factory=lambda: ["a", "b", "c"]
            )

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def nested_class_definitions(self):
        """
        Often we need to reuse parts of data structures. Form control supports nested class definitions, like `Movie`
        in the example below.

        [code]
        """
        @dataclass
        class Movie:
            title: str
            year: int

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            email: str = "some@email.com"
            favorite_movie: Movie = Movie(
                title="My Little Pony: The Movie",
                year=2017,
            )

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def several_nested_objects(self):
        """
        Several nested objects are supported with a List annotation. Lists rely on a sensible `str()` implementation to
        display nicely, open up in a separate panel for editing, and support adding and deleting items.

        [code]

        Note that to support adding more movies, we need to provide default values for Movie attributes.
        """
        @dataclass
        class Movie:
            title: str = ""
            year: int = 2000

            def __str__(self):
                return f"{self.title} ({self.year})"

        movies = [
            Movie(title="The Name of the Rose", year=1986),
            Movie(title="My Little Pony: The Movie", year=2017),
        ]

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            email: str = "some@email.com"
            favorite_movies: List[Movie] = field(default_factory=lambda: movies)

        return Form(value=DataclassDataModel, width=500, on_submit=show_submitted_data)

    def styling_and_dimensions(self):
        """
        Form is a Stack control, and inherits all the
        [attributes of Stacks](https://pglet.io/docs/controls/stack#properties).

        You can toggle the switch on the left to experiment with the light and dark themes.

        Example below shows using:
        - `title` to add a form title at the top,
        - `control_style` to define an alternative general style, with underlined text boxes,
        - `toggle_for_bool` to use a toggle instead of a checkbox for boolean values, and
        - a standard Stack attribute `gap` to add extra space between the lines.

        [code]
        """

        @dataclass
        class DataclassDataModel:
            name: str = "Dataclass Person"
            birthdate: datetime.date = "2000-01-01"
            address: str = "Some Street 1, Some Town, Some Country"
            age: int = 33
            happy_today: bool = True
            email: str = "some@email.com"

        form = Form(
            value=DataclassDataModel,
            title="Your information",
            control_style="line",
            toggle_for_bool=True,
            gap=24,
            width=500,
            on_submit=show_submitted_data,
        )

        return form

    def customizing_controls(self):
        """
        There are several ways to customize controls in a `Form`:

        1. Additional parameters for a specific field, as part of the data definition
        2. Additional parameters for a specific field, in an `__init__` parameter
        3. Map a data type to a specific control, for a single form
        4. Map a data type to a specific control, globally

        Example below covers the first 3:

        1. For dataclasses, the contents of the pglet metadata dictionary is passed as parameters for the control.
           Here we turn a single-line Textbox into a multiline one.
        2. If you do not want to mix your data model with UI specifics, you can pass a parameter to the Form,
           containing extra control initialization kwargs by field name.
        3. If you want to set all fields of a type to be mapped to a specific control, provide additional type to
           control mappings to Form. Here we want all `Amount`s have two decimals in the UI.

        [code]

        For option #4, you can set a type/control mapping globally by updating the mapping in the `Form` class directly,
        like this:

        ```
        Form.default_data_to_control_mapping["Amount"] = partial(
            SpinButton, step=.01,
        )
        ```

        ... or, of course, by subclassing the Form and setting values in the subclass `__init__`.
        """
        from dataclasses import field
        from functools import partial
        from typing import NewType

        Amount = NewType('Amount', float)

        @dataclass
        class DataclassDataModel:
            item: str = field(
                default="",
                metadata={"pglet": {
                    "multiline": True  # <<< 1
                }}
            )
            description: str = ""
            amount: Amount = Amount(0)

        form = Form(
            value=DataclassDataModel,
            control_kwargs={
                'description': {'multiline': True}  # <<< 2
            },
            control_mapping={
                "Amount": partial(  # <<< 3
                    SpinButton, step=.01,
                )
            },
            width=500,
            on_submit=show_submitted_data,
        )

        return form

    def introducing_pydantic(self):
        """
        Pydantic is not a dependency of pglet nor of the Form control, so you need to install it separately with:

        ```
        pip install pydantic
        ```

        If you need validation of email addresses, you should also install:

        ```
        pip install pydantic[email]
        ```

        Once you have pydantic, your data can inherit from `pydantic.BaseModel` instead of being decorated as a `dataclass`.

        [code]

        Next pages will cover the benefits of having pydantic in place.
        """
        from pydantic import BaseModel

        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            email: str = "some@email.com"

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    def change_the_labels(self):
        """
        Normally, the labels on each line derived from the attribute names in your data. If you need something
        different, for example punctuation, use `Field.title`.

        [code]

        See pydantic docs for the full documentation on the
        [Field function](https://pydantic-docs.helpmanual.io/usage/schema/#field-customization).
        """
        from pydantic import Field

        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            happy: bool = Field(True, title="Are you happy today?")

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    def placeholders(self):
        """
        If an input does not have text, a placeholder is shown, defined by `Field.description`.
        """
        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            email: str = Field("", description="Enter a valid email address")

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    def customizing_pydantic_fields(self):
        """
        Pydantic equivalent of the `dataclasses.field.metadata["pglet"]` is `Field.pglet`.

        Here we use this option for the `notes` attribute to get a multiline `Textbox`.

        [code]
        """
        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            notes: str = Field("", pglet={'multiline': True})

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    def validation(self):
        """
        Main benefit of adding pydantic is that get your data validated with the minimum of boilerplate code.

        Experiment with this view to see how the validation works.

        [code]

        Errors are pydantic validation errors, in English.

        Check pydantic documentation on the available
        [validating field types](https://pydantic-docs.helpmanual.io/usage/types/) and their usage.

        Form control currently maps the field types below to specific controls - everything else is a Textbox.

        - ConstrainedDecimal
        - ConstrainedFloat
        - ConstrainedInt
        - EmailStr
        - FutureDate
        - NegativeFloat
        - NegativeInt
        - PastDate
        - PositiveFloat
        - PositiveInt
        - StrictBool
        - StrictFloat
        - StrictInt
        """
        from pydantic import conint
        from pydantic import EmailStr

        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            birthdate: datetime.date = "2000-01-01"
            age: conint(ge=0, lt=150) = 0
            email: EmailStr = Field("", description="Enter a valid email address")

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    def cross_field_validation(self):
        """
        Pydantic supports defining more complex relationships between fields.

        In this example, email must be filled (and a valid email) if newsletters have been requested.

        If validator returns an error, the capitalised str value of the error is shown to the user as the error
        message under the field. Pydantic standard validation errors are in English.

        [code]

        Again, pydantic docs contain a lot more information about
        [validators](https://pydantic-docs.helpmanual.io/usage/validators/).
        """
        from pydantic import validator

        class PydanticDataModel(BaseModel):
            name: str = "Pydantic Person"
            newsletter_ok: bool = Field(
                False, title="Send me the monthly newsletter"
            )
            email: Union[EmailStr, Literal[""]] = Field("", description="Valid email needed for newsletter")

            @validator('email', pre=True, allow_reuse=True)
            def email_filled_if_needed(cls, value, values):
                if values.get("newsletter_ok") and not value:
                    raise ValueError("Need email for newsletter")
                return value

        return Form(value=PydanticDataModel(), width=500, on_submit=show_submitted_data)

    cross_field_validation.display_name = "Cross-field validation"

    def status(self):
        """
        The status of the Form control is: **early Proof of Concept for discussion and feedback**

        *Some* todo items remain.

        [no code]
        """
        todo = '''
        Ordering of lists
        Slider option for number ranges
        Proper Documentation
        Responsive layout
        Align/integrate with Grid control
        Dates with DatePicker
        Manage decimal.Decimal values
        '''

        done = '''
        Lists for basic types
        Toggle as an alternative for Checkbox
        Support custom control parameters (e.g. multiline)
        Support customising controls for types
        Multiple selection from enums
        Some tests
        Documentation in the shape of this demo
        '''

        return Stack(
            padding=20,
            width=400,
            controls=[
                Checkbox(label=label.strip(), value=False, disabled=True)
                for label in todo.strip().splitlines()
            ] + [
                Checkbox(label=label.strip(), value=True, disabled=True)
                for label in done.strip().splitlines()
            ]
        )

    def grande_finale(self):
        """
        As one last thing, let's combine the Form control, replit database utility and the pglet graph control into a
        quick poll.

        Please select the options you are interested in and click "OK".

        [code]

        The `db` object here is a replit database that is essentially a dict with per-program persistent contents.
        """
        @dataclass
        class PollData:
            pglet: bool = False
            python: bool = False
            pglet_with_python: bool = False
            pglet_with_some_other_language: bool = False
            pglet_with_forms: bool = False
            pydantic: bool = False
            pydantic_form_validation: bool = False

        chart = BarChart(
            data_mode='fraction',
            padding=20,
            width=210,
            points=[]
        )

        def update_chart():
            chart.points.clear()
            values = list(reversed(sorted(
                [
                    (field, db.get(field, 0))
                    for field in PollData.__annotations__
                    if field != "answers"
                ],
                key=lambda value: value[1]
            )))
            max_value = values[0][1]
            for field, value in values:
                display_name = field.replace("_", " ").capitalize()
                chart.points.append(
                    Point(legend=display_name, x=value, y=max_value),
                )
        update_chart()

        poll = Form(
            value=PollData,
            title="I am interested in...",
            width=300,
            label_width="100%",
            control_width="fit-content",
        )

        def update_db_values(event):
            value = event.control.value
            for key, value in asdict(value).items():
                if value:
                    db[key] = db.get("key", 0) + 1
            db["answers"] = db.get("answers", 0) + 1

            update_chart()
            poll.submit_button.disabled = True
            event.control.page.update()

        poll.on_submit = update_db_values

        stack = Stack(controls=[poll, chart])

        return stack

    grande_finale.display_name = "Grande Finale"

def show_submitted_data(event):
    value = event.control.value
    event.control.page.add(Dialog(open=True, title="Submitted data", blocking=True, controls=[
        Text(value=str(value.__dict__))
    ]))

content = [value for attribute, value in Content.__dict__.items() if callable(value)]

{'Image': 'pglet.Image', 'Stack': 'pglet.Stack', 'Text': 'pglet.Text', 'field': 'dataclasses.field', 'partial': 'functools.partial', 'NewType': 'typing.NewType', 'BaseModel': 'pydantic.BaseModel', 'Field': 'pydantic.Field', 'conint': 'pydantic.conint', 'EmailStr': 'pydantic.EmailStr', 'validator': 'pydantic.validator'}.__dict__.items


# -*- coding: utf-8 -*-
import scrapy

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector
from cwcraigs.items import CraigslistSampleItem

class CraigsSpider(CrawlSpider):
    name = "craigs"
    allowed_domains = ["sfbay.craigslist.org"]
    start_urls = ["http://sfbay.craigslist.org/search/vnn"]

    # the below is next page url
    # 'http://sfbay.craigslist.org/search/npo?s=100'
    # 'http://sfbay.craigslist.org/search/npo?s=200'

    rules = (
        # Rule(SgmlLinkExtractor(allow=(), restrict_xpaths=('//div[@class="pageNumbers"]/a',)), callback="parse_items", follow= True),
        Rule(SgmlLinkExtractor(allow=(), restrict_xpaths=('//span[@class="buttons"]/a',)), callback="parse_items",
             follow=True),
    )

    def __init__(self, name=None, **kwargs):
        from cwcraigs.DBUtils import DBUtils
        self.dbutils = DBUtils(kwargs['mongo_uri'], kwargs['mongo_db'], kwargs['collection_name'])
        self.dbutils.open_spider()
        super(CraigsSpider, self).__init__(name, **kwargs)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        return super(CraigsSpider, cls).from_crawler(crawler,
                                                         args,
                                                         mongo_uri=crawler.settings.get('MONGODB_SERVER'),
                                                         mongo_db=crawler.settings.get('MONGODB_DB', 'items'),
                                                         collection_name=(
                                                             "__history" + crawler.settings.get('MONGODB_COLLECTION'))
                                                         )

    def parse_items(self, response):
        hxs = HtmlXPathSelector(response)
        titles = hxs.xpath('//span[@class="pl"]')
        items = []

        count = 0
        for title in titles:
            item = CraigslistSampleItem()
            item["url"] = (response.url+"-{0}").format(count)
            item["title"] = title.xpath('a/span/text()').extract()
            item["link"] = title.xpath("a/@href").extract()
            items.append(item)

            count += 1
        return(items)


import typing, netaddr
from box import Box

from netsim import api

"""
Enable 'proxy_arp' attribute per VRF
"""
def init(topology: Box) -> None:
  topology.defaults.attributes.vrf.proxy_arp = None

"""
Add 'proxy_arp' flag and trigger custom config file for any interfaces that are
part of a VRF with proxy ARP enabled
"""
def post_transform(topology: Box) -> None:

  if not topology.get('vrfs',[]):
    return

  target_vrfs = [ v for v,d in topology.vrfs.items() if 'proxy_arp' in d ]

  config_name = api.get_config_name(globals())

  # Iterate over node[x].interfaces that are part of a VRF with proxy_arp enabled
  for n, ndata in topology.nodes.items():
    for i in ndata.interfaces:
      if i.type in ['lan','p2p','svi'] and 'vrf' in i and i.vrf in target_vrfs \
        and ('ipv4' in i or 'ipv6' in i):
          i.proxy_arp = True
          api.node_config(ndata,config_name)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
drawRegion.py
Draw a ROI region by click the left button of the mouse.
'''

import numpy as np
import cv2 as cv

def on_mouse(event, x, y, flags, param):
    '''
    Callback function to respond the mouse left button down action
    Args:
        event:  mouse event code
        x:      image col index of the mouse point location
        y:      image col index of the mouse point location
    '''
    if event == cv.EVENT_LBUTTONDOWN:
        '''respond the left button click action'''
        ptls.append([x, y])
        cmimg = cmsrc.copy()
        cmmask = np.zeros((cmW, cmH), np.uint8)
        if len(ptls) > 1:
            for i in range(len(ptls)-1):
                # draw lines if the number of clicked points lager than 1
                cv.line(cmimg, tuple(ptls[i]), tuple(ptls[i+1]), green, 5)
            if len(ptls) > 2:
                # draw polygon if the number of clicked points lager than 2
                cv.polylines(cmimg, np.array([ptls], np.int32), True, black, 3)
                cv.fillPoly(cmmask, np.array([ptls], np.int32), white, 1)
                '''draw the polygon by the clicked point of the image'''
                image, contours, hierarchy = cv.findContours(cmmask, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE)
                cv.drawContours(cmmask, contours, -1, white, -1)
                cv.drawContours(cmimg, contours, -1, white, 1)
                '''
                Find the extreme outer contours of the polygon to fill the overlap holes.
                Comment the upper 3 lines if you want to show the overlap holes.
                '''
                cmdst = cv.bitwise_and(cmsrc, cmsrc, mask=cmmask)
                cv.imshow('dst', cmdst)
                cv.imshow('mask', cmmask)
            cv.imshow('src', cmimg)
            cv.waitKey(0)

if __name__ == '__main__':
    ptls = []
    green = (0, 255, 0)
    black = (0, 0, 0)
    white = (255, 255, 255)
    '''define the line colors'''
    cmsrc = cv.imread('lena.jpg')
    cmW, cmH, cmC = cmsrc.shape
    cmimg = cmsrc.copy()
    '''load the image'''
    cv.imshow('src', cmimg)
    cv.setMouseCallback('src', on_mouse)
    '''show image and set the callback function'''
    cv.waitKey(0)

    cv.destoryAllWindows()


import pytest
import torch

from hysteresis.meshing import create_triangle_mesh
from hysteresis.states import (
    get_states,
    switch,
    sweep_up,
    sweep_left,
    predict_batched_state,
)

class TestStateCalc:
    def test_grad(self):
        h = torch.tensor(1.0, requires_grad=True)
        dummy = h * 4.0
        dummy.backward()

    def test_switches(self):
        mesh = torch.tensor(create_triangle_mesh(1.0))
        h = torch.ones(1, requires_grad=True)
        flip_flag = switch(h, mesh[:, 1])

        flip_flag[0].backward()
        assert h.grad is not None

    def test_sweeps(self):
        mesh = torch.tensor(create_triangle_mesh(0.1))
        h = torch.tensor(0.5, requires_grad=True)
        initial_state = torch.ones_like(mesh[:, 1]) * -1.0
        second_state = sweep_up(h, mesh, initial_state)

        h2 = torch.tensor(0.75, requires_grad=True)
        thrid_state = sweep_up(h2, mesh, second_state)

        h3 = torch.tensor(0.4, requires_grad=True)
        final_state = sweep_left(h3, mesh, thrid_state)

        # final_state[0].backward()
        total = torch.sum(final_state)
        total.backward()
        assert h.grad is not None
        assert h2.grad is not None
        assert h3.grad is not None
        assert not torch.isnan(h.grad)
        assert not torch.isnan(h2.grad)
        assert not torch.isnan(h3.grad)

    def test_get_states(self):
        mesh = torch.tensor(create_triangle_mesh(0.1))
        h = torch.tensor((0.5, 0.75, 0.4, 0.5), requires_grad=True)
        states = get_states(h, mesh)

        total = torch.sum(states[-1])
        total.backward()
        assert not torch.any(torch.isnan(h.grad))

        # plt.tripcolor(*mesh.T, states[-1].detach())
        # plt.show()

    def test_error_handling(self):
        mesh = torch.tensor(create_triangle_mesh(0.1))
        h = torch.tensor((0.5, 0.75, 0.4, 0.5, 1.0, 0.0))
        states = get_states(h, mesh_points=mesh)

        h = torch.tensor((0.5, 1.75, 0.4, 0.5))
        with pytest.raises(RuntimeError):
            states = get_states(h, mesh_points=mesh)

        h = torch.tensor((-0.5, 0.75, 0.4, 0.5))
        with pytest.raises(RuntimeError):
            states = get_states(h, mesh_points=mesh)

    def test_batched(self):
        mesh = torch.tensor(create_triangle_mesh(0.5))
        current_state = torch.ones(mesh.shape[0]) * -1.0
        current_field = torch.tensor(-0.01)
        h = torch.linspace(0.0, 1.0, 20, requires_grad=True)
        out = predict_batched_state(
            h.reshape(20, 1), mesh, current_state, current_field
        )
        assert out.shape == torch.Size([20, 1, len(mesh)])
        out[0][0][0].backward()
        assert not torch.any(torch.isnan(h.grad))

        # test different sized tensors for h - requires last dim to be 1
        tests = [torch.rand(10, 20), torch.rand(10), torch.rand(2, 4, 5)]
        for t in tests:
            out = predict_batched_state(t, mesh, current_state, current_field)
            assert out.shape == torch.Size([*t.shape, len(mesh)])


# Libraries
import streamlit as st
from PIL import Image
import pandas as pd
import plotly.express as px
# Layout

st.set_page_config(page_title='Developer Activity in 2022 on NEAR Protocol', page_icon=':bar_chart:', layout='wide')
st.title('Developer activity')

#CSS
with open('style.css')as f:
    st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html = True)

@st.cache(ttl=1)
def get_data(query):
    if query == 'Top Open PR':
        return pd.read_json('https://node-api.flipsidecrypto.com/api/v2/queries/4877ed5c-f392-4c30-b987-6bd0e3f9d783/data/latest')
    elif query == 'Top Close PR':
        return pd.read_json('https://node-api.flipsidecrypto.com/api/v2/queries/76e9f3ce-8ffc-4f63-b752-ab341e14b1e9/data/latest')
    # elif query == 'test':
    #     return pd.read_json('https://node-api.flipsidecrypto.com/api/v2/queries/4b9fa244-4679-4fe7-9fda-55cb3fedb140/data/latest')
    # elif query == 'Transactions Fee Payers':
    #     return pd.read_json('https://node-api.flipsidecrypto.com/api/v2/queries/7eae69ea-2387-420d-b4b9-6eceeb5ef22d/data/latest')
    return None

#Get Data
Top_Open_PR = get_data('Top Open PR')
Top_Close_PR = get_data('Top Close PR')

theme_plotly = None # None or streamlit

# st.table(Top_Open_PR)
c1,c2 = st.columns(2)
with c1:
    st.info('Developer Activity - Top Open PR')
    st.table(Top_Open_PR)
with c2:
    st.info('Developer Activity - Top Close PR')
    st.table(Top_Close_PR)


import argparse
import numpy as np
import random
import os, sys
import time
import matplotlib
matplotlib.use('agg')  # use matplotlib without GUI support
import matplotlib.pyplot as plt

import torch
from torch.utils.data import DataLoader
import torch.optim as optim

sys.path.append('./auxiliary/')
from auxiliary.model import PoseEstimator
from auxiliary.dataset import Pascal3D
from auxiliary.utils import AverageValueMeter, load_checkpoint, KaiMingInit, rotation_acc, get_pred_from_cls_output
from auxiliary.loss import DeltaLoss, CELoss, SmoothCELoss
from evaluation import val

# =================PARAMETERS=============================== #
parser = argparse.ArgumentParser()

## training param
parser.add_argument('--resume', type=str, default=None, help='optional resume model path')
parser.add_argument('--save_dir', type=str, default='save_models', help='save directory')
parser.add_argument('--lr', type=float, default=0.0001, help='learning rate of optimizer')
parser.add_argument('--batch_size', type=int, default=16, help='input batch size')
parser.add_argument('--workers', type=int, help='number of data loading workers', default=4)
parser.add_argument('--n_epoch', type=int, default=200, help='number of epochs to train for')
parser.add_argument('--lr_step', type=int, default=200, help='step to decrease lr')
parser.add_argument('--print_freq', type=int, default=100, help='frequency of output print')

## dataset param
parser.add_argument('--setting', type=str, default=None, choices=['IntraDataset', 'InterDataset'])
parser.add_argument('--root_dir_train', type=str, default=None, help='training dataset directory')
parser.add_argument('--annot_train', type=str, default=None, help='training dataset annotation file')
parser.add_argument('--novel', action='store_true', help='whether to exclude novel classes during training')
parser.add_argument('--keypoint', action='store_true', help='use only samples with keypoint annotations')
parser.add_argument('--shot', type=int, default=None, help='few shot number')

## method param
parser.add_argument('--input_dim', type=int, default=224, help='input image dimension')
parser.add_argument('--img_feature_dim', type=int, default=512, help='feature dimension for images')
parser.add_argument('--point_num', type=int, default=2500, help='number of points used in each sample')
parser.add_argument('--shape_feature_dim', type=int, default=512, help='feature dimension for shapes')
parser.add_argument('--bin_size', type=int, default=15, help='bin size for the euler angle classification')
parser.add_argument('--smooth', type=float, default=0.2, help='activate label smoothing in classification')

opt = parser.parse_args()
print(opt)
# ========================================================== #

# =================CREATE DATASET=========================== #
if opt.novel:
    print('Excluding novel classes in training')
else:
    print('Including novel classes in training')

# 20 novel classes on ObjectNet3D
if opt.setting == 'IntraDataset':
    test_cats = ['bed', 'bookshelf', 'calculator', 'cellphone', 'computer', 'door', 'filing_cabinet', 'guitar', 'iron',
                 'knife', 'microwave', 'pen', 'pot', 'rifle', 'shoe', 'slipper', 'stove', 'toilet', 'tub', 'wheelchair']
    dataset_train = Pascal3D(train=True, root_dir=opt.root_dir_train, annotation_file=opt.annot_train,
                             cls_choice=test_cats, input_dim=opt.input_dim, point_num=opt.point_num,
                             keypoint=opt.keypoint, novel=opt.novel, shot=opt.shot)

# 12 novel classes on Pascal3D
elif opt.setting == 'InterDataset':
    test_cats = ['aeroplane', 'bicycle', 'boat', 'bottle', 'bus', 'car',
                 'chair', 'diningtable', 'motorbike', 'sofa', 'train', 'tvmonitor']
    dataset_train = Pascal3D(train=True, root_dir=opt.root_dir_train, annotation_file=opt.annot_train,
                             cls_choice=test_cats, input_dim=opt.input_dim, point_num=opt.point_num,
                             keypoint=opt.keypoint, novel=opt.novel, shot=opt.shot)
else:
    sys.exit('Wrong setting!')

train_loader = DataLoader(dataset_train, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers, drop_last=True)

print('train data consist of {} samples'.format(len(dataset_train)))
# ========================================================== #

# ================CREATE NETWORK============================ #
azi_classes, ele_classes, inp_classes = int(360 / opt.bin_size), int(180 / opt.bin_size), int(360 / opt.bin_size)

model = PoseEstimator(shape_feature_dim=opt.shape_feature_dim, img_feature_dim=opt.img_feature_dim,
                      azi_classes=azi_classes, ele_classes=ele_classes, inp_classes=inp_classes)
model.cuda()
if opt.resume is not None:
    load_checkpoint(model, opt.resume)
else:
    model.apply(KaiMingInit)
# ========================================================== #

# ================CREATE OPTIMIZER AND LOSS================= #
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.lr, weight_decay=5e-4)
lrScheduler = optim.lr_scheduler.MultiStepLR(optimizer, [opt.lr_step], gamma=0.1)

criterion_azi = SmoothCELoss(360, 24, opt.smooth) if opt.smooth is not None else CELoss(360)
criterion_ele = SmoothCELoss(180, 12, opt.smooth) if opt.smooth is not None else CELoss(180)
criterion_inp = SmoothCELoss(360, 24, opt.smooth) if opt.smooth is not None else CELoss(360)
criterion_reg = DeltaLoss(opt.bin_size)
# ========================================================== #

# =============DEFINE stuff for logs ======================= #
result_path = os.path.join(os.getcwd(), opt.save_dir)
if not os.path.exists(result_path):
    os.makedirs(result_path)
logname = os.path.join(result_path, 'training_log.txt')
with open(logname, 'a') as f:
    f.write(str(opt) + '\n' + '\n')
    f.write('training set: ' + str(len(dataset_train)) + '\n')
    # f.write('evaluation set: ' + str(len(dataset_eval)) + '\n')

# arrays for saving the losses and accuracies
losses = np.zeros((opt.n_epoch, 2))  # training loss and validation loss
accuracies = np.zeros((opt.n_epoch, 2))  # train and val accuracy for classification and viewpoint estimation
# ========================================================== #

# =================== DEFINE TRAIN ========================= #
def train(data_loader, model, bin_size, criterion_azi, criterion_ele, criterion_inp, criterion_reg, optimizer):
    train_loss = AverageValueMeter()
    train_acc_rot = AverageValueMeter()

    model.train()

    data_time = AverageValueMeter()
    batch_time = AverageValueMeter()
    end = time.time()
    for i, data in enumerate(data_loader):
        # load data and label
        im, shapes, label, _ = data
        im, shapes, label = im.cuda(), shapes.cuda(), label.cuda()
        data_time.update(time.time() - end)

        # forward pass
        out = model(im, shapes)

        # compute losses and update the meters
        loss_azi = criterion_azi(out[0], label[:, 0])
        loss_ele = criterion_ele(out[1], label[:, 1])
        loss_inp = criterion_inp(out[2], label[:, 2])
        loss_reg = criterion_reg(out[3], out[4], out[5], label.float())
        loss = loss_azi + loss_ele + loss_inp + loss_reg
        train_loss.update(loss.item(), im.size(0))

        # compute rotation matrix accuracy
        preds = get_pred_from_cls_output([out[0], out[1], out[2]])
        for n in range(len(preds)):
            pred_delta = out[n + 3]
            delta_value = pred_delta[torch.arange(pred_delta.size(0)), preds[n].long()].tanh() / 2
            preds[n] = (preds[n].float() + delta_value + 0.5) * bin_size
        acc_rot = rotation_acc(torch.cat((preds[0].unsqueeze(1), preds[1].unsqueeze(1), preds[2].unsqueeze(1)), 1),
                               label.float())
        train_acc_rot.update(acc_rot.item(), im.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure bacth time
        batch_time.update(time.time() - end)
        end = time.time()

        if (i + 1) % opt.print_freq == 0:
            print("\tEpoch %3d --- Iter [%d/%d] Train loss: %.2f || Train accuracy: %.2f" %
                  (epoch, i + 1, len(data_loader), train_loss.avg, train_acc_rot.avg))
            print("\tData loading time: %.2f (%.2f)-- Batch time: %.2f (%.2f)\n" %
                  (data_time.val, data_time.avg, batch_time.val, batch_time.avg))

    return [train_loss.avg, train_acc_rot.avg]
# ========================================================== #

# =============BEGIN OF THE LEARNING LOOP=================== #
# initialization
best_acc = 0.

for epoch in range(opt.n_epoch):
    # update learning rate
    lrScheduler.step()

    # train
    train_loss, train_acc_rot = train(train_loader, model, opt.bin_size,
                                      criterion_azi, criterion_ele, criterion_inp, criterion_reg, optimizer)

    # save checkpoint
    torch.save({'epoch': epoch,
                'model': model.state_dict(),
                'optimizer': optimizer.state_dict(),
                'losses': losses},
               os.path.join(result_path, 'checkpoint.pth'))

    # save losses and accuracies into log file
    with open(logname, 'a') as f:
        text = str('Epoch: %03d || train_loss %.2f || train_acc %.2f \n \n' % (epoch, train_loss, train_acc_rot))
        f.write(text)
# ========================================================== #

import pickle
import collections

data_loader = DataLoader(dataset_train, batch_size=1, shuffle=False, num_workers=1)

with torch.no_grad():
    model.eval()

    class_data = collections.defaultdict(list)
    for i, data in enumerate(data_loader):
        # load data and label
        im, shapes, label, cls = data
        im, shapes = im.cuda(), shapes.cuda()

        cls_data = model(im, shapes, get_mean_cls_data=True)

        class_data[cls[0]].append(cls_data.squeeze())

# calculate mean attention vectors of every class
mean_class_data = {k: sum(v) / len(v) for k, v in class_data.items()}

with open(os.path.join(result_path, 'mean_class_data.pkl'), 'wb') as f:
    pickle.dump(mean_class_data, f, pickle.HIGHEST_PROTOCOL)
print('save mean class data done!')


import json
import re
from lxml import etree
import pymongo
import time
import requests
import ssl
context = ssl._create_unverified_context()
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

client = pymongo.MongoClient('mongodb://localhost:27017')
db = client['douban']
table = db['douban-250']

url = ['https://movie.douban.com/top250?start={}'.format(str(i)) for i in range(0,250,25)]
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}

def download_urls(url):
    response = requests.get(url,headers=header,verify=False)
    selector = etree.HTML(response.text)
    url_list = selector.xpath('//a[@class=""]/@href')
    return url_list

def get_content(url):
    #print(url)
    response = requests.get(url,headers=header,verify=False)
    selector = etree.HTML(response.text)
    name = selector.xpath('//span[@property="v:itemreviewed"]/text()')[0]
    director = selector.xpath('//a[@rel="v:directedBy"]/text()')[0]
    #主演只取前三个
    actor = selector.xpath('//a[@rel="v:starring"]/text()')[:3]
    actor = r';'.join(actor)
    type = selector.xpath('//span[@property="v:genre"]/text()')
    type = r';'.join(type)
    runtime = selector.xpath('//span[@property="v:runtime"]/text()')[0]
    #制片国家使用re
    country = re.findall('制片国家/地区:(.+)
', response.text, re.S)
    level = selector.xpath('//strong[@class="ll rating_num"]/text()')[0]

    info = {
        '电影':name,
        '导演':director,
        '主演':actor,
        '类型':type,
        '国家':country,
        '时长':runtime,
        '评价':level }

    table.insert_one(info)

if __name__=='__main__':
    start = time.time()
    url_lists = []  #全部链接的列表
    for i in url:
        url_lists.extend(download_urls(i))
    for url in url_lists:
    #有几个链接是无效的 ，使用try...except...
        try:
            get_content(url)
        except Exception as e:
            continue

    stop = time.time()
    print('run time:', (stop - start))


#!/usr/bin/python3
""" Send request """

if __name__ == "__main__":
    import requests
    import sys

    response = requests.get(sys.argv[1])
    statusCode = response.status_code

    if statusCode >= 400:
        print("Error code:", statusCode)
    else:
        print(response.text)


"""

49. Group Anagrams

Given an array of strings, group anagrams together.

Example:

Input: ["eat", "tea", "tan", "ate", "nat", "bat"],
Output:
[
  ["ate","eat","tea"],
  ["nat","tan"],
  ["bat"]
]
Note:

All inputs will be in lowercase.
The order of your output does not matter.

"""

import collections

def group_anagrams(strs):
    dic = collections.defaultdict(list)
    ans = []
    for each_word in strs:
        dic[tuple(sorted(each_word))].append(each_word)
    for key, values in dic.items():
        ans.append(values)
    return ans

group_anagrams(["eat", "tea", "tan", "ate", "nat", "bat"])