import sys

from flask import Flask, request, jsonify
import sqlalchemy
from db import db_session
from db import init_db
from models import Queries, Users, Roles

app = Flask(__name__)

@app.teardown_appcontext
def shutdown_session(exception=None):
    db_session.remove()

def check_rights(required_role, action):
    if required_role not in [i.name for i in db_session.query(Users).filter(Users.name == request.args.get('user')).first().role]:
        return {'result': False,
                'error': {'text': "You have to be " + required_role + " to " + action}}
    return {'result': True,
            'text': "OK"}

@app.route('/create')
def create():
    is_allowed = check_rights('User', "create Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    new_request = Queries(text=request.args.get('text'), state=1, author=request.args.get('user'))
    db_session.add(new_request)
    db_session.commit()

    res = db_session.query(Queries).get(new_request.id)
    return res.serialize

@app.route('/edit')
def edit():
    is_allowed = check_rights('User', "edit Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    query_to_edit = db_session.query(Queries).filter(Queries.id == request.args.get('id'), Queries.author == request.args.get('user')).first()

    if not query_to_edit:
        return jsonify({'text': "0 queries found. Check author and id"})

    query_to_edit.text = request.args.get('new_text')
    db_session.commit()

    res = db_session.query(Queries).get(query_to_edit.id)
    return res.serialize

@app.route('/send')
def send():
    is_allowed = check_rights('User', "send Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    query_to_update = db_session.query(Queries).filter(Queries.id == request.args.get('id'), Queries.author == request.args.get('user')).first()

    if not query_to_update:
        return jsonify({'text': "0 queries found. Check author and id"})

    if query_to_update.state == 1:
        query_to_update.state = 2
        db_session.commit()
    else:
        return "query is already sent to operator"
    return jsonify(query_to_update.serialize)

@app.route('/view')
def view():
    is_allowed = check_rights('User', "view your Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    requester_query = db_session.query(Queries).filter(Queries.author == request.args.get('user')).all()
    return jsonify([i.serialize for i in requester_query])

@app.route('/requested')
def list_req():
    is_allowed = check_rights('Operator', "view requested Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    result_list = db_session.query(Queries).filter(Queries.state == 2).order_by(sqlalchemy.desc(Queries.created)).all()
    if not result_list:
        return jsonify([i.serialize for i in result_list])

    result_list = [i.serialize for i in result_list]
    for i in result_list:
        i['text'] = "-".join(i['text'])

    print(result_list[0]['text'], file=sys.stderr)
    return jsonify(result_list)

@app.route('/accept')
def accept():
    is_allowed = check_rights('Operator', "accept Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    query_to_accept = db_session.query(Queries).get(request.args.get('id'))

    if not query_to_accept:
        return jsonify({'text': "0 queries found. Check id"})

    if query_to_accept.state == 2:
        query_to_accept.state = 3
        db_session.commit()
    else:
        return jsonify({'text': "query is already processed"})

    return query_to_accept.serialize

@app.route('/decline')
def decline():
    is_allowed = check_rights('Operator', "decline Queries")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    query_to_decline = db_session.query(Queries).get(request.args.get('id'))

    if not query_to_decline:
        return jsonify({'text': "0 queries found. Check id"})

    if query_to_decline.state == 2:
        query_to_decline.state = 4
        db_session.commit()
    else:
        return jsonify({'text': "query is already processed"})

    return query_to_decline.serialize

@app.route('/list_users')
def list_users():
    is_allowed = check_rights('Admin', "view Users")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    all_users = db_session.query(Users).all()

    roles = [[i.name for i in db_session.query(Users).filter(Users.id == j.id).first().role] for j in all_users]
    users = [i.serialize for i in all_users]

    for i in range(len(users)):
        users[i]['role'] = roles[i]

    return jsonify(users)

@app.route('/define_rights')
def define_rights():
    is_allowed = check_rights('Admin', "define Roles")
    if not is_allowed['result']:
        return jsonify({'text': is_allowed['error']})

    user_to_define_rights = db_session.query(Users).get(request.args.get('id'))

    if 'Operator' not in [i.name for i in user_to_define_rights.role]:
        operator_role = db_session.query(Roles).filter(Roles.name == "Operator").first()
        user_to_define_rights.role.append(operator_role)
        db_session.commit()
    else:
        return jsonify({'text': "User is already processed"})
    return user_to_define_rights.serialize

if __name__ == "__main__":
    init_db()
    app.run(host='0.0.0.0', debug=True)


import logging
import couchdb
import urlparse
import json
import  urllib2
import threading
import re
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from lr.model import LRNode as sourceLRNode, \
            NodeServiceModel, ResourceDataModel, LRNodeModel, defaultCouchServer, appConfig
from lr.lib.base import BaseController, render
from lr.lib import helpers as h
import base64
import pprint

import Queue

log = logging.getLogger(__name__)

class DistributeController(BaseController):
    __TARGET_NODE_INFO = 'taget_node_info'
    __OK = 'ok'
    __ERROR = 'error'

    def __before__(self):
        self.resource_data = appConfig['couchdb.db.resourcedata']

    """REST Controller styled on the Atom Publishing Protocol"""
    # To properly map this controller, ensure your config/routing.py
    # file has a resource setup:
    #     map.resource('distribute', 'distribute')

    def destination(self):
        """GET /destination: return node information"""
        # url('distribute')
        response = {self.__OK: True}

        try:
            response[self.__TARGET_NODE_INFO] = sourceLRNode.distributeInfo
        except Exception as ex:
            log.exception(ex)
            response["error":"Internal error"]

        log.info("received distribute request...returning: \n"+pprint.pformat(response, 4))
        return json.dumps(response)

    def _getDistinationInfo(self, connection):
        # Make sure we only have one slash in the url path. More than one
        #confuses pylons routing libary.
        destinationURL = urlparse.urljoin(connection.destination_node_url.strip(),
                                                                "destination")

        request = urllib2.Request(destinationURL)
        credential = sourceLRNode.getDistributeCredentialFor(destinationURL)

        if credential is not None:
            base64string = base64.encodestring('%s:%s' % (credential['username'],credential['password'])).replace("\n", "")
            request.add_header("Authorization", "Basic %s" % base64string)

        log.info("\n\nAccess destination node at: "+pprint.pformat(request.__dict__))
        return json.load(urllib2.urlopen(request))

    def _canDistributeTo(self, connection, sourceNodeInfo):

        if  not connection.active:
            return {self.__OK: False,
                         'connection_id': connection.connection_id,
                         self.__ERROR: 'Inactive connection'}

        result={self.__OK:True, 'connection_id': connection.connection_id }
        sourceNodeInfo = h.dictToObject(sourceNodeInfo)
        try:
            destinationNodeInfo = h.dictToObject(self._getDistinationInfo(connection)[self.__TARGET_NODE_INFO])
            result['destinationNodeInfo'] = destinationNodeInfo
            # Don't bother going through all the filter out rules if the source and
            # destionation nodes are on the same community and network.
            if((sourceNodeInfo.community_id == destinationNodeInfo.community_id) and
                (sourceNodeInfo.network_id == destinationNodeInfo.network_id) and
                not (sourceNodeInfo.gateway_node and destinationNodeInfo.gateway_node)):
                pass

            elif sourceNodeInfo.node_id == destinationNodeInfo.node_id:
                result[self.__ERROR] = "Source and destination node must be different node."

            elif ((sourceNodeInfo.gateway_node or destinationNodeInfo.gateway_node)  != connection.gateway_connection):
                result[self.__ERROR] = " 'gateway_connection' mismatch between nodes and connection data"

            elif ((sourceNodeInfo.community_id != destinationNodeInfo.community_id) and
                    ((not sourceNodeInfo.social_community) or (not destinationNodeInfo.social_community))):
                result[self.__ERROR] = 'cannot distribute across non social communities'

            elif ((sourceNodeInfo.network_id != destinationNodeInfo.network_id) and
                    ((not sourceNodeInfo.gateway_node)or(not destinationNodeInfo.gateway_node))):
                result[self.__ERROR] = 'cannot distribute across networks (or communities) unless gateway'

            elif ((sourceNodeInfo.gateway_node and destinationNodeInfo.gateway_node)
                    and (sourceNodeInfo.network_id == destinationNodeInfo.network_id)):
                result[self.__ERROR]  = 'gateways must only distribute across different networks'

            elif (sourceNodeInfo.gateway_node and not destinationNodeInfo.gateway_node):
                result[self.__ERROR]  = 'gateways can only distribute to gateways'
        except urllib2.URLError as ex:
            log.exception(ex)
            result[self.__ERROR] = "Cannot reach destination node.  "+str(ex.reason)
        except Exception as ex:
            log.exception(ex)
            result[self.__ERROR] = "Internal error. Cannot process destination node info"

        if result.has_key(self.__ERROR):
            result[self.__OK] = False

        return result

    def _getDistributeDestinations(self):
        """"Method to test the connections and returns a list of destionation node
             if the connections are valid"""
        gatewayConnectionList = []
        connectionsStatusInfo = {self.__OK:True, 'connections':[]}

        for connection in sourceLRNode.connections:
            # Make sure that the connection is active
            connectionsStatusInfo['connections'].append(self._canDistributeTo(connection, sourceLRNode.distributeInfo))

            if (connectionsStatusInfo['connections'][-1][self.__OK] and
                sourceLRNode.distributeInfo['gateway_node'] and
                connectionsStatusInfo['connections'][-1]['destinationNodeInfo'].gateway_node and
                connection.gateway_connection):
                gatewayConnectionList.append(connection)
              # Only one gateway  connection is allowed, faulty network description
            if len(gatewayConnectionList) > 1:
                log.info("***Abort distribution. More than one gateway node connection")
                connectionsStatusInfo[self.__ERROR] ="only one active gateway connection is allowed, faulty network description"
                break

        if len (sourceLRNode.connections) == 0:
            connectionsStatusInfo[self.__ERROR] ="No connection present for distribution"

        if connectionsStatusInfo.has_key(self.__ERROR) :
            connectionsStatusInfo[self.__OK] = False

        return connectionsStatusInfo

    def create(self):
        """POST / distribute start distribution"""

        distributeResults = Queue.Queue()

        def doDistribution(connectionInfo, server, sourceUrl):
            # We want to always use the replication filter function to replicate
            # only distributable doc and filter out any other type of documents.
            # However we don't have any query arguments until we test if there is any filter.
            replicationOptions={'filter':ResourceDataModel.REPLICATION_FILTER,
                                             'source':sourceUrl,
                                             'connection_id':  connectionInfo['connection_id'],
                                             'query_params': None}
            # If the destination node is using an filter and is not custom use it
            # as the query params for the filter function
            if ((connectionInfo['destinationNodeInfo'].filter_description is not None ) and
                 (connectionInfo['destinationNodeInfo'].filter_description.get('custom_filter') == False)):
                     replicationOptions['query_params'] =connectionInfo['destinationNodeInfo'].filter_description

            #if distinationNode['distribute service'] .service_auth["service_authz"] is not  None:
                #log.info("Destination node '{}' require authentication".format(destinationUrl))
                #Try to get the user name and password the url
            #destinationUrl = connectionInfo['destinationNodeInfo'].resource_data_url
            destinationUrl = connectionInfo['destinationNodeInfo'].incoming_url

            credential = sourceLRNode.getDistributeCredentialFor(destinationUrl)
            if credential is not None:
                parsedUrl = urlparse.urlparse(destinationUrl)
                destinationUrl = destinationUrl.replace(parsedUrl.netloc, "{0}:{1}@{2}".format(
                                                credential['username'], credential['password'], parsedUrl.netloc))

            if replicationOptions['query_params'] is  None:
                del replicationOptions['query_params']

            replicationOptions['target'] = destinationUrl

            authz_header = h.getBasicAuthHeaderFromURL(appConfig['couchdb.url.dbadmin']);
            authz_header.update( { 'Content-Type': 'application/json'})
            request = urllib2.Request(urlparse.urljoin(appConfig['couchdb.url'], '_replicator'),
                                    headers=authz_header,
                                    data = json.dumps(replicationOptions))

            log.info("\n\nReplication started\nSource:{0}\nDestionation:{1}\nArgs:{2}".format(
                            sourceUrl, destinationUrl, pprint.pformat(replicationOptions)))

            results = json.load(urllib2.urlopen(request))
            connectionInfo['replication_results'] = results
            distributeResults.put(connectionInfo)
            log.debug("Replication results: " + pprint.pformat(results))

        log.info("Distribute.......\n")
        ##Check if the distribte service is available on the node.
        #if(sourceLRNode.isServiceAvailable(NodeServiceModel.DISTRIBUTE) == False):
            #log.info("Distribute not available on node ")
            #return
        if((sourceLRNode.connections is None) or
            (len(sourceLRNode.connections) ==0)):
            log.info("No connection present for distribution")
            return json.dumps({self.__ERROR:''})
        log.info("Connections: \n{0}\n"+pprint.pformat([c.specData for c in sourceLRNode.connections]))

        connectionsStatusInfo = self._getDistributeDestinations()
        log.debug("\nSource Node Info:\n{0}".format(pprint.pformat(sourceLRNode.distributeInfo)))
        log.debug("\n\n Distribute connections:\n{0}\n\n".format(pprint.pformat(connectionsStatusInfo)))

        for connectionStatus in  connectionsStatusInfo['connections']:
            if connectionsStatusInfo.has_key(self.__ERROR) or connectionStatus.has_key(self.__ERROR) == True:
                distributeResults.put(connectionStatus)
            else:
                replicationArgs = (connectionStatus, defaultCouchServer, self.resource_data )
                    # Use a thread to do the actual replication.
                replicationThread = threading.Thread(target=doDistribution, args=replicationArgs)
                replicationThread.start()
                replicationThread.join()
            log.debug("\n\n\n---------------------distribute threads end--------------------\n\n\n")

        log.debug("\n\n\n----------Queue results Completed size: {0}--------------\n\n\n".format(distributeResults.qsize()))
        connectionsStatusInfo['connections'] = []

        while distributeResults.empty() == False:
            connectionsStatusInfo['connections'].append(distributeResults.get())
        log.debug("\n\n======== DISTRIBUTE RESULTS ============\n\n")
        log.debug(pprint.pformat(connectionsStatusInfo))
        return json.dumps(connectionsStatusInfo, indent=4)


"""
This file initialises, allows the modification of parameters and runs the evolutionary algorithm
"""
import numpy as np
import matplotlib.pyplot as plt
import random
from celluloid import Camera
from matplotlib import colors
from scipy.stats import sem

from ea.encoding import real_number_encoding
from ea.selection import tournament_selection
from ea.crossover_mutation import choose_pair, one_point_crossover, uniform_crossover, arithmetic_crossover, mutation
from ea.reproduction import generational_rollover
from ea.evaluation import get_xy_phenotype, cost_rosenbrock, cost_rastrigin
from ea.robot_evaluation import evaluate_genotype

import time

output_directory = "animations/ea_benchmark/diyon/"
plot_directory = "plots/"
phenotype_history = []
population_dictionary = {}

population_size = 30

generations = 100

offsprings_per_generation = 25

# Variables for the encoding strategy
genotype_length = 210
genotype_min_range = -10
genotype_max_range = 10
whole_numbers = False

# Variables for the selection strategy
tournament_k = 6

# Variables for the crossover/mutation strategies
mutation_rate = 5

"""
IMPORTANT:

Any time you change one of these strategies/functions, be sure to check if the arguments you pass are
actually those that the strategy/functions require

"""
encoding_strategy = real_number_encoding
phenotype_computer = get_xy_phenotype
cost_function = cost_rosenbrock
selection_strategy = tournament_selection
reproduction_strategy = generational_rollover

def initialise(room):
    """
    This function populates the population dictionary by using the specified encoding strategy.
    Each individual has a key (integer), a genotype array and a fitness.

    The dictionary will be of the form:

    key : value
    int : [[list], int]
    <ID> : [genotype], fitness

    :return: none
    """

    # generation = []
    for individual in range(population_size):

        genotype = encoding_strategy(genotype_length, genotype_min_range, genotype_max_range, integers=whole_numbers)

        population_dictionary[individual] = [genotype, evaluate_genotype(genotype, individual, room)]

def initialise_rooms(rooms):
    """
    This function populates the population dictionary by using the specified encoding strategy.
    Each individual has a key (integer), a genotype array and a fitness.

    The dictionary will be of the form:

    key : value
    int : [[list], int]
    <ID> : [genotype], fitness

    :return: none
    """

    # generation = []
    for individual in range(population_size):

        genotype = encoding_strategy(genotype_length, genotype_min_range, genotype_max_range, integers=whole_numbers)

        fitness = 0
        for room in rooms:
            print("Room: " + str(room[2]))
            fitness += evaluate_genotype(genotype, individual, room)

        population_dictionary[individual] = [genotype, fitness / len(rooms)]

def run_algorithm(room, file):
    """
    Initialises the data, and then runs evaluation, selection, reproduction and mutations for the specified amount
    of generations

    :param room: Room list
    :param file: An open file to write to
    :return: a max fitness array and an average fitness array, both of length equal to number of generations + 1
            (to include "generation 0" after initialisation but before any reproduction has taken place), containing
            max fitness and average fitness of the population at every generation
    """

    max_fitness_history = []
    avg_fitness_history = []

    start = time.time()
    print(f"Running for {generations} generations with {population_size} genotypes\n"
          f"Initialising all genotypes")

    file.write(f"This file contains a run for {generations} generations with {population_size} genotypes\n"
               f"offspring per generation: {offsprings_per_generation}\n"
               f"k: {tournament_k}\n"
               f"mutation rate: {mutation_rate}\n\n")

    initialise(room)

    print(f"Done initialising in {time.time() - start} seconds")

    file.write("Initialised genotypes:\n")
    write_dictionary(population_dictionary, file)
    file.write("############################\n")

    start = time.time()

    # Get the max and the avg fitness at the start
    features = [individual[1] for individual in population_dictionary.values()]
    max_fitness_history.append(max(features))
    avg_fitness_history.append(sum(features) / len(features))

    for i in range(generations):

        print(f"Evaluating generation: {i+1}")
        file.write(f"GENERATION: {i+1}\n")

        # SELECTION
        # For the number of desired offspring per generation, choose the best parent
        # and momentarily copy them over in a dedicated offspring dictionary
        offspring_dictionary = {}

        for j in range(offsprings_per_generation):

            best_parent = selection_strategy(population_dictionary, k=tournament_k)

            # offspring is initialised (genotype is set as empty, fitness is set to 0)
            offspring_dictionary[j] = [[], 0]

            # genotype of offspring is copied over from parent
            for gene in population_dictionary[best_parent][0]:
                offspring_dictionary[j][0].append(gene)

        # CROSS OVER
        crossover_events = int(random.uniform(0.1, 0.4)*offsprings_per_generation)

        for _ in range(crossover_events):

            offspring_pair = choose_pair(offspring_dictionary)
            rng = random.randint(0, 99)

            if rng < 33:
                one_point_crossover(offspring_dictionary, offspring_pair)
            elif rng < 66:
                uniform_crossover(offspring_dictionary, offspring_pair)
            else:
                arithmetic_crossover(offspring_dictionary, offspring_pair, integers=False)

        # MUTATION
        for target_offspring in offspring_dictionary:
            mutation(offspring_dictionary, target_offspring, mutation_rate, integers=False)

        # REPRODUCTION
        # before offsprings are added into the population, but after they have undergone crossover and mutation events
        # (so that their genotype is "finalised"), their fitness and phenotype is computed
        for offspring in offspring_dictionary:

            genotype = offspring_dictionary[offspring][0]

            offspring_dictionary[offspring][1] = evaluate_genotype(genotype, offspring, room)

        reproduction_strategy(population_dictionary, offspring_dictionary)

        # max fitness and average fitness of the population are extracted and appended to the max fitness and
        # average fitness arrays
        features = [individual[1] for individual in population_dictionary.values()]

        max_fitness = max(features)
        avg_fitness = sum(features) / len(features)
        max_fitness_history.append(max_fitness)
        avg_fitness_history.append(avg_fitness)

        end = time.time()
        print(f"Finished generation {i+1} in {end - start} seconds.\n "
              f"Max fitness: {max_fitness}\n "
              f"Avg. fitness: {avg_fitness}")

        start = time.time()

        write_dictionary(population_dictionary, file)
        file.write("############################\n")

    best_genotype = max(population_dictionary.values(), key=lambda x: x[1])
    print(f"The best genotype is:\n"
          f"{best_genotype}")

    file.write(f"The best genotype is: {best_genotype}")

    return max_fitness_history, avg_fitness_history

def write_dictionary(dictionary, file):

    for genotype_id in dictionary:

        line = f"ID: {str(genotype_id)}\n" \
               f"Genotype: {str(dictionary[genotype_id][0])}\n" \
               f"Fitness: {str(dictionary[genotype_id][1])}\n\n"

        file.write(line)
        file.write("------------------------------------\n")

def animate_evolution():
    """
    A function to animate the plotting of graphs

    :return: None
    """
    fig, ax = plt.subplots()
    camera = Camera(fig)

    n = 100
    X = np.linspace(genotype_min_range, genotype_max_range, n)
    Y = np.linspace(genotype_min_range, genotype_max_range, n)
    X, Y = np.meshgrid(X, Y)

    Z = cost_function(X, Y)

    for i in range(len(phenotype_history)):

        generation = phenotype_history[i]

        # Holds all phenotypes of all genotypes
        x, y = [], []
        for phenotype in generation:
            # Get the phenotype
            phenotype_x, phenotype_y = phenotype
            x.append(phenotype_x)
            y.append(phenotype_y)

        # Plot all phenotypes
        ax.plot(x, y, 'bo', c="#000000")
        ax.set_xlim([genotype_min_range, genotype_max_range])
        ax.set_ylim([genotype_min_range, genotype_max_range])
        ax.text(0.4, 1.01, f"Generation {i+1}", transform=ax.transAxes)
        camera.snap()

    im = plt.contourf(X, Y, Z, levels=50, cmap="jet") if cost_function == cost_rastrigin else \
         plt.pcolor(X, Y, Z, norm=colors.LogNorm(vmin=Z.min(), vmax=Z.max()), cmap='jet', shading='auto')

    fig.colorbar(im, orientation='vertical', extend='max')
    ax.set_xlabel("x")
    ax.set_ylabel("y")

    # Animates
    animation = camera.animate(interval=200, repeat=False, repeat_delay=500)
    # animation.save(output_directory +
    #                f"{cost_function.__name__}_pop{population_size}_offs{offsprings_per_generation}_gens{generations}_"
    #                f"tk{tournament_k}_mr{mutation_rate}.gif")
    plt.show()

def test_helper(results, value, tests, room):
    """
    A helper function to make code more concise
    Initialises average max fitness and average average fitness arrays, runs the evolutionary algorithm multiple times
    and fills said arrays across iterations and appends said arrays into the dictionary of results

    :param results: The dictionary of results
    :param value: The value for the parameter being investigated
    :param tests: The number of times the evolutionary algorithm should be run
    :return: None
    """

    avg_max_fitnesses = [0 for _ in range(generations+1)]
    avg_avg_fitnesses = [0 for _ in range(generations+1)]
    results[value] = []

    for itter in range(tests):
        max_fitness_history, avg_fitness_history = run_algorithm(room)
        for i in range(len(max_fitness_history)):
            avg_max_fitnesses[i] += max_fitness_history[i]
            avg_avg_fitnesses[i] += avg_fitness_history[i]

    # the dictionary of results is modified in place
    results[value].append(avg_max_fitnesses)
    results[value].append(avg_avg_fitnesses)

def testing_routine(parameter_set, parameter, room, tests=100, short=False):
    """
    The function that handles running multiple experiments with different values for a given parameter
    to be investigated

    :param parameter_set: A list of values for the parameter to be investigated
    :param parameter: The parameter of the evolutionary algorithm under investigation
    :param tests: The number of times the evolutionary algorithm should be run to test a single value for the parameter
    :param short: A boolean to indicate whether the user wants to run a simple and quick testing routine

    :return: A dictionary of results, the keys are values for a parameter being investigated in a testing routine
            and values are lists of two arrays, the first containing average max fitness at every generation
            of the genetic algorithm and the second containing average average fitness at every generation
    """

    if short:
        global generations
        generations = 10
        tests = 10

    results = {}

    if parameter == "tournament_k":

        for value in parameter_set:

            global tournament_k
            tournament_k = value
            test_helper(results, value, tests, room)
            print("Finished with value " + str(value))

    elif parameter == "offsprings_per_generation":

        for value in parameter_set:

            global offsprings_per_generation
            offsprings_per_generation = value
            test_helper(results, value, tests, room)
            print("Finished with value " + str(value))

    elif parameter == "mutation_rate":

        for value in parameter_set:

            global mutation_rate
            mutation_rate = value
            test_helper(results, value, tests, room)
            print("Finished with value "+str(value))

    return results

def run_algorithm_with_parameters(rooms, gens, k, num_offspring, mr, file):
    """
    This function allows the value of parameters to be changed by passing in values

    Initialises the data, and then runs evaluation, selection, reproduction and mutations for the specified amount
    of generations

    :param file: An open file object
    :param mr: Integer for mutation rate
    :param num_offspring: Integer for number of offspring per generation
    :param k: Integer for tournament k
    :param gens: Integer for number of generations
    :param room: Room list

    :return: a max fitness array and an average fitness array, both of length equal to number of generations + 1
            (to include "generation 0" after initialisation but before any reproduction has taken place), containing
            max fitness and average fitness of the population at every generation
    """

    max_fitness_history = []
    avg_fitness_history = []

    start = time.time()
    print(f"Running for {gens} generations with {population_size} genotypes\n"
          f"Initialising all genotypes")

    file.write(f"This file contains a run for {gens} generations with {population_size} genotypes\n"
               f"offspring per generation: {num_offspring}\n"
               f"k: {k}\n"
               f"mutation rate: {mr}\n\n")

    initialise_rooms(rooms)

    print(f"Done initialising in {time.time() - start} seconds")

    file.write("Initialised genotypes:\n")
    write_dictionary(population_dictionary, file)
    file.write("############################\n")

    start = time.time()

    # Get the max and the avg fitness at the start
    features = [individual[1] for individual in population_dictionary.values()]
    max_fitness_history.append(max(features))
    avg_fitness_history.append(sum(features) / len(features))

    for i in range(gens):

        print(f"Evaluating generation: {i + 1}")
        file.write(f"GENERATION: {i + 1}\n")

        # SELECTION
        # For the number of desired offspring per generation, choose the best parent
        # and momentarily copy them over in a dedicated offspring dictionary
        offspring_dictionary = {}

        for j in range(num_offspring):

            best_parent = selection_strategy(population_dictionary, k=k)

            # offspring is initialised (genotype is set as empty, fitness is set to 0)
            offspring_dictionary[j] = [[], 0]

            # genotype of offspring is copied over from parent
            for gene in population_dictionary[best_parent][0]:
                offspring_dictionary[j][0].append(gene)

        # CROSS OVER
        crossover_events = int(random.uniform(0.1, 0.4)*num_offspring)

        for _ in range(crossover_events):

            offspring_pair = choose_pair(offspring_dictionary)
            rng = random.randint(0, 99)

            if rng < 33:
                one_point_crossover(offspring_dictionary, offspring_pair)
            elif rng < 66:
                uniform_crossover(offspring_dictionary, offspring_pair)
            else:
                arithmetic_crossover(offspring_dictionary, offspring_pair, integers=False)

        # MUTATION
        for target_offspring in offspring_dictionary:
            mutation(offspring_dictionary, target_offspring, mr, integers=False)

        # REPRODUCTION
        # before offsprings are added into the population, but after they have undergone crossover and mutation events
        # (so that their genotype is "finalised"), their fitness and phenotype is computed
        for offspring in offspring_dictionary:

            genotype = offspring_dictionary[offspring][0]
            fitness = 0
            for room in rooms:
                fitness += evaluate_genotype(genotype, offspring, room)
            offspring_dictionary[offspring][1] = fitness / len(rooms)

        reproduction_strategy(population_dictionary, offspring_dictionary)

        # max fitness and average fitness of the population are extracted and appended to the max fitness and
        # average fitness arrays
        features = [individual[1] for individual in population_dictionary.values()]

        max_fitness = max(features)
        avg_fitness = sum(features) / len(features)
        max_fitness_history.append(max_fitness)
        avg_fitness_history.append(avg_fitness)

        end = time.time()
        print(f"Finished generation {i+1} in {end - start} seconds.\n "
              f"Max fitness: {max_fitness}\n "
              f"Avg. fitness: {avg_fitness}")

        write_dictionary(population_dictionary, file)
        file.write("############################\n")

        start = time.time()

    best_genotype = max(population_dictionary.values(), key=lambda x: x[1])
    print(f"The best genotype is:\n"
          f"{best_genotype}")

    file.write(f"The best genotype is: {best_genotype}")
    file.write(f"\n")
    file.write(f"{max_fitness_history}\n")
    file.write(f"{avg_fitness_history}\n")

    return max_fitness_history, avg_fitness_history

def test_algorithm_with_parameters(room, file, plot_name, gens=generations, k=tournament_k, num_offspring=offsprings_per_generation, mr=mutation_rate, tests=100):
    """
    Plots the average max fitness and average fitness from all run tests over a number of generations.

    Plots contain error bars

    :param room: Room list
    :param gens: Integer for number of generations
    :param k: Integer for tournament k
    :param num_offspring: Integer for number of offspring per generation
    :param mr: Integer for mutation rate
    :param tests: Integer for number of tests
    :return: None - saves figure
    """
    # Holds lists of max and avg fitness per run test
    max_fitness = []
    avg_fitness = []

    print(f"Running the algorithm for {tests} tests, each containing {gens} generations.\n"
          f"Parameters:\n"
          f"k = {tournament_k}\n"
          f"offspring per gen = {num_offspring}\n"
          f"mutation rate = {mr}")

    for i in range(tests):

        print(f"TEST {i+1}:")
        file.write(f"TEST {i+1}\n")

        max_fitness_history, avg_fitness_history = run_algorithm_with_parameters(room, gens, k, num_offspring, mr, file)

        max_fitness.append(max_fitness_history)
        avg_fitness.append(avg_fitness_history)

        file.write("##############################################\n")

    # Holds the average max fitness and average average fitness over all tests, per generation
    avg_max_fitness = []
    avg_avg_fitness = []

    # Holds the errors
    max_fitness_error = []
    avg_fitness_error = []

    # For each generation, go over all run tests and compute the average + errors
    for j in range(gens):

        total_max_fitness = []
        total_avg_fitness = []

        for k in range(tests):

            total_max_fitness.append(max_fitness[k][j])
            total_avg_fitness.append(avg_fitness[k][j])

        avg_max_fitness.append(sum(total_max_fitness) / tests)
        avg_avg_fitness.append(sum(total_avg_fitness) / tests)

        max_fitness_error.append(sem(total_max_fitness))
        avg_fitness_error.append(sem(total_avg_fitness))

    generation_list = [gen for gen in range(gens)]

    # Plot
    plt.errorbar(generation_list, avg_max_fitness, yerr=max_fitness_error, c='b', ls='-', fmt='.', label="Max Fitness")
    plt.errorbar(generation_list, avg_avg_fitness, yerr=avg_fitness_error, c='r', ls='-', fmt='.', label="Average Fitness")

    plt.xlabel("Generations")
    plt.ylabel("Fitness")

    plt.xlim(0, gens)
    plt.legend(loc='lower right')

    # Save
    plt.savefig(plot_directory+plot_name)


import cmath
import numpy as np
import math
import pylab
import matplotlib.pyplot as plt

alpha=5.0/4.0
beta=-(25.0/32.0)
H= []
fase= []
w = np.arange(0,2*math.pi,0.01)
for i in range(0,len(w)):
    z= cmath.exp(complex(0,w[i]))
    H.append( 0.5* (1.0/ ((z**2)-alpha*z-beta) ) )
    fase.append( math.degrees(cmath.phase(H[i])))

H_array = np.array(H)
mag_db =20*np.log10(np.abs(H_array))
plt.subplot(2,1,1)
plt.title("Respuesta en frecuencia")
plt.plot(w,mag_db)
plt.xlabel("w(rad/seg")
plt.ylabel("|H(w)|(dB)")
plt.grid(True)

plt.subplot(2,1,2)
plt.plot(w,fase)
plt.xlabel("w(rad/seg")
plt.ylabel("fase de H(w)")
plt.grid(True)

plt.show()


#%%
import requests
from credentials import credentials
from apps import apps

class ironsource:

    def ironsourceAuthentication(self):
        auth_url = 'https://platform.ironsrc.com/partners/publisher/auth'

        print(auth_url)
        auth_headers = {
            'secretkey': credentials['ironsource_secretKey_jc'],
            'refreshToken': credentials['ironsource_refreshToken_jc']
        }
        payload = ''
        authToken = requests.request('GET', auth_url, headers= auth_headers, data=payload)
        print('Authenicated Complete')
        return authToken.text.strip('\"')

    def getMediationData(self , startDate, endDate, appkey, breakdown):

        url = 'https://platform.ironsrc.com/partners/publisher/mediation/applications/v6/stats?startDate='+ startDate +'&endDate='+endDate+'&appKey='+appkey+'&breakdown=' + breakdown

        print(self.ironsourceAuthentication)
        authorizedToken = self.ironsourceAuthentication()
        headers = {
            'Authorization': 'Bearer '+ authorizedToken
        }
        payload = ''
        data = requests.request('GET', url, headers = headers, data = payload)
        print('Data Pulled Complete')

        return data.json()


from rpg_api.dungeon.character import Character
import random
from config import CONFIG

class Mob(Character):
    def __init__(self, **kwargs):
        super(Mob, self).__init__(**kwargs)

    @staticmethod
    def create_mob(mobs, reporter):
        mob = random.choice(CONFIG['mobs'])
        mob_name = mob["name"]
        mobs[mob_name] += 1
        atk = random.randrange(*mob["atk"])
        deff = random.randrange(*mob["def"])
        health = random.randrange(*mob["health"])
        return Mob(health=health, atk=atk, deff=deff, potions=[], rep=reporter, name='{}{}'.format(mob_name.capitalize(), mobs[mob_name]))


from urllib import response
import requests
import pytest

class TestSchedules:

    user_id = 2
    payload = {
        'title': 'that\'s a test'
    }

    payload_update = {
        'title': 'that\s was a test'
    }

    def test_add_non_default_schedule(self, url):
        response = requests.post(
            f'{url}/user/{self.user_id}/schedules?default=false', json=self.payload)
        assert response.status_code == 200
        assert response.json()['status'] == 'successful request'

    def test_add_default_schedule(self, url):
        response = requests.post(
            f'{url}/user/{self.user_id}/schedules?default=true', json=self.payload)
        assert response.status_code == 200
        assert response.json()['status'] == 'successful request'

    def test_get_schedules(self, url):
        response = requests.get(f'{url}/user/{self.user_id}/schedules')
        data = response.json()['schedules']
        assert response.status_code == 200
        assert len(data) > 0
        assert data[0]['title']
        assert data[0]['free_slots']
        pytest.test_slot_id = data[-1]['_id']

    def test_update_schdule(self, url):
        response = requests.put(
            f'{url}/user/{self.user_id}/schedule/{pytest.test_slot_id}/update', json=self.payload_update)
        assert response.status_code == 200

    def test_delete_schdule(self, url):
        response = requests.delete(
            f'{url}/schedule/{pytest.test_slot_id}/delete', json={'schedule_id': pytest.test_slot_id})
        assert response.status_code == 200


import random
import pprint as pp
import logging as log
import ctpy

log.basicConfig(level=log.DEBUG, format='%(asctime)s %(levelname)s: %(message)s')

def build_even_dimension(simconfig, num_modes):
    mode_boundaries = []
    mode_width = float(simconfig.MAXALLELES) / float(num_modes)
    lower_val = 0.0
    upper_val = 0.0
    for mode in range(0, num_modes):
        upper_val += mode_width
        mode_boundaries.append(dict(lower=lower_val, upper=upper_val))
        lower_val = upper_val
    log.debug("boundaries: %s", mode_boundaries)
    return mode_boundaries

def build_random_dimension(simconfig, num_modes):
    iboundaries = []
    mode_boundaries = []
    num_internal_boundaries = num_modes - 1
    for i in range(0, num_internal_boundaries):
        random_variate = random.random()
        iboundaries.append(random_variate)

    # add the final upper boundary
    iboundaries.append(1.0)

    lower_val = 0.0
    upper_val = 0.0
    iboundaries.sort()

    for mode in range(0, num_modes):
        lower_val = lower_val * simconfig.MAXALLELES
        upper_val = iboundaries[mode] * simconfig.MAXALLELES
        mode_boundaries.append(dict(lower=lower_val, upper=upper_val))
        lower_val = iboundaries[mode]

    log.debug("boundaries: %s", mode_boundaries)
    return mode_boundaries

# # TODO:  needs to deal with cases where maxalleles % num_modes leaves a remainder...
#
#
# def build_even_partitions_all_dimensions(num_modes, sim_param):
#     """
#
#
#     :param num_modes:
#     :param sim_param:
#     :return:
#     """
#     dimensions = {}
#
#     # to start, we partition into quarters for each locus (dimension)
#     mode_width = sim_param["maxalleles"] / num_modes
#
#     for dimension in range(0, sim_param["numloci"]):
#         mode_boundaries_dict = {}
#         lower_val = 0.0
#         upper_val = 0.0
#         for mode in range(0,num_modes):
#             upper_val += mode_width
#             mode_boundaries_dict[mode] = dict(lower=lower_val, upper=upper_val)
#             lower_val = upper_val
#             dimensions[dimension] = mode_boundaries_dict
#     return dimensions
#
#
# def build_random_partitions_all_dimensions(num_modes, sim_param):
#     """
#     For k desired modes, generate random mode boundaries within maxalleles.
#     Algorithm generates k-1 "internal" boundaries on the unit interval [0,1]
#     and then scales maxalleles by the unit interval partitions.  Upper
#     and lower internal boundaries are equivalent, since they will be
#     interpreted with open/closed interval semantics.
#
#
#     :param num_modes:
#     :param sim_param:
#     :return: dict of dimension-specific dicts, within each of which a mode maps to a dict of upper and lower boundaries
#     """
#     dimensions = {}
#     num_internal_boundaries = num_modes - 1
#
#     for dimension in range(0, sim_param["numloci"]):
#         tempboundary = list()
#         mode_boundaries_dict = {}
#         maxalleles = sim_param["maxalleles"]
#
#         for i in range(0, num_internal_boundaries):
#             random_variate = random.random()
#             tempboundary.append(random_variate)
#
#         # add the final upper boundary
#         tempboundary.append(1.0)
#
#         lower_val = 0.0
#         upper_val = 0.0
#         tempboundary.sort()
#
#         for mode in range(0, num_modes):
#             lower_val = int(lower_val * maxalleles)
#             upper_val = int(tempboundary[mode] * maxalleles)
#             mode_boundaries_dict[mode] = dict(lower=lower_val, upper=upper_val)
#             lower_val = tempboundary[mode]
#
#         dimensions[dimension] = mode_boundaries_dict
#         # TODO:  missing logic for scaling to maxalleles, need to debug this first...
#     return dimensions
#
#
# if __name__ == "__main__":
#     sim_param = {}
#     sim_param["numloci"] = 3
#     sim_param["maxalleles"] = 100000000
#
#
#     print "Testing random partitions for 3 dimensions, 4 modes"
#     result_dict = build_random_partitions_all_dimensions(4, sim_param)
#     pp.pprint(result_dict)
#
#     print "Testing even partitions for 3 dimensions, 4 modes"
#     result_dict = build_even_partitions_all_dimensions(4, sim_param)
#     pp.pprint(result_dict)


import asyncio

import nest_asyncio
import slack
#For jupyter
nest_asyncio.apply()
import logging

class SlackMessenger:

    def __init__(self, api_token):
        self._api_token = api_token
        self._client = slack.WebClient(token=self._api_token)

    def message_channel(self, message, channel):
        response = self._client.chat_postMessage(
            channel=channel,
            text=message)
        logging.info(f' Result of response successful: {response["ok"]}')


#!/usr/bin/env python3

import logging
import numpy as np
import onnx
import os
import tempfile
import torch
import unittest

from fairseq import models
from fbtranslate import rnn  # noqa
from fbtranslate.ensemble_export import (
    CombinedDecoderEnsemble,
    CombinedEncoderEnsemble,
    DecoderBatchedStepEnsemble,
    DecoderStepEnsemble,
    EncoderEnsemble,
)
from fbtranslate.test import utils as test_utils

from caffe2.python.onnx import backend as caffe2_backend

logger = logging.getLogger(__name__)

class TestONNX(unittest.TestCase):

    class BeamSearch(torch.jit.ScriptModule):

        def __init__(self, test_args, src_dict, tgt_dict):
            super().__init__(False)
            # Create models
            self.num_models = 3
            self.model_list = []

            self.incremental_states = [[{} for _ in range(len(self.model_list))]]
            for _ in range(self.num_models):
                self.model_list.append(models.build_model(
                    test_args, src_dict, tgt_dict))
            self.encoder_ensemble = CombinedEncoderEnsemble(self.model_list)
            self.decoder_ensemble = CombinedDecoderEnsemble(self.model_list)

            def encoder(src_tokens, src_lengths):
                self.incremental_states[0] = [{} for _ in range(len(self.model_list))]
                enc_outs = self.encoder_ensemble(
                    src_tokens.repeat(6, 1), src_lengths.repeat(6))
                return enc_outs

            def decoder(input_tokens, encoder_outs, final_hidden, final_cell,
                        src_lengths, src_tokens):
                dec_outs = self.decoder_ensemble(
                    input_tokens, encoder_outs, final_hidden, final_cell,
                    src_lengths, src_tokens, self.incremental_states[0])
                return dec_outs

            def reorder_states(new_order):
                for i, model in enumerate(self.decoder_ensemble.models):
                    model.decoder.reorder_incremental_state(
                        self.incremental_states[0][i], new_order.squeeze(dim=0))
                return torch.zeros(1, 2, 3)  # prevent DCE

            self.define('''
            def forward(
                self, src_tokens, src_lengths, scores_t, hypo_t, tokens_t, attention_t
            ):
                encoder_outs, final_hidden, final_cells, src_lengths, src_tokens\
                 = encoder(src_tokens, src_lengths)

                output_token_beam_list = tokens_t
                output_prev_index_beam_list = hypo_t
                output_score_beam_list = scores_t
                output_attention_weights_beam_list = attention_t
                dce = tokens_t

                timestep = 0

                while timestep < 20:
                    log_probs, attn = decoder(output_token_beam_list,
                        encoder_outs, final_hidden, final_cells, src_lengths,
                        src_tokens)

                    best_scores_per_hypo, best_tokens_per_hypo = topk(
                        log_probs, k=6, dim=-1, largest=1, sorted=1)

                    output_scores = best_scores_per_hypo + \
                        squeeze(scores_t, dim=0)

                    output_scores_flattened = view(output_scores, size=[-1])
                    output_scores_flattened_slice = output_scores_flattened
                    if timestep == 0:
                        output_scores_flattened_slice = slice(
                            output_scores_flattened, dim=0, end=6, start=0,
                            step=1)
                    else:
                        output_scores_flattened_slice = slice(
                            output_scores_flattened, dim=0, end=-1, start=0,
                            step=1)
                    output_scores_flattened_slice = view(
                        output_scores_flattened_slice, size=[1, -1])

                    scores_t, best_indices = topk(
                        output_scores_flattened_slice, k=6, dim=-1,
                        largest=1, sorted=1)

                    hypo_t_int64 = best_indices / 6

                    attention_t = index_select(
                        attention_t, squeeze(hypo_t_int64, dim=0), dim=0)
                    tokens_t_int64 = view(best_tokens_per_hypo, size=[-1])
                    tokens_t_int64 = index_select(
                        tokens_t_int64, squeeze(best_indices, dim=0), dim=0)
                    tokens_t_int64 = view(tokens_t_int64, size=[-1, 1])

                    output_token_beam_list = cat(
                        output_token_beam_list, tokens_t_int64, dim=1)
                    output_prev_index_beam_list = cat(
                        output_prev_index_beam_list, hypo_t_int64, dim=0)
                    output_score_beam_list = cat(
                        output_score_beam_list, scores_t, dim=0)
                    output_attention_weights_beam_list = cat(
                        output_attention_weights_beam_list, attention_t, dim=0)

                    timestep += 1

                    dce = reorder_states(hypo_t_int64)

                return output_token_beam_list, output_prev_index_beam_list, \
                    output_score_beam_list, output_attention_weights_beam_list,\
                    dce
            ''')

    def test_beam_search(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )

        # Create data
        samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
        sample = next(samples)
        # TODO batched beam search
        src_tokens = sample['net_input']['src_tokens'][0]
        src_tokens = torch.unsqueeze(src_tokens, dim=0)
        src_lengths = sample['net_input']['src_lengths'][0]
        src_lengths = torch.unsqueeze(src_lengths, dim=0)

        bs = TestONNX.BeamSearch(test_args, src_dict, tgt_dict)

        scores_t = torch.zeros(1, 6)
        hypo_t = torch.zeros(1, 6).type(torch.LongTensor)
        eos = bs.model_list[0].dst_dict.eos()
        tokens_t = (torch.ones(6, 1) * eos).type(torch.LongTensor)
        attention_t = torch.nn.functional.softmax(torch.randn(6, 20), dim=-1)

        bs.forward(src_tokens, src_lengths, scores_t, hypo_t, tokens_t, attention_t)

    def _test_ensemble_encoder_export(self, test_args):
        samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)

        num_models = 3
        model_list = []
        for _ in range(num_models):
            model_list.append(models.build_model(test_args, src_dict, tgt_dict))
        encoder_ensemble = EncoderEnsemble(model_list)

        tmp_dir = tempfile.mkdtemp()
        encoder_pb_path = os.path.join(tmp_dir, 'encoder.pb')
        encoder_ensemble.onnx_export(encoder_pb_path)

        # test equivalence
        # The discrepancy in types here is a temporary expedient.
        # PyTorch indexing requires int64 while support for tracing
        # pack_padded_sequence() requires int32.
        sample = next(samples)
        src_tokens = sample['net_input']['src_tokens'][0:1].t()
        src_lengths = sample['net_input']['src_lengths'][0:1].int()

        pytorch_encoder_outputs = encoder_ensemble(src_tokens, src_lengths)

        with open(encoder_pb_path, 'r+b') as f:
            onnx_model = onnx.load(f)
        onnx_encoder = caffe2_backend.prepare(onnx_model)

        caffe2_encoder_outputs = onnx_encoder.run(
            (
                src_tokens.numpy(),
                src_lengths.numpy(),
            ),
        )

        for i in range(len(pytorch_encoder_outputs)):
            caffe2_out_value = caffe2_encoder_outputs[i]
            pytorch_out_value = pytorch_encoder_outputs[i].data.numpy()
            np.testing.assert_allclose(
                caffe2_out_value,
                pytorch_out_value,
                rtol=1e-4,
                atol=1e-6,
            )

        encoder_ensemble.save_to_db(
            os.path.join(tmp_dir, 'encoder.predictor_export'),
        )

    def test_ensemble_encoder_export_default(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        self._test_ensemble_encoder_export(test_args)

    def test_ensemble_encoder_export_vocab_reduction(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        lexical_dictionaries = test_utils.create_lexical_dictionaries()
        test_args.vocab_reduction_params = {
            'lexical_dictionaries': lexical_dictionaries,
            'num_top_words': 5,
            'max_translation_candidates_per_word': 1,
        }

        self._test_ensemble_encoder_export(test_args)

    def _test_ensemble_encoder_object_export(self, encoder_ensemble):
        tmp_dir = tempfile.mkdtemp()
        encoder_pb_path = os.path.join(tmp_dir, 'encoder.pb')
        encoder_ensemble.onnx_export(encoder_pb_path)

        src_dict = encoder_ensemble.models[0].src_dict
        token_list = [src_dict.unk()] * 4 + [src_dict.eos()]
        src_tokens = torch.LongTensor(
            np.array(token_list, dtype='int64').reshape(-1, 1),
        )
        src_lengths = torch.IntTensor(
            np.array([len(token_list)], dtype='int32'),
        )

        pytorch_encoder_outputs = encoder_ensemble(src_tokens, src_lengths)

        with open(encoder_pb_path, 'r+b') as f:
            onnx_model = onnx.load(f)
        onnx_encoder = caffe2_backend.prepare(onnx_model)

        srclen = src_tokens.size(1)
        beam_size = 1

        src_tokens = src_tokens.repeat(1, beam_size).view(-1, srclen).numpy()
        src_lengths = src_lengths.repeat(beam_size).numpy()

        caffe2_encoder_outputs = onnx_encoder.run(
            (
                src_tokens,
                src_lengths,
            ),
        )

        for i in range(len(pytorch_encoder_outputs)):
            caffe2_out_value = caffe2_encoder_outputs[i]
            pytorch_out_value = pytorch_encoder_outputs[i].data.numpy()
            np.testing.assert_allclose(
                caffe2_out_value,
                pytorch_out_value,
                rtol=1e-4,
                atol=1e-6,
            )

        encoder_ensemble.save_to_db(
            os.path.join(tmp_dir, 'encoder.predictor_export'),
        )

    def _test_full_ensemble_export(self, test_args):
        samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)

        num_models = 3
        model_list = []
        for _ in range(num_models):
            model_list.append(models.build_model(test_args, src_dict, tgt_dict))
        encoder_ensemble = EncoderEnsemble(model_list)

        # test equivalence
        # The discrepancy in types here is a temporary expedient.
        # PyTorch indexing requires int64 while support for tracing
        # pack_padded_sequence() requires int32.
        sample = next(samples)
        src_tokens = sample['net_input']['src_tokens'][0:1].t()
        src_lengths = sample['net_input']['src_lengths'][0:1].int()

        pytorch_encoder_outputs = encoder_ensemble(src_tokens, src_lengths)

        decoder_step_ensemble = DecoderStepEnsemble(
            model_list,
            beam_size=5,
        )

        tmp_dir = tempfile.mkdtemp()
        decoder_step_pb_path = os.path.join(tmp_dir, 'decoder_step.pb')
        decoder_step_ensemble.onnx_export(
            decoder_step_pb_path,
            pytorch_encoder_outputs,
        )

        # single EOS
        input_token = torch.LongTensor(
            np.array([[model_list[0].dst_dict.eos()]]),
        )
        timestep = torch.LongTensor(np.array([[0]]))

        pytorch_decoder_outputs = decoder_step_ensemble(
            input_token,
            timestep,
            *pytorch_encoder_outputs
        )

        with open(decoder_step_pb_path, 'r+b') as f:
            onnx_model = onnx.load(f)
        onnx_decoder = caffe2_backend.prepare(onnx_model)

        decoder_inputs_numpy = [input_token.numpy(), timestep.numpy()]
        for tensor in pytorch_encoder_outputs:
            decoder_inputs_numpy.append(tensor.detach().numpy())

        caffe2_decoder_outputs = onnx_decoder.run(tuple(decoder_inputs_numpy))

        for i in range(len(pytorch_decoder_outputs)):
            caffe2_out_value = caffe2_decoder_outputs[i]
            pytorch_out_value = pytorch_decoder_outputs[i].data.numpy()
            np.testing.assert_allclose(
                caffe2_out_value,
                pytorch_out_value,
                rtol=1e-4,
                atol=1e-6,
            )

        decoder_step_ensemble.save_to_db(
            os.path.join(tmp_dir, 'decoder_step.predictor_export'),
            pytorch_encoder_outputs,
        )

    def test_full_ensemble_export_default(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        self._test_full_ensemble_export(test_args)

    def test_full_ensemble_export_vocab_reduction(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        lexical_dictionaries = test_utils.create_lexical_dictionaries()
        test_args.vocab_reduction_params = {
            'lexical_dictionaries': lexical_dictionaries,
            'num_top_words': 5,
            'max_translation_candidates_per_word': 1,
        }

        self._test_full_ensemble_export(test_args)

    def _test_batched_beam_decoder_step(self, test_args):
        beam_size = 5
        samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)

        num_models = 3
        model_list = []
        for _ in range(num_models):
            model_list.append(models.build_model(test_args, src_dict, tgt_dict))
        encoder_ensemble = EncoderEnsemble(model_list)

        # test equivalence
        # The discrepancy in types here is a temporary expedient.
        # PyTorch indexing requires int64 while support for tracing
        # pack_padded_sequence() requires int32.
        sample = next(samples)
        src_tokens = sample['net_input']['src_tokens'][0:1].t()
        src_lengths = sample['net_input']['src_lengths'][0:1].int()

        pytorch_encoder_outputs = encoder_ensemble(src_tokens, src_lengths)

        decoder_step_ensemble = DecoderBatchedStepEnsemble(
            model_list,
            beam_size=beam_size,
        )

        tmp_dir = tempfile.mkdtemp()
        decoder_step_pb_path = os.path.join(tmp_dir, 'decoder_step.pb')
        decoder_step_ensemble.onnx_export(
            decoder_step_pb_path,
            pytorch_encoder_outputs,
        )

        # single EOS in flat array
        input_tokens = torch.LongTensor(
            np.array([model_list[0].dst_dict.eos()]),
        )
        prev_scores = torch.FloatTensor(np.array([0.0]))
        timestep = torch.LongTensor(np.array([0]))

        pytorch_first_step_outputs = decoder_step_ensemble(
            input_tokens,
            prev_scores,
            timestep,
            *pytorch_encoder_outputs
        )

        # next step inputs (input_tokesn shape: [beam_size])
        next_input_tokens = torch.LongTensor(
            np.array([i for i in range(4, 9)]),
        )

        next_prev_scores = pytorch_first_step_outputs[1]
        next_timestep = timestep + 1
        next_states = pytorch_first_step_outputs[4:]

        step_inputs = []

        # encoder outputs need to be replicated for each input hypothesis
        for encoder_rep in pytorch_encoder_outputs[:len(model_list)]:
            step_inputs.append(encoder_rep.repeat(1, beam_size, 1))

        if model_list[0].decoder.vocab_reduction_module is not None:
            step_inputs.append(pytorch_encoder_outputs[len(model_list)])

        step_inputs.extend(list(next_states))

        pytorch_next_step_outputs = decoder_step_ensemble(
            next_input_tokens,
            next_prev_scores,
            next_timestep,
            *step_inputs
        )

        with open(decoder_step_pb_path, 'r+b') as f:
            onnx_model = onnx.load(f)
        onnx_decoder = caffe2_backend.prepare(onnx_model)

        decoder_inputs_numpy = [
            next_input_tokens.numpy(),
            next_prev_scores.detach().numpy(),
            next_timestep.detach().numpy(),
        ]
        for tensor in step_inputs:
            decoder_inputs_numpy.append(tensor.detach().numpy())

        caffe2_next_step_outputs = onnx_decoder.run(
            tuple(decoder_inputs_numpy),
        )

        for i in range(len(pytorch_next_step_outputs)):
            caffe2_out_value = caffe2_next_step_outputs[i]
            pytorch_out_value = pytorch_next_step_outputs[i].data.numpy()
            np.testing.assert_allclose(
                caffe2_out_value,
                pytorch_out_value,
                rtol=1e-4,
                atol=1e-6,
            )

    def test_batched_beam_decoder_default(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        self._test_batched_beam_decoder_step(test_args)

    def test_batched_beam_decoder_vocab_reduction(self):
        test_args = test_utils.ModelParamsDict(
            encoder_bidirectional=True,
            sequence_lstm=True,
        )
        lexical_dictionaries = test_utils.create_lexical_dictionaries()
        test_args.vocab_reduction_params = {
            'lexical_dictionaries': lexical_dictionaries,
            'num_top_words': 5,
            'max_translation_candidates_per_word': 1,
        }
        self._test_batched_beam_decoder_step(test_args)


### Pegando Blogs HashTag Treinamentos
# Criando um navegador
import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Criando o navegador
servico = Service(ChromeDriverManager().install())
navegador = webdriver.Chrome(service=servico)

# Abrindo o link
navegador.get('https://www.hashtagtreinamentos.com/')
time.sleep(2)  # Esperar carregar

# Esperar o PopUp aparecer na tela
elemento = WebDriverWait(navegador, 15).until(EC.presence_of_element_located((By.CLASS_NAME, "eicon-close")))
time.sleep(1)  # Espera o PopUp carregar totalmente na tela
elemento.click()

# Pegando a lista de links da classe nav-link
links = navegador.find_elements(By.CLASS_NAME, 'nav-link')
# print(links)
for link in links:
    #print(link.text)
    if 'blog' in link.text.lower():  # transformando o nome do link em minusculo para comparar
        link_inicial = link.get_attribute('href')
        link.click()  # Abrindo o link do Blog localizado
        break
#### Pegando o Assunto e o link de cada artigo
lista_assunto = []
lista_link = []
i = 1
while navegador.find_elements(By.CLASS_NAME, 'cont'):
    list_artigos = navegador.find_elements(By.CLASS_NAME, 'cont')  # Localizando tag que possui os links
    # print(list_artigos)

    for artigo in list_artigos:
        assuntos = artigo.find_elements(By.TAG_NAME, 'h4')  # para cada artigo quero pegar a tag h4 que é o assunto
        links = artigo.find_elements(By.TAG_NAME, 'a')  # para cada artigo quero pegar a tag a
        for assunto in assuntos:
            if assunto.text is None:
                lista_assunto.append('None')
            lista_assunto.append(assunto.text)
        for link in links:
            if link.get_attribute('href') is None:
                lista_link.append('None')
            lista_link.append(link.get_attribute('href'))  # para cada link quero pegar o link

    i += 1
    pages = link_inicial + f'/page/{i}'
    navegador.get(pages)
# Fechar o navegador
navegador.quit()

time.sleep(2)
# Validação para criar um Dataframe as listas devem ter o mesmo tamanho
tamanho_assunto = len(lista_assunto)
tamanho_link = len(lista_link)

if tamanho_assunto < tamanho_link:
    lista_assunto = lista_assunto + [None] * (len(lista_link) - len(lista_assunto))
else:
    lista_link = lista_link + [None] * (len(lista_assunto) - len(lista_link))

dic_blogs = {
    'assunto': lista_assunto,
    'link_blog': lista_link
}
print(dic_blogs)

# Exportar para csv/excel
df = pd.DataFrame(dic_blogs)
df.to_excel('Blogs Hashtag.xlsx', index=False)
print('Arquivo exportado com sucesso!')


# Clam Bot
import discord
from discord.ext import commands
import random
import datetime
import urllib.request
import urllib.parse
import re
import json
from googleapiclient.discovery import build
from google.cloud import secretmanager

# Setup or the discord BOT python framework
intents = discord.Intents.default()
intents.members = True
#intents.message_content = True
bot = commands.Bot(command_prefix='$', description='Sentient Clam', intents=intents)

# This method runs when the bot is started up available for botting things.
@bot.event
async def on_ready():
    await bot.change_presence(activity=discord.Streaming(name="Deep Sea Clamming", url="https://https://www.twitch.tv/mdsolarflare"))
    print(f'The clam is now present. user:{bot.user} id:{bot.user.id}')

# This method occurs whenever the bot receives a message, ultimately will handle the conversational aspect.
@bot.listen()
async def on_message(message):
    if message.author == bot.user:
        return

    if message.content.startswith('$hello'):
        channel_name = message.channel.name
        is_bot_channel = 'smarter-clam' == channel_name
        print(f'channel name is [{channel_name}] is bot channel:{is_bot_channel}')

        if is_bot_channel:
            #this later will called process_clam(message) from the clam lib
            await message.channel.send('you are in the bot channel')
        else:
            await message.channel.send('Hello!')

# This method shares the source location for the bot for contributors
@bot.command()
async def source(ctx):
    await ctx.send('Hi, the deep sea location of my source code is here: https://github.com/mdsolarflare/clambot-discord')

# In-Progress method for pulling list of media urls from a given subreddit.
def get_reddit(ctx, subreddit, limit=1000):
    try:
        json_blob = json.loads(urllib.request.urlopen(f'https://www.reddit.com/r/{subreddit}/.json?limit={limit}').read())
    except Exception as e:
        print(f'{e}')
        ctx.send(f'You asked for something but I could not find a subreddit for that.')

    # unpack the json to get a list of urls
    url_list = ""
    return url_list

# In-Progress bot command for posting random non-reoccuring bot images. (mayble later even some kind of hash checking mechanism?)
@bot.command()
async def find(ctx, search_term):
    # get the def for get_reddit(ctx, search_term) working first
    await ctx.send(f'Sorry, this feature is still in progress, will look for {search_term} later')

# Helper method for pulling youtube bits.
def get_service():
    # Get developer key from "credentials" tab of api dashboard
    return build("youtube", "v3", developerKey="key")

# Helper method for pulling youtube bits
def search_on_youtube(term, channel):
    service = get_service()
    resp = service.search().list(
        part="id",
        q=term,
        safeSearch="none" if channel.is_nsfw() else "moderate",
        videoDimension="2d",
    ).execute()
    return resp["items"][0]["id"]["videoId"]

# Bot command to search youtube
@bot.command()
async def youtube(ctx, *, search):
    await ctx.send(search_on_youtube(search, ctx))

# Gets the token only available to my user on the instance.
def get_local_token():
    token = open('./tok.tok', 'r').read()
    return token

# In-Progress method to pull the token value from GCP Secret Manager
def get_gcp_secret():
    # Setup the Secret manager Client
    #google_secret_manager = secretmanager.SecretManagerServiceClient()
    # get the secret
    #resource_name = 'projects/heyy-dev-site/secrets/heyy-dev-discord/versions/latest'
    #response = google_secret_manager.access_secret_version(name=resource_name)
    #discord_key = response.payload.data.decode('UTF-8')
    #print('secret: {0}'.format(discord_key)
    print('not implemented')

# START MASTER CLAM
bot.run(get_local_token())


from ..model.db import session
from ..model.tweet import Tweet as tw
from datetime import datetime
from ..tw_message import tw_message
from sqlalchemy.exc import *
from sqlalchemy.sql.expression import func
from sqlalchemy import desc
import datetime

class Tweet():

    """
    tweet関連DB操作クラス
    """

    def __init__(self):
        pass

    # 投稿対象データ
    def get_all_enable_status(self):
        """未投稿データを返却

        Args:
            self(lib.datase.Tweet):

            Returns:
                [lib.model.Tweet]: 未投稿データ
        """
        users = session.query(tw).filter( tw.post_status == 0 ).\
            order_by(tw.created_date.asc()).all()
        return users

    def add_values(self,tweets):
        """読み取ったデータを格納します

        以下のデータは格納されません
         - tweet_idが重複しているデータ
         - created_dateが全レコードの最大値より過去日
           => つまり、後から参加した人のツイートは除外する

        Args:
            self(lib.datase.Tweet):

        Returns:
            None
        """
        maxtimestamp = self.get_maxtimestamp()
        for tweet in tweets.get_messages():
            try:
                date_str = tweet['created_date'].replace(" +0000", "")
                created_date = datetime.datetime.strptime(date_str, '%c')
                if maxtimestamp is None or maxtimestamp <= created_date:
                    # 新しいデータであれば格納
                    tw_data = tw(user=tweet['user'], tid=tweet['tid'],\
                        url=tweet['url'], screen_name=tweet['screen_name'], \
                            created_date=tweet['created_date'], post_status = 0)
                    session.add(tw_data)
                    session.flush()
                    session.commit()

            except IntegrityError:
                # もうここはにぎにぎしてつぶしちゃう。
                # 発生するとしたら整合性エラーだったり、DB書き込みエラーだったりだし
                # リトライするとか正気じゃない
                session.rollback()
        session.close()

    def get_maxtimestamp(self):
        """現在保持しているレコード中、created_dateの最大値

        Args:
            self (lib.datase.Tweet):

        Returns:
            String: 最大のtid
            None: 結果なし
        """
        max_timestamp = session.query(func.max(tw.created_date)).one()
        return max_timestamp[0]

    def get_maxtimestamp_tweet(self):
        """現在保持しているレコード中、created_dateが最大のtidを返却

        Args:
            self (lib.datase.Tweet):

        Returns:
            String: 最大のtid
            None: 結果なし
        """
        max_timestamp = session.query(tw.tid).order_by(desc(tw.created_date)).limit(1).all()
        if len(max_timestamp) == 0:
            pass
        else:
            return max_timestamp.pop()


import re
import threading

import requests, os, time
from bs4 import BeautifulSoup
from prettytable import PrettyTable
from queue import Queue

def get_(way='page', num=1, tag=None):
	if not tag:
		url = 'https://www.mzitu.com/{}/{}'.format(way, str(num))
	else:
		url = tag + 'page/' + str(num)
	#	print(url)
	headers = {
		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
		"Referer": "https://www.mzitu.com/"}
	r = req(url, headers=headers)
	soup = BeautifulSoup(r.text, features='html.parser')
	if soup.find('title').get_text().find('404') != -1:
		return
	try:
		num = soup.find(class_='nav-links').find_all('a')[-2].get_text()
	except AttributeError:
		num = 1
	try:
		li = soup.find(id='pins').find_all('li')
		name = [i.find('a').find('img').get('alt') for i in li]
		list = [i.find('a').get('href').split('/')[-1] for i in li]
	except AttributeError:
		li = soup.find(class_='tags').find_all('dd')
		#		web,name=li[0].find('a').get('href'),li[0].find('a').get_text()
		#		print(web,name)
		list = [(i.find('a').get('href'), i.find('a').get_text()) for i in li]
		li = soup.find(class_='tags')
		a = str(li).split('dt>')
		b = (a[1].replace('</', ''), a[2].count('
'))
		c = (a[3].replace('</', ''), a[4].count('
'))
		date = (b, c)
		# print(date)
		return (date, list)
	return (num, list, name)

def get_2(way='page', n_=1, tag=None):
	#	if not tag:
	#		date=get_(way,n_)
	#	else:
	#		date=get_(tag=tag)
	date = get_(way, n_) if not tag else get_(tag=tag)
	if date:
		num, list, name = date
		if int(num) <= 10:
			print('正在获取网址，请耐心等待')
			for i in range(2, 1 + int(num)):
				#				if not tag:
				#					num,li,na=get_(way,i)
				#				else:
				#					num,li,na=get_(num=i,tag=tag)
				num, li, na = get_(way, i) if not tag else get_(num=i, tag=tag)
				list += li
				name += na
				return (1, list, name)
		return (num, list, name)

def all_page(way='page', tag=None):
	def main_(n_=1):
		num, list, name = get_2(way, n_, tag)
		n = 0
		#		print(list)
		print('*' * 20)
		print('     共有{}个项目'.format(str(len(list))))
		print('*' * 20)
		for i in list:
			print('-' * 10 + '分割线' + '-' * 10)
			print('编号为:' + str(i))
			print('名字为:' + name[n])
			print('-开始下载-')
			get_pic(i)
			n += 1
		return True if int(num) != n_ else None

	n = 1
	while True:
		if main_(n):
			n += 1
		else:
			break

# 主页面的

def hot_page():
	all_page('hot')

#	date=get_2('hot')
# 最新的

def recommend_page():
	all_page('best')

#	date=get_2('best')
# 推荐的

def zhuanti_page():
	def print_fox(data=None, way=0):
		x = PrettyTable(['编号', '标题名称' if not way else '妹子名字'])
		n = 1
		for i in data:
			x.add_row([str(n), i[1]])
			n += 1
		print(x)
		return n

	date = get_('zhuanti')
	if date:
		num, list = date
		list = [(num[0][0], list[:int(num[0][-1])]), (num[1][0], list[int(num[0][-1]):])]
		#	print(list)
		print('查看标签/1   人物/2')
		n = chi(1, 2) - 1
		# print(list[n][-1])
		n_ = print_fox(list[n][-1], n)
		print('输入图集编号')
		cho = chi(next=n_)
		print('选择了:第{}个，名字为:{}'.format(str(cho), list[n][-1][cho - 1][1]))
		url = list[n][-1][cho - 1][0]
		# print(url)
		num, list, name = all_page(tag=url)

# zhuanti_page()

def chi(first=1, next=4):
	try:
		n = int(input('请输入:'))
		if first <= n <= next:
			return n
		print('输入应该在{}~{}中'.format(str(first), str(next)))
		return chi(first, next)
	except ValueError:
		print('请输入数字')
		return chi(first, next)

def get_pic(num=1, p=1):
	url = 'https://www.mzitu.com/'
	# num=1
	headers = {
		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
		"Referer": "https://www.mzitu.com/"}
	r = req(url + str(num), headers=headers)
	r.encoding = 'utf_8'
	soup = BeautifulSoup(r.text, features='html.parser')
	# print(soup)
	if soup.find('title').get_text().find('404') != -1:
		return
	d = soup.find(class_='pagenavi').find_all('a')[-2]
	num = d.find('span').get_text()
	if p == 1:
		print('检测到:一共有{}页'.format(str(num)))
	#	web=''
	#	for i in d['href'].split('/')[:-1]:
	#		web+=i+'/'
	web = '/'.join(d['href'].split('/')[:-1]) + '/'
	#	print(web)

	list = [(web + str(n), n) for n in range(1, int(num) + 1)]
	# print(num,list)
	info = soup.find('p').find('a').find('img')
	name, url = info.get('alt'), info.get('src')

	'''
	web,num = list[0]
	print(web,num)
	d=requests.get(web,headers=headers)
	print(d.text)
	'''
	count = 5
	Qiubai = QiubaiSpider(list,name,headers,(count,count,count))
	Qiubai.run()
	while Qiubai.event:
		time.sleep(0.5)

	# for web, num in list:
	# 	if os.path.isdir(name) != True:
	# 		os.makedirs(name)
	# 	#	print(web)
	# 	r = req(web, headers)
	# 	soup = BeautifulSoup(r.text, features='html.parser')
	# 	url = soup.find('p').find('a').find('img').get('src')
	# 	p = req(url, headers=headers)
	# 	na = '{}/{}.jpg'.format(name, str(num))
	# 	with open(na, 'wb')as f_p:
	# 		f_p.write(p.content)
	# 	print(num)
	# 	time.sleep(.5)

def run_forever(func):
	def wrapper(obj):
		while obj.event:
			func(obj)

	return wrapper

class QiubaiSpider:
	def __init__(self, li_list,name,header, count=(1, 1, 1)):
		self.event = True
		self.li_list = li_list
		self.header = header
		self.name = name
		self.all_num = len(li_list)
		self.now_num = 0
		self.count = count
		self.now_time = 1
		# url 队列
		self.url_queue = Queue()
		# 响应队列
		self.page_queue = Queue()
		# 数据队列
		self.data_queue = Queue()

	def add_url_to_queue(self):
		# 把URL添加url队列
		for i in self.li_list:
			a , b = i
			i = b ,a
			self.url_queue.put(i)

	@run_forever
	def add_page_to_queue(self):
		# 发送请求获取数据
		try:
			url_all = self.url_queue.get()
			number = url_all[0]
			url = url_all[-1]
			response = req(url,headers=self.header)
			if response.status_code != 200:
				self.url_queue.put(url_all)
			else:
				data = (number, response)
				self.page_queue.put(data)
			time.sleep(2)
		# 完成当前URL任务

		except BaseException as e:
			print(e)
			self.url_queue.put(url_all)
			now = re.findall('443', str(e))
			if len(now) > 0:
				if now[0] == '443':
					if self.now_time >= 5:
						print('网络连接超时')
						self.now_time = 1
					else:
						self.now_time += 1
			time.sleep(0.5)
		finally:
			self.url_queue.task_done()

	@run_forever
	def add_dz_to_queue(self):
		try:
			page_all = self.page_queue.get()
			number = page_all[0]
			page = page_all[-1]
			page.encoding = 'utf-8'
			soup = BeautifulSoup(page.text, features='html.parser')
			url = soup.find('p').find('a').find('img').get('src')
			p = req(url,headers=self.header)
			self.path = '{}/{}/{}.jpg'.format('妹子图片',self.name, str(number))

			data = (number, p.content)
			self.data_queue.put(data)
			time.sleep(2)
		except BaseException as e:
			print(e)
			self.page_queue.put(page_all)
			time.sleep(0.5)
		finally:
			self.page_queue.task_done()

	@run_forever
	def save_dz_list(self):
		# 把信息保存到文件中
		try:
			dz_list_all = self.data_queue.get()
			number = dz_list_all[0]
			dz_list = dz_list_all[-1]

			f,s,t=self.path.split('/')
			if not os.path.exists(f):
				os.mkdir(f)
			if not os.path.exists(f+os.path.sep+s):
				os.mkdir(f+os.path.sep+s)
			with open(self.path, 'wb')as f:
				f.write(dz_list)
			# print('finish')
			self.now_num += 1

			time.sleep(1)
		except BaseException as e:
			print(e)
			self.data_queue.put(dz_list_all)
			time.sleep(0.5)
		finally:
			self.data_queue.task_done()

	def run_use_more_task(self, func, count=1):
		# 把func放到线程中执行,count:开启多少线程执行
		for i in range(0, count):
			t = threading.Thread(target=func)
			t.setDaemon(True)
			t.start()

	def verification_event(self):
		while True:
			print(self.now_num,self.all_num)
			if self.now_num == self.all_num:
				self.event = False
				break
			else:
				time.sleep(0.5)

	def run(self):
		# 开启线程执行上面的几个方法
		url_t = threading.Thread(target=self.add_url_to_queue)
		url_t.setDaemon(True)
		url_t.start()

		p = threading.Thread(target=self.verification_event)
		p.setDaemon(True)
		p.start()

		self.run_use_more_task(self.add_page_to_queue, self.count[0])
		self.run_use_more_task(self.add_dz_to_queue, self.count[1])
		self.run_use_more_task(self.save_dz_list, self.count[2])

		self.url_queue.join()
		self.page_queue.join()
		self.data_queue.join()

def req(url, headers):
	try:
		# return requests.get(url, headers=headers,proxies=False)
		return requests.get(url, headers=headers)
	except requests.exceptions.ConnectionError as e:
		print(e)
		time.sleep(1)
		return req(url, headers)

# get_pic(i)

'''
for i in zhuanti_page():
	print('---------'+str(i))
	get_pic(i)
'''

if __name__ == '__main__':
	print('说明:1/主菜单 2/最新的 3/推荐的 4/一个大分类')
	n = chi()
	try:
		if n == 1:
			all_page()
		elif n == 2:
			hot_page()
		elif n == 3:
			recommend_page()
		elif n == 4:
			zhuanti_page()
	except BaseException as e:
		print(e)
	print('二十秒钟过后自动关闭')
	time.sleep(20)


from django.urls import path
from rest_framework import routers

from . import views
from .api_views import ExpenseViewSet

app_name = "e"

urlpatterns = [
    path("random/", views.RandomExpenseView.as_view(), name="random"),
    path("demo/", views.MyDemoView.as_view(), name="demo"),
    path("name/", views.RandomNameView.as_view(), name="random_name"),
    path("foo/", views.FooView.as_view(), name="foo"),
    path("", views.ExpenseListView.as_view(), name="list"),
    path("create/", views.ExpenseCreateView.as_view(), name="create"),
    path("<int:pk>/", views.ExpenseDetailView.as_view(), name="detail"),
    path("<int:pk>/edit/", views.ExpenseUpdateView.as_view(), name="update"),
    path("<int:pk>/delete/", views.ExpenseDeleteView.as_view(), name="delete"),
]


import cv2 as cv

image = cv.imread('images/input.jpg')
cv.imshow('original',image)
cv.waitKey()

# Let's make our image 3/4 of it's original size
image_scaled = cv.resize(image, None, fx=0.75, fy=0.75)
cv.imshow('Scaling - Linear Interpolation', image_scaled)
cv.waitKey()

# Let's double the size of our image
img_scaled = cv.resize(image, None, fx=2, fy=2, interpolation = cv.INTER_CUBIC)
cv.imshow('Scaling - Cubic Interpolation', img_scaled)
cv.waitKey()

# Let's skew the re-sizing by setting exact dimensions
img_scaled = cv.resize(image, (900, 400), interpolation = cv.INTER_AREA)
cv.imshow('Scaling - Skewed Size', img_scaled)
cv.waitKey()

# Image Pyramids
smaller = cv.pyrDown(image)
larger = cv.pyrUp(smaller)

cv.imshow('Smaller ', smaller )
cv.waitKey(0)
cv.imshow('Larger ', larger )
cv.waitKey(0)

cv.destroyAllWindows()


import torch
from pathlib import Path
import numpy as np
import os.path as osp
import shutil
import os

from mmcv.runner.hooks.logger import NeptuneLoggerHook
from loguru import logger
from .builder import MMYLOGGER
from .base import *

@MMYLOGGER.register_module()
class MyNeptuneLogger(NeptuneLoggerHook):
    def __init__(self,*args, **kwargs):
        super().__init__(*args, **kwargs)
        from neptune.new.types import File
        self.File=File

    def initialize(self):
        if self.init_kwargs:
            self.run = self.neptune.init(**self.init_kwargs)
        else:
            self.run = self.neptune.init()

    @logger.catch
    def log(self, tag_name:str, tag_value, append=True):
        if append:
            self.run[tag_name].log(tag_value)
        else:
            self.run[tag_name]=(tag_value)

    @logger.catch
    def log_bs(self,dict_to_log={},append=True):
        for key,val in dict_to_log.items():
            # if self.is_scalar(val):
                self.log(key,val,append=append)

    @logger.catch
    def log_image(self,key:str,img):
        img=img_preprocess(img)
        self.run[key].upload(self.File.as_image(img))

    @logger.catch
    def update_config(self,config_dict):
        self.run['parameters']=config_dict

    @logger.catch
    def create_proj(self,
                    workshops='lithiumice',
                    proj_name='smplifyx',
    ):
        from neptune import management
        api_token=self.init_kwargs.api_token

        management.get_project_list(api_token=api_token)
        management.create_project(
            name=f'{workshops}/{proj_name}',
            key=proj_name,
            api_token=api_token,
            visibility='pub')


#!/bin/python2.7

from naoqi import ALProxy
import sys
import time
import math
import pandas as pd # Needed for gathering data and writing to csv
from enum import Enum
from vocab import *

#ip = "192.168.208.210" # Change based on session
#port = 9559

ip = str(sys.argv[1])
port = int(sys.argv[2])

# Constants
DEFAULT_MEM = ['', -3.0]
FILE = str(sys.argv[3])
DELAY=1.5
VOL=1.0

if str(sys.argv[4]) == 'espex':
    DIALOG = espexDialog
    VOCAB = espexVocab
else:
    DIALOG = pilotDialog
    VOCAB = pilotVocab

CHANGE = math.ceil(200.0/len(DIALOG[1]))

# Enumeration to make colors more readable
class Color(Enum):
    RED = 0x00ff0000
    GREEN = 0x0000ff00
    BLUE = 0x000000ff

# import proxies to control robot
try:
    speechRec = ALProxy("ALSpeechRecognition", ip, port)
    tts = ALProxy("ALTextToSpeech", ip, port)
    animSay = ALProxy("ALAnimatedSpeech", ip, port)
    mem = ALProxy("ALMemory", ip, port)
    leds = ALProxy("ALLeds", ip, port)
    posture = ALProxy("ALRobotPosture", ip, port)
except:
    print("ERROR")
    sys.exit(1)

def recSpeech(vocab, wordSpot=True):
    """
    Recognizes speech using speech recognition and memory proxies.

    Parameters:
        vocab - List of words to be recognized;
        wordSpot - Whether the NAO can recognize it in a phrase/sentence;

    Returns:
        True if speech was recognized, False after 5 seconds and nothing recognized
    """
    speechRec.pause(True)
    speechRec.setVocabulary(vocab, wordSpot)

    speechRec.subscribe("RobotB")
    response = DEFAULT_MEM
    counter = 0

    # Keep checking until the response changes or 5 seconds has elapsed
    while response == DEFAULT_MEM and counter < 500:
        time.sleep(0.01)
        # Check memory for the word
        response = mem.getData("WordRecognized")
        counter += 1
    speechRec.unsubscribe("RobotB")
    print("RESPONSE = %s" % response)

    # Record word and confidence data
    word_list.append(response[0])
    conf_list.append(response[1])

    if response == DEFAULT_MEM:
        return False
    else:
        return True

# state 0 - initialize robot
state = 0
i = 0 # dialog counter
j = 0 # vocab counter
mood_rating = -100
word_list = [DEFAULT_MEM[0]]
conf_list = [DEFAULT_MEM[1]]

while True:
    if state == 0:
        # stand up
        posture.goToPosture("StandInit", 1.0)
        # eye color
        ledsGroup = "eyes"
        ledsNames = ["FaceLeds"]
        leds.createGroup(ledsGroup, ledsNames)
        leds.on(ledsGroup)
        leds.fadeRGB(ledsGroup, Color.GREEN, 0.1)
        # set speaker volume
        tts.setVolume(VOL)
        state = 1
    elif state == 1:
        while mood_rating < 0:
            animSay.say(DIALOG[1][i], "contextual")
            time.sleep(DELAY)
            mood_rating += CHANGE
            i += 1
        leds.fadeRGB(ledsGroup, Color.BLUE, 0.1)
        state = 2
    elif state == 2:
        while mood_rating < 100:
            if recSpeech(VOCAB[1][j]):
                j += 1
                mood_rating += CHANGE
                time.sleep(DELAY)
                animSay.say(DIALOG[1][i], "contextual")
                i += 1
            else:
                animSay.say(DIALOG[1][i], "contextual")
        leds.fadeRGB(ledsGroup, Color.RED, 0.1)
        print("DONE")
        break

d = {'word':word_list, 'confidence':conf_list}
df = pd.DataFrame(d)
df.to_csv(FILE)


#!/usr/bin/python3
import pymongo
#面向文档存储MongoDB

#连接
client = pymongo.MongoClient(host='127.0.0.1',port=27017)

#指定数据库
db = client.test

#指定集合
connection = db.pymongo

#数据对象
student={
    "name":"MissLee",
    "age":29
}
#插入对象
result = connection.insert(student)

#打印结果，返回id
print(result)

#查询大于($eq))或者小于($lt)的数据
find = connection.find({"age": {"$lt":30}})

# result = connection.aggregate({"$group":{"count":{"$sum":1}}})

print(find)

for item in find:
    print(item)


# -*- coding: utf-8 -*-
import scrapy
from biquge.items import BiqugeItem
from .sjzh import Cn2An,get_tit_num

class XsphspiderSpider(scrapy.Spider):
    name = "xsphspider"
    allowed_domains = ["qu.la"]
    start_urls = ['http://www.qu.la/paihangbang/']
    novel_list = []

    def parse(self, response):

        # 找到各类小说排行榜名单
        books = response.xpath('.//div[@class="index_toplist mright mbottom"]')

        # 找到每一类小说排行榜的每一本小说的下载链接
        for book in books:
            links = book.xpath('.//div[2]/div[2]/ul/li')
            for link in links:
                url = 'http://www.qu.la' + \
                    link.xpath('.//a/@href').extract()[0]
                self.novel_list.append(url)

        # 简单的去重
        self.novel_list = list(set(self.novel_list))

        for novel in self.novel_list:
           yield scrapy.Request(novel, callback=self.get_page_url)

    def get_page_url(self, response):
        '''
        找到章节链接
        '''
        page_urls = response.xpath('.//dd/a/@href').extract()

        for url in page_urls:
           yield scrapy.Request('http://www.qu.la' + url,callback=self.get_text)

    def get_text(self, response):
        '''
        找到每一章小说的标题和正文
        并自动生成id字段，用于表的排序
        '''
        item = BiqugeItem()

        # 小说名
        item['bookname'] = response.xpath(
            './/div[@class="con_top"]/a[2]/text()').extract()[0]

        # 章节名 ,将title单独找出来，为了提取章节中的数字
        title = response.xpath('.//h1/text()').extract()[0]
        item['title'] = title

        #  找到用于排序的id值
        item['order_id'] = Cn2An(get_tit_num(title))

        # 正文部分需要特殊处理
        body = response.xpath('.//div[@id="content"]/text()').extract()

        # 将抓到的body转换成字符串，接着去掉\t之类的排版符号，
        text = ''.join(body).strip().replace('\u3000', '')

        item['body'] = text

        return item