# -*- coding: utf-8 -*-
'''
Implements a correlated topic model, similar to that described by Blei
but using the Bouchard product of sigmoid bounds instead of Laplace
approximation.

Created on 17 Jan 2014

@author: bryanfeeney
'''

import os # Configuration for PyxImport later on. Requires GCC
# os.environ['CC']  = os.environ['HOME'] + "/bin/cc"

from math import log
from math import pi
from math import e

from collections import namedtuple
import numpy as np
import numpy.random as rd
import scipy.special as fns
import sys
import time

import sidetopics.model.lda_vb_fast as compiled

from sidetopics.util.sparse_elementwise import sparseScalarProductOfSafeLnDot
from sidetopics.util.overflow_safe import safe_log
from sidetopics.util.misc import constantArray, converged, clamp
from sidetopics.model.lda_cvb0 import toWordList
from sidetopics.model.evals import perplexity_from_like

# ==============================================================
# CONSTANTS
# ==============================================================

DTYPE=np.float32 # A default, generally we should specify this in the model setup

LN_OF_2_PI   = log(2 * pi)
LN_OF_2_PI_E = log(2 * pi * e)

DEBUG=False

MODEL_NAME="lda/vb"

# ==============================================================
# TUPLES
# ==============================================================

TrainPlan = namedtuple ( \
    'TrainPlan',
    'iterations epsilon logFrequency fastButInaccurate debug')

QueryState = namedtuple ( \
    'QueryState', \
    'W_list docLens topicDists'\
)

ModelState = namedtuple ( \
    'ModelState', \
    'K topicPrior vocabPrior wordDists dtype name'
)

# ==============================================================
# PUBLIC API
# ==============================================================

def newModelFromExisting(model):
    '''
    Creates a _deep_ copy of the given model
    '''
    return ModelState(\
        model.K, \
        model.topicPrior.copy(), \
        model.vocabPrior, \
        None if model.wordDists is None else model.wordDists.copy(), \
        model.dtype,       \
        model.name)

def newModelAtRandom(data, K, topicPrior=None, vocabPrior=None, dtype=DTYPE):
    '''
    Creates a new LDA ModelState for the given training set and
    the given number of topics. Everything is instantiated purely
    at random. This contains all parameters independent of of
    the dataset (e.g. learnt priors)

    Param:
    data - the dataset of words, features and links of which only words are used in this model
    K - the number of topics
    topicPrior - the prior over topics, either a scalar or a K-dimensional vector
    vocabPrior - the prior over vocabs, either a scalar or a T-dimensional vector
    dtype      - the datatype to be used throughout.

    Return:
    A ModelState object
    '''
    assert K > 1, "There must be at least two topics"
    T = data.words.shape[1]

    if topicPrior is None:
        topicPrior = constantArray((K,), 50.0 / K, dtype) # From Griffiths and Steyvers 2004
    if vocabPrior is None:
        vocabPrior = 1.1 # Also from G&S

    vocabPriorVec = constantArray((T,), vocabPrior, dtype)
    wordDists = rd.dirichlet(vocabPriorVec, size=K).astype(dtype)

    # Peturb to avoid zero probabilities
    wordDists += 1./T
    wordDists /= (wordDists.sum(axis=1))[:,np.newaxis]

    return ModelState(K, topicPrior, vocabPrior, wordDists, dtype, MODEL_NAME)

def newQueryState(data, modelState, debug: bool = False):
    '''
    Creates a new LDA QueryState object. This contains all
    parameters and random variables tied to individual
    datapoints.

    Param:
    data - the dataset of words, features and links of which only words are used in this model
    modelState - the model state object

    Return:
    A CtmQueryState object
    '''
    D,_ = data.words.shape
    print("Converting Bag of Words matrix to List of List representation... ", end="")
    W_list, docLens = toWordList(data.words)

    # Initialise the per-token assignments at random according to the dirichlet hyper
    topicDists = rd.dirichlet(modelState.topicPrior, size=D).astype(modelState.dtype)

    return QueryState(W_list, docLens, topicDists)

def newTrainPlan(iterations=100, epsilon=2, logFrequency=10, fastButInaccurate=False, debug=DEBUG):
    '''
    Create a training plan determining how many iterations we
    process, how often we plot the results, how often we log
    the variational bound, etc.

    epsilon is oddly measured, we just evaluate the angle of the line segment between
    the last value of the bound and the current, and if it's less than the given angle,
    then stop.
    '''
    return TrainPlan(iterations, epsilon, logFrequency, fastButInaccurate, debug)

def train (data, modelState, queryState, trainPlan):
    '''
    Infers the topic distributions in general, and specifically for
    each individual datapoint.

    Params:
    data - the dataset of words, features and links of which only words are used in this model
    modelState - the actual LDA model. In a training run (query = False) this
                 will be mutated in place, and then returned.
    queryState - the query results - essentially all the "local" variables
                 matched to the given observations. This will be mutated in-place
                 and then returned.
    trainPlan  - how to execute the training process (e.g. iterations,
                 log-interval etc.)
    query      -

    Return:
    The updated model object (note parameters are updated in place, so make a
    defensive copy if you want it)
    The query object with the update query parameters
    '''
    iterations, epsilon, logFrequency, fastButInaccurate, debug = \
        trainPlan.iterations, trainPlan.epsilon, trainPlan.logFrequency, trainPlan.fastButInaccurate, trainPlan.debug
    W_list, docLens, topicDists = \
        queryState.W_list, queryState.docLens, queryState.topicDists
    K, topicPrior, vocabPrior, wordDists, dtype = \
        modelState.K, modelState.topicPrior, modelState.vocabPrior, modelState.wordDists, modelState.dtype

    W   = data.words
    D,T = W.shape

    # Quick sanity check
    if np.any(docLens < 1):
        raise ValueError ("Input document-term matrix contains at least one document with no words")

    # Book-keeping for logs
    logPoints    = 1 if logFrequency == 0 else iterations // logFrequency
    boundIters   = np.zeros(shape=(logPoints,))
    boundValues  = np.zeros(shape=(logPoints,))
    likelyValues = np.zeros(shape=(logPoints,))
    bvIdx = 0

    # Instead of storing the full topic assignments for every individual word, we
    # re-estimate from scratch. I.e for the memberships z which is DxNxT in dimension,
    # we only store a 1xNxT = NxT part.
    z_dnk = np.empty((docLens.max(), K), dtype=dtype, order='F')

    # Select the training iterations function appropriate for the dtype
    current_micro_time = lambda: int(time.time())
    do_iterations = compiled.iterate_f32 \
                    if modelState.dtype == np.float32 \
                    else compiled.iterate_f64
#    do_iterations = iterate # pure Python

    # Iterate in segments, pausing to take measures of the bound / likelihood
    segIters  = logFrequency
    remainder = iterations - segIters * (logPoints - 1)
    totalItrs = 0
    for segment in range(logPoints - 1):
        start = current_micro_time()
        totalItrs += do_iterations (segIters, D, K, T, \
                 W_list, docLens, \
                 topicPrior, vocabPrior, \
                 z_dnk, topicDists, wordDists)

        duration = current_micro_time() - start

        boundIters[bvIdx]   = segment * segIters
        boundValues[bvIdx]  = var_bound(data, modelState, queryState)
        likelyValues[bvIdx] = log_likelihood(data, modelState, queryState)
        perp = perplexity_from_like(likelyValues[bvIdx], W.sum())
        bvIdx += 1

        if converged (boundIters, boundValues, bvIdx, epsilon, minIters=20):
            boundIters, boundValues, likelyValues = clamp (boundIters, boundValues, likelyValues, bvIdx)
            return ModelState(K, topicPrior, vocabPrior, wordDists, modelState.dtype, modelState.name), \
                QueryState(W_list, docLens, topicDists), \
                (boundIters, boundValues, likelyValues)

        print ("Segment %d/%d Total Iterations %d Duration %d Perplexity %4.0f Bound %10.2f Likelihood %10.2f" % (segment, logPoints, totalItrs, duration, perp, boundValues[bvIdx - 1], likelyValues[bvIdx - 1]))

    # Final batch of iterations.
    do_iterations (remainder, D, K, T, \
                 W_list, docLens, \
                 topicPrior, vocabPrior, \
                 z_dnk, topicDists, wordDists)

    boundIters[bvIdx]   = iterations - 1
    boundValues[bvIdx]  = var_bound(data, modelState, queryState)
    likelyValues[bvIdx] = log_likelihood(data, modelState, queryState)

    return ModelState(K, topicPrior, vocabPrior, wordDists, modelState.dtype, modelState.name), \
           QueryState(W_list, docLens, topicDists), \
           (boundIters, boundValues, likelyValues)

MaxInnerItrs = 20
MinInnerItrs = 3
#@jit never really worked very well...
def iterate (iterations, D, K, T, \
             W_list, docLens, \
             topicPrior, vocabPrior, \
             z_dnk, topicDists, wordDists):

    totalItrs = 0
    epsilon = 0.01 / K
    oldWordDists = np.empty(wordDists.shape, wordDists.dtype)
    newWordDists = wordDists

    for _ in range(iterations):
        oldWordDists, newWordDists = newWordDists, oldWordDists
        lnWordDists = safe_log(oldWordDists, out=oldWordDists)
        newWordDists.fill (vocabPrior)

        for d in range(D):
            oldTopics = topicDists[d,:].copy()
            topicDists[d,:]= 1./ K
            lnWordProbs = lnWordDists[:,W_list[d,0:docLens[d]]]

            innerItrs = 0
            while ((innerItrs < MaxInnerItrs) or (np.sum(np.abs(oldTopics - topicDists[d,:])) > epsilon)) \
            and (innerItrs < MaxInnerItrs):
                diTopic     = fns.digamma(topicDists[d,:])
                z_dnk[:docLens[d],:] = lnWordProbs.T + diTopic[np.newaxis,:]

                # We've been working in log-space till now, before we go to true
                # probability space rescale so we don't underflow everywhere
                maxes  = z_dnk.max(axis=1)
                z_dnk -= maxes[:,np.newaxis]
                np.exp(z_dnk, out=z_dnk)

                # Now normalize so probabilities sum to one
                sums   = z_dnk.sum(axis=1)
                z_dnk /= sums[:,np.newaxis]            # Update vocabulary: hard to do with a list representation

                # Now use it to infer the topic distribution
                topicDists[d,:] = topicPrior + np.sum(z_dnk[:docLens[d],:], axis=0)
                topicDists[d,:] /= np.sum(topicDists[d,:])

                innerItrs += 1

            totalItrs += innerItrs
            for k in range(K):
                for n in range(docLens[d]):
                    newWordDists[k,W_list[d,n]] += z_dnk[n,k]
            newWordDists /= newWordDists.sum(axis=1)[:,np.newaxis]

    return totalItrs

def query(data, modelState, queryState, queryPlan):
    '''
    Given a _trained_ model, attempts to predict the topics for each of
    the inputs.

    Params:
    data - the dataset of words, features and links of which only words are used in this model
    modelState - the _trained_ model
    queryState - the query state generated for the query dataset
    queryPlan  - used in this case as we need to tighten up the approx

    Returns:
    The model state and query state, in that order. The model state is
    unchanged, the query is.
    '''
    W_list, docLens, topicDists = \
        queryState.W_list, queryState.docLens, queryState.topicDists
    K, topicPrior, vocabDists, dtype = \
        modelState.K, modelState.topicPrior, modelState.wordDists, modelState.dtype

    W   = data.words
    D,T = W.shape
    z_dnk = np.empty((docLens.max(), K), dtype=dtype, order='F')

    if modelState.dtype == np.float32:
        for _ in range(queryPlan.iterations):
            compiled.query_f64 (D, T, K, \
                     W_list, docLens, \
                     topicPrior, z_dnk, topicDists,
                     vocabDists)
    else:
        for _ in range(queryPlan.iterations):
            compiled.query_f64 (D, T, K, \
                     W_list, docLens, \
                     topicPrior, z_dnk, topicDists,
                     vocabDists)

    queryState.topicDists
    return modelState, queryState

def wordDists (modelState):
    '''
    The K x T matrix of  word distributions inferred for the K topics
    '''
    result = modelState.wordDists
    norm   = np.sum(modelState.wordDists, axis=1)
    result /= norm[:,np.newaxis]

    return result

def topicDists (queryState):
    '''
    The D x K matrix of topics distributions inferred for the K topics
    across all D documents
    '''
    result = queryState.topicDists
    norm   = np.sum(queryState.topicDists, axis=1)
    result /= norm[:,np.newaxis]

    return result

def log_likelihood (data, modelState, queryState):
    '''
    Return the log-likelihood of the given data W according to the model
    and the parameters inferred for the entries in W stored in the
    queryState object.

    Actually returns a vector of D document specific log likelihoods
    '''
    return sparseScalarProductOfSafeLnDot(data.words, topicDists(queryState), wordDists(modelState)).sum()

def var_bound(data, modelState, queryState, z_dnk = None):
    '''
    Determines the variational bounds.
    '''
    # Unpack the the structs, for ease of access and efficiency
    W_list, docLens, topicDists = \
        queryState.W_list, queryState.docLens, queryState.topicDists
    K, topicPrior, vocabPrior, _, dtype = \
        modelState.K, modelState.topicPrior, modelState.vocabPrior, modelState.wordDists, modelState.dtype

    W   = data.words
    D,T = W.shape
    maxN = docLens.max()
    if z_dnk == None:
        z_dnk = np.empty(shape=(maxN, K), dtype=dtype)

    wordDistsMatrix = wordDists(modelState)

    diWordDists = fns.digamma(wordDistsMatrix.copy()) - fns.digamma(wordDistsMatrix.sum(axis=1))[:,np.newaxis]
    lnWordDists = np.log(wordDistsMatrix)

    bound = 0

    # Expected Probablity
    #

    # P(topics|topicPrior)
    diTopicDists = fns.digamma(topicDists) - fns.digamma(topicDists.sum(axis=1))[:,np.newaxis]
    ln_b_topic = fns.gammaln(topicPrior.sum()) - fns.gammaln(topicPrior).sum()
    bound += D * ln_b_topic \
           + np.sum((topicPrior - 1) * diTopicDists)

    # and its entropy
    ent = fns.gammaln(topicDists.sum(axis=1)).sum() - fns.gammaln(topicDists).sum() \
        + np.sum ((topicDists - 1) * diTopicDists)

    bound -= ent

    # P(z|topic) is tricky as we don't actually store this. However
    # we make a single, simple estimate for this case.
    # NOTE COPY AND PASTED FROM iterate_f32  / iterate_f64 (-ish)
    for d in range(D):
        lnWordProbs = lnWordDists[:,W_list[d,0:docLens[d]]]
        diTopic     = fns.digamma(topicDists[d,:])
        z_dnk[0:docLens[d],:] = lnWordProbs.T + diTopic[np.newaxis,:]

        # We've been working in log-space till now, before we go to true
        # probability space rescale so we don't underflow everywhere
        maxes  = z_dnk.max(axis=1)
        z_dnk -= maxes[:,np.newaxis]
        np.exp(z_dnk, out=z_dnk)

        # Now normalize so probabilities sum to one
        sums   = z_dnk.sum(axis=1)
        z_dnk /= sums[:,np.newaxis]
#        z_dnk[docLens[d]:maxN,:] = 0 # zero probablities for words that don't exist

        # Now use to calculate  E[ln p(Z|topics), E[ln p(W|Z) and H[Z] in that order
        diTopic -= fns.digamma(np.sum(topicDists[d,:]))
        bound += np.sum(z_dnk * diTopic[np.newaxis,:])
        bound += np.sum(z_dnk[0:docLens[d],:].T * diWordDists[:,W_list[d,0:docLens[d]]])
        bound -= np.sum(z_dnk[0:docLens[d],:] * safe_log(z_dnk[0:docLens[d],:]))

    # p(vocabDists|vocabPrior)

    ln_b_vocab = fns.gammaln(T * vocabPrior) - T * fns.gammaln(vocabPrior)
    bound += K * ln_b_vocab \
           + (vocabPrior - 1) * np.sum(diWordDists)

    # and its entropy
    ent = fns.gammaln(wordDistsMatrix.sum(axis=1)).sum() - fns.gammaln(wordDistsMatrix).sum() \
        + np.sum ((wordDistsMatrix - 1) * diWordDists)

    bound -= ent

    return bound

def vocab(modelState):
    '''
    Return the vocabulary inferred by this model as a KxT matrix of T
    terms for each of the K topics
    '''
    return modelState.wordDists

# ==============================================================
# PUBLIC HELPERS
# ==============================================================

def printStderr(msg):
    sys.stdout.flush()
    sys.stderr.write(msg + '\n')
    sys.stderr.flush()

def static_var(varname, value):
    def decorate(func):
        setattr(func, varname, value)
        return func
    return decorate

@static_var("old_bound", 0)
def _debug_with_bound (itr, var_value, var_name, W, K, topicMean, sigT, vocab, dtype, means, varcs, A, n):
    if np.isnan(var_value).any():
        printStderr ("WARNING: " + var_name + " contains NaNs")
    if np.isinf(var_value).any():
        printStderr ("WARNING: " + var_name + " contains INFs")
    if var_value.dtype != dtype:
        printStderr ("WARNING: dtype(" + var_name + ") = " + str(var_value.dtype))

    old_bound = _debug_with_bound.old_bound
    bound     = var_bound(W, ModelState(K, topicMean, sigT, vocab, A, dtype, MODEL_NAME), QueryState(means, varcs, n))
    diff = "" if old_bound == 0 else "%15.4f" % (bound - old_bound)
    _debug_with_bound.old_bound = bound

    if int(bound - old_bound) < 0:
        printStderr ("Iter %3d Update %-15s Bound %22f (%15s)" % (itr, var_name, bound, diff))
    else:
        print ("Iter %3d Update %-15s Bound %22f (%15s)" % (itr, var_name, bound, diff))

def _debug_with_nothing (itr, var_value, var_name, W, K, topicMean, sigT, vocab, dtype, means, varcs, A, n):
    pass


"""Author: Brandon Trabucco, Copyright 2019
Serializes the conceptual captions dataset into tf records."""

import tensorflow as tf
import math
import numpy as np
import pickle as pkl
import os
import time
import collections
import sys
import threading
import multiprocessing
import FasterRCNN.inference
from FasterRCNN.inference import create_r101fpn_mask_rcnn_model_graph as create_model
from nltk.tokenize import wordpunct_tokenize as word_tokenize

tf.logging.set_verbosity(tf.logging.INFO)
tf.flags.DEFINE_string("image_folder", "./train_images", "Path to the image files.")
tf.flags.DEFINE_string("tsv_filename", "./Train_GCC-training.tsv", "Path to the TSV file.")
tf.flags.DEFINE_string("vocab_filename", "./word.vocab", "Path to the vocab file.")
tf.flags.DEFINE_string("output_dir", "./train_mask_tfrecords/", "Output data directory.")
tf.flags.DEFINE_integer("num_threads", 16, "Number of threads to use when serializing the dataset.")
tf.flags.DEFINE_integer("num_examples_per_file", 5096, "Number of threads to use when serializing the dataset.")
tf.flags.DEFINE_integer("queue_size", 64, "The number of examples to extract at once.")
tf.flags.DEFINE_integer("image_size", 600, "The size of the image smaller dimension.")
tf.flags.DEFINE_integer("start_at_file_index", 0, "The number of tfrecords to skip.")
tf.flags.DEFINE_string("visible_gpus", "0,0,1", "Which GPU to use.")
tf.flags.DEFINE_string("gpus_memory", "0.4,0.4,1", "Which GPU to use.")
FLAGS = tf.flags.FLAGS

TrainingExample = collections.namedtuple("TrainingExample", (
    "image_id", "image", "caption", "boxes", "masks", "labels",
    "r1", "r2", "r3", "r4", "m1", "m2", "m3", "m4"))

def rescale_shorter_edge(images, new_shorter_edge):
    def compute_longer_edge(height, width, new_shorter_edge):
        return tf.cast(width * new_shorter_edge / height, tf.int32)
    height, width = tf.shape(images)[0], tf.shape(images)[1]
    height_smaller_than_width = tf.less_equal(height, width)
    new_height_and_width = tf.cond(
        height_smaller_than_width,
        lambda: (new_shorter_edge, compute_longer_edge(height, width, new_shorter_edge)),
        lambda: (compute_longer_edge(width, height, new_shorter_edge), new_shorter_edge))
    return tf.image.resize_images(images, new_height_and_width)

class Extractor(object):
    """Helper class for extracting features from images."""
    def __init__(self, list_of_images, gpu_id=0, memory_fraction=1):
        """Creates handles to the TensorFlow computational graph."""
        self.list_of_images = list_of_images
        self.list_of_filenames = []
        for x in list_of_images:
            self.list_of_filenames.append(x.image)
        self.graph = tf.Graph()
        with self.graph.as_default():
            self.image_tensor = self.get_image_tensor(self.list_of_filenames, gpu_id)
            # Load the image tensor using a queue for massive parallelization.
            batch_size = tf.shape(self.image_tensor)[0]
            # Create a single TensorFlow Session for all image decoding calls.
            self.sess = tf.Session(
                graph=self.graph,
                config=tf.ConfigProto(
                    gpu_options=tf.GPUOptions(
                        visible_device_list=str(gpu_id),
                        per_process_gpu_memory_fraction=memory_fraction,
                        )))
            # Creates the mask rcnn model
            self.fetch_dict = create_model(self.image_tensor, self.sess)
            self.boxes = self.fetch_dict["boxes"]
            self.labels = self.fetch_dict["labels"]
            self.masks = self.fetch_dict["masks"]
            self.r1 = self.fetch_dict["region_features_1"]
            self.r2 = self.fetch_dict["region_features_2"]
            self.r3 = self.fetch_dict["region_features_3"]
            self.r4 = self.fetch_dict["region_features_4"]
            self.m1 = self.fetch_dict["mask_features_1"]
            self.m2 = self.fetch_dict["mask_features_2"]
            self.m3 = self.fetch_dict["mask_features_3"]
            self.m4 = self.fetch_dict["mask_features_4"]
            # store a handle to how many images were extracted so far
            self.num_images_so_far = tf.Variable(0)
            self.increment_by_batch_size = tf.assign(self.num_images_so_far, self.num_images_so_far + batch_size)
            self.sess.run(tf.variables_initializer([self.num_images_so_far]))
    def get_image_tensor(self, list_of_filenames, gpu_id):
        """Loads the images using the CPU and coppies them to the GPU.
        Args:
            list_of_bytes: a list of filename strings for encoded images.
        Returns:
            a batch representing an image tensor."""
        # Create a queue of strings
        dataset = tf.data.Dataset.from_tensor_slices(list_of_filenames)
        # Create a lazy image loading function to save memory.
        def lazy_load_image(image_filename):
            with tf.gfile.FastGFile(image_filename, "rb") as f:
                return f.read()
        # Convert from a filename to an image tensor.
        def filename_to_tensor(image_filename):
            image_bytes = tf.py_func(lazy_load_image, [image_filename], tf.string)
            image_tensor = tf.image.decode_jpeg(image_bytes, channels=3)
            return rescale_shorter_edge(image_tensor, FLAGS.image_size)
        # Convert each element to a tensor
        dataset = dataset.map(filename_to_tensor, num_parallel_calls=16)
        dataset = dataset.apply(tf.contrib.data.ignore_errors())
        # Create a batch with the specified size
        dataset = dataset.batch(1)
        def prepare_final_batch(image_tensor):
            return tf.transpose(tf.cast(image_tensor, tf.float32), [0, 3, 1, 2])
        # Copy the data to the GPU ahead of time
        dataset = dataset.map(prepare_final_batch, num_parallel_calls=16)
        dataset = dataset.apply(tf.contrib.data.prefetch_to_device("/gpu:0", buffer_size=2))
        image_tensor = dataset.make_one_shot_iterator().get_next()
        return image_tensor
    def get_num_extracted_so_far(self):
        """Extracts a single batch of image features.
        Returns:
            a integer representing the number of examples processed."""
        return int(self.sess.run(self.num_images_so_far))
    def extract(self):
        """Extracts a single batch of image features.
        Returns:
            a list of ImageMetadata objects."""
        position_before = self.get_num_extracted_so_far()
        (boxes, masks, labels,
            r1, r2, r3, r4, m1, m2, m3, m4, _) = self.sess.run([
            self.boxes, self.masks, self.labels,
            self.r1, self.r2, self.r3, self.r4,
            self.m1, self.m2, self.m3, self.m4,
            self.increment_by_batch_size])
        position_after = self.get_num_extracted_so_far()
        list_of_preextracted_images = []
        for i in range(position_after - position_before):
            image_meta = self.list_of_images[i + position_before]
            list_of_preextracted_images.append(
                TrainingExample(
                    image_id=image_meta.image_id,
                    image=image_meta.image,
                    caption=image_meta.caption,
                    boxes=boxes,
                    masks=masks,
                    labels=labels,
                    r1=r1, r2=r2, r3=r3, r4=r4,
                    m1=m1, m2=m2, m3=m3, m4=m4))
        return list_of_preextracted_images

def process_string(input_string, vocab):
    stage_two = word_tokenize(input_string.strip().lower())
    return [vocab[word] if word in vocab else vocab["<unk>"] for word in (["<s>"] + stage_two + ["</s>"])]

def get_all_training_examples(image_folder, tsv_filename):
    start_time = time.time()
    training_examples = []
    with open(tsv_filename, "r", encoding="utf-8") as tsv_file:
        for i, line in enumerate(tsv_file):
            training_examples.append(
                TrainingExample(
                    i, os.path.join(image_folder, "{0}.jpg".format(i)),
                    line.strip().split("\t")[0], None, None, None,
                        None, None, None, None, None, None, None, None))
    end_time = time.time()
    print("Finished loading training examples, took {0} seconds.".format(end_time - start_time))
    return training_examples

def to_sequence_example(metadata, image_bytes, vocab):
    num_regions = metadata.boxes.shape[0]
    return tf.train.SequenceExample(
        context=tf.train.Features(feature={
            "image/image_id": tf.train.Feature(int64_list=tf.train.Int64List(value=[metadata.image_id])),
            "image/data": tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_bytes])),
        }), feature_lists=tf.train.FeatureLists(feature_list={
            "image/caption": tf.train.FeatureList(feature=[tf.train.Feature(int64_list=tf.train.Int64List(
                value=[word])) for word in process_string(metadata.caption, vocab)]),
            "image/labels": tf.train.FeatureList(feature=[tf.train.Feature(int64_list=tf.train.Int64List(
                value=metadata.labels[i:(i + 1)].ravel())) for i in range(num_regions)]),
            "image/boxes": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.boxes[i, :].ravel())) for i in range(num_regions)]),
            "image/masks": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.masks[i, :, :].ravel())) for i in range(num_regions)]),
            "image/r1": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.r1[i, :].ravel())) for i in range(num_regions)]),
            "image/r2": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.r2[i, :].ravel())) for i in range(num_regions)]),
            "image/r3": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.r3[i, :].ravel())) for i in range(num_regions)]),
            "image/r4": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.r4[i, :].ravel())) for i in range(num_regions)]),
            "image/m1": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.m1[i, :].ravel())) for i in range(num_regions)]),
            "image/m2": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.m2[i, :].ravel())) for i in range(num_regions)]),
            "image/m3": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.m3[i, :].ravel())) for i in range(num_regions)]),
            "image/m4": tf.train.FeatureList(feature=[tf.train.Feature(float_list=tf.train.FloatList(
                value=metadata.m4[i, :].ravel())) for i in range(num_regions)]),
        })).SerializeToString()

def write_all_training_examples(training_examples, vocab_filename, output_dir, num_files_so_far,
        num_examples_per_file, queue_is_finished):
    start_time = time.time()
    print("Starting to write training examples.")
    with open(vocab_filename, "rb") as f:
        vocab = {word: i for i, word in enumerate(list(pkl.load(f)) + ["<s>", "</s>", "<unk>"])}
    while len(training_examples) > 0 or not all(queue_is_finished):
        num_examples_in_this_file = 0
        print("Starting to write a file.")
        output_filename = "%.5d.tfrecord" % (num_files_so_far[0])
        num_files_so_far[0] += 1
        writer = tf.python_io.TFRecordWriter(os.path.join(output_dir, output_filename))
        while num_examples_in_this_file < num_examples_per_file and (len(training_examples) > 0 or not all(queue_is_finished)):
            try:
                x = training_examples.pop(0)
            except Exception as e:
                time.sleep(.5)
                continue
            with tf.gfile.FastGFile(x.image, "rb") as f:
                image_bytes = f.read()
            writer.write(to_sequence_example(x, image_bytes, vocab))
            num_examples_in_this_file += 1
        writer.close()
        sys.stdout.flush()
        print("Finished writing a file.")
    end_time = time.time()
    print("Finished writing training examples, took {0} hours.".format((end_time - start_time) / 3600))

def launch_extractor_thread(thread_id, input_metadata, output_metadata, total_count, queue_is_finished, gpu_id, memory_fraction):
    print("Loading images for the Extractor.")
    try:
        extractor = Extractor(input_metadata, gpu_id=gpu_id, memory_fraction=memory_fraction)
    except Exception as e:
        print("An error was encountered when creating extractor: {0}".format(str(e)))
        exit()
    print("Starting the GPU thread for Extractor.")
    time_since_last_extract = time.time()
    num_extracted_so_far = 0
    while time.time() - time_since_last_extract < 3600.0 and num_extracted_so_far < total_count:
        if len(output_metadata) >= FLAGS.queue_size:
            time.sleep(.5)
            continue
        try:
            buffer = extractor.extract()
            output_metadata.extend(buffer)
            num_extracted_so_far += len(buffer)
            print("Extractor {0} extracted {1:07d} examples: {2:03.3f}% complete: eta {3:05.3f} hours.".format(
                thread_id,
                num_extracted_so_far, num_extracted_so_far / total_count * 100.0,
                (time.time() - time_since_last_extract) / len(buffer) * (total_count - num_extracted_so_far) / 3600.0 ))
            time_since_last_extract = time.time()
        except Exception as e:
            #print("An error was encountered when extracting extractor: {0}".format(str(e)))
            time.sleep(.1)
        queue_is_finished[thread_id] = len(input_metadata) == 0

if __name__ == "__main__":
    if not tf.gfile.IsDirectory(FLAGS.output_dir):
        tf.gfile.MakeDirs(FLAGS.output_dir)
    training_examples = get_all_training_examples(FLAGS.image_folder, FLAGS.tsv_filename)
    training_examples = training_examples[(FLAGS.start_at_file_index * FLAGS.num_examples_per_file):]
    manager = multiprocessing.Manager()
    extracted_training_examples = manager.list()
    num_files_so_far = [FLAGS.start_at_file_index]
    gpu_ids = [int(x) for x in FLAGS.visible_gpus.split(",")]
    memory_fractions = [float(x) for x in FLAGS.gpus_memory.split(",")]
    queue_is_finished = [False for i in gpu_ids]
    split_length = math.ceil(len(training_examples) / len(gpu_ids))
    split_training_examples = [training_examples[i:i + split_length] for i in range(
        0, len(training_examples), split_length)]
    extractor_threads = [
        multiprocessing.Process(
            target=launch_extractor_thread,
            args=(i, z, extracted_training_examples,
                len(z), queue_is_finished, x, y)) for i, x, y, z in zip(
                    list(range(len(gpu_ids))), gpu_ids, memory_fractions, split_training_examples)]
    serializer_threads = [
        threading.Thread(
            target=write_all_training_examples,
            args=(extracted_training_examples, FLAGS.vocab_filename, FLAGS.output_dir, num_files_so_far,
                FLAGS.num_examples_per_file, queue_is_finished)) for i in range(FLAGS.num_threads)]
    for thread in extractor_threads:
        thread.start()
    for thread in serializer_threads:
        thread.start()
    coord = tf.train.Coordinator()
    coord.join(extractor_threads + serializer_threads)


from scrapy import Spider
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.linkextractors import LinkExtractor
import scrapy
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
from UW_Madison.items import UwMadisonItem

# CrawlSpider
class Madison_courses( scrapy.Spider ):

    name = 'uw_madison_final'

    allowed_domains = ['wisc.edu']

    start_urls = [

            "http://guide.wisc.edu/courses/acct_i_s/",
            #"http://guide.wisc.edu/courses/zoology/",

    ]

    def  start_requests( self ):

        for u in self.start_urls:

            yield scrapy.Request( u, callback = self.parse_httpbin,
                    errback = self.errback_httpbin,
                    dont_filter = True )

    def parse_httpbin( self, response ):

        self.logger.info("Got successful response {}".format(response.url) )
        courses = response.css('span.courseblockcode::text').extract()
        #prerequisites  = response.css('p.courseblockextra.noindent.clearfix > span.cbextra-data > .bubblelink::text').extract()
        #prerequisites = response.xpath('//div[@class="cb-extras"]/p/span[@class="cbextra-data"]/a/@title').extract()

        prerequisites = response.xpath('//div[@class="cb-extras"]/p/span[@class="cbextra-data"]/a[1]/@title[1]').extract()

        title = response.css('div.sc_sccoursedescs > div.courseblock > p.courseblocktitle > strong::text').extract()
        unit = response.css('.courseblockcredits::text').extract()
        description =  response.css('.courseblockdesc::text').extract()

        final_courses = []
        final_prerequisites = []
        final_title = []
        final_unit = []
        final_description = []

        for course_set in courses:

            if course_set == '\n' or course_set == ' ':
                continue
            final_courses.append(course_set)

        for prerequisites_set in prerequisites:

            if prerequisites_set == '\n' or prerequisites_set == ' ' :
                continue
            final_prerequisites.append(prerequisites_set)

        for title_set in title:

            if title_set == '\n' or title_set == ' ':
                continue
            final_title.append(title_set)

        for unit_set in unit:

            if unit_set == '\n' or unit_set == ' ':
                continue
            final_unit.append(unit_set)

        for description_set in description:

            if description_set == '\n' or description_set == ' ':
                continue
            final_description.append(description_set)

        course_size = len(final_courses)
        items = []
        track_index = 0

        print("Course_Size:", course_size )
        print("prerequisites_size", len(final_prerequisites))
        print("title_size", len(final_title))
        print("Unit_size", len(final_unit))
        print("description", len(final_description))

        while track_index < course_size:

            print("Track_index", track_index )

            item = UwMadisonItem()

            #courses = response.css('span.courseblockcode::text').extract()[ track_index ].strip()
            #title = response.css('div.sc_sccoursedescs > div.courseblock > p.courseblocktitle > strong::text').extract()[ track_index ].strip()
            #unit = response.css('.courseblockcredits::text').extract()[ track_index ].strip()
            #description =  response.css('.courseblockdesc::text').extract()[ track_index ].strip()

            item['course'] = final_courses[ track_index ]
            item['title'] = final_title[ track_index ]
            item['unit'] = final_unit[ track_index ]
            item['description'] = final_description[ track_index ]

            try:

                item['prerequisites'] = final_prerequisites[ track_index ]

            except IndexError:

                item['prerequisites'] = 'None'

            items.append(item)
            track_index += 1

        return items

    def errback_httpbin( self, failure):

        # log all failures
        self.logger.error(repr(failure))

        # in case you want to do something special for some errors,
        # you may need the failure's type:

        if failure.check(HttpError):
            # These exception come from HttpError spider middleware
            # you can get the non-200 response
            response = failure.value.response
            self.logger.error("HttpError on %s", response.url )

        elif failure.check(DNSLookupError):
            # This is the original request
            request = failure.request
            self.logger.error('DNSLookupError on %s', request.url )

        elif failure.check(TimeoutError, TCPTimeOutError ):

            request = failure.request
            self.logger.error('TimeoutError on %s', request.url)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
import re
import requests
from bs4 import BeautifulSoup

def get_soup (url):
    page = requests.get(url).content.decode()
    soup = BeautifulSoup(page,'html.parser')
    return  soup

def get_nexturl (soup):
    pages = soup.find('div', {'class': 'page'}).find_all('a')
    url_exist = pages[-1].text
    if url_exist == '下一页':
        url = pages[-1].get('href')
        print('加载下一页：'+ url)

        return url
    else:
        return None

url = 'http://www.qizuang.com/city/'
soup = get_soup(url)
A_url = soup.find('div',{'class':'p2 clearfix'}).find_all('div',{'class':'con1 acti'})[4].find_all('div',{'class':'span1'})[0].find_all('li')
for url in A_url :

    url = url.find('a').get('href')
    url = url + '/company/?p=1'
            #print(url)
            #每个城市的公司
    try:
        while url:
             print(url)
             soup = get_soup(url)  # 已经获取到每个城市某一页的soup
             city_name = soup.find('div',{'class':'city'}).find('strong').text #获取城市名称
             #print(city_name)
             companies = soup.find('ul', {'class': 'gongslist'}).find_all('li')

             for company in companies:
                 u = company.find('a', {'class': 'company-link'}).get('href')
                 compage = get_soup(u)

                 name = compage.find('h1',{'class':'com-tit'}).text#获取公司名称

                 infos = compage.find('ul',{'class':'c-tab-send'})#获取公司信息：地址，电话1，电话2
                 att = infos.find('li', {'class': 'att'}).text
                 tel = infos.find('li',{'class':'tel'}).text
                 mob = infos.find('li',{'class':'mob'}).text
                 list = [name,att,mob,tel,]
                 print(list)

                 file_name = '/Users/zhongyi/Desktop/d/'+ city_name + '.csv'
                 f = open(file_name, 'a+', encoding='utf-8')
                 for info in list:
                     f.write(info)
                     f.write(',')
                 f.write('\n')
                 f.close()

             url = get_nexturl(soup)  # 获取下一页的url

    except AttributeError  :
        pass


#
# Pyserini: Reproducible IR research with sparse and dense representations
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
from pyserini.search.lucene.irst import LuceneIrstSearcher
from pyserini.search.lucene.ltr._base import SpacyTextParser
from typing import List
from transformers import AutoTokenizer
from pyserini.analysis import Analyzer, get_lucene_analyzer
from tqdm import tqdm

def normalize(scores: List[float]):
    low = min(scores)
    high = max(scores)
    width = high - low
    if width != 0:
        return [(s-low)/width for s in scores]
    else:
        return scores

def query_loader(data):
    queries = {}
    nlp = SpacyTextParser('en_core_web_sm', keep_only_alpha_num=True, lower_case=True)
    analyzer = Analyzer(get_lucene_analyzer())
    bert_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
    inp_file = open(data)
    ln = 0
    for line in tqdm(inp_file):
        ln += 1
        line = line.strip()
        if not line:
            continue
        fields = line.split('\t')
        if len(fields) != 2:
            print('Misformated line %d ignoring:' % ln)
            print(line.replace('\t', '<field delimiter>'))
            continue
        did, query = fields
        query_lemmas, query_unlemm = nlp.proc_text(query)
        analyzed = analyzer.analyze(query)
        for token in analyzed:
            if ' ' in token:
                print(analyzed)
        query_toks = query_lemmas.split()
        if len(query_toks) >= 0:
            query = {"raw" : query,
                "text": query_lemmas,
                "text_unlemm": query_unlemm,
                "analyzed": ' '.join(analyzed),
                "text_bert_tok": ' '.join(bert_tokenizer.tokenize(query.lower()))}
            queries[did] = query

        if ln % 10000 == 0:
            print('Processed %d queries' % ln)

    print('Processed %d queries' % ln)
    return queries

def baseline_loader(base_path: str):
    result_dic = {}
    with open(base_path, 'r') as f:
        for line in f:
            tokens = line.strip().split()
            t = tokens[0]
            doc_id = tokens[2]
            score = float(tokens[-2])
            if t in result_dic.keys():
                result_dic[t][0].append(doc_id)
                result_dic[t][1].append(score)
            else:
                result_dic[t] = [[doc_id], [score]]

    return result_dic

def sort_dual_list(pred: List[float], docs: List[str]):
    zipped_lists = zip(pred, docs)
    sorted_pairs = sorted(zipped_lists)

    tuples = zip(*sorted_pairs)
    pred, docs = [list(tuple) for tuple in tuples]

    pred.reverse()
    docs.reverse()
    return pred, docs

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='use ibm model 1 feature to rerank the base run file')
    parser.add_argument('--tag', type=str, default="ibm",
                        metavar="tag_name", help='tag name for resulting Qrun')
    parser.add_argument('--base-path', type=str, required=False,
                        metavar="path_to_base_run", help='path to base run')
    parser.add_argument('--tran-path', type=str, default="../ibm/ibm_model/text_bert_tok_raw",
                        metavar="directory_path", help='directory path to source.vcb target.vcb and Transtable bin file')
    parser.add_argument('--topics', type=str, help='path to query topics', required=True)
    parser.add_argument('--index', type=str, default="../ibm/index-msmarco-passage-ltr-20210519-e25e33f",
                        metavar="path_to_lucene_index", help='path to lucene index folder')
    parser.add_argument('--output', type=str, default="./ibm/runs/result-colbert-test-alpha0.3.txt",
                        metavar="path_to_reranked_run", help='the path to store reranked run file')
    parser.add_argument('--field-name', type=str, default="text_bert_tok",
                        metavar="type of field", help='type of field used for training')
    parser.add_argument('--alpha', type=float, default="0.3",
                        metavar="type of field", help='interpolation weight')
    parser.add_argument('--num-threads', type=int, default="12",
                        metavar="num_of_threads", help='number of threads to use')
    parser.add_argument('--max-sim', default=False, action="store_true",
                        help='whether we use max sim operator or avg instead')
    parser.add_argument('--hits', type=int, metavar='number of hits generated in runfile',
                        required=False, default=1000, help="Number of hits.")
    args = parser.parse_args()

    print('Using max sim operator or not:', args.max_sim)

    f = open(args.output, 'w')

    reranker = LuceneIrstSearcher(
        args.tran_path, args.index, args.field_name)
    queries = query_loader(args.topics)
    i = 0
    for topic in queries.keys():
        if i % 100 == 0:
            print(f'Reranking {i}')
        query_text_field = queries[topic][args.field_name]
        query_text = queries[topic]['raw']
        if args.base_path:
            baseline_dic = baseline_loader(args.base_path)
            docids, rank_scores, base_scores = reranker.rerank(
                query_text, query_text_field, baseline_dic[topic], args.max_sim)
        else:
            docids, rank_scores, base_scores = reranker.search(
                query_text, query_text_field, args.hits, args.max_sim)
        ibm_scores = normalize([p for p in rank_scores])
        base_scores = normalize([p for p in base_scores])

        interpolated_scores = [a * args.alpha + b * (1-args.alpha) for a, b in zip(base_scores, ibm_scores)]

        preds, docs = sort_dual_list(interpolated_scores, docids)
        i = i+1
        for index, (score, doc_id) in enumerate(zip(preds, docs)):
            rank = index + 1
            f.write(f'{topic} Q0 {doc_id} {rank} {score} {args.tag}\n')
    f.close()


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/bacpypes/task.py
# Compiled at: 2018-06-20 21:27:44
"""
Task
"""
import sys
from time import time as _time
from heapq import heapify, heappush, heappop
import itertools
from .singleton import SingletonLogging
from .debugging import DebugContents, Logging, ModuleLogger, bacpypes_debugging
_debug = 0
_log = ModuleLogger(globals())
_task_manager = None
_unscheduled_tasks = []
if sys.platform in ('linux2', 'darwin'):
    from .event import WaitableEvent

    class _Trigger(WaitableEvent, Logging):

        def handle_read(self):
            if _debug:
                _Trigger._debug('handle_read')
            data = self.recv(1)
            if _debug:
                _Trigger._debug('    - data: %r', data)

else:
    _Trigger = None

class _Task(DebugContents, Logging):
    _debug_contents = ('taskTime', 'isScheduled')

    def __init__(self):
        self.taskTime = None
        self.isScheduled = False
        return

    def install_task(self, when=None, delta=None):
        global _task_manager
        global _unscheduled_tasks
        if when is None and delta is not None:
            if not _task_manager:
                raise RuntimeError('no task manager')
            when = _task_manager.get_time() + delta
        if when is None:
            when = self.taskTime
        if when is None:
            raise RuntimeError("schedule missing, use zero for 'now'")
        self.taskTime = when
        if not _task_manager:
            _unscheduled_tasks.append(self)
        else:
            _task_manager.install_task(self)
        return

    def process_task(self):
        raise RuntimeError('process_task must be overridden')

    def suspend_task(self):
        if not _task_manager:
            _unscheduled_tasks.remove(self)
        else:
            _task_manager.suspend_task(self)

    def resume_task(self):
        _task_manager.resume_task(self)

    def __lt__(self, other):
        return id(self) < id(other)

class OneShotTask(_Task):

    def __init__(self, when=None):
        _Task.__init__(self)
        self.taskTime = when

class OneShotDeleteTask(_Task):

    def __init__(self, when=None):
        _Task.__init__(self)
        self.taskTime = when

@bacpypes_debugging
def OneShotFunction(fn, *args, **kwargs):

    class OneShotFunctionTask(OneShotDeleteTask):

        def process_task(self):
            OneShotFunction._debug('process_task %r %s %s', fn, repr(args), repr(kwargs))
            fn(*args, **kwargs)

    task = OneShotFunctionTask()
    if not _task_manager:
        _unscheduled_tasks.append(task)
    else:
        task.install_task(delta=0)
    return task

def FunctionTask(fn, *args, **kwargs):
    _log.debug('FunctionTask %r %r %r', fn, args, kwargs)

    class _FunctionTask(OneShotDeleteTask):

        def process_task(self):
            _log.debug('process_task (%r %r %r)', fn, args, kwargs)
            fn(*args, **kwargs)

    task = _FunctionTask()
    _log.debug('    - task: %r', task)
    return task

@bacpypes_debugging
class RecurringTask(_Task):
    _debug_contents = ('taskInterval', 'taskIntervalOffset')

    def __init__(self, interval=None, offset=None):
        if _debug:
            RecurringTask._debug('__init__ interval=%r offset=%r', interval, offset)
        _Task.__init__(self)
        self.taskInterval = interval
        self.taskIntervalOffset = offset

    def install_task(self, interval=None, offset=None):
        if _debug:
            RecurringTask._debug('install_task interval=%r offset=%r', interval, offset)
        if interval is not None:
            self.taskInterval = interval
        if offset is not None:
            self.taskIntervalOffset = offset
        if self.taskInterval is None:
            raise RuntimeError('interval unset, use ctor or install_task parameter')
        if self.taskInterval <= 0.0:
            raise RuntimeError('interval must be greater than zero')
        if not _task_manager:
            if _debug:
                RecurringTask._debug('    - no task manager')
            _unscheduled_tasks.append(self)
        else:
            now = _task_manager.get_time() + 1e-06
            interval = self.taskInterval / 1000.0
            if self.taskIntervalOffset:
                offset = self.taskIntervalOffset / 1000.0
            else:
                offset = 0.0
            if _debug:
                RecurringTask._debug('    - now, interval, offset: %r, %r, %r', now, interval, offset)
            self.taskTime = now - offset + interval - (now - offset) % interval + offset
            if _debug:
                RecurringTask._debug('    - task time: %r', self.taskTime)
            _task_manager.install_task(self)
        return

@bacpypes_debugging
def RecurringFunctionTask(interval, fn, *args, **kwargs):
    if _debug:
        RecurringFunctionTask._debug('RecurringFunctionTask %r %r %r', fn, args, kwargs)

    class _RecurringFunctionTask(RecurringTask):

        def __init__(self, interval):
            RecurringTask.__init__(self, interval)

        def process_task(self):
            if _debug:
                RecurringFunctionTask._debug('process_task %r %r %r', fn, args, kwargs)
            fn(*args, **kwargs)

    task = _RecurringFunctionTask(interval)
    if _debug:
        RecurringFunctionTask._debug('    - task: %r', task)
    return task

@bacpypes_debugging
def recurring_function(interval, offset=None):

    def recurring_function_decorator(fn):

        class _RecurringFunctionTask(RecurringTask):

            def process_task(self):
                if _debug:
                    recurring_function._debug('process_task %r', fn)
                fn()

            def __call__(self, *args, **kwargs):
                fn(*args, **kwargs)

        task = _RecurringFunctionTask(interval, offset)
        task.install_task()
        return task

    return recurring_function_decorator

class TaskManager(SingletonLogging):

    def __init__(self):
        global _task_manager
        if _debug:
            TaskManager._debug('__init__')
        self.tasks = []
        if _Trigger:
            self.trigger = _Trigger()
        else:
            self.trigger = None
        _task_manager = self
        self.counter = itertools.count()
        if _unscheduled_tasks:
            for task in _unscheduled_tasks:
                task.install_task()

        return

    def get_time(self):
        if _debug:
            TaskManager._debug('get_time')
        return _time()

    def install_task(self, task):
        if _debug:
            TaskManager._debug('install_task %r @ %r', task, task.taskTime)
        if task.taskTime is None:
            raise RuntimeError('task time is None')
        if task.isScheduled:
            self.suspend_task(task)
        heappush(self.tasks, (task.taskTime, next(self.counter), task))
        if _debug:
            TaskManager._debug('    - tasks: %r', self.tasks)
        task.isScheduled = True
        if self.trigger:
            self.trigger.set()
        return

    def suspend_task(self, task):
        if _debug:
            TaskManager._debug('suspend_task %r', task)
        for i, (when, n, curtask) in enumerate(self.tasks):
            if task is curtask:
                if _debug:
                    TaskManager._debug('    - task found')
                del self.tasks[i]
                task.isScheduled = False
                heapify(self.tasks)
                break
        else:
            if _debug:
                TaskManager._debug('    - task not found')
            if self.trigger:
                self.trigger.set()

    def resume_task(self, task):
        if _debug:
            TaskManager._debug('resume_task %r', task)
        self.install_task(task)

    def get_next_task(self):
        """get the next task if there's one that should be processed,
        and return how long it will be until the next one should be
        processed."""
        if _debug:
            TaskManager._debug('get_next_task')
        now = _time()
        task = None
        delta = None
        if self.tasks:
            when, n, nxttask = self.tasks[0]
            if when <= now:
                heappop(self.tasks)
                task = nxttask
                task.isScheduled = False
                if self.tasks:
                    when, n, nxttask = self.tasks[0]
                    delta = max(when - now, 0.0)
            else:
                delta = when - now
        return (task, delta)

    def process_task(self, task):
        if _debug:
            TaskManager._debug('process_task %r', task)
        task.process_task()
        if isinstance(task, RecurringTask):
            task.install_task()
        elif isinstance(task, OneShotDeleteTask):
            del task


import os
from pathlib import Path

from click.termui import style

GLOBAL_CONFIG_DIR = os.path.expanduser("~/.bento")
GLOBAL_CONFIG_PATH = os.path.join(GLOBAL_CONFIG_DIR, "config.yml")

DEFAULT_LOG_PATH = os.path.join(GLOBAL_CONFIG_DIR, "last.log")

TERMS_OF_SERVICE_KEY = "terms_of_service"
TERMS_OF_SERVICE_VERSION = "0.3.0"

BENTO_TEMPLATE_HASH = "3a04e0f0cd9243d20b1e33da7ac13115"

GREP_CONFIG_PATH = ".bento-grep.yml"

BENTO_EMAIL_VAR = "BENTO_EMAIL"
QA_TEST_EMAIL_ADDRESS = "test@returntocorp.com"
SUPPORT_EMAIL_ADDRESS = "support@r2c.dev"

ARGS_TO_EXCLUDE_FROM_METRICS = {"check": ["paths"]}

### messages ###

UPGRADE_WARNING_OUTPUT = f"""
╭─────────────────────────────────────────────╮
│  🎉 A new version of Bento is available 🎉  │
│  Try it out by running:                     │
│                                             │
│       {style("pip3 install --upgrade bento-cli", fg="blue")}      │
│                                             │
╰─────────────────────────────────────────────╯
"""

### Content ###

REGISTRATION_CONTENT_PATH = (
    Path(os.path.dirname(__file__)) / "resources" / "register-content.yml"
)

INIT_CONTENT_PATH = Path(os.path.dirname(__file__)) / "resources" / "init-content.yml"


from django.db.models import fields
from courses.models import UserDetail
from django import forms
from django.contrib.auth.models import User
from django.contrib.auth.forms import UserCreationForm
from django.forms import ModelForm

class UserRegisterForm(UserCreationForm):
    email = forms.EmailField()

    class Meta:
        model = User
        fields = ['username', 'email', 'password1', 'password2']

class UserDetailForm(ModelForm):
    class Meta:
        model = UserDetail
        fields = '__all__'

        widgets = {
            'telephone_number': forms.TextInput(attrs={'class': 'form-control'}),
            'address': forms.TextInput(attrs={'class': 'form-control'}),
            'profession': forms.TextInput(attrs={'class': 'form-control'}),
        }


import string
import nltk
import pymorphy2

with open('Морфемно-орфографический словарь А. Н. Тихонова.txt', encoding='windows-1251') as f:
    morphs = []
    for line in f:
        line = line.split(' | ')[1].strip().split()[0]
        if line[-1] in string.punctuation:
            line = line[:-1]
        if line[-1].isdigit():
            line = line[:-1]
        line = line.replace("'", '').split('/')
        for morph in line:
            if morph.startswith('-'):
                morph = morph[1:]
            if morph != '':
                morphs.append(morph)

def maxmatch(s):
    i = 0
    j = len(s)
    word_parts = []
    while True:
        if s[i:j] in morphs:
            word_parts.append(s[i:j])
            i = j
            j = len(s)
        else:
            j -= 1
        if i == j:
            break
    return '/'.join(word_parts)

word = ''
while True:
    word_or_text = input('Выберите режим работы: слово (с) или текст (т) (для выхода наберите "exit"): ')
    if word_or_text == 'exit':
        break
    if word_or_text == 'с':
        word = input('Введите слово: ')
        print(maxmatch(word))
    elif word_or_text == 'т':
        file_name = input('Укажите название файла: ')
        morph = pymorphy2.MorphAnalyzer()
        with open(file_name, encoding='utf-8') as f:
            text = f.read().lower()
            text_tokenized = nltk.word_tokenize(text)
            new_text = ''
            for t in text_tokenized:
                p_t = morph.parse(t)[0]
                if t.isalpha() and p_t.tag.POS not in {'PREP', 'CONJ', 'PRCL', "INTJ"}:
                    word_parts = maxmatch(t)
                    new_text += t + ' | ' + word_parts + '\n'
        annotation_file = input('Введите название файла, в который Вам нужно сохранить результаты аннотирования: ')
        with open(annotation_file, 'w', encoding='utf-8') as f:
            f.write(new_text[:-1])
            print('В файл {} были записаны результаты аннотирования.'.format(annotation_file))


from django.core.exceptions import ObjectDoesNotExist,MultipleObjectsReturned
from django.db import models
from django import forms
from quotes.models import UseCase
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.contrib.auth.signals import user_logged_in

def get_full_name(self):
    return self.first_name + ' ' + self.last_name

User.add_to_class("__str__", get_full_name)

class SpecFileForm(forms.Form):
    spec = forms.FileField()

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    use_cases = models.ManyToManyField(
        UseCase,
        related_name='profile_use_cases',
        blank=True
    )
    selected_use_case = models.ForeignKey(
        UseCase,
        on_delete=models.CASCADE,
        related_name='profile_selected_use_case',
        blank=True, null=True
    )
    phone = models.CharField(max_length=18, null=True, blank=True)
    birth_date = models.DateField(null=True, blank=True)

    def __str__(self):
        return self.user.first_name + ' ' + self.user.last_name

@receiver(post_save, sender=User)
def create_user_profile(sender, instance, created, **kwargs):
    if created:
        Profile.objects.create(user=instance)

@receiver(post_save, sender=User)
def save_user_profile(sender, instance, **kwargs):
    profile,created = Profile.objects.get_or_create(user=instance)
    instance.profile.save()


#!/usr/bin/env python

# server.py: the grlc server

from flask import Flask, request, jsonify, render_template, make_response
import logging

# grlc modules
import grlc.static as static
import grlc.utils as utils

# The Flask app
app = Flask(__name__)

# Set logging format
log_level = logging.DEBUG if static.LOG_DEBUG_MODE else logging.INFO
logging.basicConfig(level=log_level, format=static.LOG_FORMAT)
app.debug_log_format = static.LOG_FORMAT
glogger = logging.getLogger(__name__)

# Server routes
@app.route('/')
def grlc():
    resp = make_response(render_template('index.html'))
    return resp

@app.route('/api/local/local/<query_name>', methods=['GET'])
def query_local(query_name):
    return query(user=None, repo=None, query_name=query_name)

@app.route('/api/<user>/<repo>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/<query_name>.<content>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/commit/<sha>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/commit/<sha>/<query_name>.<content>', methods=['GET', 'POST'])
def query(user, repo, query_name, sha=None, content=None):
    glogger.info("-----> Executing call name at /{}/{}/{} on commit {}".format(user, repo, query_name, sha))
    glogger.debug("Request accept header: " + request.headers["Accept"])

    requestArgs = request.args
    acceptHeader = request.headers['Accept']
    requestUrl = request.url
    formData = request.form

    query_response, status, headers = utils.dispatch_query(user, repo, query_name,
                                                           sha=sha, content=content, requestArgs=requestArgs,
                                                           acceptHeader=acceptHeader,
                                                           requestUrl=requestUrl, formData=formData)

    if isinstance(query_response, list):
        query_response = jsonify(query_response)

    return make_response(query_response, status, headers)

@app.route('/api/local', methods=['GET'])
def api_docs_local():
    return api_docs(user='local', repo='local', sha=None)

@app.route('/api/local/local/spec', methods=['GET'])
def swagger_spec_local():
    return swagger_spec(user=None, repo=None, sha=None, content=None)

@app.route('/api/<user>/<repo>', strict_slashes=False)
@app.route('/api/<user>/<repo>/api-docs')
@app.route('/api/<user>/<repo>/commit/<sha>')
@app.route('/api/<user>/<repo>/commit/<sha>/api-docs')
def api_docs(user, repo, sha=None):
    return render_template('api-docs.html', user=user, repo=repo, sha=sha)

@app.route('/api/<user>/<repo>/spec', methods=['GET'])
@app.route('/api/<user>/<repo>/commit/<sha>/spec')
def swagger_spec(user, repo, sha=None, content=None):
    glogger.info("-----> Generating swagger spec for /{}/{} on commit {}".format(user, repo, sha))

    swag = utils.build_swagger_spec(user, repo, sha, static.SERVER_NAME)

    if 'text/turtle' in request.headers['Accept']:
        resp_spec = make_response(utils.turtleize(swag))
        resp_spec.headers['Content-Type'] = 'text/turtle'
    else:
        resp_spec = make_response(jsonify(swag))
        resp_spec.headers['Content-Type'] = 'application/json'

    resp_spec.headers['Cache-Control'] = static.CACHE_CONTROL_POLICY  # Caching JSON specs for 15 minutes

    glogger.info("-----> API spec generation for /{}/{} on commit {} complete".format(user, repo, sha))
    return resp_spec

if __name__ == '__main__':
    app.run(host=static.DEFAULT_HOST, port=static.DEFAULT_PORT, debug=static.LOG_DEBUG_MODE)


from random import seed, shuffle
import numpy as np
import time
import json

# global
DATA_SIZE = 64

''' Random set '''
seed(time.time())
nums = list(range(1, DATA_SIZE+1))
shuffle(nums)
colors = [(0, 0.5 + n/(2*DATA_SIZE), 1 - n/(2*DATA_SIZE), 1) for n in nums]
data = [[nums[i], colors[i]] for i in range(len(nums))]

fo = open("random_data.json", "w")
json.dump(obj=data, fp=fo)

''' Decending set '''
nums = list(range(DATA_SIZE, 0, -1))
colors = [(0, 0.5 + n/(2*DATA_SIZE), 1 - n/(2*DATA_SIZE), 1) for n in nums]
data = [[nums[i], colors[i]] for i in range(len(nums))]

fo = open("decending_data.json", "w")
json.dump(obj=data, fp=fo)

''' Read data '''
# fr = open("random_data.json")
# raw_data = json.load(fp=fr)
# print(raw_data)


"""Shared Entity definition for UniFi Protect Integration."""
from __future__ import annotations

from pyunifiprotect.data import ProtectAdoptableDeviceModel, StateType

from homeassistant.core import callback
import homeassistant.helpers.device_registry as dr
from homeassistant.helpers.entity import DeviceInfo, Entity, EntityDescription

from .const import DEFAULT_ATTRIBUTION, DEFAULT_BRAND, DOMAIN
from .data import ProtectData

class ProtectDeviceEntity(Entity):
    """Base class for UniFi protect entities."""

    _attr_should_poll = False

    def __init__(
        self,
        data: ProtectData,
        device: ProtectAdoptableDeviceModel | None = None,
        description: EntityDescription | None = None,
    ) -> None:
        """Initialize the entity."""
        super().__init__()
        self.data: ProtectData = data

        if device and not hasattr(self, "device"):
            self.device: ProtectAdoptableDeviceModel = device

        if description and not hasattr(self, "entity_description"):
            self.entity_description = description
        elif hasattr(self, "entity_description"):
            description = self.entity_description

        if description is None:
            self._attr_unique_id = f"{self.device.id}"
            self._attr_name = f"{self.device.name}"
        else:
            self._attr_unique_id = f"{self.device.id}_{description.key}"
            name = description.name or ""
            self._attr_name = f"{self.device.name} {name.title()}"

        self._attr_attribution = DEFAULT_ATTRIBUTION
        self._async_set_device_info()
        self._async_update_device_from_protect()

    async def async_update(self) -> None:
        """Update the entity.

        Only used by the generic entity update service.
        """
        await self.data.async_refresh()

    @callback
    def _async_set_device_info(self) -> None:
        self._attr_device_info = DeviceInfo(
            name=self.device.name,
            manufacturer=DEFAULT_BRAND,
            model=self.device.type,
            via_device=(DOMAIN, self.data.api.bootstrap.nvr.mac),
            sw_version=self.device.firmware_version,
            connections={(dr.CONNECTION_NETWORK_MAC, self.device.mac)},
            configuration_url=self.device.protect_url,
        )

    @callback
    def _async_update_device_from_protect(self) -> None:
        """Update Entity object from Protect device."""
        if self.data.last_update_success:
            assert self.device.model
            devices = getattr(self.data.api.bootstrap, f"{self.device.model.value}s")
            self.device = devices[self.device.id]

        self._attr_available = (
            self.data.last_update_success and self.device.state == StateType.CONNECTED
        )

    @callback
    def _async_updated_event(self) -> None:
        """Call back for incoming data."""
        self._async_update_device_from_protect()
        self.async_write_ha_state()

    async def async_added_to_hass(self) -> None:
        """When entity is added to hass."""
        await super().async_added_to_hass()
        self.async_on_remove(
            self.data.async_subscribe_device_id(
                self.device.id, self._async_updated_event
            )
        )


import torch
import torchvision
import argparse
import cv2
import numpy as np
import sys
import os
sys.path.append('./')
import random
#--model_path F:\Masters2019\lh\dl\objectdetection\faster_r_cnn\transdata\output\myout\model_19.pth
def get_args():
    parser = argparse.ArgumentParser(description='Pytorch Faster-rcnn Detection')

    parser.add_argument('--model_path', type=str, default='.'+os.sep+'model_19.pth', help='model path')
    parser.add_argument('--model', default='fasterrcnn_resnet50_fpn', help='model')
    parser.add_argument('--score', type=float, default=0.01, help='objectness score threshold')
    parser.add_argument('--gpu', type=str, default='0', help='gpu cuda')

    args = parser.parse_args()

    return args

def test(img_path, anno_path):
    args = get_args()

    names = {'0': 'background', '1': 'core', '2': 'coreless'}

    # 创建输出文件
    core_file = open('det_test_带电芯充电宝.txt','w')
    coreless_file = open('det_test_不带电芯充电宝.txt','w')

    # Model creating
    print("Creating model")
    model = torchvision.models.detection.__dict__[args.model](num_classes=3, pretrained=False)
    if args.gpu =='0':
        model = model.cpu()
    else:
        model = model.cuda()

    model.eval()
    if args.gpu =='0':
        save = torch.load(args.model_path,map_location='cpu')
    else:
        save = torch.load(args.model_path)

    model.load_state_dict(save['model'])

    # 获取要测试的图像文件夹下所有图像文件的文件名
    img_list = os.listdir(img_path)
    # 遍历所有图像
    imgint = 0
    for img_file in img_list:
        imgint = imgint + 1
        print(imgint)

        src_img = cv2.imread(os.path.join(img_path,img_file))
        img = cv2.cvtColor(src_img,cv2.COLOR_BGR2RGB)

        if args.gpu == '0':
            img_tensor = torch.from_numpy(img / 255.).permute(2, 0, 1).float()
        else:
            img_tensor = torch.from_numpy(img / 255.).permute(2, 0, 1).float().cuda()
        out = model([img_tensor])
        boxes = out[0]['boxes']
        labels = out[0]['labels']
        scores = out[0]['scores']

        for idx in range(boxes.shape[0]):
            if scores[idx] > args.score:
                x1, y1, x2, y2 = boxes[idx][0].detach().numpy(), boxes[idx][1].detach().numpy(), boxes[idx][2].detach().numpy(), boxes[idx][3].detach().numpy()
                name = names.get(str(labels[idx].item()))

                # 按照要求的格式，将结果写入到相应的输出文件中
                # 文件名不带后缀，置信度保留小数点后3位，坐标保留一位小数点
                # 注2： Windows下可能需要修改成 \n\r作为换行符！！！
                str_write = "%s %.3f %.1f %.1f %.1f %.1f\n" % (img_file[:-4], scores[idx].detach().numpy(), x1, y1, x2, y2)

                if name == 'core':
                    core_file.write(str_write)
                elif name == 'coreless':
                    coreless_file.write(str_write)

    core_file.close()
    coreless_file.close()

if __name__ == '__main__':
    test('D:\\sysfile\\desktop\\mlbighomework\\finally\\Image_test\\', None)


from __future__ import print_function

import unittest
from backtest import Backtest
from data import SecurityMasterDataHandler
from execution import *
from portfolio import Portfolio
from order import *

class StrategiesTest(unittest.TestCase):

    def setUp(self):

        import sys

        if '../../SqlConnWraper' not in sys.path:
            sys.path.append('../../SqlConnWraper')

        if '../Strategies' not in sys.path:
            sys.path.append('../Strategies')

        from BuildSQLConnection import build_sql_conn

        self.cxcn = build_sql_conn('../config.ini')

    def test_mac_strategy(self):
        """
        Unit test for Moving Average Crossover Strategy

        :return:
        """

        from mac import MovingAverageCrossStrategy

        symbol_list = ['AAPL']
        initial_capital = 100000.0
        heartbeat = 0.0

        data_handler = SecurityMasterDataHandler(symbol_list, self.cxcn)

        start_date = data_handler.start_dt

        order_method = EquityWeightOrder

        backtest = Backtest(
            symbol_list, initial_capital, heartbeat,
            start_date, data_handler, SimulatedExecutionHandlerWithCommision,
            Portfolio, MovingAverageCrossStrategy, order_method
        )

        backtest.simulate_trading()

        print(backtest.portfolio.summary_stats)

        self.assertAlmostEqual(backtest.portfolio.summary_stats['Total Return'],
                               -11.908907924000056, places=7, msg=None, delta=None)

        self.assertAlmostEqual(backtest.portfolio.summary_stats['Sharpe Ratio'],
                               0.17652509997281265, places=7, msg=None, delta=None)

        self.assertAlmostEqual(backtest.portfolio.summary_stats['Max Drawdown'],
                               74.337676325221068, places=7, msg=None, delta=None)

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='HotWordCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('hot_word', models.CharField(max_length=50)),
                ('counter', models.IntegerField()),
            ],
        ),
        migrations.CreateModel(
            name='SearchHistory',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('user_id', models.IntegerField()),
                ('key_word', models.CharField(max_length=50)),
                ('created_at', models.DateTimeField(default=datetime.datetime(2016, 7, 22, 17, 21, 3, 335076))),
            ],
        ),
    ]


import numpy as np
import healpy as hp
from astropy import units as u
from astropy import constants as c

# Longest baseline
b_max = 1000*u.m
# Highest frequency
nu_max = 200.*u.MHz
# Lowest frequency
nu_min = 100.*u.MHz
# Highest resolution
res_max = np.degrees(1./(b_max * nu_max/c.c).to(u.dimensionless_unscaled).value)*60.
# Oversampling of synthesizbeam
pixel_oversamp = 2.

# Calculate FoV
FoV = np.degrees(((c.c/nu_min)/(14.*u.m)).to(u.dimensionless_unscaled).value)
FoV_oversamp = 2.

# Calculate number of healpix pixels
nside = 1024
npix_hp = hp.nside2npix(nside)
res = hp.nside2resol(nside,arcmin=True)

area_arcmin2 = np.power(res_max/pixel_oversamp,2)

# Calculate number of pixels in 12 hour equatorial map
sq_arcmin_eq = (FoV*60.*FoV_oversamp) * (180.*60.)
npix_eq = sq_arcmin_eq/area_arcmin2

# Calcualte number of pixels in 1 hour snapshot
sq_arcmin_snap = np.power(FoV*60.*FoV_oversamp,2)
npix_snap = sq_arcmin_snap/area_arcmin2

# Number of channels
nchan = 200.

# Conversion to size
bytes_per_float = 8
mb_per_vox = bytes_per_float / 1e6

size_eq = npix_eq * nchan * mb_per_vox
size_snap = npix_snap * nchan * mb_per_vox


from typing import (
    Any,
    List,
    Optional,
    Tuple,
    Union,
)

import pytest

from saltyrtc.server import (
    DEFAULT_DROP_REASON,
    INITIATOR_ADDRESS,
    SERVER_ADDRESS,
    Address,
    AddressType,
    ClientAddress,
    CloseCode,
    DropReason,
    InitiatorAddress,
    MessageError,
    ResponderAddress,
    ServerAddress,
    validate_drop_reason,
    validate_responder_id,
    validate_subprotocol,
    validate_subprotocols,
)

class TestDropReason:
    """
    A drop reason must be a :class:`CloseCode` (or an integer when
    parsed) and be allowed as a drop reason.
    """
    invalid_reasons = [
        '1',
        b'\x01',
        'meow',
        float(3.3333),
        None,
        object(),
        999,
        1001,
        3000,
    ]  # type: List[Any]
    valid_reasons = list(DropReason)  # type: List[DropReason]

    @pytest.mark.parametrize('reason', invalid_reasons)
    def test_invalid_reason(self, reason: Any) -> None:
        with pytest.raises(ValueError) as exc_info:
            DropReason(reason)
        assert 'is not a valid DropReason' in str(exc_info.value)

    def test_default_reason(self):
        assert DEFAULT_DROP_REASON == DropReason.drop_by_initiator

    @pytest.mark.parametrize('reason', valid_reasons)
    def test_reason_is_valid_close_code(self, reason: DropReason) -> None:
        CloseCode(reason)

    @pytest.mark.parametrize('reason', [reason for reason in invalid_reasons
                                        if reason is not None])
    def test_received_invalid_reason(self, reason: Any) -> None:
        with pytest.raises(MessageError) as exc_info:
            validate_drop_reason(reason)
        assert 'Invalid drop reason' in str(exc_info.value)

    @pytest.mark.parametrize('reason', valid_reasons + [None])
    def test_received_valid_reason(self, reason: Optional[int]) -> None:
        assert isinstance(validate_drop_reason(reason), DropReason)

class TestAddress:
    """
    An address must be in the valid range of SaltyRTC addresses.
    """
    invalid_addresses = [
        '1',
        b'\x01',
        'meow',
        float(3.3333),
        None,
        object(),
        -1,
        0x100,
    ]  # type: List[Any]
    valid_addresses = [0x00, 0x01, 0x02, 0x30, 0xff]  # type: List[int]
    addresses_type_mapping = [
        (0x00, AddressType.server),
        (0x01, AddressType.initiator),
        (0x02, AddressType.responder),
        (0x30, AddressType.responder),
        (0xff, AddressType.responder),
    ]

    @pytest.mark.parametrize('address', invalid_addresses)
    def test_invalid_address(self, address: Any) -> None:
        with pytest.raises(ValueError) as exc_info:
            Address(address)
        assert 'Invalid address' in str(exc_info.value)

    @pytest.mark.parametrize('address', valid_addresses)
    def test_valid_address_equality(self, address: int) -> None:
        assert Address(address) == address

    @pytest.mark.parametrize('address,expected', addresses_type_mapping)
    def test_to_type(self, address: int, expected: AddressType) -> None:
        address = Address(address)
        assert address.type == expected

class TestServerAddress:
    """
    A server address must be exactly 0x00.
    """
    def test_create(self) -> None:
        assert ServerAddress() == SERVER_ADDRESS

    def test_equality(self) -> None:
        assert SERVER_ADDRESS == 0x00
        assert SERVER_ADDRESS == Address(0x00)

class TestClientAddress:
    """
    A client address must be an integer in the valid range of client
    ids (0x01..0xff).
    """
    invalid_addresses = TestAddress.invalid_addresses + [0x00]  # type: List[Any]
    valid_addresses = [0x01, 0x02, 0x30, 0xff]  # type: List[int]

    @pytest.mark.parametrize('address', invalid_addresses)
    def test_invalid_address(self, address: Any) -> None:
        with pytest.raises(ValueError) as exc_info:
            ClientAddress(address)
        assert 'Invalid address' in str(exc_info.value)

    @pytest.mark.parametrize('address', valid_addresses)
    def test_valid_address_equality(self, address: int) -> None:
        assert ClientAddress(address) == address

class TestInitiatorAddress:
    """
    An initiator address must be exactly 0x01.
    """
    def test_create(self) -> None:
        assert InitiatorAddress() == INITIATOR_ADDRESS

    def test_equality(self) -> None:
        assert INITIATOR_ADDRESS == 0x01
        assert INITIATOR_ADDRESS == Address(0x01)
        assert INITIATOR_ADDRESS == ClientAddress(0x01)

    def test_to_type(self) -> None:
        assert INITIATOR_ADDRESS.type == AddressType.initiator

class TestResponderAddress:
    """
    A responder address must be an integer in the valid range of
    responder ids (0x02..0xff).
    """
    invalid_addresses = TestAddress.invalid_addresses + [0x00, 0x01]  # type: List[Any]
    valid_addresses = [0x02, 0x30, 0xff]  # type: List[int]

    @pytest.mark.parametrize('address', invalid_addresses)
    def test_invalid_address(self, address: Any) -> None:
        with pytest.raises(ValueError) as exc_info:
            ResponderAddress(address)
        assert 'Invalid address' in str(exc_info.value)

    @pytest.mark.parametrize('address', valid_addresses)
    def test_valid_address_equality(self, address: int) -> None:
        assert ResponderAddress(address) == address

    @pytest.mark.parametrize('id_', valid_addresses)
    def test_received_invalid_id(self, id_: int) -> None:
        validate_responder_id(id_)

    @pytest.mark.parametrize('id_', invalid_addresses)
    def test_received_valid_id(self, id_: Any) -> None:
        with pytest.raises(MessageError) as exc_info:
            validate_responder_id(id_)
        assert 'Invalid responder id' in str(exc_info.value)

class TestSubprotocol:
    """
    A sub-protocol must be a string.
    """
    invalid_subprotocols = [b'h3h3', 1, float(3.3333)]  # type: List[any]
    valid_subprotocols = ['', 'kittie-protocol-3000']  # type: List[str]

    @pytest.mark.parametrize('subprotocol', invalid_subprotocols)
    def test_received_invalid_subprotocol(self, subprotocol: Any) -> None:
        with pytest.raises(MessageError) as exc_info:
            validate_subprotocol(subprotocol)
        assert 'Invalid sub-protocol' in str(exc_info.value)

    @pytest.mark.parametrize('subprotocol', valid_subprotocols)
    def test_received_valid_subprotocol(self, subprotocol: str) -> None:
        validate_subprotocol(subprotocol)

class TestSubprotocols:
    """
    A sub-protocol must contain a list of strings.
    """
    not_list_like = (TestSubprotocol.valid_subprotocols +
                     TestSubprotocol.invalid_subprotocols)  # type: List[Any]
    invalid_subprotocols_list = [TestSubprotocol.invalid_subprotocols]

    @pytest.mark.parametrize('subprotocols', not_list_like)
    def test_received_invalid_not_list_like(self, subprotocols: Any) -> None:
        with pytest.raises(MessageError) as exc_info:
            validate_subprotocols(subprotocols)
        assert 'Sub-protocols not list or tuple' in str(exc_info.value)

    @pytest.mark.parametrize('subprotocols', invalid_subprotocols_list)
    def test_received_invalid_subprotocols(
            self,
            subprotocols: Union[List[Any], Tuple[Any]],
    ) -> None:
        with pytest.raises(MessageError) as exc_info:
            validate_subprotocols(subprotocols)
        assert 'Invalid sub-protocol' in str(exc_info.value)

    @pytest.mark.parametrize('subprotocols', [
        tuple(TestSubprotocol.valid_subprotocols),
        list(TestSubprotocol.valid_subprotocols)
    ])
    def test_received_valid_subprotocols(
            self,
            subprotocols: Union[List[str], Tuple[str]],
    ) -> None:
        validate_subprotocols(subprotocols)


from lxml import etree

NS = {"w": "http://www.mediawiki.org/xml/export-0.5/"}
PAGE_TAG = "{%s}page" % NS["w"]

def extract_pages(f):
    """Extract pages from Wikimedia database dump.

    Returns
    -------
    pages : iterable over (int, str, str)
        Generates (page_id, title, content) triples.
    """
    for event, elem in etree.iterparse(f, events=["end"]):
        if elem.tag == PAGE_TAG:
            text = elem.xpath("./w:revision/w:text", namespaces=NS)[0].text \
                    or ""   # may be None
            yield (int(elem.xpath("./w:id", namespaces=NS)[0].text),
                   elem.xpath("./w:title", namespaces=NS)[0].text,
                   text)

            # http://www.ibm.com/developerworks/xml/library/x-hiperfparse/
            # We do this only for <page>s, since we need to inspect the
            # ./revision/text element. That shouldn't matter since the pages
            # comprise the bulk of the file.
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

if __name__ == "__main__":
    import sys

    for pageid, title, text in extract_pages(sys.stdin):
        print("%d '%s' (%s)" % (pageid, title, text[:40].replace("\n", "_")))


#!/usr/bin/python3
# implement TF model which will be used to classify MNIST digits using keras.
#
# Name: R. Melton
# email: rnm@pobox.com
# date: 1/3/2021
# based on: https://www.youtube.com/watch?v=bee0GrKBCrE
# you will need sudo apt install nvidia-cuda-toolkit

import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt
import numpy as np
import time,sys,os,platform,distro,vers
from tensorflow.python.client import device_lib

#try to install TF 2 if it will run on this HW
# try:
#   tensorflow_version   #<< this function only exists in google colab VM
#   print('installed TF 2.x')
# except Exception:
#   print('could not install TF 2')
#   pass

if __name__ == '__main__':
  vers.show_versions_info()

#see if GPU is available
physical_devices = tf.config.experimental.list_physical_devices('GPU')
print('num GPUs ',len(physical_devices))
# if len(physical_devices) > 0:
#   tf.config.experimental.set_memory_growth(physical_devices[0],True)

objects = tf.keras.datasets.mnist    # many images of digits 0-9
(training_images, training_labels),(test_images,test_labels) = objects.load_data()

device_lib.list_local_devices()

#print some of the digit images
for i in range(9):
  plt.subplot(330+1+i)
  plt.imshow(training_images[i])

# print dimensions for training data
print(training_images.shape)

#print 1st training image of one digit
print(training_images[0])

#now normalize all training and test input values to 0 - 1
training_images = training_images / 255.0
test_images = test_images / 255.0

m = tf.keras.models.Sequential([tf.keras.layers.Flatten(input_shape=(28,28)),
  tf.keras.layers.Dense(128,activation='relu'),
  tf.keras.layers.Dense(10,activation=tf.nn.softmax)])

m.summary() #show summary

m.compile(optimizer = tf.keras.optimizers.Adam(),
  loss = 'sparse_categorical_crossentropy',
  metrics=['accuracy'])

#train the model
t = time.time()
m.fit(training_images,training_labels,epochs=5,shuffle=True) #5 itterations over all data

print(f'training duration: {time.time() - t}s')
start_test_time = time.time()
m.evaluate(test_images,test_labels)
print(f'test duration: {time.time() - start_test_time}s')
# write out one hot data values so we know what they are.
# test_images.class_indices

#show first image from test data
plt.imshow(test_images[0])
prediction=m.predict(test_images)  # do all test images
print('predicted number0: ',np.argmax(prediction[0]))
print(': ',prediction[0])
print('predicted number1: ',np.argmax(prediction[1]))
print(': ',prediction[1])
print('predicted number2: ',np.argmax(prediction[2]))
print(': ',prediction[2])
print('predicted number3: ',np.argmax(prediction[3]))
print(': ',prediction[3])
print('predicted number4: ',np.argmax(prediction[4]))
print(': ',prediction[4])


__author__ = 'haibane'
import pymongo
import sys
conn = pymongo.MongoClient(host=["mongodb://192.168.1.42:27001",
                                 "mongodb://192.168.1.42:27002",
                                 "mongodb://192.168.1.42:27003"], replicaSet="abc", w=1, j=True)
db = conn.m101
people = db.people
try:
    print("inserting")
    people.insert({"name": "andrew", "favority_color": "blue"})
except:
    print("Unexcepted error:", sys.exc_info()[0])
print("completed inserting")


import argparse

from . import encode

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("N",
                        help="Display the namehash for the given number",
                        type=int)
    args = parser.parse_args()
    print(encode(args.N))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

from mongo_cache import MongoCache
from mongo_queue import MongoPackage
from pypi_parser import PackageParser
from downloader import Downloader
import threading
import multiprocessing
from urllib import parse
from os import path
import time
SEED_URL = 'https://pypi.python.org'
SLEEP_TIME = 1

def threaded_crawler(seed_url, delay = 5, cache = None, scrape_callback = None, user_agent = 'wswp', proxies = None, num_retries = 1, max_threads = 10, timeout = 60):
    '''
    Crawl using multiple threads
    '''
    package_queue = MongoPackage()
    #crawl_queue.clear()
    #crawl_queue.push(seed_url)
    D = Downloader(delay, user_agent, proxies, num_retries, timeout, cache = cache)

    def process_queue():
        while True:
            try:
                url = package_queue.pop()
                #url is like '/pypi/xxx/'
                full_url = parse.urljoin(seed_url, url + 'json')
                print(full_url)
            except KeyError:
                break
            else:
                html = D(full_url)
                '''
                Here handle the html
                '''
                with open(path.join('../results/pypi_packinfo', url.split('/')[-2] + '.json'), 'wb') as f:
                    f.write(html)
                package_queue.complete(url)
    threads = []
    while threads or package_queue:
        # the crawl is still active
        for th in threads:
            if not th.is_alive():
                # remove the stopped threads
                threads.remove(th)
        while len(threads) < max_threads and package_queue.peek():
            # can start some more threads
            th = threading.Thread(target = process_queue, daemon = True)
            th.start()
            threads.append(th)
        # sleep temporarily so CPU can focus execution on other threads
        time.sleep(SLEEP_TIME)

def process_crawler(args, **kwargs):
    num_cpus = multiprocessing.cpu_count()
    print('Starting %s processes' % num_cpus)
    processes = []
    for i in range(num_cpus):
        p = multiprocessing.Process(target = threaded_crawler, args=(args,), kwargs = kwargs)
        p.start()
        processes.append(p)
    for p in processes:
        p.join()

def normalize(seed_url, link):
    '''
    Normalize this URL by removing hash and adding domain
    seed_url: the domain
    link: the relative path
    '''
    link, _ = parse.urldefrag(link)
    return parse.urljoin(seed_url, link)

def generate_packinfo():
    cache = MongoCache()
    cache.clear()
    process_crawler(SEED_URL, cache = cache)

if __name__ == '__main__':
    generate_packinfo()


"""Admin for news-related models."""

import wtforms

from pygotham.admin.utils import model_view
from pygotham.news import models

__all__ = ('AnnouncementModelView',)

AnnouncementModelView = model_view(
    models.Announcement,
    'Announcements',
    'News',
    column_default_sort='published',
    column_list=('title', 'published', 'active'),
    form_columns=('title', 'content', 'active', 'published'),
    form_overrides={
        'published': wtforms.DateTimeField,
    },
)


#!/usr/bin/env python

from os import getenv
import paho.mqtt.client as mqtt
import json
import bot

TALK_TO_BOT = 'uldis:talk_to_bot'
INTERRUPT = 'uldis:interrupt'

INTENT = TALK_TO_BOT

def on_connect(mqtt_client, userdata, flags, rc):
    print('[snips] connected')
    mqtt_client.subscribe('hermes/intent/#')

slot_name_is = lambda x: lambda s: s['slotName'] == x
first_slot_value = lambda s: s['value']['value']

CONTINUE_SESSION = 'hermes/dialogueManager/continueSession'

def continue_session(id, text, bot_name):
    return json.dumps({
        'sessionId': id,
        'text': text,
        'intentFilter': [INTERRUPT],
        'customData': bot_name,
        'sendIntentNotRecognized': False
    })

def on_message(client, _, msg):
    data = json.loads(msg.payload)
    intent_name = data['intent']['intentName']
    sessionId = data['sessionId']

    if (intent_name == TALK_TO_BOT):
        slot = filter(
            slot_name_is('bot'),
            data['slots']
        )[0]
        bot_name = first_slot_value(slot)

        print('[snips] user wants to talk to {}'.format(bot_name))
        on_continue = lambda response: snips_client.publish(
            CONTINUE_SESSION,
            continue_session(
                sessionId,
                response,
                bot_name
            )
        )
        bot.start(bot_name, on_continue)
    else:
        debug(data)

def on_log(client, userdata, level, buf):
    print('[snips] {}'.format([client, userdata, level, buf]))

def debug(data):
    slots = data['slots']
    intent_name = data['intent']['intentName']
    print('[snips] Intent {}'.format(intent_name))
    for slot in slots:
        slot_name = slot['slotName']
        raw_value = slot['rawValue']
        value = slot['value']['value']
        print('[snips] Slot {} -> \n\tRaw: {} \tValue: {}'.format(slot_name, raw_value, value))

def start():
    MQTT_HOST = getenv('MQTT_HOST', 'pi')
    MQTT_PORT = getenv('MQTT_PORT', 1883)
    mqtt_client = mqtt.Client()
    if getenv('DEBUG'):
        mqtt_client.on_log = on_log
    mqtt_client.on_connect = on_connect
    mqtt_client.on_message = on_message
    mqtt_client.connect(MQTT_HOST, MQTT_PORT)
    mqtt_client.loop_forever()

if __name__ == '__main__':
    print('[snips] starting...')
    start()


# -*- coding: utf-8 -*-
"""
Tencent is pleased to support the open source community by making 蓝鲸智云(BlueKing) available.
Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.

celery 任务示例

本地启动celery命令: python  manage.py  celery  worker  --settings=settings
周期性任务还需要启动celery调度命令：python  manage.py  celerybeat --settings=settings
"""
import datetime
import json

from celery import task
from celery.schedules import crontab
from celery.task import periodic_task

from blueking.component.shortcuts import get_client_by_user, logger
from home_application.models import *
from home_application.utils import *

#
# @periodic_task(run_every=crontab(minute='*', hour='*', day_of_week="*"))
# def run_task():
#     try:
#         pass
#     except Exception as e:
#         logger.exception("get_servers_perform error: %s" % e.message)

@periodic_task(run_every=crontab(minute='*/1', hour='*', day_of_week="*"))
def get_data():
    hosts = HostInfo.objects.all()
    if not hosts:
        return
    username = hosts[0].created_by
    client = get_client_by_user(username)
    biz_ids = list(set([i.bk_biz_id for i in hosts]))
    script_content = """
# !/bin/bash
cat /proc/loadavg
free -m
df -h
    """
    for bk_biz_id in biz_ids:
        biz_hosts = HostInfo.objects.filter(bk_biz_id=bk_biz_id)
        ip_list = [{'ip':j.ip,'bk_cloud_id':j.bk_cloud_id} for j in biz_hosts]
        data_mes = fast_execute_script(client, bk_biz_id, username, ip_list, script_content)
        log_data = data_mes['data']
        for i in log_data:
            ip = i['ip']
            host_obj = HostInfo.objects.get(ip=ip)
            log_content = i['log_content']
            cpu_mes = log_content.split('\n')[0]
            real_cpu_load = [float(cpu_mes.split(" ")[z]) for z in xrange(3)]
            cpu_use = (real_cpu_load[0] + real_cpu_load[1] + real_cpu_load[2])/3
            mem_mes = log_content.split('\n')[2]
            mem_list = [zx for zx in mem_mes.split(" ") if zx]
            mem_use = float(mem_list[2])
            mem_free = float(mem_list[3])
            disk_mes = log_content.split('\n')[4:]
            real_disk_mes = []
            for disk in disk_mes[1:-1]:
                disk_list = [k for k in disk.split(" ") if k]
                real_disk_mes.append({'Filesystem': disk_list[0],
                                      'Size': disk_list[1],
                                      'Used': disk_list[2],
                                      'Avail': disk_list[3],
                                      'Useage': disk_list[4],
                                      'Mounted': disk_list[5],})
            HostData.objects.create(host=host_obj,cpu_use=cpu_use,mem_use=mem_use,mem_free=mem_free,disk_mes=json.dumps(real_disk_mes),when_created=timezone.now())