import json
import plotly
import joblib
import pandas as pd
import numpy as np
import re

import nltk
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize

from flask import Flask
from flask import render_template, request, jsonify
from plotly.graph_objs import Bar
from sqlalchemy import create_engine

import sys

sys.path.append("models")

app = Flask(__name__)

#Load data+model
from misc import load_data, keep_genres

df = load_data("data/Disaster-Messages-Categories.db")
model = joblib.load("models/classifier.pkl")

# index webpage displays cool visuals and receives user input text for model
@app.route('/')
@app.route('/templates/master.html')
def index():
    """Produce graph data for genres/categories plots"""
    # extract data needed for visuals
    #Graph 1
    genres = keep_genres(df)
    genres.loc[:,'genre_other']=~(genres.any(axis=1))

    genre_counts=genres.sum().transpose().copy()
    genre_names = [x.split('_')[1] for x in genre_counts.index]
    genre_counts=genre_counts.tolist()

    related_counts= list(df.pivot_table(columns=['genre_social','genre_news'],
                    values='related', aggfunc='sum').values[0][::-1])

    #Graph 2

    cat_counts=df.iloc[:,4:].sum().sort_values(ascending=False)
    cats=list(cat_counts.index)
    cat_counts=cat_counts.values

    # create visuals
    # TODO: Below is an example - modify to create your own visuals
    graphs = [
        {
            'data': [
                Bar(
                    x=genre_names,
                    y=genre_counts,
                    name='Recieved Messages'
                ),
                Bar(
                x=genre_names,
                y=related_counts,
                name='Relevant Messages'
                )
            ],

            'layout': {
                'title': 'Distribution of Message Genres',
                'yaxis': {
                    'title': "Count"
                },
                'xaxis': {
                    'title': "Genre"
                }
            }
        },

        {
            'data': [
                Bar(
                    x=cats,
                    y=cat_counts,
                    name='Categories')
            ],

            'layout': {
                'title': 'Distribution of Message Types',
                'margin': {
                    'l': 50,
                    'r': 50,
                    'b': 100,
                    't': 100,
                    'pad': 4
                },

                'yaxis': {
                    'title': {
                        'text': "Count",
                        'standoff': 0
                        }
                },

                'xaxis': {
                    'title': {
                        'text': "Message Category",
                        'standoff': 100
                        },
                    'tickangle': 30
                }
            }
        }
    ]

    # encode plotly graphs in JSON
    ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
    graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)

    # render web page with plotly graphs
    return render_template('master.html', ids=ids, graphJSON=graphJSON, genre_data=(genre_names, genre_counts))

# web page that handles user query and displays model results
@app.route('/go')
def go():
    """Make model prediction and return results"""
    # save user input in query
    cols=['message', 'genre_social', 'genre_news']

    message = request.args.get('query', '')
    input=pd.DataFrame([[message, 1, 0]], columns=cols)

    # use model to predict classification for query
    prediction=model.predict(input).transpose()[0].sort_values(ascending=False)
    classification_results = prediction.to_dict()

    # This will render the go.html Please see that file.
    return render_template(
        'go.html',
        query=message,
        classification_result=classification_results
    )

def main():
    app.run(host='0.0.0.0', port=3001, debug=True)

if __name__ == '__main__':
    main()


""" Tool for finding counter data for heroes """
try:
    from urllib.request import Request
    from urllib.request import urlopen
except:
    from urllib2 import Request
    from urllib2 import urlopen
import os
import json
import sys
from multiprocessing import Pool
from bs4 import BeautifulSoup

HOST = 'http://www.owfire.com'

def get_soup(url):
    """ Request url, parse into soup, return """
    req = Request(url)
    req.add_header('User-Agent',
                   'Mozilla/5.0 (X11; Linux i686; rv:10.0)'
                   + 'Gecko/20100101 Firefox/10.0')
    resp = urlopen(req)
    html_doc = resp.read()
    return BeautifulSoup(html_doc, 'html.parser')

def get_hero_data(url):
    """ Get hero counter data based on url """
    soup = get_soup(url)
    hero_name = url.split('/')[-2]

    hero_data = []
    for desc in soup.select('.counters')[0].select('.desc'):
        counter_hero = desc.select('.comments')[0].get('href').split('/')[-1]
        up_votes = float(desc.select('.up')[0].text.strip())
        down_votes = float(desc.select('.down')[0].text.strip())
        hero_data.append({'up': up_votes, 'down': down_votes, 'counter_hero': counter_hero})

    return {hero_name: hero_data}

def get_counters():
    """ Return dict of heroes and how well they counter other heroes """
    if os.path.isfile('.counter_cache'):
        with open('.counter_cache') as cache:
            return json.load(cache)
    counters = {}

    index_soup = get_soup(HOST + '/overwatch/counters')
    pool = Pool()
    hero_data = pool.map(get_hero_data, (HOST + link.get('href')
                                         for link in index_soup.select('.heroes a')))

    for hero in hero_data:
        hero_name = list(hero.keys())[0]
        counters[hero_name] = {hero_name: 0.5}
        for counter in list(hero.values())[0]:
            counter_hero = counter['counter_hero']
            up_votes = counter['up']
            down_votes = counter['down']
            counters[hero_name][counter_hero] = up_votes / (up_votes + down_votes)

    for hero in counters:
        for counter in counters[hero]:
            winrate1 = counters[hero][counter]
            winrate2 = 1.0 - counters[counter][hero]
            counters[hero][counter] = (winrate1 + winrate2) / 2.0
            counters[counter][hero] = 1 - ((winrate1 + winrate2) / 2.0)

    with open('.counter_cache', 'w') as cache:
        json.dump(counters, cache)
    return counters

def main():
    """ print hero counters """
    counters = get_counters()
    requested = counters if len(sys.argv) < 2 else sys.argv[1:]
    for hero in requested:
        print(hero)
        for enemy, winrate in sorted(counters[hero].items(), key=lambda n: n[1]):
            print('\t', enemy, winrate)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import numpy as np
from keras.utils import np_utils
from settings import *

print("Loading data...")
(X_train, y_train)= load_data("train")
print("Done!")

X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_train /= 255

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, NUM_CLASS)

model = build_model()
model.fit(X_train, Y_train, batch_size=BATCH_SIZE, nb_epoch=NUM_EPOCH,
          show_accuracy=True,shuffle=True,verbose=1, validation_split=0.2)
loss , acc = model.evaluate(X_train,Y_train,show_accuracy=True)
model.save_weights("model/{:.2f}.hdf5".format(acc),overwrite=True)


import inspect
import sys
from functools import wraps

# inspired by: http://stackoverflow.com/a/6618825
def flo(string):
    '''Return the string given by param formatted with the callers locals.'''
    callers_locals = {}
    frame = inspect.currentframe()
    try:
        outerframe = frame.f_back
        callers_locals = outerframe.f_locals
    finally:
        del frame
    return string.format(**callers_locals)

# does not work if called from another package (with other globals)
def doc1():
    '''Return the first line of the (callers) docstring.'''
    return globals()[inspect.stack()[1][3]].__doc__.splitlines()[0]

def _wrap_with(color_code):
    '''Color wrapper.

    Example:
        >>> blue = _wrap_with('34')
        >>> print(blue('text'))
        \033[34mtext\033[0m
    '''
    def inner(text, bold=False):
        '''Inner color function.'''
        code = color_code
        if bold:
            code = flo("1;{code}")
        return flo('\033[{code}m{text}\033[0m')
    return inner

black = _wrap_with('30')
red = _wrap_with('31')
green = _wrap_with('32')
yellow = _wrap_with('33')
blue = _wrap_with('34')
magenta = _wrap_with('35')
cyan = _wrap_with('36')
white = _wrap_with('37')
default_color = _wrap_with('0')

def print_doc1(*args, **kwargs):
    '''Decorator, print the first line of the decorated functions docstring.

    May be invoked as a simple, argument-less decorator (i.e. ``@print_doc1``)
    or with named arguments ``color`` or ``bold`` (eg.
    ``@print_doc1(color=utils.red, bold=True)``).

    Example:
        >>> @print_doc1
        ... def foo():
        ...     """First line of docstring.
        ...
        ...     another line.
        ...     """
        ...     pass
        ...
        >>> foo()
        \033[34mFirst line of docstring.\033[0m
    '''
    color = kwargs.get('color', blue)
    bold = kwargs.get('bold', False)

    # for decorator with arguments see: http://stackoverflow.com/a/5929165
    def real_decorator(func):
        '''real decorator function'''
        @wraps(func)
        def wrapper(*args, **kwargs):
            '''the wrapper function'''
            try:
                print(color(func.__doc__.splitlines()[0], bold))
            except AttributeError as error:
                name = func.__name__
                print(red(flo('{name}() has no docstring')))
                raise(error)
            return func(*args, **kwargs)
        return wrapper

    invoked = bool(not args or kwargs)
    if not invoked:
        # invoke decorator function which returns the wrapper function
        return real_decorator(func=args[0])
    return real_decorator

def print_full_name(*args, **kwargs):
    '''Decorator, print the full name of the decorated function.

    May be invoked as a simple, argument-less decorator (i.e. ``@print_doc1``)
    or with named arguments ``color``, ``bold``, or ``prefix``
    (eg. ``@print_doc1(color=utils.red, bold=True, prefix=' ')``).
    '''
    color = kwargs.get('color', default_color)
    bold = kwargs.get('bold', False)
    prefix = kwargs.get('prefix', '')

    def real_decorator(func):
        '''real decorator function'''
        @wraps(func)
        def wrapper(*args, **kwargs):
            '''the wrapper function'''
            first_line = ''
            try:
                first_line = func.__module__ + '.' + func.__qualname__
            except AttributeError as exc:
                first_line = func.__name__
            print(color(prefix + first_line, bold))
            return func(*args, **kwargs)
        return wrapper

    invoked = bool(not args or kwargs)
    if not invoked:
        # invoke decorator function which returns the wrapper function
        return real_decorator(func=args[0])

    return real_decorator

# taken from: http://stackoverflow.com/a/3041990
def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
    It must be "yes" (the default), "no", or None (which means an answer
    of the user is required).

    The "answer" return value is True for "yes" or False for "no".
    """
    valid = {"yes": True, "y": True, "ye": True, '1': True,
             "no": False, "n": False, '0': False,}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        sys.stdout.write(question + prompt)
        choice = raw_input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write("Please respond with 'yes' or 'no' "
                             "(or 'y' or 'n').\n")

def query_input(question, default=None, color=default_color):
    """Ask a question for input via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.

    The "answer" return value is a str.
    """
    if default is None or default == '':
        prompt = ' '
    elif type(default) == str:
        prompt = flo(' [{default}] ')
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        sys.stdout.write(color(question + prompt))
        choice = raw_input()
        if default is not None and choice == '':
            return default
        if choice != '':
            return choice

def filled_out_template_str(template, **substitutions):
    '''Return str template with applied substitutions.

    Example:
        >>> template = 'Asyl for {{name}} {{surname}}!'
        >>> filled_out_template_str(template, name='Edward', surname='Snowden')
        'Asyl for Edward Snowden!'

        >>> template = '[[[foo]]] was substituted by {{foo}}'
        >>> filled_out_template_str(template, foo='bar')
        '{{foo}} was substituted by bar'

        >>> template = 'names wrapped by {single} {curly} {braces} {{curly}}'
        >>> filled_out_template_str(template, curly='remains unchanged')
        'names wrapped by {single} {curly} {braces} remains unchanged'
    '''
    template = template.replace('{', '{{')
    template = template.replace('}', '}}')
    template = template.replace('{{{{', '{')
    template = template.replace('}}}}', '}')
    template = template.format(**substitutions)
    template = template.replace('{{', '{')
    template = template.replace('}}', '}')
    template = template.replace('[[[', '{{')
    template = template.replace(']]]', '}}')
    return template

def filled_out_template(filename, **substitutions):
    '''Return content of file filename with applied substitutions.'''
    res = None
    with open(filename, 'r') as fp:
        template = fp.read()
        res = filled_out_template_str(template, **substitutions)
    return res

if __name__ == '__main__':
    import doctest
    doctest.testmod()


# coding=utf-8

import re

from argparse import ArgumentDefaultsHelpFormatter
from argparse import ArgumentParser
# from ConfigParser import ConfigParser

def check_galaxy_config(current_config_path, sample_config_path):
    # TODO: check config paths

    current_active_config, current_inactive_config = _parse_config(current_config_path)
    sample_active_config, sample_inactive_config = _parse_config(sample_config_path)

    new_default_configs = sorted([sample_active_config[key]
                                  for key in sample_active_config.keys()
                                  if key not in current_active_config.keys()],
                                 key=lambda config_option: config_option.line)
    if len(new_default_configs) > 0:
        print(u"New default configs discovered in %s" % sample_config_path)
        for config_option in new_default_configs:
            print(u"%s (line %s)" % (config_option.text, config_option.line))
        print(u"")

    new_config_options = sorted([sample_inactive_config[key]
                                 for key in sample_inactive_config.keys()
                                 if key not in current_inactive_config.keys() and
                                 key not in current_active_config.keys()],
                                key=lambda config_option: config_option.line)
    if len(new_config_options) > 0:
        print(u"New config options discovered in %s" % sample_config_path)
        for config_option in new_config_options:
            print(u"%s (line %s)" % (config_option.text, config_option.line))
        print(u"")

    deprecated_config_options = sorted([current_active_config[key]
                                        for key in current_active_config.keys()
                                        if key not in sample_active_config.keys() and
                                        key not in sample_inactive_config.keys()],
                                       key=lambda config_option: config_option.line)
    if len(deprecated_config_options) > 0:
        print(u"Deprecated config options present in %s" % current_config_path)
        for config_option in deprecated_config_options:
            print(u"%s (line %s)" % (config_option.text, config_option.line))

class ConfigOption(object):

    def __init__(self, key, value, text, line):
        self.key = key
        self.value = value
        self.text = text
        self.line = line

def _parse_config(config_path):

    active_config_pattern = r"^([^#]\S*)\s?=\s?(.*)$"
    inactive_config_pattern = r"^#(\S*)\s?=\s?(.*)$"

    active_config = {}
    inactive_config = {}

    parse = False
    position = 0

    with open(config_path) as current_config_file:
        for line in current_config_file:
            position += 1
            line = line.rstrip('\n')

            if not parse:
                if u"[app:main]" in line:
                    parse = True
                continue
            active_match = re.search(active_config_pattern, line)
            if active_match is not None:

                config_option = ConfigOption(key=active_match.group(1),
                                             value=active_match.group(2),
                                             text=line,
                                             line=position)

                active_config[active_match.group(1)] = config_option
            else:
                inactive_match = re.search(inactive_config_pattern, line)
                if inactive_match is not None:
                    config_option = ConfigOption(key=inactive_match.group(1),
                                                 value=inactive_match.group(2),
                                                 text=line,
                                                 line=position)
                    inactive_config[inactive_match.group(1)] = config_option

    return active_config, inactive_config

def _parse_cli_options():
    """
    Parse command line options, returning `parse_args` from `ArgumentParser`.
    """
    parser = ArgumentParser(usage="usage: %(prog)s <options>",
                            epilog='Example usage: check_galaxy_config '
                                   '-c galaxy/config/galaxy.ini -s galaxy/config/galaxy.ini.sample',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-c", "--current_config",
                        required=True,
                        dest="current_config",
                        help="current galaxy config file path")
    parser.add_argument("-s", "--sample_config",
                        required=True,
                        dest="sample_config",
                        help="reference sample galaxy config file path")
    return parser.parse_args()

def main():
    options = _parse_cli_options()
    check_galaxy_config(current_config_path=options.current_config,
                        sample_config_path=options.sample_config)


from collections import Counter
from typing import Union, Sequence

import torch
from torch import Tensor

from .settings import DEFAULT_TYPE, tc_type

INDEX_NAMES = ['b', 'i', 'j', 'k', 'l', 'm', 'n']

class UniqueName(str):      # todo add a lot of tests
    __slots__ = ()

    _COUNTS = Counter()

    def __new__(cls, prefix: str = None):
        assert prefix is None or isinstance(prefix, str)
        prefix = prefix or cls.__name__[:1]

        counts = UniqueName._COUNTS
        if prefix in counts:
            suffix = counts[prefix]
            while prefix + str(suffix) in counts:
                suffix += 1
            name = prefix + str(suffix)
        else:
            name = prefix

        assert name not in counts
        assert (name.lower() != name) or len(
            name) > 1, f'Lower-case single letters are reserved for index variables.'
        counts[name] += 1
        return super(UniqueName, cls).__new__(cls, name)

    def __init__(self, *args, **kwargs):
        super(UniqueName, self).__init__()

class Size(UniqueName):
    __slots__ = '_num',

    def __new__(cls, var: Union[int, str] = None) -> 'Size':
        if isinstance(var, str):
            obj = super(Size, cls).__new__(cls, prefix=var)
            obj._num: int = None
        else:
            obj = super(Size, cls).__new__(cls, prefix=None)
            obj._num: int = var
        return obj

    def __str__(self):
        if self._num is not None:
            return str(self._num)
        else:
            return self

    def __repr__(self):
        return f'Size({self})'

    @property
    def num(self):
        return self._num

    @num.setter
    def num(self, v):
        assert self._num is None or self._num == v, \
            f"Trying to reset the value of {''.join(self)}={self._num} to {v}."
        self._num = v

    def add(self, *v: Union[str, int]) -> str:
        summant = 0
        remaining = []
        for i in v:
            if isinstance(i, int):
                summant += i
            elif i.num is not None:
                summant += i.num
            else:
                remaining.append(i)

        if self._num is not None:
            out = str(self._num + summant)
        else:
            out = self
            if summant > 0:
                out += f' + {summant}'
            elif summant < 0:
                out += f' - {-summant}'

        for i in remaining:
            out += f' + {i}'

        return out

    def sub(self, *v: Union[str, int]) -> str:  # todo tests
        sub_total = 0
        remaining = []
        for i in v:
            if isinstance(i, int):
                sub_total += i
            elif i.num is not None:
                sub_total += i.num
            else:
                remaining.append(i)

        if self._num is not None:
            out = str(self._num - sub_total)
        else:
            out = self
            if sub_total > 0:
                out += f' - {sub_total}'
            elif sub_total < 0:
                out += f' + {-sub_total}'

        for i in remaining:
            out += f' - {i}'

        return out

class TensorName(UniqueName):
    __slots__ = '_sizes', '_type'

    def __new__(cls,
                dim: int,
                type: str = DEFAULT_TYPE,
                sizes: Sequence[Union[str, int]] = None,
                prefix: str = 'T') -> 'TensorName':
        obj = super(TensorName, cls).__new__(cls, prefix=prefix)
        assert type.lower() in ('double', 'float', 'long')
        if sizes is not None:
            assert not isinstance(sizes, str)
            assert len(sizes) == dim
        else:
            sizes = tuple('S' for _ in range(dim))

        obj._sizes: Sequence[Size] = tuple(v if isinstance(v, Size) else Size(v) for v in sizes)
        obj._type = type

        return obj

    @property
    def arg(self):
        return f"{self._type}({', '.join(str(s) for s in self._sizes)}) {self}"

    @property
    def dim(self):
        return len(self._sizes)

    @property
    def indices(self) -> Sequence[str]:
        indices = INDEX_NAMES[:self.dim]
        if len(indices) < self.dim:
            indices += tuple(indices[-1] + str(i) for i in range(self.dim - len(indices)))

        # Sanity checks
        assert len(indices) == self.dim
        assert len(set(indices)) == self.dim
        return indices

    @property
    def sizes(self) -> Sequence[Size]:
        return self._sizes

    @property
    def type(self):
        return self._type

    @staticmethod
    def new_from(tensor: Tensor, prefix: str = None):
        return TensorName(dim=tensor.dim(), type=tc_type(tensor), sizes=tensor.shape, prefix=prefix)

    @staticmethod
    def make_pair(sizes: Sequence[int], prefix: str = None):
        return TensorName(dim=len(sizes), type=DEFAULT_TYPE, sizes=sizes, prefix=prefix), \
               torch.rand(*sizes)


import pygame

from screens import StartScreen

class PressSpaceToJump():
    def __init__(self):
        ## Useful numbers and variables
        self.WORLD_HEIGHT = 200
        self.WORLD_WIDTH = 800
        self.offset = 50
        self.points = 0

        self.rate_create = 0

        ## Colors
        self.bg_color = pygame.Color(50, 87, 116)
        self.obs_color = pygame.Color(242, 224, 236)
        self.text_color = pygame.Color(242, 224, 236)

        ## Setup pygame
        pygame.init()

        self.display = pygame.display
        self.window = self.display.set_mode((self.WORLD_WIDTH, self.WORLD_HEIGHT))
        self.display.set_caption('Press Space To Jump')
        pygame.mixer.music.load('music.ogg')
        pygame.mixer.music.play(-1)

        self.font = pygame.font.Font('fast99.ttf', 16)
        self.fps_clock = pygame.time.Clock()
        self.screen = StartScreen(self.window, self)

    ## Main Game
    def main(self):
        while True:
            self.screen.process()
            self.screen.render()

if(__name__ == '__main__'):
    PressSpaceToJump().main()


''' Uses the Turtle Trading system to make trades '''

from collections import deque
from enum import Enum
import os
import pandas as pd

ATR_PERIOD = 20
ENTER_PERIOD = 20
EXIT_PERIOD = 10

InvState = Enum('InvState', 'OUT LONG SHORT')
init_funds = 10000000.00

def main():

    # Define symbols and price/point
    symbols = {'GE': 2500, 'ES': 50, 'CHF': 125000, 'GBP': 62500,
        'CAD': 100000, 'GC': 100, 'SI': 5000, 'HG': 25000, 'RB': 42000}

    true_ranges = deque(maxlen=ATR_PERIOD)
    enter_deque = deque(maxlen=ENTER_PERIOD)
    exit_deque = deque(maxlen=EXIT_PERIOD)
    positions = {}

    csv_files = [f for f in os.listdir('.') if f.endswith('.csv')]
    for csv_file in csv_files:

        # Initialize values
        inv_state = InvState.OUT
        funds = init_funds
        last_price = 0.0
        old_close = -1
        old_atr = -1.0
        true_ranges.clear()
        enter_deque.clear()
        exit_deque.clear()
        positions.clear()

        # Contract-specific information
        symbol = csv_file.split('.')[0]
        contract_size = symbols[symbol]
        df = pd.read_csv(csv_file)

        # Iterate through bars
        for i, bar in df.iterrows():

            # Find true range
            if old_close != -1:
                true_range = max(bar['HIGH'] - bar['LOW'],
                    abs(bar['HIGH'] - old_close),
                    abs(bar['LOW'] - old_close))
                true_ranges.append(true_range)
                old_close = bar['CLOSE']
            else:
                old_close = bar['CLOSE']
                continue

            # Compute the average true range (ATR)
            if len(true_ranges) == ATR_PERIOD:
                N = ((ATR_PERIOD-1) * old_atr + true_range)/ATR_PERIOD
                old_atr = N
            else:
                old_atr = sum(true_ranges)/len(true_ranges)
                continue

            # Initialize parameters
            price = bar['CLOSE']
            unit_size = int(0.01 * funds/(N * contract_size))

            # Check for entry
            if inv_state == InvState.OUT and len(enter_deque) == ENTER_PERIOD:

                # Buy 1 unit at 20-day high
                if price > max(enter_deque):
                    positions[price] = unit_size
                    last_price = price
                    inv_state = InvState.LONG

                # Short 1 unit at 20-day low
                elif price < min(enter_deque):
                    positions[price] = unit_size
                    last_price = price
                    inv_state = InvState.SHORT

            # Exit position if price at 10-day low/high
            elif (inv_state == InvState.LONG and price < min(exit_deque)) or \
                (inv_state == InvState.SHORT and price > max(exit_deque)):

                for p in positions:
                    if inv_state == InvState.LONG:
                        change = positions[p] * contract_size * (price - p)
                    else:
                        change = positions[p] * contract_size * (p - price)
                    funds += change
                positions.clear()
                last_price = 0.0
                inv_state = InvState.OUT

            # Exit position if the price falls/rises by 2N
            elif (inv_state == InvState.LONG and price < last_price - 2*N) or \
                (inv_state == InvState.SHORT and price > last_price + 2*N):

                # Apply stop condition
                price = last_price - 2*N if inv_state == InvState.LONG \
                    else last_price + 2*N
                for p in positions:
                    if inv_state == InvState.LONG:
                        change = positions[p] * contract_size * (price - p)
                    elif inv_state == InvState.SHORT:
                        change = positions[p] * contract_size * (p - price)
                    funds += change
                positions.clear()
                last_price = 0.0
                inv_state = InvState.OUT

            # Increase position if the price rises/falls by N/2
            elif ((inv_state == InvState.LONG and price > last_price + N/2) or \
                (inv_state == InvState.SHORT and price < last_price - N/2)):

                # Make sure position doesn't exceed 4 units
                tot_position = sum(positions.values())
                if tot_position + unit_size < 4 * unit_size:
                    if price in positions:
                        positions[price] += unit_size
                    else:
                        positions[price] = unit_size
                    last_price = price

            enter_deque.append(price)
            exit_deque.append(price)

        # Determine return
        for p in positions:
            if inv_state == InvState.LONG:
                change = positions[p] * contract_size * (price - p)
            elif inv_state == InvState.SHORT:
                change = positions[p] * contract_size * (p - price)
            funds += change
        ret = funds/init_funds
        print('Return for {0}: {1:.4f}'.format(symbol, ret))

if __name__ == '__main__':
    main()


import os
import numpy as numpy
from PIL import Image
import csv, json
import pandas as pd
import logging
import torch
import torch.utils.data as data
from torchvision import transforms
from torch.nn import functional as F
import pdb
import struct
import networkx as nx

ROOT = '/home/mschiappa/Desktop/VisualRelationshipsDetection/data/open-images/'
CLASS_PATH = os.path.join(ROOT,'class-descriptions.csv')

VAL_DATA_PATH = os.path.join(ROOT, 'validation/validation_image_data.json')
VAL_LABEL_PATH = os.path.join(ROOT, 'validation/validation_relationships.json')
VAL_IMAGES_PATH = os.path.join(ROOT, 'validation/images')

TRAIN_DATA_PATH = os.path.join(ROOT, 'train/train_image_data.json')
TRAIN_LABEL_PATH = os.path.join(ROOT, 'train/train_relationships.json')
TRAIN_IMAGES_PATH = os.path.join(ROOT, 'train/images')

class OpenImages2019(data.Dataset):
    def __init__(self, image_data_path, label_data_path, images_path, root=ROOT, class_description_path=CLASS_PATH, filenames=None, detection=False, validation=True, transform=None):
        super(OpenImages2019, self).__init__()

        # Data Initialization
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.detection = detection
        self.data_which = 'validation' if validation else 'train'
        self.image_dir = images_path
        self.dir = os.path.join(root, self.data_which)
        self.image_data = json.load(open(image_data_path))

        if filenames is None:
            self.filenames = os.listdir(images_path)
        else:
            self.filenames = filenames
        self.image_ids = [img.replace('.jpg', '') for img in self.filenames]
        self.rel_mapping = {'at':1, 'on': 2, 'holds': 3, 'plays': 4, 'interacts_with': 5, 'wears': 6, 'is': -1, 'inside_of': 7, 'under': 8, 'hits': 9}

        # Transformations for images
        if transform is None:
            normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                    std=[0.229, 0.224, 0.225])
            self.transform=transforms.Compose([transforms.ToTensor(), normalize])
        else:
            self.transform = transform
        shape = (256, 170)
        self.bounding_transform = transforms.Compose([transforms.Resize(shape)])

        # Add ground truths for either detection or relationship learning
        self.classes = pd.read_csv(class_description_path, header=None)
        self.classes['int_label'] = self.classes.index.values  # create int class label for objects
        self.classes = self.classes.set_index(0)
        classes_path = os.path.join(root, 'classes_with_int_labels.csv')
        if not os.path.exists(classes_path):
            self.classes.to_csv(classes_path)
        self.classes = self.classes.to_dict(orient='index')

        if self.detection:
            self.labels = self.preprocess_labels(os.path.join(self.dir, self.data_which+'-annotations-bbox.json'))
        else:
            self.labels = json.load(open(label_data_path))

        # Get remaining image IDs
        self.image_ids = [im for im in self.image_ids if im in self.labels.keys()]

        # Add length info
        self.length = len(self.image_ids)

    def __len__(self):
        return self.length

    def preprocess_labels(self, filepath):
        if os.path.isfile(filepath):
            return json.load(open(filepath))
        else:
            print("Pre-processing labels because json file did not exist.")
            df = pd.read_csv(filepath.replace('.json', '.csv')).set_index('ImageID')
            json_obj = df.to_dict(orient='index')
            with open(filepath, 'w') as f:
                json.dump(json_obj, f)
        return json_obj

    def __getitem__(self, index):
        """
        If detection:
            boxes (FloatTensor[N, 4]): the ground-truth boxes in [x1, y1, x2, y2] format,
                        with values between 0 and H and 0 and W

            labels (Int64Tensor[N]): the class label for each ground-truth box
        Else:
            Objects (IntT64Tensor[N, 3, W, H]): A group of objects present in image.
                Every 2 objects have a relationshop between them in order Object and Subject.

            labels (Int64[N/2]): Labels/predicates for each pair of objects in an image
        """
        # Read in Image and transform to a normalized tensor
        image_id = self.image_ids[index]
        image_path = image_id+'.jpg'
        image = Image.open(os.path.join(self.image_dir, image_path)).convert('RGB')
        image = self.transform(image)

        # Get height and width of image to extract bounding box, assuming all not the same
        height = self.image_data[image_id]['height']
        width = self.image_data[image_id]['width']

        # Return either ground truth bounding boxes for detection or relationships
        labels = dict()
        if self.detection:
            bbox_labels = self.labels[image_id]
            labels['boxes'] = [float(bbox_labels['XMax']), float(bbox_labels['YMax']), float(bbox_labels['XMin']), float(bbox_labels['YMin'])]
            labels['labels'] = [self.classes[bbox_labels['LabelName']]['int_label']]
            return image, [labels]
        else:
            # Initialize graph for adjacency matrix
            # gt = nx.from_edgelist(self.labels[image_id]['edge_list'])
            gt = nx.Graph()
            # Iterate through the relationships and extract the ROI
            objects = list()
            classes = list()
            nodes = list()
            images = dict()
            rel_dict = self.labels[image_id]['relationships']

            for rel in  rel_dict:
                # Get object and subject image regions and make them same dimensions
                images[rel['object']['node']] = F.interpolate(self.extract_ROI(rel['object'], image, width, height).unsqueeze_(0), size=(256, 170))  #.to(self.device)
                images[rel['subject']['node']] = F.interpolate(self.extract_ROI(rel['subject'], image, width, height).unsqueeze_(0), size=(256, 170))  # .to(self.device)

                # Add classes
                classes.append(self.classes[rel['object']['name']]['int_label'])
                classes.append(self.classes[rel['subject']['name']]['int_label'])

                # Add edge weights representing the predicate
                gt.add_edge(rel['object']['node'], rel['subject']['node'], weight=self.rel_mapping[rel['predicate']])
                # gt[rel['object']['node']][rel['subject']['node']]['weight'] =  self.rel_mapping[rel['predicate']]

            # Get adjacency matrix
            gt = torch.tensor(nx.adjacency_matrix(gt).todense())

            # Store remaining
            labels['gt'] = gt  # length is number of  relationships
            labels['labels'] = classes # length is number of relationships*2

            return images, labels

    def extract_ROI(self, bbox, image, im_width, im_height):
        (left, right, bottom, top) = (float(bbox['x']) * im_width, (float(bbox['x'])+bbox['w']) * im_width,
                              float(bbox['y']) * im_height, (float(bbox['y'])+bbox['h']) * im_height)
        obj = image[: , int(bottom):int(top), int(left):int(right)]
        return obj

def test_functionality():
    from torch.utils.data import DataLoader
    from torchvision import models
    from torch import nn
    test = OpenImages2019(validation=True, image_data_path=VAL_DATA_PATH, label_data_path=VAL_LABEL_PATH,
                            images_path=VAL_IMAGES_PATH, detection=False)
    test_loader = DataLoader(test, shuffle=False, batch_size=1)# , collate_fn=test_collate2)
    n = 10
    for inputs, targets in test:
        backbone = models.resnet18(pretrained=True)
        modules=list(backbone.children())[:-1]
        backbone =nn.Sequential(*modules)
        for p in backbone.parameters():
            p.requires_grad = False
        pdb.set_trace()
        break

if __name__=='__main__':
    test_functionality()


import logging
import threading
import time
import random

from configs import platform_list
from secret.bot_secret import discord_token

chats = {
    'fu6kme_':'666666',
    'lsls8787':'都看到一堆手電筒了還選隱鬼是斗M喔',
    'Km55489':'我不喜歡吃苦瓜',
    'Eeggggg0':'66666666',
    '溫蒂':'我叫我阿嬤來打都比你強，而且她已經過世了',
    '散步牛奶':'帶手電筒跟著鬼跑的都小王八蛋',
    'mad_Steve':'What\'s this game?',
    '魔法阿公':'苦瓜是歷史的罪人',
    'yukiko88':'請問小屋到底要怎麼追人？',
    '東亞病夫':'這個不修機的在幹嘛啦',
    'ssgew859':'?????',
    'james9576':'我每次鋼筋鐵骨都會被吃掉',
    '看衰仔':'這場應該四殺',
    'labmen007':'ZZZZZZZ'
}
class demo_bot(threading.Thread):
    platform = platform_list[0]
    title_tag = '@title'
    def __init__(self, main, args):
        super().__init__()
        self.main = main
        self.channel = args['channel']
        self.texts = {}
        self.EMOTE_MODE = self.main.from_setting(self.platform, 'emote', 'bool')
        self.TAKING = False
        self.TOKEN = discord_token
        self.BUSY = False
        self.go = True

    def login_and_run(self):
        self.start()

    def run(self):
        self.texts[self.title_tag] = ['已成功連結']
        # self.main.set_platform_when_logining(self.platform)
        self.main.ui.bot_login_signal.click()
        logging.info('Logged on demo')
        while self.go:
            for name in chats:
                self.texts[name] = [chats[name]]
                time.sleep(random.random() + 1)

    def fetch_text(self):
        _list = self.texts
        self.TAKING = True
        self.texts = {}
        self.TAKING = False
        return _list

    def too_busy(self):
        self.BUSY = True
        logging.info('bot is busy now.')

    def close(self):
        self.go = False
        self.join()


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/cameron/Dev/kanban-dev/django-autotask/djautotask/migrations/0046_auto_20200207_1002.py
# Compiled at: 2020-02-28 16:41:52
# Size of source mod 2**32: 773 bytes
from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):
    dependencies = [
     ('djautotask', '0045_resourceroledepartment_resourceservicedeskrole')]
    operations = [
     migrations.AddField(model_name='resource',
       name='default_service_desk_role',
       field=models.ForeignKey(blank=True, null=True, on_delete=(django.db.models.deletion.SET_NULL), to='djautotask.Role')),
     migrations.AddField(model_name='ticket',
       name='role',
       field=models.ForeignKey(blank=True, null=True, on_delete=(django.db.models.deletion.SET_NULL), to='djautotask.Role'))]


# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf

from patchwork._distill import _build_student_model, distill

def test_student_model_with_premade_model():
    inpt = tf.keras.layers.Input((None, None, 3))
    net = tf.keras.layers.GlobalMaxPool2D()(inpt)
    net = tf.keras.layers.Dense(5, activation="sigmoid")(net)
    student0 = tf.keras.Model(inpt, net)

    student1 = _build_student_model(student0, 5)

    assert isinstance(student1, tf.keras.Model)
    assert len(student0.layers) == len(student1.layers)
    assert student0.output_shape == student1.output_shape

def test_student_model_without_premade_model():
    student = _build_student_model("VGG16", 5, imshape=(32,32))

    assert isinstance(student, tf.keras.Model)
    assert student.output_shape[-1] == 5

def test_distill(test_png_path):
    inpt = tf.keras.layers.Input((None, None, 3))
    net = tf.keras.layers.GlobalMaxPool2D()(inpt)
    net = tf.keras.layers.Dense(5, activation="sigmoid")(net)
    student0 = tf.keras.Model(inpt, net)

    filepaths = [test_png_path, test_png_path]
    ys = 0.5*np.ones((2,5), dtype=np.float32)

    student1, trainloss = distill(filepaths, ys, student0, epochs=1,
                                  imshape=(32,32), batch_size=1,
                                  augment=False)

    assert isinstance(student1, tf.keras.Model)
    assert len(student0.layers) == len(student1.layers)
    assert student0.output_shape == student1.output_shape

    assert isinstance(trainloss, dict)
    assert isinstance(trainloss["train_loss"][0], np.float32)


import math

def vier_kwadraten(getal):
    a=b=c=d = getal *0.25

    a = math.sqrt(a)
    b = math.sqrt(b)
    c = math.sqrt(c)
    d = math.sqrt(d)
    l = (a, b, c, d)
    print(l)

    h = a**2,b**2,c**2,d**2
    p = sum(h)
    if p//2 == 0:
        print("hoi")
    else:
        print('koi')
    print(p)
    return l

"""==============================================[ HU TESTRAAMWERK ]====================================================
Onderstaand staan de tests voor je code -- hieronder mag je niets wijzigen!
Je kunt je code testen door deze file te runnen of met behulp van pytest.
"""
from functools import reduce
import random
from time import perf_counter

def test_vier_kwadraten():
    # Simulated test cases
    for cnt in range(10):
        n = random.randrange(1, 10)

        for _ in range(4):
            n *= 10
            n += random.randrange(0, 10)

        lst = vier_kwadraten(n)
        assert n == reduce(lambda x, y: x + y, (map(lambda x: x ** 2, lst))), \
            f"Fout: vier_kwadraten({n}) geeft {lst}, maar {lst[0]}^2 + {lst[1]}^2 + {lst[2]}^2 + {lst[3]}^2 != {n}"

def test_vier_kwadraten_tijd():
    # Test cases
    testcases = [36624, 73504, 54296, 40923, 33504, 42627, 70798, 90815, 55367, 52699]

    for case in testcases:
        print(case)
        lst = vier_kwadraten(case)
        assert case == reduce(lambda x, y: x + y, (map(lambda x: x ** 2, lst))), \
            f"Fout: vier_kwadraten({case}) geeft {lst}, maar {lst[0]}^2 + {lst[1]}^2 + {lst[2]}^2 + {lst[3]}^2 != {case}"

if __name__ == '__main__':
    try:
        print("\x1b[0;32m")
        test_vier_kwadraten()
        print("Je functie vier_kwadraten(getal) werkt goed!\n")

        print("\x1b[0;30m")
        print("Timing van 10 getallen...")

        start_time = perf_counter()
        test_vier_kwadraten_tijd()
        delta_time = perf_counter() - start_time
        print(f"Totale tijd: {delta_time*1000:.0f}ms")

    except AssertionError as ae:
        print("\x1b[0;31m")
        print(ae)


# -*- coding: utf-8 -*-
from ..tnt import torchnet as tnt
import torch
from ..utils import kaldi_io
import argparse
import os
from ..utils.kaldi_dataloader2 import KaldiStreamDataloader as KaldiStreamDataloader2
import logging
from .DNN import DNN
from torch.autograd import Variable
import numpy as np
from tqdm import tqdm
from torch.optim.lr_scheduler import ReduceLROnPlateau
from datetime import datetime
torch.manual_seed(7)

def save_checkpoint(state,is_best, savedir):
    filename = os.path.join(savedir, 'checkpoint.th')
    torch.save({'state_dict': state['model'].state_dict(),
                'epoch': state['epoch']}, filename)
    if is_best:
        torch.save(state['model'].state_dict(),
                   os.path.join(savedir, 'model_best.param'))
        with open(os.path.join(savedir, 'acc.txt'), 'w') as facc:
            facc.write('acc:'+str(state['acc'])+'\n')

def load_mean_var(fpath):
    mv = np.loadtxt(fpath)
    return torch.FloatTensor(mv[0]), torch.FloatTensor(mv[1])

def get_logger():
    logging.basicConfig(level=logging.INFO,
                        format="[ %(levelname)s : %(asctime)s ] - %(message)s")
    logger = logging.getLogger("train")
    return logger

def parse_arg():
    parser = argparse.ArgumentParser()
    parser.add_argument("traindata", type=str)
    parser.add_argument("trainlabel", type=lambda x:{
        k: v for k, v in kaldi_io.read_ali_ark(x)
    })
    parser.add_argument("cvdata", type=str)
    parser.add_argument("cvlabel", type=lambda  x:{
        k: v for k, v in kaldi_io.read_ali_ark(x)
    })
    #parser.add_argument("traincounts", type=argparse.FileType("r"))
    #parser.add_argument("cvcounts", type=argparse.FileType("r"))
    parser.add_argument("inputdim", type=int)
    parser.add_argument("outputdim", type=int)
    parser.add_argument("meanvar", type=argparse.FileType("r"))
    parser.add_argument("--nocuda", action="store_true", default=False)
    parser.add_argument("--nonorm", action="store_true", default=False)
    parser.add_argument("--pretraindir", type=str)
    parser.add_argument("-ep", "--epochs", default=50, type=int)
    parser.add_argument("-lr", '--learningrate', default=1e-2, type=float)
    parser.add_argument("-bs", '--batchsize', default=256, type=int)
    parser.add_argument("-o", "--output", type=str, required=True)

    args = parser.parse_args()
    os.makedirs(args.output, exist_ok=True)
    return args

def main():

    args = parse_arg()
    logger = get_logger()
    traindataloader = KaldiStreamDataloader2(
        args.traindata, args.trainlabel, num_outputs=args.outputdim,
        batchsize=args.batchsize, cachesize=200, shuffle=True)

    cvdataloader = KaldiStreamDataloader2(
        args.cvdata, args.cvlabel, num_outputs=args.outputdim,
        batchsize=args.batchsize, cachesize=200, shuffle=False)
    '''
    print('xx'*10)
    for x in tqdm(traindataloader, total=traindataloader.nsamples // args.batchsize):
        pass
    print('yy'*10)
    '''
    logger.info("train_nsample:"+str(traindataloader.nsamples))
    logger.info("cv_nsample:"+str(cvdataloader.nsamples))

    model = DNN(args.inputdim, args.outputdim, activation='sigm')
    # devide_ids = [0, 1]
    # model = torch.nn.DataParallel(model, device_ids=devide_ids)
    # print("CUDA_VISIBLE_DEVICES", str(os.environ['CUDA_VISIBLE_DEVICES']))

    epoch = 0
    if args.pretraindir:
        print(args.pretraindir, "pretraindir")
        loaded_stat = torch.load(args.pretraindir)
        epoch = int(loaded_stat['epoch'])
        model.load_state_dict(loaded_stat['state_dict'])

    if not args.nocuda:
        model = model.cuda()
    print(model)
    tr_mean, tr_var = load_mean_var(args.meanvar)

    epochsize = int(traindataloader.nsamples * 1.0 // args.batchsize)
    cvsize = int(cvdataloader.nsamples * 1. // args.batchsize)
    engine = tnt.engine.Engine()
    criterion = torch.nn.CrossEntropyLoss(size_average=False)
    #optimizer = torch.optim.Adam(model.parameters(), lr=args.learningrate, weight_decay=1e-8)
    optimizer = torch.optim.SGD(model.parameters(), lr=args.learningrate, momentum=0.9,weight_decay=1e-8)
    sched = ReduceLROnPlateau(optimizer,mode="max", factor=0.5, patience=1)

    # Statistics
    time_meter = tnt.meter.TimeMeter(False)
    meter_acc = tnt.meter.AverageValueMeter()
    meter_loss = tnt.meter.AverageValueMeter()

    def lossfunction(sample):
        x, y = sample
        if not args.nonorm:
            x = torch.add(x, -tr_mean)
            x = torch.div(x, tr_var)
        if not args.nocuda:
            x, y = x.cuda(), y.cuda()
        x, y = Variable(x), Variable(y)
        outputs = model(x)
        loss = criterion(outputs, y)
        return loss, outputs

    def evalfunction(sample):
        x, y = sample
        if not args.nonorm:
            x = torch.add(x, -tr_mean)
            x = torch.div(x, tr_var)
        if not args.nocuda:
            x, y = x.cuda(), y.cuda()
        x_var, y_var = Variable(x, volatile=True), Variable(y, volatile=True)
        outputs = model(x_var)
        loss = criterion(outputs,y_var)
        _, predicted = torch.max(outputs.data, 1)
        acc = (predicted == y).sum() * 1. / len(y)
        return {'loss': loss, 'acc': acc}, outputs

    def reset_meters():
        meter_acc.reset()
        meter_loss.reset()
        time_meter.reset()

    def on_start(state):
        state['best_acc'] = state[
            'best_acc'] if 'best_acc' in state else 0.

    def on_end_epoch(state):
        # Output trainmessge
        trainmessage = 'Training Epoch {:>3d}: Time: {:=6.1f}s/{:=4.1f}m Loss: {:=.4f} LR: {:=3.1e}'.format(
            state['epoch'], time_meter.value(), time_meter.value()/60, meter_loss.value()[0],optimizer.param_groups[0]['lr'])
        print(trainmessage)

        # evaluate procedure
        reset_meters()
        model.eval()
        engine.hooks['on_forward'] = on_forward_test
        engine.test(evalfunction, tqdm(cvdataloader, total=cvsize))
        engine.hooks['on_forward'] = on_forward
        acc = meter_acc.value()[0]
        loss = meter_loss.value()[0]
        evalmessage = 'CV Epoch {:>3d}: Time: {:=.2f}s/{:=.2f}m, acc:{:=.4f}, loss:{:=.8f}'.format(
            state['epoch'], time_meter.value(), time_meter.value()/60, acc, loss)
        print(evalmessage)
        sched.step(acc)

        # save model
        isbest = acc > state['best_acc']
        state['best_acc'] = acc if isbest else state['best_acc']
        save_checkpoint({
            'model': model,
            'epoch': state['epoch'],
            'acc' : acc
        }, isbest, args.output
        )

        # Quit training if learning rate is below 1e-8
        if optimizer.param_groups[0]['lr'] < 1e-8:
            state['epoch'] = 1e30
            print('lr<1e-9 stop training')
            return

    def on_start_epoch(state):
        model.train()
        reset_meters()
        state['iterator'] = tqdm(
            state['iterator'], total=epochsize, unit="batch", leave=False)

    def on_forward(state):
        meter_loss.add(state['loss'].data[0])

    def on_forward_test(state):
        meter_acc.add(state['loss']['acc'])
        meter_loss.add(state['loss']['loss'].data[0])

    engine.hooks['on_start'] = on_start
    engine.hooks['on_forward'] = on_forward
    engine.hooks['on_start_epoch'] = on_start_epoch
    engine.hooks['on_end_epoch'] = on_end_epoch

    engine.train(lossfunction, traindataloader,
                 maxepoch=args.epochs, optimizer=optimizer, epoch=epoch)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('app', '0002_auto_20150510_1558'),
    ]

    operations = [
        migrations.CreateModel(
            name='InstitucionEducativa',
            fields=[
                ('codigo', models.CharField(serialize=False, max_length=10, primary_key=True)),
                ('nombre', models.CharField(max_length=100)),
                ('sede', models.CharField(max_length=100)),
                ('municipio', models.CharField(max_length=50)),
                ('zona', models.PositiveSmallIntegerField(choices=[(0, 'Urbana'), (1, 'Urbana marginal'), (2, 'Rural'), (3, 'Rural de difícil acceso')])),
                ('modalidad', models.PositiveSmallIntegerField(choices=[(0, 'Académica'), (1, 'Ténica'), (2, 'Agropecuaria'), (3, 'Comercial'), (4, 'Promoción Social'), (5, 'Finanzas'), (6, 'Administación'), (7, 'Ecología'), (8, 'Medio Ambiente'), (9, 'Industrial'), (10, 'Informática'), (11, 'Minería'), (12, 'Salud'), (13, 'Recreación'), (14, 'Turismo'), (15, 'Deporte'), (16, 'Otro')])),
                ('slug', models.SlugField()),
            ],
        ),
        migrations.CreateModel(
            name='SecretariaEducacion',
            fields=[
                ('codigo', models.CharField(serialize=False, max_length=15, primary_key=True)),
                ('entidad', models.PositiveSmallIntegerField(choices=[(0, 'Municipal'), (1, 'Departamental')])),
                ('nombre', models.CharField(max_length=50)),
                ('slug', models.SlugField()),
            ],
            options={
                'ordering': ['nombre'],
            },
        ),
    ]


import tensorflow as tf
from sklearn.feature_extraction.text import TfidfVectorizer
import os
import pickle
import numpy as np
from data_preparation import TOP_K

NGRAM_RANGE = (1, 2)
TOKEN_MODE = 'word'
CATEGORY_LIST = ['Autism', 'Ethnicity', 'Gender Equality', 'LGBTQ+', 'Religion']
THRESHHOLD_OFFSET_FACTOR=1.5

path = os.path.join(os.path.dirname(__file__), "TAG_mlp_model.h5")
my_model = tf.keras.models.load_model(path)

kwargs = {
        'ngram_range': NGRAM_RANGE,  # Use 1-grams + 2-grams.
        'dtype': 'int32',
        'strip_accents': 'unicode',
        'decode_error': 'replace',
        'analyzer': TOKEN_MODE,  # Split text into word tokens.
}

vectorizer = TfidfVectorizer(**kwargs, max_features = TOP_K)
feature_list = pickle.load(open("feature.pkl", "rb"))
vectorizer.fit(feature_list)

def is_covid_problem(problem):
    problem = problem.lower()
    corona_words = ['corona', 'pandemic', 'quarantine', 'epidemic', 'virus', 'covid']
    if any(corona_word in problem for corona_word in corona_words):
        return True

def predict_category(problem):
    list_x = [problem]
    x = vectorizer.transform(list_x)
    x = x.toarray()
    result = my_model.predict(x)
    result_list = list(result[0])
    number_of_tags = len(CATEGORY_LIST)
    threshhold = (1/number_of_tags) * THRESHHOLD_OFFSET_FACTOR
    tags = []
    print(result_list)
    for tag in result_list:
        if tag > threshhold:
            tags.append(CATEGORY_LIST[result_list.index(tag)])
    if is_covid_problem(problem):
        tags.append("Covid-19")
    return tags

my_model.summary()
while True:
    input_x = input("What bothers you?: ")
    print(predict_category(input_x))

AI_Dev/AI versions/KI sub v1/predict_category.py


from __future__ import unicode_literals

import ffmpeg
import os
from track import crawl_timestamps
from configure import configure_tracks

def splitAudioFile(filename, tracks, outputFolder = "./Tracks"):
	if not os.path.exists(outputFolder):
		os.makedirs(outputFolder)

	numTracks = len(tracks)
	for i in range(numTracks):
		track = tracks[i]
		startTime = track.startTime
		name = track.filename()

		endTime = tracks[i + 1].startTime if i + 1 < numTracks else None

		t = track.pretty()
		print(f" - Making Track ({i + 1}/{numTracks}): {t}")
		trimOne(filename, f"{outputFolder}/{name}", startTime, endTime)

def trimOne(inputFile, outputFile, startTime, endTime):
	inputArgs = {
		"ss": startTime,
		"loglevel": "warning"
	}

	if endTime != None:
		inputArgs["t"] = endTime - startTime

	stream = ffmpeg.input(inputFile, **inputArgs)
	stream = ffmpeg.output(stream, outputFile)
	ffmpeg.run(stream);

def splitUpTrack(track, desc_path, desc_format, outputFolder, metadata):
	with open(desc_path, "r") as f:
		description = f.read()

		tracks = crawl_timestamps(description, desc_format)

		print()
		for t in tracks:
			print(t.pretty())

		if metadata["artist"] == None:
			i = input("\nAre these track names & artists correct? [y/n, f=flip] ")

			if i == 'f':
				print("Ok, flipping artists and tracks for new tracklist:\n")
				separator = desc_format.split("artist")[1].split("track")[0]
				comps = desc_format.split(separator)
				rev = comps[1] + separator + comps[0]
				tracks = crawl_timestamps(description, rev)
				for t in tracks:
					print(t.pretty())
			elif i != 'y':
				print("Ok, ignoring artists...")
				tracks = crawl_timestamps(description, None)

		outputFolder = outputFolder if outputFolder != None else "./Tracks"

		print()
		splitAudioFile(track, tracks, outputFolder)

		print("\n=> Configuring Tracks...\n")
		configure_tracks(outputFolder, tracks, metadata)

		print("\n=> Done! Your tracks are at " + outputFolder)

if __name__ == "__main__":
	track = "test resources/test.m4a"
	description = "test resources/test.description"
	thumbnail = "test resources/test.jpg"

	desc_format = "artist - title"
	outputFolder = "./Tracks"

	metadata = {
		"thumbnail": thumbnail,
		"artist": None,
		"album": None
	}

	splitUpTrack(track, description, desc_format, outputFolder, metadata)


from itertools import product

import pytest
import numpy as np
from numpy.testing import assert_array_almost_equal, assert_allclose
from scipy.spatial.distance import pdist, squareform, mahalanobis
from sklearn import clone
from sklearn.cluster import DBSCAN
from sklearn.utils import check_random_state
from sklearn.utils.testing import set_random_state

from metric_learn._util import make_context
from metric_learn.base_metric import (_QuadrupletsClassifierMixin,
                                      _PairsClassifierMixin)

from test.test_utils import (ids_metric_learners, metric_learners,
                             remove_y_quadruplets)

RNG = check_random_state(0)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_score_pairs_pairwise(estimator, build_dataset):
  # Computing pairwise scores should return a euclidean distance matrix.
  input_data, labels, _, X = build_dataset()
  n_samples = 20
  X = X[:n_samples]
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))

  pairwise = model.score_pairs(np.array(list(product(X, X))))\
      .reshape(n_samples, n_samples)

  check_is_distance_matrix(pairwise)

  # a necessary condition for euclidean distance matrices: (see
  # https://en.wikipedia.org/wiki/Euclidean_distance_matrix)
  assert np.linalg.matrix_rank(pairwise**2) <= min(X.shape) + 2

  # assert that this distance is coherent with pdist on embeddings
  assert_array_almost_equal(squareform(pairwise), pdist(model.transform(X)))

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_score_pairs_toy_example(estimator, build_dataset):
    # Checks that score_pairs works on a toy example
    input_data, labels, _, X = build_dataset()
    n_samples = 20
    X = X[:n_samples]
    model = clone(estimator)
    set_random_state(model)
    model.fit(*remove_y_quadruplets(estimator, input_data, labels))
    pairs = np.stack([X[:10], X[10:20]], axis=1)
    embedded_pairs = pairs.dot(model.transformer_.T)
    distances = np.sqrt(np.sum((embedded_pairs[:, 1] -
                               embedded_pairs[:, 0])**2,
                               axis=-1))
    assert_array_almost_equal(model.score_pairs(pairs), distances)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_score_pairs_finite(estimator, build_dataset):
  # tests that the score is finite
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  pairs = np.array(list(product(X, X)))
  assert np.isfinite(model.score_pairs(pairs)).all()

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_score_pairs_dim(estimator, build_dataset):
  # scoring of 3D arrays should return 1D array (several tuples),
  # and scoring of 2D arrays (one tuple) should return an error (like
  # scikit-learn's error when scoring 1D arrays)
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  tuples = np.array(list(product(X, X)))
  assert model.score_pairs(tuples).shape == (tuples.shape[0],)
  context = make_context(estimator)
  msg = ("3D array of formed tuples expected{}. Found 2D array "
         "instead:\ninput={}. Reshape your data and/or use a preprocessor.\n"
         .format(context, tuples[1]))
  with pytest.raises(ValueError) as raised_error:
    model.score_pairs(tuples[1])
  assert str(raised_error.value) == msg

def check_is_distance_matrix(pairwise):
  assert (pairwise >= 0).all()  # positivity
  assert np.array_equal(pairwise, pairwise.T)  # symmetry
  assert (pairwise.diagonal() == 0).all()  # identity
  # triangular inequality
  tol = 1e-12
  assert (pairwise <= pairwise[:, :, np.newaxis] +
          pairwise[:, np.newaxis, :] + tol).all()

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_embed_toy_example(estimator, build_dataset):
    # Checks that embed works on a toy example
    input_data, labels, _, X = build_dataset()
    n_samples = 20
    X = X[:n_samples]
    model = clone(estimator)
    set_random_state(model)
    model.fit(*remove_y_quadruplets(estimator, input_data, labels))
    embedded_points = X.dot(model.transformer_.T)
    assert_array_almost_equal(model.transform(X), embedded_points)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_embed_dim(estimator, build_dataset):
  # Checks that the the dimension of the output space is as expected
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  assert model.transform(X).shape == X.shape

  # assert that ValueError is thrown if input shape is 1D
  context = make_context(estimator)
  err_msg = ("2D array of formed points expected{}. Found 1D array "
             "instead:\ninput={}. Reshape your data and/or use a "
             "preprocessor.\n".format(context, X[0]))
  with pytest.raises(ValueError) as raised_error:
    model.score_pairs(model.transform(X[0, :]))
  assert str(raised_error.value) == err_msg
  # we test that the shape is also OK when doing dimensionality reduction
  if type(model).__name__ in {'LFDA', 'MLKR', 'NCA', 'RCA'}:
    # TODO:
    #  avoid this enumeration and rather test if hasattr n_components
    #  as soon as we have made the arguments names as such (issue #167)
    model.set_params(num_dims=2)
    model.fit(*remove_y_quadruplets(estimator, input_data, labels))
    assert model.transform(X).shape == (X.shape[0], 2)
    # assert that ValueError is thrown if input shape is 1D
    with pytest.raises(ValueError) as raised_error:
        model.transform(model.transform(X[0, :]))
    assert str(raised_error.value) == err_msg

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_embed_finite(estimator, build_dataset):
  # Checks that embed returns vectors with finite values
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  assert np.isfinite(model.transform(X)).all()

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_embed_is_linear(estimator, build_dataset):
  # Checks that the embedding is linear
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  assert_array_almost_equal(model.transform(X[:10] + X[10:20]),
                            model.transform(X[:10]) +
                            model.transform(X[10:20]))
  assert_array_almost_equal(model.transform(5 * X[:10]),
                            5 * model.transform(X[:10]))

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_get_metric_equivalent_to_explicit_mahalanobis(estimator,
                                                       build_dataset):
  """Tests that using the get_metric method of mahalanobis metric learners is
  equivalent to explicitely calling scipy's mahalanobis metric
  """
  rng = np.random.RandomState(42)
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  metric = model.get_metric()
  n_features = X.shape[1]
  a, b = (rng.randn(n_features), rng.randn(n_features))
  expected_dist = mahalanobis(a[None], b[None],
                              VI=model.get_mahalanobis_matrix())
  assert_allclose(metric(a, b), expected_dist, rtol=1e-15)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_get_metric_is_pseudo_metric(estimator, build_dataset):
  """Tests that the get_metric method of mahalanobis metric learners returns a
  pseudo-metric (metric but without one side of the equivalence of
  the identity of indiscernables property)
  """
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  metric = model.get_metric()

  n_features = X.shape[1]
  for seed in range(10):
    rng = np.random.RandomState(seed)
    a, b, c = (rng.randn(n_features) for _ in range(3))
    assert metric(a, b) >= 0  # positivity
    assert metric(a, b) == metric(b, a)  # symmetry
    # one side of identity indiscernables: x == y => d(x, y) == 0. The other
    # side of the equivalence is not always true for Mahalanobis distances.
    assert metric(a, a) == 0
    # triangular inequality
    assert (metric(a, c) < metric(a, b) + metric(b, c) or
            np.isclose(metric(a, c), metric(a, b) + metric(b, c), rtol=1e-20))

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_metric_raises_deprecation_warning(estimator, build_dataset):
  """assert that a deprecation warning is raised if someones wants to call
  the `metric` function"""
  # TODO: remove this method in version 0.6.0
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))

  with pytest.warns(DeprecationWarning) as raised_warning:
    model.metric()
  assert (str(raised_warning[0].message) ==
          ("`metric` is deprecated since version 0.5.0 and will be removed "
           "in 0.6.0. Use `get_mahalanobis_matrix` instead."))

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_get_metric_compatible_with_scikit_learn(estimator, build_dataset):
  """Check that the metric returned by get_metric is compatible with
  scikit-learn's algorithms using a custom metric, DBSCAN for instance"""
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  clustering = DBSCAN(metric=model.get_metric())
  clustering.fit(X)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_get_squared_metric(estimator, build_dataset):
  """Test that the squared metric returned is indeed the square of the
  metric"""
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  metric = model.get_metric()

  n_features = X.shape[1]
  for seed in range(10):
    rng = np.random.RandomState(seed)
    a, b = (rng.randn(n_features) for _ in range(2))
    assert_allclose(metric(a, b, squared=True),
                    metric(a, b, squared=False)**2,
                    rtol=1e-15)

@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
                         ids=ids_metric_learners)
def test_transformer_is_2D(estimator, build_dataset):
  """Tests that the transformer of metric learners is 2D"""
  input_data, labels, _, X = build_dataset()
  model = clone(estimator)
  set_random_state(model)
  # test that it works for X.shape[1] features
  model.fit(*remove_y_quadruplets(estimator, input_data, labels))
  assert model.transformer_.shape == (X.shape[1], X.shape[1])

  # test that it works for 1 feature
  trunc_data = input_data[..., :1]
  # we drop duplicates that might have been formed, i.e. of the form
  # aabc or abcc or aabb for quadruplets, and aa for pairs.
  if isinstance(estimator, _QuadrupletsClassifierMixin):
    for slice_idx in [slice(0, 2), slice(2, 4)]:
      pairs = trunc_data[:, slice_idx, :]
      diffs = pairs[:, 1, :] - pairs[:, 0, :]
      to_keep = np.where(np.abs(diffs.ravel()) > 1e-9)
      trunc_data = trunc_data[to_keep]
      labels = labels[to_keep]
  elif isinstance(estimator, _PairsClassifierMixin):
    diffs = trunc_data[:, 1, :] - trunc_data[:, 0, :]
    to_keep = np.where(np.abs(diffs.ravel()) > 1e-9)
    trunc_data = trunc_data[to_keep]
    labels = labels[to_keep]
  model.fit(*remove_y_quadruplets(estimator, trunc_data, labels))
  assert model.transformer_.shape == (1, 1)  # the transformer must be 2D


from sklearn.model_selection import train_test_split
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from datetime import datetime
import bert
from bert import run_classifier
from bert import optimization
from bert import tokenization
from tensorflow import keras
import os
import re
from ConexionBaseDeDatos import ConexionBaseDeDatos
from ExtractData import ExtractData

from datetime import date, datetime, timedelta

'''
Rate cambiado
'''
def datespan(startDate, endDate, delta=timedelta(days=1)):
    currentDate = startDate
    while currentDate < endDate:
        yield currentDate
        currentDate += delta

def create_tokenizer_from_hub_module():
    """Get the vocab file and casing info from the Hub module."""
    with tf.Graph().as_default():
        bert_module = hub.Module(BERT_MODEL_HUB)
        tokenization_info = bert_module(signature="tokenization_info", as_dict=True)
        with tf.Session() as sess:
            vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
                                                  tokenization_info["do_lower_case"]])

    return bert.tokenization.FullTokenizer(
        vocab_file=vocab_file, do_lower_case=do_lower_case)

'''

create_model hace esto a continuación.
Primero, vuelve a cargar el módulo concentrador BERT tf (esta vez para extraer el gráfico
de cálculo). A continuación, crea una nueva capa única que será entrenada para adaptar BERT
a nuestra tarea de sentimiento (es decir, clasificar si una crítica de película es positiva
o negativa). Esta estrategia de usar un modelo mayormente entrenado se llama ajuste fino.
'''
def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,
                 num_labels):
  """Creates a classification model."""

  bert_module = hub.Module(
      BERT_MODEL_HUB,
      trainable=True)
  bert_inputs = dict(
      input_ids=input_ids,
      input_mask=input_mask,
      segment_ids=segment_ids)
  bert_outputs = bert_module(
      inputs=bert_inputs,
      signature="tokens",
      as_dict=True)

  # Use "pooled_output" for classification tasks on an entire sentence.
  # Use "sequence_outputs" for token-level output.
  output_layer = bert_outputs["pooled_output"]

  hidden_size = output_layer.shape[-1].value

  # Create our own layer to tune for politeness data.
  output_weights = tf.get_variable(
      "output_weights", [num_labels, hidden_size],
      initializer=tf.truncated_normal_initializer(stddev=0.02))

  output_bias = tf.get_variable(
      "output_bias", [num_labels], initializer=tf.zeros_initializer())

  with tf.variable_scope("loss"):

    # Dropout helps prevent overfitting
    output_layer = tf.nn.dropout(output_layer, rate=1-0.9)

    logits = tf.matmul(output_layer, output_weights, transpose_b=True)
    logits = tf.nn.bias_add(logits, output_bias)
    log_probs = tf.nn.log_softmax(logits, axis=-1)

    # Convert labels into one-hot encoding
    one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)

    predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))
    # If we're predicting, we want predicted labels and the probabiltiies.
    if is_predicting:
      return (predicted_labels, log_probs)

    # If we're train/eval, compute loss between predicted and actual label
    per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
    loss = tf.reduce_mean(per_example_loss)
    return (loss, predicted_labels, log_probs)

'''
A continuación, incluiremos nuestra función de modelo en una función model_fn_builder
 que adapta nuestro modelo para que funcione con fines de capacitación, evaluación y predicción.
'''
def model_fn_builder(num_labels, learning_rate, num_train_steps,
                     num_warmup_steps):
    """Returns `model_fn` closure for TPUEstimator."""

    def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument
        """The `model_fn` for TPUEstimator."""

        input_ids = features["input_ids"]
        input_mask = features["input_mask"]
        segment_ids = features["segment_ids"]
        label_ids = features["label_ids"]

        is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)

        # TRAIN and EVAL
        if not is_predicting:

            (loss, predicted_labels, log_probs) = create_model(
                is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)

            train_op = bert.optimization.create_optimizer(
                loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)

            # Calculate evaluation metrics.
            def metric_fn(label_ids, predicted_labels):
                accuracy = tf.metrics.accuracy(label_ids, predicted_labels)
                '''
                f1_score = tf.contrib.metrics.f1_score(
                    label_ids,
                    predicted_labels)
                '''
                '''
                auc = tf.metrics.auc(
                    label_ids,
                    predicted_labels)
                '''
                recall = tf.metrics.recall(
                    label_ids,
                    predicted_labels)
                precision = tf.metrics.precision(
                    label_ids,
                    predicted_labels)
                true_pos = tf.metrics.true_positives(
                    label_ids,
                    predicted_labels)
                true_neg = tf.metrics.true_negatives(
                    label_ids,
                    predicted_labels)
                false_pos = tf.metrics.false_positives(
                    label_ids,
                    predicted_labels)
                false_neg = tf.metrics.false_negatives(
                    label_ids,
                    predicted_labels)
                return {
                    "eval_accuracy": accuracy,
                    #"f1_score": f1_score,
                    #"auc": auc,
                    "precision": precision,
                    "recall": recall,
                    "true_positives": true_pos,
                    "true_negatives": true_neg,
                    "false_positives": false_pos,
                    "false_negatives": false_neg
                }

            eval_metrics = metric_fn(label_ids, predicted_labels)

            if mode == tf.estimator.ModeKeys.TRAIN:
                return tf.estimator.EstimatorSpec(mode=mode,
                                                  loss=loss,
                                                  train_op=train_op)
            else:
                return tf.estimator.EstimatorSpec(mode=mode,
                                                  loss=loss,
                                                  eval_metric_ops=eval_metrics)
        else:
            (predicted_labels, log_probs) = create_model(
                is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)

            predictions = {
                'probabilities': log_probs,
                'labels': predicted_labels
            }
            return tf.estimator.EstimatorSpec(mode, predictions=predictions)

    # Return the actual model function in the closure
    return model_fn

def getPrediction(in_sentences):
    labels = [0, 1, 2]
    input_examples = [run_classifier.InputExample(guid="", text_a=x, text_b=None, label=0) for x in
                      in_sentences]  # here, "" is just a dummy label
    input_features = run_classifier.convert_examples_to_features(input_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
    predict_input_fn = run_classifier.input_fn_builder(features=input_features, seq_length=MAX_SEQ_LENGTH,
                                                       is_training=False, drop_remainder=False)
    predictions = estimator.predict(predict_input_fn)
    indice=0
    resultado = []

    try:
        for prediction in predictions:
            resultado.append((in_sentences[indice], prediction['probabilities'], labels[prediction['labels']]))

            print(str(indice) +"###"+str(len(in_sentences)))
            indice = indice + 1
            '''
            if indice>100:
                return resultado
            '''

    except Exception as e:
        print(e)
        return resultado

    return resultado
    '''
    try:
        return [(sentence, prediction['probabilities'], labels[prediction['labels']]) for sentence, prediction in
                zip(in_sentences, predictions)]
    except Exception as e:
        print(e)
    '''

if __name__ == '__main__':
    '''
    Configuración
    '''
    OUTPUT_DIR = 'Modelo_Sia'#@param {type:"string"}
    # Compute train and warmup steps from batch size
    # These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)
    BATCH_SIZE = 32
    LEARNING_RATE = 2e-5
    NUM_TRAIN_EPOCHS = 3.0
    # Warmup is a period of time where hte learning rate
    # is small and gradually increases--usually helps training.
    WARMUP_PROPORTION = 0.1
    # Model configs
    SAVE_CHECKPOINTS_STEPS = 500
    SAVE_SUMMARY_STEPS = 100

    # Compute train and warmup steps from batch size
    # These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)
    BATCH_SIZE = 32
    LEARNING_RATE = 2e-5
    NUM_TRAIN_EPOCHS = 3.0
    # Warmup is a period of time where hte learning rate
    # is small and gradually increases--usually helps training.
    WARMUP_PROPORTION = 0.1
    # Model configs
    SAVE_CHECKPOINTS_STEPS = 500
    SAVE_SUMMARY_STEPS = 100

    # Compute # train and warmup steps from batch size

    '''
    Genero el test y el train, importante todos los twwets  son lower
    '''
    DATA_COLUMN = 'sentence'
    LABEL_COLUMN = 'polarity'
    label_list = [0, 1, 2]
    datos= ExtractData()
    tweets=datos.cargarDatos()
    '''
    Fin carga de datos
     '''

    '''

    Data Preprocessing
    Necesitaremos transformar nuestros datos en un formato que BERT entienda.
    Esto implica dos pasos.

    Primero, creamos InputExample usando el constructor provisto en la biblioteca BERT.
        text_a es el texto que queremos clasificar,
        etiqueta es la etiqueta de nuestro ejemplo, es decir, Verdadero, Falso

    '''
    # Use the InputExample class from BERT's run_classifier code to create examples from the data
    train_InputExamples = tweets[0].apply(lambda x: bert.run_classifier.InputExample(guid=None,
                                                                                # Globally unique ID for bookkeeping, unused in this example
                                                                                text_a=x[DATA_COLUMN],
                                                                                text_b=None,
                                                                                label=x[LABEL_COLUMN]), axis=1)

    test_InputExamples = tweets[1].apply(lambda x: bert.run_classifier.InputExample(guid=None,
                                                                               text_a=x[DATA_COLUMN],
                                                                               text_b=None,
                                                                               label=x[LABEL_COLUMN]), axis=1)

    '''
       A continuación, debemos preprocesar nuestros datos para que coincidan con los datos
       en los que BERT recibió capacitación. Para esto, tendremos que hacer un par de cosas :

            Minúsculas en nuestro texto (si estamos usando un modelo de minúsculas BERT)
            Tokenize (es decir, "sally dice hola" -> ["sally", "dice", "hola"))
            Divida las palabras en WordPieces (es decir, "llamando" -> ["call", "## ing"])
            Asigne nuestras palabras a los índices utilizando un archivo de vocabulario que proporciona BERT
            Agregue tokens especiales "CLS" y "SEP" (consulte el archivo Léame)
            Agregue tokens de "índice" y "segmento" a cada entrada (vea el documento BERT)
            Afortunadamente, no tenemos que preocuparnos por la mayoría de estos detalles.

        Para comenzar, necesitaremos cargar un archivo de vocabulario y
        una información de minúsculas directamente desde el módulo del concentrador BERT tf:
    '''
    BERT_MODEL_HUB = "https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1"
    tokenizer = create_tokenizer_from_hub_module()
    print(tokenizer.tokenize("This here's an example of using the BERT tokenizer"))

    '''
    Usando nuestro tokenizer, llamaremos run_classifier.convert_examples_to_features
     en nuestros InputExamples para convertirlos en características que BERT entiende.
    '''

    # We'll set sequences to be at most 128 tokens long.
    ''' Modifico a 300 '''
    MAX_SEQ_LENGTH = 300
    # Convert our train and test features to InputFeatures that BERT understands.
    train_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH,
                                                                      tokenizer)
    test_features = bert.run_classifier.convert_examples_to_features(test_InputExamples, label_list, MAX_SEQ_LENGTH,
                                                                     tokenizer)

    '''
    importante verificar
    num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)
    '''
    num_train_steps = int(len(train_features))
    num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)

    # Specify outpit directory and number of checkpoint steps to save
    run_config = tf.estimator.RunConfig(
        model_dir=OUTPUT_DIR,
        save_summary_steps=SAVE_SUMMARY_STEPS,
        save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)
    model_fn = model_fn_builder(
        num_labels=len(label_list),
        learning_rate=LEARNING_RATE,
        num_train_steps=num_train_steps,
        num_warmup_steps=num_warmup_steps)

    estimator = tf.estimator.Estimator(
        model_fn=model_fn,
        config=run_config,
        params={"batch_size": BATCH_SIZE})

    '''A continuación, creamos una función de creación de entradas que toma nuestro
    conjunto de funciones de entrenamiento (train_features) y produce un generador.
    Este es un patrón de diseño bastante estándar para trabajar con Tensorflow Estimators.'''

    # Create an input function for training. drop_remainder = True for using TPUs.
    train_input_fn = bert.run_classifier.input_fn_builder(
        features=train_features,
        seq_length=MAX_SEQ_LENGTH,
        is_training=True,
        drop_remainder=False)
    print("Beginning Training!")
    current_time = datetime.now()
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
    print("Training took time ", datetime.now() - current_time)
    test_input_fn = run_classifier.input_fn_builder(
        features=test_features,
        seq_length=MAX_SEQ_LENGTH,
        is_training=False,
        drop_remainder=False)

    #print(estimator.evaluate(input_fn=test_input_fn, steps=None))

    '''
    calculate auc multiclass python
    parte de fechas
    '''

    conexionBase = ConexionBaseDeDatos()
    anterior=datetime(2019, 4,1, 1, 30)
    result = {}
    result["positivo"] = []
    result["neutral"] = []
    result["negativo"] = []
    result["total"] = []
    result["precio"] = []
    result["sentimiento_positivo"] = []
    result["sentimiento_negativo"] = []
    result["fecha"] = []

    indice=0
    for timestamp in datespan( datetime(2019, 4,1, 2, 30),  datetime(2019, 7,31, 23, 30),delta=timedelta(hours=1)):

        print("Iteracion"+str(timestamp))
        if conexionBase.existe_hora(timestamp):
            pass
        else:
            pred_sentences = conexionBase.ObtenerEntreHoras(anterior, timestamp)
            auxiliar = []
            for i in pred_sentences:
                auxiliar.append(i[0])
            sentimiento = []
            positivo=0
            negativo=0

            for z in pred_sentences:
                if z[1]=='positive':
                    positivo=positivo+1
                else:
                    negativo=negativo+1
                sentimiento.append(z[1])

            if len(auxiliar)>0:
                predictions = getPrediction(auxiliar)
                '''print(type(predictions).__name__)'''
                df = pd.DataFrame(list(predictions), sentimiento[0:len(predictions)])
                precio = conexionBase.ObtenerPrecio(timestamp)
                result["positivo"].append(len(df.loc[df[2] == 1]))
                result["neutral"].append(len(df.loc[df[2] == 2]))
                result["negativo"].append(len(df.loc[df[2] == 0]))
                result["total"].append(len(pred_sentences))
                for total_precio in precio:
                    result["precio"].append(total_precio[0].replace(".", ","))
                    break
                result["sentimiento_positivo"].append(positivo)
                result["sentimiento_negativo"].append(negativo)
                result["fecha"].append(str(timestamp))
                conexionBase.almacenar_datos_valoracion(result["precio"][indice],result["positivo"][indice],result["negativo"][indice],  result["neutral"][indice], result["sentimiento_negativo"] [indice], result["sentimiento_positivo"][indice],  result["total"][indice],result["fecha"][indice])
            else:
                precio = conexionBase.ObtenerPrecio(timestamp)
                result["positivo"].append(0)
                result["neutral"].append(0)
                result["negativo"].append(0)
                result["total"].append(0)
                for total_precio in precio:
                    result["precio"].append(total_precio[0].replace(".", ","))
                    break
                result["sentimiento_positivo"].append(positivo)
                result["sentimiento_negativo"].append(negativo)
                result["fecha"].append(str(timestamp))
                conexionBase.almacenar_datos_valoracion(result["precio"][indice],result["positivo"][indice],result["negativo"][indice],  result["neutral"][indice], result["sentimiento_negativo"] [indice], result["sentimiento_positivo"][indice],  result["total"][indice],result["fecha"][indice])
            indice=indice+1
        anterior = timestamp


from sklearn.datasets import make_moons
import numpy as np
import pdb

def pad_targets(xy):
    """
    Pad the targets to be 1hot.
    :param xy: A tuple containing the x and y matrices.
    :return: The 1hot coded dataset.
    """
    x, y = xy
    classes = np.max(y) + 1
    tmp_data_y = np.zeros((x.shape[0], classes))
    for i, dp in zip(range(len(y)), y):
        r = np.zeros(classes)
        r[dp] = 1
        tmp_data_y[i] = r
    y = tmp_data_y
    return x, y

def _download(centered):
    train_x, train_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)
    test_x, test_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)
    valid_x, valid_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)

    if centered:
        train_x += np.abs(train_x.min())
        test_x += np.abs(test_x.min())
        valid_x += np.abs(valid_x.min())

    train_set = (train_x.astype('float32'), train_t.astype('int32'))
    test_set = (test_x.astype('float32'), test_t.astype('int32'))
    valid_set = (valid_x.astype('float32'), valid_t.astype('int32'))
    return train_set, test_set, valid_set

def load_semi_supervised(centered):
    """
    Load the half moon dataset with 6 fixed labeled data points.
    """

    train_set, test_set, valid_set = _download(centered)

    # Add 6 static labels.
    train_x_l = np.zeros((6, 2))
    train_t_l = np.array([0, 0, 0, 1, 1, 1])
    # Top halfmoon
    train_x_l[0] = [.7, 1.7]  # left
    train_x_l[1] = [1.6, 2.6]  # middle
    train_x_l[2] = [2.7, 1.7]  # right

    # Bottom halfmoon
    train_x_l[3] = [1.6, 2.0]  # left
    train_x_l[4] = [2.7, 1.1]  # middle
    train_x_l[5] = [3.5, 2.0]  # right
    if not centered:
        train_x_l -= np.abs(train_set[0].min())

    train_set_labeled = (train_x_l, train_t_l)

    train_set_labeled = pad_targets(train_set_labeled)
    train_set = pad_targets(train_set)
    test_set = pad_targets(test_set)
    if valid_set is not None:
        valid_set = pad_targets(valid_set)

    return train_set, train_set_labeled, test_set, valid_set


from django.conf.urls import url
from blog import views

app_name = 'blog'
urlpatterns = [
    url(r'^post/(?P<post_id>\d+)$', views.show_post, name='show_post'),
    url(r'^add-post$', views.AddPostView.as_view(), name='add_post'),
    url(r'^tag/add/(?P<post_id>\d+)$', views.AddTagView.as_view(), name='add_tag'),
    url(r'^like/(?P<post_id>\d+)$', views.LikeView.as_view(), name='like'),
    url(r'^dislike/(?P<post_id>\d+)$', views.DislikeView.as_view(), name='dislike'),
    url(r'^delete/(?P<post_id>\d+)$', views.DeleteView.as_view(), name='delete'),
    url(r'^report/(?P<post_id>\d+)$', views.ReportView.as_view(), name='report'),
]


import json
import sys
import re
from get_directories import get_fichier_suffixe

final_result = set()
for date in json.loads(sys.argv[2]):
    with get_fichier_suffixe(str(sys.argv[1]), str(sys.argv[3]), str(date)) as file:

        '''
            get all the dates when analysing data by age growth by node for a BE type
            and a node given
        '''

        be_type = sys.argv[4]
        node = sys.argv[5]

        #if the structure of the csv files is not always the same, do:

        '''premiere_ligne = file.readline().split(",")
        premiere_ligne = [x.strip() for x in premiere_ligne]
        result = (x.split(",") for x in file)
        result = map(lambda i: [x.strip() for x in i], result)
        result = (x[premiere_ligne.index("start")] for x in result if (x[premiere_ligne.index("BE type")] == be_type and x[premiere_ligne.index("node")] == node))
        '''

        expression = re.compile('(?<='+ be_type + ',' + node + ',)[^,]*')
        result = filter(lambda i: True if (be_type in i and node in i) else False, file)
        result = map(lambda i: expression.search(i).group(), result)

        result = set(result)
        final_result = final_result.union(result)
print(json.dumps(list(final_result)))


from threading import Thread

import cv2

from obj_detection.tf_api.tf_object_detection_api import TFObjectDetectionAPI, \
    PRETRAINED_faster_rcnn_inception_v2_coco_2018_01_28, PRETRAINED_mask_rcnn_inception_v2_coco_2018_01_28, \
    PRETRAINED_faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28
from obj_detection.yolo_api.yolo_keras_object_detection_api import YOLOObjectDetectionAPI
from tf_session.tf_session_runner import SessionRunner
from tf_session.tf_session_utils import Inference

cap = cv2.VideoCapture(-1)
# cap = cv2.VideoCapture('/home/developer/Downloads/Hitman Agent 47 - car chase scene HD.mp4')

session_runner = SessionRunner()
while True:
    ret, image = cap.read()
    if ret:
        break

detection = YOLOObjectDetectionAPI('yolo_api', True)
detector_ip = detection.get_in_pipe()
detector_op = detection.get_out_pipe()
detection.use_session_runner(session_runner)
detection.use_threading()

session_runner.start()
detection.run()

while True:
    ret, image = cap.read()
    if not ret:
        continue
    detector_ip.push(Inference(image.copy()))
    detector_op.wait()
    ret, inference = detector_op.pull()
    if ret:
        i_dets = inference.get_result()
        # print(i_dets.get_masks()[0].shape)
        frame = i_dets.anotate()
        cv2.imshow("", i_dets.anotate())
        cv2.waitKey(1)
        # cv2.imwrite("/home/developer/Desktop/folder/" + (str(count).zfill(5)) + ".jpg", frame)

# Thread(target=detect_objects).start()


import argparse
import math
import numpy as np
from keras.models import Model
from keras.applications.vgg16 import VGG16
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input, decode_predictions
from pythonosc import osc_message_builder
from pythonosc import udp_client
import cv2
import pyautogui
from pythonosc import dispatcher
from pythonosc import osc_server
from PIL import Image
import time
import tensorflow as tf
import random

print("I'm working...")
model = VGG16(weights='imagenet')
graph = tf.get_default_graph()

# Fang og vis webcam:
video_capture = cv2.VideoCapture(0)

client = udp_client.SimpleUDPClient("127.0.0.1", 1234)

def send_osc_handler(unused_addr, args, message):
    print("unused: {}".format(unused_addr))
    print("args: {}".format(args))
    print("message: {}".format(message))
    global graph
    with graph.as_default():

        ret, frame = video_capture.read()

        cv2.imwrite('webcam.png',frame)

        img_path = 'webcam.png'
        img = cv2.resize(cv2.imread(img_path), (224, 224))
        cv2.imwrite("cv_output.png", img)
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)

        preds = model.predict(x)

        out = decode_predictions(preds, top=int(message))[0]

        print(out)
        output = str(out[0][1])
        cleaned = output.replace("_", " ")
        prob = str("{:.2%}".format(out[0][2]))
        print(cleaned)
        client.send_message("/isadora/1", "{}".format(cleaned))
        client.send_message("/isadora/2", "{}".format(prob))

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument("--ip",
      default="127.0.0.1", help="The ip to listen on")
  parser.add_argument("--port",
      type=int, default=5005, help="The port to listen on")
  args = parser.parse_args()

  dispatcher = dispatcher.Dispatcher()
  dispatcher.map("/filter", print)
  dispatcher.map("/matt", send_osc_handler, "Miklo")

  server = osc_server.ThreadingOSCUDPServer(
      (args.ip, args.port), dispatcher)
  print("Serving on {}".format(server.server_address))
  server.serve_forever()


import logging, os, sys
import paho.mqtt.client as mqtt
from importlib import import_module
from json import dumps as jsonify
from threading import Timer

SERVER = os.getenv('MQTT_SERVER')
PORT = int(os.getenv('MQTT_PORT'))
SECRET = os.getenv('SECRET')

headers = {'X-Secret': SECRET}

deviceClass = os.getenv('DEVICE_CLASS')
Device = import_module('{}.device'.format(deviceClass)).Device

device = Device()

# Logging setup
logging.basicConfig(format='%(asctime)s %(message)s', filename='/var/log/lightberry.log', level=logging.INFO)

def heartbeat(mqttc):
    mqttc.publish("{}/heartbeat".format(device.getId()), 'OK')
    Timer(3.0, heartbeat, [mqttc]).start()

def handleConnect(mqttc, obj, flags, rc):
    mqttc.subscribe('host/+', 0)
    mqttc.subscribe('{}/+'.format(device.getId()), 0)
    device.registerMqtt(mqttc)
    heartbeat(mqttc)

def handleServerMessage(mosq, obj, msg):
    device.registerMqtt(mqttc)

def logMessage(mosq, obj, msg):
    logging.info('Recieved message on %s', msg.topic)

mqttc = mqtt.Client(client_id=device.getId())

# Add message callbacks that will only trigger on a specific subscription match.
mqttc.message_callback_add('#', logMessage)
mqttc.message_callback_add('host/online', handleServerMessage)

mqttc.on_connect = handleConnect
mqttc.connect(SERVER, PORT, 60)

mqttc.loop_forever()