import unittest
from unittest.mock import Mock, PropertyMock, AsyncMock, AsyncMock, call
from beep import BeepBoop, Listener, NodeRunner, EventHandler, zemke_id, NodeRunnerError
from sseclient import Event
from discord import Guild, TextChannel
import json
import asyncio

message = {
  "id": 5900,
  "category": "SHOUTBOX",
  "author": {"username": "Zemke"},
  "body": "Here is a message",
}

class Env:

  def __init__(self, env):
    self.env = env

  def getenv(self, field):
    return self.env[field] if field in self.env else None

# todo append slash if it's not already there to SCRIPT env
#  and test this of course

class TestBeepBoop(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    cls.listener_mock = Mock()
    cls.listener_factory = lambda x: listener_mock()

  def test_init(self):
    client_mock = Mock()
    env = {"DISCORD_TOKEN": "fdasfdsa/"}
    beepBoop = BeepBoop(client_mock, Env(env).getenv, self.listener_mock)
    client_mock.run.assert_not_called()
    client_mock.listen.assert_not_called()

  def test_listen_not(self):
    client_mock = Mock()
    env = {}
    thread_factory_mock = Mock()
    beepBoop = BeepBoop(
        client_mock,
        Env(env).getenv,
        self.listener_mock,
        thread_factory_mock)
    thread_factory_mock.inst.assert_not_called()

  def test_listen_not(self):
    client_mock = Mock()
    endpoint = "http://example.com"
    env = {"LISTEN": "1", "CHANNEL": "1234", "CWT_MESSAGE_SSE_ENDPOINT": endpoint}
    thread_factory_mock = Mock()
    listener_mock = Mock()
    listener_factory = lambda x: listener_mock(*x, endpoint, None, None)
    beepBoop = BeepBoop(
        client_mock,
        Env(env).getenv, listener_factory,
        thread_factory_mock)
    thread_factory_mock.inst.assert_called_once()
    listener_mock.assert_called_once_with(
          client_mock, int(env["CHANNEL"]), endpoint, None, None)

class EventHandlerTest(unittest.TestCase):

  def create_channel_mock(self):
    channel_mock = Mock()
    guild_mock = Mock()
    type(guild_mock).id = PropertyMock(return_value=5)
    type(channel_mock).guild = guild_mock
    type(channel_mock).id = 1234
    type(channel_mock).send = AsyncMock()
    return channel_mock

  def create_message_mock(self, content, channel_mock):
    message_mock = Mock()
    author_mock = Mock()
    type(author_mock).display_name = PropertyMock(return_value="Zemke")
    type(message_mock).author = PropertyMock(return_value=author_mock)
    type(message_mock).content = PropertyMock(return_value=content)
    type(message_mock).channel = PropertyMock(return_value=channel_mock)
    return message_mock

  def test_on_message_beep_bop(self):
    client_mock = Mock()
    type(client_mock).user = PropertyMock(return_value=Mock())
    event_handler = EventHandler(client_mock, None, None)
    channel_mock = self.create_channel_mock()
    message_mock = self.create_message_mock("!cwt", channel_mock)
    actual = asyncio.get_event_loop().run_until_complete(
        event_handler.on_message(message_mock))
    message_mock.channel.send.assert_called_once_with(BeepBoop.say_beep_boop)

  def test_on_message_self(self):
    client_mock = Mock()
    type(client_mock).user = PropertyMock(return_value=Mock())
    node_runner_mock = Mock()
    expected = "whatevernodereturns"
    node_runner_mock.handle = Mock(return_value=expected)
    event_handler = EventHandler(client_mock, node_runner_mock, None)
    channel_mock = self.create_channel_mock()
    message_mock = self.create_message_mock("!cwtcommands", channel_mock)
    actual = asyncio.get_event_loop().run_until_complete(
        event_handler.on_message(message_mock))
    message_mock.channel.send.assert_called_once_with(expected)

  def test_on_direct_message_other_user(self):
    channel_mock = self.create_channel_mock()
    message_mock = self.create_message_mock("Hello, there", channel_mock)
    node_runner_mock = Mock()
    handle_return_value = "these are the commands"
    node_runner_mock.handle = Mock(return_value=handle_return_value)
    client_mock = Mock()
    eventHandler = EventHandler(client_mock, node_runner_mock, None)
    asyncio.get_event_loop().run_until_complete(
        eventHandler.on_direct_message(message_mock))
    calls = [call(EventHandler.other_user_dm_response), call(handle_return_value)]
    message_mock.channel.send.assert_has_calls(calls)
    self.assertEqual(len(message_mock.channel.send.mock_calls), 2)

  def test_on_direct_message_send_to_all_channels(self):
    channel_mock = self.create_channel_mock()
    content = "hello to all"
    message_mock = self.create_message_mock("!adminannounce - " + content, channel_mock)
    node_runner_mock = Mock()
    client_mock = Mock()
    user_n_author_mock = Mock()
    type(user_n_author_mock).id = PropertyMock(return_value=zemke_id)
    type(client_mock).user = PropertyMock(return_value=user_n_author_mock)
    type(message_mock).author = PropertyMock(return_value=user_n_author_mock)
    text_channel_mock = Mock(spec=TextChannel)
    text_channel_mock.send = AsyncMock()
    all_channels = [text_channel_mock]
    client_mock.get_all_channels = Mock(return_value=all_channels)
    eventHandler = EventHandler(client_mock, node_runner_mock, None)
    asyncio.get_event_loop().run_until_complete(
        eventHandler.on_direct_message(message_mock))
    # send to all channels
    text_channel_mock.send.assert_called_once_with(content)
    # send to dm
    message_mock.channel.send.assert_called_once_with(EventHandler.confirm_all_channels)

  def test_on_direct_message_send_to_env_channel(self):
    channel_mock = self.create_channel_mock()
    content = "hello to env channel"
    message_mock = self.create_message_mock("!adminannounce x " + content, channel_mock)
    node_runner_mock = Mock()
    client_mock = Mock()
    user_n_author_mock = Mock()
    type(user_n_author_mock).id = PropertyMock(return_value=zemke_id)
    type(client_mock).user = PropertyMock(return_value=user_n_author_mock)
    type(message_mock).author = PropertyMock(return_value=user_n_author_mock)
    target_channel = Mock()
    type(target_channel).id = PropertyMock(return_value=4321)
    target_channel.send = AsyncMock()
    client_mock.get_channel = Mock(return_value=target_channel)
    eventHandler = EventHandler(client_mock, node_runner_mock, target_channel.id)
    asyncio.get_event_loop().run_until_complete(
        eventHandler.on_direct_message(message_mock))
    client_mock.get_channel.assert_called_once_with(target_channel.id)
    # send to env channel
    target_channel.send.assert_called_once_with(content)
    # send to dm
    message_mock.channel.send.assert_called_once_with("Message has been sent.")

  def test_on_direct_message_send_to_specific_channel(self):
    channel_mock = self.create_channel_mock()
    target_channel = Mock()
    target_channel_id = 72608314
    type(target_channel).id = PropertyMock(return_value=target_channel_id)
    target_channel.send = AsyncMock()
    content = "hello to specific channel"
    message_mock = self.create_message_mock(
        "!adminannounce %s %s " % (str(target_channel_id), content), channel_mock)
    node_runner_mock = Mock()
    client_mock = Mock()
    user_n_author_mock = Mock()
    type(user_n_author_mock).id = PropertyMock(return_value=zemke_id)
    type(client_mock).user = PropertyMock(return_value=user_n_author_mock)
    type(message_mock).author = PropertyMock(return_value=user_n_author_mock)
    client_mock.get_channel = Mock(return_value=target_channel)
    eventHandler = EventHandler(client_mock, node_runner_mock, channel_mock.id)
    asyncio.get_event_loop().run_until_complete(
        eventHandler.on_direct_message(message_mock))
    client_mock.get_channel.assert_called_once_with(target_channel.id)
    # send to specific channel
    target_channel.send.assert_called_once_with(content)
    # send to dm
    message_mock.channel.send.assert_called_once_with(EventHandler.confirm_specific_channel)

class NodeRunnerTest(unittest.TestCase):

  def create_runner_mock(self, stdout):
    completed_process = Mock()
    runner = Mock(return_value=completed_process)
    completed_process.check_returncode = Mock()
    stdout_type = Mock()
    stdout_type.decode = Mock(return_value=stdout)
    type(completed_process).stdout = PropertyMock(return_value=stdout_type)
    return runner

  def test_format_shoutbox(self):
    script = "~/with/trailing/slash/"
    node = "Zemke via CWT: “Here is a messsage”"
    stdout = "some other ouput\nRES xx %s\n" % node
    node_runner = NodeRunner(script, self.create_runner_mock(stdout))
    actual = node_runner.format(message)
    self.assertEqual(actual, node);
    runner_args = ["node", script + "format.js", message["category"],
                   message["author"]["username"], message["body"]]
    node_runner.runner.assert_called_once_with(runner_args)

  def test_format_news(self):
    script = "../src/no/trailing/slash"
    node = "BoolC via CWT: “BoolC 3–1 Taner”"
    stdout = "some other ouput\nRES xx %s\n" % node
    node_runner = NodeRunner(script, self.create_runner_mock(stdout))
    message = {
      "category": "NEWS",
      "author": {"username": "BoolC"},
      "body": "BoolC,Taner,3,1",
      "newsType": "REPORT"
    }
    actual = node_runner.format(message)
    self.assertEqual(actual, node)
    runner_args = ["node", script + "/format.js", message["category"],
                   message["author"]["username"], message["body"],
                   message["newsType"]]
    node_runner.runner.assert_called_once_with(runner_args)

  def test_handle(self):
    script = "../src/no/trailing/slash"
    guildId = 1234
    channelId = 12345
    node = \
        "The CWT bot commands are !cwtchat, !cwtdice, !cwthell, !cwtterrain, " \
        "!cwtwinners, !cwtcommands, !cwtschedule, !cwtplayoffs, " \
        "!cwtwhatisthisthing, !cwtrafkagrass, !cwturl, !cwtgithub."
    stdout = "getting current tournament\n" \
           "cmd !cwtcommands\n" \
           "RES xx %s\n" % node
    node_runner = NodeRunner(script, self.create_runner_mock(stdout))
    actual = node_runner.handle("!cwtcommands", "Zemke", guildId, channelId)
    self.assertEqual(actual, node)
    runner_args = ["node", script + "/handle.js", "DISCORD",
                   'https://discord.com/channels/1234/12345',
                   "Zemke", "!cwtcommands"]
    node_runner.runner.assert_called_once_with(runner_args)

  def test_format_fail(self):
    runner = Mock()
    runner.check_returncode = Mock()
    stdout = "An error happened"
    node_runner = NodeRunner('.', self.create_runner_mock(stdout))
    self.assertRaises(NodeRunnerError, node_runner.format, message)

  def test_handle_fail(self):
    runner = Mock()
    runner.check_returncode = Mock()
    stdout = "An error happened"
    node_runner = NodeRunner('.', self.create_runner_mock(stdout))
    self.assertRaises(NodeRunnerError, node_runner.handle, "!cwtcommand", "Zemke", 1234, 12345)

class ListenerTest(unittest.TestCase):

  def test_listen(self):
    script = "./"
    node_runner = Mock()
    formatted = "Zemke via CWT: “Here is a message”"
    node_runner.format = Mock(return_value=formatted)
    channel = "1234"
    endpoint = "http://example.com"
    client_mock = Mock()
    guild_mock = Mock()
    channel_mock = Mock()
    type(channel_mock).id = PropertyMock(return_value=1234)
    type(guild_mock).text_channels = PropertyMock(return_value=[channel_mock])
    type(client_mock).guilds = PropertyMock(return_value=[guild_mock])
    message = {
      "id": 5000,
      "category": "SHOUTBOX",
      "author": {"username": "Zemke"},
      "body": "Here is a message",
    }
    events = [Event(event="EVENT", data=json.dumps(message))]
    open_stream = type("SSEClient", (object,), {"events": lambda x: events })
    listener = Listener(client_mock, channel, node_runner, open_stream)
    cb_mock = Mock()
    listener.listen(1, cb_mock)
    cb_mock.assert_called_once_with(int(channel), formatted)

  def test_process_message_empty(self):
    message = {}
    channel = "1234"
    listener = Listener(Mock(), channel, Mock(), Mock())
    cb = Mock()
    with self.assertLogs('discord', level='WARN') as logged:
      listener.process_message(message, channel, cb)
    cb.assert_not_called()
    self.assertIn('WARNING:discord:Data has no ID: {}', logged.output)

  def test_process_message_other_attr_missing(self):
    message = {"body": "hello there", "id": 4000}
    node_runner = Mock()
    node_runner.format = Mock(side_effect=Exception('any error'))
    listener = Listener(Mock(), None, node_runner, Mock())
    cb = Mock()
    with self.assertLogs('root', level='ERROR') as logged:
      listener.process_message(message, "1234", cb)
    cb.assert_not_called()
    self.assertTrue(len(logged.output) == 1)
    self.assertIn('ERROR:root:Error while sending', logged.output[0])

if __name__ == '__main__':
    unittest.main()


from setuptools import setup

package_name = 'reader'
filename = package_name + '.py'

def get_long_description():
    try:
        with open('README.md', 'r') as f:
            return f.read()
    except IOError:
        return ''

setup(
    name=package_name,
    version=0.1,
    author='xyzinnovation',
    author_email='arun.prasad@xyzinnotech.com',
    description='rfid for attendance',
    url='https://github.com/Arun1930/reader',
    long_description=get_long_description(),
    py_modules=[package_name],
    entry_points={
        'console_scripts': [
            'reader = reader:main',

        ]
    },
    license='License :: OSI Approved :: MIT License',
)


from flask import Blueprint, render_template, abort
from app.core.controller_by_data_output import ControllerByDataOutput
from app.core.fileutils import is_video, is_image

module = Blueprint("analytics", __name__)

@module.route("/display/<filename>")
def analytics_show_media(filename: str) -> str:
    output = ControllerByDataOutput()
    view_data = output.get_view_data(filename)

    if is_image(filename):
        return render_template("analytics2.html", image_file=view_data.box_path, image_chart_file=view_data.chart_path, messages=view_data.messages)
    elif is_video(filename):
        return render_template("analytics.html", video_file="upload/box/" + filename)
    abort(400)


"""
@author: ZacVND

k-fold cross-validation using LOO
"""

from datetime import datetime
from os import path
import numpy as np
import random

from ie_tools.src.classifier import Classifier
from ie_tools.src import util

def run():
    # choose between TypeRF, TypeDT, TypeSVM
    classifier_type = Classifier.TypeRF
    run_count = 20
    fold_count = 8
    max_papers = 120
    paper_paths = util.get_paper_paths()[:max_papers]
    paper_count = len(paper_paths)
    step_size = int(paper_count / fold_count)

    new_data = False
    if new_data:
        new_data_dir = util.get_new_data_dir()
        new_pps = util.get_paper_paths(dir=new_data_dir)
        paper_paths.extend(new_pps)

    curr_i = 1
    running_sum = 0
    running_pre_sum = [0] * 6

    print("---- INFO: First paper will always take longer to run than the "
          "subsequent papers because we starts genia tagger.\n\n")

    for run in range(run_count):
        random.shuffle(paper_paths)
        for fold in range(fold_count):
            train_pps = paper_paths[:step_size * fold]
            train_pps += paper_paths[step_size * (fold + 1):]
            test_pps = paper_paths[step_size * fold:step_size * (fold + 1)]
            classifier = Classifier(clf_type=classifier_type)
            classifier.train(train_pps)
            current_total_loss, current_precisions = classifier.test(test_pps)

            running_avg = np.divide(running_sum, curr_i) + \
                          np.divide(current_total_loss, curr_i)

            running_sum = np.add(running_sum, current_total_loss)

            running_pre_avg = np.divide(running_pre_sum, curr_i) + \
                              np.divide(current_precisions, curr_i)

            running_pre_sum = np.add(running_pre_sum, current_precisions)

            running_pre_avg = [np.round(rpa, 4) for rpa in running_pre_avg]

            print("Running average: {}\n".format(np.round(running_avg, 4)))
            print("Running precisions average:\nA1:{}\t A2:{}\t R1:{}\t "
                  "R2:{}\t OC:{}\t P:{}\n\n".format(running_pre_avg[0],
                                                    running_pre_avg[1],
                                                    running_pre_avg[2],
                                                    running_pre_avg[3],
                                                    running_pre_avg[4],
                                                    running_pre_avg[5]))

            curr_i += 1

    avg_total_loss = np.round(np.divide(running_sum,
                                        (run_count * fold_count)), 4)
    print("average total loss for {}-fold cross validation".format(fold_count),
          avg_total_loss)

    avg_total_precisions = np.round(np.divide(running_pre_sum,
                                              (run_count * fold_count)), 4)
    print("average precision for {}-fold cross validation".format(fold_count),
          avg_total_precisions)

    date_str = datetime.now().strftime('%Y-%m-%d_%H-%M')
    out_file = path.join(util.get_result_dir(), "{}-results-{}.txt".format(
        classifier_type, date_str))
    with open(out_file, 'w+') as file:
        file.write("Average total loss across {} runs of {}-fold cross "
                   "validation:\n{}\n\n".format(run_count, fold_count,
                                                avg_total_loss))
        file.write("Average precisions across {} runs of {}-fold cross "
                   "validation:\nA1:{}\t A2:{}\t R1:{}\t R2:{}\t "
                   "OC:{}\t P:{}".format(run_count, fold_count,
                                         avg_total_precisions[0],
                                         avg_total_precisions[1],
                                         avg_total_precisions[2],
                                         avg_total_precisions[3],
                                         avg_total_precisions[4],
                                         avg_total_precisions[5]))

if __name__ == "__main__":
    run()


import pytest

# from django.contrib.auth.models import User
from django.test import TestCase
from ..models import Tag, Space, Thing

# from utils.tests.conftest import ConfTest

@pytest.mark.django_db
class TagModelTest(TestCase):
    def test_tag_create(self):
        new_tag = Tag()
        new_tag.title = "new_tag"
        new_tag.save()
        assert new_tag.title == "new_tag"

    def test_tag_read(self):
        new_tag = Tag()
        new_tag.title = "new_tag"
        new_tag.save()
        assert new_tag.title == "new_tag"

@pytest.mark.django_db
class SpaceModelTest(TestCase):
    def test_space_model_persists(self):
        new_space = Space.objects.create(title="new_space")
        assert new_space.title == "new_space"

class ThingModelTest(TestCase):
    @pytest.mark.django_db
    def test_tag_model_persists(self):
        new_thing = Thing()
        new_thing.title = "new_thing"
        new_thing.save()
        assert new_thing.title == "new_thing"


from collections import defaultdict

from ..models import Recipe, RecipeIngredient, Ingredient, Kitchen, User, Subtype

def find_recipes(db, ingredients_str):
    """
    Finds all the recipes that can be made with the given ingredients

    Args:
        db: database session
        ingredients: (List[str]) ingredients

    Returns:
        (List[Recipe])
    """
    ingredients = set()
    for ingredient_str in set(ingredients_str):
        ingredient = db.query(Ingredient).filter_by(name=ingredient_str).first()
        if ingredient:
            ingredients.add(ingredient.id)

    # Add subtypes of every ingredient
    ingredients = ingredients.union(get_subtypes(db, ingredients))

    ingredient_pieces = defaultdict(set)
    for ingredient in ingredients:
        recipe_pieces = db.query(RecipeIngredient).filter_by(ingredient_id=ingredient).all()
        for recipe_piece in recipe_pieces:
            if recipe_piece.required == 1:  # Only want to consider required ingredients
                ingredient_pieces[recipe_piece.recipe_id].add(recipe_piece)

    found_recipes = set()
    for possible_recipe_id, recipe_pieces in ingredient_pieces.items():
        full_recipe = set(db.query(RecipeIngredient).filter_by(recipe_id=possible_recipe_id,
                                                               required=1).all())
        if full_recipe == recipe_pieces:
            found_recipes.add(possible_recipe_id)

    return [format_recipe(db, found_recipe) for found_recipe in found_recipes]

def find_all_recipes(db, ingredients_str):
    ingredients = set()
    for ingredient_str in set(ingredients_str):
        ingredient = db.query(Ingredient).filter(Ingredient.name==ingredient_str).first()
        if ingredient:
            ingredients.add(ingredient.id)

    ingredients = ingredients.union(get_subtypes(db, ingredients))

    recipes = set()
    for ingredient in ingredients:
        for found_recipe in db.query(RecipeIngredient).filter(RecipeIngredient.ingredient_id==ingredient).all():
            if found_recipe:
                recipes.add(found_recipe.recipe_id)
    return [format_recipe(db, recipe) for recipe in recipes]

def format_recipe(db, found_recipe):
    """
    Formats a found recipe to print onto the page

    Args:
        db: database session
        found_recipe: (int) id of recipe

    Returns:
        (str) Pretty format of recipe
    """
    recipe = db.query(Recipe).filter(Recipe.id==found_recipe).first()
    if not recipe:
        return ''
    return_str = f'

-: {recipe.name} :-
'
    return_str += '
 Ingredients
'
    for ingredient in recipe.recipe_ingredient:
        ingredient_name = db.query(Ingredient.name).filter(Ingredient.id==ingredient.ingredient_id).first()[0]
        return_str += f'+ {ingredient_name}: {ingredient.quantity} {ingredient.unit}
'
    return_str += f'

Directions
{pretty_directions(recipe.directions)}'
    return return_str

def pretty_directions(directions_str):
    """
    Puts a newline before each number in directions.

    Args:
        directions_str: (str) Looks like "1. something. 2. Something else"

    Returns:
        (str) pretty directions that look like:
            1. something
            2. something else
    """
    step = 2
    while True:
        old_str = directions_str
        directions_str = directions_str.replace(f'{step}. ', f'
{step}. ')
        if old_str == directions_str:
            break
        step += 1
    return directions_str

def get_initial_list(db, user):
    user_id = db.query(User).filter(User.name == user).first().id
    ingredient_ids = {k.ingredient_id for k in db.query(Kitchen).filter(Kitchen.user_id == user_id).all()}
    ingredient_names = [k.name for k in db.query(Ingredient).filter(Ingredient.id.in_(ingredient_ids)).all()]
    return ingredient_names

def get_subtypes(db, ingredients):
    """
    Finds all the subtypes of an ingredient

    Returns:
        (set[int]) a set of all id's for specific ingredients
    """
    new_ingredients = set()
    for ingredient in ingredients:
        subtypes = db.query(Subtype).filter(Subtype.generic==ingredient).all()
        for subtype in subtypes:
            new_ingredients.add(subtype.specific)
    return new_ingredients


import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
import time
import copy
from torch.distributions import Normal, MultivariateNormal, Beta
from ReplayBuffer import ReplayBuffer
from utils import action_adapter
import math

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class BetaActor(nn.Module):
    def __init__(self, obs_shape, action_shape, hidden_num=128):
        super(BetaActor, self).__init__()

        self.l1 = nn.Linear(obs_shape, hidden_num)
        self.l2 = nn.Linear(hidden_num, hidden_num)
        self.alpha_head = nn.Linear(hidden_num, action_shape)
        self.beta_head = nn.Linear(hidden_num, action_shape)

    def forward(self, input):
        a = torch.tanh(self.l1(input))
        a = torch.tanh(self.l2(a))

        alpha = F.softplus(self.alpha_head(a)) + 1.0
        beta = F.softplus(self.beta_head(a)) + 1.0

        return alpha, beta

    def get_dist(self, state):
        alpha, beta = self.forward(state)
        dist = Beta(alpha, beta)
        return dist

    def dist_mode(self, state):
        alpha, beta = self.forward(state)
        mode = (alpha) / (alpha + beta)
        return mode

class actor(nn.Module):
    def __init__(self, obs_shape, action_shape, hidden_num=128):
        super(actor, self).__init__()
        # parameters
        self.obs_shape = obs_shape
        self.action_shape = action_shape

        # network
        self.net = nn.Sequential(
            nn.Linear(obs_shape, hidden_num),
            nn.Tanh(),
            nn.Linear(hidden_num, hidden_num),
            nn.Tanh()
        )

        self.mu_head = nn.Linear(hidden_num, action_shape)
        self.sigma_head = nn.Linear(hidden_num, action_shape) + 0.01

    def forward(self, input):
        x = self.net(input)
        mu = torch.sigmoid(self.mu_head(x))
        sigma = F.softplus(self.sigma_head(x)) + 0.01

        return mu, sigma

    def get_dist(self, input):
        mu, sigma = self.forward(input)
        dist = Normal(mu, sigma)
        return dist

class critic(nn.Module):
    def __init__(self, obs_shape, hidden_num=128):
        super(critic, self).__init__()
        # parameters
        self.obs_shape = obs_shape

        # network
        self.net = nn.Sequential(
            nn.Linear(obs_shape, hidden_num),
            nn.Tanh(),
            nn.Linear(hidden_num, hidden_num),
            nn.Tanh(),
            nn.Linear(hidden_num, 1)
        )

    def forward(self, input):
        value = self.net(input)
        return value

class PPO(object):
    '''
    agent类
    '''

    def __init__(
            self,
             obs_shape,
             action_shape,
             buffer=ReplayBuffer(10000),
             max_grad_norm=40,
             gamma=0.99,
             lambd=0.95,
             K_epochs=10,
             a_optim_batch_size=64,
             c_optim_batch_size=64,
             clip_param=0.2,
             l2_reg=1e-3,
             actor_lr=0.0001,
             critic_lr=0.0001,
             entropy_coef = 0.001,
             entropy_coef_decay=0.9998,
             actor=None,
             critic=None
             ):
        super(PPO, self).__init__()
        self.obs_shape = obs_shape
        self.action_shape = action_shape
        self.buffer = buffer

        self.training_step = 0
        self.max_grad_norm = max_grad_norm

        self.gamma = gamma
        self.lambd = lambd
        self.K_epochs = K_epochs
        self.a_optim_batch_size = a_optim_batch_size
        self.c_optim_batch_size = c_optim_batch_size

        self.clip_param = clip_param
        self.l2_reg = l2_reg

        self.actor_lr = actor_lr
        self.critic_lr = critic_lr
        self.actor = actor.to(device)
        self.critic = critic.to(device)

        self.entropy_coef = entropy_coef
        self.entropy_coef_decay = entropy_coef_decay

        self.data = []

        # optimizer
        self.actor_optim = torch.optim.Adam(self.actor.parameters(), self.actor_lr)
        self.critic_optim = torch.optim.Adam(self.critic.parameters(), self.critic_lr)

    def take_action(self, obs):
        '''
        产生一个动作，并返回动作和其概率值

        :param obs: agent的observation
        :return: action和action的概率
        '''

        # 将numpy格式转为tensor格式，并且增加batch维度
        with torch.no_grad():
            obs = torch.from_numpy(obs).float().unsqueeze(0).to(device)
            dist = self.actor.get_dist(obs)
            action = dist.sample()
            action_log_prob = dist.log_prob(action)
            # 最好将均值mu弄到动作空间的均值
            action = action.clamp(-2, 2)  # TODO(将此处改为参数),将action限制在[-1,1]，超过的按2算，不够的按-2算
            return action.cpu().flatten().numpy(), action_log_prob.sum().cpu().item()  # 返回np格式数据，而不是tensor格式,注意要把log概率之和加起来（log内相乘）

    def get_value(self, obs):
        '''
        获得critic值

        :param obs:
        :return:
        '''
        with torch.no_grad():
            value = self.critic(obs)
            return value

    def save_model(self, path='../param', para=None):
        '''
        保存模型参数

        :param path:
        :return:
        '''
        torch.save(self.actor.state_dict(), path + para + '_' + str(time.time())[:10] + '_actor.pkl')
        torch.save(self.critic.state_dict(), path + para + '_' + str(time.time())[:10] + '_critic.pkl')

    # def update(self, batch_size =256, type = 'clip'):
    #
    #     obs, action, reward, log_old_prob, next_obs, done = self.buffer.sample(batch_size=batch_size)
    #     for i in range(self.K_epoch):
    #         next_v = self.get_value(next_obs)
    #         current_v = self.get_value(obs)
    #         target_v = reward + self.gamma * next_v * (1 - done.float())
    #         delta = target_v - current_v  # TD error
    #         delta = delta.cpu().flatten().numpy()
    #         done = done.cpu().flatten().numpy()
    #
    #         '''计算GAE'''
    #         adv = [0]
    #         for dlt, done in zip(delta[::-1], done[::-1]):  # batch内反向计算
    #             advantage = dlt + self.gamma * self.lambd * adv[-1] * (1 - done)
    #             adv.append(advantage)
    #         adv.reverse()  # 再反向回来，就是batch内每个数据对应的advantage
    #         adv = copy.deepcopy(adv[0:-1])  # 把初始化的adv = [0]去掉
    #         adv = torch.tensor(adv).unsqueeze(1).float().to(device)  # 增加batch维度
    #         td_target = adv + current_v
    #         adv = (adv - adv.mean()) / ((adv.std() + 1e-4))  # 归一化后，训练更稳定
    #         '''结束'''
    #
    #         if type == 'clip':
    #             # 计算概率动作之比
    #             mu, sigma = self.actor(obs)
    #             n = Normal(mu, sigma)
    #             action_log_prob = n.log_prob(action).sum(dim=-1, keepdim=True)  # 概率相乘（相当于log相加）
    #             ratio = torch.exp(action_log_prob - log_old_prob)  # TODO 问题：这里两个概率值一直一样，比率为1
    #
    #             # 计算两个loss，之后选取最小值
    #             L1 = ratio * adv
    #             L2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * adv
    #             action_loss = -torch.min(L1, L2).mean()
    #             self.actor_optim.zero_grad()
    #             action_loss.backward()
    #             nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
    #             self.actor_optim.step()
    #             print('pi_loss : {}'.format(action_loss))
    #
    #         else:
    #             pass  # TODO PPO1，利用KL divergence计算
    #
    #         # 更新critic
    #         v_loss = self.critic_loss(self.critic(obs), td_target) + self.l2_reg  # TODO 这里要重新使用self.critic(obs)，这里需要梯度
    #         self.critic_optim.zero_grad()
    #         v_loss.backward()
    #         nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
    #         self.critic_optim.step()
    #         print('v_loss : {}'.format(v_loss))
    #
    #     return

    # def update(self, type='clip'):
    #     '''
    #     更新actor和critic
    #
    #     batch_size: batch_size为horizon T,一个T中可能包含多个episode，也可能多个T才时一个episode
    #     :return:
    #     '''
    #
    #     self.training_step += 1
    #
    #     # batch数据
    #     obs, action, reward, next_obs, log_old_prob, done = self.make_batch()
    #
    #     # TODO 计算优势估计,这里不需要梯度
    #     with torch.no_grad():
    #         current_v = self.critic(obs)
    #         next_v = self.critic(next_obs)
    #         target_v = reward + self.gamma * next_v * (1 - done.float())
    #
    #         delta = target_v - current_v  # TD error
    #         delta = delta.cpu().flatten().numpy()
    #         done = done.cpu().flatten().numpy()
    #
    #         '''计算GAE'''
    #         adv = [0]
    #         for dlt, d in zip(delta[::-1], done[::-1]):  # batch内反向计算
    #             advantage = dlt + self.gamma * self.lambd * adv[-1] * (1 - d)
    #             adv.append(advantage)
    #         adv.reverse()  # 再反向回来，就是batch内每个数据对应的advantage
    #         adv = copy.deepcopy(adv[0:-1])  # 把初始化的adv = [0]去掉
    #         adv = torch.tensor(adv).unsqueeze(1).float().to(device)  # 增加batch维度
    #         td_target = adv + current_v
    #         adv = (adv - adv.mean()) / ((adv.std() + 1e-4))  # 归一化后，训练更稳定
    #         '''结束'''
    #
    #     a_optim_iter_num = int(math.ceil(obs.shape[0] / self.a_optim_batch_size))
    #     c_optim_iter_num = int(math.ceil(obs.shape[0] / self.c_optim_batch_size))
    #     # 更新actor
    #     for j in range(self.K_epoch):
    #
    #         # 用batch更新，先Shuffle数据，取得一个batch数据
    #         perm = np.arange(obs.shape[0])  # 整个数据的长度
    #         np.random.shuffle(perm)
    #         perm = torch.LongTensor(perm).to(device)
    #         obs, action, td_target, adv, log_old_prob = \
    #             obs[perm].clone(), action[perm].clone(), td_target[perm].clone(), adv[perm].clone(), log_old_prob[
    #                 perm].clone()
    #
    #         for i in range(a_optim_iter_num):
    #             index = slice(i * self.a_optim_batch_size,
    #                           min((i + 1) * self.a_optim_batch_size, obs.shape[0]))  # minibatch的切片
    #             old = log_old_prob[index]
    #
    #             # if type == 'clip':
    #             # 计算概率动作之比
    #             dist = self.actor.get_dist(obs[index])
    #             action_log_prob = dist.log_prob(action[index]).sum(dim=-1, keepdim=True)  # 概率相乘（相当于log相加）
    #             dist_entropy = dist.entropy().sum(1, keepdim=True)
    #             ratio = torch.exp(action_log_prob - old)
    #
    #             # 计算两个loss，之后选取最小值
    #             L1 = ratio * adv[index]
    #             L2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * adv[index]
    #             action_loss = -torch.min(L1, L2) - self.entropy_coef * dist_entropy  # TODO action_loss修改
    #             self.actor_optim.zero_grad()
    #             action_loss.mean().backward()
    #             ####################################
    #             # for name, parms in self.actor.named_parameters():
    #             #     print('-->name:', name, '-->grad_requirs:', parms.requires_grad, \
    #             #           ' -->grad_value:', parms.grad.mean(),'-->parms', parms)
    #
    #             ####################################
    #             nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
    #             self.actor_optim.step()
    #
    #         # 更新critic
    #         for i in range(c_optim_iter_num):
    #             index = slice(i * self.c_optim_batch_size, min((i + 1) * self.c_optim_batch_size, obs.shape[0]))
    #
    #             v_loss = F.mse_loss(self.critic(obs[index]), td_target[index])
    #             for name, param in self.critic.named_parameters():
    #                 if 'weight' in name:
    #                     v_loss += param.pow(2).sum() * self.l2_reg
    #             self.critic_optim.zero_grad()
    #             v_loss.backward()
    #
    #             #####################################
    #             # for name, parms in self.critic.named_parameters():
    #             #     print('-->name:', name, '-->grad_requirs:', parms.requires_grad, \
    #             #           ' -->grad_value:', parms.grad)
    #             #####################################
    #
    #             nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
    #             self.critic_optim.step()
    #
    #     return self.training_step, v_loss, action_loss

    def update(self):
        self.entropy_coef *= self.entropy_coef_decay
        s, a, r, s_prime, logprob_a, done_mask = self.make_batch()

        ''' Use TD+GAE+LongTrajectory to compute Advantage and TD target'''
        with torch.no_grad():
            vs = self.critic(s)
            vs_ = self.critic(s_prime)

            '''dw for TD_target and Adv'''
            deltas = r + self.gamma * vs_ * (1 - done_mask) - vs

            deltas = deltas.cpu().flatten().numpy()
            adv = [0]

            '''done for GAE'''
            for dlt, mask in zip(deltas[::-1], done_mask.cpu().flatten().numpy()[::-1]):  # 这里reverse一下
                advantage = dlt + self.gamma * self.lambd * adv[-1] * (1 - mask)
                adv.append(advantage)
            adv.reverse()
            adv = copy.deepcopy(adv[0:-1])
            adv = torch.tensor(adv).unsqueeze(1).float().to(device)
            td_target = adv + vs
            adv = (adv - adv.mean()) / ((adv.std() + 1e-4))  # sometimes helps

        """Slice long trajectopy into short trajectory and perform mini-batch PPO update"""
        a_optim_iter_num = int(math.ceil(s.shape[0] / self.a_optim_batch_size))
        c_optim_iter_num = int(math.ceil(s.shape[0] / self.c_optim_batch_size))
        for i in range(self.K_epochs):

            # Shuffle the trajectory, Good for training
            perm = np.arange(s.shape[0])
            np.random.shuffle(perm)
            perm = torch.LongTensor(perm).to(device)
            s, a, td_target, adv, logprob_a = \
                s[perm].clone(), a[perm].clone(), td_target[perm].clone(), adv[perm].clone(), logprob_a[perm].clone()

            '''update the actor，用minibatch更新，i是minibatch得index'''
            for i in range(a_optim_iter_num):
                index = slice(i * self.a_optim_batch_size,
                              min((i + 1) * self.a_optim_batch_size, s.shape[0]))  # minibatch的切片
                distribution = self.actor.get_dist(s[index])
                dist_entropy = distribution.entropy().sum(1, keepdim=True)
                logprob_a_now = distribution.log_prob(a[index])
                old = logprob_a[index]
                ratio = torch.exp(
                    logprob_a_now.sum(1, keepdim=True) - old)  # a/b == exp(log(a)-log(b))

                surr1 = ratio * adv[index]
                surr2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * adv[index]
                a_loss = -torch.min(surr1, surr2) - self.entropy_coef * dist_entropy

                self.actor_optim.zero_grad()
                a_loss.mean().backward()

                ############################
                # for name, parms in self.actor.named_parameters():
                # 	print('-->name:', name, '-->grad_requirs:', parms.requires_grad, \
                # 		  ' -->grad_value:', parms.grad.mean(), '-->parms', parms)
                #####################
                torch.nn.utils.clip_grad_norm_(self.actor.parameters(), 40)
                self.actor_optim.step()

            '''update the critic'''
            for i in range(c_optim_iter_num):
                index = slice(i * self.c_optim_batch_size, min((i + 1) * self.c_optim_batch_size, s.shape[0]))
                c_loss = (self.critic(s[index]) - td_target[index]).pow(2).mean()
                for name, param in self.critic.named_parameters():
                    if 'weight' in name:
                        c_loss += param.pow(2).sum() * self.l2_reg

                self.critic_optim.zero_grad()
                c_loss.backward()
                self.critic_optim.step()

    def evaluate(self, obs):
        '''
        测试模型，训练不调用，只在测试时使用

        :return:一个动作
        '''
        with torch.no_grad():
            obs = torch.from_numpy(obs).float().unsqueeze(0).to(device)
            dist = self.actor.get_dist(obs)
            action = dist.sample()
            action = action.clamp(-2, 2)  # TODO(将此处改为参数),将action限制在[-1,1]，超过的按1算，不够的按-1算
            return action.cpu().flatten().numpy()

    def make_batch(self):
        '''
        将数据弄成batch

        :return:
        '''
        s_lst, a_lst, r_lst, s_prime_lst, logprob_a_lst, done_lst = [], [], [], [], [], []
        for transition in self.data:
            s, a, r, s_prime, logprob_a, done = transition

            s_lst.append(s)
            a_lst.append(a)
            logprob_a_lst.append([logprob_a])
            r_lst.append([r])
            s_prime_lst.append(s_prime)
            done_lst.append([done])

        self.data = []  # Clean history trajectory

        '''list to tensor'''
        with torch.no_grad():
            s, a, r, s_prime, logprob_a, done = \
                torch.tensor(s_lst, dtype=torch.float).to(device), \
                torch.tensor(a_lst, dtype=torch.float).to(device), \
                torch.tensor(r_lst, dtype=torch.float).to(device), \
                torch.tensor(s_prime_lst, dtype=torch.float).to(device), \
                torch.tensor(logprob_a_lst, dtype=torch.float).to(device), \
                torch.tensor(done_lst, dtype=torch.float).to(device)

        return s, a, r, s_prime, logprob_a, done

    def put_data(self, transition):
        self.data.append(transition)


import json

data = {
    'sales': [
        {'day' : 'Monday', 'revenue' : 100},
        {'day' : 'Tuesday', 'revenue' : 50},
        {'day' : 'Wednesday', 'revenue' : 80},
        {'day' : 'Thursday', 'revenue' : 120},
        {'day' : 'Friday', 'revenue' : 20},
        {'day' : 'Saturday', 'holiday' : True},
        {'day' : 'Sunday', 'holiday' : True},
    ],
    'expense': 14.99,
    'fires' : None
}

# write data
s = json.dumps(data)
print(s)

# read data
read_data = json.loads(s)
print(read_data)
# print(sum([day for day in read_data['sales'] if 'revenue' in day.keys()]))


import json
import pandas_datareader as pdr
import datetime
import schedule,time,datetime

def make_list():
# リストを読み込む
    tickerfile = "./ticker.txt"
    tickers = open(tickerfile, "r")
    tickerslist = json.load(tickers)
    tickers.close

# パラメータ作成
    stockTickerlist = [str(t)+'.JP' for t in tickerslist["ticker"]]
    startdate = datetime.date(2012,1,1)
    enddate = datetime.date.today()
    df = pdr.DataReader(stockTickerlist, 'stooq', startdate, enddate)
    print(len(df.index))
    stockData = df["Close"]
    stockData.to_csv("./stock_price_data.csv", encoding = "utf-8")

# 毎日10:00に実行
schedule.every().day.at("10:00").do(make_list)

while True:
    schedule.run_pending()
    time.sleep(1)


from django.conf.urls import url
from django.urls import path
from . import views
from django.contrib.auth.views import LoginView, LogoutView

urlpatterns = [
    url(r'^$', views.home),
    url(r'^test/$', views.new),
    url(r'^logout/$', LogoutView.as_view(template_name='accounts/logout.html'),name='logout'),
    url(r'^register/$', views.register,name = 'register'),
    url(r'^registerS/$', views.registerS,name = 'registerStudent'),
    url(r'^registerT/$', views.registerT,name = 'registerTutor'),
    url(r'^about/$', views.about,name = 'about'),
    #url(r'^profile/(\w+)/', views.profile, name = 'profile'),
    path('profile/', views.profile, name='profile'),
]


"""
Script:    Module to set-up logging structure
"""

# Generic
import logging
import logging.handlers
import os
import sys

_INT_TO_LEVEL = {
    5: logging.CRITICAL,
    4: logging.ERROR,
    3: logging.WARNING,
    2: logging.INFO,
    1: logging.DEBUG,
    0: logging.NOTSET,
}

def setup_logging(name, time_stamp, args):
    """
    Sets up the logger for the classification.
    :param name: Logger instance name
    :param time_stamp: The timestamp to apply to the file
    :param args: a dictionary with the arguments which are used by the classifier.
    This dict will be modified, removing items which shouldn't be sent to the classification function.
    :return: None
    """

    named_level = logging.DEBUG

    log_dir = args['LOG_DIR']
    del args['LOG_DIR']

    level = args['LOG_LEVEL']
    del args['LOG_LEVEL']

    try:
        named_level = _INT_TO_LEVEL[level]
    except IndexError:
        print('[WARNING] - Verbosity non-existent, using DEBUG')

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    print("Logging to {} named: {}".format(log_dir, name))

    log_file = '{}_{}.log'.format(name, time_stamp)

    # Logger
    my_log = logging.getLogger(name)
    my_log.propagate = False
    my_log.handlers = []
    my_log.setLevel(named_level)

    formatter = logging.Formatter('%(asctime)s [%(threadName)s] [%(levelname)s]'
                                  ' ([%(filename)s::%(funcName)s) :: %(message)s',
                                  datefmt='%m/%d/%Y %H:%M:%S')

    # Handlers
    fh_path = os.path.join(log_dir, log_file)

    file_handler = logging.FileHandler(fh_path)
    file_handler.setFormatter(formatter)

    std_handler = logging.StreamHandler(sys.stdout)
    std_handler.setFormatter(formatter)

    my_log.addHandler(file_handler)
    my_log.addHandler(std_handler)

def get_logger(name):
    """
    Returns logger for the given name
    :param name: Logger name
    :return: None
    """
    return logging.getLogger(name)

def print_imports_versions(logger):
    """
    Prints on logger the information about the version of all the imported modules
    :param logger: Logging object to be used
    :return: None
    """
    for name, module in sorted(sys.modules.items()):
        if hasattr(module, '__version__'):
            logger.debug('{0} :: {1}'.format(name, module.__version__))

def an_print(logger, string=''):
    """
    Prints if logger level is 10
    :param logger: Logger instance
    :param string: String to be printed
    :return: None
    """
    if logger is not None:
        if logger.getEffectiveLevel() == 10:
            print(string)


import matplotlib.pyplot as py
import random
x = range(50)
y_shanghai= [random.uniform(15,18) for i in x]
y_beijing= [random.uniform(1,10) for i in x]

figure,axes = py.subplots(nrows = 1,ncols=2 )

axes[0].plot(x,y_shanghai,color='r',label="shanghai")
axes[1].plot(x,y_beijing,color='b',label="beijing")

#修改xy刻度
x_label = [ "11点{}分".format(i) for i in x]   #注意用{}
axes[0].set_xticks(x[::5],x_label[::5])
axes[0].set_yticks(range(0,40,5))
axes[1].set_xticks(x[::5],x_label[::5])
axes[1].set_yticks(range(0,40,5))

#comment
axes[0].grid(True)
axes[0].set_title("summary")
axes[0].set_xlabel("x-scale")
axes[0].set_ylabel("y-scale")
axes[0].legend(loc="upper right")   #显示图例
axes[1].grid(True)
axes[1].set_title("summary")
axes[1].set_xlabel("x-scale")
axes[1].set_ylabel("y-scale")
axes[1].legend(loc="upper right")   #显示图例

py.show()


#!/usr/bin/env python
# encoding: utf-8
import datetime
import json
import logging
import threading
import time

import redis

from backend_python.settings import *
from hermes.models.HermesModels import HermesDevice

logger = logging.getLogger(__name__)

# 设置本地连接池
def create_redis_client():
    if disable_redis:
        return None
    redis_param = {}
    if not redis_host or not redis_port:
        raise Exception('redis_host and redis_port config can not be empty')
    redis_param['host'] = redis_host
    redis_param['port'] = redis_port
    if redis_auth:
        # TODO
        redis_auth['auth'] = redis_auth
    redis_param['decode_responses'] = True
    pool = redis.ConnectionPool(**redis_param)
    return redis.Redis(connection_pool=pool)

try:
    redis_client = create_redis_client()
except Exception as e:
    redis_client = None
    logger.error("create redis client failed,redis based queue will not enable %s", e)

class ServiceQueue(object):
    """队列轮训模型，在不同场景下对轮训方式进行的抽象"""

    def __init__(self, service_name):
        self.serviceName = service_name

    def online(self, mac):
        """上线一个服务"""
        pass

    def offline(self, mac):
        """下线一个服务"""
        pass

    def roll_poling(self):
        """轮训动作，头部poll，移至尾部，second->head，head->tail"""
        pass

    def mac_size(self):
        """当前已注册设备总数"""
        pass

    def record_failed(self, mac):
        """记录失败，业务方的feedback调用，由于也无方的失败判定不可信，但是能够反应资源好坏趋势，所以该接口可以对执行资源降权，但不直接下线资源"""
        pass

    def record_success(self, mac):
        """记录成功，这个也必须调用，正向打分用户维持正常的高优先级"""
        pass

    def queue_status(self):
        """显示队列内容"""
        pass

    def reset(self):
        """重制队列"""
        pass

class RedisServiceQueue(ServiceQueue):
    """基于redis的实现"""

    def __init__(self, service_name):
        super(RedisServiceQueue, self).__init__(service_name)
        self.queue_polling_key = 'hermes_admin_' + self.serviceName + '_redis_polling_v3'
        self.queue_meta_key = 'hermes_admin_' + self.serviceName + '_redis_data_v3'
        self.queue_index_key = 'hermes_admin_' + self.serviceName + '_index_v3'

    def online(self, mac):
        meta_data = redis_client.hget(self.queue_meta_key, mac)
        if meta_data is None:
            meta_data = {
                'mac': mac,
                'score': 1.0,
                'lastReportTime': time.time(),
                'online': True
            }
            # print 'new device report:' + json.dumps(meta_data)
        else:
            meta_data = json.loads(meta_data)
            meta_data['lastReportTime'] = time.time()
            meta_data['online'] = True
        redis_client.hset(self.queue_meta_key, mac, json.dumps(meta_data))
        self.set_up_poling_key(mac, meta_data['score'])

    def set_up_poling_key(self, mac, score, old_score=0):
        old_max_index = self.judge_index_by_score(old_score)
        last_index = self.judge_index_by_score(score)

        now_index = last_index
        if now_index < old_max_index:
            return
        queue_size = self.mac_size()
        all_polling_key = set(redis_client.lrange(self.queue_polling_key, 0, -1))
        while now_index >= old_max_index:
            poling_key = mac + '__' + str(now_index)
            if poling_key in all_polling_key:
                now_index -= 1
                continue
            ref_value = redis_client.lindex(self.queue_polling_key, int(queue_size * now_index / 10))
            now_index -= 1
            if not ref_value:
                redis_client.rpush(self.queue_polling_key, poling_key)
                continue
            insert_result = redis_client.linsert(self.queue_polling_key, 'AFTER', ref_value, poling_key)
            if insert_result <= 0:
                redis_client.rpush(self.queue_polling_key, poling_key)

    @staticmethod
    def judge_index_by_score(score):
        enlarge_score = int(float(score) * 10 + 0.1)
        return 1 if enlarge_score < 1 else 10 if enlarge_score > 10 else enlarge_score

    def offline(self, mac):
        """下线一个服务"""
        # 元数据不要删，这里记录了历史评估数据
        meta_data = redis_client.hget(self.queue_meta_key, mac)
        if not meta_data:
            return
        meta_data = json.loads(meta_data)
        meta_data['online'] = False
        redis_client.hset(self.queue_meta_key, mac, json.dumps(meta_data))

    def roll_poling(self):
        total_len = self.mac_size()
        if total_len == 0:
            return None
        retry = 0
        while True:
            # 理论上不会死循环，但是还是要这样做保证，避免代码内部出现逻辑漏洞
            if retry > 5:
                return None
            retry += 1

            now_index = redis_client.incr(self.queue_index_key)
            if now_index > 0x0FFFFFFF:
                # 调用次数很多了，可以清零cursor，避免溢出问题
                redis_client.delete(self.queue_index_key)
                now_index = redis_client.incr(self.queue_index_key)
            # print 'nowIndex:%d' % now_index
            resource = redis_client.lindex(self.queue_polling_key, now_index % total_len)
            if not resource:
                # 这个case是可能发生的，由于中途我们可能删除数据，导致index和实际的不一致
                resource = redis_client.lindex(self.queue_polling_key, -1)
                if not resource:
                    return None
            # print 'resource:%s' % resource
            polling_key_param = str(resource).split('__')
            mac = polling_key_param[0]
            mac_enlarge_index = int(polling_key_param[1])
            meta_data = redis_client.hget(self.queue_meta_key, mac)
            if not meta_data:
                continue
            # print 'metaData:%s' % meta_data
            meta_data = json.loads(meta_data)
            if not meta_data['online']:
                if mac_enlarge_index > 1:
                    redis_client.lrem(self.queue_polling_key, 1, resource)
                continue
            if mac_enlarge_index <= 1:
                # 永远保证始终有1个投射是有效的，如此提供自动复原的能力
                return mac
            # print 'max index for score:%d' % self.judge_index_by_score(meta_data['score'])
            # print 'now_index:%s' % mac_enlarge_index
            if self.judge_index_by_score(meta_data['score']) < mac_enlarge_index:
                redis_client.lrem(self.queue_polling_key, 1, resource)
                continue
            return mac

    def mac_size(self):
        """当前已注册设备总数"""
        # 读取queue，不要读取set，set可能有下线资源
        return redis_client.llen(self.queue_polling_key)

    def record_failed(self, mac):
        """记录失败，业务方的feedback调用，由于也无方的失败判定不可信，但是能够反应资源好坏趋势，所以该接口可以对执行资源降权，但不直接下线资源"""
        meta_data = redis_client.hget(self.queue_meta_key, mac)
        if meta_data is None:
            logger.warning('can not find meta data for device:%s' % mac)
            return
        meta_data = json.loads(meta_data)
        new_score = float(meta_data['score']) * 0.9
        # 不能让分数无限降低，这回导致资源被跑干之后无法恢复
        new_score = float(0.1) if new_score < float(0.1) else float(1) if new_score > float(1) else new_score

        meta_data['score'] = new_score
        redis_client.hset(self.queue_meta_key, mac, json.dumps(meta_data))

    def record_success(self, mac):
        """记录成功，这个也必须调用，正向打分用户维持正常的高优先级"""
        meta_data = redis_client.hget(self.queue_meta_key, mac)
        if meta_data is None:
            logger.warning('can not find meta data for device:%s' % mac)
            return
        meta_data = json.loads(meta_data)
        old_score = meta_data['score']
        new_score = float(meta_data['score']) * 0.9 + 0.1
        new_score = float(0) if new_score < float(0) else float(1) if new_score > float(1) else new_score
        meta_data['score'] = new_score
        redis_client.hset(self.queue_meta_key, mac, json.dumps(meta_data))
        self.set_up_poling_key(mac, new_score, old_score)

    def queue_status(self):
        """显示队列内容"""
        mac_list = redis_client.lrange(self.queue_polling_key, 0, -1)
        all_meta = redis_client.hgetall(self.queue_meta_key)
        # print json.dumps(all_meta)
        ret = []
        duplicate_set = set()
        # print mac_list
        for mac in mac_list:
            # print mac
            mac = str(mac).split('__')[0]
            meta_data = all_meta[mac]
            if meta_data is None:
                continue
            if mac in duplicate_set:
                continue
            duplicate_set.add(mac)
            meta_data = json.loads(meta_data)
            if not meta_data['online']:
                continue
            resource_entry = ResourceEntry(meta_data['mac'])
            resource_entry.score = meta_data['score']
            resource_entry.lastReportTime = meta_data['lastReportTime']
            ret.append(resource_entry)
        return ret

    def reset(self):
        """重制队列"""
        redis_client.delete(self.queue_polling_key)
        redis_client.delete(self.queue_meta_key)
        redis_client.delete(self.queue_index_key)

class ResourceEntry(object):
    def __init__(self, mac):
        self.mac = mac
        self.score = 1.0
        self.lastReportTime = time.time()
        self.online = True

class RamServiceQueue(ServiceQueue):
    """基于内存的实现"""

    def __init__(self, service_name):
        super(RamServiceQueue, self).__init__(service_name)
        self.queue = list()
        self.metaMap = {}
        self.lock = threading.Lock()
        self.index = -1

    def online(self, mac):
        resource_entry = self.metaMap.get(mac)
        if resource_entry is None:
            with self.lock:
                resource_entry = self.metaMap.get(mac)
                if resource_entry is None:
                    resource_entry = ResourceEntry(mac)
                    self.metaMap[mac] = resource_entry
        else:
            resource_entry.lastReportTime = time.time()
            resource_entry.online = True
        self.set_up_poling_key(mac, resource_entry.score)

    def set_up_poling_key(self, mac, score, old_score=float(0)):
        old_max_index = self.judge_index_by_score(old_score)
        # print 'set up poling key for mac:%s' % mac
        last_index = self.judge_index_by_score(score)

        now_index = last_index
        # print 'last index:%d' % last_index
        queue_size = self.mac_size()
        all_polling_key = set(self.queue)
        while now_index >= old_max_index:
            poling_key = mac + '__' + str(now_index)
            if poling_key in all_polling_key:
                now_index -= 1
                continue
            try:
                self.queue.insert(int(queue_size * now_index / 10), poling_key)
            except:
                self.queue.append(poling_key)

    @staticmethod
    def judge_index_by_score(score):
        enlarge_score = int(float(score) * 10 + 0.1)
        return 1 if enlarge_score < 1 else 10 if enlarge_score > 10 else enlarge_score

    def offline(self, mac):
        resource_entry = self.metaMap.get(mac)
        if resource_entry is None:
            return
        resource_entry.online = False

    def roll_poling(self):
        retry_times = 0
        while True:
            if retry_times > 5:
                return None
            retry_times += 1
            with self.lock:
                self.index += 1
                next_index = self.index
            if next_index > 0x0FFFFFFF:
                # 调用次数很多了，可以清零cursor，避免溢出问题
                self.index = -1
                continue
            try:
                resource = self.queue[next_index % len(self.queue)]
            except:
                continue
            polling_key_param = str(resource).split('__')
            mac = polling_key_param[0]
            mac_enlarge_index = int(polling_key_param[1])
            meta_data = self.metaMap[mac]
            if not meta_data:
                continue
            # print 'metaData:%s' % meta_data
            if not meta_data.online:
                if mac_enlarge_index > 1:
                    self.queue.remove(resource)
                continue
            if mac_enlarge_index <= 1:
                # 永远保证始终有1个投射是有效的，如此提供自动复原的能力
                return mac
            if self.judge_index_by_score(meta_data.score) < mac_enlarge_index:
                self.queue.remove(resource)
                continue
            return mac

    def mac_size(self):
        return len(self.queue)

    def record_failed(self, mac):
        """记录失败，业务方的feedback调用，由于也无方的失败判定不可信，但是能够反应资源好坏趋势，所以该接口可以对执行资源降权，但不直接下线资源"""
        meta_data = self.metaMap[mac]
        if meta_data is None:
            logger.warning('can not find meta data for device:%s' % mac)
            return
        new_score = float(meta_data.score) * 0.9
        # 不能让分数无限降低，这回导致资源被跑干之后无法恢复
        new_score = float(0.1) if new_score < float(0.1) else float(1) if new_score > float(1) else new_score
        meta_data.score = new_score

    def record_success(self, mac):
        """记录成功，这个也必须调用，正向打分用户维持正常的高优先级"""
        meta_data = self.metaMap[mac]
        if meta_data is None:
            logger.warning('can not find meta data for device:%s' % mac)
            return
        meta_data = json.loads(meta_data)
        old_score = meta_data.score
        new_score = float(meta_data.score) * 0.9 + 0.1
        new_score = float(0) if new_score < float(0) else float(1) if new_score > float(1) else new_score
        meta_data.score = new_score
        self.set_up_poling_key(mac, new_score, old_score)

    def queue_status(self):
        """显示队列内容"""
        # print json.dumps(all_meta)
        ret = []
        duplicate_set = set()
        # print mac_list
        for mac in self.queue:
            # print mac
            mac = str(mac).split('__')[0]
            meta_data = self.metaMap[mac]
            if meta_data is None:
                continue
            if mac in duplicate_set:
                continue
            duplicate_set.add(mac)
            if not meta_data.online:
                continue
            ret.append(meta_data)
        return ret

    def reset(self):
        """重制队列"""
        self.queue = list()
        self.metaMap = {}
        self.index = 0

class ServiceRegistry(object):
    def __init__(self):
        self.lock = threading.Lock()
        self.resource_queue_map = {}
        if not redis_client:
            # recovery for ram mode
            self.__recovery__()

    def __recovery__(self):
        now = datetime.datetime.now()
        start = now - datetime.timedelta(minutes=10)
        online_devices = HermesDevice.objects.filter(lastReportTime__gt=start)
        for online_device in online_devices:
            last_online_service = json.loads(online_device.lastReportService)
            for service in last_online_service:
                self.create_or_get_resource_queue(service).online(online_device.mac)

    def create_or_get_resource_queue(self, service):
        service_queue = self.resource_queue_map.get(service)
        if service_queue is not None:
            return service_queue
        with self.lock:
            service_queue = self.resource_queue_map.get(service)
            if service_queue is not None:
                return service_queue
            if redis_client:
                service_queue = RedisServiceQueue(service)
            else:
                service_queue = RamServiceQueue(service)
            self.resource_queue_map[service] = service_queue
            return service_queue

    def handle_device_report(self, report_model):
        mac = report_model['mac']
        online_service = report_model['onlineServices']
        for service in online_service:
            self.create_or_get_resource_queue(service).online(mac)

    def roll_poling(self, service):
        return self.create_or_get_resource_queue(service).roll_poling()

    def record_failed(self, mac, service):
        self.create_or_get_resource_queue(service).record_failed(mac)

    def offline(self, mac, service):
        self.create_or_get_resource_queue(service).offline(mac)

    def record_success(self, mac, service):
        self.create_or_get_resource_queue(service).record_success(mac)

    def queue_status(self, service):
        return self.create_or_get_resource_queue(service).queue_status()

    def reset_queue(self, service):
        self.create_or_get_resource_queue(service).reset()

registry = ServiceRegistry()


'''
Components that are not abstract but are still used between cogs.
'''

import logging
from discord.ui import Modal, InputText, Button
import discord.ui.view as core_view
from discord.ext.commands import Bot
from database.models import Game
from utils import config
from utils.errors import ValueAccessError

logger = logging.getLogger(f'civviebot.{__name__}')

class View(core_view.View):
    '''
    View class with overridden timeout parameters.
    '''

    def __init__(self, *args, **kwargs):
        '''
        Constructor that forces view timeout to None.
        '''
        kwargs['timeout'] = None
        super().__init__(*args, **kwargs)

class NotifyIntervalInput(InputText):
    '''
    Input text for setting a game's notify interval.

    Uses custom_id 'notify_interval'.
    '''

    def __init__(self, *args, remind_interval: int = None, **kwargs):
        '''
        The notify interval to set. Will be the global config if not passed in.
        '''
        if kwargs.get('value', None) is None:
            kwargs['value'] = str(
                config.REMIND_INTERVAL
                if remind_interval is None
                else remind_interval
            )
        if kwargs.get('label', None) is None:
            kwargs['label'] = 'Seconds between re-pings (use 0 to disable):'
        kwargs['custom_id'] = 'notify_interval'
        super().__init__(*args, **kwargs)

class MinTurnsInput(InputText):
    '''
    Input text for setting a game's minimum turns.

    Uses custom_id 'min_turns'.
    '''

    def __init__(self, *args, min_turns: int = None, **kwargs):
        '''
        The minimum turns to set. Will get the global config if not passed in.
        '''
        if kwargs.get('value', None) is None:
            kwargs['value'] = (
                config.MIN_TURNS
                if min_turns is None
                else str(min_turns)
            )
        if kwargs.get('label', None) is None:
            kwargs['label'] = 'Start notifying after turn:'
        super().__init__(custom_id='min_turns', *args, **kwargs)

class ChannelAwareModal(Modal):
    '''
    A component that stores a channel_id and bot.
    '''

    def __init__(self, channel_id: int, bot: Bot, *args, **kwargs):
        '''
        Constructor; sets the channel_id and bot.
        '''
        self._channel_id = channel_id
        self._bot = bot
        super().__init__(*args, **kwargs)

    @property
    def channel_id(self) -> int:
        '''
        The channel this URL interaction was initiated from.
        '''
        return self._channel_id

    @property
    def bot(self) -> Bot:
        '''
        Instance of CivvieBot passed in from the interaction.
        '''
        return self._bot

    def get_child_value(self, custom_id: str):
        '''
        Gets the value of a child component by its custom_id.
        '''
        try:
            component = next(
                filter(lambda c: c.custom_id == custom_id, self.children)
            )
        except StopIteration as error:
            raise IndexError(
                f'Accessed a non-existent component by custom_id {custom_id}'
            ) from error
        if component.value is None:
            raise ValueAccessError(
                (
                    f'Accessed value of component by custom_id {custom_id} '
                    'before it was set'
                )
            )
        return component.value

class GameAwareButton(Button):
    '''
    Button component that stores a Game.
    '''

    def __init__(self, game: Game, *args, **kwargs):
        '''
        Constructor; sets the game_id, channel_id and bot.
        '''
        self._game = game.id
        self._channel_id = game.webhookurl.channelid
        super().__init__(*args, **kwargs)

    @property
    def game(self) -> int:
        '''
        The game this button is tracking.
        '''
        return self._game

    @property
    def channel_id(self) -> int:
        '''
        The channel this button is in.
        '''
        return self._channel_id


#!/usr/bin/env python
_version_=3.0
from optparse import OptionParser
import optparse
try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import os
try:
        from StringIO import StringIO ## for Python 2
except ImportError:
        from io import StringIO ## for Python 3
import sys
import re
import copy
from ast import literal_eval
from logging import info, debug, warning, error, critical

class Options(object):
    version = 3.001
    def __init__(self):

        self._command_options()
        self.job = configparser.SafeConfigParser()
        self._set_paths()
        self._load_defaults()
        self._load_job()
        self._set_attr()

    def _set_paths(self):
        if __name__ != '__main__':
            self.script_dir = os.path.dirname(__file__)
        else:
            self.script_dir = os.path.abspath(sys.path[0])
        self.job_dir = os.getcwd()
        self.jobname = os.path.splitext(os.path.basename(
                                       self.input_file))[0]

    def _command_options(self):
        """Load data from the command line."""

        usage = "%prog [options] input_file"
        version = "%prog " + "%f"%(_version_)
        parser = OptionParser(usage=usage, version=version)
        group = optparse.OptionGroup(parser, "Verbosity Options")
        group.add_option("-s", "--silent", action="store_true",
                          dest="silent",
                          help="Print nothing to the console.")
        group.add_option("-q", "--quiet", action="store_true",
                          dest="quiet",
                          help="Print only warnings and errors.")
        group.add_option("-v", "--verbose", action="store_true",
                          dest="verbose",
                          help="Print everything to the console.")
        parser.add_option_group(group)
        (self.cmd_options, local_args) = parser.parse_args()

        if not local_args:
            parser.print_help()
            sys.exit(1)
        elif len(local_args) != 1:
            error("Only one argument required, the input file")
            sys.exit(1)
        else:
            self.input_file = os.path.abspath(local_args[0])

    def _load_defaults(self):
        default_path = os.path.join(self.script_dir, 'defaults.ini')
        try:
            filetemp = open(default_path, 'r')
            default = filetemp.read()
            filetemp.close()
            if not '[defaults]' in default.lower():
                default = '[defaults]\n' + default
            default = StringIO(default)
        except IOError:
            error("Error loading defaults.ini")
            default = StringIO('[defaults]\n')
        self.job.readfp(default)

    def _load_job(self):
        """Load data from the local job name."""
        if self.input_file is not None:
            try:
                filetemp = open(self.input_file, 'r')
                job = filetemp.read()
                filetemp.close()
                if not '[job]' in job.lower():
                    job = '[job]\n' + job
                job = StringIO(job)
            except IOError:
                job = StringIO('[job]\n')
        else:
            job = StringIO('[job]\n')
        self.job.readfp(job)

    def _set_attr(self):
        """Sets attributes to the base class. default options are over-written
        by job-specific options.

        """
        for key, value in self.job.items('defaults'):
            value = self.get_val('defaults', key)
            setattr(self, key, value)
        for key, value in self.job.items('job'):
            value = self.get_val('job', key)
            setattr(self, key, value)
        for key, value in self.cmd_options.__dict__.items():
            setattr(self, key, value)

    def get_val(self, section, key):
        """Returns the proper type based on the key used."""
        # known booleans
        booleans = ['verbose', 'quiet', 'silent', 'exhaustive',
                    'gen_single_bonding_set', 'calc_sbu_surface_area',
                    'calc_max_sbu_span', 'create_sbu_input_files', 'find_symmetry',
                    'relaxed_topology', 'build_from_tree']
        floats = ['distance_tolerance', 'overlap_tolerance',
                  'bond_angle_tolerance',
                  'cell_angle_cutoff', 'symmetry_precision']
        integers = ['organic_sbu_per_structure',
                    'metal_sbu_per_structure',
                    'structure_sbu_length',
                    'max_trials',
                    'max_structures']
        lists = ['topologies', 'sbu_files', 'organic_sbus',
                 'metal_sbus']
        tuple_of_tuples = ['sbu_combinations']

        if key in booleans:
            try:
                val = self.job.getboolean(section, key)
            except ValueError:
                val = False
        # known integers
        elif key in integers:
            try:
                val = self.job.getint(section, key)
            except ValueError:
                val = 0;
        # known floats
        elif key in floats:
            try:
                val = self.job.getfloat(section, key)
            except ValueError:
                val = 0.
        # known lists
        elif key in lists:
            p = re.compile('[,;\s]+')
            val = p.split(self.job.get(section, key))
            try:
                val = [int(i) for i in val if i]
            except ValueError:
                val = [i for i in val if i]

        # tuple of tuples.
        elif key in tuple_of_tuples:
            val = literal_eval(self.job.get(section, key)) \
                    if self.job.get(section, key) else None
            # failsafe if only one tuple is presented, need to embed it.
            if val is not None:
                if isinstance(val[0], int) or isinstance(val[0], float):
                    val = [val]
        else:
            val = self.job.get(section, key)
        return val

def Terminate():
    info("Genstruct terminated normally")
    sys.exit(1)


from sklearn.metrics import confusion_matrix, classification_report
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

def model_Evaluate(model,X_test,y_test):
    # Predict values for Test dataset
    y_pred = model.predict(X_test)

    # Print the evaluation metrics for the dataset.
    print(classification_report(y_test, y_pred))

    # Compute and plot the Confusion matrix
    cf_matrix = confusion_matrix(y_test, y_pred)

    categories = ['Negative', 'Positive']
    group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
    group_percentages = ['{0:.2%}'.format(value) for value in cf_matrix.flatten() / np.sum(cf_matrix)]

    labels = [f'{v1}\n{v2}' for v1, v2 in zip(group_names, group_percentages)]
    labels = np.asarray(labels).reshape(2, 2)

    sns.heatmap(cf_matrix, annot=labels, cmap='Blues', fmt='',
                xticklabels=categories, yticklabels=categories)

    plt.xlabel("Predicted values", fontdict={'size': 14}, labelpad=10)
    plt.ylabel("Actual values", fontdict={'size': 14}, labelpad=10)
    plt.title("Confusion Matrix", fontdict={'size': 18}, pad=20)


#!/usr/bin/env python3

import pytest  # noqa: F401

import r5py
import com.conveyal.r5

class TestBreakdownStat:
    @pytest.mark.parametrize(
        ["enum_member", "enum_name"],
        [
            (r5py.r5.BreakdownStat.MEAN, "MEAN"),
            (r5py.r5.BreakdownStat.MINIMUM, "MINIMUM"),
        ],
    )
    def test_breakdownstats(self, enum_member, enum_name):
        assert enum_member.name == enum_name
        assert isinstance(
            enum_member.value, com.conveyal.r5.analyst.cluster.PathResult.Stat
        )
        assert (
            enum_member.value
            == com.conveyal.r5.analyst.cluster.PathResult.Stat.valueOf(enum_name)
        )
        assert (
            enum_member.name
            == com.conveyal.r5.analyst.cluster.PathResult.Stat.valueOf(enum_name).name()
        )


import json
import sys

fileinput = "test.txt"
f = open(fileinput,"r", encoding="utf8")

d_word_tag= { }  #keeps word : tag -->
d_tag_next_tag = { } #keeps tag to next tag and count --> { tag : { nexttag : count }}
d_tagcount = { } #counts number of tags  --> { tag : count(tag)}

d_trans_count = {} #keeps total tag to nexttags count -- > { tag : count(nexttags) }
starttags = { } # keeps dict of start tags and counts ---> { starttag : count }

if f.mode=="r":
        fl = f.readlines()
        for a in fl:

            b = a.split()
            for i,c in enumerate(b):
                d_prev = c.split("/")
                s = ""
                if(len(d_prev)>2):
                    for z in range(0,len(d_prev)-1):
                        if(z == len(d_prev)-2):
                            s += str(d_prev[z])
                        else:
                            s += str(d_prev[z])+"/"
                else:
                    s = d_prev[0]

                word = s
                #print(word)
                tag = d_prev[len(d_prev)-1]

                if(i==0):

                    if tag not in starttags:
                        starttags[tag] = 1
                    else:
                        starttags[tag] += 1

                if word not in d_word_tag:
                        d_word_tag[word] = { tag : 1 }
                elif word in d_word_tag and tag not in d_word_tag[word]:
                        d_word_tag[word][tag]=  1
                else:
                        d_word_tag[word][tag] += 1

                if tag not in d_tagcount:
                    d_tagcount[tag] = 1
                else:
                    d_tagcount[tag] +=1

                if(i < len(b)-1 ):
                    d_next = b[i+1].split("/")
                    next_tag = d_next[len(d_next)-1]

                    if tag not in d_tag_next_tag:
                        d_tag_next_tag[tag] = { next_tag : 1 }
                        d_trans_count[tag] = 1
                    elif tag in d_tag_next_tag and next_tag not in d_tag_next_tag[tag]:
                        d_tag_next_tag[tag][next_tag]=  1
                        d_trans_count[tag] += 1
                    else:
                        d_tag_next_tag[tag][next_tag] += 1
                        d_trans_count[tag] += 1

total_starttags = 0
for x in starttags:
   total_starttags +=starttags[x]

total_tags = 0
for x in d_tagcount:
   total_tags +=d_tagcount[x]

    #-----------------update probablities for all ------------------------------------------

    # start probabilites -----
for x in d_tagcount:
    if x not in starttags:
        starttags[x]=0

for x in starttags:
   starttags[x] = ( starttags[x]+1) / (total_starttags + total_tags)
  # print( x + " " + str(starttags[x]))

    # transmission probabilites -----

for x in d_tagcount:
    if x not in d_tag_next_tag:
            d_tag_next_tag[x] = {}
            d_trans_count[x] = 0

#print(len(d_tag_next_tag))
for x in d_tagcount:
    for y in d_tag_next_tag:
        if x not in d_tag_next_tag[y]:
            d_tag_next_tag[y][x] = 0

for tags in d_tag_next_tag:
    for nexttags in d_tag_next_tag[tags]:

        #d_tag_next_tag[tags][nexttags] /= d_trans_count[tags]
        d_tag_next_tag[tags][nexttags] = ( d_tag_next_tag[tags][nexttags] + 1 ) / (d_trans_count[tags]+ len(d_tagcount))

    # emission probabilites -----

for words in d_word_tag:
    for tags in d_word_tag[words]:
       #d_word_tag[words][tags] = ( d_word_tag[words][tags] + 1 ) / (d_tagcount[tags]+ total_tags )
       d_word_tag[words][tags] /= d_tagcount[tags]

data_list = { "start_prob" : starttags , "trans_prob" : d_tag_next_tag , "emis_prob" : d_word_tag  }

f2 = open("hmmmodel.txt","w+", encoding="utf8")

data = json.dumps(data_list,ensure_ascii=False)
f2.write(data)


#!/usr/bin/env python2

import sys
import rospy
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QCoreApplication, Qt
from PyQt5.QtGui import *

class controlTable(QTableWidget):
	def __init__(self):
		super(QTableWidget, self).__init__()

	def itemClicked(self, ev):
			QTableView.selectRow(self,row)
class ParameterWindow(QWidget):

	def __init__(self):
		super(QWidget,self).__init__()
		self.initUI()

	def initUI(self):
		rospy.init_node('experiment')

		self.horiz_layout1 = QHBoxLayout()
		self.horiz_layout2 = QHBoxLayout()
		self.vert_layout = QVBoxLayout()
		self.setWindowTitle("Experiment UI")
		self.setGeometry(500,200,1000,600)

		self.lbl_box = QLabel(self)
		self.lbl_box.setText('Participant ID:')
		self.horiz_layout1.addWidget(self.lbl_box)

		self.textbox = QLineEdit(self)
		self.textbox.resize(280,40)
		self.horiz_layout1.addWidget(self.textbox)

		self.smt_btn = QPushButton('Submit',self)
		self.smt_btn.clicked.connect(self.submit_data)
		self.smt_btn.setMaximumWidth(120)
		self.horiz_layout1.addWidget(self.smt_btn)

		self.lbl_tbl = QLabel(self)
		self.lbl_tbl.setText('Traverse List:')

		self.table = controlTable()
		self.table = QTableWidget(5,5,self)

		self.table.setHorizontalHeaderLabels(('Goal ID', 'Pos X', 'Pos Y','Theta','Fuel'))
		self.table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
		self.table.setMaximumWidth(self.table.horizontalHeader().length()+30)
		self.table.setMaximumHeight(self.table.verticalHeader().length()+25)
		self.horiz_layout2.addWidget(self.table)

		self.go_btn = QPushButton('Go!',self)
		self.go_btn.clicked.connect(self.submit_data)
		self.go_btn.setMaximumWidth(120)
		self.horiz_layout2.addWidget(self.go_btn)

		self.buildTables()

		self.quit_btn = QPushButton('Quit',self)
		self.quit_btn.clicked.connect(self.submit_data)
		self.quit_btn.setMaximumWidth(120)

		self.buildWidgets()

	def submit_data(self):
		self.msg.id = int(self.textbox.text())
		self.hide()
		while not rospy.is_shutdown():
			self.pub.publish(self.msg)
			self.r.sleep()

	def buildTables(self):
		self.cur_trav_lbl = QLabel(self)
		self.cur_trav_lbl.setText('Current Traverse:')
		self.trav_table = QTableWidget(1,4,self)
		self.trav_table.setHorizontalHeaderLabels(('Goal ID', 'Pos X', 'Pos Y','Theta'))
		self.trav_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
		self.trav_table.setMaximumWidth(self.trav_table.horizontalHeader().length()+20)
		self.trav_table.setMaximumHeight(self.trav_table.verticalHeader().length()+25)

		self.cur_pos_lbl = QLabel(self)
		self.cur_pos_lbl.setText('Current Position:')
		self.cur_pos_table = QTableWidget(1,3,self)
		self.cur_pos_table.setHorizontalHeaderLabels(('Pos X', 'Pos Y','Theta'))
		self.cur_pos_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
		self.cur_pos_table.setMaximumWidth(self.cur_pos_table.horizontalHeader().length()+20)
		self.cur_pos_table.setMaximumHeight(self.cur_pos_table.verticalHeader().length()+25)

		self.cur_fuel = QLabel(self)
		self.cur_fuel.setText('Current Fuel:')
		self.cur_fuel_table = QTableWidget(1,1,self)
		self.cur_fuel_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
		self.cur_fuel_table.setMaximumWidth(self.cur_fuel_table.horizontalHeader().length()+20)
		self.cur_fuel_table.setMaximumHeight(self.cur_fuel_table.verticalHeader().length()+25)

	def buildWidgets(self):
		self.vert_layout.addLayout(self.horiz_layout1)
		self.vert_layout.addWidget(self.lbl_tbl)
		self.vert_layout.addLayout(self.horiz_layout2)

		self.vert_layout.addWidget(self.cur_trav_lbl)
		self.vert_layout.addWidget(self.trav_table)
		self.vert_layout.addWidget(self.cur_pos_lbl)
		self.vert_layout.addWidget(self.cur_pos_table)
		self.vert_layout.addWidget(self.cur_fuel)
		self.vert_layout.addWidget(self.cur_fuel_table)
		self.vert_layout.addWidget(self.quit_btn)

		self.vert_layout.addSpacing(10)
		self.vert_layout.setContentsMargins(30,30,30,30)
		self.vert_layout.addStretch()
		self.setLayout(self.vert_layout)

		self.show()

if __name__ == '__main__':
	app = QApplication(sys.argv)
	coretools_app = ParameterWindow()
	sys.exit(app.exec_())


from collections.abc import Sequence
from typing import Any, Optional, Union

from google.protobuf import descriptor
from google.protobuf import message
from tensorflow.tools.proto_splitter import chunk_pb2

_BYTE_UNITS = [(1, "B"), (1 << 10, "KiB"), (1 << 20, "MiB"), (1 << 30, "GiB")]

def format_bytes(b: int) -> str:
  """Formats bytes into a human-readable string."""
  for i in range(1, len(_BYTE_UNITS)):
    if b < _BYTE_UNITS[i][0]:
      n = f"{b / _BYTE_UNITS[i-1][0]:.2f}"
      units = _BYTE_UNITS[i - 1][1]
      break
  else:
    n = f"{b / _BYTE_UNITS[-1][0]:.2f}"
    units = _BYTE_UNITS[-1][1]
  n = n.rstrip("0").rstrip(".")
  return f"{n}{units}"

FieldTypes = Union[str, int, bool, Sequence[Union[str, int, bool]]]

def get_field(
    proto: message.Message, fields: FieldTypes
) -> tuple[Any, Optional[descriptor.FieldDescriptor]]:
  """Returns the field and field descriptor from the proto.

  Args:
    proto: Parent proto of any message type.
    fields: List of string/int/map key fields, e.g. ["nodes", "attr", "value"]
      can represent `proto.nodes.attr["value"]`.

  Returns:
    Tuple of (
      Field in the proto or `None` if none are found,
      Field descriptor
    )
  """
  field_proto = proto
  field_desc = None
  for field_proto, field_desc, _, _ in _walk_fields(proto, fields):
    pass
  return field_proto, field_desc

def get_field_tag(
    proto: message.Message, fields: FieldTypes
) -> Sequence[chunk_pb2.FieldIndex]:
  """Generates FieldIndex proto for a nested field within a proto.

  Args:
    proto: Parent proto of any message type.
    fields: List of string/int/map key fields, e.g. ["nodes", "attr", "value"]
      can represent `proto.nodes.attr["value"]`.

  Returns:
    A list of FieldIndex protos with the same length as `fields`.
  """
  field_tags = []
  for _, field_desc, map_key, list_index in _walk_fields(proto, fields):
    field_tags.append(chunk_pb2.FieldIndex(field=field_desc.number))
    if map_key is not None:
      key_type = field_desc.message_type.fields_by_name["key"].type
      field_tags.append(
          chunk_pb2.FieldIndex(map_key=_map_key_proto(key_type, map_key))
      )
    elif list_index is not None:
      field_tags.append(chunk_pb2.FieldIndex(index=list_index))
  return field_tags

def _walk_fields(proto: message.Message, fields: FieldTypes):
  """Yields fields in a proto.

  Args:
    proto: Parent proto of any message type.
    fields: List of string/int/map key fields, e.g. ["nodes", "attr", "value"]
      can represent `proto.nodes.attr["value"]`.

  Yields:
    Tuple of (
      Field in the proto or `None` if none are found,
      Field descriptor,
      Key into this map field (or None),
      Index into this repeated field (or None))
  """
  if not isinstance(fields, list):
    fields = [fields]

  field_proto = proto
  parent_desc = proto.DESCRIPTOR
  i = 0
  while i < len(fields):
    field = fields[i]
    field_desc = None
    map_key = None
    index = None

    if parent_desc is None:
      raise ValueError(
          f"Unable to find fields: {fields} in proto of type {type(proto)}."
      )

    if isinstance(field, int):
      try:
        field_desc = parent_desc.fields_by_number[field]
      except KeyError:
        raise KeyError(  # pylint:disable=raise-missing-from
            f"Unable to find field number {field} in {parent_desc.full_name}. "
            f"Valid field numbers: {parent_desc.fields_by_number.keys()}"
        )
    elif isinstance(field, str):
      try:
        field_desc = parent_desc.fields_by_name[field]
      except KeyError:
        raise KeyError(  # pylint:disable=raise-missing-from
            f"Unable to find field '{field}' in {parent_desc.full_name}. "
            f"Valid field names: {parent_desc.fields_by_name.keys()}"
        )
    else:  # bool (only expected as map key)
      raise TypeError("Unexpected bool found in field list.")

    i += 1
    parent_desc = field_desc.message_type
    if field_proto is not None:
      field_proto = getattr(field_proto, field_desc.name)

    # Handle special fields types (map key and list index).
    if _is_map(parent_desc) and i < len(fields):
      # Next field is the map key.
      map_key = fields[i]

      try:
        field_proto = field_proto[map_key] if field_proto is not None else None
      except KeyError:
        field_proto = None
      i += 1

      if i < len(fields):
        # The next field must be from the Value Message.
        value_desc = parent_desc.fields_by_name["value"]
        assert value_desc.message_type is not None
        parent_desc = value_desc.message_type

    elif is_repeated(field_desc) and i < len(fields):
      # The next field is the index within the list.
      index = fields[i]
      try:
        field_proto = field_proto[index] if field_proto is not None else None
      except IndexError:
        field_proto = None
      i += 1

    yield field_proto, field_desc, map_key, index

def _is_map(desc: descriptor.Descriptor) -> bool:
  return desc.GetOptions().map_entry if desc is not None else False

def is_repeated(field_desc: descriptor.FieldDescriptor) -> bool:
  return field_desc.label == descriptor.FieldDescriptor.LABEL_REPEATED

_FIELD_DESC = descriptor.FieldDescriptor
_MAP_KEY = {
    _FIELD_DESC.TYPE_STRING: lambda key: chunk_pb2.FieldIndex.MapKey(s=key),
    _FIELD_DESC.TYPE_BOOL: lambda key: chunk_pb2.FieldIndex.MapKey(boolean=key),
    _FIELD_DESC.TYPE_UINT32: lambda key: chunk_pb2.FieldIndex.MapKey(ui32=key),
    _FIELD_DESC.TYPE_UINT64: lambda key: chunk_pb2.FieldIndex.MapKey(ui64=key),
    _FIELD_DESC.TYPE_INT32: lambda key: chunk_pb2.FieldIndex.MapKey(i32=key),
    _FIELD_DESC.TYPE_INT64: lambda key: chunk_pb2.FieldIndex.MapKey(i64=key),
}

def _map_key_proto(key_type, key):
  """Returns MapKey proto for a key of key_type."""
  return _MAP_KEY[key_type](key)


from Plugin.Interface import Interface
import nltk
from nltk.collocations import *
from nltk.tokenize import word_tokenize
import string

class Nltk(Interface):

    def __init__(self, cfg, pluginParams, workflowPluginParams, frameworkParams):
        super(Nltk, self).__init__(cfg, pluginParams, workflowPluginParams, frameworkParams)

    def runSingleWords(self):
        percentage = float(self.getPluginParamValue("Percentage")) / 100.0
        minCharLength = int(self.getPluginParamValue("MinCharLength"))
        posFilter = self.getPluginParamValue("POS")
        inputContent = self.getInputContent().lower()
        punctuation = string.punctuation.replace("-", "")
        puncFilter = dict((ord(char), None) for char in punctuation)
        tokens = nltk.word_tokenize(inputContent.translate(puncFilter))
        tokensCnt = len(tokens)
        if tokensCnt < 1:
            self.raiseException("No words found")
        maxTokensCnt = int(percentage * tokensCnt)
        tags = nltk.pos_tag(tokens)
        pos = [(token, nltk.map_tag('en-ptb', 'universal', tag)) for token, tag in tags]
        filteredTokens1 = []
        for p in pos:
            if len(p[0]) < minCharLength:
                continue
            if p[1] not in posFilter:
                continue
            filteredTokens1.append(p)
        freqTokens = nltk.FreqDist(tokens)
        content = ""
        cnt = 0
        for freqToken in freqTokens.most_common(tokensCnt):
            for token in filteredTokens1:
                if freqToken[0] == token[0]:
                    content = "{0}\n{1},{2},{3}".format(content, token[0], token[1], freqToken[1])
                    cnt += 1
                    break
            if cnt >= maxTokensCnt:
                break
        content = content.strip()
        self.setAnalyzerContent(content)
        return content


from django.contrib import admin

from models import Diary
import tools.webTools as tools
from django.http import HttpResponse
# Register your models here.

class DiaryAdmin(admin.ModelAdmin):
    list_display = ('weather','author','timestamp')

    def change_view(self, request, object_id, form_url='', extra_context=None):
        try:
            tempDiary = Diary.objects.get(id=object_id)
            tools.debug("obj id ",object_id)
            tools.debug("author id",tempDiary.author.id)
            tools.debug("request author id",request.user.id)
            if tempDiary.author.id != request.user.id or \
                            "user_login" not in request.session:
                tools.debug("change diary error")
                return HttpResponse("cannot access this diary")
        except:
            tools.debug("change diary except error")
            pass

        return admin.ModelAdmin.change_view(self, request, object_id, form_url=form_url, extra_context=extra_context)

admin.site.register(Diary,DiaryAdmin)