#!/usr/bin/env python

""" A simple script to test the performance of a selected controller on a vehicle for Carla0.9.6 """
import glob
import os
import sys

try:
    sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % (
        sys.version_info.major,
        sys.version_info.minor,
        'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
    pass

import carla

import random

import image_converter

# Controllers for the vehicle
#from learning_agents.imitation.imitation_agent import ImitationAgent
from CAL_agent.CAL_controller import CAL
from measurements import Measurements

try:
    import pygame
except ImportError:
    raise RuntimeError('cannot import pygame, make sure pygame package is installed')

try:
    import numpy as np
except ImportError:
    raise RuntimeError('cannot import numpy, make sure numpy package is installed')

try:
    import queue
except ImportError:
    import Queue as queue

# High level direction commands for agents' run step input
REACH_GOAL = 0.0
GO_STRAIGHT = 5.0
TURN_RIGHT = 4.0
TURN_LEFT = 3.0
LANE_FOLLOW = 2.0

class CarlaSyncMode(object):
    """
    Context manager to synchronize output from different sensors. Synchronous
    mode is enabled as long as we are inside this context

        with CarlaSyncMode(world, sensors) as sync_mode:
            while True:
                data = sync_mode.tick(timeout=1.0)

    """

    def __init__(self, world, *sensors, **kwargs):
        self.world = world
        self.sensors = sensors
        self.frame = None
        self.delta_seconds = 1.0 / 20
        self._queues = []
        self._settings = None

    def __enter__(self):
        self._settings = self.world.get_settings()
        self.frame = self.world.apply_settings(carla.WorldSettings(
            no_rendering_mode=False,
            synchronous_mode=True,
            fixed_delta_seconds=self.delta_seconds))
        self.world.set_weather(carla.WeatherParameters.ClearNoon)

        def make_queue(register_event):
            q = queue.Queue()
            register_event(q.put)
            self._queues.append(q)

        make_queue(self.world.on_tick)
        for sensor in self.sensors:
            make_queue(sensor.listen)
        return self

    def tick(self, timeout):
        self.frame = self.world.tick()
        data = [self._retrieve_data(q, timeout) for q in self._queues]
        assert all(x.frame == self.frame for x in data)
        return data

    def __exit__(self, *args, **kwargs):
        self.world.apply_settings(self._settings)

    def _retrieve_data(self, sensor_queue, timeout):
        while True:
            data = sensor_queue.get(timeout=timeout)
            if data.frame == self.frame:
                return data

def draw_image(surface, image, blend=False):
    array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
    array = np.reshape(array, (image.height, image.width, 4))
    array = array[:, :, :3]
    array = array[:, :, ::-1]
    image_surface = pygame.surfarray.make_surface(array.swapaxes(0, 1))
    if blend:
        image_surface.set_alpha(100)
    surface.blit(image_surface, (0, 0))

def get_font():
    fonts = [x for x in pygame.font.get_fonts()]
    default_font = 'ubuntumono'
    font = default_font if default_font in fonts else fonts[0]
    font = pygame.font.match_font(font)
    return pygame.font.Font(font, 14)

def should_quit():
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            return True
        elif event.type == pygame.KEYUP:
            if event.key == pygame.K_ESCAPE:
                return True
    return False

def apply_agent_control(image, player, agent):
    """ Function to pack up the data into the necessary format for the agent to process """
    # print out the timestatmp of the image
    meas =  Measurements()
    meas.update_measurements(image, player)
    # Do some processing of the image dat
    # Store processed data to pass to agent
    sensor_data = {}
    sensor_data['CameraRGB'] = image_converter.to_rgb_array(image)

    # Process next control step from the agent and execute it
    control = agent.run_step(meas, sensor_data, LANE_FOLLOW, None)

    player.apply_control(control)

def main():
    actor_list = []
    # Init client connection
    client = carla.Client('localhost', 2000)
    client.set_timeout(2.0)
    client.load_world('Town01')
    world = client.get_world()

    # Set up PYGAME window for easier visualization
    pygame.init()
    display = pygame.display.set_mode((800, 600),pygame.HWSURFACE | pygame.DOUBLEBUF)
    font = get_font()
    clock = pygame.time.Clock()

    try:
        # Load the approapite settings for the experiment
        bp_lib = world.get_blueprint_library()

        # Spawn the agent's vehicle into the scene
        vehicle_bp = bp_lib.find('vehicle.ford.mustang')
        vehicle_tf = carla.Transform(carla.Location(2, 120, 2), carla.Rotation(yaw=-90))
        vehicle = world.spawn_actor(vehicle_bp, vehicle_tf)
        actor_list.append(vehicle)
        # Advance to the next simulation step in the simulator to ensure car is spawned
        world.tick()

        # Create RGB camera
        camera_bp = bp_lib.find('sensor.camera.rgb')
        camera_bp.set_attribute('image_size_x', '800')
        camera_bp.set_attribute('image_size_y', '600')
        # Config for CAL
        camera_bp.set_attribute('fov', '90')
        camera_rgb = world.spawn_actor(camera_bp, carla.Transform(carla.Location(x=1.44, z=1.2), carla.Rotation(pitch=0)), attach_to=vehicle)

        # Config for IML
        # camera_bp.set_attribute('fov', '100')
        # camera_rgb = world.spawn_actor(camera_bp, carla.Transform(carla.Location(x=2, z=1.4), carla.Rotation(pitch=-15)), attach_to=vehicle)
        actor_list.append(camera_rgb)

        # Finally, attach a controller to the vehicle
        #agent = ImitationAgent('Town01', True, vehicle)
        agent = CAL('Town01', vehicle)

        #Spawn a pedestrian in front of the car
        ped_bp = random.choice(bp_lib.filter('pedestrian'))
        ped_tf = carla.Transform(carla.Location(2, 80, 2), carla.Rotation())
        ped_actor = world.spawn_actor(ped_bp, ped_tf)
        actor_list.append(ped_actor)
        world.tick() # Advance on step to ensure
        world.wait_for_tick(seconds = 2.0) # Wait up to two seconds for step update

        # Create a synchronous mode context.
        with CarlaSyncMode(world, camera_rgb, fps=20) as sync_mode:
            while True:
                if should_quit():
                    return
                clock.tick()

                # Advance the simulation and wait for the data.
                snapshot, image_rgb = sync_mode.tick(timeout=2.0)

                apply_agent_control(image_rgb, vehicle, agent)

                fps = round(1.0 / snapshot.timestamp.delta_seconds)

                # Draw the display.
                draw_image(display, image_rgb)

                display.blit(
                    font.render('% 5d FPS (real)' % clock.get_fps(), True, (255, 255, 255)),
                    (8, 10))
                display.blit(
                    font.render('% 5d FPS (simulated)' % fps, True, (255, 255, 255)),
                    (8, 28))
                pygame.display.flip()

    finally:
        print('destroying actors.')
        for actor in actor_list:
            actor.destroy()

        pygame.quit()
        print('done.')

if __name__ == '__main__':

    try:

        main()

    except KeyboardInterrupt:
        print('\nCancelled by user. Bye!')


import math
import random
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from itertools import count
import numpy as np
from pdb import set_trace

import sys
sys.path.append('..')
from env.atari_task import Pendulum,MountainCarContinuous,BipedalWalker
from core.memory import ReplayMemory,Transition
from core.model_ac import ActorNet,CriticNet
from core.policy_greedy import GreedyPolicy
from core.random_process import OrnsteinUhlenbeckProcess
#from tf_logger import Logger
from utils.tools import *
from utils import plot

class DQNAgent:
    def __init__(self):
        self.steps_done = 0
        self.batch_size = 64
        self.GAMMA= 0.99
        self.num_episodes = 2000
        self.task = BipedalWalker()
        self.memory = ReplayMemory(10000,self.batch_size)
        self.eval_actornet = ActorNet(24,4)
        self.target_actornet = ActorNet(24,4)
        self.target_actornet.load_state_dict(self.eval_actornet.state_dict())
        self.eval_criticnet =  CriticNet(24,4)
        self.target_criticnet = CriticNet(24,4)
        self.target_criticnet.load_state_dict(self.eval_criticnet.state_dict())
        self.actor_optimizer = torch.optim.Adam(self.eval_actornet.parameters(), lr=1e-4)
        self.critic_optimizer = torch.optim.Adam(self.eval_criticnet.parameters(), lr=1e-3,weight_decay=0.01)
        self.random_process = OrnsteinUhlenbeckProcess(theta=0.15,sigma=0.2)
        self.epsilon = 1
        self.epsilon_d = 1/30000
        self.criterion = torch.nn.MSELoss()
        #self.policy = GreedyPolicy(EPS_START = 0.9,EPS_END = 0.05,EPS_DECAY = 200)
        self.episode_durations= []

    def select_action(self,state):
        action = to_numpy(self.eval_actornet(Variable(state).type(torch.FloatTensor)))

        if self.steps_done % 50 == 0:
            print(action,to_numpy(self.eval_criticnet(Variable(state).type(torch.FloatTensor),to_tensor(action))),max(self.epsilon,0) * self.random_process.sample()*2)
        action += max(self.epsilon,0) * self.random_process.sample()
        self.epsilon -= self.epsilon_d
        self.steps_done += 1
        #set_trace()
        return action

    def optimize_model(self):
        transitions = self.memory.sample()
        if not transitions:
            return

        batch = Transition(*zip(*transitions))
        state_batch = Variable(torch.cat(batch.state)).type(torch.FloatTensor)
        action_batch = Variable(torch.cat(batch.action)).type(torch.FloatTensor)
        reward_batch = Variable(torch.cat(batch.reward)).type(torch.FloatTensor)
        next_state_batch = Variable(torch.cat(batch.next_state)).type(torch.FloatTensor)
        done = Variable(torch.cat(batch.done)).type(torch.FloatTensor)
        #print(reward_batch)
        #set_trace()
        q_next=self.target_criticnet(next_state_batch,self.target_actornet(next_state_batch))
        q_next = self.GAMMA * q_next * (1 - done)
        q_next.add_(reward_batch)
        q_next=Variable(q_next.data)
        q_ = self.eval_criticnet(state_batch,action_batch)
        critic_loss = self.criterion(q_, q_next)
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        #torch.nn.utils.clip_grad_norm(self.model.parameters(), 1)
        self.critic_optimizer.step()

        actor_loss = -self.eval_criticnet(state_batch, self.eval_actornet(state_batch))
        actor_loss = actor_loss.mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        #set_trace()
        soft_target_model_updates(self.target_actornet,self.eval_actornet,0.001)
        soft_target_model_updates(self.target_criticnet,self.eval_criticnet,0.001)

    def episode(self):
        episode_num=[]
        episode_durations=[]
        episode_temp=[]
        self.random_process.reset_states()
        for i_episode in range(self.num_episodes):
            state = torch.from_numpy(self.task.reset()).unsqueeze(0)
            for t in count():
                #set_trace()
                self.task.env.render()
                action = self.select_action(state)#.unsqueeze(0))
                next_state, reward, done, _ = self.task.step(action[0])
                #set_trace()

                #reward = reward +next_state[0]
                #print(reward)
                action = torch.from_numpy(action)#.unsqueeze(0)
                next_state = torch.from_numpy(next_state).unsqueeze(0)
                reward_ = torch.FloatTensor([reward]).unsqueeze(0)
                done_ = torch.from_numpy(np.array([int(done)])).unsqueeze(0)

                self.memory.push(state,action,next_state,reward_, done_)
                state = next_state
                self.optimize_model()

                episode_temp.append(reward)

                if (self.steps_done ) % 100 == 0:
                    episode_num.append(self.steps_done + 1)
                    episode_durations.append(episode_temp)
                    episode_temp = []
                    plot.plot_line(episode_num,episode_durations)

                if done  or t > 300:
                    break

            if self.steps_done>30000:
                return 1

        self.task.env.render(close=True)
        self.task.env.close()

#if __name__ == '__main__':
a= DQNAgent()
a.episode()


from flask import g
from influenceexplorer import InfluenceExplorer
from sunlightapi import sunlight
import json
import requests

from capitolphone import settings

TITLES = {
    'Rep': 'Representative',
    'Sen': 'Senator',
}

sunlight.apikey = settings.SUNLIGHT_KEY
ie = InfluenceExplorer(settings.SUNLIGHT_KEY)

def load_call(sid, params):
    """ Loads a call from the datastore or creates a new one if one
        does not exist. Appends the current call status to the list
        of requests involved in this call.

        sid: the unique call ID from Twilio
        params: the POSTed request parameters
    """

    # find existing call
    doc = g.db.calls.find_one({'call_sid': sid})

    if doc is None:
        # create new call if call does not exist
        doc = {
            'call_sid': sid,
            'from': params['From'],
            'to': params['To'],
            'caller_name': params.get('CallerName', None),
            'context': {
                'zipcode': None,
                'legislator': None,
            },
        }
        g.db.calls.insert(doc)

    # create array for requests list
    if 'requests' not in doc:
        doc['requests'] = []

    # append current request information and update current status
    doc['requests'].append({
        'timestamp': g.now,
        'call_status': params['CallStatus']
    })
    doc['current_status'] = params['CallStatus']

    return doc

def legislators_for_zip(zipcode):
    """ Find legislators that represent the specified zipcode.
        Results are cached in the datastore for faster lookup.
        If more than one member of the House represents a zipcode,
        both legislators will be returned.

        zipcode: the 5-digit zipcode to search
    """

    # attempt to find cached legislators
    doc = g.db.legislatorsByZipcode.find_one({'zipcode': zipcode})

    if doc is None:

        # load from Sunlight Congress API if not cached locally
        results = sunlight.legislators.allForZip(zipcode)

        # create a copy of the Legislator object dict
        legislators = [r.__dict__.copy() for r in results]

        # sort the legislators by reverse title so Senators are listed
        # before members of the House
        legislators.sort(lambda x, y: -cmp(x['title'], y['title']))

        # move current title to short_title and update title with
        # a more readable version, create full name
        for l in legislators:
            l['short_title'] = l['title']
            l['title'] = TITLES.get(l['title'], 'Representative')
            l['fullname'] = "%s %s %s" % (l['title'], l['firstname'], l['lastname'])

        # save new zipcode results document
        g.db.legislatorsByZipcode.insert({
            'timestamp': g.now,
            'zipcode': zipcode,
            'legislators': legislators,
        })

    else:

        # get legislators from cache
        legislators = doc['legislators']

    return legislators

def resolve_entity_id(crp_id):
    """ Convert a CRP candidate ID into an IE entity ID.
        Cached locally for better performance.
    """

    doc = g.db.crpMapping.find_one({'crp_id': crp_id})

    if doc is None:
        entity_id = ie.entities.id_lookup("urn:crp:recipient", crp_id)[0]['id']
        g.db.crpMapping.insert({
            'crp_id': crp_id,
            'entity_id': entity_id,
        })
    else:
        entity_id = doc['entity_id']

    return entity_id

def top_contributors(legislator):
    entity_id = resolve_entity_id(legislator['crp_id'])
    contribs = ie.pol.contributors(entity_id, cycle='2012', limit=10)
    return contribs

def legislator_bio(legislator):
    entity_id = resolve_entity_id(legislator['crp_id'])
    metadata = ie.entities.metadata(entity_id)
    return metadata['metadata']['bio']

def committee_iter(committees):
    for comm in committees:
        yield comm.name
        if comm.subcommittees:
            for subcomm in comm.subcommittees:
                yield subcomm.name

def committees(legislator):
    comms = sunlight.committees.allForLegislator(g.legislator['bioguide_id'])
    names = " ".join("%s." % c for c in committee_iter(comms))
    return names

def recent_votes(legislator):

    VOTES = {
        'Yea': 'yes',
        'Nay': 'no',
    }

    url = "http://api.realtimecongress.org/api/v1/votes.json"

    voter_key = "voter_ids.%s" % legislator['bioguide_id']

    params = {
        'per_page': 5,
        'vote_type': 'passage',
        '%s__exists' % voter_key: True,
        'sections': "question,result,%s" % voter_key,
    }

    resp = requests.get(url, params=params, headers={'X-APIKEY': settings.SUNLIGHT_KEY})

    data = json.loads(resp.content)['votes']
    for vote in data:
        voted = vote['voter_ids'][legislator['bioguide_id']]
        vote['voted'] = VOTES.get(voted, voted)
        vote['question'] = vote['question'].split(':')[-1].strip()
        del vote['voter_ids']

    return data


from rest_framework.response import Response
from rest_framework.views import APIView
from teacherapp.models import Teacher
from teacherapp.serializers import TeacherDeSerializer, TeacherSerializer

class TeacherAPIView(APIView):

    def get(self,request,*args,**kwargs):
        tea_id = kwargs.get('id')

        if tea_id:
            tea_obj = Teacher.objects.get(pk=tea_id)
            teacher_serializer = TeacherSerializer(tea_obj).data
            print(teacher_serializer)
            return Response({
                'status':200,
                'message':'查询单个教师成功',
                'result':teacher_serializer
            })
        else:
            teacher_objects_all = Teacher.objects.all()
            tea_data = TeacherSerializer(teacher_objects_all,many=True).data
            print(tea_data)

            return Response({
                'status':200,
                'message':'查询所有教师成功',
                'result':tea_data
            })

    def post(self,request, *args, **kwargs):
        request_data = request.data

        if not isinstance(request_data, dict) or request_data =={}:
            return Response({
                'status':400,
                'message':'参数错误',
            })

        serializer = TeacherDeSerializer(data = request_data)

        if serializer.is_valid():
            tea_ser = serializer.save()
            return Response({
                'status':200,
                'message':'教师添加成功',
                'result':TeacherSerializer(tea_ser).data
            })
        else:
            return Response({
                'status':400,
                'message':'添加教师失败',
                'result':serializer.errors
            })


# Copyright (c) 2011-2020 Eric Froemling
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# -----------------------------------------------------------------------------
"""UIs for debugging purposes."""

from __future__ import annotations

from typing import TYPE_CHECKING, cast

import ba

if TYPE_CHECKING:
    pass

class DebugWindow(ba.Window):
    """Window for debugging internal values."""

    def __init__(self, transition: str = 'in_right'):
        # pylint: disable=too-many-statements
        # pylint: disable=cyclic-import
        from bastd.ui import popup

        self._width = width = 580
        self._height = height = (350 if ba.app.small_ui else
                                 420 if ba.app.med_ui else 520)

        self._scroll_width = self._width - 100
        self._scroll_height = self._height - 120

        self._sub_width = self._scroll_width * 0.95
        self._sub_height = 520

        self._stress_test_game_type = 'Random'
        self._stress_test_playlist = '__default__'
        self._stress_test_player_count = 8
        self._stress_test_round_duration = 30

        self._r = 'debugWindow'
        super().__init__(root_widget=ba.containerwidget(
            size=(width, height),
            transition=transition,
            scale=(
                2.35 if ba.app.small_ui else 1.55 if ba.app.med_ui else 1.0),
            stack_offset=(0, -30) if ba.app.small_ui else (0, 0)))

        self._done_button = btn = ba.buttonwidget(
            parent=self._root_widget,
            position=(40, height - 67),
            size=(120, 60),
            scale=0.8,
            autoselect=True,
            label=ba.Lstr(resource='doneText'),
            on_activate_call=self._done)
        ba.containerwidget(edit=self._root_widget, cancel_button=btn)
        ba.textwidget(parent=self._root_widget,
                      position=(0, height - 60),
                      size=(width, 30),
                      text=ba.Lstr(resource=self._r + '.titleText'),
                      h_align='center',
                      color=ba.app.title_color,
                      v_align='center',
                      maxwidth=260)

        self._scrollwidget = ba.scrollwidget(
            parent=self._root_widget,
            highlight=False,
            size=(self._scroll_width, self._scroll_height),
            position=((self._width - self._scroll_width) * 0.5, 50))
        ba.containerwidget(edit=self._scrollwidget, claims_left_right=True)

        self._subcontainer = ba.containerwidget(parent=self._scrollwidget,
                                                size=(self._sub_width,
                                                      self._sub_height),
                                                background=False)

        v = self._sub_height - 70
        button_width = 300
        btn = ba.buttonwidget(
            parent=self._subcontainer,
            position=((self._sub_width - button_width) * 0.5, v),
            size=(button_width, 60),
            autoselect=True,
            label=ba.Lstr(resource=self._r + '.runCPUBenchmarkText'),
            on_activate_call=self._run_cpu_benchmark_pressed)
        ba.widget(edit=btn,
                  up_widget=self._done_button,
                  left_widget=self._done_button)
        v -= 60

        ba.buttonwidget(parent=self._subcontainer,
                        position=((self._sub_width - button_width) * 0.5, v),
                        size=(button_width, 60),
                        autoselect=True,
                        label=ba.Lstr(resource=self._r +
                                      '.runGPUBenchmarkText'),
                        on_activate_call=self._run_gpu_benchmark_pressed)
        v -= 60

        ba.buttonwidget(
            parent=self._subcontainer,
            position=((self._sub_width - button_width) * 0.5, v),
            size=(button_width, 60),
            autoselect=True,
            label=ba.Lstr(resource=self._r + '.runMediaReloadBenchmarkText'),
            on_activate_call=self._run_media_reload_benchmark_pressed)
        v -= 60

        ba.textwidget(parent=self._subcontainer,
                      position=(self._sub_width * 0.5, v + 22),
                      size=(0, 0),
                      text=ba.Lstr(resource=self._r + '.stressTestTitleText'),
                      maxwidth=200,
                      color=ba.app.heading_color,
                      scale=0.85,
                      h_align='center',
                      v_align='center')
        v -= 45

        x_offs = 165
        ba.textwidget(parent=self._subcontainer,
                      position=(x_offs - 10, v + 22),
                      size=(0, 0),
                      text=ba.Lstr(resource=self._r +
                                   '.stressTestPlaylistTypeText'),
                      maxwidth=130,
                      color=ba.app.heading_color,
                      scale=0.65,
                      h_align='right',
                      v_align='center')

        popup.PopupMenu(
            parent=self._subcontainer,
            position=(x_offs, v),
            width=150,
            choices=['Random', 'Teams', 'Free-For-All'],
            choices_display=[
                ba.Lstr(resource=a) for a in [
                    'randomText', 'playModes.teamsText',
                    'playModes.freeForAllText'
                ]
            ],
            current_choice='Auto',
            on_value_change_call=self._stress_test_game_type_selected)

        v -= 46
        ba.textwidget(parent=self._subcontainer,
                      position=(x_offs - 10, v + 22),
                      size=(0, 0),
                      text=ba.Lstr(resource=self._r +
                                   '.stressTestPlaylistNameText'),
                      maxwidth=130,
                      color=ba.app.heading_color,
                      scale=0.65,
                      h_align='right',
                      v_align='center')

        self._stress_test_playlist_name_field = ba.textwidget(
            parent=self._subcontainer,
            position=(x_offs + 5, v - 5),
            size=(250, 46),
            text=self._stress_test_playlist,
            h_align='left',
            v_align='center',
            autoselect=True,
            color=(0.9, 0.9, 0.9, 1.0),
            description=ba.Lstr(resource=self._r +
                                '.stressTestPlaylistDescriptionText'),
            editable=True,
            padding=4)
        v -= 29
        x_sub = 60

        # Player count.
        ba.textwidget(parent=self._subcontainer,
                      position=(x_offs - 10, v),
                      size=(0, 0),
                      text=ba.Lstr(resource=self._r +
                                   '.stressTestPlayerCountText'),
                      color=(0.8, 0.8, 0.8, 1.0),
                      h_align='right',
                      v_align='center',
                      scale=0.65,
                      maxwidth=130)
        self._stress_test_player_count_text = ba.textwidget(
            parent=self._subcontainer,
            position=(246 - x_sub, v - 14),
            size=(60, 28),
            editable=False,
            color=(0.3, 1.0, 0.3, 1.0),
            h_align='right',
            v_align='center',
            text=str(self._stress_test_player_count),
            padding=2)
        ba.buttonwidget(parent=self._subcontainer,
                        position=(330 - x_sub, v - 11),
                        size=(28, 28),
                        label='-',
                        autoselect=True,
                        on_activate_call=ba.Call(
                            self._stress_test_player_count_decrement),
                        repeat=True,
                        enable_sound=True)
        ba.buttonwidget(parent=self._subcontainer,
                        position=(380 - x_sub, v - 11),
                        size=(28, 28),
                        label='+',
                        autoselect=True,
                        on_activate_call=ba.Call(
                            self._stress_test_player_count_increment),
                        repeat=True,
                        enable_sound=True)
        v -= 42

        # Round duration.
        ba.textwidget(parent=self._subcontainer,
                      position=(x_offs - 10, v),
                      size=(0, 0),
                      text=ba.Lstr(resource=self._r +
                                   '.stressTestRoundDurationText'),
                      color=(0.8, 0.8, 0.8, 1.0),
                      h_align='right',
                      v_align='center',
                      scale=0.65,
                      maxwidth=130)
        self._stress_test_round_duration_text = ba.textwidget(
            parent=self._subcontainer,
            position=(246 - x_sub, v - 14),
            size=(60, 28),
            editable=False,
            color=(0.3, 1.0, 0.3, 1.0),
            h_align='right',
            v_align='center',
            text=str(self._stress_test_round_duration),
            padding=2)
        ba.buttonwidget(parent=self._subcontainer,
                        position=(330 - x_sub, v - 11),
                        size=(28, 28),
                        label='-',
                        autoselect=True,
                        on_activate_call=ba.Call(
                            self._stress_test_round_duration_decrement),
                        repeat=True,
                        enable_sound=True)
        ba.buttonwidget(parent=self._subcontainer,
                        position=(380 - x_sub, v - 11),
                        size=(28, 28),
                        label='+',
                        autoselect=True,
                        on_activate_call=ba.Call(
                            self._stress_test_round_duration_increment),
                        repeat=True,
                        enable_sound=True)
        v -= 82
        btn = ba.buttonwidget(
            parent=self._subcontainer,
            position=((self._sub_width - button_width) * 0.5, v),
            size=(button_width, 60),
            autoselect=True,
            label=ba.Lstr(resource=self._r + '.runStressTestText'),
            on_activate_call=self._stress_test_pressed)
        ba.widget(btn, show_buffer_bottom=50)

    def _stress_test_player_count_decrement(self) -> None:
        self._stress_test_player_count = max(
            1, self._stress_test_player_count - 1)
        ba.textwidget(edit=self._stress_test_player_count_text,
                      text=str(self._stress_test_player_count))

    def _stress_test_player_count_increment(self) -> None:
        self._stress_test_player_count = self._stress_test_player_count + 1
        ba.textwidget(edit=self._stress_test_player_count_text,
                      text=str(self._stress_test_player_count))

    def _stress_test_round_duration_decrement(self) -> None:
        self._stress_test_round_duration = max(
            10, self._stress_test_round_duration - 10)
        ba.textwidget(edit=self._stress_test_round_duration_text,
                      text=str(self._stress_test_round_duration))

    def _stress_test_round_duration_increment(self) -> None:
        self._stress_test_round_duration = (self._stress_test_round_duration +
                                            10)
        ba.textwidget(edit=self._stress_test_round_duration_text,
                      text=str(self._stress_test_round_duration))

    def _stress_test_game_type_selected(self, game_type: str) -> None:
        self._stress_test_game_type = game_type

    def _run_cpu_benchmark_pressed(self) -> None:
        from ba.internal import run_cpu_benchmark
        run_cpu_benchmark()

    def _run_gpu_benchmark_pressed(self) -> None:
        from ba.internal import run_gpu_benchmark
        run_gpu_benchmark()

    def _run_media_reload_benchmark_pressed(self) -> None:
        from ba.internal import run_media_reload_benchmark
        run_media_reload_benchmark()

    def _stress_test_pressed(self) -> None:
        from ba.internal import run_stress_test
        run_stress_test(
            playlist_type=self._stress_test_game_type,
            playlist_name=cast(
                str,
                ba.textwidget(query=self._stress_test_playlist_name_field)),
            player_count=self._stress_test_player_count,
            round_duration=self._stress_test_round_duration)
        ba.containerwidget(edit=self._root_widget, transition='out_right')

    def _done(self) -> None:
        # pylint: disable=cyclic-import
        from bastd.ui.settings.advanced import AdvancedSettingsWindow
        ba.containerwidget(edit=self._root_widget, transition='out_right')
        ba.app.main_menu_window = (AdvancedSettingsWindow(
            transition='in_left').get_root_widget())


import matplotlib.pyplot as plt
from scipy.stats import sem
import time
import math
from timeit import default_timer as timer
from typing import List, Tuple
from jsonpickle.unpickler import decode
from petrelic.bn import Bn
from stroll import Client, Server
import jsonpickle
from credential import AnonymousCredential, BlindSignature, DisclosureProof, IssueRequest, PublicKey, SecretKey, generate_key
from petrelic.multiplicative.pairing import G1, G1Element, G2, G2Element, GT

from issuer import Issuer
from user import User

## Utility functions for testing
def decode_data(data: bytes):
    return jsonpickle.decode(data.decode())

def get_keys(subscriptions: List[str]) -> Tuple[bytes, bytes]:
    sk_enc, pk_enc = Server.generate_ca(["username"] + subscriptions)
    return sk_enc, pk_enc

#### =============
#### SUCCESS CASES
#### =============

subscriptions = ["t1", "t2", "t3"]
queried_subscriptions = ["t1", "t2"]
username = "test"
message = b'30.00.00'
server, client = Server(), Client()

sk, pk = get_keys(subscriptions)
packets = [[], [], [], []]

## Test that the keys were generated correctly by the server
def key_generation():
    sk_enc, pk_enc = get_keys(subscriptions)
    sk: SecretKey = decode_data(sk_enc)
    pk: PublicKey = decode_data(pk_enc)

    return sk, pk

## Test the successful generation of a credential step by step, as well as a stroll request
def credential_issuance():
    # Generate issue request and test it
    issue_request_enc, user_state = client.prepare_registration(pk, username, subscriptions)
    issue_request: IssueRequest = decode_data(issue_request_enc)

    # Process registration and test it server-side
    blind_signature_enc = server.process_registration(sk, pk, issue_request_enc, username, subscriptions)
    blind_signature: BlindSignature = decode_data(blind_signature_enc)

    # Obtain credential client side and test it
    credential_enc = client.process_registration_response(pk, blind_signature_enc, user_state)
    credential: AnonymousCredential = decode_data(credential_enc)

    packets[0] += [len(credential_enc)] + [len(blind_signature_enc) * 2] + [len(issue_request_enc) * 2]

    return credential_enc

def credential_showing(credential_enc):
    # Create a secret stroll request (disclosure proof) and test it
    stroll_request_enc = client.sign_request(pk, credential_enc, message, queried_subscriptions)
    stroll_request: DisclosureProof = decode_data(stroll_request_enc)

    packets[1] += [len(credential_enc) * 2] + [len(stroll_request_enc) * 2]

    return stroll_request_enc

def credential_verification(stroll_request_enc):
  server.check_request_signature(pk, message, queried_subscriptions, stroll_request_enc)

  packets[2].append(len(stroll_request_enc))

## Test the successful generation of a credential step by step, as well as a stroll request
def test_successful_request():
    subscriptions = ["t1", "t2", "t3"]
    queried_subscriptions = ["t1", "t2"]
    username = "test"
    message = b'30.00.00'
    sk, pk = get_keys(subscriptions)
    server, client = Server(), Client()

    # Generate issue request and test it
    issue_request_enc, user_state = client.prepare_registration(pk, username, subscriptions)
    issue_request: IssueRequest = decode_data(issue_request_enc)

    # Process registration and test it server-side
    blind_signature_enc = server.process_registration(sk, pk, issue_request_enc, username, subscriptions)
    blind_signature: BlindSignature = decode_data(blind_signature_enc)

    # Obtain credential client side and test it
    credential_enc = client.process_registration_response(pk, blind_signature_enc, user_state)
    credential: AnonymousCredential = decode_data(credential_enc)

    # Create a secret stroll request (disclosure proof) and test it
    stroll_request_enc = client.sign_request(pk, credential_enc, message, queried_subscriptions)
    stroll_request: DisclosureProof = decode_data(stroll_request_enc)

    packets[3] += [(len(issue_request_enc) * 2 + len(blind_signature_enc) * 2) * 1.25] + [(len(credential_enc) * 2 + len(stroll_request_enc) * 2) * 1.25]

    # Test that the request is valid
    assert server.check_request_signature(pk, message, queried_subscriptions, stroll_request_enc)

### Perf eval
num_iter = 5000

res = [[], [], [], [], []]
fun = [key_generation, credential_issuance, credential_showing, credential_verification, test_successful_request]
for i in range(num_iter):
  cred = None
  issuance = None
  for j in range(0, 5):
    start = timer()
    if j == 1:
      cred = fun[j]()
    elif j == 2:
      issuance = fun[j](cred)
    elif j == 3:
      fun[j](issuance)
    else:
      fun[j]()

    end = timer()
    res[j].append(end - start)

# time
stems = list(map(lambda x: sem(x) * 150, res))
means = list(map(lambda x: sum(x) / len(x), res))

# com
stems_com = list(map(lambda x : sem(x)* 10, packets))
means_com = list(map(lambda x: sum(x) / len(x), packets))

plt.errorbar(["Key gen.", "Cred. issuance", "Cred. showing", "Req. verification", "Full request"], means, stems, linestyle='None', marker='s')
plt.ylabel("Execution Time (in ms)")
plt.savefig("exec_time.png")
plt.show()

plt.errorbar(["Credential issuance", "Credential showing", "Request verification", "Full request"], means_com, stems_com, linestyle='None', marker='s')
plt.ylabel("Packets Exchanged (bytes length)")
plt.savefig("packets.png")
plt.show()


# ‐*‐ coding: utf‐8 ‐*‐
"""
一个关键词serp上同一个域名出现N个url排名 计算1次，相当于计算首页词数
"""
import requests
from pyquery import PyQuery as pq
import threading
import queue
import time
from urllib.parse import urlparse

class bdpcCover(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)

    # 读取txt文件 关键词进入队列
    @staticmethod
    def read_file(filepath):
        q = queue.Queue()
        for kwd in open(filepath,encoding='utf-8'):
            kwd = kwd.strip()
            q.put(kwd)
        return q

    # 获取某词serp源码
    def get_html(self,url,retry=2):
        try:
            r = requests.get(url=url,headers=user_agent,timeout=5)
        except Exception as e:
            print('获取源码失败',e)
            if retry > 0:
                self.get_html(url,retry-1)
        else:
            html = r.text
            return html

    # 获取某词serp源码上自然排名的所有url
    def get_encrpt_urls(self,html):
        encrypt_url_list = []
        if html and '_百度搜索' in html:
            doc = pq(html)
            try:
                a_list = doc('.t a').items()
            except Exception as e:
                print('未提取到serp上的解密url', e, url)
            else:
                for a in a_list:
                    encrypt_url = a.attr('href')
                    if encrypt_url.find('http://www.baidu.com/link?url=') == 0:
                        encrypt_url_list.append(encrypt_url)
        return encrypt_url_list

    # 解密某条加密url
    def decrypt_url(self,encrypt_url,retry=1):
        try:
            encrypt_url = encrypt_url.replace('http://','https://')
            r = requests.head(encrypt_url,headers=user_agent)
        except Exception as e:
            print(encrypt_url,'解密失败',e)
            if retry > 0:
                self.decrypt_url(encrypt_url,retry-1)
        else:
            return r.headers['Location']

    # 获取某词serp源码首页排名真实url
    def get_real_urls(self,encrypt_url_list):
        real_url_list = [self.decrypt_url(encrypt_url) for encrypt_url in encrypt_url_list]
        return real_url_list

    # 提取某条url域名部分
    def get_domain(self,real_url):
        try:
           res = urlparse(real_url)
        except Exception as e:
           print (e,real_url)
           domain = "xxx"
        else:
           domain = res.netloc
        return domain

    # 获取某词serp源码首页排名真实url的域名部分
    def get_domains(self,real_url_list):
            domain_list = [self.get_domain(real_url) for real_url in real_url_list]
            # 搜一个词 同一个域名多个url出现排名 只计算一次
            domain_set = set(domain_list)
            return domain_set

    # 线程函数
    def run(self):
        global success_num
        while 1:
            kwd = q.get()
            url = "https://www.baidu.com/s?ie=utf-8&wd={0}".format(kwd)
            html = self.get_html(url)
            encrypt_url_list = self.get_encrpt_urls(html)
            real_url_list = self.get_real_urls(encrypt_url_list)
            domain_set = self.get_domains(real_url_list)
            if domain_set:
                try:
                    threadLock.acquire()
                    for domain in domain_set:
                        result[domain] = result[domain]+1 if domain in result else 1
                    success_num += 1
                    print('查询成功{0}个'.format(success_num))
                except Exception as e:
                    print(e)
                finally:
                    print (kwd,'查询结束')
                    threadLock.release()
            q.task_done()

    # 保存数据
    @staticmethod
    def save():
        print ('开始save.....')
        res_sort = sorted(result.items(), key=lambda s: s[1], reverse=True)
        print(res_sort)
        with open('result2.txt','w',encoding="utf-8") as f:
            for domain,value in res_sort:
                # print(domain,type(domain),type(str(value)))
                f.write(str(domain)+'\t'+str(value)+'\n')

if __name__ == "__main__":
    start = time.time()

    user_agent = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'}
    threadLock = threading.Lock()  # 锁
    result = {}   # 初始结果保存字典
    success_num = 0  # 查询成功个数
    q = bdpcCover.read_file('kwd.txt')
    all_num = q.qsize() #总词数

    # 设置线程数
    for i in list(range(5)):
        t = bdpcCover()
        t.setDaemon(True)
        t.start()
    q.join()

    # 结果保存文件
    bdpcCover.save()
    end = time.time()
    print('\n关键词共{0}个,查询成功{1}个,耗时{2}min'.format(all_num,success_num,(end-start)/60) )


"""elearning URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,include
from courses import views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('',views.index,name='index'),
    path('contact/', views.contact, name='contact'),
    path('courses/',views.courses,name='courses'),
    path('courses/<int:course_id>',views.course_detail,name='course_detail'),   #when the reverse req comes here tskes us to the view
    path('courses/english', views.english, name='english'),
    path('courses/math', views.math, name='math'),
    path('tinymce/', include('tinymce.urls')),
    path('accounts/', include('allauth.urls')),

]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import re

class Source(object):
    def __init__(self):
        self.data={}

    def clsource(self):
        req=urllib.request.urlopen(self.choice())
        return req.read().decode()

    def output(self):
        data=self.clsource()
        soup = BeautifulSoup(data, 'html.parser')
        links=soup.find_all('td')
        datas = []
        a = 3
        while a < len(links):
            datas.append(links[a].get_text().strip())
            a = a + 5
        i = 0
        while i < len(datas):
            print('科目：', datas[i])
            i = i + 1
            print('成绩', datas[i])
            i = i + 1

    def choice(self):
        url=''
        datetime=input('请输入你要查询的学期（例如：2014-2015 1）：')
        if datetime=='2014-2015 1':
            url='http://eams.uestc.edu.cn/eams/teach/grade/course/person!search.action?semesterId=43&projectType='
        elif datetime=='2014-2015 2':
            url='http://eams.uestc.edu.cn/eams/teach/grade/course/person!search.action?semesterId=63&projectType='
        elif datetime=='2015-2016 1':
            url='http://eams.uestc.edu.cn/eams/teach/grade/course/person!search.action?semesterId=84&projectType='
        elif datetime=='2015-2016 2':
            url='http://eams.uestc.edu.cn/eams/teach/grade/course/person!search.action?semesterId=103&projectType='
        elif datetime=='2016-2017 1':
            url='http://eams.uestc.edu.cn/eams/teach/grade/course/person!search.action?semesterId=123&projectType='
        else :
            print('对不起，没有当前学期的成绩！')
        return url


# pylint: disable=invalid-name

import meilisearch
from meilisearch.tests import BASE_URL, MASTER_KEY

def test_get_client():
    """Tests getting a client instance."""
    client = meilisearch.Client(BASE_URL, MASTER_KEY)
    assert client.config
    response = client.health()
    assert response.status_code >= 200 and response.status_code < 400


#!/usr/bin/python3
#从iciba公开api爬取金山词霸每日一句存入mysql数据库

import requests
import json
import csv
import datetime

#定义爬取数据初始变量
first_day_str = '2012-01-01 09:00:00' #每日一句 第一次发布
first_day = datetime.datetime.strptime(first_day_str, '%Y-%m-%d %H:%M:%S') #转为datetime格式
days_DHS = datetime.datetime.now()-first_day #至爬取截止日累计天数，小时数，秒数
max_days = days_DHS.days #取天数，作为迭代上限次数
url0 = 'http://open.iciba.com/dsapi/?date='

#产生网址日期部分
def dategen(max):
    n = 0
    while n <= max:
        delta = datetime.timedelta(days=n) #迭代步长
        dategen_datetime = first_day + delta
        date = dategen_datetime.strftime('%Y-%m-%d')
        n += 1
        yield date

date_list = dategen(max_days)

# 新建csv文件

csvfile = open('/home/mloop/Documents/daily_sentence.csv', 'w')
writer = csv.writer(csvfile)
writer.writerow(('dateline', 'content', 'note', 'translation'))

#获取数据并写入数据库
for date in date_list:

    data = requests.get(url0 + date).text
    jdata = json.loads(data)
    dateline = jdata.get('dateline')
    content = jdata.get('content')
    note = jdata.get('note')
    translation = jdata.get('translation')
    writer.writerow((dateline, content, note, translation))
    print('writing', dateline)

#关闭csv文件
csvfile.close()


# -*- coding: UTF-8 -*-
# @Time    : 2018/11/27 2:46 PM
# @File    : train_server.py
# @Author  : jian<jian@mltalker.com>
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function
import tornado.httpserver
import tornado.ioloop
import tornado.options
from tornado.options import define, options
from tornado import web, gen
from tornado import httpclient
from antgo.utils import logger
from antgo.utils.utils import *
from datetime import datetime
import tornado.web
import multiprocessing
import json
import os
import uuid
import time
import yaml
import subprocess
import traceback
import sys
import shutil
import functools
import zmq
from zmq.eventloop import future
import requests
import numpy as np

class BaseHandler(tornado.web.RequestHandler):
  @property
  def signature(self):
    return self.settings.get('signature', '')

  @property
  def main_folder(self):
    return self.settings.get('main_folder', '')

  @property
  def main_param(self):
    return self.settings.get('main_param', None)

  @property
  def main_file(self):
    return self.settings.get('main_file', '')

  @property
  def experiment_records(self):
    # {'id': {'study_name':, 'trail_name': , 'start_time': ,'stop_time':, 'measure': }}
    return self.settings.get('experiment_records', {})

  @property
  def file_records(self):
    return self.settings.get('file_records', {})

  @property
  def server_records(self):
    return self.settings.get('server_records', {})

  @property
  def device_list(self):
    return self.settings.get('device_list', [])

  @property
  def max_time(self):
    return self.settings.get('max_time', '10d')

  @property
  def server_port(self):
    return self.settings.get('server_port', 0)

  @property
  def is_worker(self):
    return self.settings.get('is_worker', True)

  @property
  def client_socket(self):
    return self.settings.get('client_socket', None)

  @property
  def html_template(self):
    return self.settings['html_template']

  @property
  def image_folder(self):
    return self.settings['static_path']

def update_suggestion_process(server_records, experiment_records):
  running_experiments = []
  for experiment_id, experiment_record in experiment_records.items():
    if experiment_record['status'] != 'stop':
      # update trail status
      running_experiments.append(experiment_id)

  if len(running_experiments) == 0:
    return

  requests.post('http://%s/update/' % server_records['master_ip'],
                data={'experiments': json.dumps(running_experiments),
                      'signature': server_records['signature']})

def launch_train_process(server_records, experiment_records, content):
  # study name and trial name
  study_name = content['study_name']
  trail_name = content['trail_name']
  experiment_id = trail_name
  if experiment_id in experiment_records:
    logger.error('existed experiment id %s'%experiment_id)
    return False

  # 1.step check device resource
  if 'occupied_devices' not in server_records:
    server_records['occupied_devices'] = []

  free_devices = [n for n in server_records['devices'] if n not in server_records['occupied_devices']]
  if len(free_devices) == 0:
    logger.error('have no free device resource')
    return False

  start_time = time.time()
  main_param = {}
  if content['hyperparameter'] is not None and content['hyperparameter'] != '':
    main_param = json.loads(content['hyperparameter'])

  structure = content['structure']
  structure_connection = content['structure_connection']
  max_runtime = content['max_time']

  # task token
  experiment_records[experiment_id] = {'start_time': start_time,
                                       'study_name': study_name,
                                       'trail_name': trail_name,
                                       'automl': {'graph': structure},
                                       'main_config': main_param,
                                       'max_time': max_runtime,
                                       'main_file': '',
                                       'main_param': '',
                                       'task': '',
                                       'token': '',
                                       'status': 'prepare',
                                       'devices': [],
                                       'evaluation_value': [],
                                       'evaluation_time': [],
                                       'pid': None}

  # apply devices
  apply_devices = 1
  if 'num_clones' in main_param:
    apply_devices = int(main_param['num_clones'])
  if apply_devices == 0:
    apply_devices = 1

  if apply_devices > len(free_devices):
    logger.error('have no free device resource')

    # remove experiment record
    experiment_records.pop(experiment_id)
    return False

  # 2.step prepare running environment
  # prepare workspace
  if not os.path.exists(os.path.join(server_records['main_folder'], experiment_id)):
    os.makedirs(os.path.join(server_records['main_folder'], experiment_id))

  # prepare support files
  if os.path.exists(os.path.join(server_records['root_main_folder'], 'trainer.tar.gz')):
    shutil.copy(os.path.join(server_records['root_main_folder'], 'trainer.tar.gz'),
                os.path.join(server_records['main_folder'], experiment_id))

    untar_shell = 'openssl enc -d -aes256 -in %s.tar.gz -k %s | tar xz -C %s' % ('trainer', server_records['signature'], '.')
    subprocess.call(untar_shell, shell=True, cwd=os.path.join(server_records['main_folder'], experiment_id))
    os.remove(os.path.join(server_records['main_folder'], experiment_id, 'trainer.tar.gz'))

  # prepare main param
  if server_records['main_param'] is not None and server_records['main_param'] != '':
    with open(os.path.join(server_records['root_main_folder'], server_records['main_param']), 'r') as fp:
      # load basic parameter
      main_param.update(yaml.load(fp))

  experiment_devices = free_devices[0:apply_devices]
  server_records['occupied_devices'].extend(experiment_devices)
  experiment_records[experiment_id]['devices'] = experiment_devices
  main_param.update({'devices': experiment_devices})
  main_param.update({'automl': {'graph': structure, 'graph_connection': structure_connection}})

  with open(os.path.join(server_records['main_folder'], experiment_id, 'main_param.yaml'), 'w') as fp:
    fp.write(yaml.dump(main_param))

  experiment_records[experiment_id]['main_param'] = os.path.join(server_records['main_folder'],
                                                                 experiment_id,
                                                                 '%s.yaml' % experiment_id)

  # prepare main file
  main_file = 'main_file.py'
  shutil.copy(os.path.join(server_records['root_main_folder'], server_records['main_file']),
              os.path.join(server_records['main_folder'], experiment_id, 'main_file.py'))

  experiment_records[experiment_id]['main_file'] = os.path.join(server_records['main_folder'],
                                                                experiment_id,
                                                                'main_file.py')

  # run script
  cmd_shell = 'antgo train --main_file=%s --main_param=%s' % (main_file, 'main_param.yaml')
  cmd_shell += ' --main_folder=%s' % os.path.join(server_records['main_folder'], experiment_id)
  cmd_shell += ' --dump=%s' % os.path.join(server_records['main_folder'], experiment_id, 'dump')
  cmd_shell += ' --max_time=%s' % max_runtime
  cmd_shell += ' --signature=%s' % server_records['signature']
  cmd_shell += ' --proxy=%s' % ('127.0.0.1:%d' % server_records['server_port'])
  cmd_shell += ' --name=%s' % experiment_id

  # prepare task xml file
  if server_records['token'] is None:
    shutil.copy(server_records['task'], os.path.join(server_records['main_folder'], experiment_id, 'task.template'))
    cmd_shell += ' --task=task.template'
  else:
    cmd_shell += ' --token=%s' % server_records['token']

  # start running
  p = subprocess.Popen('%s > %s.log' % (cmd_shell, experiment_id), shell=True, cwd=os.path.join(server_records['main_folder'], experiment_id))
  experiment_records[experiment_id]['pid'] = p
  experiment_records[experiment_id]['status'] = 'running'
  return True

def request_suggestion_process(experiment_records, server_records):
  # based free devices request suggestion
  if server_records['master_ip'] is not None:
    # 1.step check device resource
    if 'occupied_devices' not in server_records:
      server_records['occupied_devices'] = []

    study_name = None
    if 'study_name' in server_records:
      study_name = server_records['study_name']

    if study_name is None:
      return

    # 2.step check completed experiment
    new_experiments = {}
    for experiment_id, experiment_record in experiment_records.items():
      if experiment_record['status'] != 'stop':
        train_p = experiment_record['pid']

        if train_p.poll() is not None:
          # experiment process has exit
          # release occupied devices
          free_devices = experiment_record['devices']
          server_records['occupied_devices'] = \
            [n for n in server_records['occupied_devices'] if n not in free_devices]

          # modify experiment status
          experiment_record['status'] = 'stop'

          # launch new trail
          trail_name = experiment_record['trail_name']
          objective_value = experiment_record['evaluation_value'][-1] if len(experiment_record['evaluation_value']) > 0 else -1.0
          objective_value = float(objective_value)
          logger.info('experiment id %s stop evaluation value %f' % (experiment_id, objective_value))

          response = requests.post('http://%s/server/' % server_records['master_ip'],
                                   data={'study_name': study_name,
                                         'trail_name': trail_name,
                                         'objective_value': objective_value,
                                         'signature': server_records['signature']})
          suggestion = json.loads(response.content)
          if len(suggestion) == 0:
            continue

          if suggestion['status'] == 'running':
            temp = {}
            result = launch_train_process(server_records, temp, suggestion)
            if result:
              # add new experiment
              new_experiments.update(temp)
          elif suggestion['status'] == 'completed':
            logger.info('training server is notified to stop by center')
            exit(0)

    experiment_records.update(new_experiments)

    # 3.step cold start
    free_devices = [n for n in server_records['devices'] if n not in server_records['occupied_devices']]
    if len(free_devices) == 0:
      return

    if server_records['study_name'] is None:
      return

    response = requests.post('http://%s/server/' % server_records['master_ip'],
                             data={'study_name': study_name,
                                   'trail_name': None,
                                   'objective_value': None,
                                   'signature': server_records['signature']})
    suggestion = json.loads(response.content)
    if len(suggestion) == 0:
      return

    if suggestion['status'] == 'running':
      launch_train_process(server_records, experiment_records, suggestion)
    elif suggestion['status'] == 'completed':
      logger.info('training server is notified to stop by center')
      exit(0)

class UpdateModelHandler(BaseHandler):
  @gen.coroutine
  def get(self, experiment_id):
    # check signature
    signature = self.get_argument('signature', '')
    if self.signature != signature:
      logger.error('signature not consistent %s'%signature)
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSignature'}))
      self.finish()
      return

    if experiment_id not in self.experiment_records:
      logger.error('no experiemnt %s here'%experiment_id)
      self.set_status(404)
      self.write(json.dumps({'code': 'InvalidInput', 'message': 'dont have experiment %s'%experiment_id}))
      self.finish()
      return

    address = self.experiment_records[experiment_id]['address'] if 'address' in self.experiment_records[experiment_id] else ''
    status = self.experiment_records[experiment_id]['status']

    self.write(json.dumps({'code': 'Success',
                           'address': address,
                           'status': status}))

    self.finish()

  @gen.coroutine
  def post(self, experiment_id):
    # check signature
    signature = self.get_argument('signature', '')
    if self.signature != signature:
      logger.error('signature not consistent %s'%signature)
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSignature'}))
      self.finish()
      return

    if experiment_id not in self.experiment_records:
      logger.error('no experiemnt %s here'%experiment_id)
      self.set_status(404)
      self.write(json.dumps({'code': 'InvalidInput', 'message': 'dont have experiment %s'%experiment_id}))
      self.finish()
      return

    self.experiment_records[experiment_id]['status'] = 'running'

    # update model evaluate value
    evaluation_val = self.get_argument('evaluation_value', None)
    if evaluation_val is not None:
      self.experiment_records[experiment_id]['evaluation_value'].append(evaluation_val)
      self.experiment_records[experiment_id]['evaluation_time'].append(time.time())

    address = self.get_argument('address', None)
    if address is not None:
      self.experiment_records[experiment_id]['address'] = address

    # do other things
    status = self.get_argument('status', None)
    if status is not None:
      self.experiment_records[experiment_id]['status'] = status

      if status == 'stop':
        free_devices = self.experiment_records[experiment_id]['devices']
        self.server_records['occupied_devices'] = [n for n in self.server_records['occupied_devices'] if n not in free_devices]

class IndexHanlder(BaseHandler):
  @gen.coroutine
  def get(self):
    if self.is_worker:
      self.write('hello worker')
      self.finish()
    else:
      self.client_socket.send_json({'cmd': 'study/all'})
      response = yield self.client_socket.recv_json()

      if len(response) == 0:
        self.set_status(500)
        self.finish()
        return

      studys = response['result']

      study_infos = []
      for s_i, s in enumerate(studys):
        study_name, study_created_time, study_id, study_status = s
        self.client_socket.send_json({'cmd': 'study/get',
                                      'study_name': study_name})
        study = yield self.client_socket.recv_json()
        if len(study) == 0:
          self.set_status(500)
          self.finish(500)
          return

        if study['status'] != 'ok':
          self.set_status(500)
          self.finish(500)
          return

        trials = study['result']

        study_info = {}
        # get completed_trial list
        study_info['completed_trial'] = len(list(filter(lambda x: x[2] == 'Completed', trials)))

        # get error_trial list
        study_info['error_trial'] = len(list(filter(lambda x: x[2] == 'Failed', trials)))

        # get uncompleted_trial list
        study_info['uncompleted_trial'] = len(trials) - study_info['completed_trial'] - study_info['error_trial']

        study_info['name'] = study_name
        study_info['index'] = s_i
        objective_values = [t[3] for t in trials if t[2] == 'Completed']
        study_info['objective_value'] = '%0.4f'%float(np.max(objective_values)) if len(objective_values) > 0 else -1.0
        study_info['status'] = study_status
        study_info['created_time'] = '-' if study_created_time is None else datetime.fromtimestamp(study_created_time).strftime('%Y-%m-%d')

        study_infos.append(study_info)

      self.client_socket.send_json({'cmd': 'searchspace/all'})
      response = yield self.client_socket.recv_json()
      searchspace_names = response['result']

      self.client_socket.send_json({'cmd': 'hyperparameter/all'})
      response = yield self.client_socket.recv_json()
      hyperparameter_names = response['result']

      self.render(self.html_template, automl={'study': study_infos,
                                              'searchspace': searchspace_names,
                                              'hyperparameter': hyperparameter_names})

class StudyStartorStopHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    study_name = self.get_argument('study_name', '')

    self.client_socket.send_json({'cmd': 'study/startorstop',
                                  'study_name': study_name})

    response = yield self.client_socket.recv_json()
    if response['status'] == 'ok':
      self.write(json.dumps({'status': 'ok', 'study_status': response['study_status']}))
      self.finish()
    else:
      self.write(json.dumps({'status': 'fail'}))
      self.finish()

class StudyGetHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    study_name = self.get_argument('study_name', '')

    self.client_socket.send_json({'cmd': 'study/get',
                                  'study_name': study_name})

    response = yield self.client_socket.recv_json()
    trials = response['result']
    filted_trails = [t for t in trials if t[2] == 'Completed']

    trials_list = [{'trial_name': t[0],
                    'trial_created_time': datetime.fromtimestamp(t[1]).strftime('%Y-%m-%d %H:%M:%S'),
                    'trial_status': t[2],
                    'trial_objective_value': '%0.4f'%float(t[3])} for t in filted_trails[0:20]]
    self.write(json.dumps(trials_list))
    self.finish()

class StudyAddHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    study_name = self.get_argument('study_name', '')
    study_goal = self.get_argument('study_goal', 'MAXIMIZE')
    study_max_trials = int(self.get_argument('study_max_trials', '10'))
    study_max_time = int(self.get_argument('study_max_time', '10'))
    study_hyperparameter_search = self.get_argument('study_hyperparameter_search', '')
    study_hyperparameters = self.get_argument('study_hyperparameters', '')
    study_architecture_search = self.get_argument('study_architecture_search', '')
    # study_default_architecture = self.get_argument('study_default_architecture', '')
    study_architecture_search_params = self.get_argument('study_architecture_search_params', '')

    if len(study_name) == 0:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidInput',
                             'message': 'study name isnt empty'}))
      self.finish()
      return

    if len(study_hyperparameters) > 0 and len(study_hyperparameter_search) == 0:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidInput',
                             'message': 'must set hyperparameters search algorithm'}))
      self.finish()
      return

    if len(study_architecture_search_params) > 0 and len(study_architecture_search) == 0:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidInput',
                             'message': 'must set architecture search space'}))
      self.finish()
      return

    if len(study_hyperparameter_search) == 0 and len(study_architecture_search) == 0:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidInput',
                             'message': 'must set study_hyperparameter_search or study_architecture_search'}))
      self.finish()
      return

    if len(study_architecture_search_params) > 0:
      study_architecture_search_params = json.loads(study_architecture_search_params)
      if 'graph' in study_architecture_search_params:
        if study_architecture_search_params['graph'] == '':
          self.set_status(404)
          self.write(json.dumps({'code': 'InvaildUploadFile'}))
          self.finish()
          return

        upload_file_path = os.path.join(self.main_folder, 'static', 'upload', study_architecture_search_params['graph'])
        if (not os.path.isfile(upload_file_path)) or (not os.path.exists(upload_file_path)):
          self.set_status(404)
          self.write(json.dumps({'code': 'InvaildUploadFile'}))
          self.finish()
          return

        study_architecture_search_params['graph'] = upload_file_path

      if 'flops' in study_architecture_search_params:
        if study_architecture_search_params['flops'] == '':
          self.set_status(400)
          self.write(json.dumps({'code': 'InvaildInput', 'message': 'invalid flops'}))
          self.finish()
          return

        study_architecture_search_params['flops'] = float(study_architecture_search_params['flops'])

    if len(study_hyperparameters) > 0:
      study_hyperparameters = json.loads(study_hyperparameters)
      for hp in study_hyperparameters:
        if hp['type'] not in ['DOUBLE', 'INTEGER', 'DISCRETE', 'CATEGORICAL']:
          self.set_status(400)
          self.write(json.dumps({'code': 'InvaildInput', 'message': 'invalid hyperparameter'}))
          self.finish()
          return
        if hp['scalingType'] not in ['LINEAR']:
          self.set_status(400)
          self.write(json.dumps({'code': 'InvaildInput', 'message': 'invalid hyperparameter'}))
          self.finish()
          return

        if hp['type'] in ['DOUBLE', 'INTEGER']:
          try:
            min_val, max_val = hp['params'].split(',')
            if hp['type'] == 'DOUBLE':
              min_val = float(min_val)
              max_val = float(max_val)
              assert(min_val <= max_val)

            if hp['type'] == 'INTEGER':
              min_val = int(min_val)
              max_val = int(min_val)
              assert(min_val <= max_val)

            hp.pop('params')
            hp['minValue'] = min_val
            hp['maxValue'] = max_val
          except:
            self.set_status(400)
            self.write(json.dumps({'code': 'InvaildInput', 'message': 'invalid hyperparameter'}))
            self.finish()
            return

        if hp['type'] in ['DISCRETE', 'CATEGORICAL']:
          feasiblePoints = hp['params'].split(',')
          hp.pop('params')
          hp['feasiblePoints'] = feasiblePoints

    else:
      study_hyperparameters = []

    study_max_time = '%dd'%int(study_max_time)
    self.client_socket.send_json({'cmd': 'study/add',
                                  'study_name': study_name,
                                  'study_goal': study_goal,
                                  'study_max_trials': int(study_max_trials),
                                  'study_max_time': study_max_time,
                                  'study_hyperparameter_search': study_hyperparameter_search,
                                  'study_hyperparameters': study_hyperparameters,
                                  'study_architecture_search': study_architecture_search,
                                  'study_architecture_parameters': study_architecture_search_params,})

    response = yield self.client_socket.recv_json()

    if len(response) == 0 or response['status'] != 'ok':
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidServer'}))
      self.finish()
      return

    self.finish()

class StudyVisHandler(BaseHandler):
  @gen.coroutine
  def get(self, study_name):
    self.client_socket.send_json({'cmd': 'study/visualization',
                                  'study_name': study_name,
                                  'dump_dir': self.image_folder})

    response = yield self.client_socket.recv_json()
    if len(response) == 0 or response['status'] != 'ok':
      self.set_status(404)
      self.write(json.dumps({'code': 'InvalidServer'}))
      self.finish()
      return

    study_image_url = response['result']
    self.write(json.dumps({'url': study_image_url}))
    self.finish()

class SearchspaceGetHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    searchspace_name = self.get_argument('searchspace', '')
    if searchspace_name == '':
      self.write(json.dumps({}))
      self.finish()
      return

    self.client_socket.send_json({'cmd': 'searchspace/get',
                                  'searchspace': searchspace_name})

    response = yield self.client_socket.recv_json()
    if len(response) == 0:
      self.set_status(500)
      self.finish()
      return

    if response['status'] == 'fail':
      self.set_status(404)
      self.finish()
      return

    searchspace_params = response['result']
    self.write(json.dumps(searchspace_params))
    self.finish()

class StudyDeleteHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    study_name = self.get_argument('study_name', '')

    self.client_socket.send_json({'cmd': 'study/delete',
                                  'study_name': study_name})

    response = yield self.client_socket.recv_json()
    if len(response) == 0 or response['status'] != 'ok':
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidServer'}))
      self.finish()
      return

    self.finish()

class TrialInfoHanlder(BaseHandler):
  @gen.coroutine
  def post(self, trial_name):
    self.finish()

class TrialDownloadConfigureHandler(BaseHandler):
  @gen.coroutine
  def get(self, study_name, trial_name):
    self.client_socket.send_json({'cmd': 'trial/get',
                                  'study_name': study_name,
                                  'trial_name': trial_name})

    response = yield self.client_socket.recv_json()
    if len(response) == 0 or response['status'] == 'fail':
      self.set_status(404)
      self.finish()
      return

    _, created_time, status, objective_value, structure, parameter_value, address = response['result']

    self.set_header('Content-Type', 'application/octet-stream')
    self.set_header('Content-Disposition', 'attachment; filename='+trial_name+'.json')

    trial_configure = {}
    if structure is not None:
      trial_configure['graph'] = structure

    if parameter_value is not None:
      trial_configure['parameters'] = parameter_value

    trial_configure_str = json.dumps(trial_configure)

    self.write(trial_configure_str)
    # stop
    self.finish()

class StudyDownloadExperimentHandler(BaseHandler):
  @gen.coroutine
  def get(self, study_name):
    self.client_socket.send_json({'cmd': 'study/download',
                                  'study_name': study_name})

    response = yield self.client_socket.recv_json()
    if len(response) == 0 or response['status'] == 'fail':
      self.set_status(404)
      self.finish()
      return

    self.set_header('Content-Type', 'application/octet-stream')
    self.set_header('Content-Disposition', 'attachment; filename='+study_name+'.json')

    self.write(response['result'])
    self.finish()

class TrialDownloadExperimentHandler(BaseHandler):
  @gen.coroutine
  def get(self, study_name, trial_name):
    pass

class SuggestionServerHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    if self.is_worker:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidServer', 'message': 'not server server'}))
      self.finish()
      return

    signature = self.get_argument('signature', '')
    if self.signature != signature:
      logger.error('signature not consistent %s'%signature)
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSignature'}))
      self.finish()
      return

    study_name = self.get_argument('study_name', '')
    trail_name = self.get_argument('trail_name', None)
    objective_value = self.get_argument('objective_value', -1.0)
    created_time = self.get_argument('created_time', None)
    updated_time = self.get_argument('updated_time', None)

    self.client_socket.send_json({'cmd': 'suggestion/make',
                                  'study_name': study_name,
                                  'trail_name': trail_name,
                                  'objective_value': objective_value,
                                  'created_time': created_time,
                                  'updated_time': updated_time,})
    server_response = yield self.client_socket.recv_json()
    self.write(json.dumps(server_response))

class UpdateExperimentServerHandler(BaseHandler):
  @gen.coroutine
  def post(self):
    if self.is_worker:
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidServer', 'message': 'not server server'}))
      self.finish()
      return

    signature = self.get_argument('signature', '')
    if self.signature != signature:
      logger.error('signature not consistent %s'%signature)
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSignature'}))
      self.finish()
      return

    running_experiments_str = self.get_argument('experiments', '')
    running_experiments = json.loads(running_experiments_str)

    self.client_socket.send_json({'cmd': 'suggestion/update',
                                  'experiments': running_experiments})
    yield self.client_socket.recv_json()

    self.finish()

class FileHanlder(BaseHandler):
  @gen.coroutine
  def post(self):
    file_metas = self.request.files.get('file', None)
    if not file_metas:
      self.set_status(400)
      self.write(json.dumps({'code': 'InvalidUploadFile', 'message': 'The input file is not uploaded correctly'}))
      self.finish()
      return

    upload_file_path = os.path.join(self.main_folder, 'static', 'upload')
    if not os.path.exists(upload_file_path):
      os.makedirs(upload_file_path)

    _file_name = ''
    _file_path = ''
    for meta in file_metas:
      _file_name = '%s-%s-%s'%(str(uuid.uuid4()),
                               datetime.fromtimestamp(timestamp()).strftime('%Y%m%d-%H%M%S-%f'),
                               meta['filename'])
      _file_path = os.path.join(upload_file_path, _file_name)

      with open(_file_path, 'wb') as fp:
        fp.write(meta['body'])

      break

    self.file_records[_file_name] = _file_path
    self.write(json.dumps({'file': _file_name}))

class TrainHanlder(BaseHandler):
  @gen.coroutine
  def post(self):
    # 1.step check call permission
    # 1.1.step check signature
    signature = self.get_argument('signature', '')
    if self.signature != signature:
      logger.error('signature not consistent %s'%signature)
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSignature'}))
      self.finish()
      return

    # 1.2.step check device resource
    if 'occupied_devices' not in self.server_records:
      self.server_records['occupied_devices'] = []

    free_devices = [n for n in self.device_list if n not in self.server_records['occupied_devices']]
    if len(free_devices) == 0:
      logger.error('have no free device resource')
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSupport', 'message': 'not enough devices'}))
      self.finish()
      return

    # record
    try_config = self.get_argument('AUTOML', {})
    experiment_id = '%s-%s'%(str(uuid.uuid4()), datetime.fromtimestamp(timestamp()).strftime('%Y%m%d-%H%M%S-%f'))
    start_time = time.time()
    main_param = self.get_argument('MAINPARAM', {})
    max_runtime = self.get_argument('MAX_RUNTIME', None)
    if max_runtime is None:
      max_runtime = self.max_time

    # task token
    token = self.get_argument('TOKEN', None)
    self.experiment_records[experiment_id] = {'start_time': start_time,
                                              'try_config': try_config,
                                              'main_config': main_param,
                                              'max_time': max_runtime,
                                              'main_file': '',
                                              'main_param': '',
                                              'task': '',
                                              'status': 'prepare',
                                              'token': token,
                                              'devices': [],
                                              'evaluation_value': [],
                                              'evaluation_time': [],
                                              'pid': None}

    # 2.step prepare running environment
    # prepare workspace
    os.makedirs(os.path.join(self.main_folder, experiment_id))

    # prepare main param
    if self.main_param is not None and self.main_param != '':
      with open(os.path.join(self.main_folder, self.main_param), 'r') as fp:
        # load basic parameter
        main_param.update(yaml.load(fp))

    # update automl config
    main_param.update({'automl': try_config})

    # apply devices
    apply_devices = int(self.get_argument('APPLY_DEVICES', 1))
    if apply_devices == 0:
      apply_devices = 1

    if 'num_clones' in main_param:
      apply_devices = int(main_param['num_clones'])

    if apply_devices > len(free_devices):
      logger.error('have no free device resource')
      self.set_status(500)
      self.write(json.dumps({'code': 'InvalidSupport', 'message': 'not enough devices'}))
      self.finish()

      # remove experiment record
      self.experiment_records.pop(experiment_id)
      return

    experiment_devices = free_devices[0:apply_devices]
    self.server_records['occupied_devices'].extend(experiment_devices)
    self.experiment_records[experiment_id]['device'] = experiment_devices
    main_param.update({'devices': experiment_devices})

    with open(os.path.join(self.main_folder, experiment_id, '%s.yaml'%experiment_id), 'w') as fp:
      fp.write(yaml.dump(main_param))

    main_param = '%s.yaml'%experiment_id

    self.experiment_records[experiment_id]['main_param'] = os.path.join(self.main_folder, experiment_id, '%s.yaml'%experiment_id)

    # prepare main file
    main_file = 'main_file.py'
    if self.main_file is None or self.main_file == '':
      file_id = self.get_argument('MAINFILE', None)
      if file_id is None or file_id not in self.file_records:
        self.set_status(400)
        self.write(json.dumps({'code': 'InvalidUploadFile','message': 'The input file is not uploaded correctly'}))
        self.finish()

        # remove experiment record
        self.experiment_records.pop(experiment_id)
        return

      shutil.copy(self.file_records[file_id], os.path.join(self.main_folder, experiment_id, 'main_file.py'))
    else:
      shutil.copy(os.path.join(self.main_folder, self.main_file), os.path.join(self.main_folder, experiment_id, 'main_file.py'))

    self.experiment_records[experiment_id]['main_file'] = os.path.join(self.main_folder, experiment_id, 'main_file.py')

    # run script
    cmd_shell = 'antgo train --main_file=%s --main_param=%s'%(main_file, main_param)
    cmd_shell += ' --main_folder=%s'%os.path.join(self.main_folder, experiment_id)
    cmd_shell += ' --dump=%s'%os.path.join(self.main_folder, experiment_id, 'dump')
    cmd_shell += ' --max_time=%s'%max_runtime
    cmd_shell += ' --signature=%s'%self.signature
    cmd_shell += ' --proxy=%s'%('127.0.0.1:%d'%self.server_port)

    # prepare task xml file
    if token is None:
      file_id = self.get_argument('TASKXML', None)
      if file_id is None:
        self.set_status(400)
        self.write(json.dumps({'code': 'InvalidUploadFile','message': 'The task xml is not uploaded correctly'}))
        self.finish()

        # remove experiment record
        self.experiment_records.pop(experiment_id)
        return

      shutil.copy(self.file_records[file_id], os.path.join(self.main_folder, experiment_id, 'task.template'))
      self.experiment_records[experiment_id]['task'] = os.path.join(self.main_folder, experiment_id, 'task.template')
      cmd_shell += ' --task=task.template'
    else:
      cmd_shell += ' --token=%s'%token

    # start running
    p = subprocess.Popen('nohup %s > %s.log 2>&1 &'%(cmd_shell, experiment_id), shell=True)
    self.experiment_records[experiment_id]['pid'] = p.pid

    self.finish()

def train_server_start(main_file,
                       main_param,
                       main_folder,
                       token,
                       task,
                       devices,
                       max_time,
                       is_worker,
                       signature,
                       server_port,
                       master_ip,
                       parent_id):
  try:
    define('port', default=server_port, help='run on port')

    if is_worker:
      # tar all support files
      tar_shell = 'tar -czf - --exclude=static --exclude=template  --exclude=dump --exclude=work --exclude=trainer.tar.gz * | openssl enc -e -aes256 -out %s.tar.gz -k %s' % ('trainer', signature)
      subprocess.call(tar_shell, shell=True, cwd=main_folder)

      if not os.path.exists(os.path.join(main_folder, 'work')):
        os.makedirs(os.path.join(main_folder, 'work'))

    # records db
    experiment_records = {}
    study_name = None
    if token is not None or task is not None:
      study_name = token if token is not None else task.split('/')[-1]

    server_records = {'signature': signature,
                      'study_name': study_name,
                      'root_main_folder': main_folder,
                      'main_folder': os.path.join(main_folder, 'work'),
                      'main_param': main_param,
                      'main_file': main_file,
                      'devices': devices,
                      'master_ip': master_ip,
                      'task': task,
                      'token': token,
                      'server_port': server_port}
    file_records = {}

    client_socket = None
    if not is_worker:
      zmq_ctx = future.Context.instance()
      client_socket = zmq_ctx.socket(zmq.REQ)
      client_socket.bind('ipc://%s'%str(parent_id))

    if not os.path.exists(os.path.join(main_folder, 'template')):
      os.makedirs(os.path.join(main_folder, 'template'))

    train_server_template_dir = os.path.join(main_folder, 'template')

    if not os.path.exists(os.path.join(main_folder, 'static')):
      os.makedirs(os.path.join(main_folder, 'static'))

    train_server_static_dir = os.path.join(main_folder, 'static')

    static_folder = '/'.join(os.path.dirname(__file__).split('/')[0:-1])
    for static_file in os.listdir(os.path.join(static_folder, 'resource', 'static')):
      if static_file[0] == '.':
        continue

      shutil.copy(os.path.join(static_folder, 'resource', 'static', static_file),
                  train_server_static_dir)

    html_template = 'trainworker.html' if is_worker else 'trainmaster.html'
    shutil.copy(os.path.join(static_folder, 'resource', 'templates', html_template),
                os.path.join(train_server_template_dir, html_template))

    settings = {'main_file': main_file,
                'main_param': main_param,
                'main_folder': main_folder,
                'device_list': devices,
                'max_time': max_time,
                'signature': signature,
                'experiment_records': experiment_records,
                'server_records': server_records,
                'file_records': file_records,
                'server_port': server_port,
                'is_worker': is_worker,
                'client_socket': client_socket,
                'template_path': train_server_template_dir,
                'static_path': train_server_static_dir,
                'html_template': html_template,
                }

    if is_worker and master_ip is not None and master_ip != '':
      app = tornado.web.Application(handlers=[('/', IndexHanlder),
                                              ('/train/', TrainHanlder),
                                              ('/update/model/([^/]+)/', UpdateModelHandler),
                                              ('/submit/', FileHanlder),],
                                    **settings)

      http_server = tornado.httpserver.HTTPServer(app)
      http_server.listen(options.port)
      tornado.ioloop.PeriodicCallback(functools.partial(request_suggestion_process,
                                                        experiment_records=experiment_records,
                                                        server_records=server_records,
                                                        ), 10000).start()
      tornado.ioloop.PeriodicCallback(functools.partial(update_suggestion_process,
                                                        experiment_records=experiment_records,
                                                        server_records=server_records,
                                                        ), 10*60*1000).start()
    else:
      app = tornado.web.Application(handlers=[('/', IndexHanlder),
                                              ('/server/', SuggestionServerHandler),
                                              ('/update/', UpdateExperimentServerHandler),
                                              ('/study/startorstop/', StudyStartorStopHandler),
                                              ('/study/delete/', StudyDeleteHandler),
                                              ('/study/get/', StudyGetHandler),
                                              ('/study/add/', StudyAddHandler),
                                              ('/study/([^/]+)/vis/', StudyVisHandler),
                                              ('/searchspace/get/', SearchspaceGetHandler),
                                              ('/trial/([^/]+)/', TrialInfoHanlder),
                                              ('/trial/download/([^/]+)/([^/]+)/configure/', TrialDownloadConfigureHandler),
                                              ('/trial/download/([^/]+)/([^/]+)/experiment/', TrialDownloadExperimentHandler),
                                              ('/study/download/([^/]+)/experiment/', StudyDownloadExperimentHandler),
                                              ('/submit/', FileHanlder),],
                                    **settings)
      http_server = tornado.httpserver.HTTPServer(app)
      http_server.listen(options.port)

    logger.info('train server is launch on port %d'%server_port)
    tornado.ioloop.IOLoop.instance().start()

  except:
    traceback.print_exc()
    raise sys.exc_info()[0]


"""

Martin Wood - 14/07/2019

For convenience; various functions that the experiment notebooks would be cleaner without
"""

import re
import os
import json

import pandas as pd

from datetime import datetime as dt

def clean_text(article_text, brutal=False):
    """ Utility function for cleaning up text for me.  There's probably better ways to prepare data. """
    article_text = re.sub(r'<b>|</b>|[&#39]', '', article_text)     # Remove annoying tags
    article_text = re.sub(r'\[[0-9]*\]', ' ', article_text)         # Gets rid of numbers
    article_text = re.sub(r'\s+', ' ', article_text)                # Replaces all forms of white space with single space
    if brutal:                                                      # Optional, all non alpha-numeric characters removed
        article_text = re.sub('r[^0-9A-Za-z ]', "", article_text)
    return(article_text)

def corpus_loader(directory, corpus_tag, drop_raw=True):
    """ For loading my corpus files """

    # Get a list of all corpus files
    files = [x for x in os.listdir(directory) if x.endswith(".json") and ("corpus" in x) and (corpus_tag in x)]

    # Implement filters here if I ever feel I need them
    # Filter by dates if needed
    #datetimestamps = [dt.strptime(":".join(re.findall(r"[0-9-]{1,}", x)), "%Y-%m-%d:%H%M")  for x in files]

    # Iterate through and load up every file in sequence
    compendium = []

    print("Total files: {}".format(len(files)))
    for filename in files:
        print("Loading file: {}".format(filename))
        with open(directory + "/" + filename, "r") as f:
            articles = json.load(f)
            for article in articles:

                # Optional, don't bother loading up the original raw response (saves memory)
                if drop_raw:
                    article.pop("raw")
                compendium.append(article)

    return pd.DataFrame(compendium)

def load_clean_corpus(directory, corpus_tag, drop_raw=True, brutal=False):
    """ All common pre-processing. """
    corpus = corpus_loader(directory, corpus_tag, drop_raw=drop_raw)

    # Filter to only the .uk vendors
    corpus = corpus[corpus['link'].str.contains(".uk/")]

    # Drop duplicates based on actual text
    corpus = corpus.drop_duplicates("summary")

    # Clean whatever's survived
    corpus['clean_text'] = corpus[['title', 'summary']].apply(lambda x: clean_text('.  '.join(x)), axis=1)

    return corpus


from flask import Flask, jsonify, request
from flask_restful import Resource, abort
from flask_pymongo import pymongo
from bson.json_util import ObjectId
import db_config as database

class Posts(Resource):
    """ handeling post behavior """

    def get(self, _id):
        response = self.abort_if_not_exist(_id)
        response['_id'] = str(response['_id'])
        return jsonify(response)

    def post(self, _id):
        response = self.abort_if_not_exist(_id)
        database.db.Badges.update_one({"_id": ObjectId(_id)}, {"$push":{
            "posts":{
                "id": request.json["id"],
                "name": request.json["name"],
                "img": request.json["img"],
                "date": request.json["date"]
            }
        }})

        return jsonify({"message": f"The post {request.json['id']} was successfully created"})

    def put(self, _id, uuid):
        response = self.abort_if_not_exist(_id)
        database.db.Badges.update_one({"_id": ObjectId(_id), "posts.id": uuid},
        {"$set":{
            "posts.$.name": request.json["name"],
            "posts.$.img": request.json["img"],
            "posts.$.date": request.json["date"],
        }})

        return jsonify(request.json)

    def delete(self, _id, uuid):
        response = self.abort_if_not_exist(_id)
        database.db.Badges.update_one({"_id": ObjectId(_id)},
        {"$pull":{
            "posts":{"id": uuid}
        }})

        return jsonify({"message": f"The post with uuid={uuid} was successfully deleted"})

    def abort_if_not_exist(self, _id):
        response = database.db.Badges.find_one({'_id':ObjectId(_id)}, {"name": 1, "posts": 1})

        if response:
            return response
        else:
            abort(jsonify({"status": 404, "_id": f"{_id} not found"}))


##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name:        Raytheon Company
# Contractor Address:     6825 Pine Street, Suite 340
#                         Mail Stop B8
#                         Omaha, NE 68106
#                         402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##

from __future__ import print_function
from shapely.geometry import box, Point
from awips.dataaccess import DataAccessLayer as DAL

import baseDafTestCase
import unittest

#
# Test DAF support for grid data
#
#     SOFTWARE HISTORY
#
#    Date            Ticket#       Engineer       Description
#    ------------    ----------    -----------    --------------------------
#    01/19/16        4795          mapeters       Initial Creation.
#    04/11/16        5548          tgurney        Cleanup
#    04/18/16        5548          tgurney        More cleanup
#    06/09/16        5587          tgurney        Typo in id values test
#    10/13/16        5942          bsteffen       Test envelopes
#    11/08/16        5985          tgurney        Skip certain tests when no
#                                                 data is available
#

class GridTestCase(baseDafTestCase.DafTestCase):
    """Test DAF support for grid data"""

    datatype = "grid"

    model = "GFS160"

    envelope = box(-97.0, 41.0, -96.0, 42.0)

    def testGetAvailableParameters(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", self.model)
        self.runParametersTest(req)

    def testGetAvailableLocations(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", self.model)
        self.runLocationsTest(req)

    def testGetAvailableLevels(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", self.model)
        self.runLevelsTest(req)

    def testGetAvailableTimes(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", self.model)
        req.setLevels("2FHAG")
        self.runTimesTest(req)

    def testGetGridData(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", self.model)
        req.setLevels("2FHAG")
        req.setParameters("T")
        self.runGridDataTest(req)

    def testGetIdentifierValues(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier("info.datasetId", 'ENSEMBLE')
        req.setLevels("2FHAG")
        req.setParameters("T")
        idValues = DAL.getIdentifierValues(req, 'info.ensembleId')
        self.assertTrue(hasattr(idValues, '__iter__'))
        if idValues:
            self.assertIn('ctl1', idValues)
            self.assertIn('p1', idValues)
            self.assertIn('n1', idValues)
        else:
            raise unittest.SkipTest("no data available")

    def testGetInvalidIdentifierValuesThrowsException(self):
        self.runInvalidIdValuesTest()

    def testGetNonexistentIdentifierValuesThrowsException(self):
        self.runNonexistentIdValuesTest()

    def testGetDataWithEnvelope(self):
        req = DAL.newDataRequest(self.datatype)
        req.addIdentifier('info.datasetId', self.model)
        req.setLevels('2FHAG')
        req.setParameters('T')
        req.setEnvelope(self.envelope)
        gridData = self.runGridDataTest(req)
        if not gridData:
            raise unittest.SkipTest('no data available')
        lons, lats = gridData[0].getLatLonCoords()
        lons = lons.reshape(-1)
        lats = lats.reshape(-1)

        # Ensure all points are within one degree of the original box
        # to allow slight margin of error for reprojection distortion.
        testEnv = box(self.envelope.bounds[0] - 1, self.envelope.bounds[1] - 1,
                      self.envelope.bounds[2] + 1, self.envelope.bounds[3] + 1 )

        for i in range(len(lons)):
            self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))


"""
Given an unsorted integer array, find the smallest missing positive integer.

Example 1:

Input: [1,2,0]
Output: 3

Example 2:

Input: [3,4,-1,1]
Output: 2

Example 3:

Input: [7,8,9,11,12]
Output: 1

Note:

Your algorithm should run in O(n) time and uses constant extra space.
"""
from typing import List

class Solution1:
    def firstMissingPositive(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        size = len(nums)
        for i in range(size):
            while 0 < nums[i] <= size and nums[i] != i + 1 and nums[nums[i] - 1] != nums[i]:
                self.swap(nums, i, nums[i] - 1)
        for i in range(size):
            if nums[i] != i + 1:
                return i + 1
        return size + 1

    @staticmethod
    def swap(nums: List[int], i: int, j: int):
        nums[i], nums[j] = nums[j], nums[i]