from flask import Blueprint, render_template, redirect, url_for, request
from app.main.form import TaskForm
from app import db
from app.model import Task

main = Blueprint('main', __name__, template_folder='templates')

@main.route('/', methods=['GET'])
def index():
  list_of_todos = Task.query.order_by(Task.id).all()
  return render_template('index.html', list_of_todos=list_of_todos)

@main.route('/new', methods=['GET'])
def new():
  form = TaskForm()
  return render_template('new.html', form=form)

@main.route('/show/<int:id>', methods=['GET'])
def show(id):
  todo = Task.query.filter_by(id=id).first()
  return render_template('show.html', todo=todo)

@main.route('/create', methods=['POST'])
def create():
  form = TaskForm()
  if request.method == 'POST' and form.validate_on_submit():
    todo = Task(todo=form.name.data)
    db.session.add(todo)
    db.session.commit()
  return redirect(url_for('main.index'))


######################################################################################################################
#  Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.                                           #
#                                                                                #
#  Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance     #
#  with the License. A copy of the License is located at                                                             #
#                                                                                #
#      http://www.apache.org/licenses/                                                                               #
#                                                                                #
#  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES #
#  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions    #
#  and limitations under the License.                                                                                #
######################################################################################################################
import time

import boto3

import services.dynamodb_service
from helpers.timer import Timer

class DynamoDB(object):

    def __init__(self, region=None, session=None):

        self.region = region if region is not None else boto3.Session().region_name
        self.session = session if session is not None else boto3.Session(region_name=self.region)
        self.ddb_client = self.session.client("dynamodb", region_name=self.region)
        self.ddb_service = services.dynamodb_service.DynamodbService(session=self.session)

    def wait_until_table_backups_available(self, table, timeout_seconds=30 * 60):
        with Timer(timeout_seconds=timeout_seconds, start=True) as t:

            while not t.timeout:
                try:
                    # the only way to find out if backups are available for newly created table is to try to create a backup
                    resp = self.ddb_client.create_backup(TableName=table, BackupName="is-backup-available")
                    arn = resp.get("BackupDetails", {}).get("BackupArn", None)
                    self.ddb_client.delete_backup(BackupArn=arn)
                    return True
                except Exception as ex:
                    if type(ex).__name__ != "ContinuousBackupsUnavailableException":
                        return False

                time.sleep(30)

        return False

    def get_table(self, table_name):
        return self.ddb_service.get(services.dynamodb_service.TABLE,
                                    TableName=table_name,
                                    region=self.region,
                                    tags=True)

    def create_backup(self, table_name, backup_name):
        return self.ddb_client.create_backup(TableName=table_name, BackupName=backup_name).get("BackupDetails")

    def delete_backup(self, backup_arn, exception_if_not_exists=False):
        try:
            self.ddb_client.delete_backup(BackupArn=backup_arn)
        except Exception as e:
            if e.__class__.__name__ == "BackupNotFoundException":
                if exception_if_not_exists:
                    raise e
                return
            raise e

    def delete_table_backups(self, table_name):
        for backup_arn in [s["BackupArn"] for s in self.get_table_backups(table_name) if s["BackupStatus"] == "AVAILABLE"]:
            self.delete_backup(backup_arn=backup_arn)

    def get_table_backups(self, table_name):
        return self.ddb_service.describe(services.dynamodb_service.BACKUPS, TableName=table_name)

    def create_tags(self, table_name, tags):
        arn = "arn:aws:dynamodb:{}:{}:table/{}".format(self.region, services.get_aws_account(), table_name)
        self.ddb_client.tag_resource(ResourceArn=arn, Tags=[{"Key": t, "Value": tags[t]} for t in tags])


# -*- coding:utf-8 -*-
# Author: Jay-Q
# -*- coding: utf-8 -*-
import sys
from PyQt5.QtWidgets import QApplication
import login

app = QApplication(sys.argv)
Login = login.login()
if Login.exec():
    print("登录成功")
else:
    print("登录退出")
sys.exit(app.exec())


"""
OMEGA iTHX Series Temperature and Humidity Chart Recorder.

This class is compatible with the following model numbers:

* iTHX-W3
* iTHX-D3
* iTHX-SD
* iTHX-M
* iTHX-W
* iTHX-2
"""
import os
import re
import time
import socket
import sqlite3
import datetime
try:
    ConnectionResetError
except NameError:
    ConnectionResetError = socket.error  # for Python 2.7

from msl.equipment.exceptions import OmegaError, MSLTimeoutError
from msl.equipment.connection_socket import ConnectionSocket
from msl.equipment.resources import register

@register(manufacturer=r'OMEGA', model=r'iTHX-[2DMSW][3D]*', flags=re.IGNORECASE)
class iTHX(ConnectionSocket):

    def __init__(self, record):
        """OMEGA iTHX Series Temperature and Humidity Chart Recorder.

        Do not instantiate this class directly. Use the :meth:`~.EquipmentRecord.connect`
        method to connect to the equipment.

        Parameters
        ----------
        record : :class:`~.EquipmentRecord`
            A record from an :ref:`equipment-database`.
        """
        super(iTHX, self).__init__(record)
        self.set_exception_class(OmegaError)
        self.IS_W_OR_2 = record.model.upper() in ['ITHX-2', 'ITHX-W']

    def temperature(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature of (for iTHX's that contain multiple probes).
        celsius : class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current temperature.
        """
        msg = 'TC' if celsius else 'TF'
        return self._get(msg, probe, size=nbytes)

    def humidity(self, probe=1, nbytes=None):
        """Read the percent humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the humidity of (for iTHX's that contain multiple probes).
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current percent humidity.
        """
        return self._get('H', probe, size=nbytes)

    def dewpoint(self, probe=1, celsius=True, nbytes=None):
        """Read the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the dew point of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the dew point in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current dew point.
        """
        msg = 'D' if celsius else 'DF'
        return self._get(msg, probe, size=nbytes)

    def temperature_humidity(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature and the humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature and humidity of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then the default value is 13 bytes.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        """
        if nbytes is None:
            # since the returned bytes are of the form b'019.4\r,057.0\r'
            # the _get method would stop reading bytes at the first instance of '\r'
            # therefore, we will specify the number of bytes to read to get both values
            nbytes = 13

        if self.IS_W_OR_2:
            # iTHX-W and iTHX-2 do not support the 'B' command
            n = nbytes//2
            return self.temperature(probe=probe, celsius=celsius, nbytes=n), self.humidity(probe=probe, nbytes=n)

        msg = 'B' if celsius else 'BF'
        return self._get(msg, probe, size=nbytes)

    def temperature_humidity_dewpoint(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature, the humidity and the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature, humidity and dew point (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            If :data:`True` then return the temperature and dew point in celsius, :data:`False` for fahrenheit.
        nbytes : :class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence. For example, if no termination is returned and each value is
            represented by 6 bytes then specify ``nbytes=18`` to get all 3 values.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        :class:`float`
            The current dew point.
        """
        nth = None if nbytes is None else (nbytes*2)//3
        nd = None if nbytes is None else nbytes//3
        t, h = self.temperature_humidity(probe=probe, celsius=celsius, nbytes=nth)
        return t, h, self.dewpoint(probe=probe, celsius=celsius, nbytes=nd)

    def start_logging(self, path, wait=60, nprobes=1, nbytes=None):
        """Start logging the temperature, humidity and dew point to the specified path.

        The information is logged to a SQLite_ database. To stop logging press ``CTRL+C``.

        .. _SQLite: https://www.sqlite.org/index.html

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database. If you only specify a folder then a database
            with the default filename, ``model_serial.sqlite3``, is created/opened in this folder.
        wait : :class:`int`, optional
            The number of seconds to wait between each log.
        nprobes : :class:`int`, optional
            The number of probes that the iServer has (1 or 2).
        nbytes : :class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence. For example, if no termination is used and each value is
            represented by 6 bytes then specify ``nbytes=18`` to get all three values.
        """
        if os.path.isdir(path):
            filename = self.equipment_record.model + '_' + self.equipment_record.serial + '.sqlite3'
            path = os.path.join(path, filename)

        db_timeout = 10.0

        db = sqlite3.connect(path, timeout=db_timeout)
        self.log_info('start logging to {}'.format(path))

        if nprobes == 1:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature FLOAT, '
                'humidity FLOAT, '
                'dewpoint FLOAT);'
            )
        elif nprobes == 2:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature1 FLOAT, '
                'humidity1 FLOAT, '
                'dewpoint1 FLOAT, '
                'temperature2 FLOAT, '
                'humidity2 FLOAT, '
                'dewpoint2 FLOAT);'
            )
        else:
            raise ValueError('The number-of-probes value must be either 1 or 2. Got {}'.format(nprobes))

        db.commit()
        db.close()

        msg = 'Sn:{} - {}:{} => '.format(self.equipment_record.serial, self._host, self._port)

        try:
            while True:
                t0 = time.time()
                results = [datetime.datetime.now()]

                # get the values
                try:
                    data = self.temperature_humidity_dewpoint(probe=1, celsius=True, nbytes=nbytes)
                    if nprobes == 2:
                        data += self.temperature_humidity_dewpoint(probe=2, celsius=True, nbytes=nbytes)
                        self.log_info(msg+'T1={:.1f} H1={:.1f} DP1={:.1f} T2={:.1f} H2={:.1f} DP2={:.1f}'.format(*data))
                    else:
                        self.log_info(msg+'T={:.1f} H={:.1f} DP={:.1f}'.format(*data))
                except MSLTimeoutError:
                    while True:
                        try:
                            self._connect()
                        except MSLTimeoutError:
                            pass
                        else:
                            break
                    continue
                else:
                    results.extend(data)

                # save the values to the database and then wait
                try:
                    db = sqlite3.connect(path, timeout=db_timeout)
                    if nprobes == 1:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?);', results)
                    else:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?, ?, ?, ?);', results)
                    db.commit()
                    db.close()
                except sqlite3.OperationalError as e:  # database is locked, someone is reading data
                    db.close()
                    self.log_error('sqlite3.OperationalError: ' + str(e))
                else:
                    time.sleep(max(0.0, wait - (time.time() - t0)))

        except (KeyboardInterrupt, SystemExit):
            pass

        db.close()
        self.log_info('stopped logging to {}'.format(path))

    @staticmethod
    def data(path, date1=None, date2=None, as_datetime=True, select='*'):
        """Fetch all the log records between two dates.

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database.
        date1 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp > `date1`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        date2 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp < `date2`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        as_datetime : :class:`bool`, optional
            Whether to fetch the timestamps from the database as :class:`datetime.datetime` objects.
            If :data:`False` then the timestamps will be of type :class:`str` and this function
            will return much faster if requesting data over a large date range.
        select : :class:`str` or :class:`list` of :class:`str`, optional
            The column(s) in the database to use with the ``SELECT`` SQL command.

        Returns
        -------
        :class:`list` of :class:`tuple`
            A list of ``(timestamp, temperature, humidity, dewpoint)`` log records,
            depending on the value of `select`.
        """
        if not os.path.isfile(path):
            raise IOError('Cannot find {}'.format(path))

        detect_types = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES if as_datetime else 0
        db = sqlite3.connect(path, timeout=10.0, detect_types=detect_types)
        cursor = db.cursor()

        if select != '*':
            if isinstance(select, (list, tuple, set)):
                select = ','.join(select)
        base = 'SELECT {} FROM data'.format(select)

        if date1 is None and date2 is None:
            cursor.execute(base + ';')
        elif date1 is not None and date2 is None:
            cursor.execute(base + ' WHERE timestamp > ?;', (date1,))
        elif date1 is None and date2 is not None:
            cursor.execute(base + ' WHERE timestamp < ?;', (date2,))
        else:
            cursor.execute(base + ' WHERE timestamp BETWEEN ? AND ?;', (date1, date2))

        data = cursor.fetchall()
        cursor.close()
        db.close()
        return data

    def _get(self, message, probe, size=None):
        if not 1 <= probe <= 3:
            self.raise_exception('Invalid probe number, {}. Must be either 1, 2, or 3'.format(probe))

        command = '*SR' + message
        if probe > 1:
            command += str(probe)

        try:
            ret = self.query(command, size=size)
        except ConnectionResetError:
            # for some reason the socket closes if a certain amount of time passes and no
            # messages have been sent. For example, querying the temperature, humidity and
            # dew point every >60 seconds raised:
            #   [Errno errno.ECONNRESET] An existing connection was forcibly closed by the remote host
            self._connect()  # reconnect
            return self._get(message, probe, size=size)  # retry
        else:
            values = tuple(float(v) for v in re.split(r'[,;]', ret))
            if len(values) == 1:
                return values[0]
            else:
                return values


"""
4.10 序列上索引值迭代
问题
你想在迭代一个序列的同时跟踪正在被处理的元素索引。

解决方案
内置的 enumerate() 函数可以很好的解决这个问题：
"""

my_list = ['a', 'b', 'c']
for idx, val in enumerate(my_list):
    print(idx, val)

# 为了按传统行号输出(行号从1开始)，你可以传递一个开始参数：
for idx, val in enumerate(my_list, 1):
    print(idx, val)

# enumerate() 对于跟踪某些值在列表中出现的位置是很有用的。
# 将一个文件中出现的单词映射到它出现的行号上去，可以很容易的利用 enumerate() 来完成：
from collections import defaultdict
word_summary = defaultdict(list)
with open('./test.txt') as f:
    lines = f.readlines()
for idx, line in enumerate(lines):
    words = [w.strip().lower() for w in line.split()]
    for word in words:
        word_summary[word].append(idx)
print(word_summary)


import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
from sklearn.externals.six import StringIO
from sklearn.cluster import KMeans
import pickle as pk
import pydotplus
import pensieve
import pensiedt
import robustmpc
import robustmdt
import robustlin
import argparse
import load_trace
import fixed_env as env
from multiprocessing.dummy import Pool as ThreadPool
import limes
import pensilin
import fixed_env_hotdash as env_hotdash
import hotdash
import hotdadt
import hotdlin
#from actor_wrapper import Actor_LIME
#NN_MODEL = './models/pretrain_linear_reward.ckpt'

S_INFO = 6
S_INFO_P = 6  # bit_rate, buffer_size, next_chunk_size, bandwidth_measurement(throughput and time), chunk_til_video_end
S_INFO_R = 5  # bit_rate, buffer_size, rebuffering_time, bandwidth_measurement, chunk_til_video_end
S_LEN = 8  # take how many frames in the past
A_DIM = 6
ACTOR_LR_RATE = 0.0001
CRITIC_LR_RATE = 0.001
VIDEO_BIT_RATE = [300,750,1200,1850,2850,4300]  # Kbps
BITRATE_REWARD = [1, 2, 3, 12, 15, 20]
BUFFER_NORM_FACTOR = 10.0
CHUNK_TIL_VIDEO_END_CAP = 48.0
M_IN_K = 1000.0
REBUF_PENALTY = 4.3  # 1 sec rebuffering -> 3 Mbps
SMOOTH_PENALTY = 1
DEFAULT_QUALITY = 1  # default video quality without agent
RANDOM_SEED = 42
RAND_RANGE = 1000
SUMMARY_DIR = './results'
LOG_FILE = './results/log_pensieve'
# log in format of time_stamp bit_rate buffer_size rebuffer_time chunk_size download_time reward
NN_MODEL = './models/pretrain_linear_reward.ckpt'
THREAD = 16

def split_train_test(obss, acts, train_frac):
    n_train = int(train_frac * len(obss))
    idx = np.arange(len(obss))
    np.random.shuffle(idx)
    obss_train = obss[idx[:n_train]]
    acts_train = acts[idx[:n_train]]
    obss_test = obss[idx[n_train:]]
    acts_test = acts[idx[n_train:]]
    return obss_train, acts_train, obss_test, acts_test

def get_rollouts(env, policy, args, n_batch_rollouts, lin=None):
    rollouts = []
    if lin is None:
        for i in range(n_batch_rollouts):
            rollouts.extend(policy.main(args, env, flag =1))
    else:
        for i in range(n_batch_rollouts):
            rollouts.extend(policy.main(args, env, lime = lin,flag =1))
    return rollouts

def resample(states, actions, serials, max_pts):
    idx = np.random.choice(len(states), size=max_pts)
    return states[idx], actions[idx], serials[idx]

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--abr', metavar='ABR', choices=['pensieve', 'robustmpc', 'hotdash'])
    #parser.add_argument('-n', '--leaf-nodes', type=int)
    parser.add_argument('-q', '--qoe-metric', choices=['lin', 'log', 'hd'])
    parser.add_argument('-l', '--log', action='store_true')
    parser.add_argument('-i', '--lin', action='store_true')
    parser.add_argument('-m', '--iters', type=int)
    parser.add_argument('-t', '--traces', choices=['norway', 'fcc', 'oboe'])

    args = parser.parse_args()
    n_batch_rollouts = 10
    max_iters = args.iters
    max_pts = 200000
    train_frac = 0.8
    np.random.seed(RANDOM_SEED)
    states, actions, serials = [], [], []
    precision = []
    #trees = []
    all_cooked_time, all_cooked_bw, all_file_names = load_trace.load_trace(args.traces)
    if args.abr == 'hotdash':
        net_env = env_hotdash.Environment(all_cooked_time=all_cooked_time, all_cooked_bw=all_cooked_bw,
                              all_file_names=all_file_names)
    else:
        net_env = env.Environment(all_cooked_time=all_cooked_time, all_cooked_bw=all_cooked_bw,
                              all_file_names=all_file_names)

    if args.abr == 'pensieve':
        teacher = pensieve.Pensieve()
        student = pensilin.Pensilin()
        #test = pensieve.Pensieve()
    elif args.abr == 'robustmpc':
        teacher = robustmpc.RobustMPC()
        student = robustlin.Robustlin()
    elif args.abr == 'hotdash':
        teacher = hotdash.Hotdash()
        student = hotdlin.Hotdlin()
    else:
        raise NotImplementedError

    # Step 1: Initialization for the first iteration
    trace = get_rollouts(env=net_env, policy=teacher, args=args, n_batch_rollouts=n_batch_rollouts)
    states.extend((state for state, _, _ in trace))
    actions.extend((action for _, action, _ in trace))
    serials.extend(serial for _, _, serial in trace)

    def predict_fn2(data):
        if args.abr == 'pensieve':
            input_data = np.zeros((data.shape[0], S_INFO, S_LEN))
            input_data[:, 0, -1] = data[:, 0]
            input_data[:, 1, -1] = data[:, 1]
            input_data[:, 2, :] = data[:, 2:10]
            input_data[:, 3, :] = data[:, 10:18]
            input_data[:, 4, :A_DIM] = data[:, 18:24]
            input_data[:, 5, -1] = data[:, 24]
            return teacher.actor.predict(input_data)
        elif args.abr == 'hotdash':
            input_data = np.zeros((data.shape[0], S_INFO, S_LEN))
            input_data[:, 0, -1] = data[:, 0]
            input_data[:, 1, -1] = data[:, 1]
            input_data[:, 2, :] = data[:, 2:10]
            input_data[:, 3, :] = data[:, 10:18]
            input_data[:, 4, :A_DIM] = data[:, 18:24]
            input_data[:, 5, -1] = data[:, 24]
            return teacher.actor_bitr.predict(input_data)
        else:
            pass
    #test_data=[20,21,22,23,24]
    #receive_data=[]
    #def he_test(j):
    grade=np.zeros((100,3))
    for j in range(27,30):
        for i in range(max_iters):
            # Step 2:
            print('cluster number:{},Iteration {}/{}'.format(j, i, max_iters))
            cur_states, cur_actions, cur_serials = resample(np.array(states), np.array(actions), np.array(serials), max_pts)
            print('Training student with {} points'.format(len(cur_serials)))
            serials_train, actions_train, serials_val, actions_val = split_train_test(cur_serials, cur_actions, train_frac)
            #print(serials_train.shape[0])
            #def predict_fn1(state):
                #print(state.shape)
                #print(state.shape[0])
                #result = np.zeros((state.shape[0],6))
                #for i in range (state.shape[0]):
                    #for j in range(6):
                        #if(actions_train[i] ==j ) :
                            #result[i,j] = 0.5
                        #else:
                            #result[i,j] = 0.1
                #print(result)
                #return result

            lin = limes.LIMESimpleModel(cluster_num = j, cluster_method = KMeans, random_state = None)
            lin.fit(serials_train, actions_train, predict_fn = predict_fn2, labels_num = 6)
            #print('Train accuracy: {}'.format(np.mean(actions_train == lin.predict(serials_train))))
            #print('Val accuracy: {}'.format(np.mean(actions_val == lin.predict(serials_val))))
            precision.append(np.mean(lin.predict(serials_val) == actions_val))
            print('unpruned precision', precision[-1])
            grade[i][j-27] =precision[-1]
            student_trace = get_rollouts(env=net_env, policy=student, args=args, n_batch_rollouts=n_batch_rollouts,lin= lin)
            student_states = [state for state, _, _ in student_trace]
            student_actions = [action for _, action, _ in student_trace]
            student_serials = [serial for _, _, serial in student_trace]

            if args.abr == 'pensieve':
                teacher_actions = map(teacher.predict, student_states)
            elif args.abr == 'hotdash':
                teacher_actions = map(teacher.get_abr_rl_bitrate, student_states)
            else:
                pool = ThreadPool(THREAD)
                teacher_actions = pool.map(teacher.predict, student_states)
                pool.close()
                pool.join()
                print (teacher_actions)

        # teacher_actions = []
        # for student_state in student_states:
        #     teacher_actions.append(teacher.predict(student_state))

            states.extend(student_states)
            actions.extend(teacher_actions)
            serials.extend(student_serials)
        #print('cluster number ={}, average curancy: {}',format(j,sum(grade)/max_iters))
        #trees.append(lin)
        #return(grade)
    #pool = ThreadPool(THREAD)
    #receive_data=pool.map(he_test, test_data)
    #pool.close()
    #pool.join()
    np.savetxt('dd.txt',grade,fmt='%.5f')
    #best_lin = trees[-1]
    # save decision tree to file
    #with open('lime/' + args.abr + '.pk3', 'wb') as f:
        #pk.dump(lin, f)

    #dot_data = StringIO()
    #export_graphviz(lin, out_file=dot_data, filled=True)
    #out_graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
    #out_graph.write_svg('lime/' + args.abr + '.svg')


#! /usr/bin/env python3

import argparse
import time

from qmi.core.context import QMI_Context

parser = argparse.ArgumentParser(description="Start some QMI_Contexts, waits, then stops the contexts.")

parser.add_argument("--numcontexts", action='store', default=1, type=int, help="The number of contexts to make.")
parser.add_argument("--sleep", action='store', default=60.0, type=float, help="Time keep the context alive (seconds).")

args = parser.parse_args()

num_contexts = args.numcontexts
sleeptime = args.sleep

contexts = []

print("Instantiating {} contexts ...".format(num_contexts))
for i in range(num_contexts):
    context_name = "context_{}".format(i + 1)
    context = QMI_Context(context_name)
    contexts.append(context)

print("Starting contexts ...")
for (i, context) in enumerate(contexts):
    server_port = 0  # Pick a random one.
    context.start(server_port)

try:
    print("waiting for {:.3f} seconds ...".format(sleeptime))
    time.sleep(sleeptime)
finally:
    print("Stopping contexts ...")
    for context in contexts:
        context.stop()


import numpy as np
import pandas as pd
import sys # can use sys to take command line arguments

class Recommender():
    '''
    This Recommender uses FunkSVD to make predictions of exact ratings.  And uses either FunkSVD or a Knowledge Based recommendation (highest ranked) to make recommendations for users.  Finally, if       given a movie, the recommender will provide movies that are most similar as a Content Based Recommender.
    '''
    def __init__(self, ):
        '''
        what do we need to start out our recommender system
        '''

    def fit(self, reviews_pth='train_data.csv', movies_pth= 'movies_clean.csv', learning_rate=.01, iters=1):
        '''
        fit the recommender        to your dataset and also have this save the results
        pull from when you need to make predictions
        '''

        # Read in the datasets
        self.movies = pd.read_csv(movies_pth)
        self.reviews = pd.read_csv(reviews_pth)

        # define SVD parameters
        self.learning_rate = learning_rate
        self.iters = iters
        self.latent_features = 4

        # Create user-item matrix
        usr_itm = self.reviews[['user_id', 'movie_id', 'rating', 'timestamp']]
        self.user_item_df = usr_itm.groupby(['user_id','movie_id'])['rating'].max().unstack()
        self.user_item_mat = np.array(self.user_item_df)

        # find number of users and movies
        self.n_users = self.user_item_mat.shape[0]
        self.n_movies = self.user_item_mat.shape[1]

        # initialize the user and movie matrices with random values
        self.user_mat = np.random.rand(self.n_users, self.latent_features)
        self.movie_mat = np.random.rand(self.latent_features, self.n_movies)

        # initialize sse at 0 for first iteration
        sse_accum = 0
        self.num_ratings = 0

        # keep track of iteration and MSE
        print("Optimizaiton Statistics")
        print("Iterations | Mean Squared Error ")

        # for each iteration
        for iteration in range(self.iters):

            # update our sse
            old_sse = sse_accum
            sse_accum = 0

            # For each user-movie pair
            for i in range(self.n_users):
                for j in range(self.n_movies):

                    # if the rating exists
                    if self.user_item_mat[i, j] > 0:

                        # compute the error as the actual minus the dot product of the user and movie latent features
                        diff = self.user_item_mat[i, j] - np.dot(self.user_mat[i, :], self.movie_mat[:, j])

                        # Keep track of the sum of squared errors for the matrix
                        sse_accum += diff**2

                        # update ratings counter
                        self.num_ratings += 1

                        # update the values in each matrix in the direction of the gradient
                        for k in range(self.latent_features):
                            self.user_mat[i, k] += self.learning_rate * (2*diff*self.movie_mat[k, j])
                            self.movie_mat[k, j] += self.learning_rate * (2*diff*self.user_mat[i, k])

            # print results
            print("%d \t\t %f" % (iteration+1, sse_accum / self.num_ratings))

    def predict_rating(self, user_matrix, movie_matrix, user_id, movie_id):
        '''
        INPUT:
        user_matrix - user by latent factor matrix
        movie_matrix - latent factor by movie matrix
        user_id - the user_id from the reviews df
        movie_id - the movie_id according the movies df

        OUTPUT:
        pred - the predicted rating for user_id-movie_id according to FunkSVD
        '''

        # Create series of users and movies in the right order
        user_ids_series = np.array(train_data_df.index)
        movie_ids_series = np.array(train_data_df.columns)

        # User row and Movie Column
        user_row = np.where(user_ids_series == user_id)[0][0]
        movie_col = np.where(movie_ids_series == movie_id)[0][0]

        # Take dot product of that row and column in U and V to make prediction
        pred = np.dot(user_matrix[user_row, :], movie_matrix[:, movie_col])

        return pred

    def make_recommendations(self, _id, _id_type='movie', rec_num=5):
        '''
        INPUT:
        _id - either a user or movie id (int)
        _id_type - "movie" or "user" (str)
        rec_num - number of recommendations to return (int)

        OUTPUT:
        recs - (array) a list or numpy array of recommended movies like the
                       given movie, or recs for a user_id given
        '''
        # if the user is available from the matrix factorization data,
        # I will use this and rank movies based on the predicted values
        # For use with user indexing
        rec_ids, rec_names = None, None
        if _id_type == 'user':
            if _id in self.user_ids_series:
                # Get the index of which row the user is in for use in U matrix
                idx = np.where(self.user_ids_series == _id)[0][0]

                # take the dot product of that row and the V matrix
                preds = np.dot(self.user_mat[idx,:],self.movie_mat)

                # pull the top movies according to the prediction
                indices = preds.argsort()[-rec_num:][::-1] #indices
                rec_ids = self.movie_ids_series[indices]
                rec_names = self.get_movie_names(rec_ids, self.movies)

            else:
                # if we don't have this user, give just top ratings back
                rec_names = self.popular_recommendations(_id, rec_num, self.ranked_movies)
                print("Because this user wasn't in our database, we are giving back the top movie recommendations for all users.")

        # Find similar movies if it is a movie that is passed
        else:
            if _id in self.movie_ids_series:
                rec_names = list(self.find_similar_movies(_id, self.movies))[:rec_num]
            else:
                print("That movie doesn't exist in our database.  Sorry, we don't have any recommendations for you.")

        return rec_ids, rec_names

    def make_recommendations(self, _id, _id_type='movie', rec_num=5):
        '''
        INPUT:
        _id - either a user or movie id (int)
        _id_type - "movie" or "user" (str)
        train_data - dataframe of data as user-movie matrix
        train_df - dataframe of training data reviews
        movies - movies df
        rec_num - number of recommendations to return (int)
        user_mat - the U matrix of matrix factorization
        movie_mat - the V matrix of matrix factorization

        OUTPUT:
        rec_ids - (array) a list or numpy array of recommended movies by id
        rec_names - (array) a list or numpy array of recommended movies by name
        '''

        val_users = train_data_df.index
        rec_ids = create_ranked_df(movies,train_df)

        if _id_type == "user":
            idx = np.where(val_users == _id)[0][0]
            pred = predict_rating(self.user_mat, self.movie_mat, user_id, movie_id)
            preds = np.dot(user_mat[idx,:], self.movie_mat)
            indices = preds.argsort()[-rec_num][::-1]
            rec_ids = train_data_df.columns[indices]
            rec_names = get_movie_names[rec_ids]
        elif _id_type == "movie":
            rec_ids = find_similar_movies(_id)
            rec_names = get_movie_names(rec_ids)

        else:
            print("id is either a user_id nor a movie_id.")

        return rec_ids, rec_names

    def find_similar_movies(self, movie_id):
        '''
        INPUT
        movie_id - a movie_id
        OUTPUT
        similar_movies - an array of the most similar movies by title
        '''
        # find the row of each movie id
        movie_idx = np.where(movies['movie_id'] == movie_id)[0][0]

        # find the most similar movie indices - to start I said they need to be the same for all content
        similar_idxs = np.where(dot_prod_movies[movie_idx] == np.max(dot_prod_movies[movie_idx]))[0]

        # pull the movie titles based on the indices
        similar_movies = np.array(movies.iloc[similar_idxs, ]['movie_id'])

        return similar_movies

    def get_movie_names(self, movie_ids):
        '''
        INPUT
        movie_ids - a list of movie_ids
        OUTPUT
        movies - a list of movie names associated with the movie_ids

        '''
        movie_lst = list(movies[movies['movie_id'].isin(movie_ids)]['movie'])

        return movie_lst

    def create_ranked_df(self, movies, reviews):
        '''
        INPUT
        movies - the movies dataframe
        reviews - the reviews dataframe

        OUTPUT
        ranked_movies - a dataframe with movies that are sorted by highest avg rating, more reviews,
                        then time, and must have more than 4 ratings
        '''

        # Pull the average ratings and number of ratings for each movie
        movie_ratings = reviews.groupby('movie_id')['rating']
        avg_ratings = movie_ratings.mean()
        num_ratings = movie_ratings.count()
        last_rating = pd.DataFrame(reviews.groupby('movie_id').max()['date'])
        last_rating.columns = ['last_rating']

        # Add Dates
        rating_count_df = pd.DataFrame({'avg_rating': avg_ratings, 'num_ratings': num_ratings})
        rating_count_df = rating_count_df.join(last_rating)

        # merge with the movies dataset
        movie_recs = movies.set_index('movie_id').join(rating_count_df)

        # sort by top avg rating and number of ratings
        ranked_movies = movie_recs.sort_values(['avg_rating', 'num_ratings', 'last_rating'], ascending=False)

        # for edge cases - subset the movie list to those with only 5 or more reviews
        ranked_movies = ranked_movies[ranked_movies['num_ratings'] > 4]

        return ranked_movies

    def popular_recommendations(self, user_id, n_top, ranked_movies):
        '''
        INPUT:
        user_id - the user_id (str) of the individual you are making recommendations for
        n_top - an integer of the number recommendations you want back
        ranked_movies - a pandas dataframe of the already ranked movies based on avg rating, count, and time

        OUTPUT:
        top_movies - a list of the n_top recommended movies by movie title in order best to worst
        '''

        top_movies = list(ranked_movies['movie'][:n_top])

        return top_movies

if __name__ == '__main__':
    # test different parts to make sure it works
    import recommender as r

    #instantiate recommender
    rec = r.Recommender()

    # fit recommender
    rec.fit(reviews_pth='train_data.csv', movies_pth= 'movies_clean.csv', learning_rate=.01, iters=1)

    # predict
    rec.predict_rating(user_id=8, movie_id=2844)

    # make recommendations
    print(rec.make_recommendations(8,'user')) # user in the dataset
    print(rec.make_recommendations(1,'user')) # user not in dataset
    print(rec.make_recommendations(1853728)) # movie in the dataset
    print(rec.make_recommendations(1)) # movie not in dataset
    print(rec.n_users)
    print(rec.n_movies)
    print(rec.num_ratings)


from __future__ import unicode_literals

from django.db.models.fields import PositiveIntegerField, PositiveSmallIntegerField
from django.db.models.fields.related import ForeignKey, ManyToManyField, OneToOneField
from model_fields.numeric import (
    PositiveAutoField, PositiveBigIntegerField, PositiveMediumIntegerField,
    PositiveTinyIntegerField,
)

class PositiveForeignKey(ForeignKey):
    def db_type(self, connection):
        rel_field = self.target_field
        if (isinstance(rel_field, PositiveAutoField) or
                (not connection.features.related_fields_match_type and
                isinstance(rel_field, (PositiveIntegerField,
                                       PositiveTinyIntegerField,
                                       PositiveSmallIntegerField,
                                       PositiveMediumIntegerField,
                                       PositiveBigIntegerField)))):
            return PositiveIntegerField().db_type(connection=connection)
        else:
            return super(PositiveForeignKey, self).db_type(connection)

class PositiveOneToOneField(OneToOneField):
    def db_type(self, connection):
        rel_field = self.target_field
        if (isinstance(rel_field, PositiveAutoField) or
                (not connection.features.related_fields_match_type and
                isinstance(rel_field, (PositiveIntegerField,
                                       PositiveTinyIntegerField,
                                       PositiveSmallIntegerField,
                                       PositiveMediumIntegerField,
                                       PositiveBigIntegerField)))):
            return PositiveIntegerField().db_type(connection=connection)
        else:
            return super(PositiveOneToOneField, self).db_type(connection)

class PositiveManyToManyField(ManyToManyField):
    pass


import numpy as np
import time
import sys
import math
from random import uniform

if sys.version_info.major == 2:
    import Tkinter as tk
else:
    import tkinter as tk
import itertools

UNIT = 100  # pixels
MAZE_H = 4  # grid height
MAZE_W = 4  # grid width

class Maze(tk.Tk, object):
    def __init__(self):
        super(Maze, self).__init__()
        # a = [[.1, .4, .6, -.1, -.4, -.6]]
        # b = [[.1, .4, .6, -.1, -.4, -.8]]
        a = [[.2, .4, .6, -.2, -.4, -.6]]
        b = [[.2, .4, .6, -.2, -.4, -.6]]

        c = np.vstack((a, b))
        self.data = list(itertools.product(*c))
        self.data = [str(x) for x in self.data]

        self.action_space = self.data

        self.n_actions = len(self.action_space)
        self.title('maze')
        self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))

        self._vspeed = self.my_round(uniform(-1, 1), 50)
        self._xdistance = self.my_round(uniform(-0.6, 0.6), 100)

        self.m = 0.5
        self.g = 9.8
        self.l = 2
        self.c = 0.01
        self.t = 0.02
        self.table_width = MAZE_W * UNIT * .8 - MAZE_W * UNIT * .2
        self.table_height_move = MAZE_W * UNIT * .8 - MAZE_W * UNIT * .2
        self._build_maze()

        self._time = 0

    def _build_maze(self):
        self.canvas = tk.Canvas(self, bg='white',
                                height=MAZE_H * UNIT,
                                width=MAZE_W * UNIT)

        origin = np.array([MAZE_W * UNIT * .5, MAZE_H * UNIT * .5 - 20])

        oval_center = origin + self._xdistance * (self.table_width / 2)

        self.oval = self.canvas.create_oval(
            oval_center[0] - 15, origin[1] - 15,
            oval_center[0] + 15, origin[1] + 15,
            fill='red')

        self.flatTable = self.canvas.create_line(MAZE_W * UNIT * .2, MAZE_H * UNIT * .5, MAZE_W * UNIT * .8,
                                                 MAZE_H * UNIT * .5, fill='#531B0A',
                                                 width=10)
        self.zero = self.canvas.create_line(MAZE_W * UNIT * .5, MAZE_H * UNIT * .5 - 10, MAZE_W * UNIT * .5,
                                            MAZE_H * UNIT * .5 + 10, fill='red',
                                            width=2)
        # pack all

        # self.gif1 = tk.PhotoImage(file='./Pic/robot1.gif', format="gif -index 2")
        self._numr1 = 0
        self._numr2 = 0
        self._animateagent2()
        self._animateagent1()

        bg = tk.PhotoImage(file='./Pic/bg.png')
        self.bg = self.canvas.create_image(0, 0, image=bg, anchor=tk.NW)
        self.robot1 = self.canvas.create_image(30, MAZE_H * UNIT * .5 - 50, image=self.gif1, anchor=tk.NW)
        self.robot2 = self.canvas.create_image(MAZE_W * UNIT - 80, MAZE_H * UNIT * .5 - 50, image=self.gif2,
                                               anchor=tk.NW)

        # label = tk.Label(image=gif1)
        # label.configure(image=gif1)
        # label.image = gif1  # keep a reference!
        label = tk.Label(image=bg)
        label.image = bg

        # image = ImageTk.PhotoImage(file="./Pic/bg.png")
        # self.canvas.create_image(0, 0, image=image, anchor=tk.NW)
        # self.canvas.pack(expand=tk.YES, fill=tk.BOTH)

        self.canvas.pack()

    def _animateagent2(self):
        try:
            self.gif2 = tk.PhotoImage(file="./Pic/robot2.gif",
                                      format='gif -index {}'.format(self._numr2))  # Looping through the frames
            label = tk.Label(image=self.gif2)
            label.configure(image=self.gif2)
            self.robot2 = self.canvas.create_image(MAZE_W * UNIT - 80, MAZE_H * UNIT * .5 - 50, image=self.gif2,
                                                   anchor=tk.NW)

            # label = tk.Label(image=self.gif)
            # label.image = self.gif  # keep a reference!
            self.canvas.pack()
            self._numr2 += 1
        except tk.TclError:  # When we try a frame that doesn't exist, we know we have to start over from zero
            self._numr2 = 0

        self.after(int(.1 * 1000), self._animateagent2)

    def _animateagent1(self):
        try:
            self.gif1 = tk.PhotoImage(file="./Pic/robot1.gif",
                                      format='gif -index {}'.format(self._numr1))  # Looping through the frames
            label = tk.Label(image=self.gif1)
            label.configure(image=self.gif1)
            self.robot1 = self.canvas.create_image(30, MAZE_H * UNIT * .5 - 50, image=self.gif1, anchor=tk.NW)

            # label = tk.Label(image=self.gif)
            # label.image = self.gif  # keep a reference!
            self.canvas.pack()
            self._numr1 += 1
        except tk.TclError:  # When we try a frame that doesn't exist, we know we have to start over from zero
            self._numr1 = 0

        self.after(int(.1 * 1000), self._animateagent1)

    def reset(self):
        self.update()
        self._time = 0
        time.sleep(0.7)
        self._vspeed = self.my_round(uniform(-1, 1), 50)
        self._xdistance = self.my_round(uniform(-0.6, 0.6), 100)
        self.canvas.delete(self.oval)
        self.canvas.delete(self.flatTable)
        origin = np.array([MAZE_W * UNIT * .5, MAZE_H * UNIT * .5 - 20])
        # create oval

        oval_center = origin + self._xdistance * (self.table_width / 2)

        self.oval = self.canvas.create_oval(
            oval_center[0] - 15, origin[1] - 15,
            oval_center[0] + 15, origin[1] + 15,
            fill='red')
        self.flatTable = self.canvas.create_line(MAZE_W * UNIT * .2, MAZE_H * UNIT * .5, MAZE_W * UNIT * .8,
                                                 MAZE_H * UNIT * .5, fill='black',
                                                 width=8)

        return self._xdistance, self._vspeed

    def nextState(self, v, h1, h2):
        xresult = ((-self.c * v) + (self.m * self.g * ((h1 - h2) / self.l))) / self.m
        return xresult

    def reward(self, x, xbar):
        p = 0.25 * 0.25
        return 0.8 * np.exp(-(x * x) / p) + 0.2 * np.exp(-(xbar * xbar) / p)

    def reward2(self, x, xbar):
        return 1 - x * x - xbar * xbar

    def xPos(self, a, t, v, x_0):
        return 1 / 2 * a * t * t + v * t + x_0

    def vSpeed(self, a, t, v_0):
        return a * t + v_0

    def my_round(self, x, p):
        return round(x * p) / p

    def MN(self, AN, BC, AB):
        return AN * BC / math.sqrt(math.fabs(AB * AB - BC * BC))

    def movextable(self, h1, h2):
        h = h1 - h2
        return math.sqrt(4 - h * h)

    def step(self, state, action):

        s = self.canvas.coords(self.oval)
        base_action = np.array([0.0, 0.0])
        X = state

        a = self.nextState(X[1], float(action[0]), float(action[1]))
        base_action[0] = self.xPos(a, self.t, X[1], X[0])
        v = self.vSpeed(a, self.t, X[1])

        tw = (self.table_width / 2)

        # move Ball
        moveTableHeight = self.movextable(action[0], action[1]) / 2
        AN = self.canvas.coords(self.oval)[0] - self.canvas.coords(self.flatTable)[0]
        AC = moveTableHeight * 210
        BC = (action[0] - action[1]) * (self.table_height_move / 2)
        base_action[1] = (AN * BC / AC) - action[0] * (self.table_height_move / 2)
        base_action[1] -= (s[3] - MAZE_H * UNIT * .5)
        movex = ((base_action[0] * tw) - (s[0] - MAZE_W * UNIT * .5))
        moveh = base_action[1]
        self.canvas.move(self.oval, movex, moveh)  # move agent

        # round data to discrete
        base_action[0] = self.my_round(base_action[0], 100)
        v = self.my_round(v, 50)
        # print(v)
        if v < -1:
            v = -1
        if v > 1:
            v = 1

        # Update table
        self.canvas.delete(self.flatTable)
        self.flatTable = self.canvas.create_line(
            MAZE_W * UNIT * .2 + (self.table_width / 2) - (moveTableHeight * (self.table_width / 2)),
            (-action[0] * (self.table_height_move / 2) + UNIT * MAZE_H * .5),
            (MAZE_W * UNIT * .8) - ((self.table_width / 2) - (moveTableHeight * (self.table_width / 2))),
            (-action[1] * (self.table_height_move / 2) + UNIT * MAZE_H * .5),
            fill='#531B0A',
            width=10)

        s = self.canvas.coords(self.oval)
        ss = self.canvas.coords(self.flatTable)

        s_ = (base_action[0], v)

        # print(X, s_ , action, a)
        reward = self.reward2(base_action[0], v)

        # check ball into table
        if s[2] > ss[2] + 15 or s[0] < ss[0] - 15:
            # print(X, action, a)
            done = True
            self.canvas.move(self.oval, 0, 50)  # move agent
            self._time = 0
        else:
            done = False
        return s_, reward, done

    def render(self):
        time.sleep(0.03)
        self.update()

def update():
    for t in range(10):
        env.reset()
        while True:
            env.render()
            a = 1
            s, r, done = env.step(a)
            if done:
                break

if __name__ == '__main__':
    env = Maze()
    env.after(100, update)
    env.mainloop()


import re
import json
import request
class Info:
    def __init__(self,Ip,Hyphen,Request,Day,Month,Year,Hour,Min,Sec,Zone,get,Status,SizeResponse,Referer,UserAgent,
                    Country,Latitude,Longitude,Country_code):
        self.Ip=Ip
        self.Hyphen=Hyphen
        self.Request=Request

        self.Day=Day
        self.Month=Month
        self.Year=Year
        self.Hour=Hour
        self.Min=Min
        self.Sec=Sec
        self.Zone=Zone

        self.Country=Country
        self.Latitude=Latitude
        self.Longitude=Longitude
        self.Country_code=Country_code

        self.get=str(get)
        self.Status=str(Status)
        self.SizeResponse=str(SizeResponse)
        self.Referer=str(Referer)
        self.UserAgent=str(UserAgent)
    def __str__(self):
        return self.Ip

class Storange_Logele:
    def __init__(self):
        self.storange=[]
    def Add_log(self,ip,hy,re,day,mon,year,hour,min,sec,zone,get,stat,size,ref,user,Country,Latitude,Longitude,Country_code):
        self.storange.append(Info(ip,hy,re,day,mon,year,hour,min,sec,zone,get,stat,size,ref,user,Country,Latitude,Longitude,Country_code))
    def save_file_json(self):
        data=json.dumps(self.storange,default=obj_to_dict,indent=4)
        with open('Log_data.json', 'w') as f:
            json.dump(data, f)

def load_mydata():
    with open('Log_data.json') as f:
        data=json.load(f)
    my_data=json.loads(data,object_hook=dict_to_obj)
    return my_data

def ListToStr(l):
    str1=" "
    for i in l:
        str1+=i
    return str1
def obj_to_dict(obj):
    my_dict={
        "__class__":obj.__class__.__name__,
        "__module__":obj.__module__
    }
    my_dict.update(obj.__dict__)
    return my_dict
def dict_to_obj(my_dict):
    if "__class__" in my_dict:
        class_name=my_dict.pop("__class__")
        module_name=my_dict.pop("__module__")
        module=__import__(module_name)
        class_=getattr(module,class_name)
        print(class_)
        obj= class_(**my_dict)
    else:
        obj=my_dict
    return obj


def get_as_radian(degree,minute,second=0,zodiac_index=0):
    base_angle=zodiac_index*30
    deg_angle=base_angle+degree+(minute/60.0)+(second/3600.0)
    return np.radians(deg_angle)
#returns x y coordinates of mars from triangulation
def get_coordinates(A,B,C,D):
    x=(np.tan(D)*np.cos(B)-np.tan(C)*np.cos(A)-np.sin(B)+np.sin(A))/(np.tan(D)-np.tan(C))
    y=np.sin(A)+np.tan(C)*(x-np.cos(A))
    return (x,y)

def circle_fit_objective(r,x,y):
    return np.sum((np.sqrt(x**2+y**2)-r)**2)

def ellipse_fit_objective(params,u,v):
    a=params[0]
    eccentricity=params[1]
    b=np.sqrt(a*a*(1-eccentricity**2))
    delta=params[2]
    X=u*np.cos(delta)+v*np.sin(delta)
    Y=v*np.cos(delta)-u*np.sin(delta)+np.sqrt(a**2-b**2)
    f=(X/a)**2+(Y/b)**2-1
    return np.sum(f**2)

#returns mars heliocentric lats from geocentric lats and dis of mars from sun
def get_mars_helio_lat(mars_lat_geo,radius):
    y= (radius-1)*np.tan(mars_lat_geo)
    return np.arctan(y/radius)

#returns mars' 3d coordinates from latitude(alpha) and longitude(beta)
def get_mars_3d(alpha,beta):
    X=-np.cos(alpha)*np.sin(beta)
    Y=np.cos(alpha)*np.cos(beta)
    Z=np.sin(alpha)
    return (X,Y,Z)

def cost(x,y,z,normal_dirn): #squared sum of distances of (x,y,z) from plane with normal_dirn as normal
    dis=(x*normal_dirn[0]+y*normal_dirn[1]+z*normal_dirn[2])/np.sqrt(np.sum(normal_dirn**2))
    return np.sum(dis**2)
#check pdf for computation
def grad_sig_f_sq(x,y,z,normal_dirn):
    S=np.sum(normal_dirn**2)
    A,B,C=normal_dirn
    V=(A*x+B*y+C*z)
    df_da=(1/S)*(2*np.sum(V*x))-(2*A/(S*S))*(np.sum(V**2))
    df_db=(1/S)*(2*np.sum(V*y))-(2*B/(S*S))*(np.sum(V**2))
    df_dc=(1/S)*(2*np.sum(V*z))-(2*C/(S*S))*(np.sum(V**2))
    return np.array([df_da,df_db,df_dc])

#exact line search to get best alpha using dichotomous search
def dichotomous(x,y,z,theta,grad,left,right,tolerance=1e-5):
    while(right-left>tolerance):
        mid=(left+right)/2
        mid_l=mid-tolerance/4
        mid_r=mid+tolerance/4
        cost_l=cost(x,y,z,theta-grad*mid_l)
        cost_r=cost(x,y,z,theta-grad*mid_r)
        if cost_l>cost_r:
            left=mid_l
        else:
            right=mid_r
    return (left+right)/2

#steepest descent using exact line search.
def grad_desc(x,y,z):
    theta=np.random.uniform(low=-1, high=1, size=(3,))
    eps=1e-5
    gf=grad_sig_f_sq(x,y,z,theta)
    mod_gf=np.sum(gf**2)
    i=0
    while(mod_gf>eps):
        i+=1
        mod_gf=np.sum(gf**2)
        gf=grad_sig_f_sq(x,y,z,theta)
        alpha_min=dichotomous(x,y,z,theta,gf,0,1)
        theta=theta-alpha_min*gf
    return theta,cost(x,y,z,theta)

#returns inclination of a vec from xy plane (ecliptic plane)
def get_inclination(plane):
    sin_inclination=np.sqrt(np.sum(np.cross(plane,np.array([0,0,1]))**2))/(np.sqrt(np.sum(plane**2))) #001 is normal to xy plane.(horizontal plane)
    inclination=np.arcsin(sin_inclination)
    return np.degrees(inclination)

#returns mars' 3d coordinates from equation of plane and x,y coordinates (also the projection on ecliptic)
def get_mars_plane_projection(normal_dirn,x,y):
    z=(normal_dirn[0]*x+normal_dirn[1]*y)/normal_dirn[2]
    return (x,y,z)

#given n, n>1 3d points lying on a plane and normal to the plane this method returns the points in 2d coordinates u-v on plane.
def coordinate_transform(X,Y,Z,normal_dirn):
    u=np.array([X[0]-X[1],Y[0]-Y[1],Z[0]-Z[1]]) #vector on plane
    v=np.cross(normal_dirn,u)
    u=u/np.sqrt(np.sum(u**2)) #unit vector
    v=v/np.sqrt(np.sum(v**2))
    #let U,V be the respective coordinates in this new system then,
    U=np.dot(u,(X,Y,Z))
    V=np.dot(v,(X,Y,Z))
    return (U,V)

#returns uv coordinates for given point with parameter t on ellipse with eccentricity,semi major axis, rotation and focus on origin
def get_uv(t,delta,A,ecc):
    B=A*np.sqrt(1-ecc**2)
    u_=A*np.cos(t)
    v_=B*np.sin(t)-A*ecc
    x=u_*np.cos(delta)-v_*np.sin(delta)
    y=u_*np.sin(delta)+v_*np.cos(delta)
    return (x,y)

import pandas as pd
import numpy as np
df=pd.read_csv('../data/01_data_mars_triangulation.csv')

#ques 2
A=[] #heliocentric angle of earth
B=[] #paired heliocentric angle of earth
C=[] #geocentric angle of mars
D=[] #paired geocentric angle of mars

for i in range(1,6):
    rows=df[df['PairIndex']==i]
    A.append(get_as_radian(rows.iloc[0]['DegreeEarthLocationHelioCentric'],rows.iloc[0]['MinuteEarthLocationHelioCentric']))
    B.append(get_as_radian(rows.iloc[1]['DegreeEarthLocationHelioCentric'],rows.iloc[1]['MinuteEarthLocationHelioCentric']))
    C.append(get_as_radian(rows.iloc[0]['DegreeMarsLocationGeoCentric'],rows.iloc[0]['MinuteMarsLocationGeoCentric']))
    D.append(get_as_radian(rows.iloc[1]['DegreeMarsLocationGeoCentric'],rows.iloc[1]['MinuteMarsLocationGeoCentric']))
A=np.array(A)
B=np.array(B)
C=np.array(C)
D=np.array(D)

x,y=get_coordinates(A,B,C,D)

from scipy.optimize import minimize,Bounds
r=np.random.rand(1)
res=minimize(circle_fit_objective,r,args=(x,y))
Radius=res.x
print("Radius of Circle on Ecleptic plae",Radius)

#ques 3
df_opp=pd.read_csv('../data/01_data_mars_opposition.csv')
mars_lat_geo=get_as_radian(df_opp['LatDegree'],df_opp['LatMinute'])
mars_lat_helio=get_mars_helio_lat(mars_lat_geo,Radius)
print("Mars' heliocentric latitudes(radians)",mars_lat_helio)
mars_long_helio=get_as_radian(df_opp['Degree'],df_opp['Minute'],df_opp['Second'],df_opp['ZodiacIndex'])

m_x,m_y,m_z=get_mars_3d(mars_lat_helio,mars_long_helio)
res=grad_desc(m_x,m_y,m_z)
inclination=get_inclination(res[0])
print("Inclination with ecliptic",int(inclination),"degrees", (inclination-int(inclination))*60,"minutes")
print("Cost of regression",res[1])
mars_plane=(res[0]/np.sqrt(np.sum(res[0]**2)))

#ques 4
X,Y,Z=get_mars_plane_projection(mars_plane,x,y)
print("Mars' five locations on a plane (X,Y,Z) arrays",(X,Y,Z))
U,V=coordinate_transform(X,Y,Z,mars_plane)
#fitting circle
r_mc=np.random.rand(1)
res_mc=minimize(circle_fit_objective,r_mc,args=(U,V))
print("Sum of losses for circle on mars plane",res_mc.fun)
#fitting ellipse
r_me=np.random.rand(3)
bounds=Bounds([0,0,0],[np.inf,1,2*np.pi])
res_me=minimize(ellipse_fit_objective,r_me,args=(U,V),bounds=bounds)
print("Sum of losses for ellipse on mars' plane",res_me.fun)

#plots

import matplotlib.pyplot as plt

plt.plot(0,0,'ro')

t=np.linspace(0,2*np.pi,100)

circ_x=Radius*np.cos(t)
circ_y=Radius*np.sin(t)
plt.plot(circ_x,circ_y,'y')

A,ecc,delta=res_me.x
ellip_x,ellip_y=get_uv(t,delta,A,ecc)
plt.plot(ellip_x,ellip_y)
M_x,M_y,M_z=get_mars_plane_projection(mars_plane,m_x,m_y)
M_u,M_v=coordinate_transform(M_x,M_y,M_z,mars_plane)

T=np.arctan(M_v/M_u)
m12_x,m12_y=get_uv(T,delta,A,ecc)
plt.plot(m12_x,m12_y,'o')
plt.show()


# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""BERT finetuning runner."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import json
import os
import tempfile

import tensorflow as tf
from bayes_opt import BayesianOptimization
from bayes_opt.event import Events
from bayes_opt.observer import JSONLogger
from bayes_opt.util import load_logs

import modeling
import tokenization
from run_wikijoin import (WikiJoinProcessor, model_fn_builder, file_based_convert_examples_to_features,
                          file_based_input_fn_builder, PaddingInputExample)

flags = tf.flags

FLAGS = flags.FLAGS

## Required parameters
flags.DEFINE_string(
    "data_dir", None,
    "The input data dir. Should contain the .tsv files (or other data files) "
    "for the task.")

flags.DEFINE_string(
    "bert_config_file", None,
    "The config json file corresponding to the pre-trained BERT model. "
    "This specifies the model architecture.")

flags.DEFINE_string("vocab_file", None,
                    "The vocabulary file that the BERT model was trained on.")

flags.DEFINE_string(
    "output_dirs_path", None,
    "调参时保存每次调参输出模型的路径。")

## Other parameters

flags.DEFINE_string(
    "init_checkpoint", None,
    "Initial checkpoint (usually from a pre-trained BERT model).")

flags.DEFINE_bool(
    "do_lower_case", True,
    "Whether to lower case the input text. Should be True for uncased "
    "models and False for cased models.")

flags.DEFINE_integer(
    "max_seq_length", 128,
    "The maximum total input sequence length after WordPiece tokenization. "
    "Sequences longer than this will be truncated, and sequences shorter "
    "than this will be padded.")

flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")

flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.")

flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.")

flags.DEFINE_float(
    "warmup_proportion", 0.1,
    "Proportion of training to perform linear learning rate warmup for. "
    "E.g., 0.1 = 10% of training.")

flags.DEFINE_integer("save_checkpoints_steps", 1000,
                     "How often to save the model checkpoint.")

flags.DEFINE_integer("iterations_per_loop", 1000,
                     "How many steps to make in each estimator call.")

flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")

tf.flags.DEFINE_string(
    "tpu_name", None,
    "The Cloud TPU to use for training. This should be either the name "
    "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
    "url.")

tf.flags.DEFINE_string(
    "tpu_zone", None,
    "[Optional] GCE zone where the Cloud TPU is located in. If not "
    "specified, we will attempt to automatically detect the GCE project from "
    "metadata.")

tf.flags.DEFINE_string(
    "gcp_project", None,
    "[Optional] Project name for the Cloud TPU-enabled project. If not "
    "specified, we will attempt to automatically detect the GCE project from "
    "metadata.")

tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.")

flags.DEFINE_integer(
    "num_tpu_cores", 8,
    "Only used if `use_tpu` is True. Total number of TPU cores to use.")

## Tuning parameters

flags.DEFINE_string("tuning_metric", "eval_accuracy", "调参时优化的指标。")

def main(_):
    tf.logging.set_verbosity(tf.logging.INFO)

    tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,
                                                  FLAGS.init_checkpoint)

    bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)

    if FLAGS.max_seq_length > bert_config.max_position_embeddings:
        raise ValueError(
            "Cannot use sequence length %d because the BERT model "
            "was only trained up to sequence length %d" %
            (FLAGS.max_seq_length, bert_config.max_position_embeddings))

    processor = WikiJoinProcessor()

    label_list = processor.get_labels()

    tokenizer = tokenization.FullTokenizer(
        vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)

    train_examples = processor.get_train_examples(FLAGS.data_dir)
    eval_examples = processor.get_dev_examples(FLAGS.data_dir)
    tuning_metric = FLAGS.tuning_metric

    output_dirs_path = FLAGS.output_dirs_path

    tf.gfile.MakeDirs(output_dirs_path)

    pbounds = {"learning_rate": (1e-5, 6e-5), "num_train_epochs": (1, 5)}

    partial_function = functools.partial(tuned_function, bert_config=bert_config, label_list=label_list,
                                         output_dirs_path=output_dirs_path, tokenizer=tokenizer,
                                         train_examples=train_examples,
                                         eval_examples=eval_examples, tuning_metric=tuning_metric)

    optimizer = BayesianOptimization(
        f=partial_function,
        pbounds=pbounds,
        random_state=0,
    )
    logger = JSONLogger(path=os.path.join(output_dirs_path, "logs_2.json"))
    optimizer.subscribe(Events.OPTMIZATION_STEP, logger)

    load_logs(optimizer, logs=[os.path.join(output_dirs_path, "logs.json")])
    optimizer.maximize(init_points=0, n_iter=20)
    with open(os.path.join(output_dirs_path, "best_2.json"), "w", encoding="utf-8") as fo_best:
        json.dump(optimizer.max, fo_best, ensure_ascii=False)

def tuned_function(bert_config, label_list, output_dirs_path, tokenizer, train_examples, eval_examples, tuning_metric,
                   learning_rate, num_train_epochs):
    output_dir = tempfile.mkdtemp(prefix=f"lr{learning_rate:.6f}_ep{num_train_epochs:.6f}_", dir=output_dirs_path)
    tf.gfile.MakeDirs(output_dir)
    tpu_cluster_resolver = None
    if FLAGS.use_tpu and FLAGS.tpu_name:
        tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
            FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
    is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
    run_config = tf.contrib.tpu.RunConfig(
        cluster=tpu_cluster_resolver,
        master=FLAGS.master,
        model_dir=output_dir,
        save_checkpoints_steps=FLAGS.save_checkpoints_steps,
        tpu_config=tf.contrib.tpu.TPUConfig(
            iterations_per_loop=FLAGS.iterations_per_loop,
            num_shards=FLAGS.num_tpu_cores,
            per_host_input_for_training=is_per_host))
    num_train_steps = int(
        len(train_examples) / FLAGS.train_batch_size * num_train_epochs)
    num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
    model_fn = model_fn_builder(
        bert_config=bert_config,
        num_labels=len(label_list),
        init_checkpoint=FLAGS.init_checkpoint,
        learning_rate=learning_rate,
        num_train_steps=num_train_steps,
        num_warmup_steps=num_warmup_steps,
        use_tpu=FLAGS.use_tpu,
        use_one_hot_embeddings=FLAGS.use_tpu)
    # If TPU is not available, this will fall back to normal Estimator on CPU
    # or GPU.
    estimator = tf.contrib.tpu.TPUEstimator(
        use_tpu=FLAGS.use_tpu,
        model_fn=model_fn,
        config=run_config,
        train_batch_size=FLAGS.train_batch_size,
        eval_batch_size=FLAGS.eval_batch_size,
        predict_batch_size=FLAGS.predict_batch_size)
    train_file = os.path.join(output_dir, "train.tf_record")
    file_based_convert_examples_to_features(
        train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file)
    tf.logging.info("***** Running training *****")
    tf.logging.info("  Num examples = %d", len(train_examples))
    tf.logging.info("  Batch size = %d", FLAGS.train_batch_size)
    tf.logging.info("  Num steps = %d", num_train_steps)
    train_input_fn = file_based_input_fn_builder(
        input_file=train_file,
        seq_length=FLAGS.max_seq_length,
        is_training=True,
        drop_remainder=True)
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
    num_actual_eval_examples = len(eval_examples)
    if FLAGS.use_tpu:
        # TPU requires a fixed batch size for all batches, therefore the number
        # of examples must be a multiple of the batch size, or else examples
        # will get dropped. So we pad with fake examples which are ignored
        # later on. These do NOT count towards the metric (all tf.metrics
        # support a per-instance weight, and these get a weight of 0.0).
        while len(eval_examples) % FLAGS.eval_batch_size != 0:
            eval_examples.append(PaddingInputExample())
    eval_file = os.path.join(output_dir, "eval.tf_record")
    file_based_convert_examples_to_features(
        eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file)
    tf.logging.info("***** Running evaluation *****")
    tf.logging.info("  Num examples = %d (%d actual, %d padding)",
                    len(eval_examples), num_actual_eval_examples,
                    len(eval_examples) - num_actual_eval_examples)
    tf.logging.info("  Batch size = %d", FLAGS.eval_batch_size)
    # This tells the estimator to run through the entire set.
    eval_steps = None
    # However, if running eval on the TPU, you will need to specify the
    # number of steps.
    if FLAGS.use_tpu:
        assert len(eval_examples) % FLAGS.eval_batch_size == 0
        eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size)
    eval_drop_remainder = True if FLAGS.use_tpu else False
    eval_input_fn = file_based_input_fn_builder(
        input_file=eval_file,
        seq_length=FLAGS.max_seq_length,
        is_training=False,
        drop_remainder=eval_drop_remainder)
    result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)
    return result[tuning_metric]

if __name__ == "__main__":
    flags.mark_flag_as_required("data_dir")
    flags.mark_flag_as_required("vocab_file")
    flags.mark_flag_as_required("bert_config_file")
    flags.mark_flag_as_required("output_dirs_path")
    tf.app.run()


from finviz.helper_functions.request_functions import http_request_get

STOCK_URL = 'https://finviz.com/quote.ashx'
NEWS_URL = 'https://finviz.com/news.ashx'

def get_stock(ticker):
    """
    Returns a dictionary containing stock data.

    :param ticker: stock symbol
    :type ticker: str
    :return dict
    """

    data = {}
    page_parsed, _ = http_request_get(url=STOCK_URL, payload={'t': ticker}, parse=True)
    all_rows = [row.xpath('td//text()') for row in page_parsed.cssselect('tr[class="table-dark-row"]')]

    for row in all_rows:
        for column in range(0, 11):
            if column % 2 == 0:
                data[row[column]] = row[column + 1]

    return data

def get_insider(ticker):
    """
    Returns a list of dictionaries containing all recent insider transactions.

    :param ticker: stock symbol
    :return: list
    """

    page_parsed, _ = http_request_get(url=STOCK_URL, payload={'t': ticker}, parse=True)
    table = page_parsed.cssselect('table[class="body-table"]')[0]
    headers = table[0].xpath('td//text()')
    data = [dict(zip(headers, row.xpath('td//text()'))) for row in table[1:]]

    return data

def get_news(ticker):
    """
    Returns a list of sets containing news headline and url

    :param ticker: stock symbol
    :return: list
    """

    page_parsed, _ = http_request_get(url=STOCK_URL, payload={'t': ticker}, parse=True)
    all_news = page_parsed.cssselect('a[class="tab-link-news"]')

    dates = []
    for i in range(len(all_news)):
        tr = all_news[i].getparent().getparent()
        date_str = tr[0].text.strip()
        if ' ' not in date_str:
            # This is only time, need to grab date from upper sibling news line.
            tbody = tr.getparent()
            previous_date_str = ''
            j = 1
            while ' ' not in previous_date_str:
                try:
                    previous_date_str = tbody[i-j][0].text.strip()
                except IndexError:
                    break
                j += 1
            # Combine date from earlier news with time from current news.
            date_str = ' '.join([previous_date_str.split(' ')[0], date_str])
        dates.append(date_str)

    headlines = [row.xpath('text()')[0] for row in all_news]
    urls = [row.get('href') for row in all_news]

    return list(zip(dates, headlines, urls))

def get_all_news():
    """
    Returns a list of sets containing time, headline and url

    :return: list
    """

    page_parsed, _ = http_request_get(url=NEWS_URL, parse=True)
    all_dates = [row.text_content() for row in page_parsed.cssselect('td[class="nn-date"]')]
    all_headlines = [row.text_content() for row in page_parsed.cssselect('a[class="nn-tab-link"]')]
    all_links = [row.get('href') for row in page_parsed.cssselect('a[class="nn-tab-link"]')]

    return list(zip(all_dates, all_headlines, all_links))

def get_analyst_price_targets(ticker):
    """
    Returns a list of dictionaries containing all analyst ratings and Price targets
     - if any of 'price_from' or 'price_to' are not available in the DATA, then those values are set to default 0

    :param ticker: stock symbol
    :return: list
    """

    import datetime

    page_parsed, _ = http_request_get(url=STOCK_URL, payload={'t': ticker}, parse=True)
    table = page_parsed.cssselect('table[class="fullview-ratings-outer"]')[0]
    ratings_list = [row.xpath('td//text()') for row in table[1:]]
    ratings_list = [[val for val in row if val != '\n'] for row in ratings_list] #remove new line entries

    headers = ['date', 'category', 'analyst', 'rating', 'price_from', 'price_to'] # header names
    analyst_price_targets = []

    for row in ratings_list:
        price_from, price_to = 0, 0  # defalut values for len(row) == 4 , that is there is NO price information
        if len(row) == 5:
            strings = row[4].split('→')
            #print(strings)
            if len(strings) == 1:
                price_to = int(strings[0].strip(' ').strip('$'))   # if only ONE price is avalable then it is 'price_to' value
            else:
                price_from = int(strings[0].strip(' ').strip('$'))  # both '_from' & '_to' prices available
                price_to = int(strings[1].strip(' ').strip('$'))

        elements = row[:4]  # only take first 4 elements, discard last element if exists
        elements.append(price_from)
        elements.append(price_to)
        elements[0] = datetime.datetime.strptime(elements[0], '%b-%d-%y').strftime('%Y-%m-%d') # convert date format
        data = dict(zip(headers, elements))
        analyst_price_targets.append(data)

    return analyst_price_targets


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from pandas.api.types import CategoricalDtype
from loadData import preProcess, filterMedalsOnly

# load data
hun_df = preProcess('./data/athlete_events.csv')

plt.figure(figsize=(9, 6))
ax = sns.scatterplot(x="Height",
y="Weight",
data=hun_df,
hue='Sex',
edgecolor='none',
s=20,
alpha=0.5,
palette={"Male": "#18a1cd", "Female":"#fa8c00"})

plt.title('Height and weight of Hungarian athletes', size=16, pad=20, weight='heavy')
ax.set_xlabel('Height (cm)', size=14, labelpad=10)
ax.set_ylabel('Weight (kg)', size=14, labelpad=10)
plt.tight_layout()
plt.show()


import argparse
import torch.utils.data
import random
import time
import numpy as np
from pydoc import locate
import scipy.misc
import csv
import copy
import os
import option
import models
import datasets
import utils
from point_cloud import Depth2BEV
from tqdm import tqdm

opt = option.make(argparse.ArgumentParser())

d = datasets.IntPhys(opt, 'paths_test')
indices = list(range(9999))
#indices = list(range(1200,1300))
valLoader = torch.utils.data.DataLoader(
    d,
    1,
    num_workers=opt.nThreads,
    sampler=datasets.SubsetSampler(indices)
)
opt.nbatch_val = len(valLoader)
print(opt)
np.random.seed(opt.manualSeed)
random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
if opt.gpu:
    torch.cuda.manual_seed_all(opt.manualSeed)

model = locate('models.%s' %opt.model)(opt)
if opt.load:
    model.load(opt.load, 'test')
if opt.gpu:
    model.gpu()

print('n parameters: %d' %sum([m.numel() for m in model.parameters()]))

def get_visibility_probabilities(tp_dets, fn_dets, vg, cam, ball_radius=17.5):
    """
    Args:
        tp_dets: List of (i,j,k) pixel coordinates in [250,348,35] of TP dets for an image
        fn_dets: List of (i,j,k) pixel coordinates in [250,348,35] of FN dets (gt objects
            missed by the detector)
        vg: The visibility grid with dims [1/35,250,348]
        cam: the coordinates of the camera in [250,348]
    Return:
        tp_probs, fn_probs: Lists of probabilities
    """
    # c, h, w = vg.shape
    h, w = vg.shape
    def shifted(cam_x, cam_y, obj_x, obj_y):
        if cam_y == obj_y:
            theta = np.pi/2
        else:
            theta = np.arctan((cam_x - obj_x) / (cam_y - obj_y))
        xa = ball_radius * np.cos(theta)
        yb = ball_radius * np.sin(theta)
        if obj_y <= cam_y:
            y = obj_y + yb
        else:
            y = obj_y - yb
        x = obj_x - xa
        x = min(max(int(np.floor(x)), 0), h-2)
        y = min(max(int(np.floor(y)), 0), w-2)
        return x,y

    def average(x, y, vg_):
        vg_ = vg_.squeeze()
        u = vg_[x, y+1].item()
        d = vg_[x, y-1].item()
        l = vg_[x-1, y].item()
        r = vg_[x+1, y].item()
        return np.mean(np.array([u, d, l, r, vg_[x,y].item()]))

    tp_probs, fn_probs = [], []
    for t in tp_dets:
        if type(t) == tuple:
            t = t[1]
        x = t[0]
        y = t[1]
        if type(x) == torch.Tensor:
            x = x.item()
        if type(y) == torch.Tensor:
            y = y.item()
        # shift towards camera by ball radius
        x, y = shifted(cam[0].item(), cam[1].item(), x, y)
        # take average of neighborhood
        #tp_probs.append(average(x,y,vg[t[2]]))
        tp_probs.append(average(x,y,vg))
    for f in fn_dets:
        x = f[0]
        y = f[1]
        if type(x) == torch.Tensor:
            x = x.item()
        if type(y) == torch.Tensor:
            y = y.item()
        # shift towards camera by ball radius
        x, y = shifted(cam[0].item(), cam[1].item(), x, y)
        #fn_probs.append(average(x,y,vg[f[2]]))
        fn_probs.append(average(x,y,vg))
    return tp_probs, fn_probs

def get_proximity(source_obj, idx, other_obj, other_occluders, cam,
        thresholds=[10,20,40]):
    """
    Given a source object, compute the minimum angle to
    the other objects and occluders.
    Args:
        source_obj: (i,j,k) tuple
        other_obj: List of (i,j,k) tuples (potentially empty)
        other_occluders: List of (i,j,k) tuples (potentially empty)
    """
    cam = (cam[0].item(), cam[1].item())
    def angle_to_cam(obj, cam):
        if cam[1] - obj[1] == 0:
            return 0
        return np.rad2deg(np.arctan((cam[0] - obj[0])/(cam[1] - obj[1])))

    angles = []
    if type(source_obj[0]) == torch.Tensor:
        source_obj[0] = source_obj[0].item()
        source_obj[1] = source_obj[1].item()
    src_obj_angle_to_cam = angle_to_cam(source_obj, cam)
    for idx_, o in enumerate(other_obj):
        if type(o[0]) == torch.Tensor:
            o[0] = o[0].item()
            o[1] = o[1].item()
        if idx == idx_:
            continue
        if (source_obj[0]-o[0]) == 0 and (source_obj[1]-o[1]) == 0:
            continue
        a = angle_to_cam(o, cam)
        angles.append(abs(src_obj_angle_to_cam - a))
    for o in other_occluders:
        if type(o[0]) == torch.Tensor:
            o[0] = o[0].item()
            o[1] = o[1].item()
        a = angle_to_cam(o, cam)
        angles.append(abs(src_obj_angle_to_cam - a))

    if len(angles) == 0:
        return -1
    return min(angles)

    #min_angle = min(angles)
    #if min_angle <= thresholds[0]:
    #    return thresholds[0]
    #elif min_angle <= thresholds[1]:
    #    return thresholds[1]
    #elif min_angle <= thresholds[2]:
    #    return thresholds[2]
    #else:
    #    return -1

overlap_ratios = [1.0]
conf_threshs = [0.9]
ball_radii = [opt.ball_radius]

outfile = open(os.path.join(opt.results, 'spatial_prior_experiment_results.csv'), 'w+')
fieldnames = ['frame', 'tp', 'detector_conf', 'prior_conf', 'angular_proximity', 'num_objects', 'num_occluders', 'distance_to_camera']
csv_writer = csv.DictWriter(outfile, fieldnames=fieldnames)
csv_writer.writeheader()

viz = utils.Viz(opt)
model.eval()
#i = indices[0]
n = 0
for radius in ball_radii:
    opt.ball_radius = radius
    model.bev_pixor.ball_radius = radius
    model.fv_pixor.ball_radius = radius
    for overlap_ratio in overlap_ratios:
        for conf_th in conf_threshs:
            opt.conf_thresh = conf_th
            model.bev_pixor.conf_thresh = conf_th
            model.fv_pixor.conf_thresh = conf_th
            for data in tqdm(valLoader):
                if not opt.use_occluded:
                    pc = data[0]['point_cloud'].squeeze()
                    vg = data[0]['VG'].squeeze()
                    i = data[0]['frame'][0]
                    detections, dets_px, bev_scores, fv_scores = model.predict(data, d.depth2bev)
                    objects = data[1]['objects']
                    gt_balls_px = copy.deepcopy(data[1]['objects_px']['balls'])
                    gt_walls_px = copy.deepcopy(data[1]['objects_px']['walls'])
                    found_objs = 0
                    tp_objs = []
                    for det, det_px in zip(detections, dets_px):
                        j = -1; found = False
                        for obj in objects:
                            obj = obj.numpy()[0]
                            j += 1
                            dist = np.linalg.norm(obj - det)
                            if dist/(overlap_ratio * 2 * opt.ball_radius) <= 1.:
                                found = True
                                found_objs += 1
                                break
                        if found:
                            del objects[j]
                            # get the index into the gt objs
                            # TODO: test
                            for idx, b in enumerate(data[1]['objects_px']['balls']):
                                if gt_balls_px[j] == b:
                                    break
                            del gt_balls_px[j]
                            tp_objs.append((idx,det_px))
                    # gathering statistics for experiment below
                    num_objects = len(data[1]['objects_px']['balls'])
                    num_occluders = len(gt_walls_px)
                    # either there are 2+ objects or there is at least 1 object and 1 occluder
                    if num_objects <= 1 and num_occluders == 0:
                        continue
                    tp_det_probs, fn_det_probs = [], []
                    tp_min_angle_cat, fn_min_angle_cat = [], []
                    tp_cam_dist, fn_cam_dist = [], []
                    # true positives
                    for (idx,o) in tp_objs:
                        tp_det_probs.append(o[3].item())
                        tp_min_angle_cat.append(get_proximity(
                            o[:3], idx, data[1]['objects_px']['balls'],
                            gt_walls_px, data[0]['cam'], thresholds=opt.angular_thresh))
                        tp_cam_dist.append(np.sqrt(np.square((o[0] - data[0]['cam'][0])) + np.square(o[1] - data[0]['cam'][1])).item())
                    # false negatives
                    for k in range(len(gt_balls_px)):
                        a, b = gt_balls_px[k][0], gt_balls_px[k][1]
                        a_ = int(np.floor(a / 4)) # TODO
                        b_ = int(np.floor(b / 4)) # TODO
                        bev_score = bev_scores[a_, b_]
                        fn_det_probs.append(bev_score)
                        fn_min_angle_cat.append(get_proximity(
                            gt_balls_px[k], -1, data[1]['objects_px']['balls'],
                            gt_walls_px, data[0]['cam'], thresholds=opt.angular_thresh))
                        fn_cam_dist.append(np.sqrt(np.square((a - data[0]['cam'][0])) + np.square(b - data[0]['cam'][1])).item())
                    tp_vg_probs, fn_vg_probs = get_visibility_probabilities(
                            tp_objs, gt_balls_px, vg, data[0]['cam'])
                    for tp_det_prob, tp_vg_prob, tp_angle, tp_cam in zip(tp_det_probs, tp_vg_probs, tp_min_angle_cat, tp_cam_dist):
                        #if tp_angle == -1:
                        #    tp_angle = '{}'.format(1+opt.angular_thresh[-1])
                        csv_writer.writerow({'frame': i, 'tp': 1, 'detector_conf': round(tp_det_prob,3),
                            'prior_conf': round(tp_vg_prob,3), 'angular_proximity': round(tp_angle,3),
                            'num_objects': num_objects, 'num_occluders': num_occluders, 'distance_to_camera': round(tp_cam, 3)})
                    for fn_det_prob, fn_vg_prob, fn_angle, fn_cam in zip(fn_det_probs, fn_vg_probs, fn_min_angle_cat, fn_cam_dist):
                        #if fn_angle == -1:
                        #    fn_angle = '{}'.format(1+opt.angular_thresh[-1])
                        csv_writer.writerow({'frame': i, 'tp': 0, 'detector_conf': round(fn_det_prob,3),
                            'prior_conf': round(fn_vg_prob,3), 'angular_proximity': round(fn_angle,3),
                            'num_objects': num_objects, 'num_occluders': num_occluders, 'distance_to_camera': round(fn_cam, 3)})
                    if opt.image_save:
                        #start = time.time()
                        #Depth2BEV.display_point_cloud(pc, '3d', detections, opt.ball_radius, True, name='eval_imgs/{}.png'.format(i))
                        #diff = time.time() - start
                        #print("display pc time: {}".format(diff))
                        for o in tp_objs:
                            vg_ = np.zeros((vg.shape[2], vg.shape[3], 3))
                            for a in range(vg.shape[2]):
                                for b in range(vg.shape[3]):
                                    vg_[a,b,:] = np.array([0, vg[o[2], a, b] * 255, 0])

                            vg_[o[0]-5:o[0]+5, o[1]-5:o[1]+5] = np.array([255, 0, 0])
                            vg_resized = scipy.misc.imresize(vg_, 200)
                            scipy.misc.imsave('eval_imgs/vg/{}.png'.format(n), vg_resized)
                            n += 1
                else:
                    vg = data[0]['VG'].squeeze()
                    i = data[0]['frame'][0]
                    gt_balls_px = copy.deepcopy(data[1]['objects_px']['balls'])
                    gt_walls_px = copy.deepcopy(data[1]['objects_px']['walls'])
                    # gathering statistics for experiment below
                    num_objects = len(data[1]['objects_px']['balls'])
                    num_occluders = len(gt_walls_px)
                    # either there are 2+ objects or there is at least 1 object and 1 occluder
                    if num_objects <= 1 and num_occluders == 0:
                        continue
                    _, vg_probs = get_visibility_probabilities([], gt_balls_px, vg, data[0]['cam'])
                    for k in range(len(gt_balls_px)):
                        a, b = gt_balls_px[k][0], gt_balls_px[k][1]
                        a_ = int(np.floor(a / 4)) # TODO
                        b_ = int(np.floor(b / 4)) # TODO
                        min_angle = get_proximity(
                            gt_balls_px[k], -1, data[1]['objects_px']['balls'],
                            gt_walls_px, data[0]['cam'])
                        cam_dist = np.sqrt(np.square((a - data[0]['cam'][0])) + np.square(b - data[0]['cam'][1])).item()
                        csv_writer.writerow({'frame': i, 'tp': -1, 'detector_conf': -1,
                            'prior_conf': round(vg_probs[k],3), 'angular_proximity': round(min_angle,3),
                            'num_objects': num_objects, 'num_occluders': num_occluders, 'distance_to_camera': round(cam_dist, 3)})

                #i += 1

outfile.close()
print('Done')


#coding=utf-8
import os
import sys
import logging
import torch
from torch.autograd import Variable
from visdom import Visdom

logging.basicConfig(level=logging.INFO, format='[%(asctime)s %(levelname)s] %(message)s')

class classifyexperiment():
	def __init__(self, home_dir, net, max_epoch, criterion, optimizer, scheduler, snapshot=0, use_cuda=True, visdomName=None):
		self.home_dir = home_dir
		self.net = net
		self.max_epoch = max_epoch
		self.criterion = criterion
		self.optimizer = optimizer
		self.scheduler = scheduler
		self.snapshot = snapshot
		self.use_cuda = use_cuda
		self.init_visdom(visdomName)

	def init_visdom(self, visdomName):
		if visdomName is None:
			self.viz = Visdom(env=os.path.basename(self.home_dir))
		else:
			self.viz = Visdom(env=visdomName)
		if self.viz.check_connection():
			self.viz.line(X=torch.FloatTensor([0]), Y=torch.FloatTensor([0]), win='train_loss_batch', opts={'title': 'train_loss_batch'})
			self.viz.line(X=torch.FloatTensor([0]), Y=torch.FloatTensor([0]), win='train_loss_epoch', opts={'title': 'train_loss_epoch'})
			self.viz.line(X=torch.FloatTensor([0]), Y=torch.FloatTensor([0]), win='train_acc_batch', opts={'title': 'train_acc_batch'})
			self.viz.line(X=torch.FloatTensor([0]), Y=torch.FloatTensor([0]), win='train_acc_epoch', opts={'title': 'train_acc_epoch'})
			self.viz.line(X=torch.FloatTensor([0]), Y=torch.FloatTensor([0]), win='val_acc_epoch', opts={'title': 'val_acc_epoch'})

	def set_dataloader(self, classes, train_dataloader=None, val_dataloader=None):
		self.train_dataloader = train_dataloader
		self.val_dataloader = val_dataloader
		self.classes = classes

	def get_net(self):
		return self.net

	def run(self):
		# 运行模型
		for epoch in range(0, self.max_epoch):
			self.scheduler.step()
			self.train(epoch)
			# 清除部分无用变量
			torch.cuda.empty_cache()
			if self.val_dataloader is not None:
				self.val(epoch)
				# 清除部分无用变量
				torch.cuda.empty_cache()
			if self.snapshot != 0 and epoch % self.snapshot == 0:
				self.save(epoch)

		if self.snapshot == 0 or self.max_epoch % self.snapshot != 0:
			self.save(self.max_epoch)

	def train(self, epoch):
		self.net.train(True)
		train_loss = 0.0
		correct = 0.0
		total = 0

		# batch
		batch_len = len(self.train_dataloader)
		for batch_idx, (inputs, labels) in enumerate(self.train_dataloader):
			if self.use_cuda:
				inputs, labels = inputs.cuda(), labels.cuda()

			# zero optimizer grad
			self.optimizer.zero_grad()
			# Variable
			inputs, labels = Variable(inputs), Variable(labels)
			outputs = self.net(inputs)
			loss = self.criterion(outputs, labels)
			# loss backward
			loss.backward()
			# update
			self.optimizer.step()
			train_loss += loss.data
			# statist
			_, predicted = torch.max(outputs.data, 1)
			total += labels.size(0)
			correct += predicted.eq(labels.data).cpu().sum()
			batch_acc = 100. * predicted.eq(labels.data).cpu().sum() / labels.size(0)
			sys.stdout.write('\rTrain || Batch_id: %d/%d | Loss: %.3f | Acc: %.3f%% (%d/%d)'
							 % (batch_idx, batch_len, train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
			if self.viz.check_connection():
				self.viz.line(X=torch.FloatTensor([epoch + 1. * batch_idx / batch_len])
							  , Y=torch.FloatTensor([batch_acc]), win='train_acc_batch', update='append')
				self.viz.line(X=torch.FloatTensor([epoch + 1. * batch_idx / batch_len])
							  , Y=torch.FloatTensor([loss.data]), win='train_loss_batch', update='append')
		sys.stdout.write('\n')
		sys.stdout.flush()
		logging.info('Epoch: %d || Train | lr: %f | Loss: %.3f | Acc: %.3f%% (%d/%d)'
				 % (epoch, self.scheduler.get_lr()[0], train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
		if self.viz.check_connection():
			self.viz.line(X=torch.FloatTensor([epoch])
						  , Y=torch.FloatTensor([100. * correct / total]), win='train_acc_epoch', update='append')
			self.viz.line(X=torch.FloatTensor([epoch])
						  , Y=torch.FloatTensor([train_loss / (batch_idx + 1)]), win='train_loss_epoch', update='append')

	def val(self, epoch):
		self.net.train(False)
		test_loss = 0.0
		correct = 0.0
		total = 0

		# batch
		batch_len = len(self.val_dataloader)
		for batch_idx, (inputs, labels) in enumerate(self.val_dataloader):
			if self.use_cuda:
				inputs, labels = inputs.cuda(), labels.cuda()

			# Variable
			inputs, labels = Variable(inputs), Variable(labels)
			outputs = self.net(inputs)
			loss = self.criterion(outputs, labels)
			test_loss += loss.data
			# statist
			_, predicted = torch.max(outputs.data, 1)
			total += labels.size(0)
			correct += predicted.eq(labels.data).cpu().sum()
			sys.stdout.write('\rTest || Batch_id: %d/%d | Loss: %.3f | Acc: %.3f%% (%d/%d)'
							 % (batch_idx, batch_len, test_loss / (batch_idx + 1), 100. * correct / total, correct, total))
		sys.stdout.write('\n')
		sys.stdout.flush()
		logging.info('Epoch: %d || Test | Loss: %.3f | Acc: %.3f%% (%d/%d)'
			% (epoch, test_loss/(batch_idx+1), 100. * correct / total, correct, total))
		if self.viz.check_connection():
			self.viz.line(X=torch.FloatTensor([epoch])
						  , Y=torch.FloatTensor([100. * correct / total]), win='val_acc_epoch', update='append')

	def save(self, epoch):
		if not os.path.exists(self.home_dir):
			os.makedirs(self.home_dir)
		state = {'net': self.net.state_dict(), 'optimizer': self.optimizer.state_dict(), 'epoch': epoch
			, 'classes': self.classes}
		torch.save(state, os.path.join(self.home_dir, '{}.pkl'.format(epoch)))


from django.conf.urls.defaults import *
from django.views.generic import RedirectView
import views

urlpatterns = patterns('',
    (r'^no_template_view/$', views.no_template_view),
    (r'^staff_only/$', views.staff_only_view),
    (r'^get_view/$', views.get_view),
    (r'^request_data/$', views.request_data),
    (r'^request_data_extended/$', views.request_data, {'template':'extended.html', 'data':'bacon'}),
    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    (r'^login_protected_redirect_view/$', views.login_protected_redirect_view),
    (r'^redirects/$', RedirectView.as_view(url='/test_client_regress/redirects/further/')),
    (r'^redirects/further/$', RedirectView.as_view(url='/test_client_regress/redirects/further/more/')),
    (r'^redirects/further/more/$', RedirectView.as_view(url='/test_client_regress/no_template_view/')),
    (r'^redirect_to_non_existent_view/$', RedirectView.as_view(url='/test_client_regress/non_existent_view/')),
    (r'^redirect_to_non_existent_view2/$', RedirectView.as_view(url='/test_client_regress/redirect_to_non_existent_view/')),
    (r'^redirect_to_self/$', RedirectView.as_view(url='/test_client_regress/redirect_to_self/')),
    (r'^circular_redirect_1/$', RedirectView.as_view(url='/test_client_regress/circular_redirect_2/')),
    (r'^circular_redirect_2/$', RedirectView.as_view(url='/test_client_regress/circular_redirect_3/')),
    (r'^circular_redirect_3/$', RedirectView.as_view(url='/test_client_regress/circular_redirect_1/')),
    (r'^set_session/$', views.set_session_view),
    (r'^check_session/$', views.check_session_view),
    (r'^request_methods/$', views.request_methods_view),
    (r'^check_unicode/$', views.return_unicode),
    (r'^parse_unicode_json/$', views.return_json_file),
    (r'^check_headers/$', views.check_headers),
    (r'^check_headers_redirect/$', RedirectView.as_view(url='/test_client_regress/check_headers/')),
    (r'^raw_post_data/$', views.raw_post_data),
    (r'^read_all/$', views.read_all),
    (r'^read_buffer/$', views.read_buffer),
    (r'^request_context_view/$', views.request_context_view),
)