from app import app,controller
from flask import render_template,url_for,request,redirect
from werkzeug import secure_filename
from .machine import main
import os

app.config['UPLOAD_FOLDER'] = "./server/app/static/temp"
args = {}

@app.route('/')
def index():
    return render_template("welcome.html")

@app.route('/clear')
def clear():
	#clean temp
	dest = app.config['UPLOAD_FOLDER']
	filelist = os.listdir(dest)
	for fn in filelist:
		os.remove(dest+'/'+fn)
	return render_template("welcome.html")

@app.route('/success')
def success():
    return render_template("success.html",data=args)

@app.route('/error')
def error():
    return render_template("error.html",data=args)

@app.route('/merge',methods=['GET','POST'])
def merge():
	data = {
		'basepath':app.config['UPLOAD_FOLDER'],
		'imgname' : 'noimage.png',
		'encrypt':False,
		'has_error':False,
		'key':'kunci'
	}

	if request.method == 'POST':
		dest = app.config['UPLOAD_FOLDER']

		#clean temp
		filelist = os.listdir(dest)
		for fn in filelist:
			os.remove(dest+'/'+fn)

		data['key'] = request.form["password"]

		# super encrypt
		if request.form.getlist("reqpassword"):
			data['encrypt'] = True

		imgfile = request.files["image"]
		txtfile = request.files['message']
		if controller.upload(imgfile,dest,'img') and controller.upload(txtfile,dest,'txt') :
			data['imgname'] =  secure_filename(imgfile.filename)
			data['txtname'] =  secure_filename(txtfile.filename)
			args ['imgname'] = 'http://127.0.0.1:1337'+url_for('static',filename='temp/'+data['imgname'])
			main.merge(data)
			args['act'] ='merge'
			return redirect(url_for('success'))
		else:
			args['error'] = 'Extensi File Gambar atau Text Salah!'
			return redirect(url_for('error'))

	else:
		return render_template("merge.html")

@app.route('/expand',methods=['GET','POST'])
def expand():
	data = {
		'basepath':app.config['UPLOAD_FOLDER'],
		'imgname' : 'noimage.png',
		'encrypt':False,
		'has_error':False,
		'key':'kunci'
	}
	if request.method == 'POST':
		dest = app.config['UPLOAD_FOLDER']

		#clean temp
		filelist = os.listdir(dest)
		for fn in filelist:
			os.remove(dest+'/'+fn)

		data['key'] = request.form["password"]
		# super encrypt
		if request.form.getlist("reqpassword"):
			data['encrypt'] = True
		imgfile = request.files["image"]

		if controller.upload(imgfile,dest,'img'):
			data['imgname'] =  secure_filename(imgfile.filename)
			args ['imgname'] = 'http://127.0.0.1:1337'+url_for('static',filename='temp/'+data['imgname'])
			data['newname'] = 'pesan'
			filename =  main.expand(data)
			args['txtname'] =	'http://127.0.0.1:1337'+url_for('static',filename='temp/'+filename)
			args['act'] ='expand'
			return redirect(url_for('success'))
		else:
			args['error'] = 'Extensi File Gambar Salah!'
			return redirect(url_for('error'))
	else:
		return render_template("expand.html")


#libraries
import sqlite3
import logging
import aiogram.utils.markdown as md
import keyboard as keyboard
import datetime
import calendar
import lxml
import requests
import json
import time
import random
import asyncio
import aioschedule

from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher, FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.utils import executor
from aiogram.types import ReplyKeyboardRemove, ReplyKeyboardMarkup, KeyboardButton, InlineKeyboardMarkup, InlineKeyboardButton, ParseMode
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.utils.helper import Helper, HelperMode, ListItem
from config import bot_token, api_key
from datetime import date
from bs4 import BeautifulSoup
from markdown import markdown
#Logging
logging.basicConfig(level=logging.INFO)
#States
class States(StatesGroup):
    group = State()
    on_off = State()
    main = State()
    setting = State()
    profile = State()
#Bot object
bot= Bot(token=bot_token)
#Bot dispetcher
dp = Dispatcher(bot, storage=MemoryStorage())
#Weekday and date
today_day = datetime.datetime.today().weekday()
days_naming = ["Понедельник","Вторник","Среда","Четверг","Пятница","Суббота","Воскресенье"]
days_naming_en = ["monday", "tuesday", "wednesday", "thursday", "friday","saturday", "sunday"]
today = date.today()
calendar.day_name[today.weekday()]
next_day = today_day + 1
#users database
conn = sqlite3.connect('users_database.db')
cur = conn.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS users(user_id INTEGER, group_number TEXT, notify_times TEXT)')
#Function
@dp.message_handler(commands='start')
async def start(message : types.Message):
    await message.answer(
        'Добро пожаловать в petroshedulebot, мои создатели:\nАверин Андрей\nПрохоров Евгений\nБерозко Роман\n\nCтуденты группы 39-55',
        reply_markup=keyboard.button_register
        )
    await message.answer(f'Сегодня: {today}\n{days_naming[today_day]}')
    print (next_day)

@dp.message_handler(text=['Регистрация'])
async def register(message: types.Message):
    await bot.send_message(message.from_user.id,'Кто ты?', reply_markup=keyboard.button_who)

@dp.message_handler(text=['Я студент'])
async def student_register(message: types.Message):
    await States.group.set()
    await bot.send_message(message.from_user.id, 'Напиши номер своей группы')

@dp.message_handler(state=States.group)
async def group_number(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        global group
        data['group'] = message.text
        await bot.send_message(message.chat.id,
        md.text(md.text('Ваша группа:',
        md.bold(data['group']))),
        parse_mode=ParseMode.MARKDOWN_V2,
        reply_markup= keyboard.button_go_main
        )
        grouper = md.text(md.text(md.bold(data['group'])))
        data = markdown(grouper)
        group = ''.join(BeautifulSoup(data).findAll(text=True))
        print(group)
        conn = sqlite3.connect('users_database.db')
        cur = conn.cursor()
        cur.execute(f'INSERT INTO users VALUES("{message.from_user.id}","{group}")')
        conn.commit()
        await state.finish()

@dp.message_handler(text=['Я преподаватель'])
async def teacher_register(message: types.Message):
    await bot.send_message(
        message.from_user.id,
        'Данная функция находится в разработке, пожалуйста попробуйте позднее',
        )

@dp.message_handler(text=['Настройки'])
async def setting(message: types.Message):
    await message.answer('Добро пожаловать в меню настроек',
    reply_markup=keyboard.button_notify,
    )

@dp.message_handler(text=['Сменить группу'])
async def rewrite (message: types.Message):
    con = sqlite3.connect('users_database.db')
    cur = con.cursor()
    cur.execute(f'DELETE from users WHERE user_id = "{message.from_user.id}"')
    con.commit()
    cur.close()
    await States.group.set()
    await message.answer('Группа успешно сброшена\nВведите новый номер группы:')

@dp.message_handler(text=['Время уведомлений'])
async def time_quest (message: types.Message):
    await message.answer('Функция в разработке')
#    await States.setting.set()

@dp.message_handler(text=['Вкл/Выкл уведомлений'])
async def time_set (message: types.Message):
 #  await States.on_off.set()
 #  await message.answer('Состояние успешно изменено, текущее состояние:\n')
    await message.answer('Функция в разработке')

@dp.message_handler(state=States.setting)
async def times_setting_set (message: types.Message, state = FSMContext):
    async with state.proxy() as times:
        times['times_set'] = message.text
        await bot.send_message(message.chat.id,
        md.text(md.text('Установленное время:',
        md.bold(times['times_set']))),
        parse_mode=ParseMode.MARKDOWN_V2,
        reply_markup= keyboard.button_go_main
        )
        timers = md.text(md.text(md.bold(times['times_set'])))
        rework = markdown(timers)
        setted_time = ''.join(BeautifulSoup(rework).findAll(text=True))
        connect = sqlite3.connect('users_database.db')
        cur = connect.cursor()
        cur.execute(f'INSERT INTO users VALUES WHERE user_id = "{message.from_user.id}"("{setted_time}")')
        connect.commit()
        cut.close()

@dp.message_handler(text=['Получить расписание'])
async def schedule_menu(message: types.Message):
    await message.answer('Выберите на какой день получить расписание',
    reply_markup=keyboard.button_schedule_choise,
    )

@dp.message_handler(text=['Расписание на сегодня'])
async def schedule_today(message: types.Message):
    await message.answer(f'Сегодня: {days_naming[today_day]}')
    con = sqlite3.connect('users_database.db')
    cur = con.cursor()
    cur.execute(f'SELECT * FROM users WHERE user_id = "{message.from_user.id}"')
    res = cur.fetchall()
    response = requests.get(f'https://petrocol.ru/schedule/{[list(res[0])[1]][0]}?json=1&key={api_key}')
    all_schedule = json.loads(response.text)
    try:
        schedule = all_schedule["schedule"][days_naming_en[today_day]]
    except:
        await message.answer('Нет данных о парах на сегодня, попробуйте посмотреть на сайте:\nhttps://portal.petrocollege.ru/Pages/responsiveSh-aspx.aspx')
    else:
        for i in range(len(schedule)+1):
            try:
                lesson_json = schedule[str(i+1)][0]
                lesson = lesson_json['lesson']
                teacher = lesson_json['teacher']
                classroom = lesson_json['classroom']
                period = f'Пара {i+1}:\n {lesson}, {teacher}, {classroom}\n'
                reply_message = f'{period}'
                await bot.send_message(message.from_user.id, reply_message)
            except:
                pass

@dp.message_handler(text=['Расписание на завтра'])
async def schedule_next_day(message: types.Message):
    await message.answer(f'Завтра: {days_naming[next_day]}')
    con = sqlite3.connect('users_database.db')
    cur = con.cursor()
    cur.execute(f'SELECT * FROM users WHERE user_id = "{message.from_user.id}"')
    res = cur.fetchall()
    response = requests.get(f'https://petrocol.ru/schedule/{[list(res[0])[1]][0]}?json=1&key={api_key}')
    all_schedule = json.loads(response.text)
    try:
        schedule = all_schedule["schedule"][days_naming_en[next_day]]
    except:
        await message.answer('Нет данных о парах на завтра, попробуйте посмотреть на сайте:\nhttps://portal.petrocollege.ru/Pages/responsiveSh-aspx.aspx')
    else:
        for i in range(len(schedule)+1):
            try:
                lesson_json = schedule[str(i+1)][0]
                lesson = lesson_json['lesson']
                teacher = lesson_json['teacher']
                classroom = lesson_json['classroom']
                period = f'Пара {i+1}:\n {lesson}, {teacher}, {classroom}\n'
                reply_message = f'{period}'
                await bot.send_message(message.from_user.id, reply_message)
            except:
                pass
#                await message.answer('Нет данных о парах на завтра, попробуйте посмотреть на сайте:\nhttps://portal.petrocollege.ru/Pages/responsiveSh-aspx.aspx')

@dp.message_handler(text=['zxc'])
async def zxc(message: types.Message):
    await message.answer('Ты в муте, бездарь')
    await bot.send_photo(message.from_user.id, 'https://i.pinimg.com/736x/17/b9/09/17b909bc23b89a24c163cdde88025aed.jpg')

@dp.message_handler(text=['Перейти в главное меню', 'Вернутся в главное меню', 'Назад'])
async def main_menu (message: types.Message):
    await bot.send_message(message.from_user.id,
        'Добро пожаловать в главное меню, здесь вы можете настроить автоматическое получение получение расписания или получить его вручную',
        reply_markup=keyboard.button_main,
        )

@dp.message_handler(text=['Мой профиль'])
async def get_profile(message: types.Message):
    conn = sqlite3.connect('users_database.db')
    cur = conn.cursor()
    cur.execute(f'SELECT * FROM users WHERE user_id = "{message.from_user.id}"')
    result = cur.fetchall()
    await bot.send_message(message.from_user.id, f'ID = {list(result[0])[0]}\nGroup = {[list(result[0])[1]][0]}', reply_markup=keyboard.btn_back)

@dp.message_handler(text=['Помощь'])
async def user_help (message: types.Message):
    photo = ['https://sun9-45.userapi.com/impg/L_ZjDqZoxr0-Ps0fQi0-c48PjJ-UWJk64exZqw/HrcqjPtfIjE.jpg?size=840x737&quality=96&sign=e78ad3ba428e817729e80c0f02d249df&type=album',
    'https://sun9-30.userapi.com/impg/S-bSLtCaDlC1bcUwMCDlCAyzerrNVqFgw5Ygpg/BoLXpwQ1HcY.jpg?size=608x770&quality=96&sign=e101f67bcaa95f1aec2d52a651d24cef&type=album',
    'https://sun9-64.userapi.com/impg/v9TI88OR_8UV_CJ2u2FRJlSjFiRhpoh_lFKSFg/jPCHxw5V4WQ.jpg?size=1125x1077&quality=96&sign=b62f3c74fd48e7831d66add4eb792715&type=album',
    ]
    await message.answer('Не готово...', reply_markup = keyboard.btn_back)
    await bot.send_photo(message.from_user.id, photo[random.randint(0,2)])

if __name__=='__main__':
    executor.start_polling(dp, skip_updates=True,)


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/runner/work/PyTplot/PyTplot/pytplot/QtPlotter/TVarFigureSpec.py
# Compiled at: 2020-04-24 11:05:41
# Size of source mod 2**32: 17428 bytes
import pyqtgraph as pg, numpy as np
from .. import tplot_utilities
from pyqtgraph.Qt import QtCore
import pytplot
import CustomAxis.DateAxis as DateAxis
import CustomImage.UpdatingImage as UpdatingImage
import CustomImage.ColorbarImage as ColorbarImage
import CustomAxis.BlankAxis as BlankAxis
from CustomLegend.CustomLegend import CustomLegendItem
import CustomAxis.AxisItem as AxisItem
import CustomViewBox.NoPaddingPlot as NoPaddingPlot
import CustomLinearRegionItem.CustomLinearRegionItem as CustomLinearRegionItem

class TVarFigureSpec(pg.GraphicsLayout):

    def __init__(self, tvar_name, show_xaxis=False):
        self.tvar_name = tvar_name
        self.show_xaxis = show_xaxis
        self.crosshair = pytplot.tplot_opt_glob['crosshair']
        pg.GraphicsLayout.__init__(self)
        self.layout.setHorizontalSpacing(10)
        self.layout.setContentsMargins(0, 0, 0, 0)
        self.show_xaxis = show_xaxis
        if 'show_all_axes' in pytplot.tplot_opt_glob:
            if pytplot.tplot_opt_glob['show_all_axes']:
                self.show_xaxis = True
        else:
            if self.show_xaxis:
                self.xaxis = DateAxis(orientation='bottom')
                self.xaxis.setHeight(35)
                self.xaxis.enableAutoSIPrefix(enable=False)
            else:
                self.xaxis = DateAxis(orientation='bottom', showValues=False)
                self.xaxis.setHeight(0)
                self.xaxis.enableAutoSIPrefix(enable=False)
            self.yaxis = AxisItem('left')
            self.yaxis.setWidth(100)
            vb = NoPaddingPlot()
            self.plotwindow = self.addPlot(row=0, col=0, axisItems={'bottom':self.xaxis,  'left':self.yaxis}, viewBox=vb)
            self.legendvb = pg.ViewBox(enableMouse=False)
            self.legendvb.setMaximumWidth(100)
            self.legendvb.setXRange(0, 1, padding=0)
            self.legendvb.setYRange(0, 1, padding=0)
            self.addItem(self.legendvb, 0, 1)
            self.curves = []
            self.colors = self._setcolors()
            self.colormap = self._setcolormap()
            if pytplot.tplot_opt_glob['black_background']:
                self.labelStyle = {'font-size':str(pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']['char_size']) + 'pt',
                 'color':'#FFF'}
            else:
                self.labelStyle = {'font-size':str(pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']['char_size']) + 'pt',  'color':'#000'}
        self.hoverlegend = CustomLegendItem(offset=(0, 0))
        self.hoverlegend.setItem('Date:', '0')
        self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['xaxis_opt']['crosshair'] + ':', '0')
        self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['crosshair'] + ':', '0')
        self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['crosshair'] + ':', '0')
        self.hoverlegend.setVisible(False)
        self.hoverlegend.setParentItem(self.plotwindow.vb)

    @staticmethod
    def getaxistype():
        axis_type = 'time'
        link_y_axis = False
        return (axis_type, link_y_axis)

    def _set_crosshairs(self):
        self.vLine = pg.InfiniteLine(angle=90, movable=False, pen=(pg.mkPen('k')))
        self.hLine = pg.InfiniteLine(angle=0, movable=False, pen=(pg.mkPen('k')))
        self.plotwindow.addItem((self.vLine), ignoreBounds=True)
        self.plotwindow.addItem((self.hLine), ignoreBounds=True)

    def _set_roi_lines(self):
        if 'roi_lines' in pytplot.tplot_opt_glob.keys():
            roi_1 = pytplot.tplot_utilities.str_to_int(pytplot.tplot_opt_glob['roi_lines'][0])
            roi_2 = pytplot.tplot_utilities.str_to_int(pytplot.tplot_opt_glob['roi_lines'][1])
            x = pytplot.data_quants[self.tvar_name].coords['time']
            x_sub_1 = abs(x - roi_1 * np.ones(len(x)))
            x_sub_2 = abs(x - roi_2 * np.ones(len(x)))
            x_argmin_1 = np.nanargmin(x_sub_1)
            x_argmin_2 = np.nanargmin(x_sub_2)
            x_closest_1 = x[x_argmin_1]
            x_closest_2 = x[x_argmin_2]
            roi = CustomLinearRegionItem(orientation=(pg.LinearRegionItem.Vertical), values=[x_closest_1, x_closest_2])
            roi.setBrush([211, 211, 211, 130])
            roi.lines[0].setPen('r', width=2.5)
            roi.lines[1].setPen('r', width=2.5)
            self.plotwindow.addItem(roi)

    def buildfigure(self):
        self._setxrange()
        self._setyrange()
        self._setyaxistype()
        self._setzaxistype()
        self._setzrange()
        self._visdata()
        self._setxaxislabel()
        self._setyaxislabel()
        self._addlegend()
        self._addtimebars()
        self._addmouseevents()
        self._set_crosshairs()
        self._set_roi_lines()

    def _setyaxislabel(self):
        if 'axis_subtitle' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']:
            label = pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['axis_label']
            sublabel = pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['axis_subtitle']
            (self.yaxis.setLabel)(f"{label}
 {sublabel} <\\center>", **self.labelStyle)
        else:
            (self.yaxis.setLabel)((pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['axis_label']), **self.labelStyle)

    def _setxaxislabel(self):
        if self.show_xaxis:
            (self.xaxis.setLabel)((pytplot.data_quants[self.tvar_name].attrs['plot_options']['xaxis_opt']['axis_label']), **self.labelStyle)

    def getfig(self):
        return self

    def _visdata(self):
        dataset, spec_bins = pytplot.tplot_utilities.convert_tplotxarray_to_pandas_dataframe(self.tvar_name)
        specplot = UpdatingImage(dataset, spec_bins, pytplot.data_quants[self.tvar_name].attrs['plot_options']['spec_bins_ascending'], self._getyaxistype(), self._getzaxistype(), self.colormap, self.ymin, self.ymax, self.zmin, self.zmax)
        self.plotwindow.addItem(specplot)

    def _setyaxistype(self):
        if self._getyaxistype() == 'log':
            self.plotwindow.setLogMode(y=True)
        else:
            self.plotwindow.setLogMode(y=False)

    def _addlegend(self):
        zaxis = AxisItem('right')
        if 'axis_subtitle' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']:
            label = pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['axis_label']
            sublabel = pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['axis_subtitle']
            (zaxis.setLabel)(f"{label}
 {sublabel} <\\center>", **self.labelStyle)
        else:
            (zaxis.setLabel)((pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['axis_label']), **self.labelStyle)
        if self.show_xaxis:
            emptyAxis = BlankAxis('bottom')
            emptyAxis.setHeight(35)
            p2 = self.addPlot(row=0, col=1, axisItems={'right':zaxis,  'bottom':emptyAxis}, enableMenu=False, viewBox=(self.legendvb))
        else:
            p2 = self.addPlot(row=0, col=1, axisItems={'right': zaxis}, enableMenu=False, viewBox=(self.legendvb))
            p2.hideAxis('bottom')
        p2.buttonsHidden = True
        p2.setMaximumWidth(100)
        p2.showAxis('right')
        p2.hideAxis('left')
        colorbar = ColorbarImage()
        colorbar.setImage(np.array([np.linspace(1, 2, 200)]).T)
        p2.addItem(colorbar)
        p2.setLogMode(y=(self.zscale == 'log'))
        p2.setXRange(0, 1, padding=0)
        colorbar.setLookupTable(self.colormap)
        if self.zscale == 'log':
            colorbar.setRect(QtCore.QRectF(0, np.log10(self.zmin), 1, np.log10(self.zmax) - np.log10(self.zmin)))
            p2.setYRange((np.log10(self.zmin)), (np.log10(self.zmax)), padding=0)
            p2.setYRange((np.log10(self.zmin)), (np.log10(self.zmax)), padding=0)
        else:
            colorbar.setRect(QtCore.QRectF(0, self.zmin, 1, self.zmax - self.zmin))
            p2.setYRange((self.zmin), (self.zmax), padding=0)
        colorbar.setLookupTable(self.colormap)

    def _addmouseevents(self):
        if self.plotwindow.scene() is not None:
            self.plotwindow.scene().sigMouseMoved.connect(self._mousemoved)

    def _mousemoved(self, evt):
        pos = evt
        flag = 0
        if self.plotwindow.sceneBoundingRect().contains(pos):
            mousePoint = self.plotwindow.vb.mapSceneToView(pos)
            index_x = int(mousePoint.x())
            if self._getyaxistype() == 'log':
                index_y = 10 ** round(float(mousePoint.y()), 4)
            else:
                index_y = round(float(mousePoint.y()), 4)
            dataframe, specframe = pytplot.tplot_utilities.convert_tplotxarray_to_pandas_dataframe(self.tvar_name)
            x = np.asarray(dataframe.index.tolist())
            x_sub = abs(x - index_x * np.ones(len(x)))
            x_argmin = np.nanargmin(x_sub)
            x_closest = x[x_argmin]
            if len(specframe) > 1:
                try:
                    speclength = len(specframe.iloc[x_argmin])
                    y = np.asarray(specframe.iloc[x_argmin, 0:speclength - 1])
                except:
                    speclength = len(specframe.iloc[0])
                    y = np.asarray(specframe.iloc[0, 0:speclength - 1])

            else:
                speclength = len(specframe.iloc[0])
                y = np.asarray(specframe.iloc[0, 0:speclength - 1])
            y_sub = abs(y - index_y * np.ones(y.size))
            y_argmin = np.nanargmin(y_sub)
            y_closest = y[y_argmin]
            data_point = dataframe[y_argmin][x_closest]
            pytplot.hover_time.change_hover_time((int(mousePoint.x())), name=(self.tvar_name))
            if self.crosshair:
                self.vLine.setPos(mousePoint.x())
                self.hLine.setPos(mousePoint.y())
                self.vLine.setVisible(True)
                self.hLine.setVisible(True)
            date = pytplot.tplot_utilities.int_to_str(x_closest)[0:10]
            time = pytplot.tplot_utilities.int_to_str(x_closest)[11:19]
            if self.crosshair:
                self.hoverlegend.setVisible(True)
                self.hoverlegend.setItem('Date:', date)
                self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['xaxis_opt']['crosshair'] + ':', time)
                self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['crosshair'] + ':', str(y_closest))
                self.hoverlegend.setItem(pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['crosshair'] + ':', str(data_point))
        else:
            self.hoverlegend.setVisible(False)
            self.vLine.setVisible(False)
            self.hLine.setVisible(False)

    def _getyaxistype(self):
        if 'y_axis_type' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']:
            return pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['y_axis_type']
        return 'linear'

    def _setzaxistype(self):
        if self._getzaxistype() == 'log':
            self.zscale = 'log'
        else:
            self.zscale = 'linear'

    def _getzaxistype(self):
        if 'z_axis_type' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']:
            return pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['z_axis_type']
        return 'log'

    def _setcolors(self):
        if 'line_color' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']:
            return pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']['line_color']
        return pytplot.tplot_utilities.rgb_color(['k', 'r', 'seagreen', 'b', 'darkturquoise', 'm', 'goldenrod'])

    def _setcolormap(self):
        if 'colormap' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']:
            for cm in pytplot.data_quants[self.tvar_name].attrs['plot_options']['extras']['colormap']:
                return tplot_utilities.return_lut(cm)

        else:
            return tplot_utilities.return_lut('inferno')

    def _setxrange(self):
        if 'x_range' in pytplot.tplot_opt_glob:
            self.plotwindow.setXRange(pytplot.tplot_opt_glob['x_range'][0], pytplot.tplot_opt_glob['x_range'][1])

    def _setyrange(self):
        self.ymin = pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['y_range'][0]
        self.ymax = pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['y_range'][1]
        if self._getyaxistype() == 'log' and not pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['y_range'][0] <= 0:
            if pytplot.data_quants[self.tvar_name].attrs['plot_options']['yaxis_opt']['y_range'][1] <= 0:
                self.ymin = np.nanmin(pytplot.data_quants[self.tvar_name].coords['spec_bins'].values)
                self.ymax = np.nanmax(pytplot.data_quants[self.tvar_name].coords['spec_bins'].values)
            self.plotwindow.vb.setYRange((np.log10(self.ymin)), (np.log10(self.ymax)),
              padding=0)
        else:
            self.plotwindow.vb.setYRange((self.ymin), (self.ymax),
              padding=0)

    def _setzrange(self):
        if 'z_range' in pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']:
            self.zmin = pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['z_range'][0]
            self.zmax = pytplot.data_quants[self.tvar_name].attrs['plot_options']['zaxis_opt']['z_range'][1]
        else:
            dataset_temp = pytplot.data_quants[self.tvar_name].where(pytplot.data_quants[self.tvar_name] != np.inf)
            dataset_temp = dataset_temp.where(dataset_temp != -np.inf)
            if self.zscale == 'log':
                dataset_temp = dataset_temp.where(dataset_temp > 0)
            self.zmax = dataset_temp.max().max().values
            self.zmin = dataset_temp.min().min().values

    def _addtimebars(self):
        dict_length = len(pytplot.data_quants[self.tvar_name].attrs['plot_options']['time_bar'])
        for i in range(dict_length):
            date_to_highlight = pytplot.data_quants[self.tvar_name].attrs['plot_options']['time_bar'][i]['location']
            color = pytplot.data_quants[self.tvar_name].attrs['plot_options']['time_bar'][i]['line_color']
            thick = pytplot.data_quants[self.tvar_name].attrs['plot_options']['time_bar'][i]['line_width']
            infline = pg.InfiniteLine(pos=date_to_highlight, pen=pg.mkPen(color, width=thick))
            self.plotwindow.addItem(infline)


# -*- coding: utf8 -*-
import inspect
import os
import sys
import random
import time
import multiprocessing
import logging
import socket
import select
import pickle
import zlib
import unittest

BASE_TEST_PATH = os.path.join('test', 'data')
RANDOM_SEED = 42
NUM_DIRS = 10
NUM_SUBDIRS = 10
FILES_PER_DIR = 10
FILE_SIZE = 1024*8
RANDOM_DATA_BUF_SIZE = 1024*1024*4
POLL_WAIT_SECONDS = 5
FILE_DELAY = 0.5

if __package__ is None:
    # test code is run from the ./test directory.  add the parent
    # directory to the path so that we can see all the isi_ps code.
    current_file = inspect.getfile(inspect.currentframe())
    base_path = os.path.dirname(os.path.dirname(os.path.abspath(current_file)))
    sys.path.insert(0, base_path)
    from HydraWorker import HydraWorker
    from HydraClient import HydraClient
    from HydraClient import HydraClientProcess

"""
This method creates files with random data in them using a single buffer
"""
def create_files(path, num, size, buffer = None, buf_size = 1024*1024, prefix = 'file', force_overwrite = False):
  if buffer is None:
    buffer = bytearray(random.getrandbits(8) for x in xrange(buf_size))
  for i in xrange(num):
    offset = random.randrange(buf_size)
    bytes_to_write = size
    if force_overwrite is False:
      try:
        file_lstat = os.lstat(os.path.join(path, '%s%d'%(prefix, i)))
        if file_lstat.st_size == size:
          continue
      except:
        pass
    with open(os.path.join(path, '%s%d'%(prefix, i)), 'wb') as f:
      while bytes_to_write > 0:
        remainder = buf_size - offset
        if bytes_to_write < remainder:
          f.write(buffer[offset:(offset+bytes_to_write)])
          bytes_to_write = 0
        else:
          f.write(buffer[offset:buf_size])
          bytes_to_write -= remainder

class HydraTestClassSlowFileProcess(HydraWorker):
  def __init__(self, args={}):
    super(HydraTestClassSlowFileProcess, self).__init__(args)
    # Set a default delay of 0.5 seconds per file processed
    self.file_delay = FILE_DELAY

  def setFileDelay(self, delay_in_seconds):
    self.file_delay = delay_in_seconds

  def filter_subdirectories(self, root, dirs, files):
    """
    Fill in docstring
    """
    return dirs, files

  def handle_directory_pre(self, dir):
    """
    Fill in docstring
    """
    return False

  def handle_directory_post(self, dir):
    """
    Fill in docstring
    """
    return False

  def handle_file(self, dir, file):
    """
    Fill in docstring
    """
    if file == "skip_check":
      return False
    if self.file_delay > 0:
      time.sleep(self.file_delay)
    file_lstats = os.lstat(os.path.join(dir, file))
    #logging.getLogger().debug("Proc file: %s"%os.path.join(dir, file))
    return True

  def handle_extended_ops(self, data):
    if data.get('op') == 'setdelay':
      self.file_delay = data.get('payload')
    return True

'''
class HydraClientProcess(multiprocessing.Process):
  def __init__(self, args):
    super(HydraClientProcess, self).__init__()
    self.server_addr = args['svr']
    self.server_port = args['port']
    self.client = None

  def run(self):
    self.client = HydraClient(HydraTestClassSlowFileProcess)
    self.client.connect(self.server_addr, self.server_port)
    #self.client.set_workers(4)
    self.client.serve_forever()

  def shutdown(self):
    self.client.shutdown()
'''
class TestHydraClient(unittest.TestCase):
  @classmethod
  def setUpClass(cls):
    cls.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    cls.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    cls.server.bind(('127.0.0.1', 0))
    cls.server_port = cls.server.getsockname()[1]
    cls.server.listen(1)
    #print("Local port: %d"%cls.server_port)

    cls.buffer_size = RANDOM_DATA_BUF_SIZE
    random.seed(RANDOM_SEED)
    cls.base_path = os.path.dirname(os.path.dirname(os.path.abspath(current_file)))
    cls.test_path = os.path.join(cls.base_path, BASE_TEST_PATH)
    # Check for skip file named 'skip_check' and bypass creation/check if it is present
    if os.path.isfile(os.path.join(cls.test_path, 'skip_check')):
      return
    logging.getLogger().info("Setting up file structure. This may take time")
    cls.rand_buffer = bytearray(random.getrandbits(8) for x in xrange(cls.buffer_size))
    for i in xrange(NUM_DIRS):
      cur_path = os.path.join(cls.test_path, "dir%s"%i)
      try:
        os.makedirs(cur_path, exists_ok = True)
      except:
        pass
      create_files(cur_path, FILES_PER_DIR, FILE_SIZE, cls.rand_buffer, cls.buffer_size)
      for j in xrange(NUM_SUBDIRS):
        sub_path = os.path.join(cur_path, "subdir%s"%j)
        try:
          os.makedirs(sub_path)
        except:
          pass
        create_files(sub_path, FILES_PER_DIR, FILE_SIZE)

  @classmethod
  def tearDownClass(cls):
    print("tearDownClass called")
    try:
      cls.server.close()
      cls.server = None
    except:
      pass
    cls.rand_buffer = None
    cls = None

  #@unittest.skip("")
  def test_1_spawn_client_and_shutdown(self):
    client = HydraClientProcess({'svr': '127.0.0.1', 'port': self.server_port, 'file_handler': HydraTestClassSlowFileProcess})
    client.start()
    try:
      readable, _, _ = select.select([self.server], [], [], 5)
      connection, client_address = self.server.accept()
      logging.getLogger().debug('New connection from %s, Socket: %s'%(client_address, connection))
      connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
      conn_handle = connection.makefile('wb+')
      conn_handle.flush()
      logging.getLogger().debug("Sleep 2 seconds before closing connection")
      time.sleep(2)
      conn_handle.close()
      connection.close()
      logging.getLogger().debug("Waiting for shutdown up to 10 seconds")
      client.join(10)
      try:
        self.assertFalse(client.is_alive())
      except:
        client.terminate()
        raise
    except:
      if connection:
        connection.close()

  #@unittest.skip("")
  def test_2_client_process_single_dir(self):
    client = HydraClientProcess({'svr': '127.0.0.1', 'port': self.server_port, 'file_handler': HydraTestClassSlowFileProcess})
    client.start()
    try:
      readable, _, _ = select.select([self.server], [], [], 5)
      connection, client_address = self.server.accept()
      logging.getLogger().debug('New connection from %s, Socket: %s'%(client_address, connection))
      connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
      conn_handle = connection.makefile('wb+')
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0', 'subdir0')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()
      logging.getLogger().debug("Sleep 20 seconds before closing connection")
      time.sleep(20)
      conn_handle.close()
      connection.close()
      logging.getLogger().debug("Waiting for shutdown up to 10 seconds")
      client.join(10)
      try:
        self.assertFalse(client.is_alive())
      except:
        client.terminate()
        raise
    except:
      if connection:
        connection.close()

  #@unittest.skip("")
  def test_3_client_process_multiple_simple_dir(self):
    client = HydraClientProcess({'svr': '127.0.0.1', 'port': self.server_port, 'file_handler': HydraTestClassSlowFileProcess})
    client.start()
    try:
      readable, _, _ = select.select([self.server], [], [], 5)
      connection, client_address = self.server.accept()
      logging.getLogger().debug('New connection from %s, Socket: %s'%(client_address, connection))
      connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
      conn_handle = connection.makefile('wb+')
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0', 'subdir0')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0', 'subdir1')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0', 'subdir2')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0', 'subdir3')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()
      logging.getLogger().debug("Sleep 20 seconds before closing connection")
      time.sleep(20)
      conn_handle.close()
      connection.close()
      logging.getLogger().debug("Waiting for shutdown up to 10 seconds")
      client.join(10)
      try:
        self.assertFalse(client.is_alive())
      except:
        client.terminate()
        raise
    except:
      if connection:
        connection.close()

  #@unittest.skip("")
  def test_4_dir_with_subdirs_splitting_work(self):
    client = HydraClientProcess({'svr': '127.0.0.1', 'port': self.server_port, 'file_handler': HydraTestClassSlowFileProcess})
    client.start()
    try:
      readable, _, _ = select.select([self.server], [], [], 5)
      connection, client_address = self.server.accept()
      logging.getLogger().debug('New connection from %s, Socket: %s'%(client_address, connection))
      connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
      conn_handle = connection.makefile('wb+')
      inputs = [connection]
      for i in range(2):
        logging.getLogger().debug("Waiting idle state")
        # Wait for stats update before client shutdown
        readable, _, _ = select.select(inputs, [], [], POLL_WAIT_SECONDS)
        for s in  readable:
          data = pickle.load(conn_handle)
          self.assertIsNot(data, False)
          self.assertEqual(data.get('state'), ['connected', 'idle'][i])

      for d in ['dir0', 'dir1', 'dir2', 'dir3']:
        cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, d)]}
        pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()
      logging.getLogger().debug("Wait up to 40 seconds to allow for file processing")
      start = time.time()
      for i in range(40):
        if (time.time() - start) >= 40:
          break
        logging.getLogger().debug("Waiting for stats update")
        # Wait for stats update before client shutdown
        readable, _, _ = select.select(inputs, [], [], POLL_WAIT_SECONDS)
        for s in  readable:
          data = pickle.load(conn_handle)
          self.assertIsNot(data, False)
          logging.getLogger().debug("Received interim message: :%s"%data)

      logging.getLogger().debug("Shutting down client")
      cmd = {'cmd': 'shutdown'}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()

      found_stats = False
      for i in range(15):
        logging.getLogger().debug("Waiting for stats update and shutdown")
        # Wait for stats update before client shutdown
        readable, _, _ = select.select(inputs, [], [], POLL_WAIT_SECONDS)
        for s in  readable:
          try:
            data = pickle.load(conn_handle)
          except EOFError:
            break
          self.assertIsNot(data, False)
          cmd = data.get('cmd')
          self.assertTrue(cmd is not None, msg='Invalid command from worker.')
          if cmd == 'stats':
            found_stats = True
          elif cmd == 'state':
            if data.get('state') == 'shutdown':
              break
      self.assertTrue(found_stats, msg='Expected to receive some stats from workers. None received.')

      logging.getLogger().debug("Closing server to client connection")
      conn_handle.close()
      connection.close()
      logging.getLogger().debug("Waiting for shutdown up to 10 seconds")
      client.join(10)
      try:
        self.assertFalse(client.is_alive(), msg='Client is still alive')
      except:
        client.terminate()
        raise
    except KeyboardInterrupt:
      if connection:
        connection.close()
        client.terminate()
    except Exception as e:
      logging.getLogger().exception(e)
      if connection:
        connection.close()

  #@unittest.skip("")
  def test_5_periodic_stats_query(self):
    client = HydraClientProcess({'svr': '127.0.0.1', 'port': self.server_port, 'file_handler': HydraTestClassSlowFileProcess})
    client.start()
    inputs = []
    try:
      readable, _, _ = select.select([self.server], [], [], 5)
      connection, client_address = self.server.accept()
      logging.getLogger().debug('New connection from %s, Socket: %s'%(client_address, connection))
      connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
      conn_handle = connection.makefile('wb+')
      inputs.append(connection)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir0')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir1')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir2')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      cmd = {'cmd': 'submit_work', 'paths': [os.path.join(self.test_path, 'dir3')]}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()
      logging.getLogger().debug("Sleep 10 seconds to allow for file processing")
      time.sleep(10)

      for i in range(2):
        # Wait for multiple stats update before client shutdown
        cmd = {'cmd': 'return_stats'}
        pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
        conn_handle.flush()
        readable, _, _ = select.select(inputs, [], [], POLL_WAIT_SECONDS*5)
        #self.assertIsNot(len(readable), 0)
        data = pickle.load(conn_handle)
        #self.assertIsNot(data, False)
        #self.assertEqual('stats', data.get('cmd'))
        time.sleep(3)

      # Wait for stats update before client shutdown
      logging.getLogger().debug("Waiting for shutdown")
      cmd = {'cmd': 'shutdown'}
      pickle.dump(cmd, conn_handle, pickle.HIGHEST_PROTOCOL)
      conn_handle.flush()
      inputs = [connection]
      readable, _, _ = select.select(inputs, [], [], POLL_WAIT_SECONDS*4)
      self.assertIsNot(len(readable), 0)
      data = pickle.load(conn_handle)
      self.assertIsNot(data, False)

      logging.getLogger().debug("Closing server to client connection")
      conn_handle.close()
      connection.close()
      logging.getLogger().debug("Waiting for shutdown up to 10 seconds")
      client.join(10)
      try:
        self.assertFalse(client.is_alive())
      except:
        client.terminate()
        raise
    except Exception as e:
      logging.getLogger().exception(e)
      if connection:
        connection.close()

if __name__ == '__main__':
  root = logging.getLogger()
  root.setLevel(logging.WARN)
  #root.setLevel(logging.DEBUG)
  #root.setLevel(5)
  #root.setLevel(9)

  ch = logging.StreamHandler(sys.stdout)
  formatter = logging.Formatter('%(asctime)s - %(name)s - %(process)d - %(levelname)s - %(message)s')
  ch.setFormatter(formatter)
  root.addHandler(ch)

  suite1 = unittest.TestLoader().loadTestsFromTestCase(TestHydraClient)
  #suite2 = unittest.TestLoader().loadTestsFromTestCase(TestHydraWorkerProcessDirectory)
  all_tests = unittest.TestSuite([suite1])
  unittest.TextTestRunner(verbosity=2).run(all_tests)


# from django.contrib.gis.geoip2 import GeoIP2
from collections import namedtuple
from .models import Destination, Tourist, DestinationCityDetails, DestinationMetaDetails
import folium
from geopy.geocoders import Nominatim
import socket
import geocoder
from geopy.distance import geodesic
import requests
import json
from ipregistry import IpregistryClient

#creating my project helper functions here

def get_center_coordinates(latA, longA, latB=None, longB=None):
    cord = (latA, longA)
    if latB:
        cord = [(latA+latB)/2, (longA+longB)/2]
    return cord

def get_zoom(distance):

    if distance <=20:
        return 12
    elif distance<=50:
        return 10
    elif distance <= 100:
        return 9
    elif distance > 100 and distance <= 1500:
        return 5
    elif distance > 1501 and distance <= 3000:
        return 4
    else:
        return 2

def location_module(destination_):

    city=destination_
    geolocator = Nominatim(user_agent='syedarfa')

    client = IpregistryClient("escffgkb0orli5")
    ipInfo = client.lookup()
    # print(ipInfo)
    # print(ipInfo.ip)
    # print(ipInfo.location.get('latitude'))
    # print(ipInfo.location.get('longitude'))
    mylatitude = ipInfo.location.get('latitude')
    mylongitude = ipInfo.location.get('longitude')
    mycity = ipInfo.location.get('city')

    tourist_lat, tourist_lon = mylatitude, mylongitude

    print("yahan", tourist_lat)
    touristLocation = (tourist_lat, tourist_lon)
    # to get the location details of the tourist
    touristlocdetails = geolocator.reverse(str(tourist_lat) + ',' + str(tourist_lon))
    touristaddress = touristlocdetails.raw['address']
    # print(touristaddress)
    tourist_state = touristaddress.get('state', '')
    tourist_country = touristaddress.get('country', '')
    tourist_locdetails = tourist_state + tourist_country
    # print('Details are ' + tourist_locdetails)

    # Map using folium library
    # the map will point to the location of tourist
    mapobj = folium.Map(width=380, height=600, location=touristLocation, zoom_start=15)
    # adding marker at tourist location
    folium.Marker([tourist_lat, tourist_lon], tooltip='Click to get the name of city', popup=mycity,
                  icon=folium.Icon(color='green')).add_to(mapobj)

    if geolocator.geocode(destination_) is None:
        return "not found"
    # print(destination.address)
    # print(destination.latitude)
    # if not destination.latitude:
    else:
        destination = geolocator.geocode(destination_)
        dest_lat = destination.latitude
        # print(destination.longitude)
        dest_long = destination.longitude

        destinationLocation = (dest_lat, dest_long)

        # Now calculating the distance between tourist location and distance location
        distance = geodesic(touristLocation, destinationLocation).km

        # Now to plot the route between tourist location and the destination
        mapobj = folium.Map(width=550, height=380,
                            location=get_center_coordinates(tourist_lat, tourist_lon, dest_lat, dest_long),
                            zoom_start=get_zoom(distance))
        # TouristLocation
        folium.Marker([tourist_lat, tourist_lon], tooltip='Click to get the name of city', popup=mycity,
                    icon=folium.Icon(color='green')).add_to(mapobj)
        # TouristDestination
        folium.Marker([dest_lat, dest_long], tooltip='Click to get the name of city', popup=destination,
                    icon=folium.Icon(color='blue')).add_to(mapobj)
        # Now we will draw line between touristlocation and tourist destination
        line = folium.PolyLine(locations=[touristLocation, destinationLocation], weight=2.5, color='red')
        # adding the line to our map now
        mapobj.add_child(line)
        mapobj = mapobj._repr_html_()
        dest_meta_details = DestinationMetaDetails.objects.filter(meta_destination__in=DestinationCityDetails.objects.filter(destination_name__icontains=city))
        # print("--------------------------------------,",dest_meta_details)
        # print("destttttttttttttttttttt===", dest_meta_details)
        return [distance,mapobj,dest_meta_details,tourist_lat,tourist_lon,tourist_locdetails,dest_lat,dest_long]


import csv
import pickle
import random

import numpy as np
import ujson as json
import torch
import pandas as pd
from sklearn.model_selection import train_test_split
from typing import List
import spacy as spacy

'''
Read data from file and return a list of list
every item in the list is a list of original text, response text and label
label = 0 -> not sarcasm
label = 1 -> sarcasm
'''

def read_discussion_forum(file="./data/dicussion-forum-data.csv"):
    '''Read data from discussion forum data'''
    with open(file) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        data = []
        for row in csv_reader:
            # if line_count > 100:
            #     break
            if line_count == 0:
                print('Column names are {}'.format(", ".join(row)))
                line_count += 1
            else:
                '''Remember to trim leading and trailing spaces'''
                data.append([row[3].strip(), row[4].strip(), 1 if row[1] == 'sarc' else 0])
                line_count += 1
        print('Processed {} lines.'.format(line_count))
        return data

'''
Stores indices to file in the format of [contexts, responses, labels]'''
def save_discussion_forum_data():
    data = read_discussion_forum()
    contexts, responses, labels = generate_indices(data, "discussion")
    with open('./data/discussion/discussion_forum_indices', 'wb') as f:
        pickle.dump([contexts, responses, labels], f)

def read_discussion_forum_from_file():
    with open('./data/discussion/discussion_forum_indices', 'rb') as f:
        my_list = pickle.load(f)
    d = list(zip(my_list[0], my_list[1], my_list[2]))
    random.shuffle(d)
    contexts, responses, labels = zip(*d)
    return contexts, responses, labels

def save_reddit_data():
    data = read_reddit_data()
    random.shuffle(data)
    contexts, responses, labels = generate_indices(data, "reddit")
    with open('./data/reddit/reddit_indices', 'wb') as f:
        pickle.dump([contexts, responses, labels], f)

def read_reddit_data_from_file():
    with open('./data/reddit/reddit_indices', 'rb') as f:
        my_list = pickle.load(f)
    d = list(zip(my_list[0], my_list[1], my_list[2]))
    random.shuffle(d)
    contexts, responses, labels = zip(*d)
    return contexts, responses, labels

def torch_from_json(path, dtype=torch.float32):
    """Load a PyTorch Tensor from a JSON file.

    Args:
        path (str): Path to the JSON file to load.
        dtype (torch.dtype): Data type of loaded array.

    Returns:
        tensor (torch.Tensor): Tensor loaded from JSON file.
    """

    with open(path, 'r') as fh:
        array = np.array(json.load(fh))

    tensor = torch.from_numpy(array).type(dtype)
    if torch.cuda.is_available():
        tensor = tensor.cuda()

    return tensor

'''
Separate every content to sentences, and generate index for each word. Pad the sentence and contents.
sents: a list of contents
Every content is consisted of a list: [original sentences, response sentences, label]
'''
def generate_indices(sents: List, tag):
    '''Generate index for words. If the word doesn't exist, return 1 for that position'''

    nlp = spacy.load("en_core_web_sm")
    if tag == "discussion":
        word2idxPath = './data/discussion/word2idx.json'
    else:
        word2idxPath = './data/reddit/word2idx.json'

    with open('./data/discussion/word2idx.json') as handle:
            word2idx = json.loads(handle.read())

    originals = []
    responses = []
    labels = []

    count = 0

    for content in sents:
        # separate every content into a list of sentences
        original = [sentence.text for sentence in nlp(content[0]).sents]
        response = [sentence.text for sentence in nlp(content[1]).sents]
        label = content[2]
        originals.append([[word2idx[word.text] if word.text in word2idx else 1 for word in nlp(sentence)] for sentence in original])
        responses.append([[word2idx[word.text] if word.text in word2idx else 1 for word in nlp(sentence)] for sentence in response])
        labels.append(label)
        count += 1
        if count % 1000 is 0:
            print("At count {}".format(count))
            originals = pad_sents(originals)
            responses = pad_sents(responses)
            with open('./data/reddit/reddit_indices', 'wb') as f:
                pickle.dump([originals, responses, labels], f)
            # if count > 8000:
            #     break
    originals = pad_sents(originals)
    responses = pad_sents(responses)
    # print(originals)
    # print(responses)
    print("Generated indices for contexts and responses.")
    return originals, responses, labels

'''
@return: A list of content padded
Every content has max_content_length of sentences
Every sentence has max_sentence_length of words
'''
def pad_sents(sents: List):
    # max_content_length = np.max([len(content) for content in sents])
    # max_sentence_length = [len(sentence) for sentence in content for content in sents]
    max_content_length = 10
    max_sentence_length = 50
    for content in sents:
        if len(content) > max_content_length:
            del content[max_content_length:]
        else:
            content += (max_content_length - len(content)) * [max_sentence_length * [0]]

        for sentence in content:
            if len(sentence) > max_sentence_length:
                del sentence[max_sentence_length:]
            else:
                sentence += (max_sentence_length - len(sentence)) * [0]
    return sents

def read_reddit_data(file="./data/reddit/train-balanced-sarcasm.csv"):
    with open(file) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        data = []
        for row in csv_reader:
            # if line_count > 1000:
            #     break
            if line_count == 0:
                print('Column names are {}'.format(", ".join(row)))
                line_count += 1
            else:
                '''Remember to trim leading and trailing spaces'''
                data.append([row[9].strip(), row[1].strip(), 1 if row[0] == '1' else 0])
                line_count += 1
        print('Processed {} lines.'.format(line_count))
        return data

if __name__ == '__main__':
    # read_discussion_forum()
    # save_discussion_forum_data()
    save_reddit_data()
    # with open('./data/discussion/discussion_forum_indices', 'rb') as f:
    #     my_list = pickle.load(f)
    #     print(my_list)


import time
import numpy as np
import torch

import fidelity as F
import povm as P
import ann as A
import sys

# Basic parameters

def AQT(datapath, Nq, Nep, Nl=2, dmodel=64, Nh=4, save_model=True, save_loss=True, save_pt=True, save_dm=True):

    povm = P.POVM('pauli6')
    Na = povm.Na

    if torch.cuda.is_available():
        device = torch.device("cuda:0")
    else:
        device = torch.device("cpu")

    # Load data

    data = np.load('{}.npy'.format(datapath))
    np.random.shuffle(data)

    split = 0.8
    traindata = data[:int(len(data)*split)]
    testdata = data[int(len(data)*split):]

    # Train model

    model_filetag = '{}_{}-{}-{}-{}'.format(datapath, Nep, Nl, dmodel, Nh)

    model = A.InitializeModel(Nq, Nlayer=Nl, dmodel=dmodel, Nh=Nh, Na=Na).to(device)

    t = time.time()
    model, loss = A.TrainModel(model, traindata, testdata, device, batch_size=50,lr=1e-4,Nep=Nep)
    print('Took %f minutes'%((time.time()-t)/60))

    model.to('cpu')
    if save_model:
        torch.save(model, '{}.mod'.format(model_filetag))
    if save_loss:
        np.save('{}_loss.npy'.format(model_filetag), loss)

    # Build POVM probability table

    pt_model = F.POVMProbTable(model)

    if save_pt:
        np.save('{}_pt.npy'.format(model_filetag), pt_model)

    # Reconstruct density matrix

    dm8 = np.zeros((8, 2**Nq, 2**Nq), dtype=complex)

    for xyz in range(8):
        dm8[xyz] = F.GetDMFull(pt_model, Nq, P.POVM('pauli6', xyz))

    _, negeig, dm_model = F.GetBestDM(dm8)

    if save_dm:
        np.save('{}_dm.npy'.format(model_filetag), dm_model)

    return dm_model

if __name__ == '__main__':

    datapath = 'dicke_6_2/6_72900'
    Nq = 6
    Nep = 100

    AQT(datapath, Nq, Nep)


from urllib import request as rq
from urllib.parse import urlparse
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup as bs

class BookmarkParserConfig:
    request = ''
    parse_uri = ''
    soup = ''

    def connect(self, url):
        headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}
        try:
            r = rq.Request(url, headers=headers)
            self.request = rq.urlopen(r)
        except (HTTPError, URLError):
            print("Connect error")
        self.parse_uri = urlparse(url)
        self.soup = bs(self.request, "lxml")

class BookmarkParser(BookmarkParserConfig):
    def generate_site_title(self):
        return self.soup.title.string

    def generate_site_paragraph(self):
        p = ''
        metas = self.soup.find_all('meta')
        d = [meta.attrs['content'] for meta in metas if 'name' in meta.attrs and meta.attrs['name'] == 'description']

        if metas:
            for k in metas:
                if d:
                    p = d[0]
                elif k.get('property', None) == 'og:description':
                    p = k.get('content')
        else:
            if not self.soup.p:
                p = self.soup.title.string
            else:
                p = self.soup.p.text[0:799]
        return p

    def generate_site_author(self):
        try:
            result = '{uri.netloc}'.format(uri=self.parse_uri)
        except Exception as e:
            print(e)
        return result


# encoding: utf-8

"""
twinado server.application.tornado_app

~~~~~~

对 tornado web application 的定制

"""

from tornado.web import Application as BaseApplication

class Application(BaseApplication):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        # self.db = None
        # self.db_uri = db_uri
        # self.redis = None

        self.mode = 'production'


from itertools import permutations
def nth47Number(s):
    # a = []
    # j = 4
    # while len(a) < s:
    #     if (str(j)).count('4')+(str(j)).count('7') == len(str(j)):
    #         a.append(j)
    #     # print(a)
    #     j += 1
    # return a[s-1]
    # print(a[s])
    i = 0
    while pow(2,i) - 2 < s:
        i+=1
    i-=1
    k = s - pow(2,i)+2
    a = []
    for j in range(i+1):
        b = list(permutations([4]*(i-j)+[7]*j))
        # print(b)
        c = []
        for z in b:
            if len(a) > k:
                break
            temp = int(''.join(list(map(str,z))))
            if  not temp in c:
                c.append(int(''.join(list(map(str, z)))))
        a = a + c
    # print(sorted(a))
    # print(k)
    return sorted(a)[k-1]
    # print(sorted(a)[k])
    # a = list(permutations(list[4,4,7,7]))
    # a = sorted(a)
    # print(a[k-1])

T = int(input())
for i in range(T):
    s = int(input())
    print(nth47Number(s))


import multiprocessing as mp
import numpy as np
import logging
import os
import sys
from env import Env
import ppo2 as network
import tensorflow as tf

os.environ['CUDA_VISIBLE_DEVICES'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

S_DIM = [5, 8]
A_DIM = 10
ACTOR_LR_RATE =1e-4
CRITIC_LR_RATE = 1e-3
NUM_AGENTS = 16
TRAIN_SEQ_LEN = 1500  # take as a train batch
TRAIN_EPOCH = 1000000
MODEL_SAVE_INTERVAL = 30
RANDOM_SEED = 42
RAND_RANGE = 10000
SUMMARY_DIR = './results'
MODEL_DIR = './models'
TRAIN_TRACES = './cooked_traces/'
TEST_LOG_FOLDER = './test_results/'
LOG_FILE = './results/log'
PPO_TRAINING_EPO = 5
# create result directory
if not os.path.exists(SUMMARY_DIR):
    os.makedirs(SUMMARY_DIR)

NN_MODEL = None

def central_agent(net_params_queues, exp_queues):

    assert len(net_params_queues) == NUM_AGENTS
    assert len(exp_queues) == NUM_AGENTS
    tf_config=tf.ConfigProto(intra_op_parallelism_threads=5,
                            inter_op_parallelism_threads=5)
    with tf.Session(config = tf_config) as sess, open(LOG_FILE + '_test.txt', 'w') as test_log_file:
        summary_ops, summary_vars = build_summaries()

        actor = network.Network(sess,
                state_dim=S_DIM, action_dim=A_DIM,
                learning_rate=ACTOR_LR_RATE)

        sess.run(tf.global_variables_initializer())
        writer = tf.summary.FileWriter(SUMMARY_DIR, sess.graph)  # training monitor
        saver = tf.train.Saver(max_to_keep=1000)  # save neural net parameters

        # restore neural net parameters
        nn_model = NN_MODEL
        if nn_model is not None:  # nn_model is the path to file
            saver.restore(sess, nn_model)
            print("Model restored.")

        max_reward, max_epoch = -10000., 0
        tick_gap = 0
        # while True:  # assemble experiences from agents, compute the gradients
        for epoch in range(TRAIN_EPOCH):
            # synchronize the network parameters of work agent
            actor_net_params = actor.get_network_params()
            for i in range(NUM_AGENTS):
                net_params_queues[i].put(actor_net_params)

            s, a, p, g, r = [], [], [], [], []
            for i in range(NUM_AGENTS):
                s_, a_, p_, g_, r_ = exp_queues[i].get()
                s += s_
                a += a_
                p += p_
                g += g_
                r += r_
            s_batch = np.stack(s, axis=0)
            a_batch = np.vstack(a)
            p_batch = np.vstack(p)
            v_batch = np.vstack(g)
            r_batch = np.vstack(r)

            for _ in range(PPO_TRAINING_EPO):
                actor.train(s_batch, a_batch, p_batch, v_batch, epoch)
            # actor.train(s_batch, a_batch, v_batch, epoch)

            if epoch % MODEL_SAVE_INTERVAL == 0:
                # Save the neural net parameters to disk.
                save_path = saver.save(sess, SUMMARY_DIR + "/nn_model_ep_" +
                                       str(epoch) + ".ckpt")

                avg_reward = np.mean(r_batch)

                if avg_reward > max_reward:
                    max_reward = avg_reward
                    max_epoch = epoch
                    tick_gap = 0
                else:
                    tick_gap += 1

                if tick_gap >= 10:
                    # saver.restore(sess, SUMMARY_DIR + "/nn_model_ep_" + str(max_epoch) + ".ckpt")
                    actor.set_entropy_decay()
                    tick_gap = 0

                summary_str = sess.run(summary_ops, feed_dict={
                    summary_vars[0]: actor.get_entropy(epoch),
                    summary_vars[1]: avg_reward,
                    summary_vars[2]: 0.
                })

                writer.add_summary(summary_str, epoch)
                writer.flush()

def agent(agent_id, net_params_queue, exp_queue):
    env = Env(agent_id)
    with tf.Session() as sess, open(SUMMARY_DIR + '/log_agent_' + str(agent_id), 'w') as log_file:
        actor = network.Network(sess,
                                state_dim=S_DIM, action_dim=A_DIM,
                                learning_rate=ACTOR_LR_RATE)

        # initial synchronization of the network parameters from the coordinator
        actor_net_params = net_params_queue.get()
        actor.set_network_params(actor_net_params)

        time_stamp = 0

        for epoch in range(TRAIN_EPOCH):
            obs = env.reset()
            s_batch, a_batch, p_batch, r_batch = [], [], [], []
            for step in range(TRAIN_SEQ_LEN):
                s_batch.append(obs)

                action_prob = actor.predict(
                    np.reshape(obs, (-1, S_DIM[0], S_DIM[1])))

                noise = np.random.gumbel(size=len(action_prob))
                act = np.argmax(np.log(action_prob) + noise)

                obs, rew, done, info = env.step(act)

                action_vec = np.zeros(A_DIM)
                action_vec[act] = 1
                a_batch.append(action_vec)
                r_batch.append(rew)
                p_batch.append(action_prob)
                if done:
                    break
            v_batch = actor.compute_v(s_batch, a_batch, r_batch, done)
            exp_queue.put([s_batch, a_batch, p_batch, v_batch, r_batch])

            actor_net_params = net_params_queue.get()
            actor.set_network_params(actor_net_params)

def build_summaries():
    td_loss = tf.Variable(0.)
    tf.summary.scalar("Beta", td_loss)
    eps_total_reward = tf.Variable(0.)
    tf.summary.scalar("Reward", eps_total_reward)
    entropy = tf.Variable(0.)
    tf.summary.scalar("Entropy", entropy)

    summary_vars = [td_loss, eps_total_reward, entropy]
    summary_ops = tf.summary.merge_all()

    return summary_ops, summary_vars

def main():

    np.random.seed(RANDOM_SEED)

    # inter-process communication queues
    net_params_queues = []
    exp_queues = []
    for i in range(NUM_AGENTS):
        net_params_queues.append(mp.Queue(1))
        exp_queues.append(mp.Queue(1))

    # create a coordinator and multiple agent processes
    # (note: threading is not desirable due to python GIL)
    coordinator = mp.Process(target=central_agent,
                             args=(net_params_queues, exp_queues))
    coordinator.start()

    agents = []
    for i in range(NUM_AGENTS):
        agents.append(mp.Process(target=agent,
                                 args=(i,
                                       net_params_queues[i],
                                       exp_queues[i])))
    for i in range(NUM_AGENTS):
        agents[i].start()

    # wait unit training is done
    coordinator.join()

if __name__ == '__main__':
    main()


from django.conf.urls import patterns, url

from django.conf.urls import patterns, include, url
import orderviews

urlpatterns = patterns('',
	url(r'^orders$', orderviews.orders),
	url(r'^samples$', orderviews.samples),
	url(r'^orderswkview$', orderviews.orderswkview),
	url(r'^sampleswkview$', orderviews.sampleswkview),
	url(r'^ORDER/filter$', orderviews.orderfilter),
	url(r'^SAMPLE/filter$', orderviews.samplefilter),
	url(r'^add', orderviews.add),
	url(r'^(?P<order_id>\d+)$', orderviews.show),
	url(r'^(?P<order_id>\d+)/edit', orderviews.edit),
	url(r'^(?P<order_id>\d+)/addproduct', orderviews.addproduct),
	url(r'^(?P<order_id>\d+)/submit', orderviews.submit),
	url(r'^(?P<order_id>\d+)/cancel', orderviews.cancel),
	url(r'^(?P<order_id>\d+)/approve', orderviews.approve),
	url(r'^(?P<order_id>\d+)/complete', orderviews.complete),
	# url(r'^(?P<order_id>\d+)/setdate', orderviews.setdate),
	url(r'^(?P<op_id>\d+)/deleteproduct', orderviews.deleteproduct),
	url(r'^(?P<order_id>\d+)/addnote', orderviews.addnote),
	url(r'^(?P<order_id>\d+)/income', orderviews.income),
	url(r'^(?P<order_id>\d+)/expense', orderviews.expense),
	url(r'^(?P<op_id>\d+)/opstart', orderviews.opstart),
	url(r'^(?P<op_id>\d+)/opcomplete', orderviews.opcomplete),
	url(r'^(?P<op_id>\d+)/opsetdate', orderviews.opsetdate),
)


#!/usr/bin/env python3

from bottle import route
import bottle
import tempfile
import os
import shutil
import subprocess
import json
import hashlib

def is_known_format(fmt):
    return fmt in ['std', 'export', 'simple', 'inline']

def get_out_filename(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    f =  os.path.join(outdir, get_out_filename_rel(name, fmt))
    if not os.path.abspath(f).startswith(os.path.abspath(outdir)):
        raise Exception('Path out problem', os.path.abspath(f), os.path.abspath(outdir))
    return f

def get_out_filename_rel(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    return "{}.{}.html".format(name, fmt)

def get_in_filename(name):
    f = os.path.join(indir, get_in_filename_rel(name))
    if not os.path.abspath(f).startswith(os.path.abspath(indir)):
        raise Exception('Path in problem', os.path.abspath(f), os.path.abspath(indir))
    for file_ending in [""] + file_endings:
        if os.path.exists(f + file_ending):
            return f + file_ending
    return f

def get_in_filename_rel(name):
    return name

def has_compile_file_ending(f):
    for file_ending in file_endings:
        if f.endswith(file_ending):
            return True
    return False

@route('/view/<name:path>')
def route_view(name):
    infile = get_in_filename(name)

    if os.path.exists(infile) and has_compile_file_ending(infile):
        fmt = bottle.request.query.fmt or "std"
        if not is_known_format(fmt): return 'Unknown format'

        compile_document(name, fmt)
        return bottle.static_file(get_out_filename_rel(name, fmt), outdir)

    else:
        return bottle.static_file(get_in_filename_rel(name), indir)

@route('/refresh/<name:path>')
def route_refresh(name):
    if needs_update(name):
        return "True"
    return "False"

@route('/generated/<name:path>')
def route_generated(name):
    return bottle.static_file(name, os.path.join(tempdir, "generated"))

@route('/')
def route_index():
    text = ""

    text += "<title>Panserver Index</title>"
    text += ''.format(style_basic + style_index)
    text += ""
    text += '
Panserver
'
    text += 'Serving Markdown documents rendered using pandoc. By Marcel Fischer'

    def dir_entry(dirname, toplevel = False):
        #collect markdown files recursively into a list
        #return '' if no markdown file is in the directory
        dirtext = ''
        print("dirname ", dirname)
        for name in sorted(os.listdir(os.path.join('.', dirname))):
            path = os.path.join(dirname, name)
            if os.path.isdir(path): continue
            if not has_compile_file_ending(path): continue
            d = {}
            d['name'] = os.path.basename(path)
            d['path'] = path
            dirtext += '
{name}
'.format(**d)

        for name in sorted(os.listdir(os.path.join('.', dirname))):
            path = os.path.join(dirname, name)
            if not os.path.isdir(path): continue
            subdirtext = dir_entry(path)
            #ignore directories with empty listings
            if subdirtext != '':
                dirtext += '
' + os.path.basename(path) + subdirtext + '
'

        if dirtext != '':
            dirtext = '
' + dirtext + '
'
        elif toplevel:
            dirtext = ' (no documents)'
        return dirtext

    text += "
Directory contents
"
    text += dir_entry('', toplevel=True)

    text += ''

    return text

def needs_update(name, fmt = 'std'):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if not os.path.exists(in_filename): return False
    if not os.path.exists(out_filename): return True
    return os.path.getmtime(out_filename) < os.path.getmtime(in_filename)

def compile_document(name, fmt):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if needs_update(name, fmt):
        #file is not cached, recompile
        #create directory if needed
        out_filename_dir = os.path.dirname(out_filename)
        if not os.path.exists(out_filename_dir):
            os.makedirs(out_filename_dir)

        #combine action based on format
        action = ['pandoc']
        # Format to use GFM"
        action.append("--from=gfm")

        if fmt == 'std':
            action += ['-H', headerfile, '-B', topmenufile,]

        if fmt == 'export':
            action += ['-H', headerfile_export,]

        if fmt == 'std' or fmt == 'export':
            action += ['-B', beforefile, '-A', afterfile,]
            action += ['--toc']

        if fmt != 'inline':
            action += ['-s']

        # compile
        #print(action)
        # create json in intermediate step
        json_encoding_process = subprocess.Popen(action + ['-t', 'json', in_filename], stdout=subprocess.PIPE)
        json_text, _ = json_encoding_process.communicate()
        json_text = json_text.decode('utf-8')

        # process json to extract embedded dot diagrams and others
        json_text, math_option = process_document_json(json_text, alternative_title=os.path.basename(name))

        # render to HTML
        json_decoding_process = subprocess.Popen(action + math_option + ['-f', 'json', '-o', out_filename], stdin=subprocess.PIPE)
        json_decoding_process.communicate(json_text.encode('utf-8'))

def process_document_json(json_text, alternative_title):
    math_option = ['--mathjax']
    changed = False

    doc = json.loads(json_text)

    # TODO descent recursively into json doc

    for block in doc["blocks"]:
        if block["t"] == "CodeBlock":
            if "dot" in block["c"][0][1]:
                extract_embedding(block, "dot")
                changed = True
            elif "plantuml" in block["c"][0][1]:
                extract_embedding(block, "plantuml")
                changed = True

    # Add title if missing
    if "title" not in doc["meta"] and "pagetitle" not in doc["meta"]:
        doc["meta"]["pagetitle"] = {'t': 'MetaInlines', 'c': [{'t': 'Str', 'c': alternative_title}]}
        changed = True

    # Allow user to select math option using metadata
    if "panserver_math" in doc["meta"]:
        meta_math = doc["meta"]["panserver_math"]["c"][0]["c"]
        print("math option ", meta_math)
        if meta_math == "mathml":
            math_option = ['--mathml']
        elif meta_math == "mathjax":
            math_option = ['--mathjax']
        elif meta_math == "none":
            math_option = []

    if changed:
        json_text = json.dumps(doc)
    return json_text, math_option

def extract_embedding(block, embedding_type):
    code = block["c"][1]
    md5 = hashlib.md5(code.encode('utf-8')).hexdigest()
    generation_dir = os.path.join(tempdir, "generated")
    error = None
    if not os.path.exists(generation_dir):
        os.makedirs(generation_dir)
    target = os.path.join(generation_dir, "{}.{}.png".format(md5, embedding_type))
    if not os.path.exists(target):
        if embedding_type == "dot":
            process = subprocess.Popen(["dot", "-Tpng"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        elif embedding_type == "plantuml":
            process = subprocess.Popen(["plantuml", "-pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        process_out, process_err = process.communicate(code.encode('utf-8'))
        error_code = process.wait()
        if error_code != 0:
            error = process_err.decode('utf-8')
        else:
            with open(target, "wb") as f:
                f.write(process_out)

    if error is None:
        # Replace embedding in block with image
        block["t"] = "Para"
        block["c"] = [ { "t" : "Image", "c" : [
            ["", [], []],
            [],
            [ "/generated/{}.{}.png".format(md5, embedding_type), "fig:"]
            ]}]
    else:
        # Insert error
        block["c"][0][1] = []
        block["c"][1] = error

def create_header(autorefresh):
    headertext = ""
    headertext += """{}""".format(meta_no_mobilescale, style_basic + style_document_add)
    headertext += markdown_css_link

    if autorefresh:
        headertext += """

"""

    with open(headerfile, 'w') as f:
        f.write(headertext)

def create_header_export():
    text = """{}""".format(meta_no_mobilescale, style_basic)
    text += markdown_css_link

    with open(headerfile_export, 'w') as f:
        f.write(text)

def create_topmenufile():
    text = """
    Panserver: Index
    TOC
    Format:
    Export
    Simple
    Inline

    """
    with open(topmenufile, 'w') as f:
        f.write(text)

def create_beforefile():
    text = """

    """
    with open(beforefile, 'w') as f:
        f.write(text)

def create_afterfile():
    with open(afterfile, 'w') as f:
        f.write("")

#global vars
file_endings = [".md", ".markdown", ".rst"]
tempdir = tempfile.mkdtemp()
outdir = os.path.join(tempdir, "out")
indir = os.path.abspath('.')

headerfile = os.path.join(tempdir, "header.html")
headerfile_export = os.path.join(tempdir, "header.export.html")
beforefile = os.path.join(tempdir, "before.html")
topmenufile = os.path.join(tempdir, "topmenu.html")
afterfile = os.path.join(tempdir, "after.html")

if not os.path.exists(outdir):
    os.makedirs(outdir)

meta_no_mobilescale = """<meta name="viewport" content="width=device-width, initial-scale=1.0">"""

style_basic = """
        body {
            width: 92%; max-width: 40em;  margin: auto;
            font-size: 1.2rem; line-height: 1.5; padding-bottom: 3em;}
        code { font-family: "DejaVu Sans Mono", Consolas, monospace; }
        pre { overflow: auto; }
        math, .LaTeX { font-size: 1.1em; }
        div.figure .caption {  padding: 0em 1em 1em; font-size: 0.8em; }
        ul li { list-style-type: disc; }
        li p { margin: 0.5em 0; }
        #TOC { border: 1px solid lightgray; padding: 0.5em; margin-bottom: 1em;}
        #TOC li { list-style: circle; }
        @media (min-width: 102em) {
            body { position: relative; }
            #TOC { display: block; position: absolute; left: -50%; top: 5em; max-width: 46%; width: 45%; }
        }
"""

style_document_add = """
        @media (max-width: 101em) {
            #TOC { display: none; }
        }
        .topmenu { font-size: 1em; color: lightgrey; }
        .topmenu a { color: lightgrey; }
"""
style_index = """
        ul.file-listing a { text-decoration: none; }
        ul.file-listing ul.file-listing { padding-left: 1.5em; }
        ul li.dir-entry {
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAApElEQVQ4je3UsQqBYRTG8Z9SMpjsrsAdYMI1WNyDhdnKriiD1U3YXML3SWE3uAgDX+lF8iqlPPX01qnn/55OncMvqYMUSeA1KjHAFKUH9S7GMcDkSb2Jk/vOQ2/QhzKq2F/fd52/ftzCBKZYYRnhA2ohcIb6m+PJdJv9A//AT4GND4BZtp0BBxgiFwG8zY7QgyIWLsv96gCE3uKIHeYoRDT1ZZ0BsR1JIXdX8A8AAAAASUVORK5CYII=');
        }
        ul li.file-entry{
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAA00lEQVQ4jcXUQUqCURSG4Qc0bKaUiKsIhEbazMBhOHILES3EUYIGtpBwlDODhhFCTqQdtIYGf5H+eOVeFP4P3ul7L4fzHbZziQme9jDEqYhU8IkervbwjecYaRPziIeXuIuRNvESKRQjjRV+4f6XN8xQPkQ4wO0Ga9QPEeazKER4gi6uA3T9zy1KeCZb8McAY1RThCkpRtjAh2yJd/EuG0txPzy68BxT4RM2RS1FWEZH+IS1UUoRpuQ1JPw7sDfCDcnTx0rWrJ1pYSTckDwPuNgU/ADJ7ku1B60bXwAAAABJRU5ErkJggg==');
        }
"""

markdown_css_link= """<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/3.0.1/github-markdown.css">"""

if __name__ == '__main__':
    import webbrowser
    import argparse

    parser = argparse.ArgumentParser(description='Run a local markdown compiling server')

    parser.add_argument('-a', '--no-refresh', action='store_false',
                        help="Don't refresh the page",
                        dest='autorefresh')
    parser.add_argument('-p', '--port', type=int, default=10543,
                        help="Port to be opened, default is 10543",
                       )
    parser.add_argument('-b', '--no-open-browser', action='store_false',
                        help="Don't open browser",
                        dest='open_browser')
    parser.add_argument('-r', action='store_true',
                        help="""Allow remote access to this server.
                        Warning! can be extremely dangerous!"""
                       )
    parser.add_argument('path', nargs='?', help="Path to directory")
    config = parser.parse_args()

    create_header(config.autorefresh)
    create_header_export()
    create_topmenufile()
    create_beforefile()
    create_afterfile()

    if config.path != None:
        if os.path.isdir(config.path):
            os.chdir(config.path)
            indir = os.path.abspath('.')
        else:
            raise Exception('Unknown path argument')

    if config.open_browser:
        webbrowser.get().open('http://localhost:{}/'.format(config.port))

    host = 'localhost'
    if config.r:
        host = ''

    bottle.run(host=host, port=config.port)
    shutil.rmtree(outdir)


#===============================================================================
# The functions in this file perform differnt data operations over lists
#
# Copyright (C) Julian Ramos 2013
#===============================================================================
import numpy as np
import time
import datetime
import numpy as np
import dataSearchUtils as dataS
import matplotlib.pyplot as plt
import pytz
import csv
import stringUtils as sU
import os

def alphaNumExtract(text):
    '''
    This function separates the string
    into subsequences of alpha and numeric
    values
    '''
    a=list(text)
    output=[]
    previous=[]
    for i in a:
        val=i.isdigit()
        if  val and previous==[]:
            output.append(i)
            previous=val
        if val == previous:
            output[-1]=output[-1]+i
            previous=val
        else:
            previous=val
            output.append(i)
    return output

def list2Dict(data):
    '''
    Transforms a one dimensional list (a list without any other lists as elements)
    in to a dictionary that uses the elements of the list as the key for
    the dictionary and the index as the data stored for that key
    '''
    outDict={}
    for i in range(len(data)):
        outDict[str(data[i])]=i

    return outDict

def deleteStrInds(data,inds):
    '''
    def deleteStrInds(data,inds):
    This function searches through the data and eliminates the indexes in inds
    to avoid problems it goes from the least item to the first
    Also the inds are sorted to prevent erasing earlier in the list items
    that cause to erase by mistake later items
    >>> a='asd,asd,asd,123,asd,'
    >>> inds=[3,7,11]
    >>> deleteStrInds(a,inds)
    'changes done directly on the input str'
    >>> print a
    'asdasdasd123,asd,'
    '''
    inds=sorted(inds)
    temp=''
    for i in range(len(data)-1,-1,-1):
        if len(dataS.find(inds,lambda x:x==i))>0:
            pass
        else:
           temp='%s%s'%(data[i],temp)

    return temp

def deleteInds(data,inds):
    '''
    def deleteInds(data,inds):
    This function searches through the data and eliminates the indexes in inds
    to avoid problems it goes from the least item to the first
    Also the inds are sorted to prevent erasing earlier in the list items
    that cause to erase by mistake later items
    >>> a=[1,2,3,4,5,6,7,8,9,1,2,3,4,1,2,3,4,5]
    >>> inds=[10,1,3]
    >>> deleteInds(a,inds)
    'changes done directly on the input list'
    >>> print a
    [1, 3, 5, 6, 7, 8, 9, 1, 3, 4, 1, 2, 3, 4, 5]
    '''
    inds=sorted(inds)
    for i in range(len(data)-1,-1,-1):
        if len(dataS.find(inds,lambda x:x==i))>0:
            data.pop(i)

    return 'changes done directly on the input list'

#     dataout=[data[i] for i in range(len(data)-1,-1,-1) if len(dataS.find(inds,lambda x:x==i))==0]
#     return dataout
    #getvar=lambda searchList, ind: [searchList[i] for i in ind if len(dataS.find(ind,ind==i))==0]
    #return getvar(data,inds)

#    fromApptoCode(data):
#    This function changes the symbols in the list to
#    codes and returns both the codes and the data set
#    changed to the codes representation
def fromApptoCode(data):
    appsList=list()
    newData=list()
    for i in range(len(data)):
        inds=dataS.find(appsList, lambda x:x==data[i])
        if len(inds)!=0:
            newData.append(inds)
        else:
            appsList.append(data[i])
            inds=dataS.find(appsList, lambda x:x==data[i])
            newData.append(inds)
    return [newData,appsList]

#function time2vec=timevectransform(timestamps)
#This function takes the timestamps data and transforms in a way that
#geomatrically the measurements between values will correspond to those of
#the time format. As an example: the difference in hours between 22 and 1
#is actually is 3 hours and not 21 which will be the regular difference.
#Instead what we can do is map the hours values to a circle coordinates.
#To find the equivalency we simply have two tables one with the circle
#coordinates and the other with the hours. We do not explicitely create a
#function but instead look at the hours equivalent for each value. This
#trick is useful when you want to try to do clustering and you cannot
#use a custom made distance metric. In that case you could use the
#euclidean and this transformation and it will work approximately like
#using a special distance for the time.

def time2vecHM(Hours,Minutes):
    """
    time2vecHM(Hours,Minutes):
    Hours are integers in a list
    Minutes are integers in a list
    This function works like time2vec except that receives the hours and minutes
    directly instead of using timestamps
    """

    time2vec.x=np.array([])
    time2vec.y=np.array([])
    time2vec.z=np.array([])

    if len(time2vec.x)==0 and len(time2vec.y)==0 and len(time2vec.z)==0:
        r=5
        step=0.001
        theta=np.arange(0,np.pi*2,step)

#        time2vec.x=[r*np.cos(i) for i in theta]
        time2vec.x=-r*np.cos(theta)
        time2vec.y=r*np.sin(theta)
#        time2vec.y=[r*np.sin(i) for i in theta]
#         plt.plot(time2vec.x,time2vec.y)
#         plt.show()
        step2=24./len(theta)
        time2vec.z=np.arange(0,24,step2)
#        plt.plot(time2vec.x,'r')
#        plt.plot(time2vec.y,'b')
#        plt.plot(time2vec.z,'k')
#        plt.scatter(time2vec.x,time2vec.y)

#        plt.show()

#    plt.scatter(time2vec.x, time2vec.y)
#    plt.show()
#
    #Changing from minutes to a scale that goes between 0 to 99
#    print(datetime.datetime.fromtimestamp(int(timestamps)).strftime('%Y-%m-%d %H:%M:%S'))
    hours=Hours
    minutes=Minutes
#     print hours[1:10]
#     print minutes[1:10]

    timeDec=[hours[i]+np.divide(int(minutes[i]),60.) for i in range(len(hours))]
#     print timeDec[1:10]
    timevec=list()
#    print time2vec.x.size
#    print time2vec.y.size
#    print time2vec.z.size

    for i in timeDec:
        ind=np.argmin(np.abs(time2vec.z-i));
        timevec.append([time2vec.x[ind], time2vec.y[ind]])
#
#     X=[i[0] for i in timevec]
#     Y=[i[1] for i in timevec]
#     plt.plot(X,Y,alpha=0.01)
#     plt.show()

    return timevec

def time2vec(timestamps):
    time2vec.x=np.array([])
    time2vec.y=np.array([])
    time2vec.z=np.array([])

    if len(time2vec.x)==0 and len(time2vec.y)==0 and len(time2vec.z)==0:
        r=5
        step=0.001
        theta=np.arange(0,np.pi*2,step)

#        time2vec.x=[r*np.cos(i) for i in theta]
        time2vec.x=-r*np.cos(theta)
        time2vec.y=r*np.sin(theta)
#        time2vec.y=[r*np.sin(i) for i in theta]

        step2=24./len(theta)
        time2vec.z=np.arange(0,24,step2)
#        plt.plot(time2vec.x,'r')
#        plt.plot(time2vec.y,'b')
#        plt.plot(time2vec.z,'k')
#        plt.scatter(time2vec.x,time2vec.y)

#        plt.show()

#    plt.scatter(time2vec.x, time2vec.y)
#    plt.show()
#
    #Changing from minutes to a scale that goes between 0 to 99
#    print(datetime.datetime.fromtimestamp(int(timestamps)).strftime('%Y-%m-%d %H:%M:%S'))
    hours=[int(datetime.datetime.fromtimestamp(int(i)).strftime('%H')) for i in timestamps]
    minutes=[int(datetime.datetime.fromtimestamp(int(i)).strftime('%M')) for i in timestamps]
#     print hours[1:10]
#     print minutes[1:10]

    timeDec=[hours[i]+np.divide(int(minutes[i]),60.) for i in range(len(timestamps))]
#     print timeDec[1:10]
    timevec=list()
#    print time2vec.x.size
#    print time2vec.y.size
#    print time2vec.z.size

    for i in timeDec:
        ind=np.argmin(np.abs(time2vec.z-i));
        timevec.append([time2vec.x[ind], time2vec.y[ind]])
#
#    X=[i[0] for i in timevec]
#    Y=[i[1] for i in timevec]
#    plt.plot(X,Y)
#    plt.show()

    return timevec

#timestamps=[1341884014,1341884014,1341884014,1341884014,1341884014,1341884014,1341884014]
#time2vec(timestamps)

def time2dec(timeData,basis,**kwargs):
    if 'timezone' in kwargs:
        tz=kwargs['timezone']
    else :
        tz='Asia/Seoul'
    print('This function is currently using Seoul\'s time zone when no other time zone is provided')
    H=np.array([ int(datetime.datetime.fromtimestamp(int(i),tz=pytz.timezone(tz)).strftime('%H')) for i in timeData])
    M=np.array([ int(datetime.datetime.fromtimestamp(int(i),tz=pytz.timezone(tz)).strftime('%M')) for i in timeData])
    S=np.array([ int(datetime.datetime.fromtimestamp(int(i),tz=pytz.timezone(tz)).strftime('%S')) for i in timeData])

#   H=np.array([ int(datetime.datetime.fromtimestamp(int(i),tz=pytz.timezone('Asia/Seoul')).strftime('%H')) for i in timeData])
#   M=np.array([ int(datetime.datetime.fromtimestamp(int(i)).strftime('%M')) for i in timeData])
#   S=np.array([ int(datetime.datetime.fromtimestamp(int(i)).strftime('%S')) for i in timeData])
#   dates=[ datetime.datetime.fromtimestamp(int(i)).strftime('%Y-%m-%d %H:%M:%S') for i in timeData]

    D=[datetime.datetime.fromtimestamp(int(i)).weekday() for i in timeData]

    if basis=='Daily':
        dec=np.int_(H)+np.int_(100*M/60.0)/100.0+ np.int_(100*S/60.0)/10000.0
    if basis=='Weekly':
        dec=D+np.int_(100*H/24.0)/100.0+np.int_(100*M/60.0)/10000.0+ np.int_(100*S/60.0)/1000000.0
    if basis=='Hourly':
        dec=np.int_(M)+ np.int_(100*S/60.0)/100.0
#     dec=np.int_(100*H/24.0)/100.0+np.int_(100*M/60.0)/10000.0+ np.int_(100*S/60.0)/1000000.0

    return dec

def data2csv(filein,fileout,delimiters):
    '''
    def data2csv(filename):
    This function transforms the input data set defined by filein to a comma separated file fileout
    using the delimiters specified.

    The delimiters should passed as a list, as an example:

    delimiters=[',','\t', '|']
    '''

    fout=open(fileout,'w')
    if fout==-1:
        print('Could not create the output file')
    print('Starting conversion operation for '+filein)
    with open(filein,'r') as f:
        for line in f:
            for di in delimiters:
                line=line.replace(di,',')
            line=line.replace(',,,,,',',')
            line=line.replace(',,,,',',')
            line=line.replace(',,,',',')
            line=line.replace(',,',',')
            if line[-2]==',':
                line=line[0:-2]
            #If the end of the line does not have a newline just add it
            if str(line[-1:])!=str('\n'):
                line=line+str('\n')
            fout.write(line)
    fout.close()
    print('Conversion done')

def multl2sl(*args):
    '''
    This function takes all the input lists and transform it to a
    single list
    '''
    superList=[]
    cols=len(args)
    rows=len(args[0])

    if np.std([len(args[i]) for i in range(cols)]) != 0:
        print('the number of rows is not the same accross lists')
        return None

    superList=[[ args[i][i2] for i in range(cols)] for i2 in range(rows)]
#     for i2 in range(rows):
#         temp=[]
#         for i in range(cols):
#             temp.append(args[i][i2])
#         superList.append(temp)
    return superList

def minimum(data):
    col=np.argmin(np.min(data,axis=0))
    row=np.argmin(data[:,col])
    return {'row':row,'col':col,'val':data[row,col]}

def maximum(data):
    col=np.argmax(np.max(data,axis=0))
    row=np.argmax(data[:,col])
    return {'row':row,'col':col,'val':data[row,col]}

def timestamp2Str(timestamp,**kwargs):

    #Using the desktop's local time zone

    Y=datetime.datetime.fromtimestamp(timestamp).strftime('%Y')
    m=datetime.datetime.fromtimestamp(timestamp).strftime('%m')
    d=datetime.datetime.fromtimestamp(timestamp).strftime('%d')
    H=datetime.datetime.fromtimestamp(timestamp).strftime('%H')
    M=datetime.datetime.fromtimestamp(timestamp).strftime('%M')
    S=datetime.datetime.fromtimestamp(timestamp).strftime('%S')
    F=datetime.datetime.fromtimestamp(timestamp).strftime('%f')

    return Y+'/'+m+'/'+d+'/'+H+'/'+M+'/'+S+'/'+F

def str2timestamp(timestamp,**kwargs):
    if 'timezone' in kwargs:
        tz=kwargs['timezone']
    else :
        tz='UTC'
        print('Using the default time zone UTC')
#     return datetime.datetime.strptime(timestamp, "%H:%M:%S:%f").timetuple()
    return time.mktime(datetime.datetime.strptime(timestamp,"%H:%M:%S:%f").timetuple())

def appendDicts(dictA,dictB):
    '''
    This function takes two dictionaries with the same
    keys and append their contents into one new dictionary.
    The function assumes the values for the keys are lists
    '''
    if type(dictA) is list:
        dictA=dictA[0]
    if type(dictB) is list:
        dictB=dictB[0]

    dictC=dictB
    for i in dictA.keys():
        dictC[i]+=dictA[i]

    return dictC

def incompDate2timestamp(dateStr,timestamp,**kwargs):
    '''
    Creates a timestamp from an incomplete date. Date has information
    about the Hour, minute, second and millisecond only.
    '''
    baseDate=timestamp2Str(timestamp).split('/')
    partialStr=baseDate[0]+'/'+baseDate[1]+'/'+baseDate[2]+'/'

    if 'irregular' in kwargs:
        temp2=dateStr.split(':')
        if int(temp2[0])<9:
            temp2[0]=str(int(temp2[0])+12)
        dateStr=':'.join(temp2)

    date2=time.mktime(datetime.datetime.strptime(partialStr+dateStr, "%Y/%m/%d/%H:%M:%S:%f").timetuple())
    temp=dateStr.split(':')
    return {'timestamp':date2,'milliseconds':int(temp[-1]),'milliTimestamp':date2+int(temp[-1])/float(1000)}

def combiner(randFiles,outputFilename):
    """
    This function combines all of the data sets produced from random sampling into a single
    file. ALl of the files must be in the same folder and have a common and unique prefix

    path : path to the random sample files
    outputFilename : name of the output file
    """
    print("files merged into %s"%(outputFilename))
    outfile=open(outputFilename,"w")

    for i in randFiles:
        file=open(i,"r")
        for line in file:
            outfile.write(line)
        file.close()
    outfile.close()

def splitter(filename,path,prefix="random",lines=50000,testMode=False):
    """
    Splits the data set into a number of subsets.
    filename : Original file to split
    path : path to where the files are going to be stored
    prefix : This is the name for each of the subsets plus a number for instance
    if prefix = subset then the stored file name will be subset_1.tsv, subset_2.tsv ....
    lines : Number of lines to be stored on each file created, the last file will always have at
    most this number of lines
    """

    if file_len(filename)<lines:
        print("No need to split the data set")
        return [filename]

    input_file = open(filename, 'r')
    input_reader = csv.reader(input_file, delimiter="\t")
    output_file=open(path+"/"+prefix+"_1.tsv","w")
    output_writer=csv.writer(output_file,delimiter="\t")
    count=0
    filenum=1
    filesCount=0
    files=[]
    for row in input_reader:
        output_writer.writerow(row)
        count+=1
        if count==lines:
            count=0
            output_file.close()
            if testMode:
                filesCount+=1
            if filesCount==8:
                break
            filenum+=1
            filename=path+"/"+prefix+"_%d.tsv"%(filenum)
            output_file=open(filename,"w")
            output_writer=csv.writer(output_file,delimiter="\t")
            files.append(filename)

    input_file.close()

    #This code breaks when there is a single file
    return files

def file_len(fname):
    with open(fname) as f:
        for i, l in enumerate(f):
            pass
    return i + 1

def createSplitFolder(filenames,testMode=False):
    """
    Creates folders to store files generated from split
    if they do not exist
    Mainly to be used with splitter
    """
    folderstoDelete=[]
    files=[]
    for i in filenames:
        idx=np.random.randint(0,10,5)
        idx=[str(s) for s in idx]
        idx="".join(idx)
        path=sU.extractPath(i)+"/temp_%s_"%(sU.extractFilename(i))+idx
        folderstoDelete.append(path)
        if not(os.path.isdir(path)):
            os.mkdir(path)
        files+=splitter(i,path,prefix="split",testMode=testMode)
    return folderstoDelete,files

def createFolder(filename):
    idx=np.random.randint(0,10,5)
    idx=[str(s) for s in idx]
    idx="".join(idx)
    path=sU.extractPath(filename)+"/temp_rand_%s_"%(sU.extractFilename(filename))+idx
    if not(os.path.isdir(path)):
        os.mkdir(path)
    return path

#===============================================================================
# #Test data2csv
# data2csv('K:/research_data_sets/Activity recognition study/S03_1/1360681304.dat','K:/research_data_sets/Activity recognition study/S03_1/1360681304.csv',['\t',' ',',','|','#',';'])
#===============================================================================
if __name__ == "__main__":
    import doctest
    timestamps=[]
    for i in range( 1405296000 ,  1405382399 ,3600):
        timestamps.append(i)
    coords=time2vec(timestamps)
    X=[i[0] for i in coords]
    Y=[i[1] for i in coords]
    plt.plot(X,Y,'o')
    plt.show()

#     print(time2vec([1405363573]))
#     print(time2vec([1405376269]))
    doctest.testmod()

    #Test for the splitter
    filename="/Users/ingenia/git/data/data_sampling/user_bot_Data.tsv"
    path="/Users/ingenia/git/data/data_sampling/splits"
    prefix="split_data"
    splitter(filename,path,prefix,100000)