"""
This is a trend-following system
"""

import pandas as pd
import numpy as np
import itertools, talib
import time
from get_fx_data import *
import pytz
import datetime as dt
import os
import json
from s_r import s_r
import yaml
from oandapyV20.contrib.requests import StopOrderRequest, StopLossOrderRequest, LimitOrderRequest, MarketOrderRequest, TakeProfitDetails, \
    StopLossDetails, TrailingStopLossOrderRequest, TrailingStopLossDetails, TradeCloseRequest, ClientExtensions
import oandapyV20.endpoints.orders as orders
import oandapyV20.endpoints.trades as trades
import oandapyV20.endpoints.transactions as trans
import oandapyV20.endpoints.accounts as accounts
import oandapyV20.endpoints.positions as positions
import os.path
# from get_journal import *
#import fence_swing_scanner as sc
import pygsheets
import math
import statistics

overall = CandleLogic('D', 'EUR_USD')
accountID = overall.user1.accountID
client = overall.user1.client
slippage = 0.00025

sr = s_r()
#in_play = sc.main()
#print('pairs in play', in_play)

class ScannerLogic():
    def __init__(self):
        self.n_window = 3
        self.risk_level = 0.4
        self.trades_repo = pd.DataFrame()

    def get_index(self, ele, intervals):
        '''returns the index position of matched interval in a list of intervals
           the list of invervals are listed from the smallest to the largest
           smallest intervals shows up first in the list
           largest intervals shows up last in the list
        '''
        for intv in intervals:
            if ele in intv:
                return intervals.index(intv)

    def get_indecision(self, candles):
        direction = ''
        body = np.absolute(candles['c'].iloc[-3] - candles['o'].iloc[-3])
        upper_wick = abs(candles['h'].iloc[-3] - max(candles['o'].iloc[-3], candles['c'].iloc[-3]))
        lower_wick = abs(min(candles['o'].iloc[-3], candles['c'].iloc[-3]) - candles['l'].iloc[-3])

        if ((upper_wick >= body) and (candles['c'].iloc[-3] <= candles['o'].iloc[-3])) or \
                ((upper_wick >= body) and (candles['c'].iloc[-2] <= candles['c'].iloc[-3])):
            direction = 'short'
            return direction, True
        elif ((lower_wick >= body) and (candles['c'].iloc[-3] >= candles['o'].iloc[-3])) or \
                ((lower_wick >= body) and (candles['c'].iloc[-2] >= candles['c'].iloc[-3])):
            direction = 'long'
            return direction, True
        else:
            return direction, False

    def grab_reversal(self, granularity, pair, units):
        operator = CandleLogic(granularity, pair)
        candles = operator.OHLC()

        if granularity in ['D', 'H12', 'H8', 'H6', 'H4', 'H1']:
            body = np.absolute(candles['c'].iloc[-4] - candles['o'].iloc[-4])
            upper_wick = abs(candles['h'].iloc[-4] - max(candles['o'].iloc[-4], candles['c'].iloc[-4]))
            lower_wick = abs(min(candles['o'].iloc[-4], candles['c'].iloc[-4]) - candles['l'].iloc[-4])

            if units > 0:  # if a long trade
                if ((upper_wick >= body) or (candles['c'].iloc[-3] < candles['c'].iloc[-4])) and \
                        (candles['c'].iloc[-2] < candles['c'].iloc[-3]):  # reversed in the process of a long trade
                    return True
            elif units < 0:  # if a short trade
                if ((lower_wick >= body) or (candles['c'].iloc[-3] > candles['c'].iloc[-4])) and \
                        (candles['c'].iloc[-2] > candles['c'].iloc[-3]):  # reversed in the process of a short trade
                    return True
            else:
                return False
        else:
            return False

    def hurst_f(self, series, lags_to_test=20):
        # interpretation of return value
        # hurst < 0.5 - input_ts is mean reverting
        # hurst = 0.5 - input_ts is effectively random/geometric brownian motion
        # hurst > 0.5 - input_ts is trending
        tau = []
        lagvec = []
        #  Step through the different lags
        for lag in range(2, lags_to_test):
            #  produce price difference with lag
            pp = np.subtract(np.array(series)[lag:], np.array(series)[:-lag])
            #  Write the different lags into a vector
            lagvec.append(lag)
            #  Calculate the variance of the differnce vector
            tau.append(np.sqrt(np.std(pp)))
        #  linear fit to double-log graph (gives power)
        m = np.polyfit(np.log10(lagvec), np.log10(tau), 1)
        # calculate hurst
        hurst = m[0]*2

        return hurst

    def get_confirmation(self, long_short, candles):
        if long_short == 'long' and candles['c'].iloc[-2] > candles['c'].iloc[-3]:
            return True
        elif long_short == 'short' and candles['c'].iloc[-2] < candles['c'].iloc[-3]:
            return True
        else:
            return False

    def get_last_candle(self, granularity, pair):
        operator = CandleLogic(granularity, pair)
        candles = operator.OHLC()

        if granularity in ['D', 'H12', 'H8', 'H6']:
            return candles['c'].iloc[-1]
        else:
            return candles['c'].iloc[-1]

    def GRaB_PA_scan(self, window, granularity='D'):
        markup = {}
        markdown = {}
        range_long = {}
        range_short = {}
        breaks = {}
        notification = ''

        pairs = in_play

        for pair in pairs:
            print('pair', pair)
            candle = CandleLogic(granularity, pair)
            df = candle.OHLC()
            self.account_bal = float(
                client.request(accounts.AccountSummary(accountID))['account']['balance'])
            df.set_index('datetime', inplace=True)

            df['sma_200'] = talib.SMA(df['c'], timeperiod=200)
            df['sma_100'] = talib.SMA(df['c'], timeperiod=100)
            df['zscore'] = [(df['c'][i] - df['sma_100'][i]) / np.std(df['c'][:i]) for i in range(len(df['c']))]
            df['sma_50'] = talib.SMA(df['c'], timeperiod=50)
            df['ema_high'] = talib.EMA(df['h'], timeperiod=34)
            df['ema_low'] = talib.EMA(df['l'], timeperiod=34)
            df['ema_close'] = talib.EMA(df['c'], timeperiod=34)
            df['ema_slow'] = talib.EMA(df['c'], timeperiod=21)
            df['ema_fast'] = talib.EMA(df['c'], timeperiod=8)
            df['rsi'] = talib.RSI(df['c'], timeperiod=30)

            df['markup'] = df['c'] >= df['ema_high']
            df['markdown'] = df['c'] <= df['ema_low']

            print('zscore', df['zscore'].iloc[-1])
            print('hurst', self.hurst_f(df['c'][-100:]))
            print()

            t = trades.TradesList(accountID)
            tr = client.request(t)
            tr = [(ele['instrument'], ele['currentUnits'], ele['unrealizedPL']) if 'instrument' in ele else None for ele
                  in tr['trades']]
            od = orders.OrderList(accountID)
            odr = client.request(od)
            odr = [ele['instrument'] if 'instrument' in ele else None for ele in odr['orders']]

            # breakout / breakdown strategy
            if df['c'].iloc[-2] > max(df['h'].iloc[-49:-3]) and df['rsi'].iloc[-1] > 50 and self.hurst_f(df['c'][-100:0]) > 0.50:
                # if (df['ema_fast'].iloc[-1] - df['ema_slow'].iloc[-1] >= spread_breaks*100) and df['high_col'].iloc[-1]:
                print('triggered breakouts')
                # entry = (df['ema_fast'].iloc[-1] + slip_breaks*100) if 'JPY' in pair else (df['ema_fast'].iloc[-1] + slip_breaks)
                entry = df['o'].iloc[-1]
                entry = float("{:.3f}".format(entry)) if 'JPY' in pair else float("{:.5f}".format(entry))
                stop = entry - 2 * df['atr'].iloc[-1]
                stop = float("{:.3f}".format(stop)) if 'JPY' in pair else float("{:.5f}".format(stop))
                at_risk = abs(entry - stop)
                target = entry + 4 * df['atr'].iloc[-1]
                target = float("{:.3f}".format(target)) if 'JPY' in pair else float("{:.5f}".format(target))
                order = StopOrderRequest(
                    instrument=pair,
                    units=(self.account_bal * self.risk_level / 100 / at_risk) if 'JPY' not in pair else (
                            self.account_bal * self.risk_level / 100 / at_risk * 100),
                    # units=self.account_bal * self.risk_level / 100 / at_risk,
                    price=entry,
                        takeProfitOnFill=TakeProfitDetails(price=target).data,
                        stopLossOnFill=StopLossDetails(price=stop,
                                                      clientExtensions=ClientExtensions(clientTag="mytag").data),
                        clientExtensions=ClientExtensions(clientTag="mytag").data)

                r = orders.OrderCreate(accountID, data=order.data)
                if (pair not in [ele[0] for ele in tr]) and pair not in odr:
                    try:
                        rv = client.request(r)
                        notification += '\n' + pair + '--> is having a breakout on ' + granularity
                        notification += '\n' + pair + ' at_risk ' + str(at_risk)
                    except:
                        notification += '\nDemo account problem entering breakdown on ' + pair
                        pass

            # EMA fast and slow crossed for the downside, set pending order for short
            elif df['c'].iloc[-2] < min(df['l'].iloc[-49:-3]) and df['rsi'].iloc[-1] < 50 and self.hurst_f(df['c'][-100:]) > 0.50:
                print('triggered breakdowns')

                # entry = (df['ema_fast'].iloc[-1] - slip_breaks*100) if 'JPY' in pair else (df['ema_fast'].iloc[-1] - slip_breaks)
                entry = df['o'].iloc[-1]
                entry = float("{:.3f}".format(entry)) if 'JPY' in pair else float("{:.5f}".format(entry))
                stop = entry + 2 * df['atr'].iloc[-1]
                stop = float("{:.3f}".format(stop)) if 'JPY' in pair else float("{:.5f}".format(stop))
                at_risk = abs(entry - stop)
                target = entry - 4 * df['atr'].iloc[-1]
                target = float("{:.3f}".format(target)) if 'JPY' in pair else float("{:.5f}".format(target))
                order = StopOrderRequest(
                    instrument=pair,
                    units=(-self.account_bal * self.risk_level / 100 / at_risk) if 'JPY' not in pair else (
                            -self.account_bal * self.risk_level / 100 / at_risk * 100),
                    # units=self.account_bal * self.risk_level / 100 / at_risk,
                    price=entry,
                        takeProfitOnFill=TakeProfitDetails(price=target).data,
                        stopLossOnFill=StopLossDetails(price=stop,
                                                      clientExtensions=ClientExtensions(clientTag="mytag").data),
                        clientExtensions=ClientExtensions(clientTag="mytag").data)

                r = orders.OrderCreate(accountID, data=order.data)
                print(r)
                if (pair not in [ele[0] for ele in tr]) and pair not in odr:
                    try:
                        rv = client.request(r)
                        notification += '\n' + pair + '--> is on a breakdown on ' + granularity
                        notification += '\n' + pair + ' at_risk ' + str(at_risk)
                    except:
                        notification += '\nDemo account problem entering breakdown on ' + pair
                        pass

        return candle, notification

    def manage_trade(self, final_close=False):
        text = ''

        t = trades.TradesList(accountID)
        tr = client.request(t)

        to = orders.OrderList(accountID)
        tor = client.request(to)

        curr_trades = []
        for element in tr['trades']:
            if 'instrument' in element:
                curr_trades.append((element['instrument'], element['id'], element['unrealizedPL'],
                               element['currentUnits'], element['price']))

        curr_trades = pd.DataFrame(curr_trades)
        if len(curr_trades) > 0:
            curr_trades.columns = ['instrument', 'tradeID', 'unrealizedPL', 'currentUnits', 'open_price']

        stops = []
        for element in tor['orders']:
            try:
                if 'price' in element:
                    stops.append((element['tradeID'], element['type'], element['price'], element['id']))
                else:
                    stops.append((element['tradeID'], element['type'], element['distance'], element['id']))
            except:
                pass

        stops = pd.DataFrame(stops)
        if len(stops) > 0:
            stops.columns = ['tradeID', 'type', 'stops', 'orderID']

            all_data = curr_trades.merge(stops, how='left', on='tradeID')

            if final_close:
                for ele in all_data.instrument.unique():
                    close = TradeCloseRequest()
                    r = trades.TradeClose(accountID, tradeID=int(ele), data=close.data)
                    rv = client.request(r)

            else:
                for ele in all_data.instrument.unique():
                    print()
                    print('adjusting stop for', ele)
                    subset = all_data[all_data['instrument'] == ele]
                    ele_tradeID = subset['tradeID'].unique()[0]
                    ele_orderID = subset[(subset['instrument']==ele) & (subset['type']=='STOP_LOSS')]['orderID'].unique()[0]
                    units = float(subset['currentUnits'].unique()[0])
                    if float(subset.unrealizedPL.unique()[0]) > 0:
                        gain = float(subset.unrealizedPL.unique()[0]) / abs(
                            float(subset.currentUnits.unique()[0]))  # unit gain
                        print('gain per unit', gain)

                        sub2 = subset[(subset['type'].str.contains('LOSS')) & (subset['stops'])]
                        print('check')
                        print(sub2)
                        old_perc = float(subset.open_price.unique()[0]) / float(sub2.stops.unique()[0])
                        new_perc = 0.75 * old_perc
                        print('old', old_perc)
                        print('new', new_perc)
                        print('open price', float(subset.open_price.unique()[0]))
                        print('stops', float(sub2.stops.unique()[0]))
                        print('current', self.get_last_candle('H1', ele))
                        if units > 0:
                            if float(subset.open_price.unique()[0]) > float(sub2.stops.unique()[0]):
                                old_perc = float(subset.open_price.unique()[0]) / float(sub2.stops.unique()[0])
                            else:
                                old_perc = float(sub2.stops.unique()[0]) / float(subset.open_price.unique()[0])
                            new_perc = 0.75 * old_perc
                            new_stop = self.get_last_candle('H1', ele) * (1 - (new_perc / 100))
                            print(new_stop)
                            if 'JPY' in ele:
                                new_stop = float("{:.3f}".format(new_stop))
                                print(ele, new_stop)
                                stop = StopLossOrderRequest(tradeID=ele_tradeID, price=new_stop)
                                r = orders.OrderReplace(accountID=accountID, orderID=ele_orderID, data=stop.data)
                                rv = candle.user1.client.request(r)

                            else:
                                new_stop = float("{:.5f}".format(new_stop))
                                print(ele, new_stop)
                                stop = StopLossOrderRequest(tradeID=ele_tradeID, price=new_stop)
                                r = orders.OrderReplace(accountID=accountID, orderID=ele_orderID, data=stop.data)
                                rv = candle.user1.client.request(r)
                        else:
                            if float(subset.stops.unique()[0]) > float(sub2.open_price.unique()[0]):
                                old_perc = float(subset.stops.unique()[0]) / float(sub2.open_price.unique()[0])
                            else:
                                old_perc = float(sub2.open_price.unique()[0]) / float(subset.stops.unique()[0])
                            new_perc = 0.75 * old_perc
                            new_stop = self.get_last_candle('H1', ele) * (1 + (new_perc / 100))
                            if 'JPY' in ele:
                                new_stop = float("{:.3f}".format(new_stop))
                                print(ele, new_stop)
                                stop = StopLossOrderRequest(tradeID=ele_tradeID, price=new_stop)
                                r = orders.OrderReplace(accountID=accountID, orderID=ele_orderID, data=stop.data)
                                rv = candle.user1.client.request(r)

                            else:
                                new_stop = float("{:.5f}".format(new_stop))
                                print(ele, new_stop)
                                stop = StopLossOrderRequest(tradeID=ele_tradeID, price=new_stop)
                                r = orders.OrderReplace(accountID=accountID, orderID=ele_orderID, data=stop.data)
                                rv = candle.user1.client.request(r)

        return 'Done adjusting stops'

    def trail_trade(self, final_close=False):
        text = ''

        t = trades.TradesList(accountID)
        tr = client.request(t)
        tr = [(ele['instrument'], ele['id'], ele['unrealizedPL'], ele['currentUnits']) if 'instrument' in ele else None
              for ele in tr['trades']]
        print('manage trade1', tr)

        if final_close:
            for ele in tr:
                print(ele[0], ele[1])
                close = TradeCloseRequest()
                r = trades.TradeClose(accountID, tradeID=int(ele[1]), data=close.data)
                rv = client.request(r)

        else:
            for ele in tr:
                if float(ele[2]) / abs(float(ele[3])) >= 1 / 100:
                    print(ele[0], ele[1])
                    close = TradeCloseRequest()
                    r = trades.TradeClose(accountID, tradeID=int(ele[1]), data=close.data)
                    rv = client.request(r)

                if float(ele[2]) / abs(float(ele[3])) <= -0.40 / 100:
                    print(ele[0], ele[1])
                    close = TradeCloseRequest()
                    r = trades.TradeClose(accountID, tradeID=int(ele[1]), data=close.data)
                    rv = client.request(r)

        return text

if __name__ == "__main__":
    # while True:
    os.environ['TZ'] = 'America/Los_Angeles'
    print(dt.datetime.now())

    sr = s_r()
    in_play = ['USD_CAD', 'AUD_CAD', 'AUD_CHF', 'AUD_NZD', 'CAD_CHF', 'NZD_USD', 'CAD_JPY', 'CHF_JPY',
               'EUR_CAD', 'EUR_NZD', 'NZD_CHF', 'EUR_CHF', 'EUR_AUD', 'EUR_GBP', 'EUR_JPY', 'GBP_CAD',
               'GBP_AUD', 'GBP_CHF', 'GBP_JPY', 'NZD_CAD', 'GBP_NZD', 'NZD_JPY', 'EUR_USD', 'GBP_USD',
               'USD_CHF', 'USD_JPY', 'AUD_USD', 'AUD_JPY']
    print('pairs in play', in_play)

    candle = CandleLogic('D', 'EUR_USD')
    a = accounts.AccountSummary(accountID)
    ad = client.request(a)
    nav = float(ad['account']['NAV'])
    print(nav)
    t = trades.TradesList(accountID)
    tr = client.request(t)
    tr = [(ele['instrument'], ele['currentUnits']) if 'instrument' in ele else None for ele in tr['trades']]
    scanner = ScannerLogic()
    pnl = float(ad['account']['unrealizedPL'])
    bal = float(ad['account']['balance'])
    print('pnl', pnl)
    print('bal', bal)
    text = ''
    if nav >= (candle.user1.previous_nav * 1.005):
        print('dauly target met, close all trades')
        res = scanner.manage_trade(final_close=True)
        text += res
    else:
        if dt.datetime.now().hour in (2 + 1, 14 + 1):
            res = scanner.manage_trade(final_close=False)
            text += res
            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='D')
            text += notification

            res = scanner.manage_trade(final_close=False)
            text += res
            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='H12')
            text += notification
        elif dt.datetime.now().hour in (2 + 1, 10 + 1, 18 + 1):
            res = scanner.manage_trade(final_close=False)
            text += res
            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='H8')
            text += notification
        elif dt.datetime.now().hour in (2 + 1, 8 + 1, 14 + 1, 20 + 1):
            res = scanner.manage_trade(final_close=False)
            text += res
            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='H6')
            text += notification
        elif dt.datetime.now().hour in (2 + 1, 6 + 1, 10 + 1, 14 + 1, 18 + 1, 22 + 1):
            res = scanner.manage_trade(final_close=False)
            text += res
            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='H4')
            text += notification
        else:
            res = scanner.manage_trade(final_close=False)
            text += res

            candle, notification = scanner.GRaB_PA_scan(window=34, granularity='H1')

        print(text)
        print(scanner.trades_repo)


#!/usr/bin/python3
import requests
import sys
import os

apikey = '0f1e7810502fded'
apiurl = 'https://api.imgur.com/3/upload'
deleteurl = 'https://imgur.com/delete/{}'

def upload(f: str) -> 'img url' 'delete url':
    f = f.replace('~', os.path.expanduser('~'), 1)
    try:
        with open(f, 'rb') as f:
            headers = {'Authorization': 'Client-ID {}'.format(apikey)}
            data = {'image': f.read()}
            req = requests.post(apiurl, headers=headers, data=data)
            if req.status_code == 415:
                return 'Invalid File', 'Invalid File'
            json = req.json()
            if json['success']:
                return (json['data']['link'],
                        deleteurl.format(json['data']['deletehash']))
    except Exception as e:
        print('Error: ' + e)

def main():
    if len(sys.argv) == 1:
        print('Usage: {} <image>'.format(sys.argv[0].split('/')[-1]))
        sys.exit(1)
    url, delete = upload(sys.argv[1])
    print(url)
    print(delete, file=sys.stderr)

if __name__ == '__main__':
    main()


import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

def read_data(temp_list: list, data_dir: str):
    # 根据temp_list中所指示的温度，在data_dir中找到相关文件，最终将文件的值存储在字典中并返回
    # 返回字典的格式
    # return_value[温度值] = [
    #     (波长0，强度0),
    #     (波长1，强度1),
    #     ...  ...  ...
    # ]
    tmp = []
    return_value = {}

    for temperature in temp_list:
        file_name = temperature + " 度.txt"
        file_path = data_dir + file_name

        f = open(file_path, 'r')
        lines = f.readlines()
        f.close()

        for i in range(1, len(lines)):
            try:
                tmp.append((int(lines[i].split("\t")[0]), float(lines[i].split("\t")[1])))
            except:
                break

        return_value[int(temperature)] = tmp
        tmp = []

    return return_value

def add_zeros(pair_list: list, range_num: tuple):
    # 将数据空位补齐，在lambda缺省处填零占位
    tmp = []
    if pair_list[0][0] > range_num[0]:
        for i in range(range_num[0], pair_list[0][0]):
            tmp.append((i, 0))

    tmp.extend(pair_list)

    if pair_list[-1][0] < range_num[1]:
        for i in range(pair_list[-1][0] + 1, range_num[1]):
            tmp.append((i, 0))

    return tmp

temperature_list = ["40", "50", "60", "70", "80", "90", "100", "125", "150", "175", "200", "225", "250", "275", "300"]
root_dir = "H:/lambda_temp_strength/"

data_dict = read_data(temperature_list, root_dir)
axis_temperature = []
axis_strength = []
# 温度范围限定450~750
axis_lambda = range(450, 751)

for key in data_dict.keys():
    axis_temperature.append(key)
    # 下一行测试专用
    # print(add_zeros(data_dict[key], (450, 751)))
    for lambda_strength_pair in add_zeros(data_dict[key], (450, 751)):
        axis_strength.append(lambda_strength_pair[1])
    # 下两行测试专用
    # print(len(axis_strength))
    # break

# 生成z轴数据网格
z_axis = np.array(axis_strength).reshape(len(axis_temperature), len(axis_lambda)).transpose()
del axis_strength

# 生成x轴、y轴数据网格
x_axis, y_axis = np.meshgrid(axis_temperature, axis_lambda)
del axis_temperature
del axis_lambda

############################################################
# 曲面图
# 创建图形对象
fig = plt.figure()
ax = Axes3D(fig)
ax.plot_surface(x_axis, y_axis, z_axis, cmap="rainbow")
# 设定坐标轴
ax.set_xlabel("temperature")
ax.set_ylabel("lambda")
ax.set_zlabel("strength")
# 画图
plt.show()
#############################################################
# # 散点图
# # 定义坐标轴
# fig4 = plt.figure()
# ax4 = plt.axes(projection='3d')
# # 画出散点图
# ax4.scatter(x_axis, y_axis, z_axis)
# # 设定坐标轴
# ax4.set_xlabel("temperature")
# ax4.set_ylabel("lambda")
# ax4.set_zlabel("strength")
# # 画图
# plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import urllib.request
import json

API="http://127.0.0.1:10088/openxiaoice/ask?q="

def reply(string):
	try:
		req = urllib.request.Request(API + urllib.request.quote(string.encode("utf8")))
		req = urllib.request.urlopen(req).read()
		ans = json.loads(req.decode('utf-8'))["answer"]
		print(ans)
		return ans
	except Exception as e:
		print(e)
		return "玩坏掉了Orz..."

if __name__ == "__main__":
	reply("Hello")


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

import pandas as pd
from pandas import DataFrame
import json

data1 = pd.read_json("categorized-comments.jsonl", lines = True)

import re
import nltk
nltk.download('stopwords')
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords

data1.loc[data1["cat"]=="sports", "cat"]=0
data1.loc[data1["cat"]=="science_and_technology", "cat"]=1
data1.loc[data1["cat"]=="video_games", "cat"]=2
data1.loc[data1["cat"]=="news", "cat"]=3

data1.cat.astype(int).head()

data1.dtypes

data1.head()

ps = PorterStemmer()

# =============================================================================
# corpus = []
# for i in range(0, 2347476, n_jobs = -1):
#         txt = re.sub('[^a-zA-Z]', " ", data1["txt"][0])
#         txt = txt.lower()
#         txt = txt.split()
#         txt = [ps.stem(word) for word in txt if not word in set(stopwords.words("english"))]
#         txt = " ".join(txt)
#         corpus.append(txt)
#
# =============================================================================
# =============================================================================
# data1.info(memory_usage = "deep")
#
# X = data1.iloc[:1].values
# y = data1.iloc[:0].values
# =============================================================================

# =============================================================================
# X = data1.txt
# y = data1.cat
#
# from sklearn.cross_validation import train_test_split
#
# X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.20, random_state = 1)
#
# from sklearn.feature_extraction.text import CountVectorizer
#
# vect = CountVectorizer(max_features = 1000)
# vect.fit(X_train)
# x_train_dtm = vect.transform(X_train).toarray()
#
# d = x_train_dtm[0:5, 1:10]
#
# from sklearn.linear_model import LogisticRegressionCV
#
# classifier = LogisticRegressionCV(n_jobs = -1, random_state = 0, penalty = "l2")
# classifier.fit(x_train_dtm, y_train.astype("int"))
# =============================================================================

data_0 = data1[data1["cat"]==0].iloc[0:2000]
data_1 = data1[data1["cat"]==1].iloc[0:2000]
data_2 = data1[data1["cat"]==2].iloc[0:2000]
data_3 = data1[data1["cat"]==3].iloc[0:2000]

data_condensed = pd.concat([data_0, data_1], axis=0)
data_condensed = pd.concat([data_condensed, data_2], axis=0)
data_condensed = pd.concat([data_condensed, data_3], axis=0)

corpus = []

for i in range(0, 8000):
        txt = re.sub('[^a-zA-Z]', " ", data1["txt"][i])
        txt = txt.lower()
        txt = txt.split()
        txt = [ps.stem(word) for word in txt if not word in set(stopwords.words("english"))]
        txt = " ".join(txt)
        corpus.append(txt)

from sklearn.feature_extraction.text import CountVectorizer

cv = CountVectorizer(max_features = 5000)
X = cv.fit_transform(corpus).toarray()
y = data_condensed.iloc[:, 0].values

from sklearn.cross_validation import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.25, random_state = 1)

from sklearn.linear_model import LogisticRegressionCV

classifier = LogisticRegressionCV(n_jobs = -1, random_state = 0, penalty = "l2")
classifier.fit(X_train, y_train.astype("int"))

test_predictions = classifier.predict(X_test)
train_predictions = classifier.predict(X_train)

from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, test_predictions)

accuarcy_test = accuracy_score(y_test, test_predictions)
accuarcy_train = accuracy_score(y_train, train_predictions)


# coding: UTF-8
import urllib.request
from bs4 import BeautifulSoup
from datetime import datetime
import csv
import time

f = open('madomagi_miku_sumiyoshi.csv','a')
writer = csv.writer(f,lineterminator='\n')

csv_list = []

time_ = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
csv_list.append(time_)

# アクセスするURL
url = 'https://daidata.goraggio.com/100433/unit_list?hist_num=1&model=SLOT%E9%AD%94%E6%B3%95%E5%B0%91%E5%A5%B3%E3%81%BE%E3%81%A9%E3%81%8B%E2%98%86%E3%83%9E%E3%82%AE%E3%82%AB&ballPrice=20.00&disp=1&graph=1'

try:
    # htmlオブジェクトが返ってくる
    print('httpリクエストの送信')
    html = urllib.request.urlopen(url)

    # htmlをBeautifulSoupで扱い、htmlタグのかたまりが返ってくる
    soup = BeautifulSoup(html, "html.parser")
    td = soup.find_all("td")

    i = 0
    for tag in td:
        if tag.a != None:
            csv_list.append(tag.a.string)
        elif tag.string != None:
            csv_list.append(tag.string)
        i += 1
        if i == 12:
            print(csv_list)
            writer.writerow(csv_list)
            i = 0
            csv_list = []
            time_ = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
            csv_list.append(time_)
except:
    pass

f.close()


"""

	HUC12 Navigator class for use with Route table.

	this uses pure python (no numpy) and navigates using python dictionaries.

"""

import os
import sys
import logging
import csv
from collections import defaultdict,OrderedDict
from datetime import datetime as dt
import tzlocal
# from xmljson import BadgerFish
# import xml.etree.ElementTree
from json import dumps
try:
	import cPickle as pickle
except:
	import pickle

log = logging.getLogger(__name__)

class RouteHUC12Navigator(object):

	def __init__(self):
		'''
		Constructor
		'''
		self.args = None

		self.config = None

		self.overall_log_file = logging.DEBUG

		self.file_log_level = None

		self.console_log_level = logging.WARNING

		self.navigation_file = None

		self.navigation_file_key = 'huc12'

		self.navigation_attribute_list = [ 'name', 'area_sq_km', 'water_area_sq_km' ]

		self.navigation_attributes = defaultdict( dict )

		self.navigation_file_keys = {
				'upstream':   {'from': 'huc12',    'to': 'huc12_us'},
				'downstream': {'from': 'huc12_us', 'to': 'huc12'}
		}

		self.us_huc_route = defaultdict( dict )

		self.ds_huc_route = defaultdict( dict )

		self.attribute_file = None

		self.attribute_file_key = 'HUC_12'

		self.attribute_data = defaultdict( dict )

		self.attribute_data_keys = []

		self.metadata_file = None

		self.metadata = None

		self.metadata_list_file = None

		self.metadata_list = defaultdict( dict )

		self.direction = 'Upstream'

		self.navigation_results = None

		self.results_length = 0

		self.upstream_result_length = 0

	@property
	def config(self):
		return self.__config

	@config.setter
	def config(self, config):
		if not config == None:
			self.__config = config
			if config.has_option('PATHS', 'log_file'):
				self.overall_log_file = config.get('PATHS', 'log_file')
			if config.has_option('DEFAULTS', 'file_log_level'):
				self.file_log_level = self.config.get('DEFAULTS', 'file_log_level')

	@property
	def args(self):
		return self.__args

	@args.setter
	def args(self, args):
		if not args == None:
			self.__args = args
			if self.__args:
				if hasattr(self.__args, 'logger'):
					numeric_level = getattr(logging, self.__args.logger.upper(), None)
					if not isinstance(numeric_level, int):
						raise ValueError('Invalid log level: %s' % self.__args.logger)
					else:
						self.console_log_level = self.__args.logger.upper()

	@property
	def navigation_file(self):
		return self.__navigation_file

	@navigation_file.setter
	def navigation_file(self, navigation_file):

		"""
			read the route file into two arrays - one for upstream navigation, one for downstream
			we are navigating the actual HUC12 strings rather than a numerical representation of those strings.
		"""
		def read_pickle():

			if not len(self.navigation_file):
				raise ValueError('navigation_file must be set before calling this function')
			if not os.path.exists(self.navigation_file):
				raise IOError("Unable to find navigation file\n\t%s" % (self.navigation_file))

			log.debug('Reading navigation route pickle file:\n\t%s' % (self.navigation_file))

			startTime = dt.now()
			pkl_file = open(self.navigation_file, 'rb')

			[self.navigation_attributes, self.us_huc_route, self.ds_huc_route] = pickle.load(pkl_file)

			pkl_file.close()

			log.debug(" Read in %s seconds" % ((dt.now() - startTime).total_seconds()))

			return

		def read():

			if not len(self.navigation_file):
				raise ValueError('navigation_file must be set before calling this function')
			if not os.path.exists(self.navigation_file):
				raise IOError("Unable to find navigation file\n\t%s" % (self.navigation_file))

			startTime = dt.now()

			required_fields = []

			navigation_file_keys = self.navigation_file_keys

			for direction in navigation_file_keys.keys():
				for node in navigation_file_keys[direction]:
					if not navigation_file_keys[direction][node] in required_fields:
						required_fields.append(navigation_file_keys[direction][node])

			try:
				infile = open(self.navigation_file, 'r')
				reader = csv.DictReader(infile)
				source_columns = reader.fieldnames

				'''
					check the all required fields are found in CSV file fieldnames
				'''
				matching = [s for s in source_columns if any(xs in s for xs in required_fields)]
				required_set = set(required_fields)
				source_set = set(source_columns)
				unmatched = required_set.difference(source_set)
				if len(unmatched) > 0:
					raise KeyError("required column(s) '%s' not found in CSV file\n\t%s" %
								(unmatched, self.navigation_file))

			except KeyError as err:
				raise
			except:
				ex = sys.exc_info()
				log.error('Exception 641: %s: %s' % (ex[0], ex[1]))

			row_count = 0

			log.debug('Reading navigation route file:\n\t%s' % (self.navigation_file))

			try:
				for row in reader:
					row_count += 1

					if not row[self.navigation_file_key] in self.navigation_attributes:
						self.navigation_attributes[row[self.navigation_file_key]] = defaultdict( dict )
						for attribute in self.navigation_attribute_list:
							self.navigation_attributes[row[self.navigation_file_key]][attribute] = row[attribute]
					'''
						build upstream navigator
					'''
					self.us_huc_route[row[navigation_file_keys['upstream']['from']]][row[navigation_file_keys['upstream']['to']]] = row
					'''
						build downstream navigator
					'''
					self.ds_huc_route[row[navigation_file_keys['downstream']['from']]][row[navigation_file_keys['downstream']['to']]] = row

			except KeyError as err:
				raise KeyError("required column '%s' not found in CSV file\n\t%s" % (err.message, self.navigation_file))
			except:
				raise ValueError("Unexpected error: %s", sys.exc_info())

			if not row_count > 0:
				raise ValueError("problem reading route file - no rows read using file\n\t%s" % (self.navigation_file))

			log.debug(" Read %s rows in %s seconds" % (row_count, (dt.now() - startTime).total_seconds()))

			return

		if navigation_file == None:
			pass
		else:
			if os.path.exists(str(navigation_file)):

				self.__navigation_file = str(navigation_file)
				if '.p' in navigation_file:
					read_pickle()
				else:
					read()
			else:
				raise IOError("Unable to find navigation file\n\t%s" % (navigation_file))

	@property
	def attribute_file(self):
		return self.__attribute_file

	@attribute_file.setter
	def attribute_file(self, attribute_file):

		"""

			read the attribute file into an array with 1 entry for each HUC12

		"""
		def read():

			if not len(self.attribute_file):
				log.warn('you must set the attribute_file (attribute file) before calling this function')
				exit()
			if not len(self.attribute_file_key):
				log.warn('you must set the attribute_key_field (the column to use to create the attribute table) before calling this function')
				exit()

			self.attribute_data = defaultdict( dict )

			startTime = dt.now()

			try:
				infile = open(self.attribute_file, 'r')  # CSV file
				reader = csv.DictReader(infile)

				'''
					check that the required field is found in CSV file fieldnames
				'''
				if not self.attribute_file_key in reader.fieldnames:
					raise KeyError("required column '%s' not found in CSV file\n\t%s" %
								(self.attribute_file_key, self.attribute_file))

				self.attribute_keys = reader.fieldnames
			except KeyError as err:
				raise
			except:
				ex = sys.exc_info()
				log.error('Exception 641: %s: %s' % (ex[0], ex[1]))
				exit()

			row_count = 0

			log.debug('Reading attribute file:\n\t%s' % (self.attribute_file))

			try:
				for row in reader:
					row_count += 1

					'''
						build the attribute data
					'''
					"""
						JAB NOTE: HUC12s suck when your using CSV files and excel
						the leading zero gets cut off.  Because of that, I have to test
						here and make sure that the leading zero gets added back on
					"""
					if (self.attribute_file_key == 'HUC_12' or self.attribute_file_key == 'HUC12') \
						and len(row[self.attribute_file_key]) == 11:

						row[self.attribute_file_key] = '0' + row[self.attribute_file_key]
					"""
						JAB end of HUC12 fix
					"""
					self.attribute_data[row[self.attribute_file_key]] = row
			except:
				print ("Unexpected error:", sys.exc_info()[0])
				raise

			log.debug(" Read %s rows in %s seconds" % (row_count, (dt.now() - startTime).total_seconds()))

			return

		if attribute_file == None:
			pass
		else:
			if os.path.exists(str(attribute_file)):
				self.__attribute_file = str(attribute_file)
				read()
			else:
				raise IOError("Unable to find attribute file\n\t%s" % (attribute_file))

	def attribute_file_get_columns(self, attribute_file):

		"""

			open the attribute file and return the column names

		"""
		def read():

			fieldnames = []
			try:
				infile = open(attribute_file, 'r')
				reader = csv.DictReader(infile)
				fieldnames = reader.fieldnames
				infile.close()

			except KeyError as err:
				raise
			except IOError as err:
				raise
			except:
				ex = sys.exc_info()
				log.error('Exception 641: %s: %s' % (ex[0], ex[1]))
				exit()

			return fieldnames

		if attribute_file == None:
			pass
		else:
			if os.path.exists(str(attribute_file)):
				return read()
			else:
				raise IOError("Unable to find attribute file to get columns\n\t%s" % (attribute_file))

	@property
	def metadata_list_file(self):
		return self.__metadata_list_file

	@metadata_list_file.setter
	def metadata_list_file(self, metadata_list_file):

		"""

			read the metadata_list_file into an array with 1 entry for each HUC12

		"""
		def read():

			if not len(self.metadata_list_file):
				log.warn('you must set the metadata_list_file (metadata_list_file) before calling this function')
				exit()
			startTime = dt.now()
			log.debug('Reading metadata_list_file file:\n\t%s' % (self.metadata_list_file))
			try:
				infile = open(self.metadata_list_file, 'r')  # CSV file
				reader = csv.DictReader(infile)
			except:
				ex = sys.exc_info()
				self.logger.error('Exception 641: %s: %s' % (ex[0], ex[1]))
				exit()

			row_count = 0
			try:
				for row in reader:
					row_count += 1
					self.metadata_list[row['file_name']][row['label']] = row

			except:
				print ("Unexpected error:", sys.exc_info()[0])
				raise

			log.debug(" Read %s rows in %s seconds" % (row_count, (dt.now() - startTime).total_seconds()))

			return

		if metadata_list_file == None:
			pass
		else:
			if os.path.exists(str(metadata_list_file)):
				self.__metadata_list_file = str(metadata_list_file)
				read()
			else:
				raise IOError("Unable to find metadata_list_file file\n\t%s" % (metadata_list_file))

	"""
		perform the navigation.  'comid' is used instead of 'huc12' because we might want to change and use
		the internal ID for hucs used in NHD and there it is 'comid_to' and 'comid_from' or some variation
	"""
	def huc12_navigate(self, comid):

		if self.direction.upper() == 'Upstream'.upper():
			data_ref = self.us_huc_route
		else:
			data_ref = self.ds_huc_route

		nav_comids = defaultdict( dict )

		mycomids = { comid: 1}
		try:
			while 1 == 1:
				for tocomid in list(mycomids): # .keys():
					if data_ref[tocomid] != None:
						for fromcomid in data_ref[tocomid].keys():
							if fromcomid != 0:
								mycomids[fromcomid] = 1
								nav_comids[tocomid][fromcomid] = data_ref[tocomid][fromcomid]
								#log.debug("[tocomid][fromcomid]==[%s][%s]" % (str(tocomid),str(fromcomid)))
							else:
								mycomids[fromcomid] = 1
								nav_comids[tocomid][fromcomid] = data_ref[tocomid][fromcomid]
								#log.debug("[tocomid][fromcomid]==[" + str(tocomid) + "][" + "%09d" % fromcomid + "] fromcomid==0")
					del mycomids[tocomid]

				if len(mycomids) == 0:
					break

			self.results_length = len(nav_comids)

			self.upstream_result_length = self.results_length - 1

			self.navigation_results = nav_comids
			# log.debug(nav_comids)
			return nav_comids
		except:
			print ("Unexpected error:", sys.exc_info()[0])
			raise

	def huc12_upstream_area(self, comid):

		output_dict = OrderedDict()

		output_dict = {
				'huc12': { 'name': 'HUC12', 'value': comid },
				'huc12_name': { 'name': 'HUC12 Name', 'value': '--na--' },
				'area_sq_km': { 'name': 'HUC12 Area', 'format': '.2f', 'units': 'km2' } ,
				'water_area_sq_km': { 'name': 'HUC12 Water area', 'format': '.2f', 'units': 'km2' } ,
				'us_huc12_ids': {'name': 'List of upstream HUC12s', 'value': [] },
				'upstream_count_nu': { 'name': 'Upstream HUC12 Count', 'value': 0 } ,
				'us_area_sq_km': { 'name': 'Upstream HUC12 area', 'format': '.2f', 'units': 'km2', 'value': 0 } ,
				'us_water_area_sq_km': { 'name': 'Upstream HUC12 water area', 'format': '.2f', 'units': 'km2', 'value': 0 } ,
				'huc8': { 'name': 'HUC8', 'value': comid[:8] } ,
				'upstream_huc8_count_nu': { 'name': 'Upstream HUC8 Count', 'value': 0 } ,
			};

		'''
			this is where the navigation occurs
		'''
		self.huc12_navigate(comid)

		# compute the drainage area metrics from the attributes data
		area_sq_km = 0
		water_area_sq_km = 0
		us_area_sq_km = 0
		us_water_area_sq_km = 0
		name = ''

		output_dict['us_huc12_ids']['value'] = list(self.navigation_results) # .keys()
		output_dict['upstream_count_nu']['value'] = "{:,}".format(len(self.navigation_results) - 1)

		for us_huc_id in self.navigation_results.keys():
			if us_huc_id == comid:
				if len(self.navigation_attributes[comid]['name']):
					output_dict['huc12_name']['value'] = self.navigation_attributes[comid]['name']
				output_dict['area_sq_km']['value'] = "{0:,.2f}".format(float(self.navigation_attributes[comid]['area_sq_km']))
				output_dict['water_area_sq_km']['value'] = "{0:,.2f}".format(float(self.navigation_attributes[comid]['water_area_sq_km']))
			else:
				if not self.navigation_attributes[us_huc_id]['area_sq_km'] == '':
					us_area_sq_km += float(self.navigation_attributes[us_huc_id]['area_sq_km'])
				us_water_area_sq_km += float(self.navigation_attributes[us_huc_id]['water_area_sq_km'])

		if us_area_sq_km > 0:
			format_pattern = "{0:,.2f}"
			if us_area_sq_km > 1000:
				format_pattern = "{0:,.0f}"
			output_dict['us_area_sq_km']['value'] = format_pattern.format(float(us_area_sq_km))

		if us_water_area_sq_km > 0:
			format_pattern = "{0:,.2f}"
			if us_water_area_sq_km > 1000:
				format_pattern = "{0:,.0f}"
			output_dict['us_water_area_sq_km']['value'] = format_pattern.format(float(us_water_area_sq_km))

		return output_dict

	def huc12_downstream_area(self, comid):

		output_dict = OrderedDict()

		output_dict = {
				'huc12': { 'name': 'HUC12', 'value': comid },
				'huc12_name': { 'name': 'HUC12 Name', 'value': '--na--' },
				'area_sq_km': { 'name': 'HUC12 Area', 'format': '.2f', 'units': 'km2' } ,
				'water_area_sq_km': { 'name': 'HUC12 Water area', 'format': '.2f', 'units': 'km2' } ,
				'ds_huc12_ids': {'name': 'List of downstream HUC12s', 'value': [] },
				'downstream_count_nu': { 'name': 'Downstream HUC12 Count', 'value': 0 } ,
				'ds_area_sq_km': { 'name': 'Downstream HUC12 area', 'format': '.2f', 'units': 'km2', 'value': 0 } ,
				'ds_water_area_sq_km': { 'name': 'Downstream HUC12 water area', 'format': '.2f', 'units': 'km2', 'value': 0 } ,
				'huc8': { 'name': 'HUC8', 'value': comid[:8] } ,
				'downstream_huc8_count_nu': { 'name': 'Downstream HUC8 Count', 'value': 0 } ,
			};

		'''
			this is where the navigation occurs
		'''
		self.huc12_navigate(comid)

		# compute the drainage area metrics from the attributes data
		area_sq_km = 0
		water_area_sq_km = 0
		ds_area_sq_km = 0
		ds_water_area_sq_km = 0
		name = ''

		output_dict['ds_huc12_ids']['value'] = self.navigation_results.keys()
		output_dict['downstream_count_nu']['value'] = "{:,}".format(len(self.navigation_results) - 1)

		for us_huc_id in self.navigation_results.keys():
			if us_huc_id == comid:
				if len(self.navigation_attributes[comid]['name']):
					output_dict['huc12_name']['value'] = self.navigation_attributes[comid]['name']
				output_dict['area_sq_km']['value'] = "{0:,.2f}".format(float(self.navigation_attributes[comid]['area_sq_km']))
				output_dict['water_area_sq_km']['value'] = "{0:,.2f}".format(float(self.navigation_attributes[comid]['water_area_sq_km']))
			else:
				if not self.navigation_attributes[us_huc_id]['area_sq_km'] == '':
					ds_area_sq_km += float(self.navigation_attributes[us_huc_id]['area_sq_km'])
				ds_water_area_sq_km += float(self.navigation_attributes[us_huc_id]['water_area_sq_km'])

		if ds_area_sq_km > 0:
			format_pattern = "{0:,.2f}"
			if ds_area_sq_km > 1000:
				format_pattern = "{0:,.0f}"
			output_dict['ds_area_sq_km']['value'] = format_pattern.format(float(ds_area_sq_km))

		if ds_water_area_sq_km > 0:
			format_pattern = "{0:,.2f}"
			if ds_water_area_sq_km > 1000:
				format_pattern = "{0:,.0f}"
			output_dict['ds_water_area_sq_km']['value'] = format_pattern.format(float(ds_water_area_sq_km))

		return output_dict

	"""
		make logger.  this makes 2 different loggers -
			one is in a log file, and
			the other is what is displayed to the user in the console
	"""
	def create_logger(self):

		def posix2local(timestamp, tz=tzlocal.get_localzone()):
			"""Seconds since the epoch -> local time as an aware datetime object."""
			return dt.fromtimestamp(timestamp, tz)

		# these are used to tweak the format of the logs. they allow multi-line log messages that display well
		class MultiLineFormatter(logging.Formatter):

			def converter(self, timestamp):
				return posix2local(timestamp)
			def formatTime(self, record, datefmt=None):
				default_time_format = "%Y-%m-%d %H:%M:%S"
				default_msec_format = "%s.%3d"
				dt = self.converter(record.created)
				if datefmt:
					s = dt.strftime(datefmt)
				else:
					t = dt.strftime(default_time_format)
					s = default_msec_format % (t, record.msecs)
				return s
			def format(self, record):
				str = logging.Formatter.format(self, record)
				header, footer = str.split(record.message)
				str = str.replace('\n', '\n' + header)
				return str

		class MultiLineFormatter2(logging.Formatter):
			def format(self, record):
				str = logging.Formatter.format(self, record)
				header, footer = str.split(record.message)
				str = str.replace('\n', '\n' + ' '*len(header))
				return str

		# turn off this package's INFO and DEBUG logging
		logging.getLogger("requests").setLevel(logging.WARNING)

		logging.basicConfig(level=logging.DEBUG, filename=os.devnull, format='%(levelname)s: %(message)s')

		'''
			set up the logger for the package
		'''
		self.logger = logging.getLogger(__name__)

		# create a file handler for STDOUT
		stdout_handler = logging.StreamHandler(stream=sys.stdout)
		stdout_handler.setLevel(self.console_log_level)
		# create a logging format using a custom formatter
		stdout_formatter = MultiLineFormatter2('%(levelname)-5s: %(message)s')
		stdout_handler.setFormatter(stdout_formatter)
		# add the handlers to the logger
		self.logger.addHandler(stdout_handler)

		# create a file handler for the log file
		if not self.overall_log_file == None:
			log_file_handler = logging.FileHandler(self.overall_log_file)
			log_file_handler.setLevel(self.file_log_level or logging.DEBUG)
			log_file_formatter = MultiLineFormatter('%(asctime)s - %(levelname)-5s - %(message)s')
			log_file_handler.setFormatter(log_file_formatter)
			self.logger.addHandler(log_file_handler)

		return self.logger

	def bannerize(self, string_tx, width=80):
		str_len = int((width - len(string_tx))/2)
		return_string = '%s %s %s' % ('#' * str_len, string_tx, '#' * str_len)
		return_string = return_string + ('#' * (80 - len(return_string)))
		return return_string
	"""
		return a 'human readable' version of the size of a file system object
	"""
	def sizeof_fmt(self, num, suffix='B'):
		for unit in ['','K','M','G','Ti','Pi','Ei','Zi']:
			if abs(num) < 1024.0:
				return "%.f %s%s" % (num, unit, suffix)
			num /= 1024.0
		return "%.1f %s%s" % (num, 'Yi', suffix)

if __name__ == '__main__':
	print ("this is a library")
	exit()


## written by Sarah Betti 2019
## updated 5 November 2019

import numpy as np
from astropy import units as u
from astropy import constants as const

def get_flux_mass_conv(pcdist,T, wavelength=1100):

    '''
    PURPOSE
    --------
    Return mass conversion factor for total flux in Jy.

    Parameters
    --------
    pcdist: float in pc
        distance to object in pc
    T: float in Kelvin
        temperature of object
    wavelength: float in microns
        wavelength at which to find flux.  Can only be [350, 850, 1100] microns
        default is 1100 microns (1.1 mm)

    Returns
    --------
    m_XXum: float
        conversion factor to convert total flux to XX Jy

    Raises
    -------
    ValueError if wavelength is not 350, 850, 1100

    Examples
    --------
        m_350um = get_aztec_mass_conv(800, 18, 350)

    Comments
    --------
        written in IDL by Rob Gutermith
        converted to python by Sarah Betti 2019
    '''

    h   =   const.h.cgs.value     # g cm^2 / s OR  erg s
    k   =   const.k_B.cgs.value   # erg / K
    c   =   const.c.cgs.value     # cm /s
    d   =   pcdist *3.0857e18     # cm

    if wavelength == 350.:
        ### Ossenkopf & Henning 1994 #5 opacities; "OH5"
        kappa_350um     =   0.101                   # cm^2 / g  (includes gas-to-dust)

        ### Dunham et al. 2009 in prep.
        snu_350um       =   151.2*1.0*10**-23       # erg/cm^2/s/Hz
        snu_350um_alt1  =   81.3*1.0*10**-23        # erg/cm^2/s/Hz
        snu_350um_alt2  =   34.6*1.0*10**-23        # erg/cm^2/s/Hz

        nu_350um        =   (2.99792458e14)/350.    # Hz
        planck_350um    =   (2.*h*(nu_350um**3.)/(c**2.))/((np.exp((h*nu_350um)/(k*T)))-1.)

        m_350um         =   (((d**2.)*snu_350um)/(planck_350um*kappa_350um))/1.99e33
        m_350um_alt1    =   (((d**2.)*snu_350um_alt1)/(planck_350um*kappa_350um))/1.99e33
        m_350um_alt2    =   (((d**2.)*snu_350um_alt2)/(planck_350um*kappa_350um))/1.99e33

        return m_350um

    elif wavelength == 450.:
        ### Ossenkopf & Henning 1994 #5 opacities; "OH5"
        kappa_450um     =   0.0674               # cm^2 / g  (includes gas-to-dust)

        snu_450um       =   1.0*10**-23      # erg/cm^2/s/Hz

        nu_450um        =   (2.99792458e14)/450.    # Hz
        planck_450um    =   (2.*h*(nu_450um**3.)/(c**2.))/((np.exp((h*nu_450um)/(k*T)))-1.)

        m_450um         =   (((d**2.)*snu_450um)/(planck_450um*kappa_450um))/1.99e33

        return m_450um

    elif wavelength == 850.:
        ### Ossenkopf & Henning 1994 #5 opacities; "OH5"
        kappa_850um     =   0.0114                  # cm^2 / g  (includes gas-to-dust) (OH*4*, 845um)

        snu_850um       =   1.0*10**-23             # erg/cm^2/s/Hz

        nu_850um        =   (2.99792458e14)/850.    # Hz
        planck_850um    =   (2.*h*(nu_850um**3.)/(c**2.))/((np.exp((h*nu_850um)/(k*T)))-1.)

        m_850um         =   (((d**2.)*snu_850um)/(planck_850um*kappa_850um))/1.99e33

        return m_850um

    elif wavelength == 1100.:
        ### Ossenkopf & Henning 1994 #5 opacities; "OH5"
        #kappa_1100um   =   0.0114                  # cm^2 / g  (includes gas-to-dust) (OH*4*, 845um)
        kappa_1100um    =   0.0121                  # cm^2 / g  (includes gas-to-dust) (close enough?)
        #kappa_1100um   =   0.0069                  # cm^2 / g  (includes gas-to-dust) (OH*4*, 1.1mm)

        snu_1100um      =   1.0*10**-23             # erg/cm^2/s/Hz

        nu_1100um       =   (2.99792458e14)/1100.0  # Hz
        planck_1100um   =   (2.*h*(nu_1100um**3.)/(c**2.))/((np.exp((h*nu_1100um)/(k*T)))-1.)

        m_1100um        =   (((d**2.)*snu_1100um)/(planck_1100um*kappa_1100um))/1.99e33

        return m_1100um

    else:
        raise ValueError('wavelength does not exist.  Use either 350, 345, 850, 1100 microns instead.')


"""
Crawler for UmbrellaCoUk based on ATSSpider
scrapy crawl umbrellacouk -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="http://jobs.umbrella.co.uk/jobboard/cands/jobresults.asp?c=1&ms=1&locallstIndustry=&locallstRegion=&localstrKeywords="
Sample URLS:
    http://jobs.umbrella.co.uk/jobboard/cands/jobresults.asp?c=1&ms=1&locallstIndustry=&locallstRegion=&localstrKeywords=
"""

from scrapy.http import Request
from scrapy.selector import Selector
from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import ConvertDateString
from urlparse import urljoin, urlparse
from math import ceil

class UmbrellaCoUk(ATSSpider):

    name = 'umbrellacouk'
    #download_delay = 0.25

    PAGINATION_STRING = "&pg="
    JOB_URL_STRING = "/jobs/"
    PAGE_NUMBER = 1
    JOBS_PER_PAGE = 10.0
    TOTAL_PAGES = 0
    JOB_URL = ""

    def parse(self, response):
        sel = Selector(response)
        # set expected job count
        if not self.expected_job_count_set:
            expected_job_count = sel.xpath('//p[@class="numResults numResultsTop"]/strong/text()').extract()
            if expected_job_count:
                self.expected_job_count = expected_job_count[0]
                self.TOTAL_PAGES = int(ceil(int(self.expected_job_count)/10.0))
                self.JOB_URL = self.get_job_url(response.url)
            else:
                raise CloseSpider("Could not find the expected job count. It's needed for pagination.")

        while self.PAGE_NUMBER < self.TOTAL_PAGES:
            yield Request(
                callback=self.parse_page,
                url=response.url+self.PAGINATION_STRING+str(self.PAGE_NUMBER)
            )
            self.PAGE_NUMBER += 1

    def parse_page(self, response):
        sel = Selector(response)

        job_ids = sel.xpath('//div[contains(@class, "jobInfo")]/input/@value').extract()

        for id in job_ids:
            yield Request(
                callback = self.parse_job_callback(),
                url =  self.JOB_URL+id
            )

    def parse_job(self, response):
        sel = Selector(response)
        loader = BrightcorpItemLoader(selector=sel)

        loader.add_xpath('title', '//div[@class="JobDetailsSection"]/form/div/h1/text()')
        loader.add_xpath('location', '//dd[@id="DDLocation"]/a/em/text()')
        loader.add_xpath('company', '//dd[@id="DDCompanyName"]/a/text()')
        loader.add_xpath('date', '//dd[@id="DDPosted"]/text()', ConvertDateString('%d-%b-%y'))
        loader.add_xpath('referencenumber', '//dd[@class="jobRefField"]/text()')
        loader.add_xpath('description', '//div/div[@class="jobDescription"]')
        loader.add_xpath('jobtype', '//dd[@id="DDContract"]/a/em/text()')
        loader.add_xpath('baseSalary', ['//dd[@id="DDSalary"]/text()', '//dd[@id="DDSalaryDesc"]/text()'])
        loader.add_xpath('industry', '//dd[@id="DDIndustry"]/a/em/text()')
        loader.add_value('url', response.url)
        loader.add_value('apply_url', response.url)

        yield loader.load_item()

    def get_job_url(self, url):
        parsed_url = urlparse(url)
        return parsed_url.scheme+"://"+parsed_url.netloc+self.JOB_URL_STRING


import bpy
import math
import os.path as op
import glob
import traceback
import ns_utils

bpy.types.Scene.output_path = bpy.props.StringProperty(
    name="", default="", description="Define the path for the output files", subtype='DIR_PATH')

def set_render_quality(quality):
    bpy.context.scene.quality = quality

def set_render_output_path(output_path):
    bpy.context.scene.output_path = output_path

def set_render_smooth_figure(smooth_figure):
    bpy.context.scene.smooth_figure = smooth_figure

def load_camera(camera_fname=''):
    if camera_fname == '':
        camera_fname = op.join(bpy.path.abspath(bpy.context.scene.output_path), 'camera.pkl')
    if op.isfile(camera_fname):
        X_rotation, Y_rotation, Z_rotation, X_location, Y_location, Z_location = ns_utils.load(camera_fname)
        RenderFigure.update_camera = False
        bpy.context.scene.X_rotation = X_rotation
        bpy.context.scene.Y_rotation = Y_rotation
        bpy.context.scene.Z_rotation = Z_rotation
        bpy.context.scene.X_location = X_location
        bpy.context.scene.Y_location = Y_location
        bpy.context.scene.Z_location = Z_location
        RenderFigure.update_camera = True
        update_camera()
    else:
        print('No camera file was found in {}!'.format(camera_fname))

def camera_mode():
    for obj in bpy.data.objects:
        obj.select = False
    for obj in bpy.context.visible_objects:
        if not (obj.hide or obj.hide_render):
            obj.select = True
    ret = bpy.ops.view3d.camera_to_view_selected()
    print(ret)

def grab_camera(self=None, do_save=True):
    RenderFigure.update_camera = False
    bpy.context.scene.X_rotation = X_rotation = math.degrees(bpy.data.objects['Camera'].rotation_euler.x)
    bpy.context.scene.Y_rotation = Y_rotation = math.degrees(bpy.data.objects['Camera'].rotation_euler.y)
    bpy.context.scene.Z_rotation = Z_rotation = math.degrees(bpy.data.objects['Camera'].rotation_euler.z)
    bpy.context.scene.X_location = X_location = bpy.data.objects['Camera'].location.x
    bpy.context.scene.Y_location = Y_location = bpy.data.objects['Camera'].location.y
    bpy.context.scene.Z_location = Z_location = bpy.data.objects['Camera'].location.z
    if do_save:
        if op.isdir(bpy.path.abspath(bpy.context.scene.output_path)):
            camera_fname = op.join(bpy.path.abspath(bpy.context.scene.output_path), 'camera.pkl')
            ns_utils.save((X_rotation, Y_rotation, Z_rotation, X_location, Y_location, Z_location), camera_fname)
            print('Camera location was saved to {}'.format(camera_fname))
        else:
            ns_utils.message(self, "Can't find the folder {}".format(bpy.path.abspath(bpy.context.scene.output_path)))
    RenderFigure.update_camera = True

def render_draw(self, context):
    layout = self.layout
    layout.label(text='Output Path:')
    layout.prop(context.scene, 'output_path')
    layout.prop(context.scene, "quality", text='Quality')

    # layout.operator(CameraMode.bl_idname, text="Camera Mode", icon='CAMERA_DATA')
    # layout.operator("view3d.viewnumpad", text="View Camera", icon='CAMERA_DATA').type = 'CAMERA'
    layout.operator(RenderFigure.bl_idname, text="Render", icon='SCENE')
    if len(glob.glob(op.join(bpy.path.abspath(bpy.context.scene.output_path), 'camera*.pkl'))) > 1:
        layout.operator(RenderAllFigures.bl_idname, text="Render All", icon='SCENE')
    layout.prop(context.scene, 'render_background')
    layout.prop(context.scene, 'smooth_figure')
    layout.operator(GrabCamera.bl_idname, text="Grab Camera", icon='BORDER_RECT')
    layout.operator(LoadCamera.bl_idname, text="Load Camera", icon='RENDER_REGION')
    layout.operator(MirrorCamera.bl_idname, text="Mirror Camera", icon='RENDER_REGION')

    col = layout.column(align=True)
    col.prop(context.scene, "X_rotation", text='X rotation')
    col.prop(context.scene, "Y_rotation", text='Y rotation')
    col.prop(context.scene, "Z_rotation", text='Z rotation')
    col = layout.column(align=True)
    col.prop(context.scene, "X_location", text='X location')
    col.prop(context.scene, "Y_location", text='Y location')
    col.prop(context.scene, "Z_location", text='Z location')

def update_camera(self=None, context=None):
    if RenderFigure.update_camera:
        bpy.data.objects['Camera'].rotation_euler.x = math.radians(bpy.context.scene.X_rotation)
        bpy.data.objects['Camera'].rotation_euler.y = math.radians(bpy.context.scene.Y_rotation)
        bpy.data.objects['Camera'].rotation_euler.z = math.radians(bpy.context.scene.Z_rotation)
        bpy.data.objects['Camera'].location.x = bpy.context.scene.X_location
        bpy.data.objects['Camera'].location.y = bpy.context.scene.Y_location
        bpy.data.objects['Camera'].location.z = bpy.context.scene.Z_location

def mirror():
    camera_rotation_z = bpy.context.scene.Z_rotation
    # target_rotation_z = math.degrees(bpy.data.objects['Camera'].rotation_euler.z)
    bpy.data.objects['Target'].rotation_euler.z += math.radians(180 - camera_rotation_z)
    print(bpy.data.objects['Target'].rotation_euler.z)

bpy.types.Scene.X_rotation = bpy.props.FloatProperty(
    default=0, min=-360, max=360, description="Camera rotation around x axis", update=update_camera)
bpy.types.Scene.Y_rotation = bpy.props.FloatProperty(
    default=0, min=-360, max=360, description="Camera rotation around y axis", update=update_camera)
bpy.types.Scene.Z_rotation = bpy.props.FloatProperty(
    default=0, min=-360, max=360, description="Camera rotation around z axis", update=update_camera)
bpy.types.Scene.X_location = bpy.props.FloatProperty(description="Camera x location", update=update_camera)
bpy.types.Scene.Y_location = bpy.props.FloatProperty(description="Camera y lovation", update=update_camera)
bpy.types.Scene.Z_location = bpy.props.FloatProperty(description="Camera z locationo", update=update_camera)
bpy.types.Scene.quality = bpy.props.FloatProperty(
    default=20, min=1, max=100, description="quality of figure in parentage")
bpy.types.Scene.smooth_figure = bpy.props.BoolProperty(
    name='Smooth image', description="This significantly affect rendering speed")
bpy.types.Scene.render_background = bpy.props.BoolProperty(
    name='Background rendering', description="Render in the background")

class MirrorCamera(bpy.types.Operator):
    bl_idname = "ohad.mirror_camera"
    bl_label = "Mirror Camera"
    bl_options = {"UNDO"}

    def invoke(self, context, event=None):
        mirror()
        return {"FINISHED"}

class CameraMode(bpy.types.Operator):
    bl_idname = "ohad.camera_mode"
    bl_label = "Camera Mode"
    bl_options = {"UNDO"}

    def invoke(self, context, event=None):
        camera_mode()
        return {"FINISHED"}

class GrabCamera(bpy.types.Operator):
    bl_idname = "ohad.grab_camera"
    bl_label = "Grab Camera"
    bl_options = {"UNDO"}

    def invoke(self, context, event=None):
        grab_camera(self)
        return {"FINISHED"}

class LoadCamera(bpy.types.Operator):
    bl_idname = "ohad.load_camera"
    bl_label = "Load Camera"
    bl_options = {"UNDO"}

    def invoke(self, context, event=None):
        load_camera()
        return {"FINISHED"}

class RenderFigure(bpy.types.Operator):
    bl_idname = "ohad.rendering"
    bl_label = "Render figure"
    bl_options = {"UNDO"}
    update_camera = True

    def invoke(self, context, event=None):
        render_image()
        return {"FINISHED"}

class RenderAllFigures(bpy.types.Operator):
    bl_idname = "ohad.render_all_figures"
    bl_label = "Render all figures"
    bl_options = {"UNDO"}
    update_camera = True

    def invoke(self, context, event=None):
        render_all_images()
        return {"FINISHED"}

def render_all_images():
    camera_files = glob.glob(op.join(bpy.path.abspath(bpy.context.scene.output_path), 'camera_*.pkl'))
    for camera_file in camera_files:
        load_camera(camera_file)
        camera_name = ns_utils.namebase(camera_file)
        for hemi in ns_utils.HEMIS:
            if hemi in camera_name:
                RenderingMakerPanel.addon.show_hide_hemi(False, hemi)
                RenderingMakerPanel.addon.show_hide_hemi(True, ns_utils.other_hemi(hemi))
        render_image('{}_fig'.format(camera_name[len('camera') + 1:]))

def render_image(image_name='', image_fol='', quality=0, use_square_samples=None, render_background=None):
    bpy.context.scene.render.resolution_percentage = bpy.context.scene.quality if quality == 0 else quality
    bpy.context.scene.cycles.use_square_samples = bpy.context.scene.smooth_figure if use_square_samples is None \
        else use_square_samples
    if not render_background is None:
        bpy.context.scene.render_background = render_background

    # print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
    print('use_square_samples: {}'.format(use_square_samples))
    # print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')

    cur_frame = bpy.context.scene.frame_current
    if image_name == '':
        image_name = 't{}'.format(cur_frame)
    image_fol = bpy.path.abspath(bpy.context.scene.output_path) if image_fol == '' else image_fol
    print('file name: {}'.format(op.join(image_fol, image_name)))
    bpy.context.scene.render.filepath = op.join(image_fol, image_name)
    # Render and save the rendered scene to file. ------------------------------
    print('Image quality:')
    print(bpy.context.scene.render.resolution_percentage)
    print("Rendering...")
    if not bpy.context.scene.render_background:
        bpy.ops.render.render(write_still=True)
    else:
        grab_camera()
        ns_utils.change_fol_to_mmvt_root()
        electrode_marked = RenderingMakerPanel.addon.is_current_electrode_marked()
        script = 'src.mmvt_addon.scripts.render_image'
        cmd = '{} -m {} -s {} -a {} -q {} -b {} '.format(
            bpy.context.scene.python_cmd, script, ns_utils.get_user(), bpy.context.scene.atlas,
            bpy.context.scene.render.resolution_percentage, bpy.context.scene.bipolar) + \
              '--hide_lh {} --hide_rh {} --hide_subs {} --show_elecs {} --curr_elec {} --show_only_lead {} '.format(
                  bpy.context.scene.objects_show_hide_lh, bpy.context.scene.objects_show_hide_rh,
                  bpy.context.scene.objects_show_hide_sub_cortical, bpy.context.scene.appearance_show_electrodes_layer,
                  bpy.context.scene.electrodes if electrode_marked else None,
                  bpy.context.scene.show_only_lead if electrode_marked else None) + \
              '--show_connections {}'.format(
                  RenderingMakerPanel.addon.connections_visible())
        print('Running {}'.format(cmd))
        ns_utils.save_blender_file()
        ns_utils.run_command_in_new_thread(cmd, queues=False)

    print("Finished")

class RenderingMakerPanel(bpy.types.Panel):
    bl_space_type = "GRAPH_EDITOR"
    bl_region_type = "UI"
    bl_context = "objectmode"
    bl_category = "NeuroSurf"
    bl_label = "Render"
    addon = None

    def draw(self, context):
        render_draw(self, context)

def init(addon):
    RenderingMakerPanel.addon = addon
    bpy.data.objects['Target'].rotation_euler.x = 0
    bpy.data.objects['Target'].rotation_euler.y = 0
    bpy.data.objects['Target'].rotation_euler.z = 0
    bpy.data.objects['Target'].location.x = 0
    bpy.data.objects['Target'].location.y = 0
    bpy.data.objects['Target'].location.z = 0
    grab_camera(None, False)
    register()

def register():
    try:
        unregister()
        bpy.utils.register_class(RenderingMakerPanel)
        bpy.utils.register_class(RenderAllFigures)

        # bpy.ns_utils.register_class(CameraMode)
        bpy.utils.register_class(GrabCamera)
        bpy.utils.register_class(LoadCamera)
        bpy.utils.register_class(MirrorCamera)
        bpy.utils.register_class(RenderFigure)
        # print('Render Panel was registered!')
    except:
        print("Can't register Render Panel!")
        print(traceback.format_exc())

def unregister():
    try:
        bpy.utils.unregister_class(RenderingMakerPanel)
        bpy.utils.unregister_class(RenderAllFigures)
        # bpy.ns_utils.unregister_class(CameraMode)
        bpy.utils.unregister_class(GrabCamera)
        bpy.utils.unregister_class(LoadCamera)
        bpy.utils.unregister_class(MirrorCamera)
        bpy.utils.unregister_class(RenderFigure)
    except:
        pass
        # print("Can't unregister Render Panel!")


from invoke import task, run

@task
def make():
    """Make the csvs."""
    scripts = [
        'read_messages.py',
        'read_questions.py',
        'read_responses.py',
        'survey_maker.py',
    ]
    for s in scripts:
        run("python scripts/{}".format(s))

@task
def rda():
    """Run the use-data script."""
    run("Rscript data-raw/use-data.R")

@task
def install():
    """Install the fidelity R package."""
    run("Rscript -e 'devtools::install()'")


from selenium import webdriver
from time import sleep
import os
import datetime
import tweepy
import ctypes
import sys

# Matheus Eduardo
# github.com/eumts

# - Setting variables -

driver = webdriver.Firefox()
driver.get(" ") # Your dontpad link
sleep(3)

CONSUMER_KEY = " "
CONSUMER_SECRET = " "
ACCESS_KEY = " "
ACCESS_SECRET = " "

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth, wait_on_rate_limit=True)

user = api.me()
print("Logged as " + user.name)
print()

# Words only tweets

def fav_tweet_calor():
    print()
    print("=-=-=-=-=-=-=-=-=-=-=")
    print("Looking for tweets...")
    print("=-=-=-=-=-=-=-=-=-=-=")
    print()
    results = api.search(q="calor", tweet_mode = "extended") # lang="pt-br" geocode = "xx.xxxxxx,xx.xxxxxx,xkm"      q="" its the query, the word that the script will look for
    for result in reversed(results):
        arquivo_calor = open('curtidas_calor.txt','r')
        conteudo_calor = arquivo_calor.read()
        try:
            if (str(result.id) in conteudo_calor):
                print("We already retweeted the tweet of @" + result.user.screen_name)
            else:
                if (result.user.screen_name == "bot_calor"):
                    print("myself") # Trying to interact with my tweet
                else:
                    arquivo_calor.close()
                    print()
                    print(str(result.user.screen_name) + ' - ' + result.full_text, flush=True) # @Mtss_e - Nossa que calor!!!
                    print()
                    print()
                    print("Found", flush=True)
                    print("Liking and retweeting...", flush=True)
                    print()
                    arquivo_calor = open('curtidas_calor.txt','a')
                    arquivo_calor.write(str(result.id) + "\n")
                    api.create_favorite(result.id)
                    api.retweet(result.id)
                    arquivo_calor.close()
        except tweepy.TweepError as e:
            print(e.reason)

# Mentions only funcion

def fav_tweet_menção():
    print()
    print("=-=-=-=-=-=-=-=-=-=-=")
    print("Looking for mentions...")
    print("=-=-=-=-=-=-=-=-=-=-=")
    print()
    mentions = api.mentions_timeline(tweet_mode = 'extended', lang="pt-br")
    for mention in reversed(mentions):
        arquivo_mencao = open('curtidas_mencao.txt','r')
        conteudo_mencao = arquivo_mencao.read()
        try:
            if (str(mention.id) in conteudo_mencao):
                print("We already retweeted the tweet of @" + mention.user.screen_name)
            else:
                if (mention.user.screen_name == "bot_calor"):
                        print("myself")
                else:
                    arquivo_mencao.close()
                    print()
                    print(str(mention.user.screen_name) + ' - ' + mention.full_text, flush=True) # @Mtss_e - Caramba, esse @bot_calor tá bombando mesmo né!!!
                    print()
                    print()
                    print("Found", flush=True)
                    print("Liking and retweeting...", flush=True)
                    print()
                    arquivo_mencao = open('curtidas_mencao.txt','a')
                    arquivo_mencao.write(str(mention.id) + "\n")
                    api.create_favorite(mention.id)
                    api.retweet(mention.id)
                    arquivo_mencao.close()
        except tweepy.TweepError as e:
            print(e.reason)

# Checking dontpad function

def check():
    os.system('cls')
    driver.refresh()
    textarea = driver.find_element_by_xpath('//*[@id="text"]')
    text = textarea.text
    now = datetime.datetime.now()
    data = (now.strftime("[%d-%m-%Y  %H:%M:%S]"))
    if ("ON" in text):
        print()
        print("=--=-=-=-=-=-=-=-=-=")
        print("Calor_bot on")
        print("=--=-=-=-=-=-=-=-=-=")
        print()
        print(data)
        textarea.clear()
        textarea.send_keys('''ON
the bot is running, as your computer.
to change it and shut your pc down type the password:''')
        fav_tweet_calor()
        fav_tweet_menção()
        sleep(30)

    if ("OFF" in text):
        os.system('cls')
        print()
        print("=--=-=-=-=-=-=-=-=-=")
        print("Calor_bot off")
        print("=--=-=-=-=-=-=-=-=-=")
        print()
        print(data)
        textarea.clear()
        textarea.send_keys('''OFF
the bot isnt running, but your pc does.
to change it and shut your pc down type the password:''')

    if (" " in text): # Your password (could not have the words ON and OFF, I recommend using a numer sequence.)
        os.system('cls')
        print()
        print("=--=-=-=-=-=-=-=-=-=")
        print("Shutting your computer down...")
        print("=--=-=-=-=-=-=-=-=-=")
        print()
        print(data)
        textarea.clear()
        textarea.send_keys('''OFF
the bot isnt running, and your pc isnt running too.''')
        os.startfile(r".\shutdown.bat")

# Loop

while True:
    check()
    sleep(20)


# Copyright 2016 Hewlett Packard Enterprise Development Company, LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from neutron import manager
from neutron.services.trunk import exceptions as trunk_exc
from neutron.services.trunk import plugin as trunk_plugin
from neutron.tests.unit.plugins.ml2 import test_plugin

class TrunkPluginTestCase(test_plugin.Ml2PluginV2TestCase):

    def setUp(self):
        super(TrunkPluginTestCase, self).setUp()
        self.trunk_plugin = trunk_plugin.TrunkPlugin()

    def test_delete_trunk_raise_in_use(self):
        with self.port() as port:
            trunk = {'port_id': port['port']['id'],
                     'tenant_id': 'test_tenant',
                     'sub_ports': []}
            response = (
                self.trunk_plugin.create_trunk(self.context, {'trunk': trunk}))
            core_plugin = manager.NeutronManager.get_plugin()
            port['port']['binding:host_id'] = 'host'
            core_plugin.update_port(self.context, port['port']['id'], port)
            self.assertRaises(trunk_exc.TrunkInUse,
                              self.trunk_plugin.delete_trunk,
                              self.context, response['id'])


#!/usr/bin/python
# Synthetic Photomotery
import warnings, glob, os, pickle, numpy as np, astropy.units as q, astropy.constants as ac, matplotlib.pyplot as plt
from SEDkit import utilities as u
from itertools import combinations, chain, groupby

warnings.simplefilter('ignore')
RSR = u.get_filters()

def vega(bbody=False):
    '''
    Returns the wavelength [um] and energy flux in [erg/s/cm2/A] calibrated to 10pc for Vega (
    http://www.stsci.edu/hst/observatory/cdbs/calspec.html)
    '''
    pi = 130.23
    w, f = np.genfromtxt(os.path.dirname(u.__file__)+'/Models/Vega/STSci_Vega.txt')
    w, f = (w * q.AA).to(q.um), f * q.erg / q.s / q.cm ** 2 / q.AA
    return [w, f, u.blackbody(w, 9610, Flam=False, radius=27.3)] if bbody else [w, f]

def get_eff(band, wave, flux, photon=True, calculate=False):
    '''
    Calculates the filter effective wavelength for a given *band*, *wave* and *flux*. If *wave* is out of range (
    uncalculable), returns the given value.
    '''
    filt = RSR[band]
    if (wave[0] < filt['wav'][0]) and (wave[-1] > filt['wav'][-1]) and calculate:
        I = np.interp(wave.value, filt['wav'].value, filt['rsr'].value, left=0, right=0)
        return (np.trapz(wave * flux * I * (wave / (ac.h * ac.c) if photon else 1), x=wave) / np.trapz(
            flux * I * (wave / (ac.h * ac.c) if photon else 1), x=wave)).to(q.um)
    else:
        return filt['eff']

def get_zp(band, photon=True):
    '''
    Calculates the zero point flux density for a given *band* using a flux calibrated Vega SED [erg/s/cm2/A]
    '''
    # return (np.trapz((u.rebin_spec(vega(), RSR[band]['wav'])[1]*RSR[band]['rsr']*((RSR[band]['wav']/(
    # ac.h*ac.c)).to(1/q.erg) if photon else 1)).to((1 if photon else q.erg)/q.s/q.cm**2/q.AA), x=RSR[band][
    # 'wav'])*q.um*(1 if photon else q.erg)/q.s/q.cm**2/(np.trapz(RSR[band]['rsr'], x=RSR[band]['wav'])*q.um)).to((1
    # if photon else q.erg)/q.s/q.cm**2)
    return (np.trapz((u.rebin_spec(vega(), RSR[band]['wav'])[1] * RSR[band]['rsr'] * (
    (RSR[band]['wav'] / (ac.h * ac.c)).to(1 / q.erg) if photon else 1)).to(
        (1 if photon else q.erg) / q.s / q.cm ** 2 / q.AA), x=RSR[band]['wav']) / np.trapz(RSR[band]['rsr'],
                                                                                x=RSR[band]['wav'])).to(
        (1 if photon else q.erg) / q.s / q.cm ** 2 / q.AA)

def get_mag(band, spectrum, exclude=[], photon=True, plot=False, to_flux=False, Flam=False, eff=False):
    '''
    Returns the magnitude in *band* for given *wave* in [um] and *flux* in [erg/s/cm2/A] with *airmass* corrections
    if provided. If uncertainty is provided, returns [mag, mag_uncertainty] instead.
    '''
    spectrum, filt = u.scrub(spectrum), RSR.get(band)
    if len(spectrum) == 2:
        spectrum += ['']
    if filt and np.logical_and(filt['max'] < spectrum[0][-1], filt['min'] > spectrum[0][0]) and all(
            [np.logical_or(all([i < filt['min'].value for i in rng]), all([i > filt['max'].value for i in rng])) for rng
             in exclude]):
        # Calculate synthetic magnitude
        f, sig_f = u.rebin_spec(spectrum, filt['wav'])[1:] if spectrum[2] is not '' else [
            u.rebin_spec(spectrum, filt['wav'])[1], '']
        F = (np.trapz((f * filt['rsr'] * ((filt['wav'] / (ac.h * ac.c)).to(1 / q.erg) if photon else 1)).to(
            (1 if photon else q.erg) / q.s / q.cm ** 2 / q.AA), x=filt['wav']) / (
             np.trapz(filt['rsr'], x=filt['wav']))).to(
            (1 if photon else q.erg) / q.s / q.cm ** 2 / (1 if Flam else q.AA))
        sig_F = np.sqrt(np.sum(((sig_f * filt['rsr'] * np.gradient(filt['wav']).value * (
        (filt['wav'] / (ac.h * ac.c)).to(1 / q.erg) if photon else 1)) ** 2).to(
            (1 if photon else q.erg) ** 2 / q.s ** 2 / q.cm ** 4 / (1 if Flam else q.AA ** 2)))) if sig_f else ''

        # Calculate effective wavelength
        w = get_eff(band, spectrum[0], spectrum[1], photon=photon, calculate=eff)

        return (u.flux2mag(band, F, sig_f=sig_F, filter_dict=RSR) if not to_flux else [F, sig_F]) + [w]

    else:
        return ['', '', '']

def all_mags(spectrum, bands=RSR.keys(), photon=True, to_flux=False, Flam=True, exclude=[], eff=False, to_list=False):
    magDict, magList = {}, []
    for band in bands:
        M = get_mag(band, spectrum, photon=photon, to_flux=to_flux, Flam=Flam, exclude=exclude, eff=eff)
        if M[0]:
            magDict[band], magDict[band + '_unc'], magDict[band + '_eff'] = M
            magList.append(M)
    return sorted(magList, key=lambda x: x[-1]) if to_list else magDict


from sklearn.base import BaseEstimator, TransformerMixin
import pandas as pd
from TaxiFareModel.utils import haversine_vectorized

class TimeFeaturesEncoder(BaseEstimator, TransformerMixin):
    """Extract the day of week (dow), the hour, the month and the year from a
    time column."""
    def __init__(self, time_column, time_zone_name='America/New_York'):
        self.time_column = time_column
        self.time_zone_name = time_zone_name

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        assert isinstance(X, pd.DataFrame)
        X_ = X.copy()
        X_.index = pd.to_datetime(X[self.time_column])
        X_.index = X_.index.tz_convert(self.time_zone_name)
        X_["dow"] = X_.index.weekday
        X_["hour"] = X_.index.hour
        X_["month"] = X_.index.month
        X_["year"] = X_.index.year
        return X_[['dow', 'hour', 'month', 'year']]

class DistanceTransformer(BaseEstimator, TransformerMixin):
    """Compute the haversine distance between two GPS points."""
    def __init__(self,
                start_lat="pickup_latitude",
                start_lon="pickup_longitude",
                end_lat="dropoff_latitude",
                end_lon="dropoff_longitude"):
        self.start_lat = start_lat
        self.start_lon = start_lon
        self.end_lat = end_lat
        self.end_lon = end_lon

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        assert isinstance(X, pd.DataFrame)
        X_ = X.copy()
        X_["distance"] = haversine_vectorized(
            X_,
            start_lat=self.start_lat,
            start_lon=self.start_lon,
            end_lat=self.end_lat,
            end_lon=self.end_lon
        )
        return X_[['distance']]


import sqlite3
from utils import display
from PyQt5.QtWidgets import QDialog
from PyQt5.QtCore import pyqtSlot
from PyQt5 import uic

# Classe permettant d'afficher la fonction à compléter 1
class AppFctComp5(QDialog):

    # Constructeur
    def __init__(self, data:sqlite3.Connection):
        super(QDialog, self).__init__()
        self.ui = uic.loadUi("gui/fct_comp_5.ui", self)
        self.data = data
        self.refreshResult()

    # Fonction de mise à jour de l'affichage

    @pyqtSlot()
    def refreshResult(self):

        display.refreshLabel(self.ui.label_fct_comp_5, "")
        try:
            cursor = self.data.cursor()
            result = cursor.execute(
                "SELECT noSpec, dateRep FROM LesRepresentations_base "
                "EXCEPT "
                "SELECT noSpec, dateREp FROM LesTickets ")
        except Exception as e:
            self.ui.table_fct_comp_5.setRowCount(0)
            display.refreshLabel(self.ui.label_fct_comp_5, "Impossible d'afficher les résultats : " + repr(e))
        else:
            display.refreshGenericData(self.ui.table_fct_comp_5, result)


# MIT License
#
# Copyright (c) 2019 Onur Dundar onur.dundar1@gmail.com
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import json
import numpy as np
import logging

from .detection_base_ov import InferenceBase, InferenceConfig

class AgeGenderDetectionTypes:
    MTCNN = "mtcnn_age_gender"
    OPENMODELZOO = "omz_age_gender"

class AgeGenderConfig(InferenceConfig):
    ModelType = AgeGenderDetectionTypes.OPENMODELZOO

    def parse_json(self, json_file):
        try:
            logging.log(logging.INFO, "Loading JSON File {}".format(json_file))
            logging.log(logging.INFO, "Model Type {}".format(self.ModelType))

            with open(json_file) as json_file:
                data = json.load(json_file)

                self.ModelPath = data[self.ModelType]["model_path"]
                self.ModelName = data[self.ModelType]["model_name"]

                self.TargetDevice = data[self.ModelType]["target_device"]

                if data[self.ModelType]["async"] == "True":
                    self.Async = True

                self.RequestCount = int(data[self.ModelType]["request_count"])

                self.BatchSize = int(data[self.ModelType]["batch_size"])

                if data[self.ModelType]["cpu_extension"] == "True":
                    self.CpuExtension = True

                self.CpuExtensionPath = data[self.ModelType]["cpu_extension_path"]

                if data[self.ModelType]["dynamic_batch"] == "True":
                    self.DynamicBatch = True

                if data[self.ModelType]["limit_cpu_threads"] == "True":
                    self.LimitCPUThreads = True

                self.CPUThreadNum = int(data[self.ModelType]["number_of_cpu_threads"])

                if data[self.ModelType]["bind_cpu_threads"] == "True":
                    self.LimitCPUThreads = True

                self.CPUStream = data[self.ModelType]["cpu_stream"]

        except FileNotFoundError:
            logging.log(logging.ERROR, '{} FileNotFound'.format(json_file))
            exit(-1)

    def read_dict(self, data=None):
        """
        Used When JSON Already Parsed as Dict
        :return:
        """
        if data is None:
            data = dict()
        self.ModelPath = data[self.ModelType]["model_path"]
        self.ModelName = data[self.ModelType]["model_name"]

        self.TargetDevice = data[self.ModelType]["target_device"]

        if data[self.ModelType]["async"] == "True":
            self.Async = True

        self.RequestCount = int(data[self.ModelType]["request_count"])

        self.BatchSize = int(data[self.ModelType]["batch_size"])

        if data[self.ModelType]["cpu_extension"] == "True":
            self.CpuExtension = True

        self.CpuExtensionPath = data[self.ModelType]["cpu_extension_path"]

        if data[self.ModelType]["dynamic_batch"] == "True":
            self.DynamicBatch = True

        if data[self.ModelType]["limit_cpu_threads"] == "True":
            self.LimitCPUThreads = True

        self.CPUThreadNum = int(data[self.ModelType]["number_of_cpu_threads"])

        if data[self.ModelType]["bind_cpu_threads"] == "True":
            self.LimitCPUThreads = True

        self.CPUStream = data[self.ModelType]["cpu_stream"]

class MTCNNAgeGenderConfig(AgeGenderConfig):
    ModelType = AgeGenderDetectionTypes.MTCNN

class MTCNNAgeGenderDetection(InferenceBase):

    Config = MTCNNAgeGenderConfig()

    def get_age_gender_data(self, request_id=0):
        """
        Parse Output Data for Age-Gender Detection Model
        :param request_id:
        :return:
        """
        detection = self.OpenVinoExecutable.requests[request_id].outputs[self.OutputLayer]
        # Parse detection vector to get age and gender
        gender_vector = detection[:, 0:2].flatten()
        gender = int(np.argmax(gender_vector))

        gender_text = 'female'
        if gender == 1:
            gender_text = 'male'

        age_matrix = detection[:, 2:202].reshape((100, 2))
        ages = np.argmax(age_matrix, axis=1)
        age = int(sum(ages))

        return age, gender_text

class AgeGenderDetection(InferenceBase):

    Config = AgeGenderConfig()

    def get_age_gender_data(self, request_id=0):
        """
        Parse Output Data for Age-Gender Detection Model
        :param request_id:
        :return:
        """
        age = int(self.OpenVinoExecutable.requests[request_id].outputs["age_conv3"][0][0][0][0] * 100)
        genders = self.OpenVinoExecutable.requests[request_id].outputs["prob"]
        # Parse detection vector to get age and gender

        gender_text = 'female'
        if genders[0][0][0][0] < genders[0][1][0][0]:
            gender_text = 'male'

        return age, gender_text


import json
import mock
import uuid

from django.test import TestCase
from django.utils import timezone
from facility_profile.models import SummaryLog
from morango.api.serializers import CertificateSerializer, BufferSerializer
from morango.certificates import Certificate, ScopeDefinition, Key
from morango.controller import MorangoProfileController
from morango.errors import CertificateSignatureInvalid
from morango.models import Buffer, SyncSession, TransferSession, InstanceIDModel
from morango.syncsession import NetworkSyncConnection, SyncClient

def mock_patch_decorator(func):

    def wrapper(*args, **kwargs):
        mock_object = mock.Mock(content=b"""{"id": "abc"}""", data={'signature': 'sig', 'client_fsic': '{}', 'server_fsic': '{}'})
        mock_object.json.return_value = {}
        with mock.patch.object(NetworkSyncConnection, '_request', return_value=mock_object):
            with mock.patch.object(Certificate, 'verify', return_value=True):
                    return func(*args, **kwargs)
    return wrapper

class NetworkSyncConnectionTestCase(TestCase):

    def setUp(self):
        self.profile = "facilitydata"

        self.root_scope_def = ScopeDefinition.objects.create(
            id="rootcert",
            profile=self.profile,
            version=1,
            primary_scope_param_key="mainpartition",
            description="Root cert for ${mainpartition}.",
            read_filter_template="",
            write_filter_template="",
            read_write_filter_template="${mainpartition}",
        )

        self.subset_scope_def = ScopeDefinition.objects.create(
            id="subcert",
            profile=self.profile,
            version=1,
            primary_scope_param_key="",
            description="Subset cert under ${mainpartition} for ${subpartition}.",
            read_filter_template="${mainpartition}",
            write_filter_template="${mainpartition}:${subpartition}",
            read_write_filter_template="",
        )

        self.root_cert = Certificate.generate_root_certificate(self.root_scope_def.id)

        self.subset_cert = Certificate(
            parent=self.root_cert,
            profile=self.profile,
            scope_definition=self.subset_scope_def,
            scope_version=self.subset_scope_def.version,
            scope_params=json.dumps({"mainpartition": self.root_cert.id, "subpartition": "abracadabra"}),
            private_key=Key(),
        )
        self.root_cert.sign_certificate(self.subset_cert)
        self.subset_cert.save()

        self.controller = MorangoProfileController('facilitydata')
        self.network_connection = self.controller.create_network_connection('127.0.0.1')

    @mock_patch_decorator
    def test_creating_sync_session_successful(self):
        self.assertEqual(SyncSession.objects.filter(active=True).count(), 0)
        NetworkSyncConnection._request.return_value.json.return_value = {'signature': 'sig', 'local_fsic': '{}'}
        self.network_connection.create_sync_session(self.subset_cert, self.root_cert)
        self.assertEqual(SyncSession.objects.filter(active=True).count(), 1)

    @mock_patch_decorator
    def test_creating_sync_session_cert_fails_to_verify(self):
        Certificate.verify.return_value = False
        with self.assertRaises(CertificateSignatureInvalid):
            self.network_connection.create_sync_session(self.subset_cert, self.root_cert)

    @mock_patch_decorator
    def test_get_remote_certs(self):
        # mock certs being returned by server
        certs = self.subset_cert.get_ancestors(include_self=True)
        cert_serialized = json.dumps(CertificateSerializer(certs, many=True).data)
        NetworkSyncConnection._request.return_value.json.return_value = json.loads(cert_serialized)

        # we want to see if the models are created (not saved) successfully
        remote_certs = self.network_connection.get_remote_certificates('abc')
        self.assertSetEqual(set(certs), set(remote_certs))

    @mock_patch_decorator
    def test_csr(self):
        # mock a "signed" cert being returned by server
        cert_serialized = json.dumps(CertificateSerializer(self.subset_cert).data)
        NetworkSyncConnection._request.return_value.json.return_value = json.loads(cert_serialized)
        self.subset_cert.delete()

        # we only want to make sure the "signed" cert is saved
        with mock.patch.object(Key, "get_private_key_string", return_value=self.subset_cert.private_key.get_private_key_string()):
            self.network_connection.certificate_signing_request(self.root_cert, '', '')
        self.assertTrue(Certificate.objects.filter(id=json.loads(cert_serialized)['id']).exists())

    @mock_patch_decorator
    def test_get_cert_chain(self):
        # mock a cert chain being returned by server
        certs = self.subset_cert.get_ancestors(include_self=True)
        original_cert_count = certs.count()
        cert_serialized = json.dumps(CertificateSerializer(certs, many=True).data)
        NetworkSyncConnection._request.return_value.json.return_value = json.loads(cert_serialized)
        Certificate.objects.all().delete()

        # we only want to make sure the cert chain is saved
        self.network_connection._get_certificate_chain(certs[1])
        self.assertEqual(Certificate.objects.count(), original_cert_count)

class SyncClientTestCase(TestCase):

    def setUp(self):
        session = SyncSession.objects.create(id=uuid.uuid4().hex, profile="facilitydata", last_activity_timestamp=timezone.now())
        transfer_session = TransferSession.objects.create(id=uuid.uuid4().hex, sync_session=session, filter='partition',
                                                          push=True, last_activity_timestamp=timezone.now(), records_total=3)
        conn = NetworkSyncConnection()
        self.syncclient = SyncClient(conn, session)
        self.syncclient.current_transfer_session = transfer_session
        self.chunk_size = 3
        InstanceIDModel.get_or_create_current_instance()

    def build_buffer_items(self, transfer_session, **kwargs):

        data = {
            "profile": kwargs.get("profile", 'facilitydata'),
            "serialized": kwargs.get("serialized", '{"test": 99}'),
            "deleted": kwargs.get("deleted", False),
            "last_saved_instance": kwargs.get("last_saved_instance", uuid.uuid4().hex),
            "last_saved_counter": kwargs.get("last_saved_counter", 179),
            "partition": kwargs.get("partition", 'partition'),
            "source_id": kwargs.get("source_id", uuid.uuid4().hex),
            "model_name": kwargs.get("model_name", "contentsummarylog"),
            "conflicting_serialized_data": kwargs.get("conflicting_serialized_data", ""),
            "model_uuid": kwargs.get("model_uuid", None),
            "transfer_session": transfer_session,
        }

        for i in range(self.chunk_size):
            data['source_id'] = uuid.uuid4().hex
            data["model_uuid"] = SummaryLog.compute_namespaced_id(data["partition"], data["source_id"], data["model_name"])
            Buffer.objects.create(**data)

        buffered_items = Buffer.objects.filter(transfer_session=self.syncclient.current_transfer_session)
        serialized_records = BufferSerializer(buffered_items, many=True)
        return json.dumps(serialized_records.data)

    @mock_patch_decorator
    def test_push_records(self):
        self.build_buffer_items(self.syncclient.current_transfer_session)
        self.assertEqual(self.syncclient.current_transfer_session.records_transferred, 0)
        self.syncclient._push_records(chunk_size=self.chunk_size)
        self.assertEqual(self.syncclient.current_transfer_session.records_transferred, self.chunk_size)

    @mock_patch_decorator
    def test_pull_records(self):
        resp = self.build_buffer_items(self.syncclient.current_transfer_session)
        NetworkSyncConnection._request.return_value.json.return_value = json.loads(resp)
        Buffer.objects.filter(transfer_session=self.syncclient.current_transfer_session).delete()
        self.assertEqual(Buffer.objects.filter(transfer_session=self.syncclient.current_transfer_session).count(), 0)
        self.assertEqual(self.syncclient.current_transfer_session.records_transferred, 0)
        self.syncclient._pull_records(chunk_size=self.chunk_size)
        self.assertEqual(Buffer.objects.filter(transfer_session=self.syncclient.current_transfer_session).count(), self.chunk_size)
        self.assertEqual(self.syncclient.current_transfer_session.records_transferred, self.chunk_size)

    @mock_patch_decorator
    def test_create_transfer_session_push(self):
        self.syncclient.current_transfer_session.active = False
        self.syncclient.current_transfer_session.save()
        self.assertEqual(TransferSession.objects.filter(active=True).count(), 0)
        self.syncclient._create_transfer_session(True, 'filter')
        self.assertEqual(TransferSession.objects.filter(active=True).count(), 1)

    @mock_patch_decorator
    def test_close_transfer_session_push(self):
        self.assertEqual(TransferSession.objects.filter(active=True).count(), 1)
        self.syncclient._close_transfer_session()
        self.assertEqual(TransferSession.objects.filter(active=True).count(), 0)

    @mock_patch_decorator
    def test_close_sync_session(self):
        self.assertEqual(SyncSession.objects.filter(active=True).count(), 1)
        self.syncclient._close_transfer_session()
        self.syncclient.close_sync_session()
        self.assertEqual(SyncSession.objects.filter(active=True).count(), 0)