import os
import sys
import numpy as np
import scipy.io as io
import random

"""For logging by tensorboard"""
bLog = False
try:
	sys.path.append('../utils')
	from logger import Logger
	bLog = True
except ImportError:
	pass

import torch
import torchvision
from torch import nn
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.utils import save_image
from torchvision.datasets import MNIST
import os
import argparse

#import modelZoo

def print_options(parser,opt):
    option_dict = dict()

    message = ''
    message += '----------------- Options ---------------\n'
    for k, v in sorted(vars(opt).items()):
        option_dict[k] = v

        comment = ''
        default = parser.get_default(k)
        if v != default:
            comment = '\t[default: %s]' % str(default)
        message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment)
    message += '----------------- End -------------------'
    print(message)

    return message, option_dict

import json
def save_options(checkpoints_dir, message, options_dict):

    # save to the disk
    #expr_dir = os.path.join(checkpoints_dir, 'options.txt')
    #util.mkdirs(expr_dir)
    if not os.path.exists(checkpoints_dir):
        os.makedirs(checkpoints_dir)
    file_name = os.path.join(checkpoints_dir, 'opt.txt')
    with open(file_name, 'wt') as opt_file:
        opt_file.write(message)
        opt_file.write('\n')

    file_name = os.path.join(checkpoints_dir, 'opt.json')
    with open(file_name, 'wt') as opt_file:
        json.dump(options_dict,opt_file)

parser = argparse.ArgumentParser(description='VAE MNIST Example')
parser.add_argument('--epochs', type=int, default=500001, metavar='N',
                    help='number of epochs to train (default: 50001)')

parser.add_argument('--batch', type=int, default=3072, metavar='N',
                    help='batch size (default: 3072)')

parser.add_argument('--gpu', type=int, default=0, metavar='N',
                    help='Select gpu (default: 0)')

parser.add_argument('--checkpoint_freq', type=int, default=500, metavar='N',
                    help='How frequently save the checkpoint (default: every 50 epoch)')

parser.add_argument('--model', type=str, default='autoencoder_first',
                    help='a model name in the model_zoo.py (default: autoencoder_first')

parser.add_argument('--solver', type=str, default='adam_ams',
                    help='Optimization solver. adam or sgd, adam_ams. (default: adam_ams')

parser.add_argument('--db', type=str, default='haggling_socialmodel_wl',
                    help='Database for training cmu...(default: cmu')

parser.add_argument('--finetune', type=str, default='',
                    help='if a folder is specified, then restart training by loading the last saved model files')

parser.add_argument('--check_root', type=str, default='./',
                    help='The root dir to make subfolders for the check point (default: ./) ')

parser.add_argument('--weight_kld', type=float, default='0.1',
                    help='Weight for the KLD term in VAE training (default: 0.1) ')

parser.add_argument('--autoreg', type=int, default='0',
                    help='If >0, train with autoregressive mode. (using init 150 frames input and later 150 frames as output) (default: 0')

args = parser.parse_args()

#Debug
#args.model = 'autoencoder_2convLayers'
#args.model ='autoencoder_3conv_vae'
#args.model ='autoencoder_3convLayers_vect3_64'
#args.model ='autoencoder_3convLayers_vect3_2'
#args.model ='autoencoder_3convLayers_vect3_2'
#args.model ='autoencoder_3convLayers_vect'
#args.finetune = 'autoencoder_3conv_vae'
#args.check_root = '/posefs2b/Users/hanbyulj/pytorch_motionSynth/checkpoint'
#args.weight_kld = 0.01
#args.autoreg = 1     #turn on autoregressive mode
#args.db = 'edin_loco'
#args.db = 'haggling_winner_loser'

# args.model ='autoencoder_3conv_vae'
# #args.solver = 'sgd'
# args.finetune = 'social_autoencoder_3conv_vae'
# args.check_root = '/posefs2b/Users/hanbyulj/pytorch_motionSynth/checkpoint'
# args.batch = 2
#args.weight_kld = 0.0001

torch.cuda.set_device(args.gpu)

rng = np.random.RandomState(23456)
torch.manual_seed(23456)
torch.cuda.manual_seed(23456)

"""All available dataset"""
#datapath ='/ssd/codes/pytorch_motionSynth/motionsynth_data'
datapath ='../../motionsynth_data/data/processed/'

# train_dblist = ['data_panoptic_speech_haggling_sellers_training_byFrame']
# test_dblist = ['data_panoptic_speech_haggling_sellers_testing_byFrame']
#train_dblist = ['data_panoptic_speech_haggling_sellers_training_byFrame_white_tiny']
# train_dblist = ['data_hagglingSellers_speech_face_30frm_10gap_white_training']
#train_dblist = ['data_hagglingSellers_speech_face_30frm_5gap_white_training']
#test_dblist = ['data_hagglingSellers_speech_face_30frm_10gap_white_testing']

train_dblist = ['data_hagglingSellers_speech_face_60frm_10gap_white_testing']
##train_dblist = ['data_hagglingSellers_speech_face_60frm_5gap_white_training']
test_dblist = ['data_hagglingSellers_speech_face_60frm_10gap_white_testing']

train_data = np.load(datapath + train_dblist[0] + '.npz')

train_X= train_data['clips']  #Input (3700,240,73)
train_Y = train_data['classes']  #Input (3700,240,73)

# train_X = train_X[:-1:10,:,:]
# train_Y = train_Y[:-1:10,:]

test_data = np.load(datapath + test_dblist[0] + '.npz')
test_X= test_data['clips']  #Input (1044,240,73)
test_Y = test_data['classes']  #Input (1044,240,73)

#data_all = np.concatenate( (X,Y), axis=0) #(5366,240,73)
# X = np.swapaxes(X, 1, 2).astype(np.float32) #(num, 73, 240)
# Y = np.swapaxes(Y, 1, 2).astype(np.float32) #(num, 73, 240)

# """Visualize X and Y"""
# #by jhugestar
# for frameIdx in range(1,X.shape[0],10):
#     sys.path.append('/ssd/codes/glvis_python/')
#     from Visualize_human_gl import showSkeleton,show_Holden_Data_73 #opengl visualization
#     show_Holden_Data_73([ X[frameIdx,:,:], Y[frameIdx,:,:]])

""" Training Network """
num_epochs = args.epochs #500
#batch_size = 128
batch_size = args.batch
learning_rate = 1e-3

# if args.autoreg ==1: #and "vae" in args.model:
#     model = getattr(modelZoo,args.model)(frameLeng=160).cuda()
# else:
#     model = getattr(modelZoo,args.model)().cuda()

import torch
from torch import nn

class naive_lstm2(nn.Module):
    def __init__(self, batch_size):
        super(naive_lstm2, self).__init__()

        self.hidden_dim = 20
        #self.feature_dim= 200
        self.feature_dim= 1
        self.embed_dim= 10
        self.num_layers = 1
        self.batch_size = batch_size

        self.hidden = self.init_hidden()
        self.encode = nn.Linear(self.feature_dim,self.embed_dim)
        #self.dout = nn.Dropout(0.5)
        self.lstm = nn.LSTM(self.embed_dim, self.hidden_dim, num_layers = self.num_layers, dropout=0.0, batch_first=True, bidirectional=False) #batch_first=True makes the order as (batch, frames, featureNum)
        #self.proj = nn.Linear(self.hidden_dim*2,1)
        self.proj = nn.Linear(self.hidden_dim,200)
        self.out_act = nn.Sigmoid()

    def init_hidden(self):
        # return (Variable(torch.zeros(self.num_layers*2, self.batch_size, self.hidden_dim)).cuda(),
        #         Variable(torch.zeros(self.num_layers*2, self.batch_size, self.hidden_dim)).cuda())
        return (Variable(torch.zeros(self.num_layers, self.batch_size, self.hidden_dim)).cuda(),
                 Variable(torch.zeros(self.num_layers, self.batch_size, self.hidden_dim)).cuda())
    def forward(self, input_):

        #input_ dimension: (batch, timestpes, dim). Note I used batch_first for this ordering
        #lstm_out:  (batch, timesteps, hidden_dim)
        #self.hidden (tuple with two elements):  ( (1, batch, hidden_dim),  (1, batch, hidden_dim))
        #input_encoded = self.dout(self.encode(input_))
        input_encoded = self.encode(input_)
        lstm_out, self.hidden = self.lstm(
                    input_encoded, self.hidden)

        proj = self.proj(lstm_out) #input:(batch, inputDim, outputDim ) -> output (batch, timesteps,1)
        return self.out_act(proj)

model = naive_lstm2(batch_size).cuda()

for param in model.parameters():
    print(type(param.data), param.size())

criterion = nn.BCELoss()

if args.solver == 'adam':
    print('solver: Adam')
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
elif args.solver == 'sgd':
    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
    print('solver: SGD')
elif args.solver == 'adam_ams': #only for pytorch 0.4 or later.
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5, amsgrad=True)
    print('solver: Adam with AMSGrad')
else:
    print('Unknown solver option')
    assert(False)

#checkpointFolder = './autoenc_vect/'
pretrain_epoch = 0
if args.finetune =='':
    pretrain_batch_size =args.batch  #Assume current batch was used in pretraining

    if 'socialmodel' in args.db:
        checkpointFolder = args.check_root + '/social_'+ model.__class__.__name__
    else:
        checkpointFolder = args.check_root + '/'+ model.__class__.__name__

    if not os.path.exists(checkpointFolder):
        os.mkdir(checkpointFolder)
    else: #if already exist
        tryIdx =1
        while True:
            newCheckName = checkpointFolder + '_try' + str(tryIdx)
            if not os.path.exists(newCheckName):
                checkpointFolder = newCheckName
                os.mkdir(checkpointFolder)
                break
            else:
                tryIdx += 1
else:       #FineTuning
    checkpointFolder = args.check_root+ '/' + args.finetune

    checkpointList =  [os.path.join(checkpointFolder,f) for f in sorted(list(os.listdir(checkpointFolder)))
                if os.path.isfile(os.path.join(checkpointFolder,f))
                    and f.endswith('.pth') ]

    #Fine Last Epoch
    last_epoch =0
    for name in checkpointList:
        pretrain_epoch= int(name[(name.find('checkpoint_') + 12):-4])
        if last_epoch <pretrain_epoch:
            last_epoch= pretrain_epoch
            trainResultName = name

    #checkpointList.sort(key=lambda x: os.path.getmtime(x))
    #trainResultName =checkpointList[-1]
    pretrain_epoch= int(trainResultName[(trainResultName.find('checkpoint_') + 12):-4])
    pretrain_epoch = pretrain_epoch+1

    try:
        log_file_name = os.path.join(checkpointFolder, 'opt.json')
        with open(log_file_name, 'r') as opt_file:
            options_dict = json.load(opt_file)
        pretrain_batch_size = options_dict['batch']

    except:
        pretrain_batch_size =args.batch  #Assume current batch was used in pretraining

    #Fine Last Epoch file
    print('load previous state: {0}'.format(trainResultName))
    model.load_state_dict(torch.load(trainResultName, map_location=lambda storage, loc: storage),strict=False)

    #Checking optimizer file
    trainResultName = trainResultName+'o'
    if os.path.exists(trainResultName):
        print('load previous state: {0}'.format(trainResultName))
        optimizer.load_state_dict(torch.load(trainResultName, map_location=lambda storage, loc: storage))

    model.train()
    model.eval()

if bLog:
    logger = Logger(checkpointFolder+'/logs')

""" Save Option Info """
option_str, options_dict = print_options(parser,args)

# option_str += '\nDBList: \n'
# for i, dbname in enumerate(dblist):
#     option_str +=  '{0}:  {1}\n'.format(dbname,X.shape)
# option_str += 'All: {0}'.format(X.shape)
# options_dict['dblist']= dblist

save_options(checkpointFolder, option_str, options_dict)

""" Compute mean and std """
train_X = np.swapaxes(train_X, 1, 2).astype(np.float32) #(num, 200, 1)
train_Y = train_Y.astype(np.float32)

test_X = np.swapaxes(test_X, 1, 2).astype(np.float32) #(num, 200, 1)
test_Y = test_Y.astype(np.float32)

Xmean = train_X.mean(axis=2).mean(axis=0)[np.newaxis,:,np.newaxis]
Xstd = np.array([[[train_X.std()]]]).repeat(train_X.shape[1], axis=1)

""" Data standardization """
train_X = (train_X - Xmean) / Xstd
test_X = (test_X - Xmean) / Xstd

"""Data Shuffle"""
I = np.arange(len(train_X))
rng.shuffle(I);
train_X = train_X[I]
train_Y = train_Y[I]

print('Input data size: {0}'.format(train_X.shape))

np.savez_compressed(checkpointFolder+'/preprocess_core.npz', Xmean=Xmean, Xstd=Xstd)

#stepNum =0

checkpointFolder_base = os.path.basename(checkpointFolder)

if train_X.shape[0]  < batch_size:
    batch_size = train_X.shape[0]

curBestAccu = 0
#Compute stepNum start point (to be continuos in tensorboard if pretraine data is loaded)
filelog_str = ''
stepNum = pretrain_epoch* len(np.arange(train_X.shape[0] // pretrain_batch_size))

#LSTM expected (stepSize, batch, features)
train_X
for epoch in range(num_epochs):

    #targetFrame = random.choice(range(240))
    model.train()
    batchinds = np.arange(train_X.shape[0] // batch_size)
    rng.shuffle(batchinds)

    avgLoss =0
    for bii, bi in enumerate(batchinds):

        idxStart  = bi*batch_size
        # inputDataAll = train_X[idxStart:(idxStart+batch_size),:,targetFrame]
        # outputDataAll = train_Y[idxStart:(idxStart+batch_size),targetFrame]
        #outputDataAll = outputDataAll[:,np.newaxis]

        inputDataAll = train_X[idxStart:(idxStart+batch_size),:,:]

        """Original lstm ordering
        # #Reordering from (batchsize,featureNum,frames) ->(frames, batch, features)
        # inputDataAll = np.swapaxes(inputDataAll, 0, 1) #(featureNum,batchsize,frames)
        # inputDataAll = np.swapaxes(inputDataAll, 0, 2) #(frames,batchsize,featureNum)
        #outputDataAll = train_Y[idxStart:(idxStart+batch_size),:]
        #outputDataAll = np.swapaxes(outputDataAll,0,1) #(batch,frames) -> (frame,batch)
        #outputDataAll = outputDataAll[:,:,np.newaxis]
        """

        #Reordering from (batchsize,featureNum,frames) ->(batch, frame,features)
        inputDataAll = np.swapaxes(inputDataAll, 1, 2) #(batch, frame,features)
        outputDataAll = train_Y[idxStart:(idxStart+batch_size),:]
        outputDataAll = outputDataAll[:,:,np.newaxis]

        #inputData = Variable(torch.from_numpy(inputDataAll)).cuda()
        #Add noise
        inputData = Variable(torch.from_numpy(inputDataAll))
        noise = torch.randn_like(inputData)
        inputData = inputData + noise #* 2.0
        inputData = inputData.cuda()

        outputGT = Variable(torch.from_numpy(outputDataAll)).cuda()

        # ===================forward=====================
        model.hidden = model.init_hidden() #clear out hidden state of the LSTM

        #output = model(inputData)
        inputPred = model(outputGT)
        loss = criterion(inputPred, inputData)

        # ===================backward====================
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # ===================log========================
        #print('model: {}, epoch [{}/{}], loss:{:.4f}'
        #            .format(checkpointFolder_base, epoch +pretrain_epoch, num_epochs, loss.item()))
        avgLoss += loss.item()*batch_size

        if bLog:
            # 1. Log scalar values (scalar summary)
            if int(torch.__version__[2])==2:
                info = { 'loss': loss.data[0] }
            else:
                info = { 'loss': loss.item() }

            for tag, value in info.items():
                logger.scalar_summary(tag, value, stepNum)

        stepNum = stepNum+1

        # # 2. Log values and gradients of the parameters (histogram summary)
        # for tag, value in model.named_parameters():
        #     tag = tag.replace('.', '/')
        #     logger.histo_summary(tag, value.data.cpu().numpy(), epoch+1)
        #     logger.histo_summary(tag+'/grad', value.grad.data.cpu().numpy(), epoch+1)

    temp_str = '## model: {}, epoch [{}/{}], avg loss:{:.4f}\n'.format(checkpointFolder_base, epoch +pretrain_epoch, num_epochs, avgLoss/ (len(batchinds)*batch_size) )

    """compute testing error"""
    #batch_size_test = 64
    batch_size_test = batch_size
    test_loss = 0.0
    acc= 0.0
    cnt =0.0

    model.eval()
    batchinds = np.arange(test_X.shape[0] // batch_size_test)
    for bii, bi in enumerate(batchinds):

        idxStart  = bi*batch_size_test
        # inputDataAll = test_X[idxStart:(idxStart+batch_size_test),:,-1]
        #inputDataAll = test_X[idxStart:(idxStart+batch_size),:,:]
        inputDataAll = test_X[idxStart:(idxStart+batch_size),:,:]

        #Reordering from (batchsize,featureNum,frames) ->(batch, frame,features)
        inputDataAll = np.swapaxes(inputDataAll, 1, 2) #(batch, frame,features)
        outputDataAll = test_Y[idxStart:(idxStart+batch_size),:]
        outputDataAll = outputDataAll[:,:,np.newaxis]
        inputData = Variable(torch.from_numpy(inputDataAll)).cuda()
        outputGT = Variable(torch.from_numpy(outputDataAll)).cuda()

        model.hidden = model.init_hidden() #clear out hidden state of the LSTM
        #output = model(inputData)
        #loss = criterion(output, outputGT)

        inputPred = model(outputGT)
        loss = criterion(inputPred, inputData)

        test_loss += loss.data.cpu().numpy().item()* batch_size_test # sum up batch loss

        #consider all 30 frames in the chuk
        # pred = output.data.cpu().numpy() >= 0.5
        # truth = outputGT.data.cpu().numpy() >= 0.5
        # acc += (pred==truth).sum() /float(train_Y.shape[1]) #train_Y.shape[1] is number of frames (chunk size)

        #consider the last frame

        """
        pred = output.data.cpu().numpy() >= 0.5
        truth = outputGT.data.cpu().numpy() >= 0.5
        acc += (pred[:,-1]==truth[:,-1]).sum()
        """
        cnt += batch_size

    test_loss /= len(batchinds)*batch_size_test
    newAcc = acc/cnt
    bNewBest = False
    if curBestAccu<newAcc:
        curBestAccu = newAcc
        bNewBest = True

    print('Testing: modelName: {} (epoch:{}) /Average loss: {:.4f}/Accuracy: {:.4f} (best {:.4f})\n'.format(checkpointFolder_base, epoch +pretrain_epoch,test_loss, acc/cnt,curBestAccu))
    if bLog:
        info = { 'test_acc': newAcc }

        for tag, value in info.items():
            logger.scalar_summary(tag, value, stepNum)

    if bNewBest:#(epoch + pretrain_epoch) % args.checkpoint_freq == 0:
    #if (epoch + pretrain_epoch) % 1 == 0:
        fileName = checkpointFolder+ '/checkpoint_e' + str(epoch + pretrain_epoch) + '.pth'
        torch.save(model.state_dict(), fileName)
        fileName = checkpointFolder+ '/opt_state.pth'    #overwrite
        torch.save(optimizer.state_dict(), fileName)
        #torch.save(model, fileName)

    print(temp_str)
    filelog_str +=temp_str
    if (epoch + pretrain_epoch) % args.checkpoint_freq == 0:
        file_name = os.path.join(checkpointFolder, 'log.txt')
        with open(file_name, 'a') as opt_file:
            opt_file.write(filelog_str)
            opt_file.write('\n')
            opt_file.close()
        filelog_str =''


from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Firefox(executable_path="C://Users//seker//Documents//driver//geckodriver.exe")
driver.get("https://www.amazon.com")
driver.maximize_window()
driver.implicitly_wait(10)

#Capture all the cookies created by browser and number
cookies = driver.get_cookies()
print('Number of cookies:',len(cookies))
print("Before adding cookie:",cookies)

# How to add a new cookie to the browser. Cookies are stored in key-value form
cookie = {'name':'MyCookie','value':'13579'}
driver.add_cookie(cookie)
cookies = driver.get_cookies()
print('Number of cookies:',len(cookies))
print("After adding cookie:",cookies)

#To delete the cookie you need to know the cookie name
driver.delete_cookie("MyCookie")
cookies = driver.get_cookies()
print('After deleting a cookie:',len(cookies))

driver.close()


import base64
import datetime
import io

from django_plotly_dash import DjangoDash
from dash.dependencies import Input, Output, State
import dash
import dash_table
import dash_core_components as dcc
import dash_html_components as html
from  statboard.models import Tasks
import pandas as pd
import plotly.express as px

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = DjangoDash('database', external_stylesheets=external_stylesheets)

qs = Tasks.objects.all()

task_data = [
        {
        'Task':x.task_type,
        'Task ID':x.code,
        'Responsible':x.allocated_to.name,
        'Status':x.status,
        'Allocated By':x.dispatched_by.name,
        'Account no.':x.account_number,
        'Meter no':x.meter_number,
        'Suburb':x.suburb,
        'City/Town':x.city,
        'Notes':x.work_order_notes,
        'Date Allocated':x.date_created,
        'AssignedToCompany':x.assigned_to_company
        } for x in qs
    ]

df = pd.DataFrame(task_data)

app.layout = html.Div([
    dash_table.DataTable(
        id='our-table',

        columns=[
                 {'name':'Task ID', 'id':'Task ID', 'deletable':False, 'renamable':False},
                 {'name':'Task', 'id': 'Task', 'deletable':False, 'renamable': False},
                 {'name':'QC Status', 'id': 'Status', 'deletable':False, 'renamable':False},
                 {'name':'Responsible', 'id': 'Responsible', 'deletable':False, 'renamable': False},
                 {'name':'Allocated By', 'id': 'Allocated By', 'deletable':False, 'renamable': False},
                 {'name':'Meter no', 'id': 'Meter no', 'deletable':False, 'renamable':False},
                 {'name':'Account no.', 'id': 'Account no.', 'deletable':False, 'renamable': False},
                 {'name':'City/Town', 'id': 'City/Town', 'deletable':False, 'renamable': False},
                 {'name':'Suburb', 'id': 'Suburb', 'deletable':False, 'renamable': False},
                 {'name':'Assigned To Company', 'id': 'AssignedToCompany', 'deletable':False, 'renamable': False},
                 {'name':'Notes', 'id': 'Notes', 'deletable':False, 'renamable': False}],

        data=df.to_dict('records'),
        editable=True,                  # allow user to edit data inside tabel
        row_deletable=True,             # allow user to delete rows
        sort_action="native",           # give user capability to sort columns
        sort_mode="single",             # sort across 'multi' or 'single' columns
        filter_action="native",         # allow filtering of columns
        page_action='none',             # render all of the data at once. No paging.
        style_table={'height': '300px', 'overflowY': 'auto'},
        style_cell={'textAlign': 'left', 'minWidth': '100px', 'width': '100px', 'maxWidth': '100px'},
        style_cell_conditional=[
            {
                'if': {'column_id': c},
                'textAlign': 'right'
            } for c in ['Responsible', 'Task']
        ]
    ),

    html.Div([
    html.Div(id='output-data'),
        dcc.Input(
            id='adding-rows-name',
            placeholder='Enter a column name...',
            value='',
            style={'padding': 10}
        ),
        html.Button('Add Column', id='adding-columns-button', n_clicks=0)
    ], style={'height': 50}),

    html.Button('Add Row', id='editing-rows-button', n_clicks=0),
    html.Button('Export to Excel', id='save_to_csv', n_clicks=0),

    # Create notification when saving to excel
    html.Div(id='placeholder', children=[]),
    dcc.Store(id="store", data=0),
    dcc.Interval(id='interval', interval=1000),

    dcc.Graph(id='my_graph', config= {'displaylogo': False})

])

def parse_contents(contents, filename, date):
    content_type, content_string = contents.split(',')

    decoded = base64.b64decode(content_string)
    try:
        if 'csv' in filename:
            # Assume that the user uploaded a CSV file
            df = pd.read_csv(
                io.StringIO(decoded.decode('utf-8')))
        elif 'xls' in filename:
            # Assume that the user uploaded an excel file
            df = pd.read_excel(io.BytesIO(decoded))
    except Exception as e:
        print(e)
        return html.Div([
            'There was an error processing this file.'
        ])

    return html.Div([
        html.H5(filename),
        html.H6(datetime.datetime.fromtimestamp(date)),

        dash_table.DataTable('our-table')
    ])
# ------------------------------------------------------------------------------------------------

@app.callback(Output('output-data', 'children'),
              [Input('psql-data', 'contents')],
              [State('psql-data', 'filename'),
               State('psql-data', 'last_modified')])
def update_output(list_of_contents, list_of_names, list_of_dates):
    if list_of_contents is not None:
        children = [
            parse_contents(c, n, d) for c, n, d in
            zip(list_of_contents, list_of_names, list_of_dates)]
        return children

@app.callback(
    Output('our-table', 'columns'),
    [Input('adding-columns-button', 'n_clicks')],
    [State('adding-rows-name', 'value'),
     State('our-table', 'columns')],
)
def add_columns(n_clicks, value, existing_columns):
    print(existing_columns)
    if n_clicks > 0:
        existing_columns.append({
            'name': value, 'id': value,
            'renamable': True, 'deletable': True
        })
    print(existing_columns)
    return existing_columns

@app.callback(
    Output('our-table', 'data'),
    [Input('editing-rows-button', 'n_clicks')],
    [State('our-table', 'data'),
     State('our-table', 'columns')],
)
def add_row(n_clicks, rows, columns):
    # print(rows)
    if n_clicks > 0:
        rows.append({c['id']: '' for c in columns})
    # print(rows)
    return rows

@app.callback(
    Output('my_graph', 'figure'),
    [Input('our-table', 'data')])
def display_graph(data):
    df_fig = pd.DataFrame(data)
    # print(df_fig)
    fig = px.bar(df_fig, x='Status', y='City/Town', color ='Status')
    return fig

@app.callback(
    [Output('placeholder', 'children'),
     Output("store", "data")],
    [Input('save_to_csv', 'n_clicks'),
     Input("interval", "n_intervals")],
    [State('our-table', 'data'),
     State('store', 'data')]
)
def df_to_csv(n_clicks, n_intervals, dataset, s):
    output = html.Plaintext("The data has been saved to your folder.",
                            style={'color': 'green', 'font-weight': 'bold', 'font-size': 'large'})
    no_output = html.Plaintext("", style={'margin': "0px"})

    input_triggered = dash.callback_context.triggered[0]["prop_id"].split(".")[0]

    if input_triggered == "save_to_csv":
        s = 6
        df = pd.DataFrame(dataset)
        df.to_csv("Your_Sales_Data.csv")
        return output, s
    elif input_triggered == 'interval' and s > 0:
        s = s-1
        if s > 0:
            return output, s
        else:
            return no_output, s
    elif s == 0:
        return no_output, s


import itertools
import math

class Grid(list):

    def __init__(self, max_number_of_nodes=100, min_balance=0.5):
        """Create a list of couples of dimensions.

        The nth couple is the tightest balanced rectangle having at least n cells.
        A balanced rectangle satisfies: length / width > min_balance.
        [None, (1, 1), (2, 2), (2, 2), (2, 2), (3, 2), (3, 2), (3, 3), ...]
        """
        list.__init__(self)
        max_square_side = int(math.ceil(math.sqrt(max_number_of_nodes)))
        for n in range(max_square_side * max_square_side, 0, -1):
            for i in range(n, int(math.ceil(math.sqrt(n))) - 1, -1):
                if n % i == 0 and n / i / i > min_balance:
                    self.insert(0, (i, n // i))
                    break
            else:
                self.insert(0, self[0])
        self.insert(0, None)
        self[2:4] = [(2, 1), (3, 1)] # tweak the 2- and 3-box grids

    def get_nth_next(self, index, nth):
        """Return the nth next distinct rectangle after the index-th one."""
        acc = set()
        for i in itertools.count(index):
            acc.add(self[i])
            if len(acc) > nth:
                return self[i]

    def get_markdown(self):
        """Return the grid as a nicely formatted table. For the docs only."""
        (w, h) = map(max, zip(*self[1:]))
        result = \
            [
                [""] +
                list(map(lambda i: "**%s**" % i, range(1, w + 1)))
            ] + \
            [["---"] * (w + 1)] + \
            [
                ["**%s**" % row] +
                [""] * w for row in range(1, h + 1)
            ]
        for (n, (col, row)) in enumerate(self[1:], 1):
            result[row + 1][col] += (", %s" % n if result[row + 1][col] else str(n))
        return "\n".join(["| %s |" % " | ".join(row) for row in result])

if __name__ == "__main__":
    grid = Grid()
    print(grid.get_markdown())


import json

def writeJsonData(path, data_json):
    File_out = open(path, "w", encoding="utf-8")
    File_out.write(json.dumps(data_json, ensure_ascii=False))
    File_out.close()

def readJsonData(path):
    decode_file = open(path, "r", encoding="utf-8")
    tweets = json.load(decode_file)
    decode_file.close()
    return tweets


"""Script to calculate global IOU for segmentation tasks."""
import argparse
from pathlib import Path

import numpy as np
from joblib import Parallel, delayed
from tqdm import tqdm

from iglovikov_helper_functions.utils.image_utils import load_grayscale
from iglovikov_helper_functions.utils.metrics import (
    calculate_confusion_matrix_from_arrays,
    calculate_jaccard,
    calculate_tp_fp_fn,
)

def confusion_matrix_from_files(y_true_path: Path, y_pred_path: Path, num_classes: int) -> np.array:
    y_true = load_grayscale(y_true_path)
    y_pred = load_grayscale(y_pred_path)

    if not y_true.shape == y_pred.shape:
        raise ValueError(
            f"y_true and y_pred should have the same shape. "
            f"y_true shape = {y_true.shape} y_pred.shape = {y_pred.shape} "
            f"y_pred_path = {y_pred_path} "
            f"y_true_path = {y_true_path}"
        )

    return calculate_confusion_matrix_from_arrays(y_true, y_pred, num_classes=num_classes)

def calculate_ious_global(y_true_path: Path, y_pred_path: Path, num_classes: int, num_workers: int = 1) -> np.array:
    y_true_files = sorted(y_true_path.glob("*.png"))
    y_pred_files = sorted(y_pred_path.glob("*.png"))

    if not [x.name for x in y_pred_files] == [x.name for x in y_true_files]:
        raise AssertionError("Should have the same file names for y_true and y_pred.")

    def helper(file_name):
        return confusion_matrix_from_files(
            y_true_path.joinpath(file_name), y_pred_path.joinpath(file_name), num_classes
        )

    matrices = Parallel(n_jobs=num_workers)(delayed(helper)(file_path.name) for file_path in tqdm(y_pred_files))

    confusion_matrix = np.dstack(matrices).sum(axis=2)
    tp_fp_fn_dict = calculate_tp_fp_fn(confusion_matrix)

    return calculate_jaccard(tp_fp_fn_dict)

def get_args():
    parser = argparse.ArgumentParser("Calculate IOU for segmentation masks.")
    arg = parser.add_argument
    arg("-p", "--predictions_path", type=Path, help="Json with predictions.", required=True)
    arg("-g", "--ground_truth_path", type=Path, help="Json with ground truth.", required=True)
    arg("-n", "--num_classes", type=int, help="Number of classes to use.", required=True)
    arg("-j", "--num_workers", type=int, help="Number of workers to use.", default=12)
    return parser.parse_args()

if __name__ == "__main__":
    args = get_args()

    ious = calculate_ious_global(args.ground_truth_path, args.predictions_path, args.num_classes, args.num_workers)

    print(f"Mean IOU = {np.mean(ious)}")
    for class_id, iou in enumerate(ious):
        print(f"Class {class_id}: IOU = {iou}")


from unittest.mock import Mock, PropertyMock
import pytest

from sqlalchemy.orm.session import Session

from generator2.domain.object import ObjectService

class TestObjectService:
    def test_full_object(self, dbsession, object_):
        """
        Ensures that the shape of the dict returned matches expectations.

        The object being tested for looks like:
        {
            "kind": "bovine",
            "slug": "cowslug",
            "properties": [
                {
                    "label": "The cow says",
                    "instances": ["moo"]
                },
                {
                    "label": "name",
                    "instances": ["aloysius"]
                }
            ]
        }
        """
        object_under_test = ObjectService(dbsession)
        actual_object = object_under_test.full_object('cowslug')
        assert 'bovine' == actual_object.get('kind')
        assert 'cowslug' == actual_object.get('slug')
        assert 2 == len(actual_object['properties'])

        property_ = actual_object.get('properties')[0]
        assert 'The cow says' == property_.get('label')
        assert 1 == len(property_.get('instances'))

        name = actual_object.get('properties')[1]
        assert 'name' == name.get('label')
        assert ['aloysius'] == name.get('instances')

        instance = property_.get('instances')[0]
        assert 'moo' == instance

    @pytest.fixture
    def dbsession(self, object_, additional_properties):
        dbsession = Mock(Session)
        _lookup_object = Mock()
        _lookup_object.filter.return_value.one.return_value = object_
        _object_additional_properties = Mock()
        _object_additional_properties.filter.return_value.all.return_value = additional_properties
        dbsession.query.side_effect = (_lookup_object, _object_additional_properties)
        return dbsession

    @pytest.fixture
    def instance(self):
        instance = Mock()
        type(instance).value = PropertyMock(return_value='moo')
        return instance

    @pytest.fixture
    def additional_instance(self):
        instance = Mock()
        type(instance).value = PropertyMock(return_value='aloysius')
        return instance

    @pytest.fixture
    def object_(self, property_):
        object_ = Mock()
        type(object_).kind = PropertyMock(return_value='bovine')
        type(object_).slug = PropertyMock(return_value='cowslug')
        type(object_).properties = PropertyMock(return_value=[property_])
        return object_

    @pytest.fixture
    def property_(self, instance):
        property_ = Mock()
        type(property_).label = PropertyMock(return_value='The cow says')
        type(property_).instances = PropertyMock(return_value=[instance])
        return property_

    @pytest.fixture
    def additional_properties(self, additional_instance):
        property_ = Mock()
        type(property_).label = PropertyMock(return_value='name')
        type(property_).instances = PropertyMock(return_value=[additional_instance])
        return [property_]


import os
from PyPDF2 import PdfReader

class Files():

    def __init__(self, path):
        self.path = path

    def getFiles(self):
        files = []
        try:
            for diretorio, subpastas, arquivos in os.walk(self.path):
                for arquivo in arquivos:
                    if arquivo.endswith('.pdf') or arquivo.endswith('.PDF'):
                        files.append(os.path.join(diretorio, arquivo))
        except:
             print("Arquivos não encontrados.")
        finally:
            return files

    def readFile(self, pathFile):

        try:
            pages = []
            filter = []
            reader = PdfReader(pathFile)

            for i in range(len(reader.pages)):
                #print(i)
                current = reader.pages[i].extract_text().splitlines()
                for line in current:
                    pages.append(line)

            """ for line in pages:
                split = line.split(" ")
                if "/" in split[0]:
                    i=1
                    print(pages[pages.index(line)])
                    filter.append(pages[pages.index(line)])
                    while "CFE" not in pages[pages.index(line)+i]:
                        linha = pages[pages.index(line)+i]
                        splitted = linha.split(" ")
                        #print(len(splitted[0]))
                        if "UN" in splitted[-1]:
                            print(linha)
                            filter.append(linha)

                        if "CFE" in pages[pages.index(line)+i]:
                            break

                        i += 1 """

            for i in range(len(pages)):
                split = pages[i].split(" ")
                if "/" in split[0]:
                    j=1
                    #print(pages[pages.index(pages[i])])
                    filter.append(pages[pages.index(pages[i])])
                    while "CFE" not in pages[pages.index(pages[i])+j]:

                        linha = pages[pages.index(pages[i])+j]
                        splitted = linha.split(" ")
                        if "UN" in splitted[-1]:
                            #print(linha)
                            filter.append(linha)
                        elif "CFE" in pages[pages.index(pages[i])+j]:
                            j=1
                            break
                        print(pages.index(pages[pages.index(pages[i])+j]), len(pages), j)
                        j+=1

        except Exception as e:
            print('\n')
            print(f"   Erro ao ler arquivo {pathFile.split('/')[-1]}", '\n')
            print("  ", "Erro: ", e, '\n')
            print("  ", "Pulando arquivo...", '\n')


import tensorflow as tf
import numpy as np
from dbObL import dbOb
import pickle
import sys
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
import string
import math
import collections
import random
import datetime as dt
from sklearn.decomposition import PCA
from matplotlib import pyplot
from sklearn.feature_extraction.text import TfidfVectorizer
from config import *
import re
from sklearn.neighbors import NearestNeighbors
from nltk import bigrams

def get_mysql_data():
    dobj = dbOb()
    tcursor = dobj.db.cursor()
    fetchQry = "SELECT data FROM syllabotData"
    tcursor.execute(fetchQry, [])
    allFetch = tcursor.fetchall()
    allSents = [x[0] for x in allFetch]
    # print(allSents)
    return allSents

def tokenization_cleaning(allSents):
    tokens = [word_tokenize(report.lower()) for report in allSents]
    regex = re.compile('[%s]' % re.escape(string.punctuation)) #see documentation here: http://docs.python.org/2/library/string.html
    punct_remove = []
    for sent in tokens:
        new_tokens = []
        for token in sent:
            new_token = regex.sub(u'', token)
            if not new_token == u'':
                new_tokens.append(new_token)

        punct_remove.append(new_tokens)

    stop_remove = []
    for sent in punct_remove:
        new_term_vector = []
        for word in sent:
            if not word in stopwords.words('english'):
                new_term_vector.append(word)
        stop_remove.append(new_term_vector)
    return punct_remove, stop_remove

def knn_run(vectors, subject):
    nbrs = NearestNeighbors(n_neighbors=10, algorithm='ball_tree').fit(vectors)
    distances, indices = nbrs.kneighbors([subject])
    return indices

def join_bigrams(stop_remove):
    target_bigram_list = [('cs', '252'), ('email', 'id'), ('final', 'exam'), ('midterm', 'exam')]
    replace_bigram_list = ['cs_252', 'email_id', 'final_exam', 'midterm_exam']
    new_stop_remove = []
    for sent in stop_remove:
        new_sent = []
        bgrm = list(bigrams(sent))
        for tbgrm in target_bigram_list:
            if tbgrm in bgrm:
                new_sent.append(replace_bigram_list[target_bigram_list.index(tbgrm)])
                for bw in tbgrm:
                    sent.pop(sent.index(bw))
        for word in sent:
            new_sent.append(word)
        new_stop_remove.append(new_sent)
    return new_stop_remove

def attach_important_stops(punct_remove, stop_remove, wv, wordidx):
    question_stops = set(["what", "how", "when", "where", "which"])
    new_final_tokens = []
    # print("stop_remove", stop_remove)
    stop_remove = join_bigrams(stop_remove)
    for sent in stop_remove:
        if len(sent) == 1:
            idx = stop_remove.index(sent)
            punct_map = set(punct_remove[idx])
            hit_stop = list(set(punct_map).intersection(question_stops))
            if len(hit_stop) > 0:
                for hs in hit_stop:
                    indices = knn_run(wv, wv[wordidx.index(hs)])
                    for index in indices[0][1:]:
                        if wordidx[index] not in question_stops:
                            sent.insert(0, wordidx[index])
                            break
            else:
                new_final_tokens.append(sent)
        new_final_tokens.append(sent)
    return new_final_tokens

def cleaned_input(wv, widx):
    allSents = get_mysql_data()
    punct_remove, stop_remove = tokenization_cleaning(allSents)
    final_tokens = attach_important_stops(punct_remove, stop_remove, wv, widx)
    return final_tokens

def sort_mysql_data(final_tokens):
    f = open("./data/allData", "w")
    for list_token in final_tokens:
        f.write(' '.join(list_token))
        f.write("\n")
    f.close()

def build_data():
    pf = open("./data/pairings", "r")
    pairs = pf.read().split("\n")
    allwords = []
    for pair in pairs:
        pairword = pair.split(",")
        for pw in pairword:
            if pw not in allwords:
                allwords.append(pw)
    # print(allwords)
    pf.close()
    X = np.eye(len(allwords), dtype=int)
    pair2d = []
    for pair in pairs:
        pairsplit = pair.split(",")
        pair2d.append(pairsplit)

    y = np.zeros((len(allwords), len(allwords)), np.int)

    r = 0
    for x in X:
        idx = np.where(x == 1)[0][0]
        iw = allwords[idx]
        for p1d in pair2d:
            if iw in p1d:
                for ws in p1d:
                    y[r][allwords.index(ws)] = 1
        r += 1
    return X, y, allwords

def gen_w2v(X, y):
    x_train = np.asarray(X)
    y_train = np.asarray(y)
    vocab_size = len(X)
    x = tf.placeholder(tf.float32, shape=(None, vocab_size))
    y_label = tf.placeholder(tf.float32, shape=(None, vocab_size))

    EMBEDDING_DIM = 3
    W1 = tf.Variable(tf.random_normal([vocab_size, EMBEDDING_DIM]))
    b1 = tf.Variable(tf.random_normal([EMBEDDING_DIM])) #bias
    hidden_representation = tf.add(tf.matmul(x,W1), b1)

    W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, vocab_size]))
    b2 = tf.Variable(tf.random_normal([vocab_size]))
    prediction = tf.nn.softmax(tf.add( tf.matmul(hidden_representation, W2), b2))

    sess = tf.Session()
    init = tf.global_variables_initializer()
    sess.run(init)

    cross_entropy_loss = tf.reduce_mean(-tf.reduce_sum(y_label * tf.log(prediction), reduction_indices=[1]))
    train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy_loss)

    n_iters = 10000
    for _ in range(n_iters):
        sess.run(train_step, feed_dict={x: x_train, y_label: y_train})
        print('loss is : ', sess.run(cross_entropy_loss, feed_dict={x: x_train, y_label: y_train}))

    vectors = sess.run(W1 + b1)
    return vectors

def plot_vectors(wvs, wordidx):
    pca = PCA(n_components=2)
    result = pca.fit_transform(wvs)
    pyplot.scatter(result[:, 0], result[:, 1])
    for i, word in enumerate(wordidx):
    	pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))
    pyplot.show()

def save_vocab(wordidx):
    np.save("./data/vocab", wordidx)

def main():
    buildFlag = None
    mysqlPullFlag = False
    if len(sys.argv) > 1:
        buildFlag = sys.argv[1]
        if "--data-pull=true" in sys.argv:
            mysqlPullFlag = True
    X, y, wordidx = build_data()
    save_vocab(wordidx)
    # print(np.load("./data/vocab.npy"))
    word_vec_file = "wv.pkl"
    wvs = None
    if buildFlag == '-n' or buildFlag is None:
        wvs = gen_w2v(X, y)
        with open(word_vec_file, "wb") as f:
            pickle.dump(wvs, f)
    elif buildFlag == "-o":
        with open(word_vec_file, "rb") as f:
            wvs = pickle.load(f)

    if wvs is None:
        print("error loading w2v model")
        sys.exit(-1)

    # plot_vectors(wvs, wordidx)
    if mysqlPullFlag:
        final_tokens = cleaned_input(wvs, wordidx)
        sort_mysql_data(final_tokens)

if __name__ == '__main__':
    main()


# python standard library
from abc import ABCMeta, abstractproperty, abstractmethod
from contextlib import closing
import shelve

# this package
from apcommand.baseclass import BaseClass

# this package
from commons import action_dict, radio_page
from commons import BroadcomWirelessData, BroadcomRadioData
from commons import BroadcomPages, set_page
from commons import BandEnumeration
from querier import BroadcomRadioQuerier, BroadcomSSIDQuerier

class BroadcomBaseData(object):
    """
    The base-data to hold the data-dict
    """
    def __init__(self):
        """
        Base Data constructor
        """
        super(BroadcomBaseData, self).__init__()
        return

    @classmethod
    def base_data(cls):
        """
        Returns data dictionary that only has action:Apply
        """
        return action_dict()

    @classmethod
    def base_24_ghz_data(cls):
        """
        data-dictionary to choose the 2.4 GHz interface
        """
        bwd = BroadcomWirelessData
        base_24_ghz_data = {bwd.wireless_interface: bwd.interface_24_ghz}
        return base_24_ghz_data

    @classmethod
    def base_5_ghz_data(cls):
        """
        data-dictionary to chose the 5 Ghz interface
        """
        bwd = BroadcomWirelessData
        base_5_ghz_data = {bwd.wireless_interface:bwd.interface_5_ghz}
        return base_5_ghz_data

    @classmethod
    def data(cls, action=True, band=None):
        """
        Builds a data-dict based on action and band

        :param:

         - `action`: if True, add action:Apply to the dict
         - `band`: if 2.4 or 5 add the interface selection data

        :return: base data-dict
        """
        band = str(band)
        if action:
            base_data = cls.base_data()
        else:
            base_data = {}

        if band.startswith('2'):
            base_data.update(cls.base_24_ghz_data())
        elif band.startswith('5'):
            base_data.update(cls.base_5_ghz_data())
        return base_data

    def __repr__(self):
        """
        Returns a representation of the data
        """
        return "{0}: Base: {1} 5GHz: {1} 2.4GHz: {2}".format(self.__class__.__name__,
                                                             self.base_data(),
                                                             self.base_5_ghz_data(),
                                                             self.base_24_ghz_data())

class BroadcomBaseCommand(BaseClass):
    """
    A base-class for the commands that change settings
    """
    __metaclass__ = ABCMeta
    def __init__(self, connection, band=None):
        """
        BroadcomBaseCommand constructor

        :param:

         - `connection`: A connection to the AP (HTTPConnection)
         - `band`: 2.4, 5, or None (chooses the data-dictionary)
        """
        super(BroadcomBaseCommand, self).__init__()
        self._logger = None
        self.connection = connection
        self._band = band
        self._base_data = None
        self._singular_data = None
        self._added_data = None
        self._non_base_data = None
        self._data = None
        self.previous_state = None
        self._querier = None
        self._shelf_name = None
        self._shelf_objects = None
        self._shelf_key = None
        self._asp_page = None
        return

    @abstractproperty
    def asp_page(self):
        """
        The page for the decorators to use (e.g. 'radio.asp')
        """

    @property
    def shelf_key(self):
        """
        a key to use for the cases where there's only one object to shelve
        """
        if self._shelf_key is None:
            self._shelf_key = "{0}_{1}".format(self.__module__,
                                               self.__class__.__name__)

    @property
    def shelf_objects(self):
        """
        The key:object dictionary to save (defaults to current singular-data)
        """
        return {self.shelf_key:self.singular_data}

    @property
    def shelf_name(self):
        """
        A name to use for any shelves created
        """
        if self._shelf_name is None:
            self._shelf_name = "{0}.shelve".format(__package__.split('.')[0])
        return self._shelf_name

    @property
    def querier(self):
        """
        A querier to get the previous state for the undo.

        This assumes radio.asp right now
        """
        if self._querier is None:
            self._querier = BroadcomRadioQuerier(connection=self.connection,
                                                 refresh=False,
                                                 band=self.band)
        return self._querier

    @property
    def band(self):
        """
        the band (2.4, 5, or None)
        """
        return self._band

    @band.setter
    def band(self, new_band):
        """
        sets the band and sets base_data, non_base_data to None
        """
        self.logger.debug('Setting to band: {0}'.format(new_band))
        self._band = str(new_band)
        self._data = None
        self._base_data = self._non_base_data = None
        self._querier = None
        return

    @property
    def base_data(self):
        """
        A data-dictionary to add commands to
        """
        if self._base_data is None:
            self.logger.debug("setting the base-data using band {0}".format(self.band))
            self._base_data = BroadcomBaseData.data(band=self.band)
            self.logger.debug('base-data: {0}'.format(self._base_data))
        return self._base_data

    @abstractproperty
    def singular_data(self):
        """
        A data dictionary with the command-specific data (or empty if querying)
        """
        if self._singular_data is None:
            self._singular_data = {}
        return self._singular_data

    @property
    def added_data(self):
        """
        Maintains data added from other commands
        """
        if self._added_data is None:
            self._added_data = {}
        return self._added_data

    @property
    def non_base_data(self):
        """
        The singular data updated with any added data
        """
        if self._non_base_data is None:
            self._non_base_data = self.singular_data.copy()
            self._non_base_data.update(self.added_data)
        return self._non_base_data

    @property
    def data(self):
        """
        The data dictionary to send to the server
        """
        if self._data is None:
            self._data = self.base_data.copy()
            self._data.update(self.non_base_data)
        return self._data

    def __iadd__(self, other):
        """
        Adds the other's non_base_data to this added_data

        """
        self.added_data.update(other.non_base_data)
        self._data = self._non_base_data = None
        return self

    def __add__(self, other):
        """
        Adds the other's non_base_data to this added_data

        **For this to work the sub-classes can't change the constructor interface**
        """
        new_object = self.__class__(connection=self.connection,
                                    band=self.band)
        new_object.added_data.update(other.non_base_data)
        return new_object

    def __sub__(self, other):
        """
        Removes the other's non-base data from added_data in new object

        :return: object with this data minus other's non_base_data
        """
        new_object = self.__class__(connection=self.connection,
                                    band=self.band)
        new_object._added_data = self.added_data.copy()

        for key in other.non_base_data.iterkeys():
            if key in new_object.added_data:
                del new_object.added_data[key]
        return new_object

    @set_page
    def __call__(self):
        """
        The main method to change settings
        """
        self.store()
        self.connection(data=self.data)
        return

    def undo(self):
        """
        This is an attempt to start bringing this into line with the Command Pattern

        But it isn't clear that it will be easy (or possible) in every case -
        so it isn't made a requirement
        """
        raise NotImplemented("undo not supported for this command")

    def store(self):
        """
        shelve data from this class (use in __call__, not meant for public)

        :precondition: self.shelf_objects is a dictionary of things to shelve
        :postocondition: objects is shelf_objects is shelved
        """
        with closing(shelve.open(self.shelf_name, writeback=True)) as open_shelf:
            for key, value in self.shelf_objects.iteritems():
                open_shelf[key] = value
        return

    def load(self):
        """
        Loads the shelved data for the undo method
        (to prevent circular calls using shelf_objects' keys, just uses shelf_key)

        :return: dictionary of objects using shelf_objects' keys
        :raise: KeyError if an item in shelf_objects isn't on the shelf
        """
        with closing(shelve.open(self.shelf_name, flag='r')) as open_shelf:
            data = open_shelf[self.shelf_key]
        return data

# end class BroadcomBaseCommand

class StateData(object):
    __slots__ = ()
    state = 'state'
    band = 'band'

class EnableInterface(BroadcomBaseCommand):
    """
    An interface enabler
    """
    def __init__(self, *args, **kwargs):
        super(EnableInterface, self).__init__(*args, **kwargs)
        self._enable_24_data = None
        self._enable_5_data = None
        self._disable = None
        return

    @property
    def asp_page(self):
        """
        radio.asp
        """
        if self._asp_page is None:
            self._asp_page = BroadcomPages.radio
        return self._asp_page

    @property
    def disable(self):
        """
        An interface disabler
        """
        if self._disable is None:
            self._disable = DisableInterface(connection=self.connection,
                                             band=self.band)
        return self._disable

    @property
    def shelf_objects(self):
        """
        A dictionary of shelf_key: band
        """
        previous_state = self.querier.state
        return {self.shelf_key:self.band}

    def undo(self):
        """
        Disable the interface

        This is actually just a call to DisableInterface...
        """
        try:
            self.band = self.load()
            self.disable()
        except KeyError as error:
            self.logger.debug(error)
            self.logger.debug('No entry on the shelf for {0}'.format(self.shelf_key))
        return

    @property
    def singular_data(self):
        """
        The data to enable the interface
        """
        band = str(self.band)
        if band.startswith('2'):
            return self.enable_24_data
        elif band.startswith('5'):
            return self.enable_5_data

    @property
    def enable_5_data(self):
        """
        The data to send to the connection to enable 5 GHz

        :return: dict of data-values for the connection
        """
        if self._enable_5_data is None:
            radio_data = BroadcomRadioData
            self._enable_5_data = {radio_data.interface :radio_data.radio_on}
        return self._enable_5_data

    @property
    def enable_24_data(self):
        """
        The data to send to the connection to enable 2.4 GHz

        :return: dict of data-values for the connection
        """
        if self._enable_24_data is None:
            radio_data = BroadcomRadioData
            self._enable_24_data = {radio_data.interface:radio_data.radio_on}
        return self._enable_24_data

class DisableInterface(BroadcomBaseCommand):
    """
    An interface enabler
    """
    def __init__(self, *args, **kwargs):
        super(DisableInterface, self).__init__(*args, **kwargs)
        self._disable_24_data = None
        self._disable_5_data = None
        self._enable = None
        return

    @property
    def asp_page(self):
        """
        radio.asp
        """
        if self._asp_page is None:
            self._asp_page = BroadcomPages.radio
        return self._asp_page

    @property
    def enable(self):
        """
        An interface enabler for the undo
        """
        if self._enable == None:
            self._enable = EnableInterface(connection=self.connection,
                                           band=self.band)
        return self._enable

    @property
    def querier(self):
        """
        A querier to get the previous state
        """
        if self._querier is None:
            self._querier = BroadcomQuerier(connection=self.connection,
                                            refresh=True,
                                            band=self.band)
        return self._querier

    @property
    def shelf_objects(self):
        """
        dictionary of shelf_key: band
        """
        return {self.shelf_key: self.band}

    def undo(self):
        """
        enables the interface
        """
        try:
            self.band = self.load()
            self.enable()
        except KeyError as error:
            self.logger.debug(error)
            self.logger.debug('No entry on the shelf for {0}'.format(self.shelf_key))
        return

    @property
    def singular_data(self):
        """
        The data to enable the interface

        This isn't persistent, as changing the band changes the data
        """
        band = str(self.band)
        if band.startswith('2'):
            return self.disable_24_data
        elif band.startswith('5'):
            return self.disable_5_data

    @property
    def disable_5_data(self):
        """
        The data to send to the connection to disable 5 GHz

        :return: dict of data-values for the connection
        """
        if self._disable_5_data is None:
            radio_data = BroadcomRadioData
            self._disable_5_data = {radio_data.interface: radio_data.radio_off}
        return self._disable_5_data

    @property
    def disable_24_data(self):
        """
        The data to send to the connection to enable 2.4 GHz

        :return: dict of data-values for the connection
        """
        if self._disable_24_data is None:
            radio_data = BroadcomRadioData
            self._disable_24_data = {radio_data.interface: radio_data.radio_off}
        return self._disable_24_data

class SetChannel(BroadcomBaseCommand):
    """
    A channel setter for the AP
    """
    def __init__(self, *args, **kwargs):
        super(SetChannel, self).__init__(*args, **kwargs)
        self._channel_map = None
        self._channel = None
        return

    @property
    def asp_page(self):
        """
        radio.asp
        """
        if self._asp_page is None:
            self._asp_page = BroadcomPages.radio
        return self._asp_page

    @property
    def shelf_objects(self):
        """
        A dictionary of shelf_key: channel for current band
        """
        self.querier.band = self.band
        channel = self.querier.channel
        return {self.shelf_key: channel}

    @property
    def channel(self):
        """
        returns the channel
        """
        return self._channel

    @channel.setter
    def channel(self, new_channel):
        """
        Sets the channel, the band, and singular_data based on the channel.
        """
        channel = str(new_channel)
        self.band = self.channel_map[channel]
        self._singular_data = {BroadcomRadioData.control_channel:channel}
        return

    @property
    def channel_map(self):
        """
        Map of channel to data-dictionary
        """
        if self._channel_map is None:
            channel_24 = [str(channel) for channel in range(1,12)]
            channel_24_data = ['2.4'] * len(channel_24)
            # these are the only channels that match the Atheros channels we chose
            channel_5 = BroadcomRadioData.channels_5ghz
            channel_5_data = ['5'] * len(channel_5)
            channels = channel_24 + channel_5
            data = channel_24_data + channel_5_data
            self._channel_map = dict(zip(channels, data))
        return self._channel_map

    @property
    def singular_data(self):
        """
        This is a pass-through (it has to be set when given a channel)
        """
        return self._singular_data

    def undo(self):
        """
        Sets the channels found on the shelve
        """
        try:
            self.channel = self.load()
            self()
        except KeyError as error:
            self.logger.debug(error)
            self.logger.debug("No entry on shelf for {0}".format(self.shelf_key))
        return

class SetSideband(BroadcomBaseCommand):
    """
    A side-band ('upper' or 'lower') setter
    """
    def __init__(self, *args, **kwargs):
        super(SetSideband, self).__init__(*args, **kwargs)
        self.band = '5'
        self._direction = None
        return

    @property
    def asp_page(self):
        """
        radio.asp
        """
        if self._asp_page is None:
            self._asp_page = BroadcomPages.radio
        return self._asp_page

    @property
    def singular_data(self):
        """
        This has to be set in the call when you know what the direction is
        """
        return self._singular_data

    @property
    def direction(self):
        """
        upper or lower
        """
        return self._direction

    @direction.setter
    def direction(self, new_direction):
        """
        sets the direction, band, and singular_data
        """
        direction = new_direction.lower()
        if direction.startswith('l'):
            self._direction = 'lower'
        elif direction.startswith('u'):
            self._direction = 'upper'
        #else raise some kind of error
        self._singular_data = {BroadcomRadioData.sideband:self._direction}
        return

    def __call__(self):
        """
        Sets the sideband ('upper' or 'lower') for 5GHz
        """
        self.connection(data=self.data)
        return
# end SetSideband

# python standard library
import unittest
import random
import string

# third-party
from mock import MagicMock, call, patch

random_letters = lambda: ",".join((random.choice(string.letters) for char in xrange(random.randint(1,5))))

class BadChild(BroadcomBaseCommand):
    def query(self):
        return

class BadChild2(BroadcomBaseCommand):
    def base_data(self):
        return

class EvilChild(BroadcomBaseCommand):
    def asp_page(self):
        return

    def base_data(self):
        return

    def query(self):
        return

    @property
    def singular_data(self):
        return

    def __call__(self):
        return

    def path(self):
        return

# this can't be random in the class for testing to work
# Because __add__  creates a new object
test_data = {random_letters(): random_letters()}
test_data_2 = {random_letters(): random_letters()}
test_data_3 = {random_letters(): random_letters()}
class TestChild(BroadcomBaseCommand):
    def __init__(self, *args, **kwargs):
        super(TestChild, self).__init__(*args, **kwargs)
        return

    def asp_page(self):
        return BroadcomPages.radio

    @property
    def singular_data(self):
        if self._singular_data is None:
            self._singular_data = test_data
        return self._singular_data

    def __call__(self):
        return

    def __add__(self, other):
        t = TestChild(self.connection)
        t.added_data.update(other.non_base_data)
        return t

    def path(self):
        return

class TestChild2(TestChild):
    def __init__(self, *args, **kwargs):
        super(TestChild2, self).__init__(*args, **kwargs)
        return

    @property
    def singular_data(self):
        if self._singular_data is None:
            self._singular_data = test_data_2
        return self._singular_data

class TestChild3(TestChild):
    def __init__(self, *args, **kwargs):
        super(TestChild3, self).__init__(*args, **kwargs)
        return

    @property
    def singular_data(self):
        if self._singular_data is None:
            self._singular_data = test_data_3
        return self._singular_data

class TestBroadcomCommands(unittest.TestCase):
    def setUp(self):
        self.connection = MagicMock()
        return

    def test_abstract_property(self):
        """
        Does a class without the properties defined crash on creation?
        """
        self.assertRaises(TypeError, BroadcomBaseCommand, args=(self.connection,))
        self.assertRaises(TypeError, BadChild, args=(self.connection,))
        self.assertRaises(TypeError, BadChild2, args=(self.connection,))
        EvilChild(self.connection)
        return

    def test_add(self):
        """
        Can you add two commands?
        """
        command_1 = TestChild(self.connection)
        command_2 = TestChild2(self.connection)

        data_1, data_2 = command_1.data, command_2.data

        data_changed = data_1.copy()
        data_changed.update(data_2)

        # first __add__
        command_3 = command_1 + command_2

        # creates a new object with the combined data
        self.assertEqual(data_changed, command_3.data)
        self.assertEqual(self.connection, command_3.connection)
        # leaves the originals alone
        self.assertEqual(command_1.data, data_1)
        self.assertEqual(command_2.data, data_2)

        # now __iadd__
        command_1 += command_2
        # changes LHS in place
        self.assertEqual(command_1.data, data_changed)
        self.assertEqual(command_2.data, data_2)
        return

    def test_subtract(self):
        """
        Can you subtract two commands?
        """
        command_1 = TestChild(self.connection)
        command_2 = TestChild2(self.connection)
        command_3 = TestChild3(self.connection)

        data_1, data_2, data_3 = command_1.data, command_2.data, command_3.data
        data_changed = data_1.copy()
        data_changed.update(data_2)
        data_changed.update(data_3)

        # first add so the commands are different
        command_1 += command_2
        command_1 += command_3

        # First __sub__
        command_4 = command_1 - command_2

        data_1_3 = data_1.copy()
        data_1_3.update(data_3)
        # new cleaned object
        self.assertEqual(command_4.data, data_1_3)

        command_4 = command_1 - command_2 - command_3
        self.assertEqual(command_4.data, data_1)

        # old objects unchanged
        self.assertEqual(command_1.data, data_changed)
        self.assertEqual(command_2.data, data_2)

        # now __isub__
        self.assertNotEqual(command_1.data, data_1)
        command_1 -= command_2
        self.assertEqual(command_1.data, data_1_3)
        self.assertEqual(command_2.data, data_2)

        command_1 -= command_3
        self.assertEqual(command_1.data, data_1)
        return

class TestEnableInterface(unittest.TestCase):
    def setUp(self):
        self.connection = MagicMock(name='connection')
        self.html = MagicMock(name='html')
        self.connection.return_value = self.html
        self.html.text = open('radio_asp.html').read()
        self.command = EnableInterface(connection=self.connection)
        return

    def test_previous_state(self):
        """
        Does it get the previous state correctly?
        """
        command = EnableInterface(connection=self.connection,
                                  band='2')
        command()

        command.band = '5'
        command()
        return

    def test_24ghz(self):
        """
        Does it construct the right set of data to Enable the interface?
        """
        command = EnableInterface(connection=self.connection,
                                  band='2.4')
        expected_data =  {'wl_unit':'0',
                          'action':'Apply',
                          'wl_radio':'1'}

        self.assertEqual(command.data, expected_data)
        command()
        self.assertEqual(self.connection.path, 'radio.asp')
        self.connection.assert_called_with(data=expected_data)
        return

    def test_5_ghz(self):
        """
        Does it send the 5 GHz data to the connection?
        """
        command = EnableInterface(connection=self.connection,
                                  band='5')
        expected_data ={'action':'Apply',
                        'wl_unit':'1',
                        'wl_radio':'1'}
        self.assertEqual(command.data, expected_data)

        # what happens if you change it to 2.4?
        command.band = '2.4'
        expected_data ={'action':'Apply',
                        'wl_unit':'0',
                        'wl_radio':'1'}
        self.assertEqual(command.data, expected_data)
        return

    def test_undo(self):
        """
        Does the undo disable the interface?
        """
        connection = MagicMock(name='connection')
        html = MagicMock(name='html')
        connection.return_value = html
        html.text = open('radio_asp.html').read()
        command = EnableInterface(connection=connection)

        command.band = 5
        command()

        html.text = open('radio_5_asp.html').read()

        command.undo()
        query_call = call(data={'wl_unit':'1'})
        calls = [query_call,
                 call(data={'action':'Apply',
                        'wl_unit':'1',
                        'wl_radio':'1'}),
                            call(data ={'action':'Apply',
                        'wl_unit':'1',
                        'wl_radio':'0'})]
        self.assertEqual(connection.mock_calls, calls)
        self.command.previous_state = 'Enabled'

        # the disable checks the state so we need to change the html so it is Disabled
        #self.html.text = open('radio_5_asp.html').read()
        #self.command.undo()
        #self.assertEqual(self.connection.mock_calls, calls)
        return

class TestDisableInterface(unittest.TestCase):
    def setUp(self):
        self.connection = MagicMock(name='connection')
        self.html = MagicMock(name='html')
        self.connection.return_value = self.html
        self.html.text = open('radio_5_asp.html').read()
        return

    def test_24_ghz(self):
        """
        Does it setup the right data dictionary and make the right call?
        """
        command = DisableInterface(connection=self.connection,
                                   band='2.4')
        expected_data = {'action':'Apply',
                        'wl_unit':'0',
                        'wl_radio':'0'}
        self.assertEqual(command.data, expected_data)
        command()
        self.connection.assert_called_with(data=expected_data)
        self.assertEqual(self.connection.path, 'radio.asp')
        return

    def test_5_ghz(self):
        """
        Does it set up the right data dictionary to disable the 5 Ghz interface?
        """
        command = DisableInterface(connection=self.connection,
                                   band='5')
        expected_data = {'action':'Apply',
                        'wl_unit':'1',
                        'wl_radio':'0'}
        self.assertEqual(command.data, expected_data)

class TestSetChannel(unittest.TestCase):
    def setUp(self):
        self.fake_channel = random_letters()
        self.querier = MagicMock()
        self.querier.channel = self.fake_channel
        self.connection = MagicMock()
        self.command = SetChannel(connection=self.connection)
        self.command._querier = self.querier
        return

    def test_24_ghz(self):
        """
        Does it pass in the correct data to set the (2.4 GHz) channel?
        """
        channel = random.choice(BroadcomRadioData.channels_24ghz)
        self.command.channel = channel
        expected_data = {'action':'Apply',
                        'wl_unit':'0',
                        'wl_channel':str(channel)}
        # setup the shelving
        self.command._querier = self.querier
        mock_shelf = MagicMock(spec_set=dict)
        mock_closing = MagicMock()
        mock_closing.return_value = mock_shelf
        with patch('contextlib.closing', mock_closing):
            #with patch('shelve', mock_shelf):
            self.command()
        self.connection.assert_called_with(data=expected_data)
        return

    def test_5_ghz(self):
        """
        Does it pass in the correct data to set the (5GHz) channel?
        """
        channel = random.choice(BroadcomRadioData.channels_5ghz)
        self.command.channel = channel
        expected_data = {'action':'Apply',
                        'wl_unit':'1',
                        'wl_channel':str(channel)}
        # setup the shelving
        self.command._querier = self.querier
        mock_shelf = MagicMock(spec_set=dict)
        mock_closing = MagicMock()
        with patch('contextlib.closing', mock_closing):
            #with patch('shelve', mock_shelf):
            self.command()
        self.connection.assert_called_with(data=expected_data)
        return

    def test_shelf_objects(self):
        """
        Is shelf_objects a dict of shelf_key:previous-channels dict?
        """
        querier = MagicMock()
        command = SetChannel(connection=self.connection)
        command._querier = self.querier
        shelf_objects = command.shelf_objects
        self.assertEqual(command.shelf_key, shelf_objects.keys()[0])
        self.assertEqual(self.fake_channel,
                          shelf_objects[command.shelf_key])
        return
# end class TestSetChannel

class TestSetSideband(unittest.TestCase):
    def setUp(self):
        self.connection = MagicMock()
        return

    def test_set_sideband(self):
        """
        Does it set the sideband correctly?
        """
        command = SetSideband(connection=self.connection)
        command.direction = 'lower'
        expected_data = {'action':'Apply',
                        'wl_unit':'1',
                        'wl_nctrlsb':'lower'}
        command()
        self.connection.assert_called_with(data=expected_data)
        return


from scrapy import Spider
from scrapy.http import Request

from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader

import urllib.request, urllib.parse, urllib.error

class TrendnetSpider(Spider):
    name = "trendnet"
    allowed_domains = ["trendnet.com"]
    start_urls = ["http://www.trendnet.com/support/"]

    def parse(self, response):
        for entry in response.xpath("//select[@id='subtype_id']/option"):
            if entry.xpath(".//text()"):
                text = entry.xpath(".//text()").extract()[0]
                href = entry.xpath("./@value").extract()[0]
                yield Request(
                    url=urllib.parse.urljoin(response.url, href),
                    meta={"product": text},
                    headers={"Referer": response.url},
                    callback=self.parse_product)
    def parse_product(self, response):
        product = response.xpath("//div[@class='col-lg-12 col-md-6 col-sm-6']/h2/text()").extract()[0].strip().split(" ")[0].strip()
        device_class = response.xpath("//div[@class='g-px-10--lg g-pt-10']/div/h1[@class='g-font-weight-300 mb-0']/text()").get().strip()
        for tab in response.xpath("//div[@class='row  g-pt-20 g-pb-10 g-px-15']"):
            text = tab.xpath("./div/div/a/text()").extract()
            href = tab.xpath("./div/div/a/@onclick").extract()
            version = tab.xpath("./div/p/text()").extract()
            date = tab.xpath("./div[2]/text()").extract()
            if len(text) == 0:
                continue
            if "firmware" in text[0].lower():
                if "open" in href[0]:
                    link = href[0].split('\'')[1]
                    url = urllib.parse.urljoin("https://www.trendnet.com", link)
                    url = url.replace("inc_downloading.asp?", "inc_downloading.asp?button=Continue+with+Download&Continue=yes&")
                self.logger.debug(date[1].strip())
                self.logger.debug(text[0])
                self.logger.debug(url)
                self.logger.debug(version)
                self.logger.debug(device_class)
                self.logger.debug(product)
                item = FirmwareLoader(
                    item=FirmwareImage(), response=response, date_fmt=["%m/%d/%Y"])
                item.add_value("url", url)
                item.add_value("product", product)
                item.add_value("date", date[1].strip())
                if len(version) > 0:
                    item.add_value("version", version)
                item.add_value("vendor", self.name)
                yield item.load_item()


#!/usr/bin/env python
# coding: utf-8
from __future__ import (absolute_import, print_function,
                        division, unicode_literals)

from pkg_resources import resource_string, resource_listdir
from datetime import datetime
import os
import sys
import re
import argparse

join = lambda a, b: os.path.join(os.path.abspath(a), b)

TODAY = datetime.today().strftime("%Y%m%d")
GENERATOR_MESSAGES = {
    'error': 'Generator failed to generate files',
    'message': "Generating Folder...",
    'lang': "Language found: %s",
    'lang_error': "%s generator wasn't found",
    'exists': "Dojo path already exists!",
    'create': "New %s!",
}
GENERATORS_PATH = join(os.path.dirname(__file__), 'generators')

class GeneratorException(Exception):
    value = GENERATOR_MESSAGES['error']
    def __str__(self):
        return repr(self.value)

class GeneratorNotFoundError(GeneratorException):
    def __init__(self, language):
        self.value = GENERATOR_MESSAGES['lang_error'] % language

class GeneratorPathExistsError(GeneratorException):
    value = GENERATOR_MESSAGES['exists']

def sprint(text, show):
    if show:
        print(text)

def list_to_args(args, show=True):
    opts = args.split(' ')
    if len(opts) < 2:
        raise GeneratorException()
    n = argparse.Namespace()
    n.language, n.problem = opts[:2]
    n.extra = opts[2:]
    n.path = os.path.curdir
    return n

class Generator(object):

    def __init__(self, args):
        self.language = args.language
        self.problem = args.problem
        self.extra =  ('_' + '_'.join(args.extra)) if args.extra else ''
        self.path = args.path
        self.ignore = args.ignore

        self.today = TODAY

        self.folder_name = "{date}{_extra}_{language}_{problem}".format(
            date=TODAY,
            _extra=self.extra,
            language=self.language,
            problem=self.problem,
        )

        self.folder_path = join(self.path, self.folder_name)
        self.generator_path = join(GENERATORS_PATH, self.language)

        splitted_name =  self.problem.split('_')

        snake_case = self.problem
        pascal_case = ''.join(part.capitalize() for part in splitted_name)
        down_case = ''.join(splitted_name)
        camel_case = (
            splitted_name.pop(0) +
            ''.join(part.capitalize() for part in splitted_name)
        )

        self.cases = {
            '___dojogen___' : snake_case,
            '___class_dojogen___' : pascal_case,
            '___down_dojogen___' : down_case,
            '___camel_dojogen___' : camel_case,
        }
        print(self.cases)

        self.generated = False

    def replace(self, text):
        for sub, replace in self.cases.items():
            text = re.sub(sub, replace, text)
        return text

    def copy_and_rename(self, current, folder_name, original):
        isdir = lambda x: os.path.isdir(x) and not os.path.islink(x)

        folder_path = join(current, folder_name)
        os.mkdir(folder_path)
        file_list = os.listdir(original)
        for infile in file_list:
            gen_path = join(original, infile)
            if isdir(gen_path):
                self.copy_and_rename(folder_path, infile, gen_path)
            elif not '.pyc' in gen_path:
                new_path = join(folder_path,self.replace(infile))
                with open(new_path, 'w') as w:
                    with open(gen_path, 'r') as r:
                        for line in r:
                            w.write(self.replace(line))

    def generate(self, show=True):
        if not os.path.exists(self.folder_path):
            sprint(GENERATOR_MESSAGES['message'], show)
            if self.ignore:
                os.mkdir(join(self.path, self.folder_name))
            elif os.path.exists(self.generator_path):
                sprint(GENERATOR_MESSAGES['lang'] % (self.language), show)
                self.copy_and_rename(
                    self.path, self.folder_name, self.generator_path)
                self.generated = True
            else:
                raise GeneratorNotFoundError(self.language)
            sprint(GENERATOR_MESSAGES['create'] % self.folder_name, show)
        else:
            self.generated = True
            raise GeneratorPathExistsError()

def generate(generate, directory):
    if generate:
        generator = Generator(list_to_args(generate))
        try:
            generator.generate()
        finally:
            if generator.generated:
                return generator.folder_path
    return directory

def generate_mode(args):
    generator = Generator(args)
    generator.generate()

def help_mode(args):
    help_path = join(join(GENERATORS_PATH, "help"), args.language)
    if os.path.exists(help_path):
        with open(help_path, 'r') as f:
            print(f.read())
    else:
        raise GeneratorNotFoundError(args.language)

def lang_mode(args):
    for name in os.listdir(GENERATORS_PATH):
        if os.path.isdir(join(GENERATORS_PATH, name)) and name != 'help':
            print(name)

def main():
    gen_msg = '''Generates directory for coding dojo following the pattern:
        {}[_extra]_language_problem\n'''.format(TODAY)
    help_msg = 'Describes how to prepare the environment for a language'
    lang_msg = 'Shows existing generators'
    dgen_msg, dhelp_msg, dlang_msg = gen_msg, help_msg, lang_msg

    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    for name in ['generate', 'gen', 'g']: # Aliases on Python 2.x
        parser_gen = subparsers.add_parser(
            name, description=gen_msg, help=dgen_msg)
        parser_gen.add_argument(
            'language', type=str, help='Programming Language')
        parser_gen.add_argument(
            'problem', type=str, help='Problem Name')
        parser_gen.add_argument(
            'extra', type=str, nargs='*', default='', help='Extra identifier')
        parser_gen.add_argument(
            '--path', '-p', type=str, default=os.path.curdir,
            help='Custom path')
        parser_gen.add_argument(
            '--ignore', '-i', action='store_true',
            help='Do not generate files. Create only the folder')
        parser_gen.set_defaults(func=generate_mode)
        dgen_msg = '...'

    for name in ['help', 'man', 'h']: # Aliases on Python 2.x
        parser_help = subparsers.add_parser(
            name, description=help_msg, help=dhelp_msg)
        parser_help.add_argument(
            'language', type=str, help='Programming Language')
        parser_help.set_defaults(func=help_mode)
        dhelp_msg = '...'

    for name in ['language', 'l']: # Aliases on Python 2.x
        parser_lang = subparsers.add_parser(
            name, description=lang_msg, help=dlang_msg)
        parser_lang.set_defaults(func=lang_mode)
        dlang_msg = '...'

    args, _ = parser.parse_known_args()
    try:
        args.func(args)
    except GeneratorException as err:
        print("Error: {0}".format(err))

if __name__ == '__main__':
    main()


import argparse
import random

import numpy as np
import matplotlib.pyplot as plt

from cv2 import imread,imwrite
import networkx as nx
from networkx.classes import graph
from scipy.optimize import minimize
from sklearn.neighbors import NearestNeighbors

def RRT(p_start,target_label,np_map,theta=0.3,step_size=25,thres_d=30):

	def get_random(theta):
		# random select a point as target
		if random.random()<theta:
			p_rand=p_target
		else:
			p_rand=[int(random.random()*480), int(random.random()*640)]
		return p_rand

	def get_nearest(tree,p_rand):
		# find a existed node which is the nearest one close to the target
		tree_nodes=np.array(tree.nodes())
		n = NearestNeighbors(n_neighbors=1,metric="euclidean")
		n.fit(tree)
		dis, ind = n.kneighbors([p_rand], return_distance=True)
		p_near=tree_nodes[ind.ravel()]
		return p_near.ravel(),dis.ravel()

	def get_new_node(p_near,p_rand,dis,step_size=step_size):
		# get p_new
		d=(p_rand-p_near)/dis*step_size

		p_new=p_near+d # one step along the direction
		p_new=p_new.ravel()

		p_new=[int(p_new[0]),int(p_new[1])]
		# this offset to make sure the distance<=step_size

		if d[0]<0:
			p_new[0]+=1
		if d[1]<0:
			p_new[1]+=1

		return p_new

	def check_achievable(p_near,p_new,np_map,n_sample=25):
		# check if it is achievable
		if p_new[0]>=w or p_new[0]<0 or p_new[1]>=h or p_new[1]<0:
			# out of map
			return False

		d=p_new-p_near
		d_step=d.ravel()/n_sample

		for i in range(n_sample):
			p=p_near+d_step*i
			p=[int(p[0]),int(p[1])]

			if (np_map[p[1]][p[0]]!=[1,1,1,1]).any():
				# is wall or object
				return False
		return True

	def check_mission(tree,p_target,thres_d=10):
		n = NearestNeighbors(n_neighbors=1,metric="euclidean")
		n.fit(tree)
		dis, ind = n.kneighbors([p_target], return_distance=True)
		if dis.ravel()<thres_d:
			p=np.array(tree.nodes())[ind.ravel()]
			return True,p.ravel()

		return False,None

	# map range
	h,w=np_map.shape[0],np_map.shape[1]

	# show start ans goal
	plt.imshow(np_map)
	plt.scatter(p_start[0],p_start[1], marker="v", color="green")
	plt.scatter(p_target[0],p_target[1], marker="*", color="black")
	plt.axis('off')
	plt.show()

	# create Tree
	G = nx.Graph()
	G.add_node(tuple(p_start))

	# start RRT
	count=0
	while tuple(p_target) not in G:
		(flag,p)= check_mission(G,p_target,thres_d)
		if flag:
			G.add_node(tuple(p_target))
			G.add_edge(tuple(p), tuple(p_target))
			break

		if count>=1500:
			# reset
			count=0
			G = nx.Graph()
			G.add_node(tuple(p_start))

		# start search
		p_rand=get_random(theta)
		p_near,dis=get_nearest(G,p_rand)
		p_new=get_new_node(p_near,p_rand,dis,step_size)

		if check_achievable(p_near,p_new,np_map,step_size):
			G.add_node(tuple(p_new))
			G.add_edge(tuple(p_near), tuple(p_new))

		count+=1
		if count%100==0:
			print(f"iter: {count}")
			draw_path(G,None,np_map)

	draw_path(G,None,np_map)
	return G

def get_taget_position(target_label):
	target_color={
		"refrigerator": (255, 0, 0),
		"rack": (0, 255, 133),
		"cushion": (255, 9, 92),
		"lamp": (160, 150, 20),
		"cooktop": (7, 255, 224)
		}

	target=[0,0]
	if target_label=="refrigerator":
		target=[253,249]
	elif target_label=="cushion":
		target=[505,253]
	else:
		c=target_color[target_label]
		c=(c[0]/255,c[1]/255,c[2]/255)

		mask= (np_map[:,:,0]==c[0]) & (np_map[:,:,1]==c[1]) & (np_map[:,:,2]==c[2])
		out=np.where(mask==True)
		x=int(np.mean(out[1]))
		y=int(np.mean(out[0]))
		target=[x,y]

	print(f"search for {target_label}, set target at {target}")
	return target

def onclick(event):
    global ix, iy
    ix, iy = event.xdata, event.ydata

    return (ix, iy)

def draw_path(tree,simple_pth,np_map):
	plt.imshow(np_map)

	for e in tree.edges():
		#draw edges
		pa,pb=list(e[0]),list(e[1])
		plt.plot([pa[0],pb[0]],[pa[1],pb[1]],color="black")

	for n in tree.nodes():
		#draw nodes
		plt.scatter(n[0],n[1], marker="o", color="hotpink",s=10)

	if simple_pth:
		for i in range(len(simple_pth)-1):
			pa,pb=simple_pth[i],simple_pth[i+1]
			plt.plot([pa[0],pb[0]],[pa[1],pb[1]],color="red")

	plt.scatter(p_start[0],p_start[1], marker="v", color="green")
	plt.scatter(p_target[0],p_target[1], marker="*", color="green")

	plt.axis('off')
	plt.savefig(f'./{args.target}.png')
	plt.show()

if __name__=="__main__":

	parser = argparse.ArgumentParser()
	parser.add_argument("--map_pth", default="./map.png")
	parser.add_argument("-t","--target", default="cushion",type=str,
		choices={"refrigerator", "rack", "cushion", "lamp","cooktop"},
		help="target label to achieve")
	args = parser.parse_args()

	np_map=plt.imread(args.map_pth)

	# get target position
	p_target=get_taget_position(args.target)

	# plot map to choose start point
	fig, ax = plt.subplots()
	plt.imshow(np_map)
	plt.scatter(p_target[0],p_target[1], marker="*", color="green")
	fig.canvas.mpl_connect('button_press_event', onclick)
	plt.axis('off')
	plt.show()

	# get start point
	global ix,iy
	p_start=[int(ix),int(iy)]
	print(f"start from {p_start}")

	# start searching
	search_G=RRT(p_start,p_target,np_map)
	answer=nx.all_simple_paths(search_G,tuple(p_start),tuple(p_target))
	answer=list(answer)[0]
	draw_path(search_G,answer,np_map)

	# save as txt
	with open(f"./{args.target}_Path.txt","w+") as f:
		for p in answer:
			f.write(f"{str(p)}\n")


import matplotlib
import matplotlib.pyplot as plt
import scienceplots
import numpy as np
plt.style.use('ieee')

files = ['acc_dist_1.csv', 'acc_dist_2.csv', 'acc_dist_3.csv']
def compute(infile):
    f = open(infile, 'r')
    lines = f.readlines()
    f.close()
    gamma = list(map(float, lines[0].strip().split(',')[1:-1]))
    th_0_acc = list(map(float, lines[1].strip().split(',')[1:-1]))
    th_1_acc = list(map(float, lines[2].strip().split(',')[1:-1]))
    th_2_acc = list(map(float, lines[3].strip().split(',')[1:-1]))
    th_3_acc = list(map(float, lines[4].strip().split(',')[1:-1]))
    return gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc

if __name__ == '__main__':
    fig, ax = plt.subplots(figsize=[4, 4])
    gamma_pos = -5
    dist = [1, 2, 3]
    th_0_acc_list = [0, 0, 0]
    th_1_acc_list = [0, 0, 0]
    th_2_acc_list = [0, 0, 0]
    th_3_acc_list = [0, 0, 0]
    for i in range(1, 4):
        gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc = compute(files[i-1])
        th_0_acc_list[i-1] = th_0_acc[gamma_pos]
        th_1_acc_list[i-1] = th_1_acc[gamma_pos]
        th_2_acc_list[i-1] = th_2_acc[gamma_pos]
        th_3_acc_list[i-1] = th_3_acc[gamma_pos]
    ax.plot(dist, th_0_acc_list, '-*', label='DLS')
    ax.plot(dist, th_1_acc_list, '--*', label='DLSM $\\theta_s$=1')
    ax.plot(dist, th_2_acc_list, '--*', label='DLSM $\\theta_s$=2')
    ax.plot(dist, th_3_acc_list, '--*', label='DLSM $\\theta_s$=3')
    # fig.text(0.25, 0.1, '(a)', horizontalalignment='center')
    # fig.text(0.5, 0.1, '(b)', horizontalalignment='center')
    # fig.text(0.75, 0.1, '(c)', horizontalalignment='center')
    ax.set_xlabel('Minimum Levenshtein Distance of Indexes')
    ax.set_ylabel('Accuracy (γ=%.2f)'%gamma[gamma_pos])
    ax.set_xticks(dist)
    ax.legend()
    ax.grid(which='both')
    # plt.gcf().set_dpi(600)
    plt.savefig('dist_optimization.pdf')

    fig, ax = plt.subplots(figsize=[4, 4])
    gamma_pos = -5
    dist = [1, 2, 3]
    th_0_acc_list = [0, 0, 0]
    th_1_acc_list = [0, 0, 0]
    th_2_acc_list = [0, 0, 0]
    th_3_acc_list = [0, 0, 0]
    for i in range(1, 4):
        gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc = compute(files[i-1])
        th_0_acc_list[i-1] = th_0_acc[gamma_pos]
        th_1_acc_list[i-1] = th_1_acc[gamma_pos]
        th_2_acc_list[i-1] = th_2_acc[gamma_pos]
        th_3_acc_list[i-1] = th_3_acc[gamma_pos]

    arr = np.array([th_0_acc_list, th_1_acc_list, th_2_acc_list, th_3_acc_list])
    th = [0, 1, 2, 3]
    print(arr)
    ax.plot(th, arr[:, 0], '-*', label='Min. LD = 1')
    ax.plot(th, arr[:, 1], '--*', label='Min. LD = 2')
    ax.plot(th, arr[:, 2], '--*', label='Min. LD = 3')
    # fig.text(0.25, 0.1, '(a)', horizontalalignment='center')
    # fig.text(0.5, 0.1, '(b)', horizontalalignment='center')
    # fig.text(0.75, 0.1, '(c)', horizontalalignment='center')
    ax.set_xlabel('$\\theta_s$')
    ax.set_ylabel('Accuracy (γ=%.2f)'%gamma[gamma_pos])
    ax.set_xticks(th)
    ax.legend()
    ax.grid(which='both')
    # plt.gcf().set_dpi(600)
    plt.savefig('dist_optimization_1.pdf')
    plt.show()


import os,json,sys,glob
import sqlite3
import urllib3

import sys
import base64
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
import psutil

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def dpapi_decrypt(encrypted):  # 老版本Chrome解密方法
    import ctypes
    import ctypes.wintypes

    class DATA_BLOB(ctypes.Structure):
        _fields_ = [('cbData', ctypes.wintypes.DWORD),
                    ('pbData', ctypes.POINTER(ctypes.c_char))]

    p = ctypes.create_string_buffer(encrypted, len(encrypted))
    blobin = DATA_BLOB(ctypes.sizeof(p), p)
    blobout = DATA_BLOB()
    retval = ctypes.windll.crypt32.CryptUnprotectData(
        ctypes.byref(blobin), None, None, None, None, 0, ctypes.byref(blobout))
    if not retval:
        raise ctypes.WinError()
    result = ctypes.string_at(blobout.pbData, blobout.cbData)
    ctypes.windll.kernel32.LocalFree(blobout.pbData)
    return result

def aes_decrypt(encrypted_txt):  # 新版本Chrome解密方法
    with open(os.path.join(os.environ['LOCALAPPDATA'],
                           r"Google\Chrome\User Data\Local State"), encoding='utf-8', mode="r") as f:
        jsn = json.loads(str(f.readline()))
    encoded_key = jsn["os_crypt"]["encrypted_key"]
    encrypted_key = base64.b64decode(encoded_key.encode())
    encrypted_key = encrypted_key[5:]
    key = dpapi_decrypt(encrypted_key)
    nonce = encrypted_txt[3:15]
    cipher = Cipher(algorithms.AES(key), None, backend=default_backend())
    cipher.mode = modes.GCM(nonce)
    decryptor = cipher.decryptor()
    return decryptor.update(encrypted_txt[15:])

def chrome_decrypt(encrypted_txt):  # 封装后Chrome解密函数
    if sys.platform == 'win32':
        try:
            if encrypted_txt[:4] == b'x01x00x00x00':
                decrypted_txt = dpapi_decrypt(encrypted_txt)
                return decrypted_txt.decode()
            elif encrypted_txt[:3] == b'v10':
                decrypted_txt = aes_decrypt(encrypted_txt)
                return decrypted_txt[:-16].decode()
        except WindowsError:
            return None
    else:
        raise WindowsError

def get_cookies_from_chrome(domain=None, sql=None):  # 给定域名，返回这个域名下的cookie信息
    if not sql:
        sql = f'SELECT name, encrypted_value as value FROM cookies where host_key like "%{domain}%"'
    else:
        sql = sql
    files=get_chrome_db_path()
    cookie_db =files.get('cookie_file')
    if os.environ.get('debug'):
        print("in get_cookies_from_chrome(),db=%s,sql=%s" % (cookie_db,sql))

    dsn = 'file:' + cookie_db + '?mode=ro&nolock=1'
    #con = sqlite3.connect(filename)
    con=sqlite3.connect(dsn,uri=True)
    con.row_factory = sqlite3.Row
    cur = con.cursor()
    cur.execute(sql)
    cookie = ''
    for row in cur:
        if row['value'] is not None:
            name = row['name']
            value = chrome_decrypt(row['value'])
            if value is not None:
                cookie += name + '=' + value + ';'
    return cookie

def get_url_from_chrome(url_query):  # 给定一个URL的SQL where条件查询语句，返回最后一条的url记录
    files=get_chrome_db_path()
    history_db = files.get('history_file')
    dsn = 'file:' + history_db + '?mode=ro&nolock=1'  # 设置只读方式打开，无锁方式，Chrome启动时会锁住History库，所以得nolock
    # print(dsn)
    con = sqlite3.connect(dsn, uri=True)
    con.row_factory = sqlite3.Row
    cur = con.cursor()
    sql = "SELECT * FROM urls where url like '%s' order by last_visit_time desc limit 1;" % url_query
    cur.execute(sql)
    for row in cur:
        return row['url']

def get_father_domain(
        full_domain):  # 给定类似testcase.software.ibm.com域名的字串，返回['testcase.software.ibm.com','.software.ibm.com','.ibm.com','.com']
    items = full_domain.split('.')
    N = len(items)
    res = []
    for i in range(N):
        one_item = []
        for j in range(i, N):
            one_item.append(items[j])
        one_domain = ".".join(one_item)
        if one_domain == full_domain:
            res.append(one_domain)
        else:
            res.append('.' + one_domain)
    return res

def gen_sql_clause(full_domain):
    domains = get_father_domain(full_domain)
    sql = 'SELECT name, encrypted_value as value FROM cookies'
    if not domains:
        return sql
    where_clause = ' where ' + " or ".join(["host_key='%s' or host_key='.%s'" % (domain,domain) for domain in domains])
    return sql + where_clause

def get_cookie_by_url(url):
    try:
        host = url.split('://', maxsplit=1)[1].split('/', maxsplit=1)[0]
        sql = gen_sql_clause(host)
        res = get_cookies_from_chrome(sql=sql)
    except Exception as e:
        print(e)
        return None
    return res

def get_profile_dir(): #通过后台进程，找到chrome.exe的进程，查看它带的--profile-directory=xx的参数，找到chrome的profile目录
    for process in psutil.process_iter():
        try:
            cmdline = process.cmdline()
            # cmdline类似如下数据：cmdline=['C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', '--profile-directory=Profile 1', '--restart']
            if len(cmdline)<2: #cmd至少要带个参数才行
                continue
            cmd=os.path.basename(cmdline[0])
            #print(f"{cmdline=}")
            if 'chrome.exe' in cmd.lower():
                # 获取chrome进程的所有命令行参数
                #print(f"{cmdline=}")
                #查找cmdline数组元素中，含有--profile-directory的数据元素
                profile_cmdline=[x for x in cmdline if '--profile-directory' in x]
                #print(f"{profile_cmdline=}")
                #如果找到了，数据长度>0，然后返回等号=后部分的内容
                if len(profile_cmdline)>0:
                    #print(f"{profile_cmdline=}")
                    return profile_cmdline[0].split('=')[1]
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass

def search_file_in_path(fname,path):#在一个path下面遍历所有子目录，返回找到的第一个文件绝对路径
    search_path=os.path.join(path,'**')
    search_files = glob.glob(os.path.join(search_path, fname), recursive=True)
    # 遍历所有找到的文件，输出其所在的路径
    for search_file in search_files:
        file_path = os.path.abspath(search_file)
        return file_path

def get_chrome_db_path(): #通过去profile目录里面去找cookie和history文件，返回找到的绝对路径
    # 指定要查找的文件名
    file_name = 'cookies'
    data_path = os.path.join(os.environ['LOCALAPPDATA'], r"Google\Chrome\User Data")
    pdir=get_profile_dir()
    if pdir:
        profile_path=os.path.join(data_path,pdir)
    else:
        profile_path = os.path.join(data_path, 'Default')
    # 使用 glob.glob 函数遍历当前目录及其子目录中所有符合条件的文件
    cookie_file=search_file_in_path('cookies',profile_path)
    history_file = search_file_in_path('history', profile_path)
    return {'cookie_file':cookie_file,'history_file':history_file}

if __name__ == '__main__':
    # domain = 'exmail.qq.com'  # 目标网站域名
    # cookie = get_cookies_from_chrome(domain)
    # print(cookie)
    url = 'https://i.mi.com/contacts'
    res = get_cookie_by_url(url)
    print(res)