#!/usr/bin/python3

# Author: Aleksander Vainer, April 2020

# This short python script downloads the free content from the Springer website.
# It is based on the following observation:
# the (example) url of the download page is
# https://link.springer.com/book/10.1007/978-981-13-8759-3
# it leads to download button with the following path
# https://link.springer.com/content/pdf/10.1007/978-981-13-8759-3.pdf
# and
# https://link.springer.com/download/epub/10.1007/978-981-13-8759-3.epub
# (epub does not exist for every book)

# To use this script put it and the csv-file with book_list_csv_filename in any folder and run it.

import csv
import os
import wget

book_list_csv_filename = 'SearchResults_Springer.csv'
report_filename = 'report.csv'

dir_name = os.path.dirname(__file__)
download_dir = os.path.join(dir_name, 'downloads')
if not os.path.exists(download_dir):
    os.mkdir(download_dir)

def page_url_to_pdf_url(input_url):
    output_url = input_url.replace('book', 'content/pdf') + '.pdf'
    return output_url

def page_url_to_epub_url(input_url):
    output_url = input_url.replace('book', 'download/epub') + '.epub'
    return output_url

def bool_to_word(bool_value):
    return 'SUCCESS' if bool_value else 'FAILURE'

with open(book_list_csv_filename, 'r', encoding='utf8') as file_handler:
    reader = csv.DictReader(file_handler)
    page_urls = []
    filenames = []
    for row in reader:
        page_urls.append(row['URL'])
        filename = row['Item Title'] + ' DOI' + row['Item DOI']
        filename = filename.replace('/', '--')
        filename = filename.replace(',', '')
        filenames.append(filename)
    pdf_urls = [page_url_to_pdf_url(path) for path in page_urls]
    epub_urls = [page_url_to_epub_url(path) for path in page_urls]

report_list_of_dicts = []

for page_url, pdf_url, epub_url, filename in zip(page_urls, pdf_urls, epub_urls, filenames):
    try:
        print(os.path.join(download_dir, filename + '.pdf'))
        wget.download(pdf_url, os.path.join(download_dir, filename + '.pdf'))
        dead_pdf_link_found = False
    except:
        dead_pdf_link_found = True
    try:
        wget.download(epub_url, os.path.join(download_dir, filename + '.epub'))
        dead_epub_link_found = False
    except:
        dead_epub_link_found = True
    report_list_of_dicts.append({'page_url': page_url, 'pdf_url': pdf_url, 'epub_url': epub_url,
                                 'pdf_exists': bool_to_word((not dead_pdf_link_found)),
                                 'epub_exists': bool_to_word((not dead_epub_link_found))})

keys = report_list_of_dicts[0].keys()
with open(report_filename, 'w') as file_handler:
    writer = csv.DictWriter(file_handler, keys)
    writer.writeheader()
    writer.writerows(report_list_of_dicts)


import os
import preprocessor
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.naive_bayes import MultinomialNB,BernoulliNB
from sklearn.linear_model import LogisticRegression,SGDClassifier
from sklearn.svm import SVC
import nltk
import pickle
import random
from nltk.tokenize import word_tokenize

class TrainingTheModel():

    def __init__(self, bow):
        self.bow = bow

    def most_frequent_words(self):
        freq_bow = nltk.FreqDist(self.bow)
        frequent_words = list(freq_bow.keys())[:5000]
        print("Word Features Sample",frequent_words[0])
        save_frequent_words = open("pickled_algos/frequent_words5k.pickle","wb")
        pickle.dump(frequent_words, save_frequent_words)
        save_frequent_words.close()

        return frequent_words

    def find_features(self, rev, frequent_words):
        words = word_tokenize(rev)
        features = {}
        for w in frequent_words:
            features[w] = (w in words)

        return features

    def create_featuresets(self, documents, frequent_words):
        try:
            load_feats = open("pickled_algos/featset.pickle","rb")
            featuresets = pickle.load(load_feats)
            load_feats.close()
        except:
            featuresets = [(self.find_features(rev, frequent_words), category) for (rev, category) in documents]
            random.shuffle(featuresets)
            print("FeatSet Length: " , len(featuresets))
            save_feats = open("pickled_algos/featset.pickle","wb")
            pickle.dump(featuresets, save_feats)
            save_feats.close()

        return featuresets

    def create_pickle(self, c, file_name):
        file_name = "pickled_algos/" + file_name + ".pickle"
        save_classifier = open(file_name, 'wb')
        pickle.dump(c, save_classifier)
        save_classifier.close()

    def train_many_classifiers(self, featuresets):
        training_set = featuresets[:20000]
        testing_set = featuresets[20000:]
        print( 'training_set :', len(training_set), '\ntesting_set :', len(testing_set))

        print("ONB Classifier training started...")
        ONB_clf = nltk.NaiveBayesClassifier.train(training_set)
        print("Original Naive Bayes Algo accuracy percent:", (nltk.classify.accuracy(ONB_clf, testing_set))*100)
        self.create_pickle(ONB_clf, "ONB_clf")

        print("MNB Classifier training started...")
        MNB_clf = SklearnClassifier(MultinomialNB())
        MNB_clf.train(training_set)
        print("MNB_classifier accuracy percent:", (nltk.classify.accuracy(MNB_clf, testing_set))*100)
        self.create_pickle(MNB_clf, "MNB_clf")

        print("BNB Classifier training started...")
        BNB_clf = SklearnClassifier(BernoulliNB())
        BNB_clf.train(training_set)
        print("BernoulliNB_classifier accuracy percent:", (nltk.classify.accuracy(BNB_clf, testing_set))*100)
        self.create_pickle(BNB_clf, "BNB_clf")

        print("LogReg Classifier training started...")
        LogReg_clf = SklearnClassifier(LogisticRegression())
        LogReg_clf.train(training_set)
        print("LogisticRegression_classifier accuracy percent:", (nltk.classify.accuracy(LogReg_clf, testing_set))*100)
        self.create_pickle(LogReg_clf, "LogReg_clf")

        print("SGD Classifier training started...")
        SGD_clf = SklearnClassifier(SGDClassifier())
        SGD_clf.train(training_set)
        print("SGDClassifier_classifier accuracy percent:", (nltk.classify.accuracy(SGD_clf, testing_set))*100)
        self.create_pickle(SGD_clf, "SGD_clf")

        print("SVC Classifier training started...")
        SVC_clf = SklearnClassifier(SVC())
        SVC_clf.train(training_set)
        print("SVC_classifier accuracy percent:", (nltk.classify.accuracy(SVC_clf, testing_set))*100)
        self.create_pickle(SVC_clf, "SVC_clf")


# -*- coding: utf-8 -*-

from odoo import api, fields, models, _
from odoo.exceptions import UserError
import datetime
import base64
from sped.efd.icms_ipi.arquivos import ArquivoDigital
from sped.efd.icms_ipi import registros
from sped.efd.icms_ipi.registros import Registro0100
from sped.efd.icms_ipi.registros import Registro0001
from sped.efd.icms_ipi.registros import Registro0005
from sped.efd.icms_ipi.registros import RegistroC001
from sped.efd.icms_ipi.registros import RegistroC100
from sped.efd.icms_ipi.registros import RegistroC101
from sped.efd.icms_ipi.registros import RegistroC170
from sped.efd.icms_ipi.registros import RegistroC190
from sped.efd.icms_ipi.registros import Registro9001
from sped.efd.icms_ipi.registros import RegistroE100
from sped.efd.icms_ipi.registros import RegistroE110
from sped.efd.icms_ipi.registros import RegistroE500
from sped.efd.icms_ipi.registros import RegistroE510
from sped.efd.icms_ipi.registros import RegistroE520
from sped.efd.icms_ipi.registros import Registro1001
from sped.efd.icms_ipi.registros import Registro1010

#from sped.efd.icms_ipi.registros import Registro9900

class SpedFile(models.Model):
    _name = "sped.file"
    _description = "Cria o arquivo para o Sped ICMS / IPI"

    date_start= fields.Date(string='Faturar de')
    date_end = fields.Date(string='Até')
    tipo_arquivo = fields.Selection([
        ('0', 'Remessa do arquivo original'),
        ('1', 'Remessa do arquivo substituto'),
        ], string='Finalidade do Arquivo', default='original')
    log_faturamento = fields.Html('Log de Faturamento')
    company_id = fields.Many2one('res.company', string='Company', required=True,
        default=lambda self: self.env['res.company']._company_default_get('account.account'))
    sped_file = fields.Binary(string=u"Sped")
    sped_file_name = fields.Char(
        string=u"Arquivo Sped")
    vl_sld_cred_ant_difal = fields.Float('Saldo Credor per. ant. Difal', default=0.0)
    vl_sld_cred_transp_difal = fields.Float('Saldo Credor per. seguinte Difal', default=0.0)
    vl_sld_cred_ant_fcp = fields.Float('Saldo Credor per. ant. FCP', default=0.0)
    vl_sld_cred_transp_fcp = fields.Float('Saldo Credor per. seguinte FCP', default=0.0)

    #arq = ArquivoDigital()
    fatura = 0

    @api.multi
    def create_file(self):
        #global arq

        if self.date_start > self.date_end:
            raise UserError('Erro, a data de início é maior que a data de encerramento!')
        num_mes = 1
        self.log_faturamento = 'Gerando arquivo ..
'
        if self.date_start and self.date_end:
            d1 = datetime.datetime.strptime(self.date_start, "%Y-%m-%d")
            d2 = datetime.datetime.strptime(self.date_end, "%Y-%m-%d")
            # TODO estou pegando somente NFe emissao propria aqui,
            # o correto e pegar Emissao Terceiros tbem
            inv = self.env['account.invoice'].search([
                ('date_invoice','>=',self.date_start),
                ('date_invoice','<=',self.date_end),
                ('state','in',['open','paid', 'cancel']),
                ('product_document_id.code', '=', '55'),
                ])
            self.registro0000(inv)
            self.log_faturamento = 'Arquivo gerado com sucesso.
'
        return {
            "type": "ir.actions.do_nothing",
        }

    def versao(self):
        #if fields.Datetime.from_string(self.dt_ini) >= datetime.datetime(2018, 1, 1):
        #    return '012'
        return '012'

    def transforma_data(self, data):  # aaaammdd
        data = self.limpa_formatacao(data)
        return data[6:8] + data[4:6] + data[:4]

    def limpa_formatacao(self, data):
        if data:
            replace = ['-', ' ', '(', ')', '/', '.', ':','º']
            for i in replace:
                data = data.replace(i, '')

        return data

    def formata_cod_municipio(self, data):
        return data[:7]

    def junta_pipe(self, registro):
        junta = ''
        for i in range(1, len(registro._valores)):
            junta = junta + '|' + registro._valores[i]
        return junta

    def registro0000(self, inv):
        arq = ArquivoDigital()
        cod_mun = '%s%s' %(self.company_id.state_id.ibge_code, self.company_id.city_id.ibge_code)
        dta_s = '%s%s%s' %(self.date_start[8:10],self.date_start[5:7],self.date_start[:4])
        dta_e = '%s%s%s' %(self.date_end[8:10],self.date_end[5:7],self.date_end[:4])
        arq._registro_abertura.COD_VER = self.versao()
        arq._registro_abertura.COD_FIN = self.tipo_arquivo
        arq._registro_abertura.DT_INI = self.transforma_data(self.date_start)
        arq._registro_abertura.DT_FIN = self.transforma_data(self.date_end)
        arq._registro_abertura.NOME = self.company_id.legal_name
        arq._registro_abertura.CNPJ = self.limpa_formatacao(self.company_id.cnpj_cpf)
        arq._registro_abertura.UF = self.company_id.state_id.code
        arq._registro_abertura.IE = self.limpa_formatacao(self.company_id.inscr_est)
        arq._registro_abertura.COD_MUN = self.formata_cod_municipio(cod_mun)
        arq._registro_abertura.IM = ''
        arq._registro_abertura.SUFRAMA = ''
        arq._registro_abertura.IND_PERFIL = 'A'
        arq._registro_abertura.IND_ATIV = '0'
        reg0001 = Registro0001()
        if inv:
            reg0001.IND_MOV = '0'
        else:
            reg0001.IND_MOV = '1'
        arq._blocos['0'].add(reg0001)

        reg0005 = Registro0005()
        reg0005.FANTASIA = self.company_id.name
        reg0005.CEP = self.limpa_formatacao(self.company_id.zip)
        reg0005.END = self.company_id.street
        reg0005.NUM = self.limpa_formatacao(self.company_id.number)
        reg0005.COMPL = self.company_id.street2
        reg0005.BAIRRO = self.company_id.district
        reg0005.FONE = self.limpa_formatacao(self.company_id.phone)
        reg0005.EMAIL = self.company_id.email
        arq._blocos['0'].add(reg0005)

        contabilista = Registro0100()
        cod_mun = '%s%s' %(self.company_id.accountant_id.state_id.ibge_code, self.company_id.accountant_id.city_id.ibge_code)
        contabilista.NOME = self.company_id.accountant_id.legal_name
        contabilista.CPF = self.limpa_formatacao(self.company_id.accountant_id.cnpj_cpf)
        contabilista.CRC = self.limpa_formatacao(self.company_id.accountant_id.rg_fisica)
        contabilista.END = self.company_id.accountant_id.street
        contabilista.CEP = self.limpa_formatacao(self.company_id.accountant_id.zip)
        contabilista.NUM = self.company_id.accountant_id.number
        contabilista.COMPL = self.company_id.accountant_id.street2
        contabilista.BAIRRO = self.company_id.accountant_id.district
        contabilista.FONE = self.limpa_formatacao(self.company_id.accountant_id.phone)
        #contabilista.FAX = self.company_id.accountant_id.fax
        contabilista.EMAIL = self.company_id.accountant_id.email
        contabilista.COD_MUN = cod_mun

        arq._blocos['0'].add(contabilista)

        for item_lista in self.query_registro0150():
            arq.read_registro(self.junta_pipe(item_lista))

        for item_lista in self.query_registro0190():
            arq.read_registro(self.junta_pipe(item_lista))

        for item_lista in self.query_registro0200():
            arq.read_registro(self.junta_pipe(item_lista))

        for item_lista in self.query_registro0400():
            arq.read_registro(self.junta_pipe(item_lista))

        regC001 = RegistroC001()
        if inv:
            regC001.IND_MOV = '0'
        else:
            regC001.IND_MOV = '1'
        arq._blocos['C'].add(regC001)

        query = """
                    select distinct
                        d.id, d.state, trim(d.nfe_modelo), d.product_document_id
                    from
                        account_invoice as d
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        d.date_invoice between '%s' and '%s'
                        and ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid', 'cancel')
                        and d.fiscal_position_id is not null
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        cont = 1
        for id in query_resposta:
            if id[2] == '55' and id[1] == 'cancel':
                continue
            #import pudb;pu.db
            self.fatura = id[0]
            #if self.fatura == 222:
            #    import pudb;pu.db
            # TODO C100 - Notas Fiscais - Feito
            for item_lista in self.query_registroC100(self.fatura):
                arq.read_registro(self.junta_pipe(item_lista))
                # TODO C101 - DIFAL - Feito
                for item_lista in self.query_registroC101(self.fatura):
                    arq.read_registro(self.junta_pipe(item_lista))

            # TODO C110 - Inf. Adiciontal

            # TODO C170 - Itens Nota Fiscal de Compras = Fazendo
            for item_lista in self.query_registroC170(self.fatura):
                arq.read_registro(self.junta_pipe(item_lista))

            # TODO C190 - Totalizacao por CST
            for item_lista in self.query_registroC190(self.fatura):
                arq.read_registro(self.junta_pipe(item_lista))

        # TODO BLOCO E - Apuracao ICMS
        # TODO E100 - Periodo Apuracao
        registro_E100 = RegistroE100()
        registro_E100.DT_INI = self.transforma_data(self.date_start)
        registro_E100.DT_FIN = self.transforma_data(self.date_end)
        arq._blocos['E'].add(registro_E100)

        # TODO E110 - Apuracao do ICMS
        for item_lista in self.query_registroE110():
            arq.read_registro(self.junta_pipe(item_lista))

        # TODO E300 - DIFAL
        for item_lista in self.query_registroE300():
            arq.read_registro(self.junta_pipe(item_lista))
            # TODO E310 - DIFAL - Detalhe
            for uf_lista in self.query_registroE310(self.company_id.state_id.code, item_lista.UF):
                arq.read_registro(self.junta_pipe(uf_lista))

        # TODO E500 - Apuracao IPI
        registro_E500 = RegistroE500()
        registro_E500.IND_APUR = '0' # 0 - MENSAL 1 - DECENDIAL
        registro_E500.DT_INI = self.transforma_data(self.date_start)
        registro_E500.DT_FIN = self.transforma_data(self.date_end)
        arq._blocos['E'].add(registro_E500)

        # TODO E510 - Consolidação IPI
        for item_lista in self.query_registroE510():
            arq.read_registro(self.junta_pipe(item_lista))

        # TODO E520 - Apuracao   IPI
        for item_lista in self.query_registroE520():
            arq.read_registro(self.junta_pipe(item_lista))

        registro_1001 = Registro1001()
        registro_1001.IND_MOV = '0'
        arq._blocos['1'].add(registro_1001)

        # TODO Colocar no cadastro da Empresa
        registro_1010 = Registro1010()
        registro_1010.IND_EXP = 'N'
        registro_1010.IND_CCRF = 'N'
        registro_1010.IND_COMB  = 'N'
        registro_1010.IND_USINA = 'N'
        registro_1010.IND_VA = 'N'
        registro_1010.IND_EE = 'N'
        registro_1010.IND_CART = 'N'
        registro_1010.IND_FORM = 'N'
        registro_1010.IND_AER = 'N'

        arq._blocos['1'].add(registro_1010)

        #reg9001 = Registro9001()
        #import pudb;pu.db
        #if inv:
        #    reg9001.IND_MOV = '0'
        #else:
        #    reg9001.IND_MOV = '1'
        #arq._blocos['9'].add(reg9001)
        #import pudb;pu.db
        arq.prepare()
        #self.assertEqual(txt, )
        #sped_f = codecs.open(os.path.abspath(), mode='r', encoding='utf-8')
        self.sped_file_name =  "Sped-%s_%s.txt" % (self.date_end[5:7],self.date_end[:4])
        self.sped_file = base64.encodestring(bytes(arq.getstring(), 'utf-8'))
        #self.sped_file = arq.getstring()

    def query_registro0150(self):
        query = """
                    select distinct
                        d.partner_id
                    from
                        account_invoice as d
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    left join
                        invoice_eletronic nf
                            on nf.invoice_id = d.id
                    where
                        d.date_invoice between '%s' and '%s'
                        and ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid', 'cancel')
                        and ((nf.state = 'done') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.fiscal_position_id is not null
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        for id in query_resposta:
            resposta_participante = self.env['res.partner'].browse(id[0])
            registro_0150 = registros.Registro0150()
            registro_0150.COD_PART = str(resposta_participante.id)
            registro_0150.NOME = resposta_participante.legal_name or resposta_participante.name
            registro_0150.COD_PAIS = resposta_participante.country_id.bc_code
            cpnj_cpf = self.limpa_formatacao(resposta_participante.cnpj_cpf)
            if len(cpnj_cpf) == 11:
                registro_0150.CPF = cpnj_cpf
            else:
                registro_0150.CNPJ = cpnj_cpf
            registro_0150.IE = self.limpa_formatacao(resposta_participante.inscr_est)
            cod_mun = '%s%s' %(resposta_participante.state_id.ibge_code, resposta_participante.city_id.ibge_code)
            registro_0150.COD_MUN = self.formata_cod_municipio(cod_mun)
            registro_0150.SUFRAMA = self.limpa_formatacao(resposta_participante.suframa)
            registro_0150.END = resposta_participante.street
            registro_0150.NUM = resposta_participante.number
            registro_0150.COMPL = resposta_participante.street2
            registro_0150.BAIRRO = resposta_participante.district
            lista.append(registro_0150)

        return lista

    def query_registro0190(self):
        query = """
                    select distinct
                        pu.name ,
                        substr(pu.name, 1,6)
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line as det
                            on d.id = det.invoice_id
                    inner join
                        product_uom pu
                            on pu.id = det.uom_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        d.date_invoice between '%s' and '%s'
                        and ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid', 'cancel')
                        and det.uom_id is not null
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        for id in query_resposta:
            resposta = self.env['product.uom'].search([('name','=',id[0])],limit=1)
            if resposta:
                registro_0190 = registros.Registro0190()
                if resposta.name.find('-') != -1:
                    unidade = resposta.name[:resposta.name.find('-')]
                else:
                    unidade = resposta.name
                registro_0190.UNID = unidade[:6]
                registro_0190.DESCR = resposta.description
                lista.append(registro_0190)
        return lista

    def query_registro0200(self):
        query = """
                    select distinct
                        det.product_id
                    from
                        account_invoice as d
                    left join
                        account_invoice_line as det
                            on d.id = det.invoice_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        d.date_invoice between '%s' and '%s'
                        and ((d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                """ % (self.date_start, self.date_end)

        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        #hash = {}
        lista = []
        cont = 1
        for resposta in query_resposta:
            resposta_produto = self.env['product.product'].browse(resposta[0])
            if not resposta_produto:
                continue
            #if not (resposta_produto.codigo_unico in hash):
            registro_0200 = registros.Registro0200()
            registro_0200.COD_ITEM = resposta_produto.default_code
            registro_0200.DESCR_ITEM = resposta_produto.name
            registro_0200.COD_BARRA = resposta_produto.barcode
            if resposta_produto.uom_id.name.find('-') != -1:
                unidade = resposta_produto.uom_id.name[:resposta_produto.uom_id.name.find('-')]
            else:
                unidade = resposta_produto.uom_id.name
            registro_0200.UNID_INV = unidade[:6]
            registro_0200.TIPO_ITEM = resposta_produto.type_product
            registro_0200.COD_NCM = self.limpa_formatacao(resposta_produto.fiscal_classification_id.code)

            lista.append(registro_0200)

        return lista

    def query_registro0400(self):
        query = """
                    select distinct
                        d.fiscal_position_id
                    from
                        account_invoice as d
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        d.date_invoice between '%s' and '%s'
                        and ((d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        for resposta in query_resposta:
            resposta_nat = self.env['account.fiscal.position'].browse(resposta[0])
            registro_0400 = registros.Registro0400()
            registro_0400.COD_NAT = str(resposta_nat.id)
            registro_0400.DESCR_NAT = resposta_nat.natureza_operacao
            lista.append(registro_0400)
        return lista

    def transforma_valor(self, valor):
        valor = ("%.2f" % (float(valor)))
        return str(valor).replace('.', ',')

    def query_registroC100(self, fatura):
        lista = []
        resposta = self.env['account.invoice'].browse(self.fatura)
        resposta_nfe = self.env['invoice.eletronic'].search([('invoice_id','=',self.fatura)])
        if (resposta.product_document_id or resposta.state in ['open','paid']) and \
            (resposta.nfe_modelo or resposta.product_document_id.code == '55'):
            # removendo Emissao de Terceiros canceladas
            #if not resposta.product_document_id and resposta.state == 'cancel':
            #    continue
            #if not resposta.nfe_modelo and resposta.product_document_id.code == '55':
            #    if not resposta_nfe:
            #        continue
            if resposta:
                cancel = False
                registro_c100 = registros.RegistroC100()
                if resposta.fiscal_position_id.fiscal_type == 'entrada':
                    registro_c100.IND_OPER = '0'
                else:
                    registro_c100.IND_OPER = '1'
                if resposta.product_document_id:
                    registro_c100.IND_EMIT = '0'
                else:
                    registro_c100.IND_EMIT = '1'
                #import pudb;pu.db
                registro_c100.COD_MOD = (resposta.nfe_modelo or resposta.product_document_id.code).zfill(2)
                if not resposta_nfe:
                    registro_c100.COD_SIT = '00'
                elif resposta_nfe.state == 'done':
                    registro_c100.COD_SIT = '00'
                elif resposta_nfe.state == 'cancel':
                    registro_c100.COD_SIT = '02'
                    cancel = True
                if resposta.nfe_serie:
                    registro_c100.SER = resposta.nfe_serie[:3]
                else:
                    registro_c100.SER = resposta.product_serie_id.code
                if resposta.nfe_chave:
                    if len(resposta.nfe_chave) != 44:
                        msg_err = 'Tamanho da Chave NFe invalida - Fatura %s.' %(str(resposta.number or resposta.id))
                        #raise UserError(msg_err)
                        self.log_faturamento += msg_err
                if resposta_nfe.chave_nfe:
                    if len(resposta_nfe.chave_nfe) != 44:
                        msg_err = 'Tamanho da Chave NFe invalida - Fatura %s.
' %(str(resposta.number or resposta.id))
                        #raise UserError(msg_err)
                        self.log_faturamento += msg_err
                registro_c100.CHV_NFE = resposta.nfe_chave or resposta_nfe.chave_nfe
                registro_c100.NUM_DOC = self.limpa_formatacao(str(resposta.nfe_num or resposta_nfe.numero))
                if not cancel:
                    try:
                        registro_c100.DT_DOC  = self.transforma_data(resposta.nfe_emissao or
                            resposta_nfe.data_emissao or resposta.date_invoice)
                        registro_c100.DT_E_S  = self.transforma_data(resposta.nfe_emissao or
                            resposta_nfe.data_fatura  or resposta.date_invoice)
                    except:
                        msg_err = 'Data Emissao Fatura %s , invalida.
' %(str(resposta.number or resposta.id))
                        #raise UserError(msg_err)
                        self.log_faturamento += msg_err

                    if resposta.receivable_move_line_ids:
                        if len(resposta.receivable_move_line_ids) == 1:
                            if resposta.receivable_move_line_ids.date_maturity == resposta.date_invoice:
                                registro_c100.IND_PGTO = '0'
                            else:
                                registro_c100.IND_PGTO = '1'
                        else:
                            registro_c100.IND_PGTO = '1'
                    else:
                        registro_c100.IND_PGTO = '2'

                    registro_c100.VL_MERC = self.transforma_valor(resposta.amount_total)
                    registro_c100.IND_FRT = str(int(resposta.freight_responsibility))
                    registro_c100.VL_FRT = self.transforma_valor(resposta.total_frete)
                    registro_c100.VL_SEG = self.transforma_valor(resposta.total_seguro)
                    desp = 0.0
                    if resposta.total_despesas > 0.0:
                        desp = resposta.total_despesas
                        registro_c100.VL_OUT_DA = self.transforma_valor(resposta.total_despesas)
                    elif resposta.total_despesas < -0.01:
                        desp = resposta.total_despesas
                        registro_c100.VL_DESC = self.transforma_valor(resposta.total_despesas*(-1))
                    registro_c100.VL_DOC  = self.transforma_valor(resposta.total_bruto+desp)
                    registro_c100.VL_BC_ICMS = self.transforma_valor(resposta.icms_base)
                    registro_c100.VL_ICMS = self.transforma_valor(resposta.icms_value)
                    registro_c100.VL_BC_ICMS_ST = self.transforma_valor(resposta.icms_st_base)
                    registro_c100.VL_ICMS_ST = self.transforma_valor(resposta.icms_st_value)
                    registro_c100.VL_IPI = self.transforma_valor(resposta.ipi_value)
                    registro_c100.VL_PIS = self.transforma_valor(resposta.pis_value)
                    registro_c100.VL_COFINS = self.transforma_valor(resposta.cofins_value)
                    registro_c100.COD_PART = str(resposta.partner_id.id)
                    #registro_c100.VL_PIS_ST = 0,0
                    #registro_c100.VL_COFINS_ST = 0.0
                #else:
                #    x = resposta.id
                #    print (x)
                #    print ('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')

                lista.append(registro_c100)

        return lista

    def query_registroC101(self, fatura):
        query = """
                    select
                        sum(d.valor_icms_uf_remet) as icms_uf_remet,
                        sum(d.valor_icms_uf_dest) as icms_uf_dest,
                        sum(d.valor_icms_fcp_uf_dest) as fcp_uf_dest,
                        fp.fiscal_type
                    from
                        account_invoice as d
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    where
                        d.id = '%s'
                        and ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((d.valor_icms_uf_dest > 0) or
                        (d.valor_icms_uf_remet > 0))
                    group by fp.fiscal_type
                """ % (fatura)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        cont = 1
        for id in query_resposta:
            #resposta = self.env['account.invoice'].browse(id[0])
            registro_c101 = registros.RegistroC101()
            registro_c101.VL_FCP_UF_DEST = self.transforma_valor(id[2])
            if id[3] == 'entrada':
                registro_c101.VL_ICMS_UF_DEST = self.transforma_valor(id[0])
                registro_c101.VL_ICMS_UF_REM = self.transforma_valor(id[1])
            else:
                registro_c101.VL_ICMS_UF_DEST = self.transforma_valor(id[1])
                registro_c101.VL_ICMS_UF_REM = self.transforma_valor(id[0])

            lista.append(registro_c101)

        return lista

    def query_registroC170(self, fatura):
        #query = """
        """
                    select distinct
                        d.id
                    from
                        account_invoice as d
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        d.id = '%s'
                        and ((fd.code='55') or (d.nfe_modelo = '55'))
                        and d.state in ('open','paid', 'cancel')
                        and d.fiscal_position_id is not null
        #
        #        """ % (fatura)
        #self._cr.execute(query)
        #query_resposta = self._cr.fetchall()
        lista = []
        #cont = 1
        #for id in query_resposta:
        resposta = self.env['account.invoice'].browse(self.fatura)
        r_nfe = self.env['invoice.eletronic'].search([('invoice_id','=',self.fatura)])
        if resposta.fiscal_position_id.fiscal_type == 'entrada' and not r_nfe:
            n_item = 1
            for item in resposta.invoice_line_ids:
                registro_c170 = registros.RegistroC170()
                registro_c170.NUM_ITEM = str(n_item)
                #if item.product_id.default_code == '02805':
                #    import pudb;pu.db
                registro_c170.COD_ITEM = item.product_id.default_code
                registro_c170.DESCR_COMPL = item.name.strip()
                registro_c170.QTD = self.transforma_valor(item.quantity)
                if item.uom_id.name.find('-') != -1:
                    unidade = item.uom_id.name[:item.uom_id.name.find('-')]
                else:
                    unidade = item.uom_id.name
                registro_c170.UNID = unidade[:6]
                #registro_c170.VL_ITEM = self.transforma_valor(item.price_subtotal+item.outras_despesas)

                desc = 0.0
                if item.outras_despesas < -0.01:
                    desc = item.outras_despesas*(-1)
                if item.valor_desconto:
                    desc = desc + item.valor_desconto
                registro_c170.VL_DESC = self.transforma_valor(float(desc))
                registro_c170.VL_ITEM = self.transforma_valor(item.valor_bruto-desc)
                if item.cfop_id.code in ['5922', '6922']:
                    registro_c170.IND_MOV = '1'
                else:
                    registro_c170.IND_MOV = '0'
                try:
                    registro_c170.CST_ICMS = item.product_id.origin + item.icms_cst_normal
                except:
                    msg_err = 'Sem CST na Fatura %s.
' %(str(resposta.number or resposta.id))
                    #raise UserError(msg_err)
                    self.log_faturamento += msg_err

                registro_c170.CFOP = item.cfop_id.code
                registro_c170.COD_NAT = str(resposta.fiscal_position_id.id)
                registro_c170.VL_BC_ICMS = self.transforma_valor(item.icms_base_calculo)
                if item.tax_icms_id:
                    registro_c170.ALIQ_ICMS = self.transforma_valor(item.tax_icms_id.amount)
                else:
                    registro_c170.ALIQ_ICMS = '0'
                registro_c170.VL_ICMS = self.transforma_valor(item.icms_valor)
                registro_c170.VL_BC_ICMS_ST = self.transforma_valor(item.icms_st_base_calculo)
                if item.tax_icms_st_id:
                    registro_c170.ALIQ_ST = self.transforma_valor(item.tax_icms_st_id.amount)
                registro_c170.VL_ICMS_ST = self.transforma_valor(item.icms_st_valor)
                # TODO incluir na empresa o IND_APUR
                registro_c170.IND_APUR = '0'
                registro_c170.CST_IPI = item.ipi_cst
                #
                # TODO adicionar na Fatura de entrada a opcao de informar o Cod. Enquadramento
                #
                #if item.fiscal_classification_id.codigo_enquadramento != '999':
                #    registro_c170.COD_ENQ = item.fiscal_classification_id.codigo_enquadramento
                #item_eletronic = self.env['invoice.eletronic.item'].search([('account_invoice_line_id','=', item.id)])
                #if item_eletronic:
                #    import pudb;pu.db
                #registro_c170.COD_ENQ = item_eletronic.codigo_enquadramento_ipi
                registro_c170.VL_BC_IPI = self.transforma_valor(item.ipi_base_calculo)
                if item.tax_ipi_id:
                    registro_c170.ALIQ_IPI = self.transforma_valor(item.tax_ipi_id.amount)
                registro_c170.VL_IPI = self.transforma_valor(item.ipi_valor)
                registro_c170.CST_PIS = item.pis_cst
                registro_c170.VL_BC_PIS = self.transforma_valor(item.pis_base_calculo)
                if item.tax_pis_id:
                    registro_c170.ALIQ_PIS = self.transforma_valor(item.tax_pis_id.amount)
                #registro_c170.QUANT_BC_PIS = self.transforma_valor(
                registro_c170.VL_PIS = self.transforma_valor(item.pis_valor)
                registro_c170.CST_COFINS = item.cofins_cst
                registro_c170.VL_BC_COFINS = self.transforma_valor(item.cofins_base_calculo)
                if item.tax_cofins_id:
                    registro_c170.ALIQ_COFINS = self.transforma_valor(item.tax_pis_id.amount)
                #registro_c170.QUANT_BC_COFINS = self.transforma_valor(
                registro_c170.VL_COFINS = self.transforma_valor(item.cofins_valor)
                #registro_c170.COD_CTA =
                n_item += 1

                lista.append(registro_c170)

        return lista

    def query_registroC190(self, fatura):
        query = """
                select distinct
                        pt.origin || dl.icms_cst_normal,
                        cfop.code,
                        COALESCE(at.amount, 0.0) as ALIQUOTA ,
                        sum(dl.price_subtotal+dl.outras_despesas) as VL_OPR,
                        sum(dl.icms_base_calculo) as VL_BC_ICMS,
                        sum(dl.icms_valor) as VL_ICMS,
                        sum(dl.icms_st_base_calculo) as VL_BC_ICMS_ST,
                        sum(dl.icms_st_valor) as VL_ICMS_ST,
                        case when (dl.icms_aliquota_reducao_base > 0) then
                          sum((dl.price_subtotal+dl.outras_despesas)-dl.icms_base_calculo) else 0 end as VL_RED_BC,
                        sum(dl.ipi_valor) as VL_IPI
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        invoice_eletronic il
                            on il.invoice_id = d.id
                    left join
                        account_tax at
                            on at.id = dl.tax_icms_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    inner join
                        product_product pp
                            on pp.id = dl.product_id
                    inner join
                        product_template pt
                            on pt.id = pp.product_tmpl_id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((il.state is null) or (il.state = 'done'))
                        and d.id = '%s'
                    group by
                        dl.icms_aliquota_reducao_base,
                        dl.icms_cst_normal,
                        cfop.code,
                        at.amount,
                        pt.origin
                    order by 1,2,3
                """ % (fatura)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        cont = 1
        for id in query_resposta:
            registro_c190 = registros.RegistroC190()
            registro_c190.CST_ICMS = id[0]
            registro_c190.CFOP = id[1]
            registro_c190.ALIQ_ICMS = self.transforma_valor(id[2])
            registro_c190.VL_OPR = self.transforma_valor(id[3])
            registro_c190.VL_BC_ICMS = self.transforma_valor(id[4])
            registro_c190.VL_ICMS = self.transforma_valor(id[5])
            registro_c190.VL_BC_ICMS_ST = self.transforma_valor(id[6])
            registro_c190.VL_ICMS_ST = self.transforma_valor(id[7])
            registro_c190.VL_RED_BC = self.transforma_valor(id[8])
            registro_c190.VL_IPI = self.transforma_valor(id[9])

            lista.append(registro_c190)

        return lista

    def query_registroE110(self):
        query = """
                select
                    sum(dl.icms_valor) as VL_ICMS
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    left join
                        invoice_eletronic as ie
                            on ie.invoice_id = d.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((ie.state is null) or (ie.state = 'done'))
                        and ((substr(cfop.code, 1,1) in ('5','6','7')) or (cfop.code = '1605'))
                        and d.date_invoice between '%s' and '%s'
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        registro_E110 = RegistroE110()
        sld_transp = 0.0
        for id in query_resposta:
            registro_E110.VL_TOT_DEBITOS = self.transforma_valor(id[0])
            sld_transp = id[0]

        query = """
                select
                    sum(dl.icms_valor) as VL_ICMS
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    left join
                        invoice_eletronic as ie
                            on ie.invoice_id = d.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((ie.state is null) or (ie.state = 'done'))
                        and (((substr(cfop.code, 1,1) in ('1','2','3')) and cfop.code not in ('1605')) or (cfop.code = '5605'))
                        and d.date_invoice between '%s' and '%s'
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        for id in query_resposta:
            registro_E110.VL_TOT_CREDITOS = self.transforma_valor(id[0])
            sld_transp -= id[0]
        if sld_transp < 0.0:
            sld_transp = 0.0

        registro_E110.VL_AJ_DEBITOS = '0'
        registro_E110.VL_TOT_AJ_DEBITOS = '0'
        registro_E110.VL_ESTORNOS_CRED = '0'
        registro_E110.VL_AJ_CREDITOS = '0'
        registro_E110.VL_TOT_AJ_CREDITOS = '0'
        registro_E110.VL_ESTORNOS_DEB = '0'
        registro_E110.VL_SLD_CREDOR_ANT = '0'
        registro_E110.VL_SLD_APURADO = '0'
        registro_E110.VL_TOT_DED = '0'
        registro_E110.VL_ICMS_RECOLHER = '0'
        registro_E110.VL_SLD_CREDOR_TRANSPORTAR = self.transforma_valor(sld_transp)
        registro_E110.DEB_ESP = '0'

        lista.append(registro_E110)
        return lista

    def query_registroE300(self):
        query = """
                    select distinct
                        rs.code, rp.state_id
                    from
                        account_invoice as d
                    inner join
                        res_partner as rp
                            on rp.id = d.partner_id
                    inner join
                        res_country_state as rs
                            on rs.id = rp.state_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((d.valor_icms_uf_dest > 0) or
                        (d.valor_icms_uf_remet > 0))
                        and d.date_invoice between '%s' and '%s'
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        uf_emitente = ''
        for id in query_resposta:
            if id[0] == self.company_id.state_id.code:
                uf_emitente = self.company_id.state_id.code
            registro_e300 = registros.RegistroE300()
            registro_e300.UF = self.limpa_formatacao(id[0])
            registro_e300.DT_INI = self.transforma_data(self.date_start)
            registro_e300.DT_FIM = self.transforma_data(self.date_end)
            lista.append(registro_e300)
        if not uf_emitente and query_resposta:
            registro_e300 = registros.RegistroE300()
            registro_e300.UF = self.limpa_formatacao(self.company_id.state_id.code)
            registro_e300.DT_INI = self.transforma_data(self.date_start)
            registro_e300.DT_FIM = self.transforma_data(self.date_end)
            lista.append(registro_e300)

        return lista

    def query_registroE310(self, uf_informante, uf_dif):
        query = """
                    select
                        sum(d.valor_icms_uf_remet) as icms_uf_remet,
                        sum(d.valor_icms_uf_dest) as icms_uf_dest,
                        sum(d.valor_icms_fcp_uf_dest) as fcp_uf_dest,
                        fp.fiscal_type
                    from
                        account_invoice as d
                    inner join
                        res_partner as rp
                            on rp.id = d.partner_id
                    inner join
                        res_country_state as rs
                            on rs.id = rp.state_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((d.valor_icms_uf_dest > 0) or
                        (d.valor_icms_uf_remet > 0))
                        and rs.code = '%s'
                        and d.date_invoice between '%s' and '%s'
                    group by fp.fiscal_type
                """ % (uf_informante, self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        vl_tot_creditos = 0.0
        vl_tot_debitos = 0.0
        tot_fcp_deb = 0.0
        tot_fcp_cred = 0.0
        for id in query_resposta:
            if id[3] == 'entrada':
                tot_fcp_cred += id[2]
            else:
                tot_fcp_deb += id[2]
            if id[1]:
                vl_tot_creditos += id[1]
            if id[0]:
                vl_tot_debitos += id[0]
        #cred_ent = vl_tot_creditos
        query = """
                    select
                        sum(d.valor_icms_uf_remet) as icms_uf_remet,
                        sum(d.valor_icms_uf_dest) as icms_uf_dest,
                        sum(d.valor_icms_fcp_uf_dest) as fcp_uf_dest,
                        fp.fiscal_type
                    from
                        account_invoice as d
                    inner join
                        res_partner as rp
                            on rp.id = d.partner_id
                    inner join
                        res_country_state as rs
                            on rs.id = rp.state_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and ((d.valor_icms_uf_dest > 0) or
                        (d.valor_icms_uf_remet > 0))
                        and rs.code = '%s'
                        and d.date_invoice between '%s' and '%s'
                    group by fp.fiscal_type
                """ % (uf_dif, self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        tot_deb_uf_inf = 0.0
        for id in query_resposta:
            if id[3] == 'entrada':
                tot_fcp_cred += id[2]
            else:
                tot_fcp_deb += id[2]
            if id[0]:
                vl_tot_creditos += id[0]
            if id[1]:
                tot_deb_uf_inf += id[1]
                vl_tot_debitos += id[1]
        #cred_ent += vl_tot_creditos
        tot_fcp_deb = tot_fcp_deb + tot_fcp_cred
        tot_fcp_cred = 0.0
        lista = []
        registro_e310 = registros.RegistroE310()
        sld_ant_difal = vl_tot_debitos - self.vl_sld_cred_ant_difal - vl_tot_creditos
        if sld_ant_difal < 0.0:
            sld_ant_difal = 0.0
        sld_transp_difal = self.vl_sld_cred_ant_difal - vl_tot_debitos + vl_tot_creditos
        if sld_transp_difal < 0.0:
            sld_transp_difal = 0.0

        if not query_resposta:
            registro_e310.IND_MOV_FCP_DIFAL = '0'
            registro_e310.VL_SLD_CRED_ANT_DIFAL = '0'
            registro_e310.VL_TOT_DEBITOS_DIFAL = '0'
            registro_e310.VL_OUT_DEB_DIFAL = '0'
            registro_e310.VL_TOT_DEB_FCP = self.transforma_valor(tot_fcp_deb)
            registro_e310.VL_TOT_CREDITOS_DIFAL = '0'
            registro_e310.VL_TOT_CRED_FCP = self.transforma_valor(tot_fcp_cred)
            registro_e310.VL_OUT_CRED_DIFAL = '0'
            registro_e310.VL_SLD_DEV_ANT_DIFAL = '0'
            registro_e310.VL_DEDUCOES_DIFAL = '0'
            registro_e310.VL_RECOL_DIFAL = '0'
            registro_e310.VL_SLD_CRED_TRANSPORTAR_DIFAL = self.transforma_valor(sld_transp_difal)
            registro_e310.DEB_ESP_DIFAL = '0'
            registro_e310.VL_SLD_CRED_ANT_FCP = '0'
            registro_e310.VL_OUT_DEB_FCP = '0'
            registro_e310.VL_TOT_CRED_FCP = '0'
            registro_e310.VL_OUT_CRED_FCP = '0'
            registro_e310.VL_SLD_DEV_ANT_FCP = '0'
            registro_e310.VL_DEDUCOES_FCP = '0'
            registro_e310.VL_RECOL_FCP = '0'
            registro_e310.VL_SLD_CRED_TRANSPORTAR_FCP = '0'
            registro_e310.DEB_ESP_FCP = '0'
        else:
            registro_e310.IND_MOV_FCP_DIFAL = '1'
            registro_e310.VL_SLD_CRED_ANT_DIFAL = self.transforma_valor(self.vl_sld_cred_ant_difal)
            registro_e310.VL_TOT_DEBITOS_DIFAL = self.transforma_valor(vl_tot_debitos)
            registro_e310.VL_OUT_DEB_DIFAL = '0'
            registro_e310.VL_TOT_DEB_FCP = self.transforma_valor(tot_fcp_deb)
            registro_e310.VL_TOT_CREDITOS_DIFAL = self.transforma_valor(vl_tot_creditos)
            registro_e310.VL_TOT_CRED_FCP = self.transforma_valor(tot_fcp_cred)
            registro_e310.VL_OUT_CRED_DIFAL = '0'
            registro_e310.VL_SLD_DEV_ANT_DIFAL = self.transforma_valor(sld_ant_difal)
            registro_e310.VL_DEDUCOES_DIFAL = '0'
            registro_e310.VL_RECOL_DIFAL = '0'
            registro_e310.VL_SLD_CRED_TRANSPORTAR_DIFAL = self.transforma_valor(sld_transp_difal)
            registro_e310.DEB_ESP_DIFAL = '0'
            registro_e310.VL_SLD_CRED_ANT_FCP = '0'
            registro_e310.VL_OUT_DEB_FCP = '0'
            registro_e310.VL_TOT_CRED_FCP = '0'
            registro_e310.VL_OUT_CRED_FCP = '0'
            registro_e310.VL_SLD_DEV_ANT_FCP = '0'
            registro_e310.VL_DEDUCOES_FCP = '0'
            registro_e310.VL_RECOL_FCP = '0'
            registro_e310.VL_SLD_CRED_TRANSPORTAR_FCP = '0'
            registro_e310.DEB_ESP_FCP = '0'

        lista.append(registro_e310)

        return lista

    def query_registroE510(self):
        query = """
                select distinct
                        dl.ipi_cst,
                        cfop.code,
                        sum(dl.ipi_base_calculo) as VL_BC_IPI,
                        sum(dl.ipi_valor) as VL_IPI
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        account_tax at
                            on at.id = dl.tax_icms_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and d.date_invoice between '%s' and '%s'
                    group by dl.ipi_cst,
                        cfop.code
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        cont = 1
        for id in query_resposta:
            registro_E510 = RegistroE510()
            registro_E510.CFOP = str(id[1])
            registro_E510.CST_IPI = str(id[0])
            registro_E510.VL_CONT_IPI = '0'
            registro_E510.VL_BC_IPI = self.transforma_valor(id[2])
            registro_E510.VL_IPI = self.transforma_valor(id[3])
            lista.append(registro_E510)
        return lista

    def query_registroE520(self):
        query = """
                select
                       sum(dl.ipi_valor) as VL_IPI
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        account_tax at
                            on at.id = dl.tax_icms_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and substr(cfop.code, 1,1) in ('5','6')
                        and d.date_invoice between '%s' and '%s'
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        lista = []
        registro_E520 = RegistroE520()
        for id in query_resposta:
            registro_E520.VL_DEB_IPI = self.transforma_valor(id[0])
        registro_E520.VL_SD_ANT_IPI = '0'
        registro_E520.VL_OD_IPI = '0'
        registro_E520.VL_OC_IPI = '0'
        registro_E520.VL_SC_IPI = '0'
        registro_E520.VL_SD_IPI = '0'

        query = """
                select
                       sum(dl.ipi_valor) as VL_IPI
                    from
                        account_invoice as d
                    inner join
                        account_invoice_line dl
                            on dl.invoice_id = d.id
                    left join
                        account_tax at
                            on at.id = dl.tax_icms_id
                    left join
                        br_account_fiscal_document fd
                            on fd.id = d.product_document_id
                    inner join
                        account_fiscal_position fp
                            on d.fiscal_position_id = fp.id
                    inner join
                        br_account_cfop cfop
                            on dl.cfop_id = cfop.id
                    where
                        ((fd.code='55') or (d.nfe_modelo = '55') or (d.nfe_modelo = '1'))
                        and d.state in ('open','paid')
                        and d.fiscal_position_id is not null
                        and substr(cfop.code, 1,1) in ('1','2','3')
                        and d.date_invoice between '%s' and '%s'
                """ % (self.date_start, self.date_end)
        self._cr.execute(query)
        query_resposta = self._cr.fetchall()
        for id in query_resposta:
            registro_E520.VL_CRED_IPI = self.transforma_valor(id[0])
        lista.append(registro_E520)
        return lista


import pandas as pd
import numpy as np

import visualise_data as viz
import matplotlib.pyplot as plt

            ############################
            ###   PRICE ELASTICITY   ###
            ############################

def top_n_products_corr_matrix(df, col, n, include_own_price=False, show_heatmap=False, bound=0.4):
    """

    Parameters
    ---------
    df : pos transaction data
    col : columns relevant for correlation matrix.
    include_own_price : whether to include the product's own price into the best and worst correlations.
    heatmap : whether to save heatmap as png in the output folder.
    bound : Upper and lower bound to restrict correlation values for heatmap visualisation

    Returns
    -------
    3 dictionaries with 1) { product : [correlated products] }
                        2) { product : [correlated product indexes from the correlation matrix] }
                        3) { product : [correlated values] }

    """
    matrix = transform_df(df, col)
    corr_matrix = correlation_matrix(matrix)

    elasticity_dictionary = product_cross_elasticity(corr_matrix, n, include_own_price, bound)

    # If show is False, the heatmap is saved to the output directory.
    corr_matrix_heatmap = corr_matrix.applymap(lambda x: set_to_zero(x, bound))
    viz.plot_correlation_heatmap(corr_matrix_heatmap, text=None, cmap_colour=plt.cm.plasma, show=show_heatmap)

    return elasticity_dictionary

def transform_df(df, col):
    """

    Extracts the relevant columns and pivots the DataFrame to be suitable for correlation.

    Parameters
    ----------
    df : pos transaction data
    col : columns array required for correlation matrix. ["UPC", "WEEK_END_DATE,
                                                          "UNITS", "PRICE"]

    Returns
    -------
    price-quantity matrix that contains all the raw price-sales data.

    """
    if ("UPC" and "WEEK_END_DATE") not in col:
        raise ValueError("Relevant columns not in col parameter.")

    relevant_columns_df = df[col].groupby(["UPC", "WEEK_END_DATE"]).mean().reset_index()

    quantity_matrix = relevant_columns_df.pivot(index="WEEK_END_DATE",
                                                columns="UPC",
                                                values="UNITS"
                                               )
    price_matrix    = relevant_columns_df.pivot(index="WEEK_END_DATE",
                                                columns="UPC",
                                                values="PRICE"
                                               )

    p_q_matrix = pd.concat([quantity_matrix, price_matrix], axis=1)

    return p_q_matrix

def correlation_matrix(price_quantity_matrix):
    """ Returns the correlation matrix for quantity-price (row-column) elasticity."""

    # Produce a correlation matrix with both price-price, price-quantity and quantity-quantity
    # correlation.
    correlation_matrix = price_quantity_matrix.corr()

    n = int((correlation_matrix.shape[0])/2)

    if correlation_matrix.shape[1] != (n*2):
        print("n*2 = {}, shape[1] = {}".format(n*2, correlation_matrix.shape[1]))
        raise ValueError("Correlation matrix is incorrect.")

    # Return only the quantity-price correlation
    return correlation_matrix.iloc[0:n,  n:110]

def col_idx_to_name_dict(key, value):
    """ Generates a {column_index : upc} dictionary to be used for retrieving correlated UPCs."""
#     col_idx = np.arange(len(key))
    return dict(zip(key, value))

def map_idx_to_upc(corr_idx, idx_upc_dict):
    """

    Mapping the indexes to corresponding UPCs.

    Parameters
    ----------

    corr_idx : Pandas series with UPC as index, storing an array of all column indexes that
               returned the highest positive and negative correlation.
    idx_upc_dict : stores {column_index : upc} dictionary generated from the
                   correlation matrix, used for mapping the indexes back to the UPCs.

    Returns
    -------
    Array of column indexes

    """

    return [idx_upc_dict[idx] for idx in corr_idx] # actually slower than just a for loop...

def product_cross_elasticity(matrix, n, include_own_price, bound):
    """

    Parameters
    ----------
    matrix: correlation matrix of products (rows) x prices (columns)
    n : the top 'n' positive and negative correlations
    include_own_price : whether to include the product's own price into the best and worst correlations.

    Returns
    -------
    Dictionary containing each UPC and corresponding UPCs of the 'n' most
    positive and negative correlations.

    """

    col_idx = np.arange(0, len(matrix))
    idx_upc_dict = col_idx_to_name_dict(col_idx, matrix.columns)

    row_length = matrix.shape[0]

    if include_own_price:
        corr_values = matrix.apply(top_pos_neg_corr, args=(n, row_length, "corr_value"), axis=1)
        corr_idx = matrix.apply(top_pos_neg_corr, args=(n, row_length, "upc_idx"), axis=1)
    else:
        for i in range(0, matrix.shape[0]):
            matrix.iloc[i, i] = 0

        corr_values = matrix.apply(top_pos_neg_corr, args=(n, row_length, "corr_value", bound), axis=1)
        corr_idx = matrix.apply(top_pos_neg_corr, args=(n, row_length, "upc_idx", bound), axis=1)

    corr_upc = corr_idx.apply(map_idx_to_upc, args=(idx_upc_dict, ))

    return [corr_upc.to_dict(), corr_idx.to_dict(), corr_values.to_dict()]

def top_pos_neg_corr(row, n, row_length, return_value, bound):
    """

    Return top and bottom 'n' correlations given a row of the correlation matrix

    Parameters
    ----------
    row : single row of the correlation matrix
    n : 'n' most positive and negatively correlated UPCs
    row_length : length of the row, used for creating a dictionary to map the column index
    return_value : ["upc_idx", "corr_value"] "upc_idx" returns the column idx of the most
                   correlated UPCs, which can be used to extract the UPCs later. "corr_value" returns the
                   correlation values for each UPC.
    bound : the minimum correlation (positive/negative) a cross-product must have to be included.

    Returns
    -------
    "upc_idx" returns the column idx of the most correlated UPCs, which can be used to
    extract the UPCs later. "corr_value" returns the correlation values for each UPC.

    """

    row_idx = np.arange(0, row_length)
    col_dict_for_each_row = col_idx_to_name_dict(row, row_idx)
#     print(col_dict_for_each_row)

    row_sorted = sorted(row)

    n_neg = (n)*(-1)
    most_correlated = row_sorted[0:n] + row_sorted[n_neg:]
#     print("\n\nmost_correlated: {}",format(most_correlated))

    most_correlated = [x for x in most_correlated if x > bound or x < (-1)*bound]

    if return_value == "corr_value":
        return most_correlated
    elif return_value == "upc_idx":
        return [col_dict_for_each_row[row_value] for row_value in most_correlated]

def set_to_zero(x, bound=0.4):
    """ Set all values less than the +- of the bound to zero."""
    if x < bound and x > -1 * bound:
        x = 0
        return x
    else:
        return x

if __name__ == "__main__":
    print("Running price_elasticity.py as main function")


from algorithms_knn import *
from algorithm_dbscan import *
from algorithm_svm import *
import matplotlib.pyplot as plt

mean = [0, 0]
cov = [[0.5, 0], [0.5, 1]]
mean2 = [2, 2]
cov2 = [[0.5, 0], [0, 0.6]]
mean3 = [-2, -2]
num_maj = 1000
num_min = 100
num_train = 800
X1 = np.random.multivariate_normal(mean, cov, num_maj)
X2 = np.random.multivariate_normal(mean2, cov2, num_min // 2)
X3 = np.random.multivariate_normal(mean3, cov2, num_min // 2)
Y1 = np.tile([0], num_maj)
Y2 = np.tile([1], num_min // 2)
Y3 = np.tile([1], num_min // 2)
X = np.concatenate([X1, X2, X3])
Y = np.concatenate([Y1, Y2, Y3])
#plt.scatter(X[:,0],X[:,1],c=Y)
#Y1=DBSCAN(eps=0.7,min_samples=5).fit_predict(X)
# method=SVMBasedOversample(alpha=0.05,n_steps=50)
method = DbscanBasedOversample(eps=0.7,min_pts=5)
X1, Y1 = method.fit_sample(X, Y)
plt.rcParams['figure.figsize'] = (8.0, 10.0)
fig = plt.figure()
ax1 = plt.subplot(2, 1, 1)
ax1.scatter(X[:, 0], X[:, 1], c=Y)
ax2 = plt.subplot(2, 1, 2)
ax2.scatter(X1[:, 0], X1[:, 1], c=Y1)
print(Counter(Y1))
plt.show()


import os
import motor
import functools
import tornado.gen

from pymongo import MongoClient

mongo_user  = os.getenv('MONGO_USER')
mongo_pass  = os.getenv('MONGO_PASSWORD')
mongo_host  = os.getenv('MONGO_HOST')
mongo_port  = os.getenv('MONGO_PORT')
mongo_db    = os.getenv('MONGO_DB')
mongo_limit = int(os.getenv('MONGO_LIMIT'))

mongo_uri = "mongodb://%s:%s@%s:%s/%s" % (mongo_user, mongo_pass,
        mongo_host, mongo_port, mongo_db)

def mongo_aync():
    return motor.MotorClient(host=mongo_uri).open_sync()[mongo_db]

def mongo_sync():
    return MongoClient(mongo_uri)[mongo_db]

class MongoQuery:
    def __init__(self, model=None, model_functions=None):
        self.collection = mongo_aync()[model.get_collection()]
        self.model = model
        self.model_functions = model_functions

    @tornado.gen.engine
    def execute(self, args, page=0, limit=mongo_limit, callback=None):

        arguments = self.build_mongo_query(args)
        cursor = self.collection.find(arguments)
        if not 0 < limit < mongo_limit:
            limit = mongo_limit
        cursor.limit(limit).skip(limit * page)
        results = []
        # I don't like the generator model here ... It hides what
        # the code is doing which is an async callback on the ioloop...
        # But can't see a good way to handle it with current callback model
        while (yield cursor.fetch_next):
            results.append(cursor.next_object())
        response = [self.model.build_response_dict(result) for result in results]
        callback(response)

    def build_mongo_query(self, arguments):
        # slug is a list, each with (key, value)
        if [func for func in dir(self.model_functions) if not "__" in func]:
            slug = {}
            for key, value in arguments.iteritems():
                new_key, new_value = self.attr_func_wrap(key, value)
                if new_key in slug:
                    slug[new_key].update(new_value)
                else:
                    slug[new_key] = new_value
            return slug
        return arguments

    def attr_func_wrap(self, key, value):
        func = getattr(self.model_functions, key)
        key, value = func(value)
        return key, value


import logging
import psycopg2
import psycopg2.extras
import json
import gzip
from typing import List

from . import utils

LOGGER = logging.getLogger(__name__)

# pylint: disable=missing-function-docstring,no-self-use,too-many-arguments
class FastSyncTargetPostgres:
    """
    Common functions for fastsync to Postgres
    """

    EXTRACTED_AT_COLUMN = '_SDC_EXTRACTED_AT'
    BATCHED_AT_COLUMN = '_SDC_BATCHED_AT'
    DELETED_AT_COLUMN = '_SDC_DELETED_AT'

    def __init__(self, connection_config, transformation_config=None):
        self.connection_config = connection_config
        self.transformation_config = transformation_config

    def open_connection(self):
        conn_string = "host='{}' dbname='{}' user='{}' password='{}' port='{}'".format(
            self.connection_config['host'],
            self.connection_config['dbname'],
            self.connection_config['user'],
            self.connection_config['password'],
            self.connection_config['port']
        )

        if 'ssl' in self.connection_config and self.connection_config['ssl'] == 'true':
            conn_string += " sslmode='require'"

        return psycopg2.connect(conn_string)

    def query(self, query, params=None):
        LOGGER.info('Running query: %s', query)
        with self.open_connection() as connection:
            with connection.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
                cur.execute(query, params)

                if cur.rowcount > 0 and cur.description:
                    return cur.fetchall()

                return []

    def create_schema(self, schema):
        sql = 'CREATE SCHEMA IF NOT EXISTS {}'.format(schema)
        self.query(sql)

    def create_schemas(self, tables):
        schemas = utils.get_target_schemas(self.connection_config, tables)
        for schema in schemas:
            self.create_schema(schema)

    def drop_table(self, target_schema, table_name, is_temporary=False):
        table_dict = utils.tablename_to_dict(table_name)
        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')

        sql = 'DROP TABLE IF EXISTS {}."{}"'.format(target_schema, target_table.lower())
        self.query(sql)

    def create_table(self, target_schema: str, table_name: str, columns: List[str], primary_key: List[str],
                     is_temporary: bool = False, sort_columns=False):

        table_dict = utils.tablename_to_dict(table_name)
        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')

        # skip the EXTRACTED, BATCHED and DELETED columns in case they exist because they gonna be added later
        columns = [c for c in columns if not (c.startswith(self.EXTRACTED_AT_COLUMN) or
                                              c.startswith(self.BATCHED_AT_COLUMN) or
                                              c.startswith(self.DELETED_AT_COLUMN))]

        columns += [f'{self.EXTRACTED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
                    f'{self.BATCHED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
                    f'{self.DELETED_AT_COLUMN} CHARACTER VARYING']

        # We need the sort the columns for some taps( for now tap-s3-csv)
        # because later on when copying a csv file into Snowflake
        # the csv file columns need to be in the same order as the the target table that will be created below
        if sort_columns:
            columns.sort()

        sql_columns = ','.join(columns).lower()
        sql_primary_keys = ','.join(primary_key).lower() if primary_key else None
        sql = f'CREATE TABLE IF NOT EXISTS {target_schema}."{target_table.lower()}" (' \
              f'{sql_columns}' \
              f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'

        self.query(sql)

    def copy_to_table(self, filepath, target_schema: str, table_name: str, size_bytes: int,
                      is_temporary: bool = False, skip_csv_header: bool = False):
        LOGGER.info('Loading %s into Postgres...', filepath)
        table_dict = utils.tablename_to_dict(table_name)
        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')

        with self.open_connection() as connection:
            with connection.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
                inserts = 0

                copy_sql = f"""COPY {target_schema}."{target_table.lower()}"
                FROM STDIN WITH (FORMAT CSV, HEADER {'TRUE' if skip_csv_header else 'FALSE'}, ESCAPE '"')
                """

                with gzip.open(filepath, 'rb') as file:
                    cur.copy_expert(copy_sql, file)

                inserts = cur.rowcount
                LOGGER.info('Loading into %s."%s": %s',
                            target_schema,
                            target_table.lower(),
                            json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}))

    # grant_... functions are common functions called by utils.py: grant_privilege function
    # "to_group" is not used here but exists for compatibility reasons with other database types
    # "to_group" is for databases that can grant to users and groups separately like Amazon Redshift
    # pylint: disable=unused-argument
    def grant_select_on_table(self, target_schema, table_name, role, is_temporary, to_group=False):
        # Grant role is not mandatory parameter, do nothing if not specified
        if role:
            table_dict = utils.tablename_to_dict(table_name)
            target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
            sql = 'GRANT SELECT ON {}."{}" TO GROUP {}'.format(target_schema, target_table.lower(), role)
            self.query(sql)

    # pylint: disable=unused-argument
    def grant_usage_on_schema(self, target_schema, role, to_group=False):
        # Grant role is not mandatory parameter, do nothing if not specified
        if role:
            sql = 'GRANT USAGE ON SCHEMA {} TO GROUP {}'.format(target_schema, role)
            self.query(sql)

    # pylint: disable=unused-argument
    def grant_select_on_schema(self, target_schema, role, to_group=False):
        # Grant role is not mandatory parameter, do nothing if not specified
        if role:
            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO GROUP {}'.format(target_schema, role)
            self.query(sql)

    # pylint: disable=duplicate-string-formatting-argument
    def obfuscate_columns(self, target_schema: str, table_name: str, is_temporary: bool = False):
        LOGGER.info('Applying obfuscation rules')
        table_dict = utils.tablename_to_dict(table_name)
        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
        transformations = self.transformation_config.get('transformations', [])
        trans_cols = []

        # Find obfuscation rule for the current table
        for trans in transformations:
            # Input table_name is formatted as {{schema}}.{{table}}
            # Stream name in taps transformation.json is formatted as {{schema}}-{{table}}
            #
            # We need to convert to the same format to find the transformation
            # has that has to be applied
            tap_stream_name_by_table_name = '{}-{}'.format(table_dict.get('schema_name'), table_dict.get('table_name'))
            if trans.get('tap_stream_name') == tap_stream_name_by_table_name:
                column = trans.get('field_id')
                transform_type = trans.get('type')
                if transform_type == 'SET-NULL':
                    trans_cols.append('"{}" = NULL'.format(column))
                elif transform_type == 'HASH':
                    trans_cols.append('"{}" = ENCODE(DIGEST("{}", \'sha1\'), \'hex\')'.format(column, column))
                elif 'HASH-SKIP-FIRST' in transform_type:
                    skip_first_n = transform_type[-1]
                    trans_cols.append('"{}" = CONCAT(SUBSTRING("{}", 1, {}),'
                                      'ENCODE(DIGEST(SUBSTRING("{}", {} + 1), \'sha1\'), \'hex\'))'
                                      .format(column,
                                              column,
                                              skip_first_n,
                                              column,
                                              skip_first_n))
                elif transform_type == 'MASK-DATE':
                    trans_cols.append('"{}" = TO_CHAR("{}"::DATE, \'YYYY-01-01\')::DATE'.format(column, column))
                elif transform_type == 'MASK-NUMBER':
                    trans_cols.append('"{}" = 0'.format(column))

        # Generate and run UPDATE if at least one obfuscation rule found
        if len(trans_cols) > 0:
            sql = f"""UPDATE {target_schema}."{target_table.lower()}"
            SET {','.join(trans_cols)}
            """

            self.query(sql)

    def swap_tables(self, schema, table_name):
        table_dict = utils.tablename_to_dict(table_name)
        target_table = table_dict.get('table_name')
        temp_table = table_dict.get('temp_table_name')

        # Swap tables and drop the temp tamp
        self.query('DROP TABLE IF EXISTS {}."{}"'.format(schema, target_table.lower()))
        self.query('ALTER TABLE {}."{}" RENAME TO "{}"'.format(schema, temp_table.lower(), target_table.lower()))


from django.conf.urls import patterns, url, include
from django.http import HttpResponse
from django.templatetags.static import static
from django.views.generic.base import TemplateView, RedirectView

from brambling.forms.user import (
    FloppyAuthenticationForm,
    FloppyPasswordResetForm,
    FloppySetPasswordForm,
)
from brambling.forms.organizer import (
    OrganizationPermissionForm,
    OrganizationProfileForm,
    OrganizationEventDefaultsForm,
)
from brambling.models import Discount, Invite
from brambling.views.orders import (
    AddToOrderView,
    RemoveFromOrderView,
    ApplyDiscountView,
    ChooseItemsView,
    AttendeeBasicDataView,
    AttendeesView,
    AttendeeHousingView,
    SurveyDataView,
    HostingView,
    OrderEmailView,
    SummaryView,
    RactiveShopView,
)
from brambling.views.core import (
    ExceptionView,
    DashboardView,
    InviteAcceptView,
    InviteSendView,
    InviteDeleteView,
)
from brambling.views.mail import (
    ConfirmationPreviewView,
    OrderReceiptPreviewView,
    OrderAlertPreviewView,
    InvitePreviewView
)
from brambling.views.organizer import (
    OrganizationUpdateView,
    OrganizationPaymentView,
    OrganizationDetailView,
    OrganizationRemoveEditorView,
    OrderRedirectView,
    EventCreateView,
    EventSummaryView,
    EventUpdateView,
    StripeConnectView,
    EventRemoveEditorView,
    PublishEventView,
    UnpublishEventView,
    ItemListView,
    item_form,
    DiscountListView,
    discount_form,
    CustomFormListView,
    custom_form_form,
    AttendeeFilterView,
    OrderFilterView,
    OrganizerApplyDiscountView,
    RemoveDiscountView,
    OrderDetailView,
    RefundView,
    TogglePaymentConfirmationView,
    SendReceiptView,
    FinancesView,
)
from brambling.views.payment import (
    OrganizationDwollaConnectView,
    OrderDwollaConnectView,
    UserDwollaConnectView,
    DwollaWebhookView,
)
from brambling.views.user import (
    PersonView,
    HomeView,
    SignUpView,
    EmailConfirmView,
    send_confirmation_email_view,
    CreditCardAddView,
    CreditCardDeleteView,
)

order_urlpatterns = patterns('',
    url(r'^ractive-shop/$', RactiveShopView.as_view()),
    url(r'^shop/$',
        ChooseItemsView.as_view(),
        name="brambling_event_shop"),
    url(r'^add/(?P<pk>\d+)/$',
        AddToOrderView.as_view(),
        name="brambling_event_shop_add"),
    url(r'^remove/(?P<pk>\d+)/$',
        RemoveFromOrderView.as_view(),
        name="brambling_event_shop_remove"),

    url(r'^attendees/$',
        AttendeesView.as_view(),
        name="brambling_event_attendee_list"),
    url(r'^attendees/add/$',
        AttendeeBasicDataView.as_view(),
        name="brambling_event_attendee_add"),
    url(r'^attendees/(?P<pk>\d+)/$',
        AttendeeBasicDataView.as_view(),
        name="brambling_event_attendee_edit"),

    url(r'^housing_data/$',
        AttendeeHousingView.as_view(),
        name='brambling_event_attendee_housing'),
    url(r'^survey/$',
        SurveyDataView.as_view(),
        name='brambling_event_survey'),
    url(r'^hosting/$',
        HostingView.as_view(),
        name='brambling_event_hosting'),
    url(r'^email/$',
        OrderEmailView.as_view(),
        name="brambling_event_order_email"),
    url(r'^summary/$',
        SummaryView.as_view(),
        name="brambling_event_order_summary"),
    url(r'^use-discount/(?P<discount>{})/$'.format(Discount.CODE_REGEX),
        ApplyDiscountView.as_view(),
        name="brambling_event_use_discount"),
    url(r'^dwolla_connect/$',
        OrderDwollaConnectView.as_view(),
        name="brambling_order_dwolla_connect"),
)

event_urlpatterns = patterns('',
    url(r'^$',
        RedirectView.as_view(pattern_name="brambling_event_order_summary", permanent=False),
        name="brambling_event_root"),

    url(r'^order/(?:(?P<code>[a-zA-Z0-9]{8})/)?', include(order_urlpatterns)),
    url(r'^summary/$',
        EventSummaryView.as_view(),
        name="brambling_event_summary"),
    url(r'^edit/$',
        EventUpdateView.as_view(),
        name="brambling_event_update"),
    url(r'^remove_editor/(?P<pk>\d+)$',
        EventRemoveEditorView.as_view(),
        name="brambling_event_remove_editor"),
    url(r'^publish/$',
        PublishEventView.as_view(),
        name="brambling_event_publish"),
    url(r'^unpublish/$',
        UnpublishEventView.as_view(),
        name="brambling_event_unpublish"),
    url(r'^items/$',
        ItemListView.as_view(),
        name="brambling_item_list"),
    url(r'^items/create/$',
        item_form,
        name="brambling_item_create"),
    url(r'^items/(?P<pk>\d+)/$',
        item_form,
        name="brambling_item_update"),
    url(r'^attendees/$',
        AttendeeFilterView.as_view(),
        name="brambling_event_attendees"),
    url(r'^finances/$',
        FinancesView.as_view(),
        name="brambling_event_finances"),
    url(r'^orders/$',
        OrderFilterView.as_view(),
        name="brambling_event_orders"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/$',
        OrderDetailView.as_view(),
        name="brambling_event_order_detail"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/remove_discount/(?P<discount_pk>\d+)/$',
        RemoveDiscountView.as_view(),
        name="brambling_event_remove_discount"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/apply_discount/$',
        OrganizerApplyDiscountView.as_view(),
        name="brambling_event_apply_discount"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/refund/(?P<pk>\d+)/$',
        RefundView.as_view(),
        name="brambling_event_refund"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/confirm/(?P<payment_pk>\d+)/$',
        TogglePaymentConfirmationView.as_view(),
        name="brambling_event_toggle_payment_confirmation"),
    url(r'^orders/(?P<code>[a-zA-Z0-9]{8})/send_receipt/$',
        SendReceiptView.as_view(),
        name="brambling_event_send_receipt"),

    url(r'^discount/$',
        DiscountListView.as_view(),
        name="brambling_discount_list"),
    url(r'^discount/create/$',
        discount_form,
        name="brambling_discount_create"),
    url(r'^discount/(?P<pk>\d+)/$',
        discount_form,
        name="brambling_discount_update"),

    url(r'^forms/$',
        CustomFormListView.as_view(),
        name="brambling_form_list"),
    url(r'^forms/create/$',
        custom_form_form,
        name="brambling_form_create"),
    url(r'^forms/(?P<pk>\d+)/$',
        custom_form_form,
        name="brambling_form_update"),
)

organization_urlpatterns = patterns('',
    url(r'^$',
        OrganizationDetailView.as_view(),
        name='brambling_organization_detail'),
    url(r'^edit/profile/$',
        OrganizationUpdateView.as_view(
            form_class=OrganizationProfileForm,
            template_name='brambling/organization/profile.html',
        ),
        name='brambling_organization_update'),
    url(r'^edit/event_defaults/$',
        OrganizationUpdateView.as_view(
            form_class=OrganizationEventDefaultsForm,
            template_name='brambling/organization/event_defaults.html',
            success_view_name='brambling_organization_update_event_defaults',
        ),
        name='brambling_organization_update_event_defaults'),
    url(r'^edit/permissions/$',
        OrganizationUpdateView.as_view(
            form_class=OrganizationPermissionForm,
            template_name='brambling/organization/permissions.html',
            success_view_name='brambling_organization_update_permissions',
        ),
        name='brambling_organization_update_permissions'),
    url(r'^edit/payment/$',
        OrganizationPaymentView.as_view(),
        name='brambling_organization_update_payment'),
    url(r'^create/$',
        EventCreateView.as_view(),
        name="brambling_event_create"),
    url(r'^dwolla_connect/$',
        OrganizationDwollaConnectView.as_view(),
        name="brambling_organization_dwolla_connect"),
    url(r'^remove_editor/(?P<pk>\d+)$',
        OrganizationRemoveEditorView.as_view(),
        name="brambling_organization_remove_editor"),

    url(r'^order/(?:(?P<code>[a-zA-Z0-9]{8})/)?', OrderRedirectView.as_view()),

    url(r'^(?P<event_slug>[\w-]+)/', include(event_urlpatterns)),
)

urlpatterns = patterns('',
    url(r'^favicon\.ico$', RedirectView.as_view(url=static('brambling/favicon.ico'))),

    url(r'about/', TemplateView.as_view(template_name='brambling/about.html'), name='brambling_about'),
    url(r'faq/', TemplateView.as_view(template_name='brambling/faq.html'), name='brambling_faq'),
    url(r'pricing/', TemplateView.as_view(template_name='brambling/pricing.html'), name='brambling_pricing'),
    url(r'global/', TemplateView.as_view(template_name='brambling/global.html'), name='brambling_global'),

    url(r'^$',
        DashboardView.as_view(),
        name="brambling_dashboard"),
    url(r'^stripe_connect/$',
        StripeConnectView.as_view(),
        name="brambling_stripe_connect"),

    url(r'^login/$',
        'django.contrib.auth.views.login',
        {'authentication_form': FloppyAuthenticationForm},
        name='login'),
    url(r'^password_reset/$',
        'django.contrib.auth.views.password_reset',
        {'password_reset_form': FloppyPasswordResetForm},
        name='password_reset'),
    url(r'^reset/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
        'django.contrib.auth.views.password_reset_confirm',
        {'set_password_form': FloppySetPasswordForm},
        name='password_reset_confirm'),
    url(r'^', include('django.contrib.auth.urls')),
    url(r'^signup/$',
        SignUpView.as_view(),
        name="brambling_signup"),
    url(r'^email_confirm/send/$',
        send_confirmation_email_view,
        name="brambling_email_confirm_send"),
    url(r'^email_confirm/(?P<pkb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})$',
        EmailConfirmView.as_view(),
        name="brambling_email_confirm"),
    url(r'^invite/(?P<code>[a-zA-Z0-9~-]{20})/$',
        InviteAcceptView.as_view(),
        name="brambling_invite_accept"),
    url(r'^invite/(?P<code>[a-zA-Z0-9~-]{20})/send/$',
        InviteSendView.as_view(),
        name="brambling_invite_send"),
    url(r'^invite/(?P<code>[a-zA-Z0-9~-]{20})/delete/$',
        InviteDeleteView.as_view(),
        name="brambling_invite_delete"),

    url(r'^profile/$',
        PersonView.as_view(),
        name="brambling_user_profile"),
    url(r'^profile/card/add/(?P<api_type>test|live)/$',
        CreditCardAddView.as_view(),
        name="brambling_user_card_add"),
    url(r'^profile/card/delete/(?P<pk>\d+)/$',
        CreditCardDeleteView.as_view(),
        name="brambling_user_card_delete"),
    url(r'^profile/dwolla_connect/$',
        UserDwollaConnectView.as_view(),
        name="brambling_user_dwolla_connect"),
    url(r'^home/$',
        HomeView.as_view(),
        name="brambling_home"),

    url(r'^daguerre/', include('daguerre.urls')),
    url(r'^404/$', 'django.views.defaults.page_not_found'),
    url(r'^418/$', TemplateView.as_view(template_name='418.html')),
    url(r'^500/$', 'django.views.defaults.server_error'),
    url(r'^500/raise/$', ExceptionView.as_view()),
    url(r'^ping/$', lambda r: HttpResponse('A-Okay!')),
    url(r'^mail/confirmation/$', ConfirmationPreviewView.as_view()),
    url(r'^mail/order_receipt/$', OrderReceiptPreviewView.as_view()),
    url(r'^mail/order_alert/$', OrderAlertPreviewView.as_view()),
    url(r'^mail/invite_event/$', InvitePreviewView.as_view(kind=Invite.EVENT_EDITOR)),
    url(r'^mail/invite_org/$', InvitePreviewView.as_view(kind=Invite.ORGANIZATION_EDITOR)),

    url(r'^webhooks/dwolla/$', DwollaWebhookView.as_view(), name='brambling_dwolla_webhook'),

    url(r'^api/', include('brambling.api.urls')),

    url(r'^(?P<organization_slug>[\w-]+)/', include(organization_urlpatterns)),
)


# --------------------------------------------------------
#
# PYTHON PROGRAM DEFINITION
#
# The knowledge a computer has of Python can be specified in 3 levels:
# (1) Prelude knowledge --> The computer has it by default.
# (2) Borrowed knowledge --> The computer gets this knowledge from 3rd party libraries defined by others
#                            (but imported by us in this program).
# (3) Generated knowledge --> The computer gets this knowledge from the new functions defined by us in this program.
#
# When launching in a terminal the command:
# user:~$ python3 this_file.py
# our computer first processes this PYTHON PROGRAM DEFINITION section of the file.
# On it, our computer enhances its Python knowledge from levels (2) and (3) with the imports and new functions
# defined in the program. However, it still does not execute anything.
#
# --------------------------------------------------------

import pyspark
import pyspark.sql.functions
import graphframes
import graphframes.lib

# ------------------------------------------
# FUNCTION my_aggregate_messages
# ------------------------------------------
def my_aggregate_messages(spark, my_dataset_dir):
    # 1. We load my_edgesDF from the dataset

    # 1.1. We define the Schema of our DF.
    my_schema = pyspark.sql.types.StructType(
      [pyspark.sql.types.StructField("src", pyspark.sql.types.IntegerType(), True),
       pyspark.sql.types.StructField("dst", pyspark.sql.types.StringType(), True)
       ])

    # 1.2. Operation C1: We create the DataFrame from the dataset and the schema
    my_edges_datasetDF = spark.read.format("csv") \
      .option("delimiter", " ") \
      .option("quote", "") \
      .option("header", "false") \
      .schema(my_schema) \
      .load(my_dataset_dir)

    # 1.3. Operation T1: We add the value column
    my_edgesDF = my_edges_datasetDF.withColumn("value", pyspark.sql.functions.lit(1))

    # 1.4. Operation P1: We persist it
    my_edgesDF.persist()

    # 1.5. Operation T3: We transform my_edgesDF so as to get my_verticesDF
    my_verticesDF = my_edgesDF.select(my_edgesDF["src"]) \
      .withColumnRenamed("src", "id") \
      .dropDuplicates() \
      .withColumn("value", pyspark.sql.functions.lit(1))

    # 1.6. Operation C3: We create myGF from my_verticesDF and my_edgesDF
    myGF = graphframes.GraphFrame(my_verticesDF, my_edgesDF)

    # 2. Operation T4: We aggregate messages for myGF. In particular, for each v we aggregate the sum of all neighbour

    # 2.1. We create the aggregate object, which provides a kind of triplet view: src, edge, dst
    myAO = graphframes.lib.AggregateMessages()

    # 2.2. Operation T4: We aggregate the messages in a DataFrame
    # (1) The first parameter is the merge function: How do we aggregate all messages being sent?
    #     In this case it just uses the function sum to sum them all
    # (2) The second argument is the sendToSrc messages: Given each element of the triplets view: What message do we send per edge?
    #     In this case we just decide to send the id of the destination vertex
    # (3) The third argument is the sendToDst messages: Given each element of the triplets view: What message do we send per edge?
    #     In this case we just decide to send the id of the source vertex
    my_aggregatedDF = myGF.aggregateMessages(pyspark.sql.functions.sum(myAO.msg).alias("result"),
                                 myAO.dst["id"],
                                 myAO.src["id"],
                                )

    # 3. Operation A1: We display the content of my_aggregatedDF
    my_aggregatedDF.show()

# ------------------------------------------
# FUNCTION my_collect_neighbor_ids
# ------------------------------------------
def my_collect_neighbor_ids(spark, my_dataset_dir, direction):
    # 1. We load my_edgesDF from the dataset

    # 1.1. We define the Schema of our DF.
    my_schema = pyspark.sql.types.StructType(
        [pyspark.sql.types.StructField("src", pyspark.sql.types.IntegerType(), True),
         pyspark.sql.types.StructField("dst", pyspark.sql.types.StringType(), True)
        ])

    # 1.2. Operation C1: We create the DataFrame from the dataset and the schema
    my_edges_datasetDF = spark.read.format("csv") \
        .option("delimiter", " ") \
        .option("quote", "") \
        .option("header", "false") \
        .schema(my_schema) \
        .load(my_dataset_dir)

    # 1.3. Operation T1: We add the value column
    my_edgesDF = my_edges_datasetDF.withColumn("value", pyspark.sql.functions.lit(1))

    # 1.4. Operation P1: We persist it
    my_edgesDF.persist()

    # 1.5. Operation T3: We transform my_edgesDF so as to get my_verticesDF
    my_verticesDF = my_edgesDF.select(my_edgesDF["src"]) \
        .withColumnRenamed("src", "id") \
        .dropDuplicates() \
        .withColumn("value", pyspark.sql.functions.lit(1))

    # 1.6. Operation C3: We create myGF from my_verticesDF and my_edgesDF
    myGF = graphframes.GraphFrame(my_verticesDF, my_edgesDF)

    # 2. Operation T4: We get the edges again
    my_edgesDF = None
    if (direction == True):
          my_edgesDF = myGF.edges.drop("value") \
                                 .groupBy("src") \
                                 .agg(pyspark.sql.functions.collect_list("dst").alias("out_neighbors"))
    else:
          my_edgesDF = myGF.edges.drop("value") \
                                 .groupBy("dst") \
                                 .agg(pyspark.sql.functions.collect_list("src").alias("in_neighbors"))

    # 3. Operation A1: We display my_edgesDF
    my_edgesDF.show()

# ------------------------------------------
# FUNCTION my_main
# ------------------------------------------
def my_main(spark, my_dataset_dir, option, direction):
    # We decide which function to call to based on the option
    if (option == 1):
        my_aggregate_messages(spark, my_dataset_dir)
    elif (option == 2):
        my_collect_neighbor_ids(spark, my_dataset_dir, direction)

# --------------------------------------------------------
#
# PYTHON PROGRAM EXECUTION
#
# Once our computer has finished processing the PYTHON PROGRAM DEFINITION section its knowledge is set.
# Now its time to apply this knowledge.
#
# When launching in a terminal the command:
# user:~$ python3 this_file.py
# our computer finally processes this PYTHON PROGRAM EXECUTION section, which:
# (i) Specifies the function F to be executed.
# (ii) Define any input parameter such this function F has to be called with.
#
# --------------------------------------------------------
if __name__ == '__main__':
    # 1. We use as many input arguments as needed
    option = 2
    direction = False

    # 2. Local or Databricks
    local_False_databricks_True = True

    # 3. We set the path to my_dataset and my_result
    my_local_path = "../../../../"
    my_databricks_path = "/"

    my_dataset_dir = "FileStore/tables/4_Spark_Graph_Libs/1_TinyGraph/my_dataset"

    if local_False_databricks_True == False:
        my_dataset_dir = my_local_path + my_dataset_dir
    else:
        my_dataset_dir = my_databricks_path + my_dataset_dir

    # 4. We configure the Spark Session
    spark = pyspark.sql.SparkSession.builder.getOrCreate()
    spark.sparkContext.setLogLevel('WARN')
    print("\n\n\n")

    # 5. We call to our main function
    my_main(spark, my_dataset_dir, option, direction)