import logging

from utils.classes import Circle, Mesh
from utils.schemas import Node
from utils.functions import run_simulation

from utils.classes import Store
from utils.schemas import Settings

logger = logging.getLogger(__name__)

def simulation(store: Store, settings: Settings):
    logger.info("Simulation started running")
    domain_height = 50
    domain_width = 300
    circle_radius = domain_height // 9
    inlet_velocity = 0.04
    reynolds_number = 80

    center_node = Node(x=domain_width // 5, y=domain_height // 2)
    circle = Circle(circle_radius)
    mesh = Mesh(domain_width, domain_height)
    mesh.place(circle, center_node)

    run_simulation(
        store,
        mesh.get_grid().astype(bool),
        domain_height,
        domain_width,
        circle_radius,
        inlet_velocity,
        reynolds_number,
        15000,
        True,
        1000,
    )


# -*- coding:utf-8 -*-
import os

import pandas as pd
import tushare as ts

from tenvs.common.logger import logger

class Market:
    """
    模拟市场，加载环境所需要的数据
    NOTE(wen):
        从多数实验可以发现，使用后复权的数据做为模型输入，性能更优， Market统一使用
        后复权数据做模型的输入, 而不复权的数据做买卖的判断
    indexs:
        000001.SH: 上证指数
        399001.SZ: 深证成指
        ...
    NOTE(wen): 使用tushare下载指数日线信息需要在tushare.pro帐户中有200积分
    data_dir: 存储数据文件的目录，以降低重复下载的频率
    """

    def __init__(self,
                 ts_token="",
                 start="20190101",
                 end="20200101",
                 codes=["000001.SZ"],
                 indexs=["000001.SH", "399001.SZ"],
                 data_dir="/tmp/tenvs"):
        ts.set_token(ts_token)
        self.start = start
        self.end = end
        self.codes = codes
        self.indexs = indexs
        self.data_dir = data_dir
        self.load_codes_history()
        self.load_indexs_history()
        # hfq数据: 去除不复权数据(9列) 和复权因子(1列): 10, 从第11列开始是后复权数据
        self.equity_hfq_info_start_index = 10
        self.init_market_info()
        self.init_size_info()
        # 涨停判断
        # NOTE(wen): 如果涨跌幅超过 top_pct_change则认为到达涨跌停的状态
        # TODO: 准确的涨跌停判断方式
        self.top_pct_change = 9.7

    def get_info_size(self, info_name):
        date = self.open_dates[0]
        return len(self.market_info[date][info_name])

    def init_size_info(self):
        """
        初始化数据的size
        """
        self.equity_hfq_info_size = self.get_info_size("equities_hfq_info")
        self.indexs_info_size = self.get_info_size("indexs_info")

    def get_code_history(self, code, adj=None):
        return ts.pro_bar(
            ts_code=code, adj=adj,
            start_date=self.start, end_date=self.end)

    def load_codes_history(self):
        """
        self.codes_history: dict
        每条记录由两部分数据组成: 复权因子(1列), 不复权数据(9列), 后复权数据(9列)
            复权因子: adj_factor(1列)
            不复权OHLCV, ... (9列)
            后复权OHLCV, ... (9列)

        self.codes_history[code].columns的值为:
        [u'adj_factor', u'open', u'high', u'low', u'close', u'pre_close',
        u'change', u'pct_chg', u'vol', u'amount', u'open_hfq', u'high_hfq',
        u'low_hfq', u'close_hfq', u'pre_close_hfq', u'change_hfq',
        u'pct_chg_hfq', u'vol_hfq', u'amount_hfq']
        """

        self.codes_history = {}
        for code in self.codes:
            dir = os.path.join(self.data_dir, code)
            if not os.path.exists(dir):
                try:
                    os.makedirs(dir)
                except OSError:
                    pass
            data_path = os.path.join(dir,
                                     self.start + "-" + self.end + ".csv")
            if os.path.exists(data_path):
                df = pd.read_csv(data_path)
                df = df.set_index("trade_date")
                df.index = df.index.astype(str, copy=False)
                self.codes_history[code] = df

            else:
                # 不复权
                df_bfq = self.get_code_history(code, adj=None)
                df_bfq = df_bfq.drop(columns=["ts_code"], axis=1)
                # 后复权
                df_hfq = self.get_code_history(code, adj="hfq")
                df_hfq = df_hfq.drop(columns=["ts_code"], axis=1)
                df = pd.merge(df_bfq, df_hfq,
                              on='trade_date', how='left',
                              suffixes=('', '_hfq'))
                # 拆分因子
                col_name = df.columns.tolist()
                col_name.insert(0, 'adj_factor')
                df = df.reindex(columns=col_name)
                df["adj_factor"] = df["close_hfq"] / df["close"]
                df = df.sort_values(by="trade_date", ascending=True)
                df = df.set_index("trade_date")
                df.index = df.index.astype(str, copy=False)
                df.to_csv(data_path)
                self.codes_history[code] = df

    def load_indexs_history(self):
        self.indexs_history = {}
        # 默认加载: 000001.SH(上证指数), 399001.SZ(深城证指)
        indexs = ["000001.SH", "399001.SZ"]
        for code in self.indexs:
            if code not in indexs:
                indexs.append(code)

        for code in indexs:
            dir = os.path.join(self.data_dir, "indexs", code)
            if not os.path.exists(dir):
                os.makedirs(dir)
            data_path = os.path.join(dir,
                                     self.start + "-" + self.end + ".csv")
            if os.path.exists(data_path):
                df = pd.read_csv(data_path)
                df = df.set_index("trade_date")
                df.index = df.index.astype(str, copy=False)
                self.indexs_history[code] = df
            else:
                pro = ts.pro_api()
                df = pro.index_daily(ts_code=code,
                                     start_date=self.start,
                                     end_date=self.end)
                df = df.drop(columns=["ts_code"], axis=1)
                df = df.sort_values(by="trade_date", ascending=True)
                df = df.set_index("trade_date")
                df.index = df.index.astype(str, copy=False)
                df.to_csv(data_path)
                self.indexs_history[code] = df

    def set_pre_info(self):
        date = self.open_dates[0]
        self.equities_pre_hfq_info = {}
        self.equities_pre_bfq_info = {}
        for code in self.codes:
            # 如果第一天就停牌，这里会出错，建议另外选择一天开始回测
            try:
                data = self.codes_history[code].loc[date].tolist()
                bfq_data = data[1: self.equity_hfq_info_start_index]
                # 开盘时， open=1
                bfq_data.append(1)
                self.equities_pre_bfq_info[code] = bfq_data
                # 后复权
                hfq_data = data[self.equity_hfq_info_start_index:]
                # 开盘时， open=1
                hfq_data.append(1)
                self.equities_pre_hfq_info[code] = hfq_data
            except Exception as e:
                print("%s, %s停牌，建议另外选择一天开始回测" % (code, date))
                print(e)
                exit()

    def get_equities_bfq_info(self, date):
        info = []
        for code in self.codes:
            data = None
            if date in self.codes_history[code].index:
                data = self.codes_history[code].loc[date].tolist()
                data = data[1: self.equity_hfq_info_start_index]
                # 开盘时， open=1
                data.append(1)
            else:
                # 如果停牌则使用前一开盘日信息
                data = self.equities_pre_bfq_info[code]
                data[-1] = 0
            # 更新前一开盘日信息
            self.equities_pre_bfq_info[code] = data
            info.extend(data)
        return info

    def get_equities_hfq_info(self, date):
        info = []
        for code in self.codes:
            data = None
            if date in self.codes_history[code].index:
                data = self.codes_history[code].loc[date].tolist()
                data = data[self.equity_hfq_info_start_index:]
                # 开盘时， open=1
                data.append(1)
            else:
                # 如果停牌则使用前一开盘日信息
                data = self.equities_pre_hfq_info[code]
                data[-1] = 0
            # 更新前一开盘日信息
            self.equities_pre_hfq_info[code] = data
            info.extend(data)
        return info

    def get_indexs_info(self, date):
        info = []
        for code in self.indexs:
            data = self.indexs_history[code].loc[date].tolist()
            info.extend(data)
        return info

    def init_market_info(self):
        """
        以日期为key, 将市场相关信息组织在一个dict中:
            equities_hfq_info: 合并之后的个股信息, 如果某支股停牌，则使用它前一开盘日信息
            indexs_info: 合并之后指数信息
        Note(wen): 添加其他市场相关信息，都放在这里
        """
        self.open_dates = self.indexs_history["000001.SH"].index.tolist()
        self.open_dates.sort()
        self.set_pre_info()
        self.market_info = {}
        for date in self.open_dates:
            self.market_info[date] = {}
            self.market_info[date]["equities_bfq_info"] = \
                self.get_equities_bfq_info(date)
            self.market_info[date]["equities_hfq_info"] = \
                self.get_equities_hfq_info(date)
            self.market_info[date]["indexs_info"] = self.get_indexs_info(date)

    def is_suspended(self, code='', datestr=''):
        # 是否停牌，是：返回 True, 否：返回 False
        if code not in self.codes_history:
            return True
        if datestr not in self.codes_history[code].index:
            return True
        return False

    def buy_check(self, code='', datestr='', bid_price=None):
        # 返回：OK, 成交价
        ok = False
        # 停牌
        if self.is_suspended(code, datestr):
            return ok, 0
        # 获取当天标的信息
        [open, high, low, pct_change] = self.codes_history[code].loc[
            datestr, ["open", "high", "low", "pct_chg"]]
        # 涨停封板, 无法买入
        if low == high and pct_change > self.top_pct_change:
            logger.debug(u"sell_check %s %s 涨停法买进" % (code, datestr))
            return ok, 0
        # 买入竞价低于最低价，不能成交
        if bid_price < low:
            return ok, 0
        # 买入竞价高于最低价， 可以成交
        if bid_price >= low:
            return True, min(bid_price, high)

    def sell_check(self, code='', datestr='', bid_price=None):
        # 返回：OK, 成交价
        ok = False
        # 停牌
        if self.is_suspended(code, datestr):
            return ok, 0
        # 获取当天标的信息
        [open, high, low, pct_change] = self.codes_history[code].loc[
            datestr, ["open", "high", "low", "pct_chg"]]
        # 跌停封板， 不能卖出
        if low == high and pct_change < -self.top_pct_change:
            logger.debug(u"sell_check %s %s 跌停无法卖出" % (code, datestr))
            return ok, 0
        # 卖出竞价高于最高价，不可以成交
        if bid_price > high:
            return ok, 0
        # 卖出竞价在最低最高价之间， 可以成交，按出价成交
        # NOTE: 这里卖出竞价低于最低价时，可以成交，按最低价成交
        if bid_price <= high:
            ok = True
            return ok, max(bid_price, low)

    def get_pre_close_price(self, code, datestr):
        if datestr in self.codes_history[code].index:
            return self.codes_history[code].loc[datestr]["pre_close"]
        # 如果当天停牌, 返回前一开市日的pre_close
        df = self.codes_history[code]
        df_ = df[df.index < datestr]
        return df_.iloc[-1]["pre_close"]

    def get_close_price(self, code, datestr):
        if datestr in self.codes_history[code].index:
            return self.codes_history[code].loc[datestr]["close"]
        # 如果当天停牌, 返回前一开市日收盘价
        df = self.codes_history[code]
        df_ = df[df.index < datestr]
        return df_.iloc[-1]["close"]

    def get_pre_adj_factor(self, code, datestr):
        df = self.codes_history[code]
        df_ = df[df.index < datestr]
        return df_.iloc[-1]["adj_factor"]

    def get_adj_factor(self, code, datestr):
        if self.is_suspended(code=code, datestr=datestr):
            return self.get_pre_adj_factor(code, datestr)
        else:
            return self.codes_history[code].loc[datestr]["adj_factor"]

    def get_divide_rate(self, code, datestr):
        pre_adj_factor = self.get_pre_adj_factor(code, datestr)
        current_adj_factor = self.get_adj_factor(code, datestr)
        return current_adj_factor / pre_adj_factor


from datetime import datetime

from django.db import models, transaction

#Managers
from django.db.models.aggregates import Count
from django.db.models.fields import DateTimeField
from django.utils import formats
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode
from base.api.util import serialize_query
from base.exceptions import EventDataError
from base.prodserv_models import Product, Service
from base.signals import proposal_accepted, event_booking_success
from stripe_cater.models import PaymentOrder

class EventManager(models.Manager):
    ERROR = -1
    ERRORS = [ERROR]
    def get_event_by(self, id, business=None):
        query = self.exclude(status=N_Event_Status.CANCELLED).filter(id=id)
        if business:
            query = query.filter(customer__business=business)
        return query.first()

    def create_or_update_invoice_from_event(self, data, invoice, handly = False):

        if invoice:
            response = self.create_or_update_proposal_from_event(data, proposal=invoice.proposal)
            if response in self.ERRORS:
                return response
            else:
                invoice.status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.EDITTING)
                invoice.update_due_date(data.get('due_date'))
                # changing_event_data.send(sender=Invoice,invoice=invoice)
                return invoice
        else:
           response = self.create_or_update_proposal_from_event(data, from_invoice= handly)
           if response in self.ERRORS:
                return response
           else:
                response = Invoice.objects.create_from_proposal(due_date=data.get('due_date'),proposal=response)
                if response in Invoice.objects.ERRORS:
                    return self.ERROR
                else:
                    return response

    def create_or_update_proposal_from_event(self, data, proposal=None, from_invoice=False):
        comment = data.get('comment')
        if proposal:
            response = self.create_or_update_event(data=data, instance=proposal.event)
            if response not in self.ERRORS:
                if proposal.status.id != N_Proposal_Status.DENIED:#para que se mantenga denegado
                    proposal.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.EDITTING)
                proposal.client_message = comment
                proposal.save()
                # changing_event_data.send(sender=Proposal,proposal=proposal)
                return proposal
            else:
                return response
        else:

            response = self.create_or_update_event(data=data,status=N_Event_Status.ACCEPTED)
            if response in self.ERRORS:
                return response

            try:
               proposal =  Proposal.objects.create_proposal(event=response,client_message=comment, from_invoice=from_invoice)
               return proposal
            except Exception as e:
                print(e)
                return self.ERROR

    def create_or_update_event(self, data, instance=None, status=None):
        name = data.get('name')
        # address = data.get('address')
        event_date = data.get('event_date')
        # due_date = data.get('due_date')
        customer = data.get('customer')
        address = {}
        address['first_line'] = data.get('first_line')
        address['second_line'] = data.get('second_line')
        address['zip'] = data.get('zip')
        address['city'] = data.get('city')
        address['state'] = data.get('state')

        try:
            if instance:
                instance.name = name
                instance.address.update(data=address)
                instance.event_date = event_date
                # instance.due_date = due_date
                instance.customer = customer

                instance.save()

                return instance
            else:
                status_value = N_Event_Status.objects.get(pk=N_Event_Status.ACCEPTED)
                if status:
                    status_value = status
                address_obj = Address.objects.create_address(data=address)

                event = self.create(name=name, address=address_obj, event_date = event_date,
                                    # due_date=due_date,
                                    customer=customer, status=status_value)

                return event
        except Exception as e:
            print(e)
            return self.ERROR

    def create_event_from_external_source(self, data, customer, req_platos=[], req_servicios=[]):
        data_to_save = {'name':data.get('evento_name'),
                        'address':data.get('evento_address'),
                        'event_date':data.get('evento_date'),
                        'customer':customer,
                        'first_line': data.get('evento_first_line'),
                        'second_line': data.get('evento_second_line'),
                        'city':data.get('evento_city'),
                        'zip':data.get('evento_zip'),
                        'state':data.get('evento_state')
                        }

        business = customer.business

        param_event_date = DateTimeField().to_python(value=data.get('evento_date'))
        schedule_events_at = self.number_of_events_schedule_at(event_date=param_event_date.date(),
                                                               business=business)

        can_be_accepted = business.can_accepted_one_more(schedule_events_at)

        status = N_Event_Status.objects.get(pk=N_Event_Status.PENDING)

        if can_be_accepted:
            status = N_Event_Status.objects.get(pk=N_Event_Status.ACCEPTED)

        print(can_be_accepted)
        response = self.create_or_update_event(data=data_to_save, status=status)

        if response in self.ERRORS:
            raise EventDataError()
        # asociar casa req al evento
        type = N_Request_Type.objects.get(pk=N_Request_Type.PRODUCT)
        requests = []
        for req in req_platos:
            req_obj = Request.objects.create_from_external_source(data=req, event=response, type=type)
            requests.append(req_obj)
        type = N_Request_Type.objects.get(pk=N_Request_Type.SERVICE)
        for req in req_servicios:
            req_obj = Request.objects.create_from_external_source(data=req, event=response, type=type)
            requests.append(req_obj)
        print(requests)
        Request.objects.bulk_create(requests)

        event_booking_success.send(sender=None,event=response)
        return response

    def number_of_events_schedule_at(self, event_date, business):
        return self.filter(customer__business=business).filter(event_date__date=event_date).exclude(status=N_Event_Status.CANCELLED).count()

    def get_events_(self, business=None, id=None):
        query = self.exclude(status=N_Event_Status.CANCELLED)
        if id:
            query = query.filter(id=id)
        if business:
          query = query.filter(customer__business=business)
        return query

    def bulk_delete(self, ids_list):
        status = N_Event_Status.objects.get(pk=N_Event_Status.CANCELLED)
        with transaction.atomic():
            Invoice.objects.bulk_delete_by_events(event_ids_list=ids_list)
            Proposal.objects.bulk_delete_by_evetns(event_ids_list=ids_list)
            deleted = self.filter(id__in=ids_list).update(status=status)
            return deleted

class ProposalManager(models.Manager):
    ERROR = -1
    ERROR_TOKEN = -2
    ERROR_EXPIRED = -3
    ERROR_INVOICE = -4
    ERROR_PRESUPUESTO_ACCEPTADO =-5
    ERROR_PRESUPUESTO_DENEGADO= -6
    OK = 0
    ERRORS = [ERROR, ERROR_TOKEN,ERROR_EXPIRED]
    def generate_number(self):
        return int(datetime.today().timestamp() * 1000000)
    def create_proposal(self, event, client_message=None, from_invoice=False):
        status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.EDITTING)
        if from_invoice:
            status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.ACCEPTED)
        return self.create(event=event, status=status, number=self.generate_number(), client_message=client_message)
    def create_proposal_from_external_source(self, event):
        return self.create_proposal(event=event)

    def get_by_id(self, id, business=None, hide_cancelled=True):
        query = self.filter(id=id)
        if hide_cancelled:
          query = query.exclude(status=N_Proposal_Status.CANCELLED)
        if business:
            query = query.filter(event__customer__business=business)
        p = query.first()
        if p:
            return p
        else:
            return self.ERROR

    def get_by_business(self, business, hide_cancelled=True):
        query = self.filter(event__customer__business=business)
        if hide_cancelled:
           query =  query.exclude(status=N_Proposal_Status.CANCELLED)
        return query

    def deny_proposal(self, id, token):
        p = self.filter(id=id).filter(send_token=token).filter(status=N_Proposal_Status.PENDING).first()
        if p:
            if p.has_expired():
                return self.ERROR_EXPIRED
            p.deny()
            # p.reset_token()
            # p.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.DENIED)
            # p.save()
            return p
        else:
            return self.ERROR_TOKEN

    def accpet_proposal(self, id, token):
        p = self.filter(id=id).filter(send_token=token).filter(status=N_Proposal_Status.PENDING).first()
        if p:
            if p.has_expired():
                return self.ERROR_EXPIRED
            response = p.accept()

            if response in Invoice.objects.ERRORS:
                return self.ERROR_INVOICE
            # p.reset_token()
            # p.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.ACCEPTED)
            # p.save()
            return p
        else:
            return self.ERROR_TOKEN

    def cancell_proposal(self, proposal):
        try:
            proposal.status = N_Proposal_Status.objects.get(id=N_Proposal_Status.CANCELLED)
            proposal.save()
            return self.OK
        except:
            return self.ERROR

    def bulk_delete(self, ids_list, business):
        cancelled_status = N_Proposal_Status.objects.get(id=N_Proposal_Status.CANCELLED)
        Proposal.objects.filter(id__in=ids_list).filter(event__customer__business=business).update(status=cancelled_status)

    def bulk_delete_by_evetns(self, event_ids_list):
        cancelled_status = N_Proposal_Status.objects.get(id=N_Proposal_Status.CANCELLED)
        Proposal.objects.filter(event__id__in=event_ids_list).update(status=cancelled_status)

    def get_invoices_all(self, business):
        return  self.annotate(ni=Count('invoice')).filter(ni=1).\
                filter(event__customer__business=business).filter(invoice__deleted=False)

class InvoiceManager(models.Manager):
    ERROR = -1
    ERROR_TOKEN = -2
    ERROR_EXPIRED = -3
    OK = 0
    ERRORS = [ERROR, ERROR_EXPIRED, ERROR_TOKEN]
    def generate_number(self):
        return str(int(datetime.today().timestamp() * 1000000))

    def create_from_proposal(self, proposal, handly = True, due_date=None):
        try:
            # status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.CREATED)
            # if handly:
            status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.EDITTING)

            due_date_value = proposal.event.event_date
            if due_date:
                due_date_value = due_date

            invoice = self.create(due_date=due_date_value,
                number=self.generate_number(), proposal=proposal, status=status)
            return invoice
        except Exception as e:
            print(e)
            return self.ERROR

    def get_by_id(self, id, business=None, hide_deleted=True):
        query = self.filter(id=id)
        if hide_deleted:
          query = query.exclude(deleted=True)
        if business:
            query = query.filter(proposal__event__customer__business=business)
        inv = query.first()
        if inv:
            return inv
        else:
            return self.ERROR

    def get_invoice_to_pay_by_id(self, id, token):
        invoice = self.filter(id=id).exclude(deleted=True).filter(send_token=token).first()
        print(invoice.status)
        if invoice and invoice.status.id not in [N_Invoice_Status.PAID]:
            return invoice
        else:
            return self.ERROR

    def get_by_business(self, business, hide_deleted=True):
        query = self.filter(proposal__event__customer__business=business)
        if hide_deleted:
           query = query.exclude(deleted=True)
        return query

    def delete_invoice(self, invoice):
        response = Proposal.objects.cancell_proposal(invoice.proposal)
        if response in Proposal.ERRORS:
            return self.ERROR
        try:
            invoice.deleted = True
            invoice.save()
            return self.OK
        except:
            return self.ERROR

    def get_invoice_by_payment(self, payment):
        return self.filter(payment_order=payment).first()

    def get_by_proposal(self, proposal):
        return self.filter(proposal=proposal).first()

    def chekc_due_date(self, business=None):
        today = datetime.today()
        query = Invoice.objects.filter(status=N_Invoice_Status.PENDING).filter(due_date__lt=today)
        if business:
            query = query.filter(proposal__event__customer__business=business)
        query.update(status=N_Invoice_Status.PAST_DUE)

    def bulk_delete(self, ids_list, business):
        Invoice.objects.filter(id__in=ids_list).filter(proposal__event__customer__business=business).update(deleted=True)

    def bulk_delete_by_events(self, event_ids_list):
        Invoice.objects.filter(proposal__event__id__in=event_ids_list).update(deleted=True)

class ItemManager(models.Manager):
     ERROR = -1
     ERRORS = [ERROR]
     def update_item(self, data, item):
          try:
              oferta = data.get('oferta')
              oferta = Oferta.objects.get_oferta_by_name(name=oferta, business=item.oferta.business)
              discount = data.get('discount')
              unit_cost = data.get('unit_cost')
              quantity = data.get('quantity')
              tax = data.get('tax') or item.proposal.get_tax()
              description = data.get('description')

              item.tax = tax
              item.oferta = oferta
              item.discount = discount
              item.description = description
              item.unit_cost = unit_cost
              item.quantity = quantity

              item.save()
              # changing_item_data.send(sender=None, proposal=item.proposal)
              return item
          except:
              return self.ERROR

     def delete_item(self,id):
         try:
            item = self.get(pk=id)
            # changing_item_data.send(sender=None, proposal=item.proposal)
            item.delete()
         except Exception as e:
             print(e)
             pass

     def create_item(self, item_data,proposal, flush=True):
         type_id = item_data.get('type')
         if isinstance(type_id, dict):
             type_id = type_id.get('id')
         type = N_Request_Type.objects.get(pk=type_id)

         item_entity = Item(proposal=proposal,
                                   tax=item_data.get('tax'),
                                   discount=item_data.get('discount'),
                                   unit_cost=item_data.get('unit_cost'),
                                   quantity=item_data.get('quantity'),
                                   type=type,
                                   )
         request_id = item_data.get('request')
         if isinstance(request_id, dict):
             request_id = request_id.get('id')

         if type.id == N_Request_Type.PRODUCT:
             item_entity.product = Product.objects.get(pk=request_id)
         else:
             item_entity.service = Service.objects.get(pk=request_id)

         if flush:
             item_entity.save()

         return item_entity

     def update_item(self, item_data,flush=True):
         item = self.get(pk=item_data.get('id'))

         type_id = item_data.get('type')
         if isinstance(type_id, dict):
             type_id = type_id.get('id')

         type = N_Request_Type.objects.get(pk=type_id)
         item.tax=item_data.get('tax')
         item.discount=item_data.get('discount')
         item.unit_cost=item_data.get('unit_cost')
         item.quantity=item_data.get('quantity')
         item.type = type

         if type.id == N_Request_Type.PRODUCT:
             item.product = Product.objects.get(pk=item_data.get('product').get('id'))
         else:
             item.service = Service.objects.get(pk=item_data.get('service').get('id'))

         if flush:
             item.save()

         return item

class RequestManager(models.Manager):

    def create_from_external_source(self, type,event, data, flush=False):

        req = Request(type=type, event=event, amount=data.get('amount'))
        if type.id == N_Request_Type.PRODUCT:
            req.product = Product.objects.get(pk=data.get('id'))
        else:
            req.service = Service.objects.get(pk=data.get('id'))

        if flush:
            req.save()
        return req

#Models
from base.models import Customer, Oferta, ModelSerialize, Address

class Event(models.Model, ModelSerialize):
    objects = EventManager()
    name = models.CharField(max_length=255)
    # address = models.CharField(max_length=255)
    address = models.ForeignKey(Address)
    event_date = models.DateTimeField()
    # due_date = models.DateField()
    customer = models.ForeignKey(Customer)
    status = models.ForeignKey('N_Event_Status')

    default_fields = ['name','address','event_date','customer','status','id','can_remove']

    def schedule(self):
        self.status = N_Event_Status.objects.get(pk=N_Event_Status.SCHEDULED)
        self.save()

    def accepted(self):
        self.status = N_Event_Status.objects.get(pk=N_Event_Status.ACCEPTED)
        self.save()

    def is_scheduled(self):
        return self.status.id == N_Event_Status.SCHEDULED

    def may_send_proposal(self):
        return self.status.id == N_Event_Status.SCHEDULED

    def may_create_proposal(self):
        # return self.status.id == N_Event_Status.SCHEDULED and self.proposal_set.count() == 0
        return self.proposal_set.count() == 0

    def get_requests(self):
        return self.request_set.all()

    def __str__(self):
        return self.name

    def serializable_value(self, field_name):
        if field_name not in ['customer','address','proposal','can_remove']:
            return super(Event, self).serializable_value(field_name)
        # if field_name == 'event_date':
            # return formats.date_format(self.event_date, 'Y-m-d H:m')
        if field_name == 'customer':
            return self.customer.serialize(fields=['email','first_name','last_name','suffix','prefix','cellphone','id','full_name'])
        if field_name == 'address':
            return self.address.serialize()
        if field_name == 'proposal':
            prop = self.proposal_set.first()
            if prop:
                return prop.serialize(fields=['id','status', 'invoice'])
            else:
                return None
        if field_name == 'can_remove':
            can_remove = self.can_be_removed()
            return can_remove

    def can_be_removed(self):
        return self.status.id not in [N_Event_Status.DONE]
    # def serialize(self, fields=[]):
    #
    #     if len(fields) == 0:
    #         fields = Event.default_fields
    #
    #     response = {}
    #     for fd in fields:
    #         # print(fd,self.serializable_value(fd))
    #         response[fd] = self.serializable_value(fd)
    #     return response

class Proposal(models.Model, ModelSerialize):

    DEFAULT_TOKEN_VALUE = '0'
    ERROR = -1
    ERRORS = [ERROR]
    objects = ProposalManager()
    event = models.ForeignKey(Event)
    status = models.ForeignKey('N_Proposal_Status')
    client_message = models.TextField(max_length=150, null=True, default='')
    number = models.CharField(default='0', max_length=50)
    send_token = models.CharField(default=DEFAULT_TOKEN_VALUE, max_length=50)
    created_at = models.DateTimeField(auto_now_add=True)
    denied_by_system = models.BooleanField(default=False)

    default_fields = ['id','status','event','items', 'can_send','can_edit', 'can_resend']

    def serializable_value(self, field_name):
        if field_name == 'event':
            return self.event.serialize()
        if field_name == 'items':
            return serialize_query(self.item_set.all())
        if field_name == 'can_send':
            cand_send = self.may_send_email() and self.status.id != N_Proposal_Status.PENDING
            return cand_send
        if field_name == 'can_resend':
            can_resend = self.may_send_email() and self.status.id == N_Proposal_Status.PENDING
            return can_resend
        if field_name == 'can_edit':
            can_edit = self.may_be_editted()
            return can_edit
        if field_name == 'invoice':
            invoice = self.invoice_set.first()
            if invoice:
                return invoice.serialize()
            return None
        return super(Proposal, self).serializable_value(field_name)

    def get_items(self):
        return self.item_set.all()

    def __str__(self):
        return self.event.name

    def get_item_by_id(self, id):
        try:
            return self.item_set.get(pk=id)
        except:
            return self.ERROR

    def add_item(self, data, business):
        try:
            oferta = data.get('oferta')
            oferta = Oferta.objects.get_oferta_by_name(name=oferta, business=business)
            discount = data.get('discount')
            unit_cost = data.get('unit_cost')
            quantity = data.get('quantity')
            tax = data.get('tax') or self.get_tax()
            description = data.get('description')

            item = Item.objects.create(tax=tax, quantity=quantity,oferta=oferta, discount=discount, description=description, unit_cost=unit_cost, proposal=self)
            # changing_item_data.send(sender=None, proposal=self)
            return item
        except Exception as e:
            print(e)
            return self.ERROR

    def add_items_list(self, data_items):
        lista = []

        for item in data_items:
            if item.get('id'):
                Item.objects.update_item(item_data=item, flush=False)
            else:
                item_entity = Item.objects.create_item(item_data=item, proposal=self, flush=False)
                lista.append(item_entity)
        Item.objects.bulk_create(lista)

    def create_items_from_requests(self, requests):
        item_list = []
        for req in requests:
            quantity = req.amount
            type = req.type
            tax = self.get_tax()
            # unit_cost = 1
            # if type.id == N_Request_Type.PRODUCT:
            print(req.get_unit_cost(), req.type)
            item = Item(quantity=quantity, type=type, tax=tax, discount=0, unit_cost=req.get_unit_cost(), proposal=self)
            if type.id == N_Request_Type.PRODUCT:
                item.product = req.product
            else:
                item.service = req.service
            # if req.has_own_tax():
            #     item.tax = req.get_tax()
            #
            # if req.has_own_discount():
            #     item.discount = req.get_discount()
            item_list.append(item)
        Item.objects.bulk_create(item_list)

    def get_total(self):
        return sum([item.grant_total() for item in self.get_items()])

    # def get_subtotal(self):
        return sum([item.sub_total() for item in self.get_items()])

    def get_subtotal(self):
        return sum([item.total() for item in self.get_items()])

    def get_taxes(self):
        return sum([item.sub_total()*item.tax for item in self.get_items()])/100

    def get_total_discount(self):
        return sum([item.total() - item.grant_total() for item in self.get_items()])

    def get_tax(self):
        return self.event.customer.business.tax / 100

    def may_send_email(self):
        return self.status.id in [N_Proposal_Status.DENIED, N_Proposal_Status.EDITTING, N_Proposal_Status.PENDING]

    def may_be_editted(self):
        return self.status.id not in [N_Proposal_Status.ACCEPTED, N_Proposal_Status.CANCELLED]

    def has_expired(self):
        return self.event.event_date.date() < datetime.date(datetime.today())

    def generate_token(self):
        if self.send_token == Proposal.DEFAULT_TOKEN_VALUE:
            token = int(datetime.today().timestamp() * 1000000)
            self.send_token = str(token)
            self.save()
            token = urlsafe_base64_encode(force_bytes(token))
        else:
            token = urlsafe_base64_encode(force_bytes(int(self.send_token)))
        return token

    def deny_by_business(self):
        if self.status.id == N_Proposal_Status.PENDING:
            self.deny(system=True)
            return self
        return self.ERROR

    def accept_by_business(self):
        if self.status.id in [N_Proposal_Status.PENDING, N_Proposal_Status.DENIED]:
            self.accept()

            return self
        return self.ERROR

    def deny(self, system=False):
        self.reset_token()
        self.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.DENIED)
        self.denied_by_system = system
        self.save()

    def accept(self):
          response = Invoice.objects.create_from_proposal(proposal=self, handly=False)
          if response in Invoice.objects.ERRORS:
               return response
          self.reset_token()
          self.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.ACCEPTED)
          self.event.schedule()
          self.save()
          proposal_accepted.send(sender=Proposal, proposal=self)

    def reset_token(self,restart_status=False, old_status=None):
        if restart_status:
            if not old_status:
                old_status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.EDITTING)
            self.status = old_status
        self.send_token = Proposal.DEFAULT_TOKEN_VALUE
        self.save()

    def prepare_to_send(self):
        self.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.PENDING)
        self.save()

    # def has_been_sent(self):
    #     self.status = N_Proposal_Status.objects.get(pk=N_Proposal_Status.PENDING)
    #     self.save()

class Invoice(models.Model, ModelSerialize):

    DEFAULT_TOKEN_VALUE = '0'
    objects = InvoiceManager()
    proposal = models.ForeignKey(Proposal)
    status = models.ForeignKey('N_Invoice_Status')
    # order = models.ForeignKey(Order, null=True)
    number = models.CharField(default='0', max_length=50)
    created_at = models.DateTimeField(auto_now_add=True)
    send_token = models.CharField(default=DEFAULT_TOKEN_VALUE, max_length=50)
    payment_order = models.ForeignKey(PaymentOrder, null=True)
    deleted = models.BooleanField(default=False)
    due_date = models.DateField(default=datetime.now())

    def add_items_list(self, items):
        self.proposal.add_items_list(items)

    def serializable_value(self, field_name):
        # if field_name == 'proposal':
        #     return self.proposal.serialize()
        if field_name == 'can_send':
            return self.may_send_email() and self.status.id != N_Invoice_Status.PENDING
        if field_name == 'can_resend':
            return self.may_send_email() and self.status.id == N_Invoice_Status.PENDING
        if field_name == 'can_edit':
            return self.may_be_editted()
        return super(Invoice, self).serializable_value(field_name)

    default_fields = ['id','status','number','created_at','due_date','can_send','can_edit', 'can_resend']

    def update_due_date(self, due_date):
        self.due_date = due_date
        self.save()

    def __str__(self):
        return self.proposal.event.name

    def generate_token(self):
        token = int(datetime.today().timestamp() * 1000000)
        self.send_token = str(token)
        self.save()
        token = urlsafe_base64_encode(force_bytes(token))
        return token

    # def has_been_sent(self):
    #     self.status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.PENDING)
    #     self.save()
    def prepare_to_send(self):
        self.status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.PENDING)
        self.save()

    def reset_token(self, restart_status=False, old_status=None):
        self.send_token = Invoice.DEFAULT_TOKEN_VALUE
        if restart_status:
            if not old_status:
                old_status = N_Invoice_Status.objects.get(pk=N_Invoice_Status.EDITTING)
            self.status = old_status
        self.save()

    def may_send_email(self):
        return self.status.id in [N_Invoice_Status.EDITTING, N_Invoice_Status.PENDING]

    def check_status(self):
        if self.payment_order.has_been_paid():
            self.status = N_Invoice_Status.objects.get(id=N_Invoice_Status.PAID)
            self.save()

    def set_payment(self, payment):
        self.payment_order = payment
        self.save()

    def has_payment_order(self):
        return self.payment_order != None

    def may_be_editted(self):
        return self.status.id != N_Invoice_Status.PAID

class Item(models.Model, ModelSerialize):
    objects = ItemManager()
    proposal = models.ForeignKey(Proposal)
    type = models.ForeignKey('N_Request_Type')
    product = models.ForeignKey(Product, null=True)
    service = models.ForeignKey(Service, null=True)

    tax = models.DecimalField(max_digits=4, decimal_places=2, default=0)
    quantity = models.IntegerField(default=0)
    unit_cost = models.DecimalField(max_digits=9, decimal_places=2)
    discount = models.DecimalField(max_digits=4, decimal_places=2)
    description = models.TextField(max_length=150, null=True, blank=True)

    default_fields = ['id','tax','quantity','description','discount','unit_cost', 'type', 'service', 'product']

    def serializable_value(self, field_name):

        if field_name == 'type':
            return self.type.serialize()
        if field_name == 'product':
            if self.product:
                return self.product.serialize(fields=['name','description','id'])
            else:
                return None
        if field_name == 'service':
            if self.service:
                return self.service.serialize(fields=['id','description','name'])
            else:
                return None
        if field_name == 'discount':
            return float(self.discount)
        if field_name == 'tax':
            return float(self.tax)
        if field_name == 'unit_cost':
            return float(self.unit_cost)
        return super(Item, self).serializable_value(field_name=field_name)

    def __str__(self):
        if self.type.id == N_Request_Type.PRODUCT:
            return self.product.name
        else:
            return self.service.name
        # return self.proposal.event.name

    def sub_total(self):
        return self.quantity * self.unit_cost

    def total(self):
        sub_total = self.sub_total()
        return sub_total*( 1 + self.tax/100)

    def grant_total(self):
        value = self.total()
        return value - value * self.discount / 100

class Request(models.Model):

    objects = RequestManager()

    event = models.ForeignKey(Event)
    type = models.ForeignKey('N_Request_Type')
    product = models.ForeignKey(Product, null=True)
    service = models.ForeignKey(Service, null=True)
    amount = models.DecimalField(max_digits=9, decimal_places=2)

    def get_unit_cost(self):
        if self.type.id == N_Request_Type.PRODUCT:
            print(self.product.name, self.amount)
            return self.product.get_product_price(self.amount)
        else:
            return self.service.get_service_price(self.amount)
        # return 1

    # def has_own_tax(self):
    #     # if self.type.id == N_Request_Type.PRODUCT:
    #     #     return self.product.has_own_tax()
    #     # else:
    #     #     return self.service.has_own_tax()
    #     return False

    # def get_tax(self):
    #     # if self.type.id == N_Request_Type.PRODUCT:
    #     #     return self.product.get_tax()
    #     # else:
    #     #     return self.service.get_tax()
    #     return 0

    # def has_own_discount(self):
    #     # if self.type.id == N_Request_Type.PRODUCT:
    #     #     return self.product.has_own_discount()
    #     # else:
    #     #     return self.service.has_own_discount()
    #     return False
    #
    # def get_discount(self):
    #     # if self.type.id == N_Request_Type.PRODUCT:
    #     #     return self.product.get_discount()
    #     # else:
    #     #     return self.service.get_discount()
    #
    #     return 0

#Nomenclators

class N_Proposal_Status(models.Model, ModelSerialize):
    EDITTING = 1
    PENDING = 2
    ACCEPTED = 3
    DENIED = 4
    CANCELLED = 5
    default_fields = ['name','id']
    name = models.CharField(max_length=25)

    def __str__(self):
        return self.name

    class Meta:
        app_label = 'base'

class N_Invoice_Status(models.Model, ModelSerialize):
    # CREATED = 1
    EDITTING = 2
    PENDING = 3
    PAID = 4
    PAST_DUE = 7
    # CANCELLED = 5
    # ACCEPTED = 6
    default_fields = ['name','id']
    name = models.CharField(max_length=25)

    def __str__(self):
        return self.name

    class Meta:
        app_label = 'base'

class N_Event_Status(models.Model, ModelSerialize):

    PENDING = 1
    SCHEDULED = 2
    DONE = 3
    CANCELLED = 4
    ACCEPTED = 5
    default_fields = ['name', 'id']
    name = models.CharField(max_length=25)

    def __str__(self):
        return self.name

    class Meta:
        app_label = 'base'

class N_Request_Type(models.Model, ModelSerialize):

    PRODUCT = 1
    SERVICE = 2
    default_fields = ['id','name']
    name = models.CharField(max_length=25)

    def __str__(self):
        return self.name

    class Meta:
        app_label = 'base'


# -*- coding: utf-8 -*-
""" This BuildingBlock class describes the organisation of a single
cortical layer.
"""
# @Author: mmilde
# @Date:   2020-04-01 17:36:37

import sys
import time
import numpy as np

from brian2 import ms, mV, pA, SpikeGeneratorGroup,\
    SpikeMonitor, StateMonitor, core

from teili.core.groups import Neurons, Connections
from teili.building_blocks.building_block import BuildingBlock, InhibitorySubnetwork
from teili.models.neuron_models import ExpAdaptLIF as neuron_model
from teili.models.synapse_models import AlphaStdp as synapse_model
from teili.models.synapse_models import Alpha as static_synapse_model

layer_params={
    'we_inp_exc': 1.5,
    'we_exc_exc': 0.5,
    'we_exc_inh': 1,
    'wi_inh_inh': -1,
    'wi_inh_exc': -1,
    'rp_exc': 1 * ms,
    'rp_inh': 1 * ms,
    'num_neurons': 64,
    'num_inh_neurons': None,
    'num_input_neurons': None,
    'num_inputs': 3,
    'num_inh_inputs': 3,
    'ee_connection_probability': 0.7,
    'ei_connection_probability': 0.7,
    'ie_connection_probability': 0.7,
    'ii_connection_probability': 0.1,
    'connection_dropoff': 'exponential',
    }

class CorticalLayer(BuildingBlock, InhibitorySubnetwork):
    """ This class describes the connectivity between primary excitatory
    neurons, such as pyramidal cells, and a inhibitory sub-network consisting
    of models of PV- and SST-positive interneurons.
    """

    def __init__(self,
                 name='layer*',
                 neuron_eq_builder=neuron_model,
                 synapse_eq_builder=synapse_model,
                 static_synapse_eq_builder=static_synapse_model,
                 block_params=layer_params,
                 monitor=True,
                 verbose=False):
        """ This function initialises the CorticalLayer building block.

        Args:
            name (str, optional): Name of the cortical layer
            neuron_eq_builder (class, optional): neuron class as imported from
                models/neuron_models.
            synapse_eq_builder (class, optional): synapse class as imported from
                models/synapse_models used for all synapses, unless
                static_synapse_eq_builder is not `None`, in which case all
                non-plastic synapse follow the model provided.
            static_synapse_eq_builder (class, optional): synapse class as imported from
                models/synapse_models for all non-plastic synapses.
            num_neurons (int, required): Number of excitatory neurons within
                the layer.
            num_inputs (int, optional): Number of input currents to WTA.
            spatial_kernel (str, optional): Connectivity kernel for lateral
                connectivity. Default is 'kernel_gauss_1d'.
                See tools.synaptic_kernel for more detail.
        """
        BuildingBlock.__init__(self,
                               name,
                               neuron_eq_builder,
                               synapse_eq_builder,
                               block_params,
                               verbose,
                               monitor)

        if static_synapse_eq_builder is None:
            static_synapse_eq_builder = synapse_eq_builder
        self._groups,\
        self.monitors,\
        self.standalone_parameters = gen_cortical_layer(name,
                                                        neuron_eq_builder,
                                                        synapse_eq_builder,
                                                        static_synapse_eq_builder,
                                                        **block_params)

        self.input_group
        self.output_group

def gen_cortical_layer(layer_name,
                       neuron_eq_builder=neuron_model,
                       synapse_eq_builder=synapse_model,
                       static_synapse_eq_builder=static_synapse_model,
                       we_inp_exc=1.5,
                       we_exc_exc=0.5,
                       we_exc_inh=1,
                       wi_inh_inh=-1,
                       wi_inh_exc=-1,
                       rp_exc=1 * ms,
                       rp_inh=1 * ms,
                       num_neurons=64,
                       num_inh_neurons=None,
                       num_input_neurons=None,
                       num_inputs=3,
                       num_inh_inputs=3,
                       ee_connection_probability=0.7,
                       ei_connection_probability=0.7,
                       ie_connection_probability=0.7,
                       ii_connection_probability=0.1,
                       connection_dropoff='exponential',
                       additional_statevars=[],
                       monitor=True,
                       verbose=False):
    """ Combines InhibitorySubnetwork BuildingBloc with a primary excitatory
    neuron cell type such as pyramidal cells.

    Args:
        layer_name (str, required): Name of the cortical layer.

    Returns:
        _groups (dictionary): Keys to all neuron and synapse groups.
        monitors (dictionary): Keys to all spike and state monitors.
        standalone_params (dictionary): Dictionary which holds all
            parameters to create a standalone network.

    """
    # create the input SpikeGeneratorGroup
    ts = np.asarray([]) * ms
    ind = np.asarray([])
    spike_gen = SpikeGeneratorGroup(
        num_input_neurons, indices=ind, times=ts,
        name=layer_name+ '_' + 'spike_gen')

    # create neuron groups
    n_exc = Neurons(num_neurons,
                    equation_builder=neuron_eq_builder(
                        num_inputs=3+num_inputs),
                    refractory=rp_exc,
                    name=layer_name + '__' + 'n_exc')

    s_exc_pv = Connections(n_exc,
                           interneurons._groups['n_pv'],
                           equation_builder=static_synapse_eq_builder
                           )

    s_exc_pv.connect(p=ei_connection_probability)

    Groups = {}
    Monitors = {}
    standalone_parameters = {}

    return Groups, Monitors, standalone_parameters

def set_layer_tags():
    pass


from datetime import datetime, timedelta
from pathlib import Path
from airflow import DAG
from airflow.models import Variable
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators import (StageToRedshiftOperator, LoadFactOperator, LoadDimensionOperator, DataQualityOperator)

# specify DAG default arguments
default_args = {
    'owner': 'sparkify',
    'start_date': datetime(2019, 1, 12),
    'end_date': datetime(2019, 7, 23),
    'depends_on_past': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=5),
    'catchup': False,
    'email_on_retry': False,
    'aws_conn_id': 'aws_default',
    'postgres_conn_id': 'redshift_default',
    'redshift_conn_id': 'redshift_default',
    'params': {
        's3_bucket': 'udacity-dend',
        's3_region': 'us-west-2',
        's3_json_path': 's3://udacity-dend/log_json_path.json',
        'aws_iam_role': Variable.get("aws_iam_role")
    }
}

# define DAG
with DAG(dag_id="sparkify_dag",
         description="Load and transform data from S3 into Redshift",
         default_args=default_args,
         schedule_interval="@hourly",
         template_searchpath=str(Path(__file__).parent.parent.joinpath("sql"))) as dag:

    # define tasks
    start = DummyOperator(task_id='begin_execution')

    create_tables = PostgresOperator(
        task_id="create_tables",
        sql="create_tables.sql"
    )

    stage_events_to_redshift = StageToRedshiftOperator(
        task_id="stage_events",
        aws_iam_role="{{ params.aws_iam_role }}",
        s3_bucket="{{ params.s3_bucket }}",
        s3_key="log-data",
        s3_region="{{ params.s3_region }}",
        redshift_table="staging_events",
        json_path="{{ params.s3_json_path }}",
        truncate=False
    )

    stage_songs_to_redshift = StageToRedshiftOperator(
        task_id="stage_songs",
        aws_iam_role="{{ params.aws_iam_role }}",
        s3_bucket="{{ params.s3_bucket }}",
        s3_key="song-data",
        s3_region="{{ params.s3_region }}",
        redshift_table="staging_songs",
        json_path="auto",
        truncate=False
    )

    load_songplays_table = LoadFactOperator(
        task_id="load_songplay_fact_table",
        fact_table="songplay",
        fact_cols="playid, start_time, userid, level, songid, artistid, sessionid, location, user_agent",
        sql="songplay_insert.sql",
        truncate=False
    )

    load_user_dimension_table = LoadDimensionOperator(
        task_id='load_user_dim_table',
        dim_table="users",
        dim_cols="userid, first_name, last_name, gender, 'level'",
        sql="user_insert.sql",
        truncate=False
    )

    load_song_dimension_table = LoadDimensionOperator(
        task_id='load_song_dim_table',
        dim_table="songs",
        dim_cols="songid, title, artistid, 'year', duration",
        sql="song_insert.sql",
        truncate=False
    )

    load_artist_dimension_table = LoadDimensionOperator(
        task_id='load_artist_dim_table',
        dim_table="artists",
        dim_cols="artistid, name, location, latitude, longitude",
        sql="artist_insert.sql",
        truncate=False
    )

    load_time_dimension_table = LoadDimensionOperator(
        task_id='load_time_dim_table',
        dim_table="time",
        dim_cols="start_time, 'hour', 'day', week, 'month', 'year', weekday",
        sql="time_insert.sql",
        truncate=False
    )

    run_quality_checks = DataQualityOperator(
        task_id='run_data_quality_checks',
        check_tables=["songplay", "users", "songs", "artists", "time"],
        check_sql="has_rows"
    )

    end = DummyOperator(task_id='Stop_execution')

    # define task dependencies
    staging_tasks = [stage_events_to_redshift, stage_songs_to_redshift]
    load_dim_tables = [load_song_dimension_table, load_artist_dimension_table, load_user_dimension_table,
                       load_time_dimension_table]

    start >> create_tables >> staging_tasks >> load_songplays_table >> load_dim_tables >> run_quality_checks >> end


import requests
import time
import os
import json

import pandas as pd
from pandas.tseries.offsets import BDay
from datetime import datetime
import pytz

# import matplotlib.pyplot as plt
import mplfinance as mpl

import slack_sdk as slack

SLACK_TOKEN=os.environ['SLACK_TOKEN']
SLACK_CHANNEL=os.environ['SLACK_CHANNEL']
client = slack.WebClient(token=SLACK_TOKEN)

import flask

# yahoo
style = {'base_mpl_style': 'fast',
         'marketcolors'  : {'candle': {'up': '#579FEC', 'down': '#E28E60'},
                            'edge'  : {'up': '#579FEC', 'down': '#E28E60'},
                            'wick'  : {'up': '#579FEC', 'down': '#E28E60'},
                            'ohlc'  : {'up': '#579FEC', 'down': '#E28E60'},
                            'volume': {'up': '#579FEC', 'down': '#E28E60'},
                            'vcedge': {'up': '#579FEC', 'down': '#E28E60'},
         'vcdopcod'      : True,
         'alpha'         : 0.9},
         'mavcolors'     : None,
         'facecolor'     : '#fafafa',
         'gridcolor'     : '#d0d0d0',
         'gridstyle'     : '-',
         'y_on_right'    : False,
         'rc'            : {'axes.labelcolor': '#101010',
                            'axes.edgecolor' : 'f0f0f0',
                            'axes.grid.axis' : 'y',
                            'ytick.color'    : '#101010',
                            'xtick.color'    : '#101010',
                            'figure.titlesize': 'x-large',
                            'figure.titleweight':'semibold',
                           },
         'base_mpf_style': 'yahoo'}

app = flask.Flask(__name__)
@app.route('/', methods=['GET'])
def home():

    code = flask.request.args['code']

    # req data
    au_last = (datetime.now(tz=pytz.timezone('Australia/Melbourne')) - BDay(1)).date()
    # start, end = int(time.time()), int(time.time() - 60 * 1440 * 1) # -1d
    start = int(datetime(au_last.year, au_last.month, au_last.day, 10, 0, tzinfo=pytz.timezone('Australia/Melbourne')).timestamp())
    end = int(datetime.now(tz=pytz.timezone('Australia/Melbourne')).timestamp())

    url = f"https://au.advfn.com/common/javascript/tradingview/advfn/history?symbol=ASX%5E{code}&resolution=5&v=1&from={start}&to={end}"
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    r = requests.get(url, headers=headers)
    d = json.loads(r.content)
    d = pd.DataFrame(d)
    d['t'] = pd.to_datetime(d['t'], unit='s')
    d = d.set_index('t')
    d = d.rename(columns={
        'o': 'Open',
        'h': 'High',
        'l': 'Low',
        'c': 'Close',
        'v': 'Volume',
    })
    d.index = d.index.tz_localize('UTC').tz_convert('Australia/Sydney')
    d = d.sort_index()

    # make chart
    mpl.plot(
        d,
        type="candle",
        style=style,
        savefig='chart.png',
        xrotation=30,
        fontscale=0.8,
        volume=True,
        ylabel=code,
        ylabel_lower='',
        scale_padding={
            'left': 0.5,
            'top': 0.5,
            'bottom': 0.75,
            'right': 0.25
        },
        vlines=dict(
            vlines=[_ for _ in d.index if _.time().hour == 15 and _.time().minute == 55],
            alpha=0.5, colors='grey', linestyle='dashed'
        ),
    )

    img = open('chart.png'.format(code), 'rb').read()

    # post to slack
    r2 = client.files_upload(
        channels=SLACK_CHANNEL,
        filename=code,
        content=img
    )

    return r2

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080, debug=True)


""" Named Entity Recognition """

import sys
from datetime import datetime, timedelta
import spacy  # the spacy nets have been trained on OntoNotes 5.0spacy.prefer_gpu()
# spacy.prefer_gpu()
from spacy import displacy
from collections import Counter
import pandas as pd
from itertools import chain
import neuralcoref
from datetime import datetime
import time
# import spacy_dbpedia_spotlight
import numpy as np
from collections import namedtuple
from copy import deepcopy
import os
import requests

sys.path.append("src/")
import read_data

sys.path.append("src/visualization/")
import visualization.get_viz_data as get_viz_data
import visualization.matplotlib_viz as viz
import visualization.bar_chart_race as bar_chart_race

# custom adaption of spacy_dbpedia_spotlight
sys.path.append("src/spacy_dbpedia_spotlight/")
from spacy_dbpedia_spotlight import entity_linker
from spacy_dbpedia_spotlight import initialize

from sentiment import target_based_sentiment
from sentiment import general_sentiment

class NamedEntityRecognizer:
    def __init__(self, model_size, with_sentiment=False):
        assert model_size in ["sm", "lg"]
        self.nlp = spacy.load(f"en_core_web_{model_size}")  # "eng_core_web_lg" for better but slower results
        coref = neuralcoref.NeuralCoref(self.nlp.vocab, greedyness=0.4)
        self.nlp.add_pipe(coref, name='neuralcoref')

        self.nlp_pp = spacy.load(f"en_core_web_{model_size}")

        self.nlp_dbp = spacy.load(f"en_core_web_{model_size}", disable=["ner"])
        initialize.load('en', self.nlp_dbp)

        self.nlp_nr = spacy.load(f"en_core_web_{model_size}")

        if with_sentiment:
            # Load target based sentiment
            self.tsa = target_based_sentiment.TargetSentimentAnalyzer()

            # Load general based sentiment
            self.gsa = general_sentiment.GeneralSentimentAnalyzer()

    def spacy_preprocessing(self, df):
        # TODO build separate pipelines for dbpedia vs. standard NER
        # build param for _sm / _lg model

        # TODO do we need/want entity merger?
        # merge_ents = nlp.create_pipe("merge_entities'")
        # nlp.add_pipe(merge_ents)
        print("Starting NLP...\t", str(datetime.now()))
        print("------------------------")
        print(self.nlp.pipe_names)
        print("------------------------")
        # df["nlp"] = [i for i in nlp.pipe(df["body"], batch_size=10, n_threads=6)]  # didn't really speed up anything
        # res = pd.DataFrame(df["body"].apply(nlp))
        # res.columns = ["nlp"]

        df["nlp"] = df["body"].apply(self.nlp)
        return df

    def spacy_ner(self, df):
        """ Use spacy CNN NER """
        # TODO should we try some of https://github.com/explosion/spaCy/tree/2d249a9502bfc5d3d2111165672d964b1eebe35e/bin/wiki_entity_linking
        print("Starting NLP Coref\t", str(datetime.now()))
        print("------------------------")
        print("Spacy NER pipeline:", self.nlp_pp.pipe_names)
        print("------------------------")
        df["nlp_resolved"] = df["nlp"].apply(lambda x: self.nlp_pp(x._.coref_resolved))
        print("Completed NLP by...\t", str(datetime.now()))
        return df

    def dbpedia_ner(self, df):
        """ Use DBPedia matching """

        # TODO we might be throwing out a lot of stuff in entity_linker (spacy_dbpedia_spotlight), maybe go check?

        print("Starting NLP Coref", str(datetime.now()))
        print("------------------------")
        print("DBPedia NER pipeline:", self.nlp_dbp.pipe_names)
        print("------------------------")
        def inner(x):
            return self.nlp_dbp(x)

        df["nlp_resolved"] = df["nlp"].apply(lambda x: inner(x._.coref_resolved))
        # res = pd.DataFrame(df["nlp"].apply(lambda x: inner(x._.coref_resolved)))
        # res.columns = ["nlp_resolved"]
        # res.append(df[])
        print("Completed NLP by...\t", str(datetime.now()))
        return df

    def find_most_common_entities(self, df, nlp_doc_colname:str, entity_type:str, df_name):
        """ Find most common entity for each article """
        # TODO put a similarity threshold in here somewhere!
        def find_most_common_entity(article_raw):
            article = article_raw[nlp_doc_colname]
            article_length = len(article.text)
            if article is None:
                return tuple()

            all_items = []  # mostly for debugging
            entity_counts = {}
            replacement_candidate = np.zeros(len(article.text)) -1

            for x in article.ents:
                if x.text.isspace():
                    continue
                text = x.text
                url = x.label_.split(" ")[0]
                dbpedia_labels = x.label_.split(" ")[1]
                all_items.append((text, url, dbpedia_labels))

                # resolve common disambiguitions
                if text.lower() == "washington":  # in doubt, go for location
                    standard_ner_ents = {i.text.lower(): i.label_ for i in article_raw["nlp"].ents}
                    if "washington" in [i.lower() for i in standard_ner_ents.keys()]:
                        if standard_ner_ents["washington"] == "GPE":
                            dbpedia_labels = "Location_State"
                elif text.lower() == "trudeau":  # in doubt, go for Justin
                    dbpedia_labels = "Justin Trudeau"

                if entity_type in dbpedia_labels:
                    # append last bit of url to make sure we don't get duplicates
                    last_url_tag = url.rfind("/")
                    entity_name = url[last_url_tag+1:].replace("_", " ")
                    if entity_name in entity_counts.keys():
                        entity_counts[entity_name] += 1
                    else:
                        entity_counts[entity_name] = 1
                    # set replacement candidate to index of corresponding entity in dict
                    ent_index = list(entity_counts).index(entity_name)
                    surface_form_start = article[x.start].idx
                    if x.end >= len(article):
                        surface_form_end = article_length
                    else:
                        surface_form_end = article[x.end].idx
                        # in case there is whitespace before next token, subtract
                        if article.text[article[x.end].idx-1] == " ":
                            surface_form_end -= 1
                    # TODO: possible bug here. If we have two candidates with the same names we for some reason do not replace them correcly
                    np.put(replacement_candidate, range(surface_form_start, surface_form_end), ent_index)

            if len(entity_counts) == 0:
                #TODO: changed this from:
                # return tuple()
                return ("None", 0, "".join(list(deepcopy(article.text))))

            mce = max(entity_counts)
            mce_list = list(mce)
            mce_val = entity_counts[mce]
            mce_idx = list(entity_counts).index(mce)

            in_a_row = False
            end = None
            start = None
            doc_len = article_length
            # TODO this list stuff isn't very efficient, could prolly be improved using pre-allocated arrays
            ner_resolved = list(deepcopy(article.text))
            for index, rep_idx in enumerate(replacement_candidate[::-1]):
                if (rep_idx != mce_idx) and (not in_a_row):
                    continue
                elif (rep_idx == mce_idx) and (not in_a_row):  # we have a mention of our enity
                    # we found start of surface form
                    in_a_row = True
                    end = doc_len - index
                    start = doc_len - index - 1
                elif ((rep_idx != mce_idx) or (index==doc_len-1)) and (in_a_row):
                    # we found end of surface form, replace
                    if (index==doc_len-1):
                        start -= 1
                    ner_resolved = ner_resolved[:start] + mce_list + ner_resolved[end:]
                    start = None
                    end = None
                    in_a_row = False
                elif (rep_idx == mce_idx) and in_a_row:
                    # we are still in surface form
                    start -= 1
            return (mce, mce_val, "".join(ner_resolved))

        df[[df_name, df_name+"_num", "ner_resolved_"+df_name]] = pd.DataFrame.from_records(
            df.apply(lambda x: find_most_common_entity(x), axis=1))  # apply to each row
        # df.dropna(inplace=True)
        df.reset_index(drop=True, inplace=True)
        return df

    def count_most_frequent(self, group):
        # current approach: most common of the most common - should be ok, I think
        items = [[i[0]] * i[1] for i in group if not (i is None)]
        items = list(chain.from_iterable(items))
        most_common = Counter(items).most_common(1)
        return most_common[0][0]  # get only the phrase for now

    def get_general_sentiment(self, df):
        df["g_sent"] = df["body"].apply(lambda x: self.gsa.get_general_sentiment(x))

        return df

    def get_target_sentiments(self, df_pp, targets):
        for target in targets:
            df_pp["sents_" + target] = df_pp["ner_resolved_"+ target].apply(lambda x: list(self.nlp_nr(x, disable=["tokenizer","tagger","entity","ner"]).sents))

        def get_average_sentiment(sentences, target):
            sentiment_sum = 0
            count = 0

            trim_count = 0
            c_sentence_count = 0

            for sentence in sentences:
                sentiment_tulp = self.tsa.get_sentiment(sentence = str(sentence), target = str(target)) # Convert them to strings
                # The tuple contains (sentiment, sentence trimmed)
                sentiment = sentiment_tulp[0]
                if(sentiment is None):
                    # Nothing found in this sentence
                    continue
                else:
                    c_sentence_count += 1 #The sentence contains the target word
                    if(sentiment_tulp[1]):
                        # The sentence was trimmed
                        trim_count += 1

                    sentiment_sum += sentiment
                    count += 1

            if(count != 0):
                if(c_sentence_count == 0):
                    c_sentence_count = 1
                return (sentiment_sum/count, trim_count/c_sentence_count)
            else:
                return (0, trim_count)

        def get_covid_sentiment(sentences):
            sentiment_sum = 0
            count = 0

            trim_count = 0
            c_sentence_count = 0

            for sentence in sentences:
                sentence = str(sentence)
                sentence = sentence.lower()
                covid_list = ["corona", "covid"] #Note that if any substring contains this we will take that as target, thus these are not nescessary "coronavirus","covid-19", "covid19",

                parsed_sentence = self.nlp_nr(sentence, disable=["parser","tagger","entity","ner"])

                found_covid = False

                for token in parsed_sentence:
                    tt = str(token.text)
                    if(found_covid):
                        break # Stop da loop

                    for word in covid_list:
                        loc = tt.find(word)
                        if(loc == -1):
                            # try next
                            continue
                        else:
                            # get the sentiment of the target word
                            sentiment_tulp = self.tsa.get_sentiment(sentence = str(sentence), target = str(tt)) # Convert them to strings
                            # The tuple contains (sentiment, sentence trimmed)

                            sentiment = sentiment_tulp[0]

                            if(sentiment is None):
                                # Nothing found in this sentence
                                continue
                            else:
                                c_sentence_count += 1 #The sentence contains the target word
                                if(sentiment_tulp[1]):
                                    # The sentence was trimmed
                                    trim_count += 1

                                sentiment_sum += sentiment
                                count += 1

                            found_covid = True
                            break

            if(count != 0):
                if(c_sentence_count == 0):
                    c_sentence_count = 1
                return (sentiment_sum/count, trim_count/c_sentence_count)
            else:
                return (0, trim_count)

        # print(df_pp)

        # Get the average sentiment for each target
        for target in targets:
            df_pp[[target + "_sent",target + "_tls"]] = df_pp.apply(lambda x: pd.Series(get_average_sentiment(x["sents_"+target], x[target])), axis=1)

        df_pp[["c_sent","c_tls"]] = df_pp.apply(lambda x: pd.Series(get_covid_sentiment(x["sents_"+targets[0]])), axis=1)
        # Fill this with ones in order to be able to get the average in the end
        df_pp["c_count"] = 1
        return df_pp

    def show_problems(self, article, visualize=False):
        """
        show some of our current problems in sentiment analyses
        @param artprint(tsa.get_sentiment("asdf","asdf"))icle: result of spacy nlp applied to the body of an article
        """
        # run as:
        # article = df["nlp"][0]
        # sentence = "On March 26, Johnson revealed he had tested positive and that he had been dealing with symptoms since that date."
        # displacy.serve(nlp(sentence), style='dep', options = {'distance': 120})
        # NER.show_problems(article)
        # we need to further resolve entities
        item_labels = [(x.text, x.label_) for x in article.ents if not x.text.isspace()]
        print(item_labels)

        if visualize:
            # dependency parsing
            displacy.serve(article, style='dep', options = {'distance': 120})
            # entity recognition
            displacy.serve(article, style='ent')

    def load_preloaded(self, path):
        return pd.read_csv(path)

def run_and_save(start_date, end_date, articles_per_period=None, max_length=None, with_sentiments=False, debug=False, delta_d=1):
    c_date = start_date

    # Name
    run_name = "s_" + start_date.strftime("%d_%m_%Y") + "_e_" \
        + end_date.strftime("%d_%m_%Y") + "_app_" + str(articles_per_period) \
        + "_ml_" + str(max_length) + "_" + datetime.now().strftime("d_%d_%m_t_%H_%M")
    print(f"RUN NAME: {run_name}")
    if debug:
        run_name = "debug_run_" + datetime.now().strftime("d_%d_%m_t_%H_%M")

    folder_path = "data/"+run_name
    if not os.path.isdir(folder_path):
        os.mkdir(folder_path)
    else:
        print(folder_path)
        print("Already exists, stopping")
        return

    print("Start run from date: " + start_date.strftime("%d_%m_%Y") + " to date: " + end_date.strftime("%d_%m_%Y"))
    print("Articles per period: " + str(articles_per_period))
    print("Max Length per article: " + str(max_length))
    print("Delta day steps:",delta_d)

    start_time = time.process_time()

    NER = NamedEntityRecognizer(model_size="sm", with_sentiment=with_sentiments)  # model_size="lg")

    # Increase until we hit the last day
    while(c_date < end_date):
        try:

            print("Running day: " + str(c_date.strftime("%d_%m_%Y")) + " till " + str((c_date + timedelta(days=delta_d)).strftime("%d_%m_%Y")))
            print("Loading Data...\t", str(datetime.now()))
            if not debug:
                # df = read_data.get_body_df(
                #     start_date=c_date,
                #     end_date=c_date,
                #     articles_per_period=articles_per_period, #700,
                #     max_length=max_length
                # )
                df = read_data.get_representative_df(
                    start_date=c_date,
                    end_date=c_date+timedelta(days=delta_d),
                    max_length=max_length,
                    n_samples = articles_per_period*delta_d
                )

            if debug:

                df = pd.DataFrame({
                    "body": [
                    "The professor Yann LeCun hates it when his students have purple hair.",
                    "The professor Yann LeCun does not like it when his students have purple hair.",
                    "The professor Yann LeCun does likes it when his students have purple hair.",
                    "People in Greenland do not have the burden of the coronavirus, since they are isolated from the rest of the world they do not have to worry about the disease and all its negative implications.",
                    "People in Greenland have a great time, even though the coronavirus is terrible for the rest of the world they keep their positive vibe.",
                    "People in Greenland do have the burden of the coronavirus, since they are isolated from the rest of the world they do have to worry about the disease and all its negative implications."
                    ],
                    "publication_date": ["2020-04-05","2020-04-05","2020-04-05","2020-04-05","2020-04-05","2020-04-05"]
                })
                # df = pd.DataFrame({
                #     "body": [
                #     "New Zealand's two top public health bosses will be addressing New Zealand's to update New Zealand's on the latest Covid-19 developments. Director of Public Health Dr Caroline McElnay will join Director-General of Health Ashley Bloomfield...",
                #     "Deepika has a dog. She loves him. The movie star has always been fond of animals",
                #     "The short guy Donald Trump is the worst. He does not know how the world turns.",
                #     "The tall guy Donald Trump is the best.",
                #     "There were 23 more deaths linked to Covid-19 in the Netherlands, raising the total number of people who died in the country to 5,811. Public health agency RIVM said it also knew of another ten hospitalizations for the coronavirus disease.",
                #     "There were 23 more deaths linked to Covid-19 in the Netherlands, raising the total number of people who died in the country to 5,811 public health agency RIVM said it also knew of another ten hospitalizations for the coronavirus disease, there were 23 more deaths linked to Covid-19 in the Netherlands, raising the total number of people who died in the country to 5,811 public health agency RIVM said it also knew of another ten hospitalizations for the coronavirus disease.",
                #     "Coronavirus disease 2019 (COVID-19) is an infectious disease caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2).[10] It was first identified in December 2019 in Wuhan, China, and has since spread globally, resulting in an ongoing pandemic.[11][12] As of 24 May 2020, more than 5.35 million cases have been reported across 188 countries and territories, resulting in more than 343,000 deaths. More than 2.14 million people have recovered."],

                #     "publication_date": ["2020-04-05","2020-04-05","2020-04-05","2020-04-05","2020-04-05","2020-04-05","2020-04-05"]
                # })

            df = NER.spacy_preprocessing(df) # model_size="lg")
            if with_sentiments:
                df = NER.get_general_sentiment(df)
            # df.drop(columns=["body"], inplace=True) # Drop some columns to make some space
            df = NER.dbpedia_ner(df) #model_size="lg")
            df = NER.find_most_common_entities(df, "nlp_resolved", entity_type="Person", df_name =  "mc_p")  # entity "OfficeHolder" is quite nice, "Person" works as well
            df = NER.find_most_common_entities(df, "nlp_resolved", entity_type="Country", df_name =  "mc_c")

            df.drop(columns=["nlp"], inplace=True)

            if with_sentiments:
                targets = ["mc_p","mc_c"]
                df = NER.get_target_sentiments(df, targets = targets)

                for target in targets:
                    df.drop(columns=["sents_"+target], inplace=True)
                    df.drop(columns=["ner_resolved_"+target], inplace=True)

            # print(df.to_string())
            df.drop(columns=["nlp_resolved"], inplace=True) # Drop some columns to make some space

            # df.drop(columns=["nlp_resolved","ner_resolved"], inplace=True)
            # TODO check ner_resolved
            # print("aft ts",df.head())

            # # df = df[["publication_date", "most_common_1", "most_common_1_num", "t_sent", "c_sent"]]
            # df_most_common = NER.sum_period_most_common_entities(df)
            # print(df_most_common.head())

            print(df.head())
            if(delta_d == 1):
                file_name = c_date.strftime("%d_%m_%Y")
            else:
                file_name = c_date.strftime("%d_%m_%Y") + "_to_" +  (c_date + timedelta(days=delta_d)).strftime("%d_%m_%Y")
            df.to_csv(folder_path + "/" + file_name +".csv", index=False)
        except:
            print("Failed to do:",c_date.strftime("%d_%m_%Y"))
            print("Moving on")

        # Increase the day
        c_date += timedelta(days=delta_d)

    print("Multiple day run finished")
    elapsed_time = time.process_time() - start_time
    print("Elapsed time: " + str(round(elapsed_time,2)) + " seconds")

if __name__ == "__main__":

    if not os.path.isdir("src/experiments"):
        os.mkdir("src/experiments")

    if not os.path.isdir("src/logs"):
        os.mkdir("src/logs")

    # TODO: dates 2019-11-06 and 2020-01-01 throw errors
    # start_date=datetime.strptime("2019-11-01", "%Y-%m-%d")
    # end_date=datetime.strptime("2020-04-05", "%Y-%m-%d")

    # run_and_save(start_date, end_date, articles_per_period=None,     # df_most_common = get_viz_data.load_data("s_01_02_2020_e_05_04_2020_app_500_ml_300_d_26_05_t_15_20")

        # df_most_common = prepare_viz(df_most_common, mc_column="mc_p", mc_num_column="mc_p_num",
        #         sent_col="mc_p_sent", with_sentiment=True)
        # print(df_most_common.head())
        # start_date = df_most_common.publication_date.min()
        # end_date = df_most_common.publication_date.max()
        # visualize(df_most_common, start_date, end_date, "mc_p", "rolling_sent")

    start_date=datetime.strptime("2020-04-01", "%Y-%m-%d")
    end_date=datetime.strptime("2020-04-02", "%Y-%m-%d")

    run_and_save(start_date, end_date, articles_per_period=100,
         max_length=500, with_sentiments=True, debug=True, delta_d=1)

    # start_date=datetime.strptime("2020-03-01", "%Y-%m-%d")
    # end_date=datetime.strptime("2020-04-05", "%Y-%m-%d")


import matplotlib.pyplot as plt
import numpy as np
import inspect
import Algorithm as alg
qtd = [1000, 5000, 10000, 15000, 20000, 25000, 50000, 100000]
tipos = ['aleatorios', 'crescentes', 'decrescentes']
funcoes = inspect.getmembers(alg, inspect.isfunction)
qplot = 0
for t  in tipos:
    plt.figure(qplot)
    fileName = './Resultados/results_' + str(t) + '.txt'
    afile = open(fileName, "r")
    vetor = [x for x in afile.read().split("\n")]
    for i in range(0, len(funcoes)):
        vetorplot = []
        for j in range(0, len(funcoes)-1):
            print(vetor[i+j*len(funcoes)].split(" "))
            yval = float(vetor[i+j*len(funcoes)].split(" ")[2])
            vetorplot.append(yval)
            print(str(yval))
        plt.plot(qtd, vetorplot, linewidth=1.5,  marker='o')

    plt.legend([f[1].__name__ for f in funcoes], loc='best')
    plt.title("Dados:" + t)
    plt.grid(True)
    plt.xlabel("Tamanho da entrada : n")
    plt.ylabel("Tempo: s")
    plt.savefig('./Resultados/Graficos/' + t + '.png')
    # plt.show()
    qplot = qplot+1


# Version0
# 约定前序遍历
# TC：O(N), SC:O(N)
# 两种写法都可，若迭代中不能使用全局对象，应该使用类似链表的数据记录每一步更新且递回更新节点
# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

'''
class Codec:

    def serialize(self, root):
        """
        Encodes a tree to a single string.
        :type root: TreeNode
        :rtype: str
        """
        se_str = serializeStep(root, "")
        return se_str.strip(",")

    def deserialize(self, data):
        data = data.split(",")
        return deserializeStep(data)

def serializeStep(node, se_str):
    # 先序遍历，外加None,None确定叶结点
    if node == None:
        se_str += "None,"
    else:
        se_str += str(node.val) + ","
        se_str = serializeStep(node.left, se_str)
        se_str = serializeStep(node.right, se_str)

    return se_str

def deserializeStep(data):
    val = data.pop(0)
    if val == "None":
        return

    root = TreeNode(int(val))
    root.left = deserializeStep(data)
    root.right = deserializeStep(data)

    return root
   # v2
    def deserialize(self, data):
        """
        Decodes your encoded data to tree.
        :type data: str
        :rtype: TreeNode
        """
        data = data.split(",")
        if len(data) == 1:
            root = None
            return root

        root = TreeNode(0)  # init with 0 root
        root, data = deserializeStep(root, data)
        return root

# v2
def deserializeStep(node, se_list):
    # 同源的当层节点在当前函数中更新，所以只需传递序列数组就能重建子树
    # 函数返回更改后的序列数组
    # 当左子树为空时，返回的序列数组已将左子树元素弹出，重建当前节点右子树
    # 当左右子树都为空时，返回的序列数组已经将当前子节点树元素弹出，返回父节点重建其右子树
    # 最终返回给根节点的数组是空的
    val = se_list.pop(0)
    if val != "None":  # 只有非空才是节点对象
        node = TreeNode(int(val))

    else:
        node = None
        return node, se_list

    node.left, se_list = deserializeStep(node.left, se_list)
    node.right, se_list = deserializeStep(node.right, se_list)

    return node, se_list
'''

'''
# Version1
# LL(1)型文法编码
# TC: O(N), SC: O(N)
class Codec:
    def serialize(self, root):
        if root == None:
            return "X"

        l = "(" + serialize(root.left) + ")"
        r = "(" + serialize(root.right) + ")"
        return l + str(root.val) + r

    def deserialize(self, data):
        ptr = [0]
        return parse(data, ptr)

def parse(data, ptr):
    if data[ptr[0]] == "X":
'''

# Version2
# 队列 + BFS遍历
from collections import deque
class Codec:
    def serialize(self, root):
        if not root: return None
        q = deque()
        q.append(root)
        res = ""
        while q:
            node = q.popleft()
            if node != None:
                res += str(node.val) + ","
                q.append(node.left)
                q.append(node.right)
            else:
                res += "X,"
        return res

    def deserialize(self, data):
        if not data: return None
        data = data.split(",")
        root = TreeNode(data.pop(0))
        q = [root]
        while q:
            node = q.pop(0)
            if data:
                val = data.pop(0)
                if val != "X":
                    node.left = TreeNode(val)
                    q.append(node.left)
            if data:
                val = data.pop(0)
                if val != "X":
                    node.right = TreeNode(val)
                    q.append(node.right)
        return root


import logging
from pymeasure.instruments import Instrument, Channel

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

Attenuator_11dB = {
    0: (False, False, False, False),
    1: (True, False, False, False),
    2: (False, True, False, False),
    3: (True, True, False, False),
    4: (False, False, False, True),
    5: (True, False, False, True),
    6: (False, True, False, True),
    7: (True, True, False, True),
    8: (False, False, True, True),
    9: (True, False, True, True),
    10: (False, True, True, True),
    11: (True, True, True, True),
}
""" Mapping of logical values for use with 0 - 11 dB attenuators """

Attenuator_110dB = {
    0: (False, False, False, False),
    10: (True, False, False, False),
    20: (False, True, False, False),
    30: (True, True, False, False),
    40: (False, False, False, True),
    50: (True, False, False, True),
    60: (False, True, False, True),
    70: (True, True, False, True),
    80: (False, False, True, True),
    90: (True, False, True, True),
    100: (False, True, True, True),
    110: (True, True, True, True),
}
""" Mapping of logical values for use with 0 - 110 dB attenuators """

Attenuator_70dB_3_Section = {
    0: (False, False, False, False),
    10: (True, False, False, False),
    20: (False, True, False, False),
    30: (True, True, False, False),
    40: (False, False, True, False),
    50: (True, False, True, False),
    60: (False, True, True, False),
    70: (True, True, True, False),
}
""" Mapping of logical values for use with 0 - 70 dB attenuators with 3 switching sections """

Attenuator_70dB_4_Section = {
    0: (False, False, False, False),
    10: (True, False, False, False),
    20: (False, False, False, True),
    30: (True, False, False, True),
    40: (False, True, False, True),
    50: (True, True, False, True),
    60: (False, True, True, True),
    70: (True, True, True, True),
}
""" Mapping of logical values for use with 0 - 70 dB attenuators with 4 switching sections """

class SwitchDriverChannel(Channel):
    enabled = Instrument.setting(
        "%s{ch}",
        """
        Set this channel to the polarity 'A' for True and 'B' for False.
        """,
        map_values=True,
        values={True: "A", False: "B"}
    )

class HP11713A(Instrument):
    """
    Represents the HP 11713A Switch and Attenuator Driver and provides a high-level
    interface for interacting with the instrument.

    Usually an attenuator is hooked to either X or Y or X and Y. To ease the control
    of the attenuator driver you have the possibility to set an attenuator type via
    the attribute 'ATTENUATOR_X' or 'ATTENUATOR_Y'. The hp11713a keeps different default attenuator
    mappings. After setting the attenuator type you are able to use the methods 'attenuation_x'
    and/or 'attenuation_y' to set the switch driver to the correct value for the specified
    attenuation. The attenuation values are rounded.

    .. code-block:: python

        from pymeasure.instruments.hp import HP11713A
        from pymeasure.instruments.hp.hp11713a import Attenuator_110dB

        sd = HP11713A("GPIB::1")

        sd.ATTENUATOR_Y = Attenuator_110dB
        sd.attenuation_y(10)
        sd.ch_0.enabled = True

    """

    ATTENUATOR_X = {}
    ATTENUATOR_Y = {}

    channels = Instrument.MultiChannelCreator(SwitchDriverChannel, list(range(0, 9)))

    def __init__(self, adapter, name="Hewlett-Packard HP11713A",
                 **kwargs):
        super().__init__(
            adapter,
            name,
            includeSCPI=False,
            send_end=True,
            **kwargs,
        )

    def attenuation_x(self, attenuation):
        """ Set switches according to the attenuation in dB for X

        The set attenuation will be rounded to the next available step.
        An attenuation mapping has to be set in before e.g.

        .. code-block:: python

            from pymeasure.instruments.hp.hp11713a import HP11713A, Attenuator_110dB

            instr.ATTENUATOR_X = Attenuator_110dB
            instr.attenuation_x(10)
        """
        rounding = 0
        if list(self.ATTENUATOR_X.keys())[1] == 10:
            rounding = -1

        self.ch_1.enabled, self.ch_2.enabled, self.ch_3.enabled, self.ch_4.enabled = \
            self.ATTENUATOR_X[int(round(attenuation, rounding))]

    def attenuation_y(self, attenuation):
        """ Set switches according to the attenuation in dB for Y

        The set attenuation will be rounded to the next available step.
        An attenuation mapping has to be set in before e.g.

        .. code-block:: python

            from pymeasure.instruments.hp.hp11713a import HP11713A, Attenuator_110dB

            instr.ATTENUATOR_Y = Attenuator_110dB
            instr.attenuation_y(10)
        """
        rounding = 0
        if list(self.ATTENUATOR_Y.keys())[1] == 10:
            rounding = -1

        self.ch_5.enabled, self.ch_6.enabled, self.ch_7.enabled, self.ch_8.enabled = \
            self.ATTENUATOR_Y[int(round(attenuation, rounding))]

    def deactivate_all(self):
        """
        Deactivate all switches to polarity 'B'.
        """
        self.write("B1234567890")


import os
import angr
import nose

test_location = str(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../binaries/tests'))
arches = {'x86_64'}

def main():
    test_ddg_global_var_dependencies()

def test_ddg_global_var_dependencies():
    for arch in arches:
        run_ddg_global_var_dependencies(arch)

def run_ddg_global_var_dependencies(arch):
    test_file = os.path.join(test_location, arch, 'ddg_global_var_dependencies')
    proj = angr.Project(test_file, auto_load_libs=False)
    cfg = proj.analyses.CFGEmulated(context_sensitivity_level=2, keep_state=True, state_add_options=angr.sim_options.refs)
    ddg = proj.analyses.DDG(cfg)
    main_func = cfg.functions.function(name='main')

    target_block_addr = main_func.ret_sites[0].addr
    target_block = proj.factory.block(addr=target_block_addr)
    tgt_stmt_idx, tgt_stmt = get_target_stmt(proj, target_block)
    assert tgt_stmt_idx is not None
    buf_addr = tgt_stmt.data.addr.con.value
    tgt_ddg_node = get_ddg_node(ddg, target_block_addr, tgt_stmt_idx)
    assert tgt_ddg_node is not None

    # Whether the target depends on the statement assigning 'b' to the global variable
    has_correct_dependency = False
    for pred in ddg.get_predecessors(tgt_ddg_node):
        pred_block = proj.factory.block(addr=pred.block_addr)
        stmt = pred_block.vex.statements[pred.stmt_idx]
        has_correct_dependency |= check_dependency(stmt, buf_addr, ord('b'))

        # If the target depends on the statement assigning 'a' to the global variable, it is underconstrained (this assignment should be overwritten by the 'b' assignment)
        nose.tools.assert_false(check_dependency(stmt, buf_addr, ord('a')), msg="Target statement has incorrect dependency (DDG is underconstrained)")
    nose.tools.assert_true(has_correct_dependency, msg='Target statement does not have correct dependency (DDG is overconstrained)')

def check_dependency(stmt, addr, const):
    # Check if we are storing a constant to a variable with constant address
    if stmt.tag == 'Ist_Store' and stmt.addr.tag == 'Iex_Const' and stmt.data.tag == 'Iex_Const':
        # Check if we are storing the specified constant to the specified variable address
        if stmt.addr.con.value == addr and stmt.data.con.value == const:
            return True

    return False

def get_ddg_node(ddg, block_addr, stmt_idx):
    for node in ddg.graph.nodes:
        if node.block_addr == block_addr and node.stmt_idx == stmt_idx:
            return node
    return None

def get_target_stmt(proj, block):
    for i, stmt in enumerate(block.vex.statements):
        # We're looking for the instruction that loads a constant memory address into a temporary variable
        if stmt.tag == 'Ist_WrTmp' and stmt.data.tag == 'Iex_Load' and stmt.data.addr.tag == 'Iex_Const':
            addr = stmt.data.addr.con.value
            section = proj.loader.main_object.find_section_containing(addr)
            # Confirm the memory address is in the uninitialized data section
            if section.name == '.bss':
                return i, stmt
    return None, None

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Sep 21 15:52:46 2021

@author: Ema
Bilbio para la res de esta tarea Holton pag 888
"""

import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sig
import scipy.stats as stats
import matplotlib.patches as mpatches
from scipy.fft import *

#%% Set-UP
plt.close("all")
N = 1000 # muestras
fs= 1000 #Hz

ts = 1/fs # tiempo de muestreo

t=np.arange(0,1,ts)

K0=N/4
K1=N/4+0.025
K2=N/4+0.5
#%% Config de mi seno y vector de frecuencia
#senoidal de amplitud 1
amp=1 #amplitud en volts
freq0=K0*fs/N #frecuencia en Hz

fase=0 #radianes
df=fs/N
f=np.arange(0,fs,df)
bfrec = f <= fs/2
seno_cont0 = amp*np.sin(2 * np.pi * freq0 * t + fase)

#%%Config del seno 2
freq1=K1*fs/N #frecuencia en Hz
seno_cont1 = np.sqrt(amp)*np.sin(2 * np.pi * freq1 * t + fase)

#%%Config del seno 3
freq2=K2*fs/N #frecuencia en Hz
seno_cont2 = amp*np.sin(2 * np.pi * freq2 * t + fase)
#%% Energia de mi señal
#Este es el valor de la energia en el tiempo, se debe repetir en frecuencia
#por Parseval asique lo puedo usar para normalizar o comparar que este todo bien
#No olvidar el tema de la cantidad de muestras
E0=sum(np.abs(seno_cont0)**2)
E1=sum(np.abs(seno_cont1)**2)
E2=sum(np.abs(seno_cont2)**2)

#%% Calculo la densidad espectral de potencia
#Si integro estas funciones densisdad tendrian que ser iguales a sus correspondientes
#Energias en tiempo
DEP0=np.abs(fft(seno_cont0))**2
DEP1=np.abs(fft(seno_cont1))**2
DEP2=np.abs(fft(seno_cont2))**2

DEP0_int=sum(DEP0)/N
DEP1_int=sum(DEP1)/N
DEP2_int=sum(DEP2)/N
#%% Muestro las DEP o PSD power spectral density
plt.figure(1)

plt.plot(f, DEP0/(E0*N))
plt.title("DEP0")
plt.ylabel("Potencia[W]")
plt.xlabel("frec[Hz]")
plt.xlim(0, fs/2)

plt.figure(2)
plt.plot(f, DEP1/(E1*N))
plt.title("DEP1")
plt.ylabel("Potencia[W]")
plt.xlabel("frec[Hz]")
plt.xlim(0, fs/2)

plt.figure(3)

plt.plot(f, DEP2/(E2*N))
plt.title("DEP2")
plt.ylabel("Potencia[W]")
plt.xlabel("frec[Hz]")
plt.xlim(0, fs/2)
##Al visualizar las 3 DEP puedo ver que ese pequeño desfasaje en la delta f
##Hace que mi potencia ya no se concentre en el un punto sino que se desparrama
##A otras frecuencias vecinas
#%% Config de la window
window = sig.windows.boxcar(len(t))

####Verificacion visual de la window en t
# plt.figure(0)
# plt.plot(t,window)
# plt.title("Boxcar window")
# plt.ylabel("Amplitud")
# plt.xlabel("Tiempo")

#%%Preparo las salidas para despues plotear
    #%%caso 1 no se distingue la sincoide

# fft_w=np.abs(fft(window)/len(window))
# freqs_1 = fftfreq(fft_w.size, 0.1)
# plt.plot(freqs_1, fft_w)
# plt.xlim(0, 0.5)

    #%%caso 2 se distingue la sincoide
# w_fft = fft(window, n=5000)
# freqs = fftfreq(w_fft.size, 0.1)
# ##Normalizo por su valor pico
# w_fft_db=20*np.log10(np.abs(w_fft) / np.abs(w_fft).max())
# plt.figure(1)
# plt.plot(freqs, w_fft_db,'--r')
# plt.xlim(0, 0.1)
# plt.ylim(-30,0)

# #%%fft del seno
# fft_x=np.abs(fft(seno_cont0)/len(seno_cont0))

#%%convol
# convol=sig.convolve(fft_x,fft_w)
#%%Plots

# plt.figure(1)

# plt.plot(f, 20*np.log10(np.abs(fft_w) / np.abs(fft_w).max()))
# plt.title("Boxcar window fft")
# plt.ylabel("Amplitud")
# plt.xlabel("frec")

# plt.figure(2)

# plt.plot(f[0:500],fft_x[0:500])
# plt.title("seno fft")
# plt.ylabel("Amplitud")
# plt.xlabel("frec")