from Managers import HeaderManager, MongoManager, ProxyManager
from Config import Config
import requests
from requests.exceptions import RequestException
from pyquery import PyQuery as pq
import urllib3
from urllib3.exceptions import InsecureRequestWarning
import re
import datetime
import time
import json

class Scheduler(object):

    def __init__(self, src_col, dest_result_col, tag):
        """
        初始化
        :param src_col:
        :param dest_result_col:
        """
        self.src_col = src_col
        self.dest_col = dest_result_col
        self.tag = tag
        self.mm = MongoManager.DBManager()
        self.pm = ProxyManager.Proxy()
        self.hm = HeaderManager.Headers()
        # self.neo = Neo4j.Neo4j()

    def import_data_from_txt(self, path, dest):
        """
        从txt文件中获取所需数据并存入目标数据库
        :param path:
        :param dest:
        :return:
        """
        f = open(path, 'r', encoding='ISO-8859-1')
        try:
            # 生成迭代器，初始化
            iter_f = iter(f)
            cur_url = ''
            old_url = ''
            count = 0 #记录当前循环次数
            result = []
            userID = ''
            profileName = ''
            helpfulness = ''
            score = ''
            time = ''
            summary = ''
            text = ''

            # 遍历存储全部电影id
            for data in iter_f:
                # if count > 50:
                #     break
                # 若当前行是目标行则执行数据库存储
                if data.find('product/productId') >= 0:
                    # 找到一个新的电影id，循环数加一
                    count += 1
                    cur_url = 'https://www.amazon.com/dp/' + data.replace('product/productId: ', '').strip('\n') + '/'
                    if count is 1:
                        old_url = cur_url
                        continue
                    data = {
                        'userID': userID,
                        'profileName': profileName,
                        'helpfulness': helpfulness,
                        'score': score,
                        'time': time,
                        'summary': summary,
                        'text': text
                    }
                    result.append(data)
                    # 若当前电影id与上一个不同，则应将历史数据打包存入数据库
                    if cur_url != old_url:
                        print('')
                        print('当前电影Id为', cur_url, '上一部电影id为', old_url, '，存储上一部电影的评论结果数据')
                        # 将result存入数据库
                        # for item in result:
                        #     print(item)
                        self.mm.update_attr(self.src_col, old_url, 'reviews', result)
                        result.clear()
                    # 重置旧url
                    old_url = cur_url

                elif data.find('review/userId') >= 0:
                    userID = data.replace('review/userId: ', '').strip('\n')
                elif data.find('review/profileName') >= 0:
                    profileName = data.replace('review/profileName: ', '').strip('\n')
                elif data.find('review/helpfulness') >= 0:
                    helpfulness = data.replace('review/helpfulness: ', '').strip('\n')
                elif data.find('review/score') >= 0:
                    score = data.replace('review/score: ', '').strip('\n')
                elif data.find('review/time') >= 0:
                    time = data.replace('review/time: ', '').strip('\n')
                elif data.find('review/summary') >= 0:
                    summary = data.replace('review/summary: ', '').strip('\n')
                elif data.find('review/text') >= 0:
                    text = data.replace('review/text: ', '').strip('\n')
            # 最后一组数据，循环结束后存入数据库
            data = {
                'userID': userID,
                'profileName': profileName,
                'helpfulness': helpfulness,
                'score': score,
                'time': time,
                'summary': summary,
                'text': text
            }
            result.append(data)
            self.mm.update_attr(self.src_col, old_url, 'reviews', result)
        finally:
            if f:
                # 关闭文件
                f.close()

    def get_page_detail(self, url, headers=None, proxy=''):
        """
        获取详情页信息
        :param url:
        :param headers:
        :return:
        """
        # 禁用安全请求警告
        urllib3.disable_warnings(InsecureRequestWarning)

        response = requests.get(url, headers=headers, proxies=proxy, verify=False)
        try:
            if response.status_code == 200:
                return response.text
            return None
        except RequestException:
            print('请求详情页失败', url)
            return None

    def parse_page(self, url, html, id):
        """
        解析网页信息，存入数据库
        :param url:
        :param html:
        :param id:
        :return bool:如果出现robot则返回False，否则正常页面存入数据库，返回True
        """
        doc = pq(html)
        rated = ''
        type = 'U'

        # 根据影片名CSS选择器类型判断当前页面黑白类型，并存储影片分级
        movie_name = doc('#a-page > div.avu-content.avu-section > div > div > section > h1').text()
        if movie_name != '':
            # print("黑色页面")
            rated = doc(
                '#a-page > div.avu-content.avu-section > div.av-dp-container > div.avu-page-container.avu-clearfix'
                ' > section.av-detail-section > div.av-badges > span.av-badge.av-badge-text').text()
            type = 'B'

        else:
            movie_name = doc('#productTitle').text()
            if movie_name != '':
                # print("白色页面")
                rated = doc('#bylineInfo > div > div > span.a-size-small').text()
                type = 'W'

        # 有明确分级，确定当前产品为电影，直接存入结果数据库
        if rated == 'R' or rated == 'G' or rated == 'PG' or rated == 'PG-13' or rated == 'NC-17':
            print("电影等级为", rated, "，电影名为", movie_name, "，存入结果数据库")
            data = {
                'url': url,
                'movieName': movie_name,
                'type': type
            }
            self.mm.save_data_to_dest(data)

        # 没有明确分级, 存入临时数据库
        else:
            # 获取页面详细信息并存储
            data = []
            if type == 'W':
                details = doc('#detail-bullets .bucket .content li').items()
                if details != '':
                    for item in details:
                        detail = item.text()
                        data.append(detail)

            if type == 'B':
                details = doc('.avu-page-section .a-keyvalue tr').items()
                if details != '':
                    for item in details:
                        detail = item.text()
                        data.append(detail)

            if type == 'U':
                print("##############出现其他页面, 存入临时数据库############")
                data.append({'url':url})
                data.append({'html':html})

            # 确定机器人检查
            page_title = doc('head > title').text()
            if page_title == 'Robot Check':
                print("-------------- Robot -----------------")
                return False

            # 处理结束，存入临时结果数据库
            """
                        if self.mm.save_data_to_temp(data) and self.mm.delete_url(id, self.src_col) and self.mm.delete_url(id, Config.MOVIE_TABLE):
                return True
            """

    def save_page_detail(self, url, id, html, page_type):
        """
        根据html获取网页所需属性并存入dest数据库，同时删去src数据库中的对应数据
        :param html:
        :param page_type:
        :return: 0表示操作完成，1表示遇到机器人检查，2表示出现不存在的页面类型，3表示数据更新操作出错，4表示数据删除失败
        """
        doc = pq(html)
        rating = ''
        actors = ''
        supporting_actors = ''
        directors = ''
        producers = ''
        writers = ''
        format = ''
        genres = ''
        duration = ''
        price = ''
        studio = ''
        release_time = ''
        asin = ''
        star = ''

        # 确定机器人检查
        page_title = doc('head > title').text()
        if page_title == 'Robot Check':
            print("-------------- Robot -----------------")
            return 1

        if page_type != 'W' and page_type != 'B' :
            print("error : 出现不存在的页面类型")
            return 2

        # 根据影片页面类型获取对应属性值
        if page_type == 'W':
            # print("白色页面")
            rating = doc('#bylineInfo > div > div > span.a-size-small').text()
            # print("影片等级为", rating)
            price = doc('#buyNewSection > h5 > div > div.a-column.a-span8.a-text-right.a-span-last > div > span.a-size-medium.a-color-price.offer-price.a-text-normal').text()
            if price == '':
                price = doc('#unqualifiedBuyBox > div > div.a-text-center.a-spacing-mini > span').text()
            # print("影片价格为", price)
            star = doc('#reviewsMedley > div > div.a-fixed-left-grid-col.a-col-left > div.a-section.a-spacing-none.a-spacing-top-mini > div > div > div.a-fixed-left-grid-col.a-col-right > div > span > span > a > span').text()
            # print("评级为", star)
            details = doc('#detail-bullets .bucket .content li').items()
            if details != '':
                for item in details:
                    detail = item.text()
                    # print(detail)
                    if 'Actors' in detail:
                        actors = detail.strip('Actors').strip('\n').strip(':')
                        # print("演员为", actors)
                    if 'Directors' in detail:
                        directors = detail.strip('Directors').strip('\n').strip(':')
                        # print("导演为", directors)
                    if 'Producers' in detail:
                        producers = detail.strip('Producers').strip('\n').strip(':')
                        # print("出品人为", producers)
                    if 'Writers' in detail:
                        writers = detail.strip('Writers').strip('\n').strip(':')
                        # print("原著作者为", writers)
                    if 'Format' in detail:
                        format = detail.strip('Format').strip('\n').strip(':')
                        # print("影片格式为", format)
                    if 'VHS Release Date' in detail:
                        release_time = detail.strip('VHS Release Date').strip('\n').strip(':')
                        # print("上映时间为", release_time)
                    if 'Run Time' in detail:
                        duration = detail.strip('Run Time').strip('\n').strip(':')
                        # print("影片时长为", duration)
                    if 'Studio' in detail:
                        studio = detail.strip('Studio').strip('\n').strip(':')
                        # print("制作公司为", studio)
                    if 'ASIN' in detail:
                        asin = detail.strip('ASIN').strip('\n').strip(':')
                        # print("ASIN为", asin)

        if page_type == 'B':
            # print("黑色页面")
            rating = doc(
                '#a-page > div.avu-content.avu-section > div.av-dp-container > div.avu-page-container.avu-clearfix'
                ' > section.av-detail-section > div.av-badges > span.av-badge.av-badge-text').text()
            # print("影片等级为", rating)
            price = doc('#dv-action-box > div > div.av-action-button-box.avu-full-width-phablet.av-spacer > form:nth-child(2) > button').text().strip('Buy ').strip('HD').strip('SD').strip('\n')
            # print("影片价格为", price)
            release_time = doc('#a-page > div.avu-content.avu-section > div > div > section > div.av-badges > span:nth-child(3)').text()
            # print("上映时间为", release_time)
            duration = doc('#a-page > div.avu-content.avu-section > div > div > section > div.av-badges > span:nth-child(2)').text()
            # print("影片时长为", duration)
            asin = doc('#dv-action-box > div > div.av-action-button-box.avu-full-width-phablet.av-spacer > form:nth-child(1) > input[type="hidden"]:nth-child(4)').val()
            # print("asin为", asin)
            star = doc('#reviewsMedley > div > div.a-fixed-left-grid-col.a-col-left > div.a-section.a-spacing-none.a-spacing-top-mini > div > div > div.a-fixed-left-grid-col.a-col-right > div > span > span > a > span').text()
            # print("评级为", star)
            details = doc('.avu-page-section .a-keyvalue tr').items()
            if details != '':
                for item in details:
                    detail = item.text()
                    # print(detail)
                    if 'Starring' in detail:
                        actors = detail.strip('Starring').strip('\n')
                        # print("演员为", actors)
                    if 'Supporting actors' in detail:
                        supporting_actors = detail.strip('Supporting actors').strip('\n')
                        # print("助演演员为", supporting_actors)
                    if 'Director' in detail:
                        directors = detail.strip('Director').strip('\n')
                        # print("导演为", directors)
                    if 'Producers' in detail:
                        producers = detail.strip('Producers').strip('\n')
                        # print("出品人为", producers)
                    if 'Writers' in detail:
                        writers = detail.strip('Writers').strip('\n')
                        # print("原著作者为", writers)
                    if 'Genres' in detail:
                        genres = detail.strip('Genres').strip('\n')
                        # print("影片类型为", genres)
                    if 'Format' in detail:
                        format = detail.strip('Format').strip('\n')
                        # print("影片格式为", format)
                    if 'Studio' in detail:
                        studio = detail.strip('Studio').strip('\n')
                        # print("制作公司为", studio)

        if rating != '':
            if self.mm.update_attr(self.dest_col, url, 'rating', rating) is False:
                return 3
        if price != '':
            if self.mm.update_attr(self.dest_col, url, 'price', price) is False:
                return 3
        if actors != '':
            if self.mm.update_attr(self.dest_col, url, 'actors', actors) is False:
                return 3
        if supporting_actors != '':
            if self.mm.update_attr(self.dest_col, url, 'supportingActors', supporting_actors) is False:
                return 3
        if directors != '':
            if self.mm.update_attr(self.dest_col, url, 'directors', directors) is False:
                return 3
        if producers != '':
            if self.mm.update_attr(self.dest_col, url, 'producers', producers) is False:
                return 3
        if writers != '':
            if self.mm.update_attr(self.dest_col, url, 'writers', writers) is False:
                return 3
        if format != '':
            if self.mm.update_attr(self.dest_col, url, 'format', format) is False:
                return 3
        if genres != '':
            if self.mm.update_attr(self.dest_col, url, 'genres', genres) is False:
                return 3
        if duration != '':
            if self.mm.update_attr(self.dest_col, url, 'duration', duration) is False:
                return 3
        if studio != '':
            if self.mm.update_attr(self.dest_col, url, 'studio', studio) is False:
                return 3
        if release_time != '':
            if self.mm.update_attr(self.dest_col, url, 'releaseTime', release_time) is False:
                return 3
        if asin != '':
            if self.mm.update_attr(self.dest_col, url, 'asin', asin) is False:
                return 3
        if star != '':
            if self.mm.update_attr(self.dest_col, url, 'star', star) is False:
                return 3

        if self.mm.delete_data(self.src_col, id, url) is False:
            print("删除数据出错")
            return 4

        return 0

    def sort_result_db(self):
        """
        把数据库按序排列
        :return:
        """
        print("开始排序")
        datas = self.mm.sort("movieName", 1)
        for data in datas:
            self.mm.save_data_to_dest(data)

    def bucket_processing(self):
        """
        将所有数据根据首字母分桶，方便后期搜索
        :return:
        """
        datas = self.mm.get_all_data_from_src()
        for data in datas:
            movie_name = data['movieName']
            mn = movie_name.lower()
            i = 0
            if re.match('a.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入a桶，i = ", i)
                continue
            i = i + 1
            if re.match('b.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入b桶，i = ", i)
                continue
            i = i + 1
            if re.match('c.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入c桶，i = ", i)
                continue
            i = i + 1
            if re.match('d.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入d桶，i = ", i)
                continue
            i = i + 1
            if re.match('e.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入e桶，i = ", i)
                continue
            i = i + 1
            if re.match('f.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入f桶，i = ", i)
                continue
            i = i + 1
            if re.match('g.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入g桶，i = ", i)
                continue
            i = i + 1
            if re.match('h.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入h桶，i = ", i)
                continue
            i = i + 1
            if re.match('i.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入i桶，i = ", i)
                continue
            i = i + 1
            if re.match('j.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入j桶，i = ", i)
                continue
            i = i + 1
            if re.match('k.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入k桶，i = ", i)
                continue
            i = i + 1
            if re.match('l.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入l桶，i = ", i)
                continue
            i = i + 1
            if re.match('m.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入m桶，i = ", i)
                continue
            i = i + 1
            if re.match('n.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入n桶，i = ", i)
                continue
            i = i + 1
            if re.match('o.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入o桶，i = ", i)
                continue
            i = i + 1
            if re.match('p.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入p桶，i = ", i)
                continue
            i = i + 1
            if re.match('q.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入q桶，i = ", i)
                continue
            i = i + 1
            if re.match('r.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入r桶，i = ", i)
                continue
            i = i + 1
            if re.match('s.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入s桶，i = ", i)
                continue
            i = i + 1
            if re.match('t.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入t桶，i = ", i)
                continue
            i = i + 1
            if re.match('u.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入u桶，i = ", i)
                continue
            i = i + 1
            if re.match('v.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入v桶，i = ", i)
                continue
            i = i + 1
            if re.match('w.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入w桶，i = ", i)
                continue
            i = i + 1
            if re.match('x.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入x桶，i = ", i)
                continue
            i = i + 1
            if re.match('y.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入y桶，i = ", i)
                continue
            i = i + 1
            if re.match('z.*', mn):
                self.mm.save_data(data, Config.TABLES[i])
                print("存入z桶，i = ", i)
                continue
            i = i + 1
            self.mm.save_data(data, Config.TABLES[i])
            print("存入#桶，i = ", i)

    def data_cleaning(self):
        """
        数据清洗
        :return:
        """
        datas = self.mm.get_all_data_from_src()
        formal_name = ''
        formal_url = ''
        formal_id = ''
        count = 0
        for data in datas:

            # 将电影名去掉空格、特殊符号，替换为小写，替换特殊字符
            current_name = data['movieName']
            cn = current_name.replace(' I ', '1').replace('II', '2').replace('III', '3').lower().replace('the', '').replace('and', '&').replace('[', '').replace(']', '').replace('/', '').replace('-', '').replace('~', '').replace('(', '').replace(')', '').replace(':', '').replace('\'', '').replace('vs.', 'versus').replace('.', '').replace(',', '').replace('!', '').replace('?', '')

            """
            fn = formal_name.replace(' ', '').lower().replace('and', '&').replace('[', '').replace(']', '').replace('/', '').replace('-', '').replace('~', '').replace('(', '').replace(')', '').replace(':', '')
            result = cn.replace(fn, '')
            if result == 'vhs':
                print("VHS")
                print(formal_name)
                print(current_name)
                self.mm.delete_data(data['_id'], current_name, Config.MOVIE_TABLE)
                count = count + 1
            formal_name = current_name
            formal_url = data['url']
            formal_id = data['_id']
            """

            result = re.match('.*(tv ).*', cn)
            if result is not None:
                print("匹配到 baby lesson")
                print(data['url'], current_name)
                #self.mm.delete_data(data['_id'], data['movieName'], Config.MOVIE_TABLE)
                count = count + 1

                """
                result = self.mm.delete_data_ignor_case(cn.replace('vhs', '').replace(' ', ''), data['url'])
                if result is not None:
                    #print(data['url'], data['movieName'])
                    print("目标电影：", result['url'], result['movieName'])
                    print("当前电影：", data['url'], data['movieName'])
                    self.mm.delete_data(data['_id'], data['movieName'], Config.MOVIE_TABLE)
                    print('############')
                    count = count + 1
                """

        print('')
        print("删除数据总数", count)

    def write_to_file(self, path, content):
        """
        将参数中的结果写入txt文件
        :return:
        """
        with open(path, 'a', encoding='utf-8') as f:
            # 清除txt文件
            # f.truncate()
            f.write(json.dumps(content, ensure_ascii=False) + '\n')
            f.close()

    def write_result_to_txt(self):
        """
        将结果电影名存入txt
        :return:
        """
        datas = self.mm.get_data()
        for data in datas:
            print(data)
            self.write_to_file(data['movieName'])

    def split_arr(self, _data):
        """
        处理演员和导演名
        按照逗号分割
        去除空格，替换换行符
        :param _data:
        :return:
        """
        results = []
        datas = re.split(',', _data)
        for data in datas:
            data = data.strip(' ').replace('\n', ' ')
            results.append(data)
        return results

    def add_relation_count(self, tuples, director, actors):
        count = tuples.count()

        # 没有此导演
        if count == 0:
            # 所有合作演员全部存储
            for actor in actors:
                data = {
                    'director': director,
                    'actor': actor,
                    'count': 1
                }
                self.mm.save_data(self.dest_col, data)
            return 0
        # 有此导演
        for tuple in tuples:
            if tuple['actor'] in actors:
                actors.remove(tuple['actor'])
                self.mm.update_attr(self.dest_col, tuple['_id'], 'count', tuple['count'] + 1)
            # 剩余的演员是该导演未合作过的，插入新数据项
            # print(actors)
        for actor in actors:
            data = {
                'director': director,
                'actor': actor,
                'count': 1
            }
            self.mm.save_data(self.dest_col, data)
        return 1

    def save_cooperate(self):
        """
        使用mongodb
        将信息数据库中的导演与演员一一拆分
        存入cooperation_list数据库
        :return:
        """
        datas = self.mm.get_data(self.src_col)
        count = 1
        for data in datas:
            print(count, '#####')
            if 'actors' in data and 'directors' in data:
                directors = self.process_act_dir(data['directors'])
                actors = self.process_act_dir(data['actors'])
                for director in directors:
                    tuples = self.mm.find_data(self.dest_col, director)
                    self.add_relation_count(tuples, director, actors)
            else:
                print('no actor or no directors')
            count += 1

    def save_cooperate_by_neo(self):
        """
        使用neo4j数据库
        将mongodb数据库中的信息图形化存储
        :return:
        """
        datas = self.mm.get_data(self.src_col)
        count = 1
        for data in datas:
            print(count, '-----')
            # if count > 50:
            #     break

            # 该条数据有导演和演员信息，进行处理
            if 'actors' in data and 'directors' in data:
                # 数据处理，去掉空格，替换换行符
                directors = self.process_act_dir(data['directors'])
                actors = self.process_act_dir(data['actors'])
                # 对于数据中的全部导演，一一进行处理
                for director in directors:
                    # 图中没有找到导演节点，针对该节点，创建与每个导演的关系
                    if self.neo.find_director_node(director) is False:
                        for actor in actors:
                            self.neo.add_relation(director, actor)
                    # 图中找到导演节点
                    else:
                        # 针对每个演员进行处理
                        for actor in actors:
                            # 有导演节点，没有演员节点，直接创建关系
                            if self.neo.find_actor_node(actor) is False:
                                self.neo.add_relation(director, actor)
                            # 导演和演员节点同时存在，查找关系并合并到图中
                            else:
                                print('dir and act both exist')
                                self.neo.find_relation_and_add_count(director, actor)
            # 该条数据没有导演或演员信息，直接跳过
            else:
                print('no actor or no directors')
            count += 1

    def save_coop_result_to_mongo(self):
        """
        将neo4j中的数据结果存入mongodb
        :return:
        """
        datas = self.neo.get_cooperations()
        if datas is None:
            print('Error!')
            return False
        count = 1
        for data in datas:
            if count > 10:
                break
            print(data)
            # self.mm.save_data(self.dest_col, data)
        return True

    def schedule(self):
        """
        总调度函数，爬取网页信息并存储到mongodb
        :return:
        """
        # 从目标数据库获取源数据
        datas = self.mm.get_data(self.src_col)

        # 记录当前爬取次数
        count = 1

        # 获取对应申请头
        headers = self.hm.get_headers(self.tag)

        ## 获取一个可用代理
        proxy = self.pm.get_proxy()
        while proxy is None:
            print("1 代理请求失败")
            proxy = self.pm.get_proxy()

        for i in range(0, Config.SPIDER_TIME):

            data = datas[i]
            # 打印调试信息
            print('\nspidering :', data['url'], "，当前解析页数：", count, " ", self.src_col)

            # 当前需要解析网页
            while True:
                print("当前代理", proxy)
                html = self.get_page_detail(data['url'], headers, proxy)

                # 页面出现404，从src数据库删除并存入test_404数据库，跳出循环，结束本次url查询
                if html is None:
                    print("!!!!!!!!!!!!!!!!!!!!!!!页面出现404!!!!!!!!!!!!!!!!!!!!!!!！")
                    if self.mm.delete_data(self.src_col, data['_id'], data['url']) and self.mm.save_data(
                            Config.TEST_404, data):
                        print("删除404数据，存入404数据库")
                    break

                # 解析html，将需要的信息存入数据库，删除源数据
                result = self.save_page_detail(data['url'], data['_id'], html, data['type'])
                # 遇到机器人，需要更换代理继续
                if result == 1:
                    print("-----更换代理-----")
                    proxy = self.pm.get_proxy()
                    while proxy is None:
                        print("2 代理请求失败")
                        proxy = self.pm.get_proxy()
                # 出现特殊页面
                if result == 2:
                    print("!!!!!!!!!!!!!!!!!!!!!!!出现特殊页面!!!!!!!!!!!!!!!!!!!!!!!！")
                    if self.mm.delete_data(self.src_col, data['_id'], data['url']) and self.mm.save_data(
                            Config.TEST_404, data):
                        print("删除特殊页面数据，存入404数据库")
                    break
                # 数据库操作出错，跳过当前页面
                if result == 3 or result == 4:
                    print("数据库操作出错")
                    break
                # 操作成功，不需要继续解析网页，跳出循环
                else:
                    break

            count += 1
            time.sleep(Config.SLEEP_TIME)

    def cooperations_processing(self):
        datas = self.mm.get_data(Config.COOPERATION_LIST)
        count = 0
        time = 0
        for data in datas:
            director = data['director']
            actor = data['actor']
            # if len(director) < 2 or len(actor) < 2:
            #     print('director:', director, 'actor:', actor)
            #     count += 1
                # self.mm.delete_data(Config.COOPERATION_LIST, data['_id'])
            if data['count'] > 1:
                row = {
                    'director': data['director'],
                    'actor': data['actor'],
                    'count': data['count']
                }
                self.write_to_file('CooperationCount.txt', row)
                time += 1
        print('count:', count, 'time:', time)
        print('database:', self.mm.get_count(Config.COOPERATION_LIST))

    def split_processing(self):
        datas = self.mm.get_data(self.src_col)
        count = 1
        attrs = [
            'actors',
            'producers',
            'genres',
            'directors',
            'writers',
            'studio',
            'supportingActors',
            'format'
        ]
        for data in datas:
            print(count, '----------------')
            # if count > 1:
            #     break
            for attr in attrs:
                if attr in data:
                    if isinstance(data[attr], list):
                        continue
                    if data[attr] is None:
                        continue
                    result = self.split_arr(data[attr])
                    self.mm.update_attr(self.src_col, data['_id'], attr, result)
            count += 1

    def run(self):
        start = datetime.datetime.now()

        end = datetime.datetime.now()
        print("调度总时长：", end - start)


import re
from typing import Literal

import sqlalchemy as sa

from accounting import db
from accounting.models import Account, JournalEntryLineItem
from accounting.utils.options import options, Recurring

class DescriptionAccount:
    """An account for a description tag."""

    def __init__(self, account: Account, freq: int):
        """Constructs an account for a description tag.

        :param account: The account.
        :param freq: The frequency of the tag with the account.
        """
        self.__account: Account = account
        """The account."""
        self.id: int = account.id
        """The account ID."""
        self.code: str = account.code
        """The account code."""
        self.is_need_offset: bool = account.is_need_offset
        """Whether the journal entry line items of this account need offset."""
        self.freq: int = freq
        """The frequency of the tag with the account."""

    def __str__(self) -> str:
        """Returns the string representation of the account.

        :return: The string representation of the account.
        """
        return str(self.__account)

    @property
    def title(self) -> str:
        """Returns the account title.

        :return: The account title.
        """
        return self.__account.title

    def add_freq(self, freq: int) -> None:
        """Adds the frequency of an account.

        :param freq: The frequency of the tag name with the account.
        :return: None.
        """
        self.freq = self.freq + freq

class DescriptionTag:
    """A description tag."""

    def __init__(self, name: str):
        """Constructs a description tag.

        :param name: The tag name.
        """
        self.name: str = name
        """The tag name."""
        self.__account_dict: dict[int, DescriptionAccount] = {}
        """The accounts that come with the tag, in the order of their
        frequency."""
        self.freq: int = 0
        """The frequency of the tag."""

    def __str__(self) -> str:
        """Returns the string representation of the tag.

        :return: The string representation of the tag.
        """
        return self.name

    def add_account(self, account: Account, freq: int):
        """Adds an account.

        :param account: The associated account.
        :param freq: The frequency of the tag name with the account.
        :return: None.
        """
        self.__account_dict[account.id] = DescriptionAccount(account, freq)
        self.freq = self.freq + freq

    @property
    def accounts(self) -> list[DescriptionAccount]:
        """Returns the accounts by the order of their frequencies.

        :return: The accounts by the order of their frequencies.
        """
        return sorted(self.__account_dict.values(), key=lambda x: -x.freq)

    @property
    def account_codes(self) -> list[str]:
        """Returns the account codes by the order of their frequencies.

        :return: The account codes by the order of their frequencies.
        """
        return [x.code for x in self.accounts]

class DescriptionType:
    """A description type"""

    def __init__(self, type_id: Literal["general", "travel", "bus"]):
        """Constructs a description type.

        :param type_id: The type ID, either "general", "travel", or "bus".
        """
        self.id: Literal["general", "travel", "bus"] = type_id
        """The type ID."""
        self.__tag_dict: dict[str, DescriptionTag] = {}
        """A dictionary from the tag name to their corresponding tag."""

    def add_tag(self, name: str, account: Account, freq: int) -> None:
        """Adds a tag.

        :param name: The tag name.
        :param account: The associated account.
        :param freq: The frequency of the tag name with the account.
        :return: None.
        """
        if name not in self.__tag_dict:
            self.__tag_dict[name] = DescriptionTag(name)
        self.__tag_dict[name].add_account(account, freq)

    @property
    def tags(self) -> list[DescriptionTag]:
        """Returns the tags by the order of their frequencies.

        :return: The tags by the order of their frequencies.
        """
        return sorted(self.__tag_dict.values(), key=lambda x: -x.freq)

class DescriptionRecurring:
    """A recurring transaction."""

    def __init__(self, name: str, account: Account, description_template: str):
        """Constructs a recurring transaction.

        :param name: The name.
        :param description_template: The description template.
        :param account: The account.
        """
        self.name: str = name
        """The name."""
        self.account: DescriptionAccount = DescriptionAccount(account, 0)
        """The account."""
        self.description_template: str = description_template
        """The description template."""

    @property
    def account_codes(self) -> list[str]:
        """Returns the account codes by the order of their frequencies.

        :return: The account codes by the order of their frequencies.
        """
        return [self.account.code]

class DescriptionDebitCredit:
    """The description on debit or credit."""

    def __init__(self, debit_credit: Literal["debit", "credit"]):
        """Constructs the description on debit or credit.

        :param debit_credit: Either "debit" or "credit".
        """
        self.debit_credit: Literal["debit", "credit"] = debit_credit
        """Either debit or credit."""
        self.general: DescriptionType = DescriptionType("general")
        """The general tags."""
        self.travel: DescriptionType = DescriptionType("travel")
        """The travel tags."""
        self.bus: DescriptionType = DescriptionType("bus")
        """The bus tags."""
        self.__type_dict: dict[Literal["general", "travel", "bus"],
                               DescriptionType] \
            = {x.id: x for x in {self.general, self.travel, self.bus}}
        """A dictionary from the type ID to the corresponding tags."""
        self.recurring: list[DescriptionRecurring] = []
        """The recurring transactions."""

    def add_tag(self, tag_type: Literal["general", "travel", "bus"],
                name: str, account: Account, freq: int) -> None:
        """Adds a tag.

        :param tag_type: The tag type, either "general", "travel", or "bus".
        :param name: The name.
        :param account: The associated account.
        :param freq: The frequency of the tag name with the account.
        :return: None.
        """
        self.__type_dict[tag_type].add_tag(name, account, freq)

    @property
    def accounts(self) -> list[DescriptionAccount]:
        """Returns the suggested accounts of all tags in the description editor
        in debit or credit, in their frequency order.

        :return: The suggested accounts of all tags, in their frequency order.
        """
        accounts: dict[int, DescriptionAccount] = {}
        freq: dict[int, int] = {}
        for tag_type in self.__type_dict.values():
            for tag in tag_type.tags:
                for account in tag.accounts:
                    accounts[account.id] = account
                    if account.id not in freq:
                        freq[account.id] = 0
                    freq[account.id] \
                        = freq[account.id] + account.freq
        for recurring in self.recurring:
            accounts[recurring.account.id] = recurring.account
            if recurring.account.id not in freq:
                freq[recurring.account.id] = 0
        return [accounts[y] for y in sorted(freq.keys(),
                                            key=lambda x: -freq[x])]

class DescriptionEditor:
    """The description editor."""

    def __init__(self):
        """Constructs the description editor."""
        self.debit: DescriptionDebitCredit = DescriptionDebitCredit("debit")
        """The debit tags."""
        self.credit: DescriptionDebitCredit = DescriptionDebitCredit("credit")
        """The credit tags."""
        self.__init_tags()
        self.__init_recurring()

    def __init_tags(self):
        """Initializes the tags.

        :return: None.
        """
        debit_credit: sa.Label = sa.case(
            (JournalEntryLineItem.is_debit, "debit"),
            else_="credit").label("debit_credit")
        tag_type: sa.Label = sa.case(
            (JournalEntryLineItem.description.like("_%—_%—_%→_%"), "bus"),
            (sa.or_(JournalEntryLineItem.description.like("_%—_%→_%"),
                    JournalEntryLineItem.description.like("_%—_%↔_%")),
             "travel"),
            else_="general").label("tag_type")
        tag: sa.Label = get_prefix(JournalEntryLineItem.description, "—")\
            .label("tag")
        select: sa.Select = sa.Select(debit_credit, tag_type, tag,
                                      JournalEntryLineItem.account_id,
                                      sa.func.count().label("freq"))\
            .filter(JournalEntryLineItem.description.is_not(None),
                    JournalEntryLineItem.description.like("_%—_%"),
                    JournalEntryLineItem.original_line_item_id.is_(None))\
            .group_by(debit_credit, tag_type, tag,
                      JournalEntryLineItem.account_id)
        result: list[sa.Row] = db.session.execute(select).all()
        accounts: dict[int, Account] \
            = {x.id: x for x in Account.query
               .filter(Account.id.in_({x.account_id for x in result})).all()}
        debit_credit_dict: dict[Literal["debit", "credit"],
                                DescriptionDebitCredit] \
            = {x.debit_credit: x for x in {self.debit, self.credit}}
        for row in result:
            debit_credit_dict[row.debit_credit].add_tag(
                row.tag_type, row.tag, accounts[row.account_id], row.freq)

    def __init_recurring(self) -> None:
        """Initializes the recurring transactions.

        :return: None.
        """
        recurring: Recurring = options.recurring
        accounts: dict[str, Account] \
            = self.__get_accounts(recurring.codes)
        self.debit.recurring \
            = [DescriptionRecurring(x.name, accounts[x.account_code],
                                    x.description_template)
               for x in recurring.expenses]
        self.credit.recurring \
            = [DescriptionRecurring(x.name, accounts[x.account_code],
                                    x.description_template)
               for x in recurring.incomes]

    @staticmethod
    def __get_accounts(codes: set[str]) -> dict[str, Account]:
        """Finds and returns the accounts by codes.

        :param codes: The account codes.
        :return: The account.
        """
        if len(codes) == 0:
            return {}

        def get_condition(code0: str) -> sa.BinaryExpression:
            m: re.Match = re.match(r"^(\d{4})-(\d{3})$", code0)
            assert m is not None,\
                f"Malformed account code \"{code0}\" for regular transactions."
            return sa.and_(Account.base_code == m.group(1),
                           Account.no == int(m.group(2)))

        conditions: list[sa.BinaryExpression] \
            = [get_condition(x) for x in codes]
        accounts: dict[str, Account] \
            = {x.code: x for x in
               Account.query.filter(sa.or_(*conditions)).all()}
        for code in codes:
            assert code in accounts,\
                f"Unknown account \"{code}\" for regular transactions."
        return accounts

def get_prefix(string: str | sa.Column, separator: str | sa.Column) \
        -> sa.Function:
    """Returns the SQL function to find the prefix of a string.

    :param string: The string.
    :param separator: The separator.
    :return: The position of the substring, starting from 1.
    """
    return sa.func.substr(string, 0, get_position(string, separator))

def get_position(string: str | sa.Column, substring: str | sa.Column) \
        -> sa.Function:
    """Returns the SQL function to find the position of a substring.

    :param string: The string.
    :param substring: The substring.
    :return: The position of the substring, starting from 1.
    """
    if db.engine.name == "postgresql":
        return sa.func.strpos(string, substring)
    return sa.func.instr(string, substring)


"""
Basic parser for mls standings/schedule
6 September 2015

V 1.1a

"""
import sys
import urllib.request
import datetime

sys.path.append('/beautifulsoup4-4.1.0')

from bs4 import BeautifulSoup

# opens the url and parses it into beautifulsoup
def ParseStandingsData(website):
	# feed it the url
	url = urllib.request.urlopen(website)
	stuff = url.read()
	# feed it the data
	soup = BeautifulSoup(stuff)
	# find what we're looking for
	table = soup.find_all('table', {'class': 'views-table tablesorter standings-table'})
	# still not enough, dive deeper and pass it along
	return table[0].find_all('td', {'class': 'views-field'})

def ParseScheduleData(website):
	#feed it the url
	url = urllib.request.urlopen(website)
	stuff = url.read()
	soup = BeautifulSoup(stuff)
	table = soup.find_all('div', {'class': 'schedule-page'})
	return table

def ParseStatsData(website):
	#feed it a URL
	url = urllib.request.urlopen(website)
	stuff = url.read()
	soup = BeautifulSoup(stuff)
	table = soup.find_all('div', {'class' : 'stats-table'})
	return table

#gets current standings
def FormatStandings(data, f):
	# 15 elements for each teem, need to offset by that much
	offset = 15
	# open file for writing
	# ugly reddit formatting begin!
	f.write("Pos | Team | Pts | W | L | T\n")
	f.write("---|---|---|---|---|---\n")
	for i in range(0, 10):
		# need to remove extra \n's and whitespace that appear
		# otherwise table doesn't display right on reddit
		string = data[i * offset + 1].text
		string = string.replace("\n", "")
		#string = string.replace(" ", "")
		# output
		f.write(
			data[i * offset].text + " | " +
			string + " | " +
			data[i * offset + 2].text + " | " +
			data[i * offset + 5].text + " | " +
			data[i * offset + 6].text + " | " +
			data[i * offset + 7].text + " | \n")

	f.write("\n")

#formats schedule for reddit usage
def FormatSchedule(data, f):

	f.write("Fixt | Opponent | Date | Time | Score\n")
	f.write("---|---|---|---|---\n")

	dates = data[0].find_all('h3')
	#find dates
	dateNum = []
	for i in range(0, len(dates)):
		dateNum.append(FixDates(dates[i].text))

	#fix date issue so it shows up m/d, m/dd, or mm/dd
	#keep this in mind! This is the correct date list!!
	fixedDates = FormatDates(dateNum)

	#find game time
	#times is the correct time list!!
	games = data[0].find_all('table', {'class': "views-table cols-1 schedule-table" })
	times = []
	for i in range(0, len(games)):
		time = games[i].find('div', {'class' : 'field-game-date-start-time'})
		time = time.text.replace(" " ,"")
		time = time.replace("\n", "")
		time = time.replace("EDT", "")
		times.append(time)

	#find game type, MLS regular season or US Open Cup
	#fixtures is the correct fixture list!!
	gameType = data[0].find_all('span', {'class' : 'competetion'})
	fixtures = []
	for i in range(0, len(gameType)):
		fixtureString = gameType[i].text
		if(fixtureString == "MLS Regular Season"):
			fixtures.append("MLS")
		elif(fixtureString == "U.S. Open Cup"):
			fixtures.append("USOC")

	#get teams
	#opponent is corrected list form!
	homeTeam = data[0].find_all('div', {'class' : 'field-home-team'})
	awayTeam = data[0].find_all('div', {'class' : 'field-away-team'})
	opponent = []
	for i in range(0, len(homeTeam)):
		if(homeTeam[i].text != "Philadelphia"):
			opponent.append("@" + homeTeam[i].text)
		else:
			opponent.append('-')
	placeholder = '-'
	for i in range(0, len(awayTeam)):
		if(awayTeam[i].text != "Philadelphia"):
			if(opponent[i] == placeholder):
				opponent[i] = awayTeam[i].text

	#results! finalScores will list all scores!
	finalScores = []
	scores = data[0].find_all('div', {'class' : 'field-score'})
	for i in range(0, len(scores)):
		score = scores[i].text.replace("\n", "")
		score = score.replace(" ", "")
		finalScores.append(score)

	if(len(finalScores) < len(fixtures)):
		for i in range(0, (len(fixtures) - len(finalScores))):
			finalScores.append("TBD")

	for i in range(0, len(fixtures)):
		date = ''.join(fixedDates[i])

		f.write(
			fixtures[i] + " | " +
			opponent[i] + " | " +
			date + " | " +
			times[i] + " | " +
			finalScores[i] + " | \n"
			)

	f.write("\n")

#formats goals for reddit usage
def FormatGoalStats(data, f):
	f.write("Player | Goals |\n")
	f.write("---|---|\n")
	td = data[0].find_all('td')

	offset = 15
	goalLeader = []
	numGoals = []
	for i in range(0, 5):
		goalLeader.append(td[i * offset].text)
		numGoals.append(td[i * offset + 5].text)

	for i in range(0, len(goalLeader)):
		f.write(goalLeader[i] + " | " + numGoals[i] + " | \n")

	f.write("\n")
#formats goals for reddit usage
def FormatAssistsStats(data, f):
	f.write("Player | Assists |\n")
	f.write("---|---|\n")

	td = data[0].find_all('td')
	offset = 9
	assistsLeader = []
	numAssists = []
	for i in range(0, 5):
		assistsLeader.append(td[i * offset].text)
		numAssists.append(td[i * offset + 4].text)

	for i in range(0, len(assistsLeader)):
		f.write(assistsLeader[i] + " | " + numAssists[i] + " | \n")

	f.write("\n")

#replaces string month with int month i.e january == 1 etc.
def FixDates(date):
	months = {'January' : 1, 'February' : 2, 'March' : 3, 'April' : 4,
				'May' : 5, 'June' : 6, 'July' : 7, 'August' : 8,
				'September' : 9, 'October' : 10, 'November' : 11, 'December' : 12}
	date = date.split()

	dateStr = str(months[date[1]]) + " " + date[2].replace(",", "")
	dateStr = dateStr.replace(" ", "")
	return dateStr

#just fixes date formatting
def FormatDates(dates):
	formattedDates = []
	#just for formatting dates! fuck this
	for i in range(0, len(dates)):
		#g.write("---|---|")
		date = dates[i]
		string = []
		#handle #/# dates
		if(len(date) == 2):
			for j in range(0, len(date)):
				string.append(date[j])
				if(j == 0):
					string.append("/")
		#handle #/## dates
		elif(len(date) == 3):
			for j in range(0, len(date)):
				string.append(date[j])
				if(j == 0):
					string.append("/")
		#handle ##/## dates
		elif(len(date) == 4):
			for j in range(0, len(date)):
				string.append(date[j])
				if(j == 0):
					string.append("/")
		formattedDates.append(string)

	return formattedDates

if __name__ == '__main__':

	f = open("sidebar.txt", "w+")
	date = datetime.date.today()
	month = date.month
	#year = date.year
	urlStandings = "http://www.mlssoccer.com/standings/2015" #+ str(year)
	urlSchedule = "http://www.mlssoccer.com/schedule?month=" + str(month) + "&year=2015&club=10&competition_type=all&broadcast_type=all&op=Search&form_id=mls_schedule_form"
	urlScoring = "http://www.mlssoccer.com/stats/season?season_year=2015&season_type=REG&team=5513&group=GOALS&op=Search&form_id=mls_stats_individual_form"
	urlAssists = "http://www.mlssoccer.com/stats/season?season_year=2015&season_type=REG&team=5513&group=ASSISTS&op=Search&form_id=mls_stats_individual_form"
	table = ParseStandingsData(urlStandings)
	FormatStandings(table, f)
	table2 = ParseScheduleData(urlSchedule)
	FormatSchedule(table2, f)
	table3 = ParseStatsData(urlScoring)
	FormatGoalStats(table3, f)
	table4 = ParseStatsData(urlAssists)
	FormatAssistsStats(table4, f)

	f.close()


# coding=utf-8
from json import dumps

from django.core.serializers.json import DjangoJSONEncoder

# Constantes Errores

ERROR_LOGIN_REQUERIDO = 'ERROR_LOGIN_REQUERIDO'
ERROR_LOGOUT_FALLIDO = 'ERROR_LOGOUT_FALLIDO'
ERROR_NO_TIENE_PERMISOS = 'ERROR_NO_TIENE_PERMISOS'
ERROR_DE_SISTEMA = 'ERROR_DE_SISTEMA'
ERROR_METODO_INCORRECTO = 'ERROR_METODO_INCORRECTO'
ERROR_DATOS_INCORRECTOS = 'ERROR_DATOS_INCORRECTOS'
ERROR_VALIDACION_DATOS = 'ERROR_VALIDACION_DATOS'
ERROR_DATOS_FALTANTES = 'ERROR_DATOS_FALTANTES'
ERROR_CREDENCIALES_INCORRECTAS = 'ERROR_CREDENCIALES_INCORRECTAS'

ERROR_USUARIO_YA_TIENE_ESE_ROL = 'ERROR_USUARIO_YA_TIENE_ESE_ROL'
ERROR_USUARIO_NO_ADMINISTRADOR = 'ERROR_USUARIO_NO_ADMINISTRADOR'
ERROR_USUARIO_NO_ES_ENCARGADO = 'ERROR_USUARIO_NO_ES_ENCARGADO'
ERROR_USUARIO_NO_ENCONTRADO = 'ERROR_USUARIO_NO_ENCONTRADO'
ERROR_USUARIO_NO_ENCARGADO = 'ERROR_USUARIO_NO_ENCARGADO'
ERROR_USUARIO_EXISTENTE = 'ERROR_USUARIO_EXISTENTE'
ERROR_EMAIL_EXISTENTE = 'ERROR_EMAIL_EXISTENTE'
ERROR_USUARIO_FINCA_NO_ENCONTRADO = 'ERROR_USUARIO_FINCA_NO_ENCONTRADO'
ERROR_USUARIO_NO_HABILITADO_EN_FINCA = 'ERROR_USUARIO_NO_HABILITADO_EN_FINCA'
ERROR_USUARIO_YA_TIENE_ROL_EN_FINCA = 'ERROR_USUARIO_YA_TIENE_ROL_EN_FINCA'
ERROR_FINCA_YA_EXISTENTE = 'ERROR_FINCA_YA_EXISTENTE'
ERROR_PROVEEDOR_NO_ENCONTRADO = 'ERROR_PROVEEDOR_NO_ENCONTRADO'
ERROR_FINCA_NO_ENCONTRADA = 'ERROR_FINCA_NO_ENCONTRADA'
ERROR_FINCA_YA_APROBADA = 'ERROR_FINCA_YA_APROBADA'
ERROR_FINCA_YA_HABILITADA = 'ERROR_FINCA_YA_HABILITADA'
ERROR_FINCA_YA_DESAPROBADA = 'ERROR_FINCA_YA_DESAPROBADA'
ERROR_FINCA_YA_DESHABILITADA = 'ERROR_FINCA_YA_DESHABILITADA'
ERROR_SECTOR_SUPERA_TAMANIO_FINCA = 'ERROR_SECTOR_SUPERA_TAMANIO_FINCA'
ERROR_SECTORES_SUPERAN_TAMANIO_FINCA = 'ERROR_SECTORES_SUPERAN_TAMANIO_FINCA'
ERROR_SECTOR_YA_EXISTENTE = 'ERROR_SECTOR_YA_EXISTENTE'
ERROR_SECTOR_NO_ENCONTRADO = 'ERROR_SECTOR_NO_ENCONTRADO'
ERROR_SECTOR_NO_HABILITADO = 'ERROR_SECTOR_NO_HABILITADO'
ERROR_MECANISMO_RIEGO_FINCA_NO_EXISTE = 'ERROR_MECANISMO_RIEGO_FINCA_NO_EXISTE'
ERROR_MECANISMO_RIEGO_FINCA_NO_HABILITADO = 'ERROR_MECANISMO_RIEGO_FINCA_NO_HABILITADO'
ERROR_IP_YA_USADA = 'ERROR_IP_YA_USADA'
ERROR_MECANISMO_RIEGO_FINCA_YA_EXISTE = 'ERROR_MECANISMO_RIEGO_FINCA_YA_EXISTE'
ERROR_MECANISMO_RIEGO_FINCA_SECTOR_NO_HABILITADO = 'ERROR_MECANISMO_RIEGO_FINCA_SECTOR_NO_HABILITADO'
ERROR_SECTOR_YA_TIENE_MECANISMO_HABILITADO = 'ERROR_SECTOR_YA_TIENE_MECANISMO_HABILITADO'
ERROR_MECANISMO_RIEGO_FINCA_SECTOR_NO_EXISTE = 'ERROR_MECANISMO_RIEGO_FINCA_SECTOR_NO_EXISTE'

ERROR_CULTIVO_NO_EXISTENTE = 'ERROR_CULTIVO_NO_EXISTENTE'
ERROR_CULTIVO_NO_HABILITADO = 'ERROR_CULTIVO_NO_HABILITADO'
ERROR_SUBTIPO_CULTIVO_NO_EXISTENTE = "ERROR_SUBTIPO_CULTIVO_NO_EXISTENTE"
ERROR_SUBTIPO_CULTIVO_NO_HABILITADO = "ERROR_SUBTIPO_CULTIVO_NO_HABILITADO"
ERROR_TIPO_CULTIVO_NO_HABILITADO = "ERROR_TIPO_CULTIVO_NO_HABILITADO"
ERROR_SECTOR_YA_TIENE_CULTIVO_ASIGNADO = 'ERROR_SECTOR_YA_TIENE_CULTIVO_ASIGNADO'

ERROR_TIPO_MEDICION_NO_EXISTENTE = 'ERROR_TIPO_MEDICION_NO_EXISTENTE'
ERROR_TIPO_MEDICION_NO_HABILITADA = 'ERROR_TIPO_MEDICION_NO_HABILITADA'
ERROR_TIPO_MEDICION_CLIMATICA_NO_EXISTENTE = 'ERROR_TIPO_MEDICION_CLIMATICA_NO_EXISTENTE'
ERROR_SENSOR_NO_EXISTENTE = 'ERROR_SENSOR_NO_EXISTENTE'

ERROR_SENSOR_NO_PERTENECE_A_FINCA = 'ERROR_SENSOR_NO_PERTENECE_A_FINCA'
ERROR_SENSOR_NO_HABILITADO = 'ERROR_SENSOR_NO_HABILITADO'
ERROR_SENSOR_YA_ASIGNADO = 'ERROR_SENSOR_YA_ASIGNADO'
ERROR_SENSOR_NO_ASIGNADO_A_COMPONENTE = 'ERROR_SENSOR_NO_ASIGNADO_A_COMPONENTE'
ERROR_COMPONENTE_NO_ACEPTA_MAS_SENSORES = 'ERROR_COMPONENTE_NO_ACEPTA_MAS_SENSORES'
ERROR_COMPONENTE_SENSOR_NO_EXISTENTE = 'ERROR_COMPONENTE_SENSOR_NO_EXISTENTE'
ERROR_COMPONENTE_SENSOR_NO_PERTENECE_A_FINCA = 'ERROR_COMPONENTE_SENSOR_NO_PERTENECE_A_FINCA'
ERROR_COMPONENTE_SENSOR_NO_HABILITADO = 'ERROR_COMPONENTE_SENSOR_NO_HABILITADO'
ERROR_COMPONENTE_SENSOR_YA_ASIGNADO = 'ERROR_COMPONENTE_SENSOR_YA_ASIGNADO'
ERROR_COMPONENTE_SENSOR_YA_HABILITADO = 'ERROR_COMPONENTE_SENSOR_YA_HABILITADO'
ERROR_COMPONENTE_SENSOR_NO_ASIGNADO_A_SECTOR = 'ERROR_COMPONENTE_SENSOR_NO_ASIGNADO_A_SECTOR'
ERROR_SECTOR_YA_TIENE_COMPONENTE_HABILITADO = 'ERROR_SECTOR_YA_TIENE_COMPONENTE_HABILITADO'

ERROR_FINCA_NO_HABILITADA = 'ERROR_FINCA_NO_HABILITADA'
ERROR_FINCA_NO_TIENE_PROVEEDOR_HABILITADO = 'ERROR_FINCA_NO_TIENE_PROVEEDOR_HABILITADO'
ERROR_FRECUENCIA_MAXIMA_SUPERADA = 'ERROR_FRECUENCIA_MAXIMA_SUPERADA'
ERROR_NO_EXISTE_PROVEEDOR_FINCA = 'ERROR_NO_EXISTE_PROVEEDOR_FINCA'
ERROR_FINCA_TIENE_UN_PROVEEDOR_HABILITADO = 'ERROR_FINCA_TIENE_UN_PROVEEDOR_HABILITADO'

# Modulo configuracion riego
ERROR_CRITERIO_NO_EXISTE = 'ERROR_CRITERIO_NO_EXISTE'
ERROR_CONFIGURACION_RIEGO_NO_EXISTE = 'ERROR_CONFIGURACION_RIEGO_NO_EXISTE'
ERROR_EJECUCION_RIEGO = "ERROR_EJECUCION_RIEGO"
ERROR_RIEGO_EN_EJECUCION = 'ERROR_RIEGO_EN_EJECUCION'
ERROR_MODIFICACION_CONFIGURACION_RIEGO = "ERROR_MODIFICACION_CONFIGURACION_RIEGO"

# Modulo reportes
ERROR_CONFIGURACION_EVENTO_NO_ENCONTRADA = 'ERROR_CONFIGURACION_EVENTO_NO_ENCONTRADA'
ERROR_CONFIGURACION_EVENTO_YA_DESACTIVADA = 'ERROR_CONFIGURACION_EVENTO_YA_DESACTIVADA'
ERROR_CONFIGURACION_EVENTO_YA_ACTIVADA = 'ERROR_CONFIGURACION_EVENTO_YA_ACTIVADA'
ERROR_HELADA_NO_CARGADA = 'ERROR_HELADA_NO_CARGADA'

# Detalle errores

DETALLE_ERROR_DESCONOCIDO = "Error desconocido"
DETALLE_ERROR_DATOS_INCOMPLETOS = "Faltan ingresar datos"
DETALLE_ERROR_DATOS_INCORRECTOS = "Datos incorrectos"
DETALLE_ERROR_SISTEMA = "Error procesando llamada"
DETALLE_ERROR_ENVIO_EMAIL = "Error enviando email"

# Modulo seguridad
DETALLE_ERROR_LOG_IN_FALLIDO = 'Usuario o contraseña incorrectos'
DETALLE_ERROR_LOG_OUT_FALLIDO = 'Log out fallo'
DETALLE_ERROR_TIPO_SESION_DATOS_FALTANTES = 'Faltan datos de tipo de sesion'

DETALLE_ERROR_NO_TIENE_PERMISOS = 'Usuario no tiene permisos para ejecutar esta operacion'

DETALLE_ERROR_REGISTRACION_USUARIO_EXISTENTE = 'Usuario ya existe en el sistema'
DETALLE_ERROR_REGISTRACION_USUARIO_FALTANTE = 'Falta ingresar usuario'
DETALLE_ERROR_REGISTRACION_USUARIO_INCORRECTO = 'Usuario incorrecto'
DETALLE_ERROR_REGISTRACION_CONTRASENIA_FALTANTE = 'Falta ingresar contrasena'
DETALLE_ERROR_REGISTRACION_NOMBRE_FALTANTE = 'Falta ingresar nombre'
DETALLE_ERROR_REGISTRACION_NOMBRE_INCORRECTO = 'Nombre incorrecto'
DETALLE_ERROR_REGISTRACION_APELLIDO_FALTANTE = 'Falta ingresar apellido'
DETALLE_ERROR_REGISTRACION_APELLIDO_INCORRECTO = 'Apellido incorrecto'
DETALLE_ERROR_REGISTRACION_DNI_FALTANTE = 'Falta ingresar dni'
DETALLE_ERROR_REGISTRACION_DNI_INCORRECTO = 'Formato dni ingresado es incorrecto'
DETALLE_ERROR_REGISTRACION_CUIT_FALTANTE = 'Falta ingresar CUIT'
DETALLE_ERROR_REGISTRACION_CUIT_INCORRECTO = 'Formato CUIT ingresado es incorrecto'
DETALLE_ERROR_REGISTRACION_DOMICILIO_FALTANTE = 'Falta ingresar domicilio'
DETALLE_ERROR_REGISTRACION_DOMICILIO_INCORRECTO = 'Formato domicilio ingresado es incorrecto'
DETALLE_ERROR_REGISTRACION_FECHA_NACIMIENTO_FALTANTE = 'Falta ingresar fecha de nacimiento'
DETALLE_ERROR_REGISTRACION_FECHA_NACIMIENTO_INCORRECTO = 'Formato de fecha de nacimiento ingresada es incorrecto'
DETALLE_ERROR_REGISTRACION_IMAGEN_FALTANTE = 'Falta ingresar imagen'
DETALLE_ERROR_REGISTRACION_IMAGEN_INCORRECTA = 'Formato imagen de ingresada es incorrecto'

DETALLE_ERROR_EMAIL_EXISTENTE = 'Ya existe un usuario con ese email'
DETALLE_ERROR_EMAIL_FALTANTE = 'Falta ingresar email'
DETALLE_ERROR_EMAIL_INCORRECTO = 'Formato email incorrecto'

DETALLE_ERROR_CONTRASENIA_INCORRECTA = 'Contraseña incorrecta'
DETALLE_ERROR_CONTRASENIA_FORMATO_INCORRECTO = 'Formato contraseña incorrecto: debe tener mas de 8 caracteres y' \
                                                    ' no puede ser solo numérica'

DETALLE_ERROR_CONTRASENIA_VIEJA_INCORRECTA = 'La contraseña actual ingresada es incorrecta'
DETALLE_ERROR_CAMBIAR_CONTRASENIA_VIEJA_FALTANTE = 'Falta ingresar contraseña actual'
DETALLE_ERROR_CAMBIAR_CONTRASENIA_NUEVA_FALTANTE = 'Falta ingresar contraseña nueva'

DETALLE_ERROR_CODIGO_VERIFICACION_FALTANTE = 'Falta ingresar codigo de verificacion'
DETALLE_ERROR_CODIGO_VERIFICACION_INCORRECTO = 'Codigo de verificacion incorrecto'

# Finca
DETALLE_ERROR_FINCA_DATOS_FALTANTES = "Falta ingresar datos de finca"

# Modulo configuracion riego
DETALLE_ERROR_EJECUION_RIEGO = "Ejecucion de riego fallo"
DETALLE_ERROR_DETENCION_RIEGO = "Detencion de riego fallo"

DETALLE_ERROR_MODIFICACION_CONFIGURACION_RIEGO = 'La configuracion de riego ya fue eliminada'
DETALLE_ERROR_CONFIGURACION_RIEGO_INCORRECTA = 'La configuracion de riego es incorrecta'
DETALLE_ERROR_RELACION_CONFIGURACION_RIEGO_CRITERIO = 'El criterio de riego no se corresponde con la configuracion'
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_INCORRECTO = 'El tipo de criterio riego es incorrecto'
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_INCORRECTO_MEDICION = "El criterio elegido no es por medicion"
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_INCORRECTO_HORA = "El criterio elegido no es por hora"
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_INCORRECTO_VOLUMEN = "El criterio elegido no es por volumen de agua"
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_INICIAL_INCORRECTO = 'El tipo de criterio riego no es valido como criterio inicial'
DETALLE_ERROR_TIPO_CRITERIO_RIEGO_FINAL_INCORRECTO = 'El tipo de criterio riego no es valido como criterio final'
DETALLE_ERROR_CRITERIO_RIEGO_INEXISTENTE = 'El criterio de riego seleccionado no existe'
DETALLE_ERROR_CRITERIO_RIEGO_VALOR_MEDICION_INEXISTENTE = 'Falta ingresar valor de medicion'
DETALLE_ERROR_CRITERIO_RIEGO_OPERADOR_MEDICION_INEXISTENTE = 'Falta ingresar operador criterio por medicion'
DETALLE_ERROR_CRITERIO_RIEGO_TIPO_MEDICION_INEXISTENTE = 'Falta ingresar tipo de medicion'
DETALLE_ERROR_CRITERIO_RIEGO_HORA_INICIO_INEXISTENTE = 'Falta ingresar hora de inicio'
DETALLE_ERROR_CRITERIO_RIEGO_DIA_INICIO_INEXISTENTE = 'Falta ingresar dia de inicio'
DETALLE_ERROR_CRITERIO_RIEGO_VOLUMEN_INEXISTENTE = 'Falta ingresar volumen de agua'

DETALLE_ERROR_CONFIGURACION_RIEGO_AUTOMATICA_CRITERIO = 'A una configuracion de riego automatica no se le asignan' \
                                                        ' criterios'
DETALLE_ERROR_CONFIGURACION_RIEGO_CANTIDAD_MAX_CRITERIO_INICIAL = 'La cantidad maxima de criterios iniciales fue ' \
                                                                  'alcanzada'
DETALLE_ERROR_CONFIGURACION_RIEGO_CANTIDAD_MAX_CRITERIO_FINAL = 'La cantidad maxima de criterios finales fue alcanzada'

DETALLE_ERROR_CONFIGURACION_RIEGO_EN_EJECUCION = 'Configuracion de riego en ejecucion, no se puede modificar'

# Constantes Keys

KEY_ERROR_CODE = 'error_code'
KEY_ERROR_DESCRIPTION = 'error_description'

# Metodos

def build_error(response, error_code, error_descripcion):
    content = {KEY_ERROR_DESCRIPTION: error_descripcion, KEY_ERROR_CODE: error_code}
    response.content = dumps(content, cls=DjangoJSONEncoder)

    return response

def build_bad_request_error(response, error_code, error_descripcion):
    response.status_code = 400
    return build_error(response, error_code, error_descripcion)

def build_unauthorized_error(response, error_code, error_descripcion):
    response.status_code = 401

    return build_error(response, error_code, error_descripcion)

def build_method_not_allowed_error(response, error_code, error_descripcion):
    response.status_code = 405

    return build_error(response, error_code, error_descripcion)

def build_internal_server_error(response, error_code, error_descripcion):
    response.status_code = 500

    return build_error(response, error_code, error_descripcion)


from common.model_manager.payment_manager import PaymentManager
from common.model_manager.utils import get_amount
from common.msg_service.wechat_msg_send import MerchantMessageSender
from common.voice_synthesis.baidu import get_baidu_token
from ws_service.utils import PublishToRedisChannel

def send_pay_success_wechat_message(payment):
    """付款成功微信消息推送"""

    payment = PaymentManager(payment)

    message_sender = MerchantMessageSender(payment.merchant)
    message_sender.on_pay_success(
        merchant_receive='%.2f' % get_amount(payment.paid_price),
        payment_type='普通订单' if not payment.coupon else '优惠券订单',
        datetime=payment.datetime,
        pay_serial_number=payment.serial_number,
    )

def send_voice_message_to_merchant_app(payment):
    """send payment pk to app through websocket, and then app request server a voice by the pk."""

    payment = PaymentManager(payment)

    token, error_msg = get_baidu_token()
    data = dict(
        channel_key=f'merchant_{payment.merchant_id}_new_payment',
        voice_text=payment.voice_text,
        baidu_token=token,
        error_msg=error_msg
    )

    PublishToRedisChannel.publish_to_channel(data=data)


import numpy as np
import random
import pandas as pd
import torch
import nengo
from scipy.stats import entropy, skew, kurtosis, normaltest

def get_state(player, game, agent, dim=0, turn_basis=None, coin_basis=None, turn_exp=None, coin_exp=None, representation="one-hot"):
	t = len(game.investor_give) if player=='investor' else len(game.trustee_give)
	if agent=='TQ':
		index = t if player=='investor' else t * (game.coins*game.match+1) + game.investor_give[-1]*game.match
		return index
	if agent=='DQN':
		if representation=="one-hot":
			index = t if player=='investor' else t * (game.coins*game.match+1) + game.investor_give[-1]*game.match
			vector = np.zeros((dim))
			vector[index] = 1
			return torch.FloatTensor(vector)
		elif representation=="ssp":
			c = game.coins if player=='investor' else game.investor_give[-1]*game.match
			vector = encode_state(t, c, turn_basis, coin_basis)
			return torch.FloatTensor(vector)
	if agent=="IBL":
		if player=="investor":
			return t, game.coins
		else:
			return t, game.investor_give[-1]*game.match
	if agent=="SPA":
		if representation=="ssp":
			c = game.coins if player=='investor' else game.investor_give[-1]*game.match
			vector = encode_state(t, c, turn_basis, coin_basis, turn_exp, coin_exp) if t<5 else np.zeros((dim))
			return vector

def action_to_coins(player, state, n_actions, game):
	available = game.coins if player=='investor' else game.investor_give[-1]*game.match  # coins available
	precise_give = state * available
	possible_actions = np.linspace(0, available, n_actions).astype(int)
	action_idx = (np.abs(possible_actions - precise_give)).argmin()
	action = possible_actions[action_idx]
	give = action
	keep = available - action
	return give, keep, action_idx

def generosity(player, give, keep):
	return np.NaN if give+keep==0 and player=='trustee' else give/(give+keep)

def encode_state(t, c, turn_basis, coin_basis, turn_exp=1.0, coin_exp=1.0):
	return np.fft.ifft(turn_basis**(t*turn_exp) * coin_basis**(c*coin_exp)).real.squeeze()

def make_unitary(v):
	return v/np.absolute(v)

def measure_sparsity(spikes1, spikes2):
	n_neurons = spikes1.shape[0]
	diff = []
	quiet = 0
	for n in range(n_neurons):
		if spikes1[n]+spikes2[n]>0:
			diff.append((spikes1[n]-spikes2[n]) / (spikes1[n]+spikes2[n]))
		else:
			quiet += 1
	diff = np.array(diff)
	quiet = quiet / n_neurons
	pdiff = (np.histogram(diff)[0][0] + np.histogram(diff)[0][-1]) / diff.shape[0]
	return 100*pdiff, 100*quiet

def measure_similarity(ssp1, ssp2, mode="cosine"):
    if mode=="dot":
        return np.sum(ssp1 * ssp2)
    elif mode=="cosine":
        return np.sum(ssp1 * ssp2) / (np.linalg.norm(ssp1, ord=2) * np.linalg.norm(ssp2, ord=2))

def get_rewards(player, game, w_s, w_o, w_i, normalize, gamma):
	rewards_self = np.array(game.investor_reward) if player=='investor' else np.array(game.trustee_reward)
	rewards_other = np.array(game.trustee_reward) if player=='investor' else np.array(game.investor_reward)
	rewards = w_s*rewards_self + w_o*rewards_other - w_i*np.abs(rewards_self-rewards_other)
	if normalize:
		rewards = rewards / (game.coins * game.match)
		rewards[:-1] = (1-gamma)*rewards[:-1]
	return rewards


import pygame
import sys

# Initialize Pygame and create a window
pygame.init()
screen_width, screen_height = 600, 500
screen = pygame.display.set_mode((screen_width, screen_height))
pygame.display.set_caption("Mario")

# Load the images of Mario running
plyer_image = [pygame.image.load('Bilder/oben.png').convert_alpha(),
               pygame.image.load('Bilder/unten.png').convert_alpha(),
               pygame.image.load("Bilder/rechts.png").convert_alpha(),
               pygame.image.load("Bilder/links.png").convert_alpha(),
               pygame.image.load("Bilder/left1.png").convert_alpha(),
               pygame.image.load("Bilder/right1.png").convert_alpha()]

def move_stone(x, y, plyer_rect1):
    key_s = pygame.key.get_pressed()
    if key_s[pygame.K_UP]:
        y -= speed
    if key_s[pygame.K_DOWN]:
        y += speed
    if key_s[pygame.K_LEFT]:
        x -= speed
    if key_s[pygame.K_RIGHT]:
        x += speed

        # Calculate the distance between the player and the stone
    distance = plyer_rect1.right - x

    # If the player is on the left of the stone, the distance is 5 or lower, and the player is moving to the right,
    # move the stone to the right as well
    if plyer_rect1.x < x and distance <= 5 and keys[pygame.K_d]:
        x += speed

def line(x_start, y_start, x_end, y_end):
    pygame.draw.line(screen, "black", (x_start, y_start), (x_end, y_end), width=5)

Stein_img = pygame.image.load("Bilder/Steine/1.png").convert_alpha()
stein = pygame.transform.smoothscale(Stein_img, (30, 30))
stein_rect = stein.get_rect(x=200, y=200)

# Load the background image and scale it to fit the screen
o_background = pygame.image.load("Bilder/background.jpg").convert_alpha()
background = pygame.transform.scale(o_background, (screen_width, screen_height))

# Set the player's initial position and speed
plyer = plyer_image[1].convert_alpha()
plyer = pygame.transform.scale(plyer, (30, 50))
plyer_rect = plyer_image[1].get_rect(x=100, y=200)

speed = 5
line_x = 0
# Create a clock to control the frame rate
clock = pygame.time.Clock()

# Main game loop
while True:
    # Handle events
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()

    # Update the player's position based on key input
    keys = pygame.key.get_pressed()
    if keys[pygame.K_RIGHT] and keys[pygame.K_LEFT]:
        plyer = plyer_image[1]

    if keys[pygame.K_UP]:
        plyer = plyer_image[0]
        plyer = pygame.transform.scale(plyer, (30, 50))
        plyer_rect.move_ip(0, -speed)
    if keys[pygame.K_DOWN]:
        plyer = plyer_image[1]
        plyer = pygame.transform.scale(plyer, (30, 50))
        plyer_rect.move_ip(0, speed)
    if keys[pygame.K_RIGHT] and not keys[pygame.K_LEFT]:
        if pygame.time.get_ticks() % 100 < 50:
            plyer = plyer_image[2]
            plyer = pygame.transform.scale(plyer, (30, 50))
        else:
            plyer = plyer_image[5]
            plyer = pygame.transform.scale(plyer, (30, 50))
        plyer_rect.move_ip(speed, 0)
    if keys[pygame.K_LEFT] and not keys[pygame.K_RIGHT]:
        if pygame.time.get_ticks() % 100 < 50:
            plyer = plyer_image[3]
            plyer = pygame.transform.scale(plyer, (30, 50))
        else:
            plyer = plyer_image[4]
            plyer = pygame.transform.scale(plyer, (30, 50))
        plyer_rect.move_ip(-speed, 0)

    # Here is the movement of the stone
    dist_bottom = plyer_rect.bottom - stein_rect.top
    dist_top = plyer_rect.top - stein_rect.bottom
    dist_right = plyer_rect.right - stein_rect.left
    dist_left = plyer_rect.left - stein_rect.right

    if 0 <= dist_bottom <= 80 and 10 <= dist_right <= 30 and keys[pygame.K_RIGHT]: stein_rect.x += speed
    if 0 <= dist_bottom <= 80 and -30 <= dist_left <= 0 and keys[pygame.K_LEFT]:   stein_rect.x -= speed
    if 0 <= dist_bottom <= 20 and -60 <= dist_left <= -5 and keys[pygame.K_DOWN]:  stein_rect.y += speed
    if -20 <= dist_top <= 0 and -60 <= dist_left <= -5 and keys[pygame.K_UP]:      stein_rect.y -= speed

    # print to help
    if pygame.time.get_ticks() % 1000 < 50 / 3:
        print("line: 118", "bottom:", dist_bottom)

    # The line movement
    if keys[pygame.K_e]:
        if line_x < 50:
            line_x += 1
    if keys[pygame.K_q]:
        if line_x > 0:
            line_x -= 1

    if line_x+plyer_rect.left+30 >= stein_rect.x and 31 < dist_bottom < 56:
        if keys[pygame.K_f]:
            stein_rect.x = plyer_rect.x+line_x+20
            stein_rect.y = plyer_rect.centery-10

    # Keep the player within the screen boundaries
    if plyer_rect.bottom > screen_height:
        plyer_rect.bottom = screen_height
    if plyer_rect.right > screen_width:
        plyer_rect.right = screen_width
    if plyer_rect.top < 0:
        plyer_rect.top = 0
    if plyer_rect.left < 0:
        plyer_rect.left = 0

    # Clear the screen and draw the background and player
    screen.blit(background, (0, 0))
    screen.blit(plyer, plyer_rect)
    screen.blit(stein, stein_rect)
    line(plyer_rect.centerx + 15, plyer_rect.centery, plyer_rect.centerx + line_x + 15, plyer_rect.centery)

    # Update the display
    pygame.display.update()

    # Limit the frame rate to 60 FPS
    clock.tick(60)


import numpy as np
from scipy.spatial import Delaunay
from scipy.spatial import Voronoi
import math
from itertools import compress
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
import matplotlib.pyplot as plt
from matplotlib import cm

number_of_points = 1000
area_threshold = 40
min_number_of_localizations = 5

def PolyArea(x,y):
    return 0.5*np.abs(np.dot(x,np.roll(y,1))-np.dot(y,np.roll(x,1)))

def neighbors_delauay(points):
    dt = Delaunay(points)
    vertices = dt.vertices

    vertices_hstack = np.hstack(vertices)
    vertices_hstack_sorted = np.sort(vertices_hstack)
    vertices_hstack_sorted_arg = np.argsort(vertices_hstack)
    unique_vals, indices = np.unique(vertices_hstack_sorted , return_index=True)
    vertices = vertices[divmod(vertices_hstack_sorted_arg,3)[0],:]

    neighbors = []
    for i in range(len(indices)):
        if i<len(indices)-1:
            neighbors.append(np.unique(vertices[indices[i]:indices[i+1]]))
        else:
            neighbors.append(np.unique(vertices[indices[i]:vertices.shape[0]]))
    neighbors = [list(i) for i in neighbors]
    [val.remove(idx) for idx,val in enumerate(neighbors)]
    return neighbors

def find_voronoi(points):
    vor = Voronoi(points)
    point_to_cell_index = list(vor.point_region)
    regions = vor.regions
    vertices = vor.vertices

    inf_empty_index = [regions.index(i) for i in regions if -1 in i or i ==[]]

    voronoi_cells = [vertices[i] for i in regions]

    voronoi_areas = [PolyArea(i[:,0],i[:,1]) for i in voronoi_cells]

    for i in inf_empty_index:
        voronoi_areas[i] = math.inf

    voronoi_areas = [voronoi_areas[i] for i in point_to_cell_index]
    voronoi_cells = [voronoi_cells[i] for i in point_to_cell_index]
    return voronoi_areas, voronoi_cells

def find_clusters(keep_points,neighbors):
    used_points = [False]*len(keep_points)
    clusters = []
    for i in range(len(keep_points)):
        if keep_points[i] and not used_points[i]:
            seed = []
            idx = []
            seed = list(compress(neighbors[i],[keep_points[j] for j in neighbors[i]]))
            if seed:
                while True:
                    size_one = len(seed)
                    idx = list(np.unique(np.concatenate([neighbors[i] for i in seed])))
                    idx = list(compress(idx,[keep_points[j] for j in idx]))
                    seed  = list(np.unique(np.asarray(seed+idx)))
                    size_two = len(seed)
                    if size_one==size_two:
                        break
            else:
                seed = [i]
            for j in seed:
                used_points[j] = True
            clusters.append(seed)
    return clusters

def plot_clusters(clusters_voronoi_cells,clusters_voronoi_areas):
    fig, ax = plt.subplots()
    patches = []
    color = []
    for counter,i in enumerate(clusters_voronoi_cells):
        color.append([counter]*len(i))
        for j in i:
            patches.append(Polygon(j))
    color = np.asarray([np.asarray(i) for i in color])
    color = np.hstack(color)
    collection = PatchCollection(patches, alpha = 1, edgecolor = None, cmap=cm.hsv)
    collection.set_array(color)
    # collection.set_clim(cmin, cmax)
    ax.add_collection(collection)
    fig.colorbar(collection, ax=ax)
    plt.show()
    plt.xlim(min(points[:,0]),max(points[:,0]))
    plt.ylim(min(points[:,1]),max(points[:,1]))

def plot_voronoi(points,voronoi_cells,voronoi_areas,keep_points):
    fig, ax = plt.subplots()
    patches = []
    for counter,cell in enumerate(voronoi_cells):
        if voronoi_areas[counter] != math.inf:
            patches.append(Polygon(cell))
    collection = PatchCollection(patches, alpha = 0.1, edgecolor = 'r', cmap=cm.hsv)
    color = [i for i in voronoi_areas if i!=math.inf]
    collection.set_array(np.log(np.array(color)))
    ax.add_collection(collection)
    plt.scatter(points[:,0], points[:,1], s=1, color = 'b')
    plt.show()
    plt.xlim(min(points[:,0]),max(points[:,0]))
    plt.ylim(min(points[:,1]),max(points[:,1]))

    voronoi_cells = list(compress(voronoi_cells,keep_points))
    voronoi_areas = list(compress(voronoi_areas,keep_points))
    fig, ax = plt.subplots()
    patches = []
    for counter,cell in enumerate(voronoi_cells):
        if voronoi_areas[counter] != math.inf:
            patches.append(Polygon(cell))
    collection = PatchCollection(patches, alpha = 0.1, edgecolor = 'r', cmap=cm.hsv)
    color = [i for i in voronoi_areas if i!=math.inf]
    collection.set_array(np.log(np.array(color)))
    ax.add_collection(collection)
    plt.scatter(points[:,0], points[:,1], s=1, color = 'b')
    plt.show()
    plt.xlim(min(points[:,0]),max(points[:,0]))
    plt.ylim(min(points[:,1]),max(points[:,1]))

points = np.random.random([number_of_points,2])
neighbors = neighbors_delauay(points)
voronoi_areas, voronoi_cells =  find_voronoi(points)
area_threshold = np.percentile(voronoi_areas,area_threshold)
keep_points = [True if i<=area_threshold else False for i in voronoi_areas]
clusters = find_clusters(keep_points,neighbors)
clusters = list(filter(lambda x: len(x)>=min_number_of_localizations,clusters))
clusters_voronoi_cells = [[voronoi_cells[k] for k in i] for i in clusters]
clusters_voronoi_areas = [[voronoi_areas[k] for k in i] for i in clusters]

plot_voronoi(points,voronoi_cells,voronoi_areas,keep_points)
plot_clusters(clusters_voronoi_cells,clusters_voronoi_areas)


# -*- coding: utf-8 -*-
from __future__ import division

from annotation.models import Document, Label, Score
from annotation.models import NBC_class_count, NBC_word_count_given_class, NBC_vocabulary
from django.db.models import F, FloatField, Sum, IntegerField

from nltk import word_tokenize
from math import log, exp
from operator import itemgetter, attrgetter
from itertools import groupby
from pprint import pprint
import numpy as np
import re
import sys

import pdb

def update_class_count(labels):
    # This function updates the counts for the class prior. If there
    # are existing class count objects, their counts are being
    # updated. Otherwise the objects are initialized with 0.
    for label in map(lambda l: l.label, labels):
        cc, created = NBC_class_count.objects.get_or_create(label=label)
        if not created:
            cc.count = cc.count +1
            cc.save()

def update_word_count(tokens, labelObjs):
    # To compute the conditional probability of a term given the class
    # this function counts how often each term occurs in each class of
    # the training document.
    labels = map(lambda l: l.label, labelObjs)
    for token in tokens:
        for label in labels:
            try:
                vocab, created_vocab = NBC_vocabulary.objects.get_or_create(word=token)
            except:
                duplicates = NBC_vocabulary.objects.filter(word=token)
                duplicates.delete()
                wcgc = NBC_vocabulary(word=token)
                wcgc.save()
            try:
                wcgc, created_wcgc = NBC_word_count_given_class.objects.get_or_create(
                    word=token, label=label)
            except:
                duplicates = NBC_word_count_given_class.objects.filter(word=token, label=label)
                count = sum(map(attrgetter('count'), duplicates))
                duplicates.delete()
                wcgc = NBC_word_count_given_class(word=token, label=label)
                wcgc.count = count
                wcgc.save()
                #
            if not created_wcgc:
                wcgc.count = wcgc.count +1
                wcgc.save()
                #
    ccs = list(NBC_class_count.objects.filter(label__in=labels))
    for cc in ccs:
        cc.total_word_count = cc.total_word_count + len(tokens)
        cc.save()

# def relative_term_freq(token, label):
#     # Conditional probability P(term|class)
#     vocab = NBC_vocabulary.objects.filter(word=token).first()
#     if vocab:
#         wcgc = NBC_word_count_given_class.objects.filter(
#             word=vocab, label=label).first()
#         if wcgc:
#             Tct = wcgc.count
#         else:
#             Tct = 0
#     else:
#         Tct = 0
#         #
#     Tct_sum = NBC_class_count.objects.filter(
#         label=label).first().total_word_count + NBC_vocabulary.objects.all().count()
#     return (Tct + 1) / Tct_sum

def preprocessing(raw_document):
    urls = r'(http.+?(\s|$))'
    specialchar =  r'|[^A-Za-z\s]'
    doc = raw_document.lower()
    tokens = word_tokenize(re.sub(urls+specialchar, ' ', doc))
    return tokens

def online_train(document, labels):
    update_class_count(labels)
    update_word_count(document.preprocessed.split(' '), labels)

def predict(document, saveScores=True):
    # To predict the label of a document first get the preprocessed
    # version.
    tokens = document.preprocessed.split(' ')
    scores = {} # holds the end result
    vocabulary_count = NBC_vocabulary.objects.all().count()
    # The calculation for each label can be done independently
    ccs = list(NBC_class_count.objects.all())
    labels = map(attrgetter('label'), ccs)
    #
    wcgcObjs = list(NBC_word_count_given_class.objects.filter(word__in=tokens,
                                                              label__in=labels))
    # This is a dict where each key is a label and each value is a
    # dict itself, where the keys are words and the values are the
    # counts of how often the word occured given the label.
    wcgcs = dict((label, dict((word, map(attrgetter('count'), w)[0])
                              for word, w in groupby(sorted(v, key=attrgetter('word')),
                                                     key=attrgetter('word'))))
                 for label, v in groupby(sorted(wcgcObjs, key=attrgetter('label')),
                                         key=attrgetter('label')))
    # Sum over the field 'count' in the table NBC_class_count to know
    # how many documents there are in total.
    N = sum(map(attrgetter('count'), ccs))
    for c in ccs:
        label = c.label
        scores[label] = {}
        if c.count:
            prior = log(c.count/N)
        else:
            prior = -sys.maxint -1
            #
        scores[label]['prior'] = prior
        scores[label]['term_given_label'] = 0
        Tct_tmp = wcgcs.get(label, 0)
        Tct_sum = c.total_word_count + vocabulary_count
        for token in tokens:
            if Tct_tmp:
                Tct = Tct_tmp.get(token, 0)
                    #
            relative_term_freq = (Tct + 1) / Tct_sum
            scores[label]['term_given_label'] += log(relative_term_freq)
            #
        scores[label]['total'] = scores[label]['prior']+scores[label]['term_given_label']
        #scores[label]['total'] = scores[label]['term_given_label']
        scores[label]['prior'] = exp(scores[label]['prior'])
        #scores[label]['term_given_label'] = exp(scores[label]['term_given_label'])
        scores[label]['total'] = exp(scores[label]['total'])
        #
    # ask eugen again
    for c in ccs:
        label = c.label
        scores[label]['normalized'] = np.power(scores[label]['total'], 1/(len(tokens)+1))
    # normalize
    total_sum = 0
    for c in ccs:
        label = c.label
        total_sum += scores[label]['normalized']
    for c in ccs:
        label = c.label
        if scores[label]['normalized'] != 0:
            scores[label]['normalized'] = scores[label]['normalized'] / total_sum
        else:
            scores[label]['normalized'] = 0.0
        # save scores to db
        # if saveScores:
        #     dbScore, created = Score.objects.get_or_create(document=document,
        #                                                    label=Label.objects.filter(label=c.label).first())
        #     dbScore.nbc_normalized = scores[label]['normalized']
        #     dbScore.nbc_prior = scores[label]['prior']
        #     dbScore.nbc_term_given_label = scores[label]['term_given_label']
        #     dbScore.nbc_total = scores[label]['total']
        #     dbScore.save()
    return scores

def predict_label(document, scores={}):
    # This function adds a little abstraction for convenience but more
    # importantly it decides which label is picked based on the
    # probabilities for each label.
    if scores:
        scores = map(lambda l: (l[0], l[1]['total']), scores.items())
    else:
        scores = map(lambda l: (l[0], l[1]['total']),
                     predict(document).items())
    if scores:
        predicted_label = Label.objects.filter(
            label=max(scores,key=itemgetter(1))[0]).first()
    else:
        predicted_label = None
    return predicted_label

    # predict the labels for all scores
    # pred_label = map(lambda score: predict_label(None, score), pred_scores)
    # go over all pred_scores and select the normalized score for the
    # predicted label.
    # pred_norm = map(lambda idx:
    #                 pred_scores[idx][pred_label[idx].label]['normalized'],
    #                 range(len(documents)))


# -*- coding: utf-8 -*-
import string, time, subprocess
from flask import  request
from datetime import datetime
from library.common import Common

from configparser import ConfigParser
from library.config_parser import configSectionParser

class VesselVPN(Common):

    # INITIALIZE
    def __init__(self):

        # INIT CONFIG
        self.config = ConfigParser()

        # CONFIG FILE
        self.config.read("config/config.cfg")

        self.token = configSectionParser(self.config,"TOKEN")['token']

    # GET VESSEL FUNCTION
    def vessel_vpn(self):
        """
        This API is for VPN
        ---
        tags:
          - VPN
        produces:
          - application/json
        parameters:
          - name: token
            in: header
            description: Token
            required: true
            type: string
          - name: query
            in: body
            description: VPN
            required: true
            schema:
              id: vessel_vpn
              properties:
                callback_url:
                    type: string
                vessel_number:
                    type: string
                vessel_name:
                    type: string
                job_id:
                    type: integer

        responses:
          500:
            description: Error
          200:
            description: Sending invitaion
        """

        # INIT DATA
        data = {}

        # GET DATA
        token =  request.headers.get('token')

        # GET JSON REQUEST
        query_json = request.get_json(force=True)

        job_id = query_json["job_id"]
        callback_url = query_json["callback_url"]
        vessel_number = query_json["vessel_number"]
        vessel_name = query_json["vessel_name"]

        # # CHECK TOKEN
        if not token == self.token:

            data["alert"] = "Invalid Token"
            data['status'] ='Failed'

            # RETURN ALERT
            return self.return_data(data)

        directory = '/home/vvpn/vpn_access'
        process_name = 'script_vpn_real_vessel_auto_creation.py'
        script_file = "%s/%s" % ( directory, process_name)
        print("script_file: ", script_file)
        print("job_id: ", job_id)
        print("callback_url: ", callback_url)
        print("vessel_number: ", vessel_number)
        print("vessel_name: ", vessel_name)
        print("token: ", token)
        print('python3.5', script_file, '-job_id', str(job_id), '-callback_url', str(callback_url), '-vessel_number', str(vessel_number), '-vessel_name', str(vessel_name), '-token', str(token), '-vessel_os', 'LINUX')
        subprocess.Popen(['sudo','python3.5', script_file, '-job_id', str(job_id), '-callback_url', str(callback_url), '-vessel_number', str(vessel_number), '-vessel_name', str(vessel_name), '-token', str(token), '-vessel_os', 'LINUX'])

        # print('python3.5', script_file, '-job_id', str(job_id), '-callback_url', str(callback_url), '-vessel_number', str(vessel_number), '-token', str(token))
        # subprocess.Popen(['sudo','python3.5', script_file, '-job_id', str(job_id), '-callback_url', str(callback_url), '-vessel_number', str(vessel_number), '-token', str(token)])

        data['status'] ='ok'

        return self.return_data(data)


import asyncio
import datetime
import discord
from discord.ext import commands, tasks
from ee250_final.user import User
import io
import json
import logging
import os
import paho.mqtt.client as mqtt
from PIL import Image
import schedule
import threading
import time

logging.basicConfig(level=logging.INFO)

class Commands:
    LED_OFF = "LED off"
    LED_ON = "LED on"
    FEED = "feed"

class Topics:
    FEEDER = "{username}/feeder"
    IMAGE = "rpi/webcam"
    LED = "{username}/LED"

class DiscordPub(commands.Cog):

    def __init__(self, bot: commands.Bot, channel_id, host, port, keepalive):
        self._feeder_scheduler = schedule.Scheduler()
        self._sched_stop = threading.Event()
        self._scheduler_thread = threading.Thread(target=self._feeder_timer)
        self._scheduler_thread.start()

        self._bot = bot
        self._channel_id = channel_id
        self._mqtt_client = mqtt.Client()
        self._mqtt_client.on_connect = self._on_connect
        self._mqtt_client.connect(host=host, port=port, keepalive=keepalive)
        self._mqtt_client.loop_start()
        self._users = []
        self._is_publish_image = False

        self.check_for_images_to_send.start()

    def _feeder_timer(self):
        while not self._sched_stop.is_set():
            self._feeder_scheduler.run_pending()
            time.sleep(10)

    def _on_connect(self, client, userdata, flags, rc):
        logging.info("MQTT publisher client connected to server with code {}".format(str(rc)))
        self._mqtt_client.subscribe(Topics.IMAGE)
        self._mqtt_client.message_callback_add(Topics.IMAGE, self._save_image_data)

    def _save_image_data(self, client, userdata, msg):
        with open("scripts/temp.jpg", "wb") as f:
            f.write(msg.payload)
        self._is_publish_image = True

    @tasks.loop(seconds=10.0)
    async def check_for_images_to_send(self):
        if self._is_publish_image:
            channel = await self._bot.fetch_channel(self._channel_id)
            await channel.send(file=discord.File("scripts/temp.jpg"))
            logging.info("Image sent from bot")
            os.remove("scripts/temp.jpg")
            self._is_publish_image = False

    @commands.Cog.listener()
    async def on_member_join(self, member):
        channel = member.guild.system_channel
        if channel is not None:
            await channel.send('hello, {0.mention}!'.format(member))

    @commands.command(
        name="set", help="set times to trigger the automatic feeder"
    )
    async def ask_to_set_times(self, message: discord.Message):
        name = message.author.name
        user = self._find_user(name)

        await message.channel.send("enter times (hh:mm:ss) separated by a space.")

        def is_valid_time(message):
            times_array = message.content.split()
            for value in times_array:
                try:
                    datetime.datetime.strptime(value, "%H:%M:%S")
                except ValueError:
                    return False
            return True

        try:
            times: discord.Message = await self._bot.wait_for(
                "message", check=is_valid_time, timeout=40.0
            )
        except asyncio.TimeoutError:
            return await message.channel.send(
                "sorry, you took too long. please enter the command again."
            )

        self._save_times(times, user)
        await message.channel.send("cool, thanks!")

    @commands.command(
        name="feed", help="send 'feed' command to rpi feeder"
    )
    async def send_feed_command(self, message: discord.Message):
        name = message.author.name
        user: User = self._find_user(name)
        self._mqtt_client.publish(user.feeder_topic, Commands.FEED)

    @commands.command(
        name="LED_off", help="Send message to RPI to turn off LED"
    )
    async def turn_led_off(self, message: discord.Message):
        name = message.author.name
        user = self._find_user(name)
        logging.info(user)
        logging.info(user.led_topic)
        if user is not None:
            self._mqtt_client.publish(user.led_topic, Commands.LED_OFF)

    @commands.command(
        name="LED_on", help="Send message to RPI to turn on LED"
    )
    async def turn_led_on(self, message: discord.Message):
        name = message.author.name
        user = self._find_user(name)
        self._mqtt_client.publish(user.led_topic, Commands.LED_ON)

    def add_user(self, name: str):
        user = User(name)
        self._users.append(user)
        logging.info("New user added: {}".format(name))
        return user

    def _find_user(self, name: str, create_user_if_not_exists=True):
        user_to_return = None
        for user in self._users:
            if user.name == name:
                return user
        if create_user_if_not_exists:
            user_to_return = self.add_user(name)
        return user_to_return

    def _save_times(self, message: discord.Message, user: User):
        times_array = message.content.split()
        user.add_feeding_times(times_array)
        for time in times_array:
            self._feeder_scheduler.every().day.at(time).do(
                self._mqtt_client.publish,
                user.feeder_topic,
                Commands.FEED
            )


from django.contrib import admin
from django.urls import path
from messagesApi import views

urlpatterns = [
    path("admin/", admin.site.urls),
    path('',views.api),
    path("login/", views.loginUser,name='login'),
    path("logout/", views.logoutUser),
    path("getusers/",views.getAllUsers),
    path("getMessages/",views.messagesList),
    path("getUnReadMessages/",views.UnreadmessagesList),
    path("readMessage/",views.readMessage),
    path("writeMessage/",views.writeMessage),
    path("deleteMessage/<int:id>",views.deleteMessage)
]


import os
import json
from tqdm import tqdm,trange
from udicOpenData.stopwords import *
import numpy as np

def getContent(allFileList):
    content_list = []
    for files in tqdm(allFileList):
        with open(path + files,"r",encoding='UTF-8') as f:
            jf = json.load(f)
            articles_list = jf['articles']
            for article in articles_list:
                if article['article_title']:
                    content_list.append(article['content'])
    return content_list
                    # words = jieba.cut(article['content'])
                    # for word in words:
                    #     print(word)
                    # if "韓國瑜" in article['content']:
                    #     has_han += 1
                    # else:
                    #     unhan += 1
    # print(has_han,unhan)
def content_seg(content_list):
    seg_content = []
    for content in tqdm(content_list):
        seg_content.append(list(rmsw(content)))
        # for i in list(rmsw(content)):
        #     if i in seg_content:
        #         seg_content[i] += 1
        #     else:
        #         seg_content[i] = 1
    return seg_content

if __name__ == "__main__":
    path =  "Gossiping/"
    allFileList = os.listdir(path)
    content_list = getContent(allFileList)
    seg_content = content_seg(content_list)

    # print(seg_content[0])
    np.save("seg_data",seg_content)
    # with open("data","w",encoding='UTF-8') as f:
    #     f.write(seg_content)


from django.conf.urls import url
from . import views

urlpatterns = [

    # Main project workflow urls

    url(r'^project/create$',
        views.ProjectViewSet.as_view({'post': 'create'}),
        name='project-create',
    ),
    url(r'^project/list$',
        views.ProjectViewSet.as_view({'get': 'list'}),
        name='project-list',
    ),
    url(r'^project/get$',
        views.ProjectViewSet.as_view({'get': 'retrieve'}),
        name='project-retrieve',
    ),
    url(r'^project/update$',
        views.ProjectViewSet.as_view({'post': 'update'}),
        name='project-update',
    ),

    url(r'^container/get$',
        views.ContainerViewSet.as_view({'get': 'retrieve'}),
        name='container-detail',
    ),
    url(r'^container/list_children$',
        views.ContainerViewSet.as_view({'get': 'list_children'}),
        name='container-list-children',
    ),

    url(r'^work/get$',
        views.WorkViewSet.as_view({'get': 'retrieve'}),
        name='work-detail',
    ),
    url(r'^work/start$',
        views.WorkViewSet.as_view({'get': 'start'}),
        name='work-start',
    ),
    url(r'^work/complete$',
        views.WorkViewSet.as_view({'get': 'complete'}),
        name='work-complete',
    ),
    url(r'^work/verify$',
        views.WorkViewSet.as_view({'get': 'verify'}),
        name='work-verify',
    ),
    url(r'^work/deny$',
        views.WorkViewSet.as_view({'get': 'deny'}),
        name='work-deny',
    ),
    url(r'^work/reactivate$',
        views.WorkViewSet.as_view({'get': 'reactivate'}),
        name='work-reactivate',
    ),
    url(r'^work/free-cash/list/available$',
        views.WorkViewSet.as_view({'get': 'list_free_cash_available'}),
        name='list-free-cash-available',
    ),
     url(r'^work/free-cash/list/my$',
        views.WorkViewSet.as_view({'get': 'list_free_cash_workon'}),
        name='list-free-cash-workon',
    ),
    url(r'^work/post_message$',
        views.WorkViewSet.as_view({'post': 'post_message'}),
        name='work-post-message',
    ),

    url(r'^resource/get$',
        views.ResourceViewSet.as_view({'get': 'retrieve'}),
        name='resource-get',
    ),
    url(r'^resource/update$',
        views.ResourceViewSet.as_view({'post': 'update'}),
        name='resource-update',
    ),
    url(r'^resource/file_upload$',
        views.ResourceFileViewSet.as_view({'post': 'put_file'}),
        name='resource-file-upload',
    ),

    # Config generation urls

    url(r'^project_template/create$',
        views.ProjectTemplateViewSet.as_view({'post': 'create'}),
        name='project-template-create',
    ),
    url(r'^project_template/list$',
        views.ProjectTemplateViewSet.as_view({'get': 'list'}),
        name='project-template-list',
    ),
    url(r'^project_template/get$',
        views.ProjectTemplateViewSet.as_view({'get': 'retrieve'}),
        name='project-template-retrieve',
    ),
    url(r'^project_template/update$',
        views.ProjectTemplateViewSet.as_view({'post': 'update'}),
        name='project-template-update',
    ),
]