import os
import json
import pandas as pd
from datetime import datetime

now = datetime.now()
print('start time : ', now.strftime('%Y-%m-%d %H:%M:%S'))

path = r'D:\project\구문검사결과지\test'
path_to = r'D:\project\구문검사결과지\results'
csvanme = r'results.csv'

df_total = pd.DataFrame(columns=['파일명', '오류 위치','검사 항목','검사 조건','오류 값','오류 메시지'])
list_filename = []
list_errorposition = []
list_testele = []
list_testcondition = []
list_errorvalue = []
list_errormessage = []

list_json = os.listdir(path)

totalcount_errorele = 0
totalcount_errorfile = 0
totalcount_file = len(list_json)
totalcount_ele = 0

for json_ in list_json:
    path_json = os.path.join(path, json_)

    with open(path_json, "r", encoding='UTF8') as json_ele:
        dict_jsoncontents = json.load(json_ele)

    count_errorele = 0
    count_ele = 0

    #####################################
    ###          images 검사          ###
    #####################################

    # /images/id 검사
    if dict_jsoncontents['images']['id'] == '':
        list_filename.append(json_)
        list_errorposition.append('/images/id')
        list_testele.append('필수여부')
        list_testcondition.append(' ')
        list_errorvalue.append(' ')
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /images/width 검사
    if dict_jsoncontents['images']['width'] == '' :
        list_filename.append(json_)
        list_errorposition.append('/images/width')
        list_testele.append('최소값')
        list_testcondition.append('400')
        list_errorvalue.append(dict_jsoncontents['images']['width'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1

    elif int(dict_jsoncontents['images']['width']) < 400 :
        list_filename.append(json_)
        list_errorposition.append('/images/width')
        list_testele.append('최소값')
        list_testcondition.append('400')
        list_errorvalue.append(dict_jsoncontents['images']['width'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1

    else:
        count_ele += 1

    # /images/height 검사
    if dict_jsoncontents['images']['height'] == '' :
        list_filename.append(json_)
        list_errorposition.append('/images/height')
        list_testele.append('최소값')
        list_testcondition.append('400')
        list_errorvalue.append(dict_jsoncontents['images']['height'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1

    elif int(dict_jsoncontents['images']['height']) < 400 :
        list_filename.append(json_)
        list_errorposition.append('/images/height')
        list_testele.append('최소값')
        list_testcondition.append('400')
        list_errorvalue.append(dict_jsoncontents['images']['height'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1

    else:
        count_ele += 1

    # /images/filename 검사
    if dict_jsoncontents['images']['filename'] == '':
        list_filename.append(json_)
        list_errorposition.append('/images/filename')
        list_testele.append('필수여부')
        list_testcondition.append(' ')
        list_errorvalue.append(' ')
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /images/volume 검사
    if 'volume' in  dict_jsoncontents['images'].keys():
        count_ele += 1

    #####################################
    ###          metas 검사           ###
    #####################################

    # /metas/id 검사
    if 'id' in dict_jsoncontents['metas'].keys():
        count_ele += 1

    # /metas/age 검사
    if dict_jsoncontents['metas']['age'] == '':
        list_filename.append(json_)
        list_errorposition.append('/metas/age')
        list_testele.append('필수여부')
        list_testcondition.append(' ')
        list_errorvalue.append(' ')
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/gender 검사
    if dict_jsoncontents['metas']['gender'] not in ['M', 'F']:
        list_filename.append(json_)
        list_errorposition.append('/metas/gender')
        list_testele.append('유효값')
        list_testcondition.append("\"M\",\"F\"")
        list_errorvalue.append(dict_jsoncontents['metas']['gender'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/phase 검사
    if dict_jsoncontents['metas']['phase'] not in ['Velum', 'OTE']:
        list_filename.append(json_)
        list_errorposition.append('/metas/phase')
        list_testele.append('유효값')
        list_testcondition.append("\"Velum\",\"OTE\"")
        list_errorvalue.append(dict_jsoncontents['metas']['phase'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/position 검사
    if dict_jsoncontents['metas']['position'] not in ['awake', 'supine','lateral','resupine']:
        list_filename.append(json_)
        list_errorposition.append('/metas/position')
        list_testele.append('유효값')
        list_testcondition.append("\"awake\",\"supine\",\"lateral\",\"resupine\"")
        list_errorvalue.append(dict_jsoncontents['metas']['position'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/SpO2 검사
    if dict_jsoncontents['metas']['SpO2'] == '':
        list_filename.append(json_)
        list_errorposition.append('/metas/SpO2')
        list_testele.append('필수여부')
        list_testcondition.append(' ')
        list_errorvalue.append(' ')
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/AHI 검사
    if dict_jsoncontents['metas']['AHI'] == '':
        list_filename.append(json_)
        list_errorposition.append('/metas/AHI')
        list_testele.append('필수여부')
        list_testcondition.append(' ')
        list_errorvalue.append(' ')
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/bmi 검사
    if 'bmi' in dict_jsoncontents['metas'].keys():
        count_ele += 1

    # /metas/position 검사
    if dict_jsoncontents['metas']['obstruction'] not in ['no', 'partial','complete']:
        list_filename.append(json_)
        list_errorposition.append('/metas/obstruction')
        list_testele.append('유효값')
        list_testcondition.append("\"no\",\"partial\",\"complete\"")
        list_errorvalue.append(dict_jsoncontents['metas']['obstruction'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    # /metas/cause 검사
    if dict_jsoncontents['metas']['cause'] not in ['no', 'Velum','Tongue_Base','Epiglottis']:
        list_filename.append(json_)
        list_errorposition.append('/metas/cause')
        list_testele.append('유효값')
        list_testcondition.append("\"no\",\"Velum\",\"Tongue_Base\",\"Epiglottis\"")
        list_errorvalue.append(dict_jsoncontents['metas']['cause'])
        list_errormessage.append('_')
        count_errorele +=1
        count_ele += 1
    else:
        count_ele += 1

    #####################################
    ###        annoatation 검사       ###
    #####################################

    for annotation_ele in dict_jsoncontents['annotations']:

        # /annotation/id 검사
        if annotation_ele['id'] == '':
            list_filename.append(json_)
            list_errorposition.append('/annotations/id')
            list_testele.append('필수여부')
            list_testcondition.append(' ')
            list_errorvalue.append(' ')
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/image_id 검사
        if annotation_ele['image_id'] == '':
            list_filename.append(json_)
            list_errorposition.append('/annotations/image_id')
            list_testele.append('필수여부')
            list_testcondition.append(' ')
            list_errorvalue.append(' ')
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/category_id 검사
        if annotation_ele['category_id'] not in ['6000', '6001','6002','6003','6004','6005']:
            list_filename.append(json_)
            list_errorposition.append('/annotations/category_id')
            list_testele.append('유효값')
            list_testcondition.append("\"6000\",\"6001\",\"6002\",\"6003\",\"6004\",\"6005\"")
            list_errorvalue.append(annotation_ele['category_id'])
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/category_name 검사
        if annotation_ele['category_name'] not in ['Velum', 'Airway1','Posterior_lateral_wall','Tongue_Base','Epiglottis','Airway2']:
            list_filename.append(json_)
            list_errorposition.append('/annotations/category_name')
            list_testele.append('유효값')
            list_testcondition.append("\"Velum\",\"Airway1\",\"Posterior_lateral_wall\",\"Tongue_Base\",\"Epiglottis\",\"Airway2\"")
            list_errorvalue.append(annotation_ele['category_name'])
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/type 검사
        if annotation_ele['type'] not in ['poly']:
            list_filename.append(json_)
            list_errorposition.append('/annotations/type')
            list_testele.append('유효값')
            list_testcondition.append("\"poly\"")
            list_errorvalue.append(annotation_ele['type'])
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/color 검사
        if annotation_ele['color'] == '':
            list_filename.append(json_)
            list_errorposition.append('/annotations/color')
            list_testele.append('필수여부')
            list_testcondition.append(' ')
            list_errorvalue.append(' ')
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

        # /annotation/points 검사
        if annotation_ele['points'] == []:
            list_filename.append(json_)
            list_errorposition.append('/annotations/points')
            list_testele.append('필수여부')
            list_testcondition.append(' ')
            list_errorvalue.append(' ')
            list_errormessage.append('_')
            count_errorele +=1
            count_ele += 1
        else:
            count_ele += 1

            for points_ele in annotation_ele['points']:

                # /annotation/points/[] 검사
                if points_ele == []:
                    list_filename.append(json_)
                    list_errorposition.append('/annotations/points/[]')
                    list_testele.append('필수여부')
                    list_testcondition.append(' ')
                    list_errorvalue.append(' ')
                    list_errormessage.append('_')
                    count_errorele +=1
                    count_ele += 1
                else:
                    count_ele += 1

    totalcount_errorele += count_errorele
    totalcount_ele += count_ele

    if count_errorele > 0 :
        totalcount_errorfile += 1

df_total['파일명'] = list_filename
df_total['오류 위치'] = list_errorposition
df_total['검사 항목'] = list_testele
df_total['검사 조건'] = list_testcondition
df_total['오류 값'] = list_errorvalue
df_total['오류 메시지'] = list_errormessage

path_results = os.path.join(path_to, csvanme)
df_total.to_csv(path_results, encoding = 'cp949', index = False)

print('정확도(파일): ', round(((totalcount_file-totalcount_errorfile)/totalcount_file)*100,2),'%')
print('정확도(항목): ', round(((totalcount_ele-totalcount_errorele)/totalcount_ele)*100,2),'%')
print('오류 건수(파일): ', totalcount_errorfile)
print('오류 건수(항목): ', totalcount_errorele)
print('전체 건수(파일): ', totalcount_file)
print('전체 건수(항목): ', totalcount_ele)

now = datetime.now()
print('end time : ', now.strftime('%Y-%m-%d %H:%M:%S'))


import discord, json, typing, datetime, asyncio
from discord.ext import commands
from utils import errors, checks

def get_embed(title, description='', color=0xf4fa72):
    return discord.Embed(title=title,description=description,color=color)

class profile(commands.Cog):
    def __init__(self, client):
        self.client = client
        self.pool = self.client.pool
        self.checks = checks.checks(self.pool)

        self._profile_setup.add_check(self.checks.registered)

    @commands.group(name = "프로필", invoke_without_command = True)
    async def _profile(self, ctx, user: typing.Optional[discord.Member] = None):
        if ctx.guild is None:
            await ctx.send(embed=get_embed("<a:no:698461934613168199> | 서버내에서만 사용가능한 명령어 입니다.",0xff0000))
            return
        if not user: user = ctx.author

        try:
            st = str(user.status)
            if st == "online": sta = ":green_circle: 온라인"
            elif st == "offline": sta = ":black_circle: 오프라인"
            elif st == "idle": sta = ":yellow_circle: 자리 비움"
            else: sta = ":no_entry: 방해 금지"
        except: sta = "불러오는데 실패"

        if str(user.id) in self.pool: embcolor = int(self.pool[str(user.id)]["profile"]["color"], 16)
        else: embcolor = 0xf4fa72
        embed = discord.Embed(title=f"👤 | **{user.name} 님의 프로필**", description=("" if user.name == user.display_name else f"**서버내 닉네임**: {user.display_name}\n") + f'**유저 ID**: {user.id}\n**현재 상태**: {sta}',color=embcolor)
        embed.set_thumbnail(url=user.avatar_url)

        date = datetime.datetime.utcfromtimestamp(((int(user.id) >> 22) + 1420070400000)/1000)
        embed.add_field(name="Discord 가입 일시", value=str(date.year) + "년 " + str(date.month) + "월 " + str(date.day) + "일 ")
        joat = user.joined_at.isoformat()
        embed.add_field(name="서버 가입 일시", value=joat[0:4]+'년 '+joat[5:7]+'월 '+joat[8:10]+'일')

        if user.bot:
            embed.add_field(name="봇 초대장 생성",value=f"[초대장](https://discord.com/oauth2/authorize?client_id={user.id}&scope=bot&permissions=0)")
        else:
            if user.guild_permissions.administrator: embed.add_field(name="서버 권한", value="Admin")
            else: embed.add_field(name="서버 권한", value="User")
            if str(user.id) in self.pool.keys():
                if self.pool[str(user.id)]["profile"]["money_open"]:
                    reinlist = [[k, v["starforce"]] for k, v in self.pool[str(user.id)]["reinforce"].items() if v["level"] >= 100]
                    str_ = [f':star:**x{i[1]}** {i[0]}' for i in reinlist]
                    embed.add_field(
                        name="Lemon System",
                        value=f">>> **Permission**: {self.pool[str(user.id)]['permission']}\n" +
                        ("\n**블랙리스트에 등재된 유저입니다!**" if self.pool[str(user.id)]['blacklist'] else
                        f"**Money**: {self.pool[str(user.id)]['money']}원\n" +
                        f"**bank**: {self.pool[str(user.id)]['bank']}원\n" +
                        f"**100레벨 이상 강화 아이템 갯수** : {len(reinlist)}개\n" + ', '.join(str_)
                        )
                    )
                else:
                    embed.add_field(
                        name = "Lemon System",
                        value = "프로필 비공개 유저입니다!"
                    )
        await ctx.send(embed=embed)

    @_profile.command(name = "설정")
    async def _profile_setup(self, ctx):
        color = hex(int(self.pool[str(ctx.author.id)]["profile"]["color"], 16))
        show = "공개" if self.pool[str(ctx.author.id)]["profile"]["money_open"] else "비공개"
        des = f"✏ : 프로필 색 변경 (현재 색 {color})\n🔍 : 프로필 비공개 설정 (현재 {show})\n❌ : 취소"
        msg = await ctx.send(embed=get_embed("⚙ | 프로필 설정", des))
        emjs=["✏", "🔍", "❌"]
        for i in emjs:
            await msg.add_reaction(i)
        def check(reaction, user):
            return user == ctx.author and msg.id == reaction.message.id and str(reaction.emoji) in emjs
        try:
            reaction, user = await self.client.wait_for('reaction_add', check=check, timeout=60)
        except asyncio.TimeoutError:
            await asyncio.gather(msg.delete(),ctx.send(embed=get_embed('⏰ | 시간이 초과되었습니다!',"", 0xFF0000)))
            return
        else:
            e = str(reaction.emoji)
            if e == "✏":

                await ctx.send(f"프로필 카드 색을 변경합니다!\n현재 색은 **{color}**입니다.\n변경할 색을 **0x를 붙인 HEX코드로** 적어주세요!")

                def check(author):
                    def inner_check(message):
                        if message.author != author: return False
                        else: return True
                    return inner_check

                try: msg = await self.client.wait_for('message',check=check(ctx.author),timeout=20)
                except asyncio.TimeoutError:
                    self.gaming_list.remove(ctx.author.id)
                    await ctx.send(embed=get_embed('⏰ | 시간이 초과되었습니다!',"", 0xFF0000))
                    return
                else:
                    n = msg.content
                    await ctx.send(n)
                    try: scan = str(hex(int(n, 16)))
                    except: await ctx.send("올바르지 않은 입력 양식입니다!")
                    else:
                        if (n is not scan) or not (0 < int(n,16) < 376926741):
                            self.pool[str(user.id)]["profile"]["color"] = n
                            color = self.pool[str(user.id)]["profile"]["color"]
                            await ctx.send(f"프로필 카드 색을 {color}로 변경하였습니다!")
                            return

                    await ctx.send("올바르지 않은 입력 양식입니다!")

            elif e == "🔍":
                self.pool[str(ctx.author.id)]["profile"]["money_open"] = ~self.pool[str(ctx.author.id)]["profile"]["money_open"]
                show = "공개" if self.pool[str(ctx.author.id)]["profile"]["money_open"] else "비공개"
                await ctx.send(f"프로필을 {show}상태로 변경합니다!")

            elif e == "❌":
                await ctx.send("취소되었습니다.")

def setup(client):
    client.add_cog(profile(client))


from flask import render_template, session, url_for, redirect, send_from_directory, request, flash
from health_tracker import app, db
from health_tracker.forms import HealthForm, LoginForm, RegistrationForm
from health_tracker.models import Entry, User
from health_tracker.graphics import UserData
from datetime import datetime
from flask_login import current_user, login_user, login_required, logout_user
from werkzeug.urls import url_parse
import os

@app.route('/', methods=['post', 'get'])
def login():
    if current_user.is_authenticated:
        return redirect(url_for('survey'))
    form = LoginForm()
    if form.validate_on_submit():
        user = User.query.filter_by(name=form.name.data).first()
        if user is None or not user.check_password(form.password.data):
            flash("Invalid name or password", 'error')
            return redirect(url_for('login'))
        login_user(user, remember=True)
        next_page = request.args.get('next')
        if not next_page or url_parse(next_page).netloc != '':
            next_page = url_for('survey')
        return redirect(next_page)
    return render_template('login.html', form=form)

@app.route('/logout')
def logout():
    logout_user()
    flash("Logged out.", 'info')
    return redirect(url_for('login'))

@app.route('/signup', methods=['post', 'get'])
def sign_up():
    sign_up_form = RegistrationForm()
    if sign_up_form.validate_on_submit():
        user = User(name=sign_up_form.name.data, password_hash="null")
        user.set_password(sign_up_form.password.data)
        db.session.add(user)
        db.session.commit()
        flash("Registration successful.", 'info')
        return redirect(url_for('login'))
    return render_template('sign_up.html', sign_up=sign_up_form)

@app.route('/survey', methods=['post', 'get'])
@login_required
def survey():
    name = current_user.name
    form = HealthForm()
    if request.method == 'POST' and form.validate():
        date_entry = form.date.data
        date = Entry.query.filter_by(name=name,
                                     date=datetime(date_entry.year, date_entry.month, date_entry.day)).first()
        if date:
            flash("That date already has an entry :(", "error")
            return render_template('survey.html', name=name.title(), form=form)
        entry = Entry(name=name,
                      date=datetime(date_entry.year, date_entry.month, date_entry.day),
                      hours_of_sleep=form.hours_of_sleep.data,
                      rest=form.rest.data,
                      fatigue=form.fatigue.data,
                      exercise=form.exercise.data,
                      meditation=form.meditation.data,
                      stress=form.stress.data,
                      emotion=form.emotion.data,
                      comfort=form.comfort.data,
                      arousal=form.arousal.data,
                      headache=form.headache.data,
                      cannabis=form.cannabis.data,
                      morning=form.morning.data
                      )
        db.session.add(entry)
        db.session.commit()
        return render_template('thank_you.html', name=name.title())
    else:
        return render_template('survey.html', name=name.title(), form=form)

@app.route('/data')
@login_required
def data():
    name = current_user.name
    user = UserData(name)
    user.get_data_sqlite()
    csv_file = '{}_data_{}.csv'.format(name, datetime.today().strftime("%Y"))
    xlsx_file = csv_file.replace('.csv', '.xlsx')
    if os.getcwd().endswith('health_tracker'):
        path = os.path.join(os.getcwd(), 'health_tracker', 'uploads')
    else:
        path = os.path.join(os.getcwd(), 'health_tracker', 'health_tracker', 'uploads')
    user.to_csv(os.path.join(path, csv_file))
    user.to_excel(os.path.join(path, xlsx_file))
    user.pygal_line_plot(['stress', 'fatigue', 'comfort', 'arousal', 'rest'])
    return render_template('data.html', user=user)

@app.route('/visualized_data')
@login_required
def visuals():
    name = current_user.name
    path = url_for('static', filename='{}_line_graph.svg'.format(name))
    return render_template('visuals.html', path=path)

@app.route('/download_csv')
@login_required
def download_csv():
    name = current_user.name
    file = '{}_data_{}.csv'.format(name, datetime.today().strftime("%Y"))
    if os.getcwd().endswith('health_tracker'):
        path = os.path.join(os.getcwd(), 'health_tracker', 'uploads')
    else:
        path = os.path.join(os.getcwd(), 'health_tracker', 'health_tracker', 'uploads')
    return send_from_directory(path, file, as_attachment=True)

@app.route('/download_xlsx')
@login_required
def download_xlsx():
    name = current_user.name
    file = '{}_data_{}.xlsx'.format(name, datetime.today().strftime("%Y"))
    if os.getcwd().endswith('health_tracker'):
        path = os.path.join(os.getcwd(), 'health_tracker', 'uploads')
    else:
        path = os.path.join(os.getcwd(), 'health_tracker', 'health_tracker', 'uploads')
    return send_from_directory(path, file, as_attachment=True)

@app.route('/table')
@login_required
def table():
    name = current_user.name
    user = UserData(name)
    user.get_data_sqlite()
    user.pandas_df
    html = user.build_table()
    return render_template('table.html', html=html)


import random
from datetime import datetime

from django.core.cache import cache
from django.db import connection
from django.db.models import Q, Count
from django.shortcuts import render

# Create your views here.
from django.views import View
from ArticleManageSystem.settings import DATETIME_FORMAT
from common.models import Article, Content, FeedBack, Comment, Tag, CommentFeedBack
from utils.common_mixin import FrontLoggedMixin, FrontMixin

class UserArticleView(FrontLoggedMixin, View):
    def get_template(self):
        if self.if_list:
            self.model = Article
            page = self.request.GET.get("page")
            if not page:
                most_hot = Article.objects.filter(is_deleted=0,
                                                  status=4).values("title", "id").order_by("-read_times")[:5]
                most_comment = Comment.objects.filter(article__is_deleted=0, article__status=4)\
                    .values("article__title", "article_id").annotate(count=Count("article_id")).order_by("-count")[:5]
                self.context.update({"most_hot": most_hot, "most_comment": most_comment})
            self.template = "article_o/article_list_add.html"
        else:
            id = self.request.GET.get("id")
            id = int(id)
            self.model = Content
            self.template = "article_o/article_detail_4.html"
            good_view = FeedBack.objects.filter(article_id=id, feed_back=1).count()
            bad_view = FeedBack.objects.filter(article_id=id, feed_back=2).count()
            comments = Comment.objects.filter(article_id=id)
            for comment in comments:
                comment.good_view = CommentFeedBack.objects.filter(comment_id=comment.id, feed_back=1).count()
                comment.bad_view = CommentFeedBack.objects.filter(comment_id=comment.id, feed_back=2).count()
            comm_count = comments.count()
            this_tags = tuple(Article.objects.get(id=id).tags.all().values_list("id", flat=True))
            sql = "SELECT distinct a.id, a.title FROM common_article a left join common_article_tags t " \
                  "on a.id = t.article_id WHERE t.tag_id in %s and a.id != %s and a.status=4 and is_deleted=0"
            cursor = connection.cursor()
            cursor.execute(sql, [this_tags, id])
            if_like = list(cursor.fetchall())
            random.shuffle(if_like)
            extra = {}
            extra["if_like"] = if_like[:4]
            extra["article_id"] = id
            extra["good_view"] = good_view
            extra["bad_view"] = bad_view
            extra["comments"] = comments
            extra["comm_count"] = comm_count
            self.context.update(extra)

    def get_list_values(self):
        """
        如果不需要重命名就用字符串，否则前面写跨表的方式，values跨表用双下划线，后面写要改的名字
        """
        self.list_values = [
            "id",
            "img",
            "title",
            "author",
            "pub_time",
            ["category__name", "category_name"],
            # ["content__content", "content_val"],
        ]

    def get_detail_values(self):
        """
        如果不需要重命名就用字符串，否则前面写跨表的方式，详情跨表用点，后面写要改的名字
        """
        self.detail_values = [
            "content",
            ["article.title", "title"],
            ["article_id", "id"],
            ["article.category.name", "category"], ["article.tags.all().values('name', 'id')", "tags"],
            ["article.author", "author"]
        ]

    def get_change(self, request):
        if not self.if_list:
            obj_id = request.GET.get("id")
            try:
                obj = Article.objects.get(id=obj_id)
            except Exception:
                self.template = "public/error.html"
            else:
                obj.read_times += 1
                obj.last_read_time = datetime.now().strftime(DATETIME_FORMAT)
                obj.save()

    def get(self, request):
        cur_page = request.GET.get("page", "1")
        if cur_page != "1":
            self.is_ajax = 1
        self.and_field = {"status": 4, "is_deleted": 0}
        self.detail_id_field = "article_id"
        self.get_list_values()
        self.get_detail_values()
        self.get_change(request)
        if self.template:
            return render(request, self.template)
        self.get_template()
        return super().get(request)

class DianZan(FrontMixin, View):
    def post(self, request):
        id = request.POST.get("id")
        view = int(request.POST.get("view"))
        extra = {}
        extra["is_view"] = 0
        user = self.user_id
        check = FeedBack.objects.filter(article_id=id, user_id=user)
        if check:
            obj = check.first()
            if obj.feed_back == view:
                extra["is_view"] = 1
            else:
                obj.feed_back = view
            obj.save()
        else:
            FeedBack.objects.create(article_id=id, user_id=user, feed_back=view)
        self.is_ajax = 1
        good_view = FeedBack.objects.filter(article_id=id, feed_back=1).count()
        bad_view = FeedBack.objects.filter(article_id=id, feed_back=2).count()
        extra["good_view"] = good_view
        extra["bad_view"] = bad_view
        self.context.update(extra)
        return super().post(request)

class CommentView(FrontMixin, View):
    def post(self, request):
        user = self.user_id
        id = request.POST.get("id")
        content = request.POST.get("content")
        parent_id = request.POST.get("parent_id")
        Comment.objects.create(article_id=id, parent_comment_id=parent_id, content=content, user_id=user)
        extra = {}
        extra["comments"] = Comment.objects.filter(article_id=id)
        self.is_ajax = 1
        return super().post(request)

class CommentDianZan(FrontMixin, View):
    def post(self, request):
        id = request.POST.get("id")
        view = int(request.POST.get("view"))
        extra = {}
        extra["is_view"] = 0
        user_id = self.user_id
        check = CommentFeedBack.objects.filter(comment_id=id, user_id=user_id)
        if check:
            obj = check.first()
            if obj.feed_back == view:
                extra["is_view"] = 1
            else:
                obj.feed_back = view
            obj.save()
        else:
            CommentFeedBack.objects.create(comment_id=id, user_id=user_id, feed_back=view)
        self.is_ajax = 1
        good_view = CommentFeedBack.objects.filter(comment_id=id, feed_back=1).count()
        bad_view = CommentFeedBack.objects.filter(comment_id=id, feed_back=2).count()
        extra["good_view"] = good_view
        extra["bad_view"] = bad_view
        self.context.update(extra)
        return super().post(request)


# Advent of Code 2016, Day 7
# (c) blu3r4y

import re

from more_itertools import flatten, windowed

def part1(ips):
    count = 0
    for sup, hyp in ips:
        if any(map(has_abba, sup)) and not any(map(has_abba, hyp)):
            count += 1
    return count

def part2(ips):
    count = 0
    for sup, hyp in ips:
        for a, b, _ in flatten(map(has_aba, sup)):
            bab = b + a + b
            if any([bab in e for e in hyp]):
                count += 1
                break
    return count

def has_abba(text):
    # rolling window with size = 4 and check
    for a, b, c, d in windowed(text, 4):
        if a == d and b == c and a != b:
            return True

def has_aba(text):
    matches = []
    # rolling window with size = 3 and check
    for a, b, c in windowed(text, 3):
        if a == c and a != b:
            matches.append(a + b + c)

    return matches

def load(path):
    data = []

    lines = open(path).read().splitlines()
    for line in lines:
        sup = re.split(r"\[|\]", line)  # main sequences
        hyp = re.findall(r"\[(\w+)\]", line)  # hypernet sequences
        for h in hyp:
            sup.remove(h)
        data.append((sup, hyp))

    return data

if __name__ == "__main__":
    print(part1(load(r"../assets/day7_demo1.txt")))
    print(part1(load(r"../assets/day7.txt")))

    print(part2(load(r"../assets/day7_demo2.txt")))
    print(part2(load(r"../assets/day7.txt")))


from spacy.matcher import Matcher
import re

class InformationExtractor():
    def __init__(self, nlp):
        self.nlp = nlp
        self.matcher = Matcher(self.nlp.vocab)
        self.matcher.add("SALUTATION", None, [{"LOWER": "good"}, {"LOWER": "morning"}],
                                     [{"LOWER": "good"}, {"LOWER": "evening"}],
                                     [{"LOWER": "good"}, {"LOWER": "afternoon"}],
                                     [{"LOWER": "good"}, {"LOWER": "night"}])

        mention_flag = lambda text: bool(re.compile(r'\@(\w+)').match(text))
        IS_MENTION = self.nlp.vocab.add_flag(mention_flag)

        self.matcher.add("MENTION", None, [{IS_MENTION: True}])

        self.matcher.add("PUNCT", None, [{"IS_PUNCT": True}])

        self.matcher.add("FEELING", None, [{"LOWER": "i"}, {"LEMMA":"be"},
                                         {"POS": "ADV", "OP": "*"},
                                         {"POS": "ADJ"}])

        self.matcher.add("DOING", None, [{"LOWER": "i"}, {"LEMMA": "be"},
                                         {"POS": "VERB"},
                                         {},
                                         {}])

    def extract_salutation(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city, "SALUTATION")

    def extract_mention(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city, "MENTION")

    def extract_punct(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city, "PUNCT")

    def extract_feeling(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city, "FEELING")

    def extract_doing(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city, "DOING")

    def extract_all_rules(self, text, city):
        matches = self.matcher(text)
        return self._matches_to_json(matches, text, city)

    def _matches_to_json(self, matches, txt, city, specific_rule=False):
        output = []
        if len(matches) != 0:
            for match in matches:
                rule = self.nlp.vocab.strings[match[0]]
                text = str(txt[match[1]:match[2]])

                if rule == "PUNCT":
                    if text not in ["!","?"]:
                        continue

                if not specific_rule:
                    output.append({'rule':rule,
                           'text':text,
                           'city': city})
                else:
                    if rule == specific_rule:
                        output.append({'rule':rule,
                           'text':text,
                           'city': city})
        return output


import sys
import os
import h5py
import numpy as np
import torch
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import torchvision.transforms.functional as TF
import torchvision.transforms as T
from PIL import Image

DATA_FOLDER_PATH = './dataset/'

data_file_name = {
    'nyu' : 'nyu_depth_v2_labeled.mat'
}

class MyDataset(Dataset):
  ''''
  Characterizes a dataset for PyTorch
  '''
  def __init__(self, inputs, labels, transform=None, target_transform=None):
        'Initialization'
        self.images = inputs
        self.labels = labels
        self.N = self.images.shape[0]
        self.transform = transform
        self.target_transform = target_transform

  def __len__(self):
        'Denotes the total number of samples'
        return self.N

  def __getitem__(self, index):
        'Generates one sample of data'
        # Select sample
        label = self.labels[index, :]
        img = self.images[index, :]

        #print("")
        #print(label.shape)
        #print(img.shape)
        org_size = list(img.shape)[0:2]
        img = np.moveaxis(img,0,-1)
        #img = T.ToTensor()(img)
        #label = T.ToTensor()(label)
        #print(img.shape)
        #print(label.shape)

        img = T.ToPILImage()(img)
        label = T.ToPILImage()(label)

        #print(img.size)
        #print(label.size)

        #img = np.transpose(img,(1,2,0))

        #img = T.ToPILImage()(img)
        #label = T.ToPILImage()(label)

        if self.transform is not None:

            #color_jitter = T.ColorJitter(brightness, contrast, saturation, hue)
            #img = T.RandomApply([color_jitter],p=0.6)(img)
            img = T.RandomApply([T.Grayscale(num_output_channels=3)],p=0.2)(img)
            if np.random.rand() > .5:
                #print("rot")
                r = np.random.uniform(-5,5)
                img = TF.rotate(img,r)
                label = TF.rotate(label,r)

            if np.random.rand() > 1:
                #print("scale")
                s = np.random.randint(1,3)

                size = [x*s for x in org_size]
                img = TF.resize(img,size)
                img = TF.center_crop(img,org_size)

                label = TF.resize(label,size)
                label = TF.center_crop(label,org_size)
                label = T.ToPILImage()(np.array(label)/s)

            if np.random.rand() > .4:
                #print("flip")
                img = TF.hflip(img)
                label = TF.hflip(label)

        else:
            pass
            #print("CRAP")

        img = T.ToTensor()(img)
        label = T.ToTensor()(label)

        #if self.transform is not None:
        img = T.Normalize([0.5145, 0.4555, 0.4275],[0.2541, 0.2605, 0.2748])(img)*255
        #else:
        #    img = img

        label = torch.squeeze(label)
        #print(img.shape)
        #print(label.shape)

        sample = {'image': img, 'label': label}

        return sample

  def set_transform(self, transform):
        self.transform = transform

def load_nyu_data(path):
    """
    Load NYU depth v2 dataset

    Inputs:
    - path

    Returns:
    data_x: (N, 3, H, W) numpy array of pixel value
    data_y: (N, H, W) numpy array of depth value
    """
    with h5py.File(path, 'r') as f:
        data_x = np.swapaxes(np.array(f.get('images')), 2, 3) # f shape: (N, 3, W, H)
        data_y = np.swapaxes(np.array(f.get('depths')), 1, 2)

    x_train_data = data_x[:960, ]
    y_train_data = data_y[:960, ]
    x_val_data = data_x[960:1200, ]
    y_val_data = data_y[960:1200, ]
    x_test_data = data_x[1200:,]
    y_test_data = data_y[1200:,]

    # transform = T.Compose([
    #         T.ToTensor(),
    #         T.Normalize((125.0117, 107.3517, 101.7845), (67.4572, 69.5167, 71.9448))
    #     ])
    # data_x = np.zeros((1449, 3, 480, 640))
    # data_y = np.zeros((1449, 480, 640))

    train_data = MyDataset(data_x[:1040, ], data_y[:1040, ])
    val_data = MyDataset(data_x[1040:1300, ], data_y[1040:1300, ])
    test_data = MyDataset(data_x[1300:,], data_y[1300:,]) # use first image for visualization!

    #loader = DataLoader(train_data, batch_size=1, num_workers=1)
    #data = next(iter(loader))

    #print(np.array(data['image'].view(x_train_data.shape[0],x_train_data.shape[1],-1)*1.0).shape)
    #xmean = torch.sum((T.ToTensor()(np.array(data['image'].view(x_train_data.shape[0],x_train_data.shape[1],-1)*1.0))).mean(1),0)/x_train_data.shape[0]
    #xstd = torch.sum((T.ToTensor()(np.array(data['image'].view(x_train_data.shape[0],x_train_data.shape[1],-1)*1.0))).std(1),0)/x_train_data.shape[0]
    #ymean = torch.sum((T.ToTensor()(np.array(data['label'].view(y_train_data.shape[0],-1)*1.0))).mean(0),0)/y_train_data.shape[0]
    #ystd = torch.sum((T.ToTensor()(np.array(data['label'].view(y_train_data.shape[0],-1)*1.0))).std(0),0)/y_train_data.shape[0]

    #print(xmean)
    #print(ymean)
    #print(xstd)
    #print(ystd)
    #stop

    return train_data, val_data, test_data

def load_data(dataset):
    """
    Load dataset

    Inputs:
    - dataset: name of the dataset, which should match the key of `data_file_name`.

    Returns:
    - data_x: (N, C, H, W) numpy array of pixel value
    - data_y: (N, H, W) numpy array of depth value
    """
    data_path = os.path.join(DATA_FOLDER_PATH, data_file_name[dataset])
    train_data, val_data, test_data = load_nyu_data(data_path)
    return train_data, val_data, test_data

def calMeanStd(dataset):
    loader = DataLoader(
        dataset,
        batch_size=10,
        num_workers=1,
        shuffle=False
    )

    mean = 0.
    std = 0.
    nb_samples = 0.
    for d in loader:
        data = d['image'].to(dtype=torch.float32)
        batch_samples = data.size(0)
        data = data.view(batch_samples, data.size(1), -1)
        mean += data.mean(2).sum(0)
        std += data.std(2).sum(0)
        nb_samples += batch_samples

    mean /= nb_samples
    std /= nb_samples
    return mean, std


# coding: utf-8

from pymysql import OperationalError
import sys
import os
import json
import shutil
from db.MysqlDatabaseConnector import MysqlDatabaseConnector
from service.SchemaComparision import SchemaComparison
from util.Configer import Configer

class DataBaseSchemaSync(object):
    """
    :type db_config: dict
    :type source: entity.MysqlDatabaseConnector
    :type target: entity.MysqlDatabaseConnector
    """
    def __init__(self, conf_path=None):
        try:
            configure = Configer(conf_path)
        except IOError:
            sys.stderr.write('configure file not exist.')
            sys.exit(1)

        self.db_config = json.loads(configure.config)

        try:
            self.source = MysqlDatabaseConnector(self.db_config['source'])
            self.source.fetch()
        except OperationalError:
            sys.stderr.write('database source connect error.')
            sys.exit(1)

        try:
            self.target = MysqlDatabaseConnector(self.db_config['target'])
            self.target.fetch()
        except OperationalError:
            sys.stderr.write('database target connect error.')
            sys.exit(1)

    def compare(self):
        if os.path.exists('update'):
            shutil.rmtree('update')
            os.mkdir('update')
        sc = SchemaComparison(self.source, self.target)
        sc.compare()

if __name__ == '__main__':
    foo = DataBaseSchemaSync()
    # for table in foo.source.database.tables:
    #     print(table.name)
    if len(sys.argv) <= 1:
        foo = DataBaseSchemaSync()
    else:
        foo = DataBaseSchemaSync(sys.argv[1])

    foo.compare()


import os
import asyncio
import copy
import json
import heapq
import logging
import operator
from collections import Counter
from collections.abc import Mapping
from .backends.message import SendMessage, TellMessage
from ..typing import BandType

logger = logging.getLogger(__name__)

MARS_ENABLE_PROFILING = int(os.environ.get("MARS_ENABLE_PROFILING", 0))

class _ProfilingOptionDescriptor:
    def __init__(self, _type, default):
        self._name = None
        self._type = _type
        self._default = default

    def __get__(self, obj, cls):
        if obj is None:
            return self
        v = obj._options.get(self._name)
        if v is None:
            v = os.environ.get(f"MARS_PROFILING_{self._name.upper()}", self._default)
        if v is not None:
            v = self._type(v)
        # Cache the value.
        obj.__dict__[self._name] = v
        return v

    def set_name(self, name: str):
        self._name = name

class _ProfilingOptionsMeta(type):
    def __init__(cls, name, bases, classdict):
        super(_ProfilingOptionsMeta, cls).__init__(name, bases, classdict)
        for k, v in classdict.items():
            if isinstance(v, _ProfilingOptionDescriptor):
                v.set_name(k)

class _ProfilingOptions(metaclass=_ProfilingOptionsMeta):
    debug_interval_seconds = _ProfilingOptionDescriptor(float, default=None)
    slow_calls_duration_threshold = _ProfilingOptionDescriptor(int, default=1)
    slow_subtasks_duration_threshold = _ProfilingOptionDescriptor(int, default=10)

    def __init__(self, options):
        if isinstance(options, Mapping):
            invalid_keys = options.keys() - type(self).__dict__.keys()
            if invalid_keys:
                raise ValueError(f"Invalid profiling options: {invalid_keys}")
            self._options = options
        elif options in (True, False, None):
            self._options = {}
        else:
            raise ValueError(f"Invalid profiling options: {options}")

class DummyOperator:
    @staticmethod
    def set(key, value):
        pass

    @staticmethod
    def inc(key, value):
        pass

    @staticmethod
    def nest(key):
        return DummyOperator

    @staticmethod
    def values():
        return []

    @staticmethod
    def empty():
        return True

class ProfilingDataOperator:
    __slots__ = ("_target",)

    def __init__(self, target):
        self._target = target

    def set(self, key, value):
        self._target[key] = value

    def inc(self, key, value):
        old = self._target.get(key, 0)
        self._target[key] = old + value

    def nest(self, key):
        v = self._target.setdefault(key, {})
        if not isinstance(v, dict):
            raise TypeError(
                f"The value type of key {key} is {type(v)}, but a dict is expected."
            )
        return ProfilingDataOperator(v)

    def values(self):
        return self._target.values()

    def empty(self):
        return len(self._target) == 0

class _CallStats:
    def __init__(self, options: _ProfilingOptions):
        self._options = options
        self._call_counter = Counter()
        self._slow_calls = []

    def collect(self, message, duration: float):
        key = (message.actor_ref.uid, message.content[0])
        self._call_counter[key] += 1
        if duration < self._options.slow_calls_duration_threshold:
            return
        key = (
            duration,
            message.actor_ref.uid,
            message.actor_ref.address,
            message.content,
        )
        try:
            if len(self._slow_calls) < 10:
                heapq.heappush(self._slow_calls, key)
            else:
                heapq.heapreplace(self._slow_calls, key)
        except TypeError:
            pass

    def to_dict(self) -> dict:
        most_calls = {}
        for name_tuple, count in self._call_counter.most_common(10):
            uid, method_name = name_tuple
            most_calls[f"{uid.decode('utf-8')}.{method_name}"] = count
        slow_calls = {}
        for duration, uid, address, content in sorted(
            self._slow_calls, key=operator.itemgetter(0), reverse=True
        ):
            method_name, _batch, args, kwargs = content
            slow_calls[
                f"[{address}]{uid.decode('utf-8')}.{method_name}(args={args}, kwargs={kwargs})"
            ] = duration
        return {"most_calls": most_calls, "slow_calls": slow_calls}

class _SubtaskStats:
    def __init__(self, options: _ProfilingOptions):
        self._options = options
        self._band_counter = Counter()
        self._slow_subtasks = []

    def collect(self, subtask, band: BandType, duration: float):
        band_address = band[0]
        self._band_counter[band_address] += 1
        if duration < self._options.slow_subtasks_duration_threshold:
            return
        key = (duration, band_address, subtask)
        try:
            if len(self._slow_subtasks) < 10:
                heapq.heappush(self._slow_subtasks, key)
            else:
                heapq.heapreplace(self._slow_subtasks, key)
        except TypeError:
            pass

    def to_dict(self) -> dict:
        band_subtasks = {}
        key = operator.itemgetter(1)
        if len(self._band_counter) > 10:
            items = self._band_counter.items()
            band_subtasks.update(heapq.nlargest(5, items, key=key))
            band_subtasks.update(reversed(heapq.nsmallest(5, items, key=key)))
        else:
            band_subtasks.update(
                sorted(self._band_counter.items(), key=key, reverse=True)
            )
        slow_subtasks = {}
        for duration, band, subtask in sorted(
            self._slow_subtasks, key=operator.itemgetter(0), reverse=True
        ):
            slow_subtasks[f"[{band}]{subtask}"] = duration
        return {"band_subtasks": band_subtasks, "slow_subtasks": slow_subtasks}

class _ProfilingData:
    def __init__(self):
        self._data = {}
        self._call_stats = {}
        self._subtask_stats = {}
        self._debug_task = {}

    def init(self, task_id: str, options=None):
        options = _ProfilingOptions(options)
        logger.info(
            "Init profiling data for task %s with debug interval seconds %s.",
            task_id,
            options.debug_interval_seconds,
        )
        self._data[task_id] = {
            "general": {},
            "serialization": {},
            "most_calls": {},
            "slow_calls": {},
            "band_subtasks": {},
            "slow_subtasks": {},
        }
        self._call_stats[task_id] = _CallStats(options)
        self._subtask_stats[task_id] = _SubtaskStats(options)

        async def _debug_profiling_log():
            while True:
                try:
                    r = self._data.get(task_id, None)
                    if r is None:
                        logger.info("Profiling debug log break.")
                        break
                    r = copy.copy(r)  # shadow copy is enough.
                    r.update(self._call_stats.get(task_id).to_dict())
                    r.update(self._subtask_stats.get(task_id).to_dict())
                    logger.warning("Profiling debug:\n%s", json.dumps(r, indent=4))
                except Exception:
                    logger.exception("Profiling debug log failed.")
                await asyncio.sleep(options.debug_interval_seconds)

        if options.debug_interval_seconds is not None:
            self._debug_task[task_id] = task = asyncio.create_task(
                _debug_profiling_log()
            )
            task.add_done_callback(lambda _: self._debug_task.pop(task_id, None))

    def pop(self, task_id: str):
        logger.info("Pop profiling data of task %s.", task_id)
        debug_task = self._debug_task.pop(task_id, None)
        if debug_task is not None:
            debug_task.cancel()
        r = self._data.pop(task_id, None)
        if r is not None:
            r.update(self._call_stats.pop(task_id).to_dict())
            r.update(self._subtask_stats.pop(task_id).to_dict())
        return r

    def collect_actor_call(self, message, duration: float):
        if self._call_stats:
            message_type = type(message)
            if message_type is SendMessage or message_type is TellMessage:
                for stats in self._call_stats.values():
                    stats.collect(message, duration)

    def collect_subtask(self, subtask, band: BandType, duration: float):
        if self._subtask_stats:
            stats = self._subtask_stats.get(subtask.task_id)
            if stats is not None:
                stats.collect(subtask, band, duration)

    def __getitem__(self, item):
        key = item if isinstance(item, tuple) else (item,)
        v = None
        d = self._data
        for k in key:
            v = d.get(k, None)
            if v is None:
                break
            else:
                d = v
        return DummyOperator if v is None else ProfilingDataOperator(v)

ProfilingData = _ProfilingData()


import pickle
from sqlitedict import SqliteDict
import zlib
import numpy as np
import time
import redis

conn = redis.StrictRedis('localhost', 6379, charset="utf-8", decode_responses=True)

# def decompress_set(obj):
#     return pickle.loads(zlib.decompress(bytes(obj)))
#
#
#
# res = []
# with SqliteDict('enlv_for_docs.sqlite', decode=decompress_set) as enlv_dct:
#     total_keys = len(enlv_dct)
#     print(total_keys)
#
#     start_time = time.time()
#     for i in range(total_keys):
#         if i%100000 == 0:
#             end_time = time.time()
#             print(i, end_time-start_time)
#             start_time = time.time()
#         res.append(enlv_dct[i])
#
# res = np.array(res, dtype=np.float32)
#
# with open('enlv_for_documents.pickle', 'wb') as handle:
#     pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL)

def test_tf_idf(file_id):
    # start with getting the id of a sample document from one of the docs e.g. 1928_in_association_football
    # Get the index of that document in the matrix (for the above example: 3917198)

    conn = redis.StrictRedis('localhost', 6379, charset="utf-8", decode_responses=True)
    val = int(conn.hget('doc_name_index', file_id))

    # Lets get the vector for this document from the doc_term_matrix
    with open('doc_term_matrix.pickle', 'rb') as input_file:
        doc_term_matrix = pickle.load(input_file)
        req_row = doc_term_matrix.getrow(val)

        col_indices_for_vocab = np.split(req_row.indices, req_row.indptr)[1:-1]

        document_frequency = doc_term_matrix.col
        uniq_indices, document_frequency = np.unique(document_frequency, return_counts=True)

        # Get respective idf values for the terms in this doc
        idf_vec = []
        for tkn_i in col_indices_for_vocab[0]:
            idf_val = document_frequency[tkn_i]
            idf_vec.append(idf_val)

        idf_vec = np.array(idf_vec)
        idf_vec = np.log10(5396041/idf_vec)

        # Get the normalised term frequency
        data_items = req_row.data
        print(data_items)
        data_items = data_items/(data_items.sum())
        print(data_items)

        tf_idf_vec = np.multiply(data_items, idf_vec)
        print(tf_idf_vec)

test_tf_idf('Nikolaj')

# with open('inverted_doc_name_dict.pickle', 'rb') as handle:
#     han = pickle.load(handle)
#     print(han['3568862'])


from PIL import Image
import numpy as np

def __convert_to_grayscale(image):
    """
    rgb to grayscale using the pillow library
    """
    return image.convert('L')

def __pixel_to_ascii_char(image):
    """
    converts the pixel value into a designated ascii char
    """
    ascii_list = []

    # list of values ranging from 0-255
    list_of_pixel_vals = list(image.getdata())

    for i in range(len(list_of_pixel_vals)):
        val = list_of_pixel_vals[i] / 255

        if 0.0 <= val and val < 0.1:
            ascii_list.append('.')
        elif 0.1 <= val and val < 0.2:
            ascii_list.append(',')
        elif 0.2 <= val and val < 0.3:
            ascii_list.append(';')
        elif 0.3 <= val and val < 0.4:
            ascii_list.append('!')
        elif 0.4 <= val and val < 0.5:
            ascii_list.append('v')
        elif 0.5 <= val and val < 0.6:
            ascii_list.append('l')
        elif 0.6 <= val and val < 0.7:
            ascii_list.append('L')
        elif 0.7 <= val and val < 0.8:
            ascii_list.append('F')
        elif 0.8 <= val and val < 0.9:
            ascii_list.append('E')
        else:
            #  0.9 <= val and val < 1.0:
            ascii_list.append('$')

    width, height = image.size

    # reshape the 1D List into a 2D array based on image dimensions
    ascii_list = np.reshape(ascii_list, (width, height))

    return ascii_list

def __write_to_txt_file(grid, filename):
    """
    converts a 2D Array into a text file
    """
    with open(filename, 'w') as ascii_art:
        for h in range(len(grid)):
            for r in range(len(grid[h])):
                print(grid[h][r], file=ascii_art, end='')
            print('', file=ascii_art)

def generate_ascii_art(image):

    image_name = image.split('.')[0]
    image = Image.open(image)
    image = __convert_to_grayscale(image)
    image = __pixel_to_ascii_char(image)
    __write_to_txt_file(image, f"{image_name}_ascii_art.txt")

def main():
    filename = "hotdog.png"
    generate_ascii_art(filename)

if __name__ == "__main__":
    main()


import argparse
import random
from typing import Callable
import pandas as pd
import pickle
import numpy as np
import yaml
from PIL import Image
import torch
import logging
from tqdm import tqdm
from collections import defaultdict

from facenet_pytorch import MTCNN
import albumentations as A
from albumentations.pytorch import ToTensorV2 as ToTensor

def get_train_augmentations(image_size: int = 224, mean: tuple = (0, 0, 0), std: tuple = (1, 1, 1)):
    return A.Compose(
        [
            # A.RandomBrightnessContrast(brightness_limit=32, contrast_limit=(0.5, 1.5)),
            # A.HueSaturationValue(hue_shift_limit=18, sat_shift_limit=(1, 2)),
            # A.CoarseDropout(20),
            A.Rotate(10),

            A.Resize(image_size, image_size),
            # A.RandomCrop(image_size, image_size, p=0.5),

            A.LongestMaxSize(image_size),
            A.Normalize(mean=mean, std=std),
            A.HorizontalFlip(),
            A.PadIfNeeded(image_size, image_size),
            # A.Transpose(),
            ToTensor(),
        ]
    )

def get_test_augmentations(image_size: int = 224, mean: tuple = (0, 0, 0), std: tuple = (1, 1, 1)):
    return A.Compose(
        [
            A.Resize(image_size, image_size),
            A.LongestMaxSize(image_size),
            A.Normalize(mean=mean, std=std),
            A.PadIfNeeded(image_size, image_size),
            ToTensor(),
        ]
    )

class Dataset(torch.utils.data.Dataset):
    def __init__(
        self,
        df: "pd.DataFrame",
        root: str,
        transforms: Callable,
        face_detector: dict = None,
        with_labels: bool = True,
        bookkeeping_path: str = None,
        train: bool = True
    ):
        self.df = df.sample(frac=1).reset_index(drop=True)
        self.root = root
        self.transforms = transforms
        self.with_labels = with_labels
        self.face_extractor = None

        self._bookkeeping_path = bookkeeping_path

        if face_detector is not None:
            face_detector["keep_all"] = True
            face_detector["post_process"] = False
            self.face_extractor = MTCNN(**face_detector)

    def generate_bookkeeping(self):
        indices_to_labels = defaultdict(int)

        for index in tqdm(range(0, len(self.df))):
            label = self[index][1]

            indices_to_labels[index] = label

        outfile = open('indices_to_labels', 'wb')
        pickle.dump(indices_to_labels, outfile)
        outfile.close()

    def __len__(self):
        return len(self.df)

    def __getitem__(self, item: int):
        # updated for absolute paths
        path = self.df.iloc[item].path

        image = Image.open(path)
        if self.with_labels:
            target = self.df.iloc[item].target

        if self.face_extractor is not None:
            faces, probs = self.face_extractor(image, return_prob=True)
            if faces is None:
                logging.warning(f"{path} doesn't containt any face!")
                image = self.transforms(image=np.array(image))["image"]
                if self.with_labels:
                    return image, target
                else:
                    return image
            if faces.shape[0] != 1:
                logging.warning(
                    f"{path} - {faces.shape[0]} faces detected"
                )
                face = (
                    faces[np.argmax(probs)]
                    .numpy()
                    .astype(np.uint8)
                    .transpose(1, 2, 0)
                )
            else:
                face = faces[0].numpy().astype(np.uint8).transpose(1, 2, 0)
            image = self.transforms(image=face)["image"]
        else:
            image = self.transforms(image=np.array(image))["image"]

        if self.with_labels:
            return image, target
        else:
            return image

    @property
    def bookkeeping_path(self):
        return self._bookkeeping_path

if __name__ == "__main__":
    print("Running dataset analysis...")

    parser = argparse.ArgumentParser()
    parser.add_argument("-config", "--config_file_path", required=True, help="Config file path.")
    args = parser.parse_args()

    with open(args.config_file_path, 'r') as stream:
        configs = yaml.safe_load(stream)

    data_df = pd.read_csv(configs['train_df'])

    mean = (configs['mean']['r'], configs['mean']['g'], configs['mean']['b'])
    std = (configs['std']['r'], configs['std']['g'], configs['std']['b'])

    random.seed(configs['seed'])
    np.random.seed(configs['seed'])
    torch.manual_seed(configs['seed'])

    dataset = Dataset(data_df, configs['dataset_root'], transforms=get_train_augmentations(mean=mean, std=std))
    print("Length of dataset: ", len(dataset))

    print("Starting bookkeeping...")
    dataset.generate_bookkeeping()


import json

import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

# 1.brandorder――柱状
def brandorder():
    plt.figure(figsize=(13, 9))
    with open("./json/brandOrder.json", 'r') as load_f:
        brandorder_list = json.load(load_f)
    x, y = [], []
    for br in brandorder_list:
        if br[1] > 1000:
            x.append(br[0])
            y.append(br[1])
    plt.bar(x, y, color='pink')
    ax1 = plt.gca()
    ax1.set_facecolor('#E6E6E6')
    plt.grid(b=True, color='#FFFFFF')
    for key, spine in ax1.spines.items():
        if key == 'right' or key == 'top' or key == 'left' or key == 'bottom':
            spine.set_visible(False)
    plt.legend(("各品牌用户数",))
    plt.xticks(rotation=30)
    plt.xlabel("品牌", size=18)
    plt.ylabel("用户数", size=18)
    plt.savefig("./img/brandorder.png")
    print("brandorder图保存成功！")
    plt.close()

# 2.brandsumofprice――柱状
def brandsumofprice():
    plt.figure(figsize=(13, 9))
    with open("./json/brandSumOfPrice.json", 'r') as load_f:
        brandsumofprice_list = json.load(load_f)
    x, y = [], []
    for br in brandsumofprice_list:
        if len(str(br[0])) > 7:
            x.append(br[0][:7] + "\n" + br[0][7:])
        else:
            x.append(br[0])
        y.append(br[1])

    plt.bar(x, y, color='pink')
    ax2 = plt.gca()
    ax2.set_facecolor('#E6E6E6')
    plt.grid(b=True, color='#FFFFFF')
    for key, spine in ax2.spines.items():
        if key == 'right' or key == 'top' or key == 'left' or key == 'bottom':
            spine.set_visible(False)

    plt.legend(("各品牌销售数据",))
    plt.xticks(rotation=30)
    plt.xlabel("品牌", size=18)
    plt.ylabel("销售额（千万）", size=18)
    plt.title("平台每月销量", size=18)
    plt.savefig("./img/brandsumofprice.png")
    print("brandsumofprice图保存成功！")
    plt.close()

# 3.categorysumofprice――柱状
def categorySumOfPrice():
    plt.figure(figsize=(13, 9))
    with open("./json/categorySumOfPrice.json", 'r') as load_f:
        categorySumOfPrice_list = json.load(load_f)
    x, y = [], []
    for br in categorySumOfPrice_list:
        x.append(br[0])
        y.append(br[1])
    plt.bar(x, y, color='#F781F3')
    plt.legend(("类别",))
    plt.savefig("./img/categorySumOfPrice.png")
    print("categorySumOfPrice图保存成功！")
    plt.close()

# 4.categoryuser――柱状
def categoryUser():
    plt.figure(figsize=(13, 9))
    with open("./json/categoryUser.json", 'r') as load_f:
        categoryUser_list = json.load(load_f)
    x, y = [], []
    for br in categoryUser_list:
        x.append(br[0])
        y.append(br[1])
    plt.bar(x, y)
    plt.legend(("类别",))
    plt.savefig("./img/categoryUser.png")
    print("categoryUser图保存成功！")
    plt.close()

# 5.hourorder――折线
def hourorder():
    plt.figure(figsize=(13, 9))
    with open("./json/HourOrder.json", 'r') as load_f:
        hourorder_list = json.load(load_f)
    x, y = [], []
    for br in hourorder_list:
        x.append(br[0])
        y.append(br[1])
    plt.plot(x, y, color='red')
    plt.xlabel("时间", size=18)
    plt.ylabel("用户购买量", size=18)
    plt.xticks(range(0, 25))
    plt.savefig("./img/hourorder.png")
    print("hourorder图保存成功！")
    plt.close()

# 6.monthorder――折线
def monthorder():
    with open("./json/MonthOrder.json", 'r') as load_f:
        monthorder_list = json.load(load_f)
    x, y = [], []
    for br in monthorder_list:
        x.append(br[0])
        y.append(br[1])
    plt.plot(x, y, color='pink')
    plt.xlabel("月份", size=18)
    plt.ylabel("销量", size=18)
    plt.savefig("./img/monthorder.png")
    print("monthorder图保存成功！")
    plt.close()

# 7.monthsumofprice――折线
def monthsumofprice():
    with open("./json/MonthSumOfPrice.json", 'r') as load_f:
        monthsumofprice_list = json.load(load_f)
    x, y = [], []
    for br in monthsumofprice_list:
        x.append(br[0])
        y.append(br[1])
    plt.title("每月消费总金额（单位/元）", size=18)
    ax3 = plt.gca()
    ax3.set_facecolor('#E6E6E6')
    plt.grid(b=True, color='#FFFFFF', axis='y')
    for key, spine in ax3.spines.items():
        if key == 'right' or key == 'top' or key == 'left' or key == 'bottom':
            spine.set_visible(False)
    plt.plot(x, y, color='pink')
    plt.savefig("./img/monthsumofprice.png")
    print("monthsumofprice图保存成功！")
    plt.close()

# 8.monthuser――折线
def monthuser():
    with open("./json/MonthUser.json", 'r') as load_f:
        monthuser_list = json.load(load_f)
    x, y = [], []
    for br in monthuser_list:
        x.append(br[0])
        y.append(br[1])
    plt.title("每月消费人数", size=18)
    ax4 = plt.gca()
    ax4.set_facecolor('#E6E6E6')
    plt.grid(b=True, color='#FFFFFF', axis='y')
    for key, spine in ax4.spines.items():
        if key == 'right' or key == 'top' or key == 'left' or key == 'bottom':
            spine.set_visible(False)
    plt.plot(x, y, color='pink')
    plt.savefig("./img/monthuser.png")
    print("monthuser图保存成功！")
    # plt.close()
    plt.show()

if __name__ == '__main__':
    # brandorder()
    # brandsumofprice()
    # categorySumOfPrice()
    # categoryUser()
    # hourorder()
    # monthorder()
    # monthsumofprice()
    monthuser()


from django.contrib import admin
from acervo.models import Acervo

class AcervoAdmin(admin.ModelAdmin):
    list_display = ('id', 'codigo', 'titulo', 'sigla', 'ordem_exibicao')
    list_display_links = ('id', 'codigo', 'titulo', 'sigla', 'ordem_exibicao')
    search_fields = ('titulo', 'descricao', 'info_adicionais')

admin.site.register(Acervo, AcervoAdmin)


import aiohttp
import bencode
import ipaddress
import struct

from urllib.parse import urlencode

class Tracker:
    def __init__(self, url):
        self.url = url.decode("utf-8")
        self.interval = 0
        self.http_session = aiohttp.ClientSession()
        self.req_count = 0

    async def request_peers(self, client_id, port, info_hash):
        async with self.http_session.get(self._build_request(client_id, port, info_hash)) as resp:
            if resp.status != 200:
                #logging.debug("Couldn't resolve Tracker")
                return
            #logging.debug("Recived Tracker response")
            response = bencode.loads(await resp.read())

        try:
            self.interval = response[b"interval"]
        except KeyError:
            pass

        self.req_count += 1

        return response

    def decode_peers_list(self, raw_str):
        return [(self._decode_ip(raw_str[i:i + 4]),
                    self._decode_port(raw_str[i + 4:i + 6]))
                                for i in range (0, len(raw_str), 6)]

    async def close(self):
        await self.http_session.close()

    def _build_request(self, client_id, port, info_hash):
        params = {
            "info_hash": info_hash,
            "peer_id": client_id,
            "port": port,
            "uploaded": 0,
            "downloaded": 0,
            "left": 0,
            "compact": 1
        }

        return self.url + "?" + urlencode(params)

    def _decode_ip(self, data):
        return str(ipaddress.ip_address(data))

    def _decode_port(self, data):
        return struct.unpack(">H", data)[0]


from typing import TYPE_CHECKING, List, Optional

from pydeus.types._converter.abc import ABCConverter
from pydeus.types.search import Building, Room

if TYPE_CHECKING:
    from pydeus.types.responses.room import BuildingResponseData, RoomResponse, RoomResponseData

class RoomConverter(ABCConverter):
    def __init__(self, room_response: 'RoomResponse') -> None:
        self.room_response = room_response

    def convert_list(self) -> Optional[List[Room]]:
        if not self.room_response.rooms:
            return None

        return [self._build_one_room(room) for room in self.room_response.rooms]

    def _build_one_room(self, room: 'RoomResponseData') -> Room:
        building = self._build_one_building(room.building)
        return Room(
            id=room.id,
            room=room.name,
            project_available=room.project_availiable,
            building=building,
            capacity=room.capacity,
        )

    def _build_one_building(self, raw_building: 'BuildingResponseData') -> Building:
        return Building(
            id=raw_building.id,
            name=raw_building.name,
            address=raw_building.address,
        )


from cuesdk import CueSdk
from cuesdk import CorsairLedId
import threading
import queue
import time
from keyboard import Keyboard

def read_keys(inputQueue):
    while (True):
        input_str = input()
        inputQueue.put(input_str)

def life_happens(keyboard: Keyboard):
    """The rules of the game of life are followed to move forward one step.

    If a cell is ON and has fewer than two neighbors that are ON, it turns OFF
    If a cell is ON and has either two or three neighbors that are ON, it remains ON.
    If a cell is ON and has more than three neighbors that are ON, it turns OFF.
    If a cell is OFF and has exactly three neighbors that are ON, it turns ON.
    """
    on_color = (255, 0, 0)
    off_color = (0, 0, 0)
    for key in keyboard._keyboard:
        neighbors = keyboard.get_key_neighbors(key)
        neighbors_on = keyboard.num_match_color(neighbors, on_color)

        # 1
        if keyboard.get_key_color(key) == on_color:
            keyboard.set_key_buffer(key, on_color)
            if neighbors_on < 2:
                keyboard.set_key_buffer(key, off_color)

        # 3
            elif neighbors_on > 4:
                keyboard.set_key_buffer(key, off_color)

        # 4
        else:
            if neighbors_on == 3 or neighbors_on == 4:
                keyboard.set_key_buffer(key, on_color)

    keyboard.set_led_colors_flush_buffer()

def main():
    inputQueue = queue.Queue()

    inputThread = threading.Thread(target=read_keys,
                                   args=(inputQueue, ),
                                   daemon=True)
    inputThread.start()

    keyboard = Keyboard()
    keyboard.set_layer_priority(126)

    print("Working... Use \"+\" or \"-\" to increase or decrease speed.\n" +
          "Press \"q\" to close program...")
    while (True):
        if (inputQueue.qsize() > 0):
            input_str = inputQueue.get()

            if input_str == "q" or input_str == "Q":
                print("Exiting.")
                break

        #keyboard.set_key(CorsairLedId.K_G, (255, 0, 0))

        life_happens(keyboard)

        time.sleep(.5)

if __name__ == "__main__":
    main()


'''
  @Author: Uthsavi KP
  @Date: 2021-02-03 13:15:20
  @Last Modified by:  Uthsavi KP
  @Last Modified time: 2021-02-03 21:03:42
  @Title: Azure cosmosDB using python
'''

import uuid
def get_andersen_family_item():
    andersen_item = {
    'id': 'Andersen_' + str(uuid.uuid4()),
    'lastName': 'Andersen',
    'district': 'WA5',
    'parents': [
        {
            'familyName': None,
            'firstName': 'Thomas'
        },
        {
            'familyName': None,
            'firstName': 'Mary Kay'
        }
    ],
    'children': None,
    'address': {
        'state': 'WA',
        'county': 'King',
        'city': 'Seattle'
    },
    'registered': True
}
    return andersen_item


import datasets
import numpy as np
from rouge import Rouge
from rouge_score import rouge_scorer

_CITATION = """\
@inproceedings{lin-2004-rouge,
    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
    author = "Lin, Chin-Yew",
    booktitle = "Text Summarization Branches Out",
    month = jul,
    year = "2004",
    address = "Barcelona, Spain",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/W04-1013",
    pages = "74--81",
}
"""

_DESCRIPTION = """\
ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
evaluating automatic summarization and machine translation software in natural language processing.
The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.
Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
This metrics is a wrapper around Google Research reimplementation of ROUGE:
https://github.com/google-research/google-research/tree/master/rouge
"""

_KWARGS_DESCRIPTION = """
Calculates average rougeL scores for a list of hypotheses and references
Args:
    predictions: list of predictions to score. Each prediction
        should contain an 'id' and a
        'prediction_text': string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should contain an 'id' and an
        'answers' list of string with tokens separated by spaces.
Returns:
    rougeL: rouge_l (precision, recall, f1),
Examples:
    >>> rouge = datasets.load_metric('rouge')
    >>> predictions = [{"id":1, "prediction_text" : "hello there"}, "{"id":2, "prediction_text": "general kenobi"]
    >>> references = [{"id":1, "answers" : ["hello there"]}, {"id":2, "answers": ["general kenobi"]}]
    >>> results = rouge.compute(predictions=predictions, references=references)
    >>> print(list(results.keys()))
    ['rougeL', 'gen_len']
"""

import sys

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class ROUGE(datasets.Metric):

    def _info(self):
        self._hf_rouge = rouge_scorer.RougeScorer(rouge_types=['rougeLsum'], split_summaries=True)
        self._kilt_rouge = Rouge(metrics=['rouge-l'])
        sys.setrecursionlimit(20000)

        return datasets.MetricInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    "predictions": {"id":datasets.Value("string", id = None),"prediction_text":datasets.Value(dtype='string', id=None)},
                    "references": {"id":datasets.Value("string", id = None),"answers":datasets.Sequence(datasets.Value(dtype='string', id=None))},
                }
            ),
            codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"],
            reference_urls=[
                "https://en.wikipedia.org/wiki/ROUGE_(metric)",
                "https://github.com/google-research/google-research/tree/master/rouge",
            ],
        )

    def _rougel_score(self,prediction, ground_truth):
        try:
            hf_scores = self._hf_rouge.score(ground_truth, prediction)
            kilt_scores = self._kilt_rouge.get_scores(prediction, ground_truth, avg=True)
        except ValueError:  # "Hypothesis is empty."
            return 0.0, 0.0
        return hf_scores['rougeLsum'].fmeasure, kilt_scores["rouge-l"]["f"]

    def _metric_max_over_ground_truths(self, prediction, ground_truths):
        kilt_scores_for_ground_truths = []
        google_scores_for_ground_truths = []
        for ground_truth in ground_truths:
            hf_score, kilt_score = self._rougel_score(prediction, ground_truth)
            google_scores_for_ground_truths.append(hf_score)
            kilt_scores_for_ground_truths.append(kilt_score)
        return max(google_scores_for_ground_truths), max(kilt_scores_for_ground_truths)

    def _compute(self, predictions, references, **kwargs):
        # adopted KILT standard evaluation from
        # https://github.com/facebookresearch/KILT/blob/main/kilt/eval_downstream.py
        total_count = 0
        kilt_rougel = 0
        google_rougel = 0

        for pred,ref in zip(predictions,references):
            _id = pred["id"]
            _pred = pred["prediction_text"]
            assert ref["id"] == _id
            total_count += 1
            _refs = ref["answers"]
            google_local_rougel, kilt_local_rougel = self._metric_max_over_ground_truths(_pred, _refs)
            kilt_rougel += kilt_local_rougel
            google_rougel += google_local_rougel

        result = {"kilt_rougeL": (kilt_rougel/total_count)*100, "google_rougeL": (google_rougel/total_count)*100}
        prediction_lens = [pred["prediction_text"].count(' ') for pred in predictions]
        result["gen_len"] = np.mean(prediction_lens)
        result = {k: round(v, 4) for k, v in result.items()}
        return result


import requests
from lxml import etree
from time import sleep
from opencontext_py.libs.generalapi import GeneralAPI

class OaiPmhClientAPI():
    """ A simple, not fully functional client API for OAI-PMH services
    """

    NAMESPACES = {
        'oai': 'http://www.openarchives.org/OAI/2.0/',
        'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
        'dc': 'http://purl.org/dc/elements/1.1/',
        'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
    }
    SLEEP_TIME = .5

    def __init__(self):
        self.request_error = False
        self.delay_before_request = self.SLEEP_TIME
        self.graph = False
        self.request_url = False

    def get_list_records(self, url, resumption_token=None):
        """
        gets OAI-PMH list records, with an optional resumption_token
        """
        xml = None
        params = None
        if 'verb=ListRecords' not in url:
            params = {}
            params['verb'] = 'ListRecords'
        if isinstance(resumption_token, str):
            if '?' in url:
                # do this to avoid URL encoding the resumption token
                url += '&resumptionToken=' + resumption_token
            else:
                url += '?resumptionToken=' + resumption_token
        if self.delay_before_request > 0:
            # default to sleep BEFORE a request is sent, to
            # give the remote service a break.
            sleep(self.delay_before_request)
        url_content = None
        if isinstance(params, dict):
            try:
                gapi = GeneralAPI()
                r = requests.get(url,
                                 params=params,
                                 timeout=240,
                                 headers=gapi.client_headers)
                self.request_url = r.url
                r.raise_for_status()
                url_content = r.content
            except:
                self.request_error = True
                url_content = None
        else:
            try:
                gapi = GeneralAPI()
                r = requests.get(url,
                                 timeout=240,
                                 headers=gapi.client_headers)
                self.request_url = r.url
                r.raise_for_status()
                url_content = r.content
            except:
                self.request_error = True
                url_content = None
        return url_content


from celery.signals import beat_init
from django.db import transaction

from core.integration.operations import *
from core.models import Configuration
from core.models import Order
from integration_tiny.celery import app

@app.task(rate_limit='10/m')
def task_get_order_in_integrador(order_id):
    order = Order.objects.get(
        id=order_id
    )

    GetOrderInIntegrator(
        order,
    ).execute()

@app.task(rate_limit='10/m')
def task_send_cancelation_to_integrador(order_id):
    order = Order.objects.get(
        id=order_id
    )

    SendRequestCancelationToIntegrator(
        order,
    ).execute()

@app.task(rate_limit='4/m')
def task_send_order_to_integrador(order_id):
    order = Order.objects.get(
        id=order_id
    )

    SendRequestToIntegrator(
        order,
    ).execute()

@app.task(rate_limit='4/m')
def task_update_order(order_id):
    try:
        order = Order.objects.get(
            id=order_id
        )
        try:
            UpdateOrder(
                order.configuration,
                order
            ).execute()
        except OperationError as error:
            logger.warning(
                f'[Order {order}] - Save labels: {error}'
            )
        finally:
            order.set_running(False)

    except Order.DoesNotExist:
        pass

@app.task(rate_limit='1/s')
def task_update_orders():
    orders = Order.objects.search_update_orders()
    ids = list(
        orders.values_list('id', flat=True)
    )
    orders.update(running=True)

    for _id in ids:
        transaction.on_commit(
            lambda: task_update_order.delay(
                _id
            )
        )

@app.task
def task_search_expedition(order_id):
    try:
        order = Order.objects.get(
            id=order_id
        )
        try:
            SaveExpeditionInfo(
                order.configuration,
                order
            ).execute()
        except OperationError as error:
            logger.warning(
                f'[Order {order}] - Save labels: {error}'
            )

        finally:
            order.set_running(False)
    except Order.DoesNotExist:
        pass

@app.task
def task_search_expeditions():
    orders = Order.objects.search_expedition()
    ids = list(
        orders.values_list('id', flat=True)
    )
    orders.update(running=True)

    for _id in ids:
        transaction.on_commit(
            lambda: task_search_expedition.delay(
                _id
            )
        )

@app.task
def task_sync_processed_orders(configuration_id):
    queryset = Configuration.objects.filter(
        is_active=True
    ).all()
    if configuration_id > 0:
        queryset = queryset.filter(
            id=configuration_id
        )

    for configuration in queryset:
        try:
            GetProcessedOrderInIntegrator(
                configuration
            ).execute()

        except OperationError as error:
            logger.warning(
                f'[{configuration}] - Sync orders: {error}'
            )

@app.task
def task_sync_cancelled_orders(configuration_id):
    queryset = Configuration.objects.filter(
        is_active=True
    ).all()
    if configuration_id > 0:
        queryset = queryset.filter(
            id=configuration_id
        )

    for configuration in queryset:
        try:
            GetCancelledOrders(
                configuration
            ).execute()

        except OperationError as error:
            logger.warning(
                f'[{configuration}] - Sync orders: {error}'
            )

@app.task
def task_sync_orders(configuration_id):
    queryset = Configuration.objects.filter(
        is_active=True
    ).all()
    if configuration_id > 0:
        queryset = queryset.filter(
            id=configuration_id
        )

    for configuration in queryset:
        try:
            SaveOrders(configuration).execute()

            task_update_orders.delay()

        except OperationError as error:
            logger.warning(
                f'[{configuration}] - Sync orders: {error}'
            )

@app.task
def task_send_labels():
    queryset = Order.objects.pending_send_labels()

    for order in queryset:
        SendRequestLabelToIntegrator(
            order
        ).execute()

@app.task
def task_send_orders_awaiting_integration():
    queryset = Order.objects.filter(
        status=Order.AWAITING_INTEGRATION,
        running=False
    )

    for order in queryset:
        SendRequestToIntegrator(
            order
        ).execute()

@beat_init.connect
def task_callback_start_beat(**kwargs):
    Order.objects.filter(
        running=True
    ).update(running=False)


import json
import argparse
from utils import *

def main(args):
    settings = dict()
    settings["LocationSelection"] = dict()
    settings["Modifications"] = dict()
    settings["ResultsAndExport"] = dict()

    print("标定开始")
    print("Location selection菜单下，选好城市")
    print("鼠标放在Next上，按下回车")
    while True:
        if get_one_key() == 13:
            p = get_mouse_point()
            time.sleep(0.01)
            break
    settings["LocationSelection"]["Next"] = p
    print("点击Next")

    print("鼠标放在Inclination框上，按下回车")
    while True:
        if get_one_key() == 13:
            p = get_mouse_point()
            time.sleep(0.01)
            break
    settings["Modifications"]["Inclination"] = p

    print("鼠标放在Results and export菜单栏上，按下回车")
    while True:
        if get_one_key() == 13:
            p = get_mouse_point()
            time.sleep(0.01)
            break
    settings["ResultsAndExport"]["Menu"] = p
    print("点击Results and export菜单栏，等待计算完成")

    print("点击Data table")
    print("鼠标放在Year行数据上，按下回车")
    while True:
        if get_one_key() == 13:
            p = get_mouse_point()
            time.sleep(0.01)
            break
    settings["ResultsAndExport"]["Year"] = p

    print("鼠标放在Modifications & data import菜单栏上，按下回车")
    while True:
        if get_one_key() == 13:
            p = get_mouse_point()
            time.sleep(0.01)
            break
    settings["Modifications"]["Menu"] = p
    print("标定完成")

    string = json.dumps(settings, indent=4)
    with open(args.settings_file, 'w')as f:
        f.write(string)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-s", dest="settings_file", default="./settings.json")
    args = parser.parse_args()
    print(args)
    main(args)


from unicodedata import name
from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('gallery', views.gallery, name='gallery'),
    path('<int:card_id>', views.card, name='card'),
    path('search', views.search, name='search'),
    path('filter_page', views.filter_page, name='filter_page'),
    path('collection', views.collection, name='collection')

]


from models.utils import get_parameters

import logging
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
import datetime

def train(x_train, x_test, y_train, y_test, sensor_data, return_values):
    default_knn = KNeighborsClassifier()
    default_knn.fit(x_train, y_train)
    best_knn = default_knn
    max_accuracy = default_knn.score(x_test, y_test)
    best_params = 'Default'
    method_params_found = 'Default'
    y_score = default_knn.predict(x_test)

    # n_samples = np.array(x_train).shape[0]
    # n = n_samples if n_samples < 200 else 200
    # n_neighbours = list(range(1, n))
    # parameters = {'n_neighbors': n_neighbours,
    #               'weights': ('uniform', 'distance'),
    #               'algorithm': ('auto', 'ball_tree', 'kd_tree', 'brute'),
    #               'leaf_size': [no_leaves for no_leaves in range(5, 27)]
    #               }
    #
    # knn, knn_method_params_found = get_parameters(parameters, KNeighborsClassifier(), x_train, y_train)
    # knn_accuracy = knn.score(x_test, y_test)
    #
    # if knn_accuracy > max_accuracy:
    #     best_knn = knn
    #     max_accuracy = knn_accuracy
    #     best_params = knn.best_params_
    #     y_score = knn.score(x_test)
    #     method_params_found = knn_method_params_found

    date_time = datetime.datetime.now()
    msg = '{0}: The best KNN classifier was trained with the parameters {1} which produced an accuracy score of: {2}' \
          'for {3} data. These parameters were found using the {4} method.'.format(str(date_time), str(best_params),
                                                                                str(max_accuracy), str(sensor_data),
                                                                                str(method_params_found))
    logging.info(msg)
    print(msg)

    return_values[2] = {'model': best_knn,
                        'model_type': 'KNN',
                        'sensor_data': sensor_data,
                        'max_accuracy': max_accuracy,
                        'best_params': best_params,
                        'method': method_params_found,
                        'y_score': y_score,
                        'y_test': y_test
                        }


from apis.reddit.reddit_posts import RedditPosts
from apis.io.post_io import load_posts, save_posts
import datatable as dt
import pandas as pd

import logging

logger = logging.getLogger(__name__)

NEW_POSTS_LISTING = "new"
TOP_POSTS_LISTING = "top"

NEW_POSTS_COUNT = 100
TOP75_POSTS_COUNT = 75

POST_COLUMNS = [
    "author_fullname",
    "title",
    "name",
    "score",
    "created",
    "view_count",
    "id",
    "author",
    "url",
    "created_utc"
    ]

POST_COLUMNS_X = {POST_COLUMNS[i]+"_x":POST_COLUMNS[i] for i in range(len(POST_COLUMNS))}
POST_COLUMNS_Y = {POST_COLUMNS[i]+"_y":POST_COLUMNS[i] for i in range(len(POST_COLUMNS))}

""" The Reddit posts processor

get redit posts as json from reddit.com
load posts from last execution
save posts to disk (for next execution)
use dataframes to compare latest posts against last program run
"""
class RedditProcessor:
    def __init__(self):
        self.reddit_new = RedditPosts(listing=NEW_POSTS_LISTING, limit=NEW_POSTS_COUNT, timeframe="hour")
        self.reddit_top = RedditPosts(listing=TOP_POSTS_LISTING, limit=TOP75_POSTS_COUNT, timeframe="hour")
        self.df_new_posts = self.load_posts(self.reddit_new, NEW_POSTS_LISTING)
        self.df_top_posts = self.load_posts(self.reddit_top, TOP_POSTS_LISTING)

    def load_posts(self, reddit_posts, listing ):
        logger.info(f"====> {listing} POSTS:")
        latest_posts = reddit_posts.load(columns=POST_COLUMNS)
        logging.info(f"latest {listing} posts count {latest_posts.nrows}")

        previous_posts = load_posts(listing=listing, columns=POST_COLUMNS)
        logging.info(f"previous {listing} posts count {previous_posts.nrows}")

        # save for next program execution
        save_posts(listing=listing, posts=latest_posts)

        # use pandas to merge into a single dataframe with all posts from latest and previous exeuction
        df_latest = latest_posts.to_pandas()
        df_previous = previous_posts.to_pandas()
        df = pd.merge(df_latest, df_previous, on=['id'], how="outer", indicator=True)
        return df

    def get_new_posts(self):
        df_new_since_last = self.df_new_posts[self.df_new_posts['_merge'] == 'left_only']
        df_new_since_last = df_new_since_last.rename(POST_COLUMNS_X, axis=1)
        df_new_since_last = df_new_since_last[POST_COLUMNS]
        logger.info(f"============> {df_new_since_last.shape[0]} LATEST NEW POSTS")
        logger.info(df_new_since_last[['id','created_utc','score']].head)
        return df_new_since_last

    def get_not_top75_posts(self):
        df_no_longer_top75 = self.df_top_posts[self.df_top_posts['_merge'] == 'right_only']
        df_no_longer_top75 = df_no_longer_top75.rename(POST_COLUMNS_Y, axis=1)
        df_no_longer_top75 = df_no_longer_top75[POST_COLUMNS]

        logger.info(f"============> {df_no_longer_top75.shape[0]} POSTS NO LONGER TOP 75")
        logger.info(df_no_longer_top75[['id','created_utc','score']].head)
        return df_no_longer_top75

    def get_scores_changed_posts(self):
        df_scores = pd.concat([self.df_top_posts, self.df_new_posts])
        df_scores.drop_duplicates(subset=['id'])

        df_scores = df_scores[(df_scores['_merge'] == 'both') & (df_scores['score_x'] != df_scores['score_y'])]

        df_scores = df_scores.rename(POST_COLUMNS_X, axis=1)
        df_scores = df_scores[POST_COLUMNS+['score_y']]
        df_scores['score_change'] = df_scores.score - df_scores.score_y

        logger.info(f"============> {df_scores.shape[0]} POSTS WITH SCORE CHANGES")
        logger.info(df_scores[['id','score','score_y','score_change']])
        return df_scores


from typing import Optional

from pydantic import EmailStr
from starlette.exceptions import HTTPException
from starlette.status import (
    HTTP_422_UNPROCESSABLE_ENTITY,
)

from .dc import get_dc
from .deployment import get_deployment
from .deployment_type import get_deployment_type
from .provider import get_provider
from .region import get_region
from .sdl_template import get_sdl_template
from .tag import get_tag
from .user import get_user, get_user_by_email
from ..db.mongodb import AsyncIOMotorClient

async def check_free_username_and_email(
        conn: AsyncIOMotorClient, username: Optional[str] = None, email: Optional[EmailStr] = None
):
    if username:
        user_by_username = await get_user(conn, username)
        if user_by_username:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="User with this username already exists",
            )
    if email:
        user_by_email = await get_user_by_email(conn, email)
        if user_by_email:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="User with this email already exists",
            )

async def check_free_tag_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        tag_by_title = await get_tag(conn, title)
        if tag_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="Tag with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Tag must have title",
        )

async def check_free_sdl_template_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        sdl_tmp_by_title = await get_sdl_template(conn, title)
        if sdl_tmp_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="SDL Template with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="SDL Template must have title",
        )

async def check_free_provider_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        provider_by_title = await get_provider(conn, title)
        if provider_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="Provider with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Provider must have title",
        )

async def check_free_region_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        region_by_title = await get_region(conn, title)
        if region_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="Region with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Region must have title",
        )

async def check_free_deployment_type_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        deployment_type_by_title = await get_deployment_type(conn, title)
        if deployment_type_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="Deployment Type with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Deployment Type must have title",
        )

async def check_free_deployment_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        deployment_by_title = await get_deployment(conn, title)
        if deployment_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="Deployment with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Deployment must have title",
        )

async def check_free_dc_title(
        conn: AsyncIOMotorClient, title: Optional[str] = None
):
    if title:
        dc_by_title = await get_dc(conn, title)
        if dc_by_title:
            raise HTTPException(
                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
                detail="DC with this title already exists",
            )
    else:
        raise HTTPException(
            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
            detail="DC must have title",
        )