__all__ = ['pdftoppm_exe', 'convert_exe', 'pdf_to_image']

import subprocess, io

#: The name of the :program:`pdftoppm` executable. Either a full path,
#: or a program that the shell can find on the :envvar:`PATH` is
#: necessary.
pdftoppm_exe = 'pdftoppm'

#: The name of the `ImageMagick`_ :program:`convert` executable. Either
#: a full path, or a program that the shell can find on the
#: :envvar:`PATH` is necessary.
convert_exe = 'convert'

def pdf_to_image(input_path, output_path, format=None):
    """
    Convert a PDF document into an image file.

    This function uses the :py:mod:`subprocess` module to operate. It
    requires the presence of the :program:`pdftoppm` program as well as
    :program:`convert` from `ImageMagick`_.

    `input_path` may be a string path or a file-like object.

    `output_path` may be a string, a file-like object or :py:obj:`None`.
    If :py:obj:`None`, an :py:class:`io.BytesIO` object is returned
    containing the image. `format` defaults to ``'png'`` if not set
    explicitly in this case.

    Return the name of the output file, or an in-memory file-like object
    (:py:class:`io.BytesIO`) if `output_path` is :py:obj:`None`.

    The idea for behind this conversion mechanism comes from
    http://stackoverflow.com/a/2002436/2988730. The implementation
    details are described in
    http://stackoverflow.com/a/4846923/2988730.

    .. include:: /link-defs.rst
    """
    if isinstance(input_path, str):
        step1 = [pdftoppm_exe, input_path]
        stdin = None
    else:
        step1 = [pdftoppm_exe]
        stdin = input_path

    if output_path is None:
        # Output to memory
        out = '-'
        if format is None:
            format = 'png'
    elif isinstance(output_path, str):
        # Output to path
        out = output_path
    else:
        # Output to file-like object
        out = '-'

    if format is not None:
        out = ':'.join((format, out))

    step2 = [convert_exe, '-', out]

    proc1 = subprocess.Popen(step1, stdin=stdin, stdout=subprocess.PIPE)
    proc2 = subprocess.Popen(step2, stdin=proc1.stdout, stdout=subprocess.PIPE)
    # Force error in case proc2 dies unexpectedly
    proc1.stdout.close()
    # Ignore output
    sysout, _ = proc2.communicate()

    if isinstance(output_path, str):
        return output_path
    elif output_path is None:
        outputFile = io.BytesIO()
        rewind = True
    else:
        outputFile = output_path
        rewind = False
    outputFile.write(sysout)
    if rewind:
        outputFile.seek(0)
    return outputFile

if __name__ == '__main__':
    from sys import argv, stderr, exit
    from shutil import copyfileobj

    if len(argv) <= 1:
        print('Usage: python -m haggis.files.pdf input.pdf '
              '[output.* | ""] [format]', file=stderr)
        exit(1)

    in_path = argv[1]
    out_path = None
    fmt = None
    if len(argv) > 2:
        if argv[2]:
            out_path = argv[2]
        if len(argv) > 3:
            fmt = argv[3]
    else:
        out_path = None

    x = pdf_to_image(in_path, out_path, fmt)

    if isinstance(x, str):
        print(x)
    else:
        print('In memory object -> tmp.x')
        with open('tmp.x', 'wb') as tmp:
            copyfileobj(x, tmp)


from django.contrib import admin

# Register your models here.
from ideas.models import Vote, Idea
from django.utils.html import format_html

#
# admin.site.register(Vote)
# admin.site.register(Idea)

class VoteInline(admin.TabularInline):
    model = Vote

@admin.register(Idea)
class IdeaAdmin(admin.ModelAdmin):
    list_display = ['title', 'status', 'show_youtube_url']
    list_filter = ('status',)
    search_fields = ('title',)

    inlines = [VoteInline]

    def show_youtube_url(self, obj):
        if obj.youtube_url:
            return format_html(f"{obj.youtube_url}")

@admin.register(Vote)
class VoteAdmin(admin.ModelAdmin):
    list_display = ['id', 'idea', 'reason']
    list_filter = ['idea']


import requests
import json

def weather_city(city):
    #和风天气免费开发api https://devapi.qweather.com/v7/weather/now?location=101010100&key=你的KEY
    # 单个插件测试请将API_KEY.json复制到api目录下
    with open("API_KEY.json",'r') as load_f:
        load_dict = json.load(load_f)
        key = load_dict["WEATHER_KEY"]
    """查询城市id"""
    data_id = requests.get("https://geoapi.qweather.com/v2/city/lookup?location={}&key={}".format(city, key))
    content_id = json.loads(data_id.text)
    """城市id"""
    id = content_id['location'][0]["id"]
    """城市名字"""
    name = content_id['location'][0]["name"]
    """以id查询城市天气"""
    data_we = requests.get("https://devapi.qweather.com/v7/weather/now?location={}&key={}".format(id, key))
    content_we = json.loads(data_we.text)
    """实时温度"""
    temp = content_we["now"]["temp"]
    """体感温度"""
    feelsLike = content_we["now"]["feelsLike"]
    """天气"""
    text = content_we["now"]["text"]
    """风向"""
    windDir = content_we["now"]["windDir"]
    """风级数"""
    windScale = content_we["now"]["windScale"]
    """相对湿度"""
    humidity = content_we["now"]["humidity"]
    """能见度"""
    vis = content_we["now"]["vis"]
    """云量"""
    cloud = content_we["now"]["cloud"]
    """降水量"""
    precip = content_we["now"]["precip"]
    """更新时间"""
    time_con = content_we['updateTime']
    date = time_con[0:10]
    time = time_con[11:16]
    """汇总"""
    sum_str = '今天{}天气为{}\n{}{}级\n气温{}度\n体感温度{}度\n相对湿度{}\n可见度{}\n降水量{}\n更新时间{} {}'.format(name, text, windDir, windScale, temp, feelsLike, humidity, vis, precip, date, time)
    return sum_str

if __name__ == '__main__':
    city = '上海'
    print(weather_city(city))


import cv2
import numpy as np
import os
import argparse
from tqdm import tqdm

# template matching method
def stitch(img_prev, img_curr, img):
    ratio=15

    if img_prev is None or img_curr is None:
        raise FileNotFoundError
    if img_prev.shape != img_curr.shape:
        raise ValueError

    # to ignore sidebar, shifting 20px both left and right
    img1 = img_prev[:, 20:img_prev.shape[1] - 20, :]
    img2 = img_curr[:, 20:img_curr.shape[1] - 20, :]

    if img1.shape[2] == 3:
        gray1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
        gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
    else:
        gray1 = img1
        gray2 = img2

    h, w = gray1.shape

    thresh1 = cv2.adaptiveThreshold(gray1, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 11, 2)
    thresh2 = cv2.adaptiveThreshold(gray2, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 11, 2)
    sub = thresh1 - thresh2
    sub = cv2.medianBlur(sub, 3)
    sub = sub // 255

    height = h
    min_height = h*0.97
    # if the white pixel < thresh, we thought they are the same area(bottom status bar)
    thresh = w // 10
    for i in range(h - 1, 0, -1):
        if np.sum(sub[i]) > thresh and height < min_height:
            break
        height = height - 1
    # block is the pre-defined template
    block = sub.shape[0] // ratio
    templ = gray1[height - block:height, ]
    # don't pocess similar picture
    if templ.shape[0] < block:
        return img

    res = cv2.matchTemplate(gray2, templ, cv2.TM_SQDIFF_NORMED)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    if min_val < 0.05:
        return draw(img_prev, img_curr, img, min_loc[1] / h, h - height + block)
    else:
        return img

def draw(img1, img2, img=None, top=0, bottom_height=0):
    if img is None:
        img = img1

    h, w, c = img2.shape

    top = int(top * h)

    roi2 = img2[top:h, ]
    roi = img[:-bottom_height, ]

    image = np.concatenate((roi, roi2), axis=0)

    return image

def from_video(path):
    (name, ext) = os.path.splitext(path)
    cap = cv2.VideoCapture(path)
    fs = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    img = None

    if cap.isOpened():
        ret, prev = cap.read()
        for i in tqdm(range(1, fs)):
            ret, curr = cap.read()
            if ret==True:
                if i%20 == 0:
                    img = stitch(prev, curr, img)
                    prev = curr

    cv2.imwrite('{}_concat.jpg'.format(name), img)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--mode', default='record', help='record or images')
    parser.add_argument('--data', default='test.mp4', help='record or images')

    opt = parser.parse_args()
    if opt.mode == 'record':
        from_video(opt.data)


# -*- coding: utf-8 -*-

import os
import sys

import cv2

from nics_lpirc.api import APIAdapter

class ValAPI(APIAdapter):
    def __init__(self, cfg):
        self.local_dir = cfg.valapi.pic_dir
        self.val_file = cfg.valapi.val_file
        self.num_limit = cfg.valapi.num_limit if cfg.valapi.num_limit >= 0 else None

        # open result file
        self.res_file = cfg.valapi.res_file
        self._res_file = open(self.res_file, "w")

        with open(self.val_file, "r") as f:
            lines = f.read().strip().split("\n")
        self.file_list = [l.split(" ", 1)[0] for l in lines]
        self.num_files = len(self.file_list)
        self.index = 0

    def get_image(self):
        fname = self.file_list[self.index]
        if not fname.endswith(".JPEG"):
            fname += ".JPEG"
        abs_fname = os.path.join(self.local_dir, fname)
        #print abs_fname

        # index in val.txt is 1-based
        im_id = "%d" % (self.index + 1)
        try:
            im = cv2.imread(abs_fname)
        except Exception as e:
            print >>sys.stderr, "im_id: %d: Ignore this image! Exception in reading image: %s: " % (im_id, abs_fname), e
        if im is None:
            print >>sys.stderr, "im_id %s: Ignore this image! Readed image is `None`, please confirm the image exists." % im_id

        self.index += 1
        return im_id, im

    def commit_result(self, im_id, class_ids, dets):
        for index in range(len(class_ids)):
            self._res_file.write("%s %d %f %f %f %f %f\n" % (im_id, class_ids[index],
                                                             dets[index][4], dets[index][0],
                                                             dets[index][1], dets[index][2],
                                                             dets[index][3]))

    def done(self):
        return self.index >= self.num_files or (self.num_limit is not None and self.index >= self.num_limit)

    @classmethod
    def get_image_by_id(cls, cfg, im_id):
        index = int(im_id)
        val_file = cfg.valapi.val_file
        ldir = cfg.valapi.pic_dir
        with open(val_file, "r") as f:
            for line in f:
                index -= 1
                if index <= 0:
                    fname = line.split(" ", 1)[0]
                    if not fname.endswith(".JPEG"):
                        fname += ".JPEG"
                    break
        try:
            abs_fname = os.path.join(ldir, fname)
            im = cv2.imread(abs_fname)
            if im is None:
                print >>sys.stderr, "Failed to read a valid image, ignore this image %s: " % abs_fname, e
        except Exception as e:
            print >>sys.stderr, "Exception in reading image, ignore this image %s: " % abs_fname, e
            return None
        return im


# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from crawlspider.items import CrawlspiderItem
import re

class CrawlspiderSpider(CrawlSpider):
    name = 'Crawlspider'
    allowed_domains = ['sbiquge.com']
    start_urls = ['https://www.sbiquge.com/biqukan/']

    rules = (
        Rule(LinkExtractor(allow=r'/\d+?_\d+?/',unique=True), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        chap_list = response.xpath('.//*[@class="listmain"]/dl/dd')
        for chapter in chap_list:
            print('aaaaa00000000')
            novel_name = chapter.xpath('//*[@id="book"]/div[1]/div/a[2]/text()').extract_first()
            chapter_name = chapter.xpath('./a/text()').extract_first()
            chapter_link = chapter.xpath('./a/@href').extract_first()

            if chapter_name:
                print('a00000000')
                print(chapter_name)
                item = CrawlspiderItem(chapter_title=chapter_name,novel_name=novel_name)
                url = response.urljoin(chapter_link)
                request = scrapy.Request(url=url,callback=self.parse_body)
                request.meta['key'] = item
                yield request

    def parse_body(self,response):
        item = response.meta['key']
        content_list = response.xpath('.//*[@id="content"]') # 匹配到的是一个列表
        print('aaaaaaaaaa')
        print(content_list)
        print('bbbbbbbbbb')
        content_list = content_list.re('([\u4e00-\u9fa5]|
)+?')
        print(content_list)
        print('ccccccccc')
        # 利用re直接匹配小说的汉字内容.正则可以匹配标签下的任何内容，这样我们可以提取我们想要的数据
        content_str = ''.join(content_list)
        content = re.sub('

','\n  ',content_str)
        # 对匹配的章节进行分段
        item['content'] = content
        yield item
        # item = {}
        # item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
        # item['name'] = response.xpath('//div[@id="name"]').get()
        # item['description'] = response.xpath('//div[@id="description"]').get()
        # return item


# _*_ coding:utf-8 _*_

import numpy as np
import cv2
import dlib

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

def get_68_key_point(img):
    """
    根据图像利用dlib获取68个关键点，保存成列表输出
    x = [(索引0, 坐标0), (索引1, 坐标1)]
    y = [x, 灰度图]
    """
    data = []

    # 取灰度
    img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 人脸数rects
    rects = detector(img_gray, 0)
    for i in range(len(rects)):
        #print("正在检查人脸")
        landmarks = np.matrix([[p.x, p.y] for p in predictor(img,rects[i]).parts()])
        for idx, point in enumerate(landmarks):
            # 68点的坐标
            #pos = (point[0, 0], point[0, 1])
            data.append(point[0, 0])
            data.append(point[0, 1])
            bdata = np.reshape(data, [-1])
        data = [bdata, img_gray]
    return data

    #应该把没有识别出来的图片自动不保存


from django import forms

from . import models

class OrderForm(forms.ModelForm):
    class Meta:
        model = models.Order    #直接取用models中的類別
        fields = ['full_name', 'address', 'postal_code', 'phone']
    def __str__(self, *args, **kwargs):
        super(OrderForm, self).__init__(*args, **kwargs)
        self.fields['full_name'].label = '收件者姓名'
        self.fields['address'].label = '收件地址'
        self.fields['postal_code'].label = '郵遞區號'
        self.fields['phone'].label = '聯絡電話'


import boto3,json

class AwsAsg:
    def __init__(self, cluster):
        print("Init AwsAsg Class")
        self.cluster = cluster
        self.activeInstances = []
        self.newInstances = []
        self.status = ""
        self.snsTopicArn = ""
        self.snsSubject = ""
        self.snsMessage = ""
        self.lifecycleHookName = ""
        self.lifecycleActionToken = ""
        self.autoscalingClient = boto3.client('autoscaling')
        self.ec2Client = boto3.client('ec2')

    def complete_lifecycle_action(self, lifecycleActionResult):
        try:
            response = self.autoscalingClient.complete_lifecycle_action(LifecycleHookName=self.lifecycleHookName,AutoScalingGroupName=self.cluster,LifecycleActionToken=self.lifecycleActionToken,LifecycleActionResult=lifecycleActionResult)
            if response:
                return response
        except BaseException as e:
            print("Finish manual run.  This is not a Lifecycle event: " + str(e))

    def check_event_status(self, event, context):
        print("Test Event for type")
        self.event = event
        self.context = context
        eventStatus = ""

        try:
            print("Test if this was called from an ASG lifecycle event or SNS message")
            print(event['Records'][0]['Sns']['TopicArn'])
            snsMessage=json.loads(event['Records'][0]['Sns']['Message'])
            try:
                print("Test for ASG event")
                print(snsMessage['Event'])
                eventStatus = "ASG"
            except BaseException as e:
                print("This is a SNS message")
                print(str(e))
                eventStatus = "SNS"
        except BaseException as e:
            print(str(e))

        if (eventStatus == "SNS") or (eventStatus == "ASG"):
            self.snsTopicArn=event['Records'][0]['Sns']['TopicArn']
            self.snsSubject=event['Records'][0]['Sns']['Subject']
            self.snsMessage=json.loads(event['Records'][0]['Sns']['Message'])

            if eventStatus == "ASG":
                self.snsEvent=snsMessage['Event']

                if self.snsEvent == "autoscaling:TEST_NOTIFICATION":
                    print("Ignore: this is a test event")
                    self.status = "exit"
                else:
                    self.lifecycleHookName=snsMessage['LifecycleHookName']
                    self.lifecycleActionToken=snsMessage['LifecycleActionToken']
                    self.lifecycleTransition=snsMessage['LifecycleTransition']

                    if self.lifecycleTransition == "autoscaling:EC2_INSTANCE_TERMINATING":
                        # self.status = "backup"
                        self.status = "exit"
                    elif self.lifecycleTransition == "autoscaling:EC2_INSTANCE_LAUNCHING":
                        self.status = "heal"
                    else:
                        self.status = "update"
        else:
            self.status = "install"

    def check_instance_status(self):
            #Get all instances for an ASG
        filters = [{
            'Name': 'tag:aws:autoscaling:groupName',
            'Values': [self.cluster]
        }]

        print("Print instances in autoscaling group")
        for reservation in self.ec2Client.describe_instances(Filters=filters)['Reservations']:
            # print(reservation['Instances'])

            for instance in reservation['Instances']:
                #Check to see if instance is healthy
                response = self.autoscalingClient.describe_auto_scaling_instances(
                    InstanceIds=[
                        instance['InstanceId']
                    ]
                )

                for asgInstance in response['AutoScalingInstances']:
                    #Pretty print json of instance from AWS autoscaling group
                    print(json.dumps(asgInstance, indent=4, sort_keys=True))

                    if (asgInstance['LifecycleState'] == 'InService'):
                        print("This instance is good to go!")
                        self.activeInstances.append(instance)
                    elif (asgInstance['LifecycleState'] == 'Pending') or (asgInstance['LifecycleState'] == 'Pending:Wait') or (asgInstance['LifecycleState'] == 'Pending:Proceed'):
                        print("We have a new instance.  Welcome!")
                        self.newInstances.append(instance)
                        self.activeInstances.append(instance)
                        self.status = "heal"
                    else:
                        print("This instance is bad, we will ignore.")


#DHCPAdmin.py

from socket import *
import re, uuid
import sys

serverName = 'localhost'
serverPort = 12001
clientSocket = socket(AF_INET, SOCK_DGRAM)
clientMacAddr =  macAddr = ':'.join(re.findall('..', '%012x' % uuid.getnode()))
clientBroadCast = '255.255.255'
currClientIPAddr = '0.0.0.0'

def sendMessage(outgoingMessage):
    clientSocket.sendto(outgoingMessage.encode(),(serverName, serverPort))

#get computers MAC Address
def getMACAddr():
    macAddr = ':'.join(re.findall('..', '%012x' % uuid.getnode()))
    return macAddr

#checks if the entered mac address is its own mac address
def checkMACAddr(address):
    if address == clientMacAddr:
        return True
    else:
        return False

#receive message and figure out what kind it is (ACK or OFFER)
def receiveMessage():
    serverMessage, serverAddress = clientSocket.recvfrom(2048)
    serverMessage = serverMessage.decode()
    serverMessage = serverMessage.split(',')
    msg_type = serverMessage[0]
    msg_mac_addr = serverMessage[1]
    msg_IP = serverMessage[2]
    print(serverMessage)
    if 'END' in serverMessage:
        sys.exit()
    else:
        pass

outmessage = 'LIST' + ',' + clientMacAddr + ',' + currClientIPAddr
sendMessage(outmessage)
while 1:
    receiveMessage()


import copy

from utils_ak.numeric import *
from utils_ak.simple_vector import *

from utils_ak.block_tree.parallelepiped_block import ParallelepipedBlock
from utils_ak.block_tree.block import Block
from utils_ak.block_tree.validation import validate_disjoint_by_axis
from utils_ak.block_tree.pushers.pushers import simple_push

from loguru import logger

def dummy_push(
    parent, block, validator, max_tries=24, start_from="max_end", iter_props=None
):
    axis = parent.props["axis"]

    if is_int_like(start_from):
        cur_start = int(float(start_from))
    elif start_from == "beg":
        cur_start = block.props["x_rel"][axis]
    else:
        if not parent.children:
            cur_start = 0
        else:
            if start_from == "max_beg":
                cur_start = max(
                    [child.props["x_rel"][axis] for child in parent.children]
                )
            elif start_from == "max_end":
                cur_start = max(
                    [
                        (child.props["x_rel"] + child.size)[axis]
                        for child in parent.children
                    ]
                )
            else:
                raise Exception("Unknown beg type")

    iter_props = iter_props or [{}]

    cur_x = cast_simple_vector(block.n_dims)

    cur_try = 0

    # logger.debug('Inserting blocks', parent=parent.props['cls'], block=block.props['cls'])

    while cur_try < max_tries:
        dispositions = []
        for props in iter_props:
            # try to push
            props = copy.deepcopy(props)
            cur_x[axis] = cur_start
            props["x"] = cur_x
            # logger.debug('Trying to push from ', cur_start=cur_start)
            res = simple_push(parent, block, validator=validator, new_props=props)

            if isinstance(res, Block):
                # success
                # logger.debug('Success')
                return block
            else:
                assert isinstance(res, dict)

                if "disposition" in res:
                    dispositions.append(res["disposition"])

        disposition = min(dispositions) if len(dispositions) == len(iter_props) else 1
        # logger.debug('Disposition', disposition=disposition)
        cur_start += disposition
    raise Exception("Failed to push element")

def test_dummy_push():
    from utils_ak.loguru import configure_loguru_stdout

    configure_loguru_stdout()

    def brute_validator(parent, block):
        for c in parent.children[:-1]:
            validate_disjoint_by_axis(c, block, axis=parent.props["axis"])

    logger.debug("Dummy push test")
    root = ParallelepipedBlock("root", n_dims=1, x=[2], axis=0)
    a = ParallelepipedBlock("a", n_dims=1, size=[4], axis=0)
    b = ParallelepipedBlock("b", n_dims=1, size=[3], axis=0)
    dummy_push(root, a, brute_validator)
    dummy_push(root, b, brute_validator, start_from=0)
    logger.debug("Root", root=root)

if __name__ == "__main__":
    test_dummy_push()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('control_acero', '0033_remove_frente_estatusfrente'),
    ]

    operations = [
        migrations.AddField(
            model_name='material',
            name='nombre',
            field=models.CharField(max_length=100, blank=True),
        ),
    ]


from flask import Flask, jsonify, request
from flask_pymongo import PyMongo, MongoClient
import conn

app = Flask(__name__)

base = conn.conn_to_database()

@app.route('/', methods=['GET'])
def home():
    res = "access /games - to consult all games<br/>access /games/[game_num] to consult this game"
    return res

@app.route('/games', methods=['GET'])
def get_all_games():
    games = base.games_parsed

    output = []

    for g in games.find():
        output.append({
            'game_id': g['id_game'],
            'game_value': g['game']
        })
    if len(output):
        return jsonify({'result':output})
    else:
        return jsonify({'result': "No games found"})

@app.route('/games/<game_id>', methods=['GET'])
def get_game(game_id):
    games = base.games_parsed
    game_id_concat = 'game_%s' % game_id
    # return game_id_concat
    output = []

    find_game = games.find_one({"id_game": game_id_concat})

    if find_game:
        output = { 'game_value': find_game['game'] }
    else:
        output = "Game not found"

    return jsonify({'result':output})

if __name__ == "__main__":
    app.run(debug=True)
    # print(app)


from google.cloud import bigquery
from external_api.gcp_biquery import BQWriter
import pytest
from mock import patch, Mock

TESTDATASET_NAME = "tds"
TESTTABLE_NAME = "ttn"

@patch('external_api.gcp_biquery.bigquery.Client')
def test_delete_sql_statement_executes(mock_bq_client):
    bq = BQWriter(TESTDATASET_NAME, TESTTABLE_NAME)

    mock_bq_client_instance = mock_bq_client.return_value
    bq.delete_image_data('test_image')

    expectedSql = "DELETE FROM {}.{} WHERE imagepath = '{}'".format(TESTDATASET_NAME, TESTTABLE_NAME, 'test_image')
    mock_bq_client_instance.query.assert_called_with(expectedSql)

@patch('external_api.gcp_biquery.bigquery.Client')
@pytest.mark.parametrize("rows_found, expected_result",
     [(1,True),
    (0,False),
    (40,True),
    (-4,False)])
def test_image_data_exists(mock_bq_client, rows_found, expected_result):
    bq = BQWriter(TESTDATASET_NAME, TESTTABLE_NAME)
    queryJobMock = Mock()
    queryJobMock.result.return_value.total_rows = rows_found

    mock_bq_client_instance = mock_bq_client.return_value
    mock_bq_client_instance.query.return_value = queryJobMock

    exists = bq.image_data_exists('test_image')

    assert exists == expected_result
    expectedSql = "SELECT * FROM {}.{} WHERE imagepath = '{}'".format(TESTDATASET_NAME, TESTTABLE_NAME, 'test_image')
    mock_bq_client_instance.query.assert_called_with(expectedSql)


#!python
print("Content-Type: text/html") #헤더라고 부름.
print()
import sys
import codecs
import cgi, os, view, html_sanitizer

sanitizer = html_sanitizer.Sanitizer()

def getList():
    files = os.listdir('data')
    listStr=''
    for item in files:
        listStr = listStr + '
{name}
'.format(name=item)
    return listStr

form = cgi.FieldStorage()

if 'id' in form:
    title = pageId = form["id"].value
    description = open('data/'+pageId, 'r').read()
    #description = description.replace('<', '<') //XSS 위협에 대한 보안
    #description = description.replace('>', '>') //이것도
    title =  sanitizer.sanitize(title)
    description = sanitizer.sanitize(description) #pip 사용
    update_link = 'update'.format(pageId)
    delete_action = '''
        <form action="process_delete.py" method="post">
            <input type="hidden" name="pageId" value="{}">
            <input type="submit" value="delete">
        </form>
    '''.format(pageId)
else:
    title = pageId = 'Welcome'
    description = 'Hello, web'
    update_link = ''
    delete_action = ''

print('''<!doctype html>

  <title>WEB1 - Welcome</title>
  <meta charset="utf-8">

WEB

    {listStr}

  create
  {update_link}
  {delete_action}

{title}

{desc}

'''.format(
    title = title,
    desc = description,
    listStr=view.getList(),
    update_link=update_link,
    delete_action=delete_action))


from flask import Flask,render_template,request
import requests
import random
import json
from faker import Faker
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

@app.route('/lotto')
def lotto(): #로또 당첨번호 for문으로 들고오기
    url ="https://www.dhlottery.co.kr/common.do?method=getLottoNumber&drwNo=837"#동행로또링크
    res = requests.get(url).text
    lotto_dict = json.loads(res)#딕셔너리를 lotto_dict에 저장
    print(lotto_dict["drwNoDate"])
    week_format_num = []
    num_list = range(1,46)#1~45값 뽑아옴
    pick= random.sample(num_list,6)#sample 함수를 pick에 저장

    for i in range(1,7):
        week_format_num.append(lotto_dict["drwtNo{}".format(i)])#한줄로 코드만드는 방법
        #포맷팅 방법을이용해 i에 값을 넣는다
        #포멧팅하는 방식1
        #"오{}".format(b) #a=오 b=창희
        #결과값 오창희

    #print(type(res))
    #print(type(json.loads(res))) #json이 res를 읽어서 딕셔너리로 바꿔줘
    t=0
    for j in range(1,7):
        for w in range(1,7):
            if(week_format_num[j-1]==pick[w-1]):
                t= t+1
    f= 6-t
    print(t)
    print(f)

    rank = ["꼴등","5등","4등","3등","2등","1등"]
    a = ""
    for i in range(6):
        if (t==i):
            print(rank[i])
            a = str(rank[i])

    return render_template("lotto.html",lotto=pick, week_format_num=week_format_num , t=t, a=a ,f=f)
    #pick 을 lotto에 담아서 lotto.html로보냄
    #html 하나를 사용자한테 보내줌
    #templates라는 폴더를 만들어줘야됨
    #규칙이기때문에 lotto.html을 templates에서 읽음
    #그리고 templates폴더에 lotto.html 파일생성
    #render_template는 플라스크에서 쓰는 함수

@app.route('/ping')
def ping():
     return render_template("ping.html")

@app.route('/pong')
def pong():
    input_name=request.args.get('name')#requests와 다름 파이썬에 기본적으로있는 모듈 불러오기
    #args 데이터를 의미 get가져옴
    #input_name에 넣음
    fake = Faker('ko_KR')
    fake_job = fake.city()
    return render_template("pong.html",html_name=input_name, fake_job=fake_job, fake= fake)


import logging

from torch import nn

from .utils import count_parameters

"""
ViT

Pipeline only support nn.Sequential partitions thus a layer cannot take more than 1 preceding layers as input.
This model architecture is normal in practice, such as Transformer Multi Head Attention layer, and the skip connection in ResNet.

Without such support, the partition can only be coarse-grained, which is hard to make the word load balanced in pipeline.
"""

class MultiHeadAttentionLayer(nn.Module):
    def __init__(self, attention_norm, attn):
        super(MultiHeadAttentionLayer, self).__init__()
        self.attention_norm = attention_norm
        self.attn = attn

    def forward(self, x):
        h = x
        x = self.attention_norm(x)
        x, weights = self.attn(x)
        x = x + h
        return x

class MLPLayer(nn.Module):
    def __init__(self, ffn_norm, ffn):
        super(MLPLayer, self).__init__()
        self.ffn_norm = ffn_norm
        self.ffn = ffn

    def forward(self, x):
        h = x
        x = self.ffn_norm(x)
        x = self.ffn(x)
        x = x + h
        return x

class ViTOutputHead(nn.Module):
    def __init__(self, hidden_size, num_classes):
        super(ViTOutputHead, self).__init__()
        self.head = nn.Linear(hidden_size, num_classes)

    def forward(self, x):
        logits = self.head(x[:, 0])
        return logits

class FrozenLayer(nn.Module):
    def __init__(self, num_frozen_layer, frozen_emb, frozen_layer_list):
        super().__init__()
        self.num_frozen_layer = num_frozen_layer
        self.embedding = frozen_emb
        self.layers = nn.ModuleList()
        for layer_i in range(num_frozen_layer):
            self.layers.append(frozen_layer_list[layer_i])

    def forward(self, x, layer_id=0):
        # logging.info(x)
        if layer_id == self.num_frozen_layer:
            logging.info("no need to recompute")
            return x
        if layer_id == 0:
            logging.info("compute from layer 0")
            x = self.embedding(x)
            for id in range(0, self.num_frozen_layer):
                x = self.layers[id](x)
            return x
        else:
            logging.info("compute from layer %d-%d" % (layer_id, self.num_frozen_layer - 1))
            for id in range(layer_id, self.num_frozen_layer):
                x = self.layers[id](x)
            return x

def create_pipe_styled_model_vit(model_config, model_backbone, num_layer_in_total, num_frozen_layer):
    """
    Optimization:
        Pin Memory: https://pytorch.org/docs/stable/notes/cuda.html#use-pinned-memory-buffers
        Prepare a Pin Memory model
    """
    #    logging.info(model_backbone)
    frozen_model = None
    pipe_model = nn.Sequential()

    parameters_size_frozen = 0.0
    parameters_list_pipe = []

    if num_frozen_layer > 0:
        for param in model_backbone.transformer.embeddings.parameters():
            param.requires_grad = False

        frozen_emb = model_backbone.transformer.embeddings

        size_embedding = count_parameters(model_backbone.transformer.embeddings, False)
        parameters_size_frozen += size_embedding

        frozen_layer_list = nn.ModuleList()
        for frozen_layer_index in range(num_frozen_layer):
            layer_block = model_backbone.transformer.encoder.layer[frozen_layer_index]
            for param in layer_block.parameters():
                param.requires_grad = False
            frozen_layer_list.append(layer_block)

            size_layer_block = count_parameters(layer_block, False)
            parameters_size_frozen += size_layer_block

        frozen_model = FrozenLayer(num_frozen_layer, frozen_emb, frozen_layer_list)
    else:
        pipe_model.add_module("embedding", model_backbone.transformer.embeddings)
        size_embedding = count_parameters(model_backbone.transformer.embeddings, False)
        parameters_list_pipe.append(size_embedding)

    # add transformer blocks needed to be trained
    for layer_index in range(num_frozen_layer, num_layer_in_total):
        layer_block = model_backbone.transformer.encoder.layer[layer_index]
        multihead_attention_layer = MultiHeadAttentionLayer(layer_block.attention_norm, layer_block.attn)
        mlp_layer = MLPLayer(layer_block.ffn_norm, layer_block.ffn)
        pipe_model.add_module("multihead_attention_layer" + str(layer_index), multihead_attention_layer)
        pipe_model.add_module("mlp_layer" + str(layer_index), mlp_layer)

        size_multihead_attention_layer = count_parameters(multihead_attention_layer, False)
        parameters_list_pipe.append(size_multihead_attention_layer)

        size_mlp_layer = count_parameters(mlp_layer, False)
        parameters_list_pipe.append(size_mlp_layer)

    pipe_model.add_module("encoder_norm", model_backbone.transformer.encoder.encoder_norm)
    size_encoder_norm = count_parameters(model_backbone.transformer.encoder.encoder_norm, False)
    parameters_list_pipe.append(size_encoder_norm)

    output_head = ViTOutputHead(model_config.hidden_size, model_config.output_dim)
    pipe_model.add_module("head", output_head)
    size_output_model = count_parameters(output_head, False)
    parameters_list_pipe.append(size_output_model)

    # logging.info(frozen_model)
    # logging.info(parameters_size_frozen)
    # logging.info(pipe_model)
    # logging.info(parameters_list_pipe)

    return frozen_model, parameters_size_frozen, pipe_model, parameters_list_pipe

def freeze_vit_only(model_backbone, num_frozen_layer):
    """
    Optimization:
        Pin Memory: https://pytorch.org/docs/stable/notes/cuda.html#use-pinned-memory-buffers
        Prepare a Pin Memory model
    """
    for param in model_backbone.transformer.embeddings.parameters():
        param.requires_grad = False
    for frozen_layer_index in range(num_frozen_layer):
        layer_block = model_backbone.transformer.encoder.layer[frozen_layer_index]
        for param in layer_block.parameters():
            param.requires_grad = False


import os
import random
import pandas as pd
import glob2
import numpy as np
from numpy import argmax, mean, std
import librosa
import librosa.display as ld
import pylab
import matplotlib.pyplot as plt
from matplotlib import figure
from time import time
# import pickle
import gc
import csv
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras import models
from timeit import default_timer as timer
from scipy.io import wavfile as wav
from matplotlib.pyplot import specgram
from tensorflow.keras.models import Sequential
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score
from sklearn import preprocessing
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split, ParameterGrid
from sklearn.preprocessing import LabelEncoder
from tune_sklearn import TuneSearchCV

from ray import tune
from ray.tune.suggest.hyperopt import HyperOptSearch
from ray.tune.suggest.basic_variant import BasicVariantGenerator
from hyperopt import tpe, hp, fmin, STATUS_OK, Trials
from hyperopt.pyll.base import scope

from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import classification_report
import _pickle as cPickle
import gzip
import numpy, scipy, matplotlib.pyplot as plt, sklearn, librosa, urllib.request, urllib.parse, urllib.error
from sklearn.model_selection import cross_val_score

def load_sound_files(file_paths):
    raw_sounds = []
    labels = []
    sr_=[]
    signal = []
    for fp in  file_paths:
        X, sample_rate = librosa.load(fp)
        # print(sample_rate)
        # stft = np.abs(librosa.stft(X))
        mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=16).T,axis=0)
        mfccs = sklearn.preprocessing.scale(mfccs, axis=0)
        # chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T,axis=0)
        mel = np.mean(librosa.feature.melspectrogram(X, sr=sample_rate).T,axis=0)
        raw_sounds.append(mfccs)
        sr_.append(sample_rate)
        signal.append(X)
    raw_sounds = np.array(raw_sounds)
    sr_ = np.array(sr_)
    signal = np.array(signal)
    # print(raw_sounds.shape())
    return raw_sounds

#train test split
def split_train_test(dataframe, target):
    train_set, test_set = train_test_split(dataframe, test_size=0.2, random_state=42)
    X_train, X_test = train_set.drop(target, axis=1), test_set.drop(target, axis=1)
    y_train, y_test = train_set[target], test_set[target]

    return X_train, X_test, y_train, y_test

def model():

    Data_dir = (load_sound_files(np.array(glob2.glob("/home/shul/audio/FoR/for-rerecorded/training/real/*"))))

    original_class = []
    name = []
    labelName1 = []
    original_feature = []
    i = 0
    for file in Data_dir[i : i + 5000]:

        name.append(0)
    # labelName = np.asarray(original_class)
    original_class = np.asarray(name)

    label_encoder = LabelEncoder()
    label_concoded_original = label_encoder.fit_transform(original_class)

    sounds_file_original = glob2.glob("/home/shul/audio/FoR/for-rerecorded/training/real/*.wav", recursive=True)
    original_feature = load_sound_files(sounds_file_original[:5000])
    norm_original_feature = preprocessing.normalize(original_feature, norm='l2')
    print((norm_original_feature))
    original_feature = norm_original_feature.reshape(5000, 16)

    Data_dir2 = np.asarray(glob2.glob("/home/shul/audio/FoR/for-rerecorded/training/fake/*"))
    fake_class = []
    label_name2 = []
    fake_feature = []
    j = 0

    for file in Data_dir2[j: j + 5000]:
        fake_class.append(1)

    fake_class = np.asarray(fake_class)

    # print(fake_class)
    label_encoder2 = LabelEncoder()
    label_encoded_fake = label_encoder2.fit_transform(fake_class)
    sounds_file_fake = glob2.glob("/home/shul/audio/FoR/for-rerecorded/training/fake/*.wav", recursive=True)
    fake_feature = load_sound_files(sounds_file_fake[:5000])
    norm_fake_feature = preprocessing.normalize(fake_feature, norm='l2')

    fake_feature = norm_fake_feature.reshape(5000, 16)

    df1 = pd.DataFrame(data=original_feature)
    df2 = pd.DataFrame(data=fake_feature)

    df1['label'] = pd.Series([0 for x in range(len(df1.index))])
    df2['label'] = pd.Series([1 for x in range(len(df2.index))])

    #append dataframes
    final_data = df1.append(df2, ignore_index=True)

    #shuffle rows
    final_data = final_data.sample(frac = 1)

    X_train, X_test, y_train, y_test = (split_train_test(final_data, 'label'))

    print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)

    start = 0
    x_train = []
    x_test = []
    Y_train = []
    Y_test = []
    for i in range(0, len(X_train)):
        result = X_train.values.reshape(8000, 4, 4, 1)
    for i in range(0, len(X_test)):
        test = X_test.values.reshape(2000, 4, 4, 1)
    for i in range(0, len(X_train)):
        ytrain = y_train.values.reshape(8000)
    for i in range(0, len(X_train)):
        ytest = y_test.values.reshape(2000)

    x_train = np.array(result)
    x_test = np.array(test)
    Y_train = np.array(ytrain)
    Y_test = np.array(ytest)
    newx_train = x_train.reshape(8000, 16)
    newy_train = Y_train

    batch_size = 64
    epochs = 200
    nb_layers = 3
    model = tf.keras.Sequential()
    model.add(layers.Conv2D(64, (2, 2),input_shape=(4, 4, 1)))
    model.add(layers.Activation('relu'))
    model.add(layers.Conv2D(120, (2, 2)))
    model.add(layers.Activation('relu'))
    model.add(layers.MaxPool2D(2, 2))
    model.add(layers.Dropout(0.25))
    model.add(layers.Flatten())
    model.add(layers.Dense(32))
    model.add(layers.Activation('relu'))
    model.add(layers.Dropout(0.25))
    # model.add(layers.Dense(64))
    # model.add(layers.Activation('relu'))
    # model.add(layers.Dropout(0.4))
    model.add(layers.Activation('softmax'))

    model.summary()
    adam = tf.keras.optimizers.Adam(lr=0.001)
    model.compile(loss =  'sparse_categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])

    model.fit(x_train, Y_train, validation_split=0.1, shuffle=True, verbose = 1, batch_size=batch_size, epochs = epochs)

    # return model, x_train, Y_train

    space = {
            "n_estimators" : hp.choice("n_estimators", [100,200,300,400]),
            "max_depth" : hp.quniform("max_depth", 1, 15, 1),
            "critetion" : hp.choice("criterion", [ "entropy"]),
        }

    # def hyperparameter_tuning(params):
    #     model, x_train, Y_train = model()
    #     acc = cross_val_score(model, x_train, Y_train, scoring="accuracy").mean()
    #     return {"loss": acc, "status":STATUS_OK}

    trials = Trials()

    best = fmin(
        fn = model,
        space = space,
        algo = tpe.suggest,
        max_evals = 100,
        trials = trials
    )

    print(best)
    print(trials.result)

model()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 14 18:04:10 2021

purpose: scan killifish proteome and look protein with KFERQ-containing peptides
The motif is always flanked by
a glutamine on one of the sides (as the pentapeptide functions as a targeting sequence in both directions)
and contains one or two of the positive residues K and R,
one or two of the hydrophobic residues F, L, I or V and
one of the negatively charged E or D residues

@author: yiwenchen
"""

import os
import pandas as pd
import numpy as np
import re
import sys
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import ttest_ind
import matplotlib as mpl

mpl.rcParams['pdf.fonttype'] = 42
mpl.rcParams.update({'font.size': 20})
sns.set(style="white")

#Functions
# Read a fasta file and output a proteome_dict = {proteinA:proteinA_sequence, proteinB:proteinB_sequence...}
# File_Name = '2016.02.12 - nfurzeri_proteins_final_mline_symbol.fasta'
def read_old_fasta(File_Name, motif = None):
    f = open(File_Name)
    Raw_Data = f.read()
    f.close()
    Raw_Data = Raw_Data.split('>')[1:]
    Len_Prot = len(Raw_Data)
    Proteome_Dict = dict()
    for i in np.arange(0, Len_Prot):
        Entry = Raw_Data[i]
        Prot_Last_Index = Entry.find('\n')
        Protein = Entry[:Prot_Last_Index].replace(' ','')
        Sequence = Entry[Prot_Last_Index:].replace('\n','').replace(' ' ,'')
        if motif == None:
            Proteome_Dict[Protein] = str(Sequence)
        else:
            for m_i in motif:
                regexp = re.compile(m_i)
                if regexp.search(Sequence):
                    Proteome_Dict[Protein] = str(Sequence)
    return Proteome_Dict

def new_rank(df, fc = 'logFC', pval = 'Qval'):
    df['rankstats'] = df[fc]*(-1)*np.log10(df[pval])
    return df

# ########## Section #1: Look for KFERQ-like peptide in killifish proteome ##########
# motif = ['Q[(K,R)]{1,2}[(F,L,I,V)]{1,2}[E|D]', '[(K,R)]{1,2}[(F,L,I,V)]{1,2}[E|D]Q',\
#          'Q[(K,R)]{1,2}[E|D][(F,L,I,V)]{1,2}', '[(K,R)]{1,2}[E|D][(F,L,I,V)]{1,2}Q',\
#          'Q[(F,L,I,V)]{1,2}[(K,R)]{1,2}[E|D]', '[(F,L,I,V)]{1,2}[(K,R)]{1,2}[E|D]Q',\
#          'Q[(F,L,I,V)]{1,2}[E|D][(K,R)]{1,2}', '[(F,L,I,V)]{1,2}[E|D][(K,R)]{1,2}Q',\
#          'Q[E|D][(F,L,I,V)]{1,2}[(K,R)]{1,2}', '[E|D][(F,L,I,V)]{1,2}[(K,R)]{1,2}Q',\
#          'Q[E|D][(K,R)]{1,2}[(F,L,I,V)]{1,2}', '[E|D][(K,R)]{1,2}[(F,L,I,V)]{1,2}Q']
# CMA_Substrate_Dict = read_old_fasta('KillifishResources/longest-Protein-seqs_Nfu-ncbi100_andMito.fasta', motif = motif)
# XP_list = [i[-14:] for i in CMA_Substrate_Dict.keys()]
# np.savetxt('GeneSets/Killifish/CMA_Substrate.csv', XP_list, delimiter =',', fmt='%s')

# ########## Section #2: Given list of XP, output raw MS values ##########
# # df_symbol = pd.read_csv('KillifishResources/nfur_updated_symbols_20161013.csv')
# # Find all related proteins in the fish proteome
# geneset_cateogry = 'CMA_Substrate'
# geneset_fnames = ['CMA_Substrate_Killifish.csv']
# # sample_type can be WCL, or AGG, or Prop
# sample_type = 'Prop' #AGG, WCL, Prop
# pairs = [('Young','Old'),('Young','Tert'),('Old','Tert')]
# fpath = 'GeneSets/Killifish/Autophagy'
# foutpath = 'GeneSets/MS/%s_%s'%(geneset_cateogry, sample_type)
# if not os.path.exists(foutpath): os.makedirs(foutpath)
# all_geneset = []
# geneset_list = []
# df_input = pd.read_csv(os.path.join('', 'kf_combined_prop.csv'))
# for geneset_fname in geneset_fnames:
#     with open(os.path.join(fpath, geneset_fname), 'r') as f: geneset = f.read()
#     geneset = geneset.replace('\r','\n')
#     geneset = geneset.replace('\n\n','\n')
#     geneset = list(set(geneset.split('\n')))
#     all_geneset += geneset
#     geneset_list.append(geneset)
# for pair in pairs:
#     age_pair = '%sv%s'%(pair[1][0], pair[0][0])
#     for geneset, set_name, i in zip(geneset_list, geneset_fnames, range(len(geneset_list))):
#         #### generate table that's convenient for heatmap or clustering ####
#         if sample_type == 'Prop':
#             df_data = df_input[(df_input['N. furzeri Protein Id'].isin(geneset)) & (df_input['Sample.Type']=='AGG')]
# #            fc = 'Delta_Propensity_%s-%s'%(pair[1], pair[0])
# #            pval = 'Propensity_%s-%s_pval'%(pair[1], pair[0])
#             fc = '%s_prop_logFC'%age_pair
#             pval = '%s_prop_pval'%age_pair
#         else:
#             df_data = df_input[(df_input['N. furzeri Protein Id'].isin(geneset)) & (df_input['Sample.Type']==sample_type)]
#             fc = '%s_logFC'%age_pair
#             pval = '%s_pval'%age_pair
#         if (df_data[fc].dropna(how='all').empty) == False:
#             df_data = new_rank(df_data, fc = fc, pval = pval)
#             df_out = df_data.pivot_table(index = 'Human', columns = 'Tissue', values = [fc, pval, 'rankstats'])
#             df_out.insert(0, 'classification', set_name.replace('_Killifish.csv','').replace('_All',''))
#             df_out_flat = df_data[['Human', 'Tissue', fc, pval, 'rankstats']]
#             df_out_flat.insert(0, 'classification', set_name.replace('_Killifish.csv','').replace('_All',''))
#             df_out_flat.to_csv(os.path.join(foutpath, set_name.replace('Killifish.csv', '%s_%s_flat.csv'%(age_pair,sample_type))),index = False)
#             if i == 0:
#                 df_allout = df_out.copy(deep = True)
#                 df_allout_flat = df_out_flat.copy(deep = True)
#             else:
#                 df_allout = df_allout.append(df_out, ignore_index = False)
#                 df_allout_flat = df_allout_flat.append(df_out_flat, ignore_index = False)
#             out_name = set_name.replace('_Killifish', '_%s_%s'%(age_pair,sample_type))
#             df_out.to_csv(os.path.join(foutpath, out_name),index = True)
# ############# End of Section 2 ##############

# ########## Section #3: Given list of XP, output raw MS values ##########
# df_symbol = pd.read_csv('KillifishResources/nfur_updated_symbols_20161013.csv')
# Find all related proteins in the fish proteome
fpath = 'GeneSets/Killifish/Autophagy'
geneset_cateogry = 'CMA_Substrate'
geneset_fname = 'CMA_Substrate_Killifish.csv'
geneset = open(os.path.join(fpath, geneset_fname), 'r').read().split('\n')
# sample_type can be WCL, or AGG, or Prop
sample_type = 'AGG' #AGG, WCL, Prop
pairs = [('Young','Old'),('Young','Tert'),('Old','Tert')]
tissue_order = ['Brain', 'Gut', 'Heart', 'Liver', 'Muscle', 'Skin']#,'Testis']
foutpath = 'GeneSets/MS/%s_%s'%(geneset_cateogry, sample_type)
if not os.path.exists(foutpath): os.makedirs(foutpath)
df_input = pd.read_csv(os.path.join('', 'kf_combined_prop.csv'))
df_input['CMA_Sub']=df_input['N. furzeri Protein Id'].isin(geneset)
ttest_out = np.zeros((len(pairs), len(tissue_order)))
for p_i, pair in enumerate(pairs):
    age_pair = '%sv%s'%(pair[1][0], pair[0][0])
    if sample_type == 'Prop':
        fc = '%s_prop_logFC'%age_pair
        df_plot = df_input[df_input['Sample.Type']=='AGG']
    else:
        fc = '%s_logFC'%age_pair
        df_plot = df_input[df_input['Sample.Type']==sample_type]
    for t_i, tissue in enumerate(tissue_order):
        df_tissue = df_plot[(df_plot['Tissue']==tissue)&(df_plot[fc].notna())]
        tissue_ttest = ttest_ind(*df_tissue.groupby('CMA_Sub')[fc].apply(lambda x: list(x)), equal_var = False)
        ttest_out[p_i,t_i] = tissue_ttest[1]
    fig, ax = plt.subplots(figsize=(15,10))
    sns.boxplot(data = df_plot, x = 'Tissue', y = fc, hue = 'CMA_Sub',\
                order = tissue_order, ax = ax, palette="Set3", fliersize = 0)
    sns.swarmplot(data = df_plot, x = 'Tissue', y = fc, hue = 'CMA_Sub',\
                  order = tissue_order, ax = ax, dodge=True, size = 1, palette="Set3")
    fig.savefig(os.path.join(foutpath, 'CMA_Substrate_%s_%s.pdf'%(sample_type, age_pair)))
df_ttest = pd.DataFrame(data = ttest_out, index = pairs, columns = tissue_order)
df_ttest.to_csv(os.path.join(foutpath, 'CMA_Substrate_%s_ttestp.csv'%(sample_type)))

# # ########## Section #4: Compare brain TL and AGG substrate differences ##########
# # df_symbol = pd.read_csv('KillifishResources/nfur_updated_symbols_20161013.csv')
# # Find all related proteins in the fish proteome
# fpath = 'GeneSets/Killifish/Autophagy'
# geneset_cateogry = 'CMA_Substrate'
# geneset_fname = 'CMA_Substrate_Killifish.csv'
# geneset = open(os.path.join(fpath, geneset_fname), 'r').read().split('\n')
# pairs = [('Young','Old'),('Young','Tert'),('Old','Tert')]
# tissue = 'Brain' #_order = ['Brain', 'Gut', 'Heart', 'Liver', 'Muscle', 'Skin','Testis']
# foutpath = 'GeneSets/MS/%s_TL'%(geneset_cateogry)
# if not os.path.exists(foutpath): os.makedirs(foutpath)
# df_input = pd.read_csv(os.path.join('', 'kf_combined_prop.csv'))
# df_input['CMA_Sub']=df_input['N. furzeri Protein Id'].isin(geneset)
# for p_i, pair in enumerate(pairs):
#     age_pair = '%sv%s'%(pair[1][0], pair[0][0])
#     df_plot = df_input[df_input['Tissue']==tissue]
#     fc = '%s_logFC'%age_pair
#     fig, ax = plt.subplots(figsize=(6,8))
#     sns.boxplot(data = df_plot, x = 'Sample.Type', y = fc, hue = 'CMA_Sub', order = ['TL','AGG'], ax = ax,\
#                 fliersize = 0, palette="Set3")
#     sns.swarmplot(data = df_plot, x = 'Sample.Type', y = fc, hue = 'CMA_Sub', order = ['TL','AGG'], ax = ax, \
#                   dodge=True, size = 1.5, palette='Set3')
#     fig.savefig(os.path.join(foutpath, 'CMA_Substrate_%s_%s.pdf'%(tissue, age_pair)))


"""
PyBlock
"""

import os
import sys
from setuptools import setup, find_packages
from distutils.sysconfig import get_python_lib

# - Pull the header into a variable
doclines = __doc__.split("\n")

VERSION = '1.4'

DATADIR = os.sep.join([os.path.dirname(__file__), 'data'])

# - Set variables for setup
PACKAGES = ['pyblock']
package_dir = {'': 'pyblock'}

# - Run setup
setup(
      name='pyblock',
      version=VERSION,
      url='http://github.com/tjlang',
      author='Timothy Lang',
      author_email='timothy.j.lang@nasa.gov',
      description=doclines[0],
      packages=PACKAGES,
      package_data={'pyblock': ['data/*.txt']},
      include_package_data=True,
      classifiers=["""
        Development Status :: Beta,
        Programming Language :: Python",
        Topic :: Scientific/Engineering
        Topic :: Scientific/Engineering :: Atmospheric Science
        Operating System :: Unix
        Operating System :: POSIX :: Linux
        Operating System :: MacOS
        """],
      long_description="""Blockage Correction Tools""",
      install_requires=['numpy'],
      )


import numpy as np
import eikonalfm
import matplotlib.pyplot as plt

def eikonal_path_grad(tau, source, target, org_mat=None):

    path_points = np.array([target], ndmin=2)
    path_indices = np.array([target], ndmin=2)
    gy, gx = np.gradient(tau)
    while not (path_indices[-1, 0], path_indices[-1, 1]) == source:
        i = path_indices[-1, 0]
        j = path_indices[-1, 1]
        x = path_points[-1, 0]
        y = path_points[-1, 1]
        curr_point = np.array([x, y])
        grad_point = np.array([gy[i, j], gx[i, j]]).reshape([1, 2])
        next_point = curr_point - grad_point/(np.linalg.norm(grad_point))
        next_indices = np.round(next_point).astype('int').reshape([1, 2])

        path_indices = np.append(path_indices, next_indices, axis=0)
        path_points = np.append(path_points, next_point, axis=0)

    if not (org_mat is None):
        image_file_path = np.copy(org_mat)
        path_array = np.array(path_indices)
        plt.imshow(image_file_path,  cmap='jet')
        plt.scatter(path_array[:, 1], path_array[:, 0], s=3, c='white')
        plt.title('Negative Gradient - Shortest path from ' + str(source) + ' to ' + str(target))

    return path_indices, path_points

def networkx_path(path, source, target, image):
    dimX = np.shape(image)[1]
    path_array = [(int(node/dimX), node % dimX) for node in path]
    path_array = np.array(path_array)

    plt.imshow(image, cmap='jet')
    plt.scatter(path_array[:, 1], path_array[:, 0], s=3, c='white')
    plt.title('Graph Dijkstra - Shortest path from ' + str(source) + ' to ' + str(target))

## eikonal_path not used - experimental
def eikonal_path(tau, source, target):

    path = [target]
    while not path[-1] == source:
        i = path[-1][0]
        j = path[-1][1]
        T_north = tau[i - 1, j] if i > 0 else float('inf')
        T_south = tau[i + 1, j] if i < dimY - 1 else float('inf')
        T_east = tau[i, j - 1] if j > 0 else float('inf')
        T_west = tau[i, j + 1] if j < dimX - 1 else float('inf')
        tuples = [(i - 1, j), (i + 1, j), (i, j - 1), (i, j + 1)]
        ind_min = np.argmin([T_north, T_south, T_east, T_west])
        next_node = tuples[ind_min]
        path.append(next_node)

    image_file_path = np.copy(image_file_original)
    path_array = np.array(path)
    plt.imshow(image_file_path)
    plt.scatter(path_array[:, 1], path_array[:, 0], s=3, c='red')
    plt.title('Negative Gradient - Shortest path from ' + str(source) + ' to ' + str(target))


from ._component import Component
from base_obj import BaseObj
from physical_obj import PhysObj
from events.verb_events import EVENT_VERB_EAT
from globals import qdel
from ..verbs._verb_names import VERB_EAT

class ComponentEdible(Component):
    id = "component_edible"

    def __init__(self, args_dict = dict[str]) -> None:
        super().__init__()

        total_bites: int = self.arg_set(args_dict, "total_bites", int)

        # How many bites this edible object has left
        self.remaining_bites: int = total_bites
        # The message sent to the user when this is eaten
        self.eat_message: str = self.arg_set(args_dict, "eat_message", str)
        # The message send to the user when they've eaten the final bite
        self.final_eat_message: str = self.arg_set(args_dict, "final_eat_message", str)

    def attach_to_parent(self, object_to_attach: BaseObj) -> bool:
        if not isinstance(object_to_attach, PhysObj):
            return False

        object_to_attach: PhysObj

        if not super().attach_to_parent(object_to_attach):
            return False

        self.register_event(object_to_attach, EVENT_VERB_EAT, self.on_eat)
        object_to_attach.add_verb(VERB_EAT)

    def detach_from_parent(self):
        phys_parent: PhysObj = self.parent

        if phys_parent:
            self.unregister_event(phys_parent, EVENT_VERB_EAT)
            phys_parent.remove_verb(VERB_EAT)

        return super().detach_from_parent()

    def on_eat(self, source):
        """
        ### EVENT FUNCT
        """
        if self.remaining_bites > 1:
            print(self.eat_message)
        else:
            print(self.final_eat_message)

        self.remaining_bites -= 1
        if self.remaining_bites <= 0:
            qdel(self.parent)


# %% IMPORTS
# Package imports
import astropy.units as u
import numpy as np
import pytest

# IMAGINE imports
from imagine.fields import (
    DummyField, MagneticField, ThermalElectronDensityField, UniformGrid)
from imagine.observables import Measurements, TabularDataset
from imagine.simulators import Simulator

# Marks tests in this module as quick
pytestmark = pytest.mark.quick

# %% HELPER DEFINITIONS

# First, we create some fields with the structure:
# A - independent - electron density
# B - independent - electron density
# C -> B - dummy
# D -> (C, B) - magnetic field
# E -> electron density (A,B) - magnetic field
# F - independent - magnetic field

class A(ThermalElectronDensityField):
    """Independent electron density"""

    NAME = 'A'
    PARAMETER_NAMES = []

    def compute_field(self, seed):
        return np.ones(self.data_shape)*self.units

class B(ThermalElectronDensityField):
    """Independent electron density"""

    NAME = 'B'
    PARAMETER_NAMES = []

    def compute_field(self, seed):
        self.secret = 9  # Example of shared information
        return np.ones(self.data_shape)*self.units/2.

class C(DummyField):
    """Dummy field dependent on B"""

    NAME = 'C'
    FIELD_CHECKLIST = {}
    SIMULATOR_CONTROLLIST = {}
    DEPENDENCIES_LIST = [B]

class D(MagneticField):
    """
    Magnetic field, dependent on B and C

    Each component takes the secret number saved during the evaluation of B
    """

    NAME = 'D'
    PARAMETER_NAMES = []
    DEPENDENCIES_LIST = [B, C]

    def compute_field(self, seed):
        result = np.ones(self.data_shape)*self.units

        return self.dependencies[B].secret * result

class E(MagneticField):
    """
    Magnetic field, dependent total thermal electron density

    Each component takes the numerical value of the electron density
    """

    NAME = 'E'
    PARAMETER_NAMES = []
    DEPENDENCIES_LIST = ['thermal_electron_density']

    def compute_field(self, seed):
        te_density = self.dependencies['thermal_electron_density']
        B = np.empty(self.data_shape)
        for i in range(3):
            B[..., i] = te_density.value
        return B*u.microgauss

class F(MagneticField):
    """Independent magnetic field"""

    NAME = 'F'
    DEPENDENCIES_LIST = [B, C]
    PARAMETER_NAMES = []

    def compute_field(self, seed):
        return np.ones(self.data_shape)*0.1*u.microgauss

# We initalize a common grid for all the tests
grid = UniformGrid([[0, 1]]*3*u.kpc, resolution=[1]*3)

class DummySimulator(Simulator):
    # Class attributes
    SIMULATED_QUANTITIES = ['nothing']
    REQUIRED_FIELD_TYPES = ['dummy', 'magnetic_field',
                            'thermal_electron_density']
    ALLOWED_GRID_TYPES = ['cartesian']

    def simulate(self, key, coords_dict, realization_id, output_units):

        results = np.zeros(coords_dict['lat'].size)

        print('The supplied (total) fields are:\n')
        for name in self.fields:
            print(name)
            print('\t', self.fields[name])
            if name != 'dummy':
                results += self.fields[name].value.sum()
        print()

        return results*output_units

# %% PYTEST DEFINITIONS
def test_Field_dependency():
    # Initializes the fields
    a = A(grid)
    b = B(grid)
    c = C(grid)
    d = D(grid)
    e = E(grid)

    # Checks whether dependencies on classes are working
    b.get_data()  # Needs to evaluate these once
    c.get_data(dependencies={B: b})  # Needs to evaluate these once
    result = d.get_data(dependencies={B: b, C: c})
    assert np.all(result == [[[9]*3]]*u.microgauss)

    # Checks whether dependencies on types are working
    te_density = a.get_data()+b.get_data()
    result = e.get_data(dependencies={'thermal_electron_density': te_density})
    assert np.all(result == [[[1.5]*3]]*u.microgauss)

def test_Simulator_dependency_resolution():
    dat = TabularDataset({'data': [0], 'lat': 0, 'lon': 0, 'err': 0.1},
                         name='nothing',
                         units=u.rad,
                         data_col='data',
                         err_col='err')
    mea = Measurements()
    mea.append(dat)

    sim = DummySimulator(mea)

    fields_list = list(map(lambda x: x(grid), (F, E, D, C, B, A)))
    obs = sim(fields_list)

    assert obs[('nothing', None, 'tab', None)].global_data[0][0] == 33.3


from web3 import Web3

w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:8545"))

def create_contract():
    # print(__name__*10)
    contract_interface = {}
    with open('./backend/bin/copyrights_sol_UserRecords.bin',  'r') as f:
        bytecode = f.read()

    with open("./backend/bin/copyrights_sol_UserRecords.abi",  'r') as f:
        abi = f.read()

    w3.eth.defaultAccount = w3.eth.accounts[0]
    Contract = w3.eth.contract(abi=abi, bytecode=bytecode)
    tx_hash = Contract.constructor().transact()
    tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)
    # print(tx_receipt)
    contract = w3.eth.contract(address=tx_receipt.contractAddress,abi=abi)

    return contract

# contract_interface = {}
# with open('./backend/bin/copyrights_sol_UserRecords.bin',  'r') as f:
#         contract_interface['bin'] = f.read()

# with open("./backend/bin/copyrights_sol_UserRecords.abi",  'r') as f:
#     contract_interface['abi'] = f.read()

# contract = create_contract(contract_interface['ab'] , contract_interface['bin'])
# print(list(contract.functions))


'''
x = [-3, 31, -11, 4, 0, 22, -2, -5, -25, -14]
y = [-7, 61, -23, 7, -1, 43, -5, -11, -51, -29]

import pandas as pd
df = pd.DataFrame({'X':x, 'Y':y})
print(df.shape)

train_features = ['X']
target_cols = ['Y']
x_train = df.loc[:, ['X']]
y_train = df.loc[:, ['Y']]
print(x_train, y_train)

from sklearn.linear_model import LinearRegression
lr = LinearRegression()
lr.fit(x_train, y_train)

print("기울기 : ", lr.coef_[0][0])
print("y절편 : ", lr.intercept_[0])

import numpy as np
x_new = np.array([11]).reshape(1, 1)
print(lr.predict(x_new))

x_data = np.arange(11, 16, 1).reshape(-1, 1)
print(lr.predict(x_data))
'''

x = [-3, 31, -11, 4, 0, 22, -2, -5, -25, -14]
y = [-7, 61, -23, 7, -1, 43, -5, -11, -51, -29]

import matplotlib.pyplot as plt

'''
plt.plot(x, y)
plt.show()
'''

import pandas as pd
df = pd.DataFrame({'X':x, 'Y':y})
print(df.shape)

# df.head()
# df.tail()

train_features = ['X']
train_cols = ['Y']
x_train = df.loc[:, train_features]
y_train = df.loc[:, train_cols]

from sklearn.linear_model import LinearRegression
lr = LinearRegression()
lr.fit(x_train, y_train)
print(lr.coef_[0][0])
print(lr.intercept_[0])

import numpy as np
x_new = np.array([11]).reshape(1, 1)
print(lr.predict(x_new))

y1_new = np.arange(11, 16, 1).reshape(-1, 1)
y2_new = np.arange(11, 16, 1).reshape(1, -1)

print(y1_new.shape)
print(y2_new.shape)
print(lr.predict(y1_new))

from sklearn import datasets
iris = datasets.load_iris()
print(iris['DESCR'])
print(iris)


import matplotlib.pyplot as plot
import numpy as np
import math as mt
import random as rd

participants = np.arange(1,31,1)

height = []
weight = []

for x in participants:
    height.append(rd.randint(153, 203))
    weight.append(rd.randint(51, 93))

plot.plot(participants, weight)
plot.plot(participants, height)
plot.grid(True)
plot.title("Wyjkres wzrostu/wagi uczestników")
plot.xlabel("Uczestnicy")
plot.ylabel("Wzrost [cm] / Waga [kg]")
plot.savefig("z6x.png")
# plot.legend(weight, height)
plot.show()


## \cond
#################################################################
#
#  GLthread.py
#     Threaded asyncore implementation
#
#  REVISION HISTORY:
#     29 JAN 2008 jcs  Created
#      6 DEC 2022 jcs  gatea.net.GLthread
#
#  (c) 1994-2022, Gatea Ltd.
#################################################################
## \endcond
import sys, time, string, math, traceback
import asyncore, threading
import gatea.net.GLeventQueue as GLeventQueue
import gatea.sys.GLcommon     as GLcommon

## @namespace pyRT.gatea.net.GLthread
#
# A native asyncore-driven thread capable of dispatching:
# -# read/write and timer events into GLconn
# -# Events from GLeventQueue
#

## \cond
# Bind() function identifiers

EXCEPTION  = 0x31

_bCatch = True

########################
# Global Fcns
########################
def CurrentID():
   """GLthread.CurrentID() returns the thread ID of the calling thread.  This
function has been moved to GLcommon and is kept for backward- compatibility
only.
"""
   return GLcommon.CurrentThreadID()

_thrs = []

def GetStatus():
   """GLthread.GetStatus() polls all GLthread's for state.  Returns a dict
of the form:
      { 'Thread1' : [ tPoll, tExcept ],
        'Thread2' : [ tPoll, tExcept ],
        . . .
      }
"""
   rtn = {}
   for thr in _thrs:
      pn = thr._name
      if not len( pn ):
         pn = 'tid-%d' % thr._tid
      rtn[pn] = thr.GetState()
   return rtn

def GetAffinity( hProc=None ):
   """GLthread.GetAffinity() gets the process affinity
"""
   try:
      import win32process as w32

      if not hProc:
         hProc = w32.GetCurrentProcess()
      rtn = w32.GetProcessAffinityMask( hProc )[0]
   except:
      rtn = 0
   return rtn

def SetAffinity( nCpu ):
   """GLthread.SetAffinity() ties this process to a specific CPU.
"""
   try:
      import win32process as w32

      hProc = w32.GetCurrentProcess()
      mask  = ( 1 << nCpu )
      w32.SetProcessAffinityMask( hProc, mask )
      rtn = GetAffinity( hProc )
   except:
      rtn  = 0
   return rtn

def ClearAffinity():
   """GLthread.ClearAffinity() unties this process from a specific CPU.
"""
   try:
      import win32process as w32

      hProc = w32.GetCurrentProcess()
      w32.SetProcessAffinityMask( hProc, 0xff )
      rtn = GetAffinity( hProc )
   except:
      rtn  = 0
   return rtn

def ShutdownAllThreads( tTmr=None ):
   """GLthread.ShutdownAllThreads() shuts down all GLthread's.
"""
   # Pre-condition

   rtn = ''
   if not len( _thrs ): return rtn

   # Walk 'em all

   d0 = GLcommon.MicroTime()
   tmp = []
   for thr in _thrs: tmp.append( thr )
   for thr in tmp:
      pn = thr._name
      t0 = GLcommon.MicroTime()
      thr.Shutdown( tTmr )
      t1   = GLcommon.MicroTime()
      rtn += '%6dmS to Shutdown( %s )\n' % ( 1000.0 * ( t1-t0 ), pn )
   d1 = GLcommon.MicroTime()
   rtn += '%6.1fmS to ShutdownAllThreads()\n' % ( 1000.0 * ( d1-d0 ) )
   return rtn
## \endcond

## \cond
###############################
#                             #
#      G L t h r e a d        #
#                             #
###############################
## \endcond
## @class GLthread
#
# A native asyncore-driven thread capable of dispatching both
# -# read / write events from registered GLconn-based channels
# -# timer events,
# -# user-defined events from a GLeventQueue.
#
# GLthread operation is as follows:
# -# The GLthread main loop - GLthread.run() - loops around asyncore.poll(),
# waiting in poll() for any registered GLconn to become readable (or
# writable, if GLconn.Send() could not send() all bytes).
#
# -# If a GLconn is readable (or writable), asyncore.poll() will call the
# GLconn.handle_read() or GLconn.handle_write() notification functions.
#
# -# Each iteration through poll(), GLthread determines how much time as
# elapsed in the loop, then updates the internal timer table of registered
# timers via GLthread._tmUpdate().
#
# -# If a registered timer has expired, GLthread._tmExpire() will call the
# notification handler that was specified when the timer was registered
# via AddTimer().
#
# Each GLthread has a single GLeventQueue, that is implemented as a pipe with
# a mutex-protected deque.  Users wishing to process events in this GLthread
# would call self.EvtQ().AddEvent() to add an event to this GLthread's event
# queue.
#
# Each GLthread maintains a _state list, which contains 2 values:
# -# Timestamp when the thread last entered asyncore.poll(),
# -# Timestamp when the thread last caught an exception
#
# The GetState() method exposes the _state list to enquiring minds.
#
# The GLthread.run() method is wrapped in a try / except block, so that any
# exception not caught by the GLthread implemention is caught in this class.
# Users may Bind( GLthread.EXCEPTION ) notification functions to be called if
# an unhandled exception is caught by the GLthread.
#
# GLthread supports 3 types of timers:
# -# Recurring, relative timer (e.g., every 60 seconds)
# -# One-shot absolute timer (e.g., expire at Unix time 1212085320.351)
# -# Recurring, cron-based timer (e.g., 1st Friday of each month at 0830 EDT);
#
# Users may specify which thread to execute the timer handler in when they
# register the timer via AddXxxTimer()
#
# The GLthread.AddXxxTimer() timer registration methods return a unique timer
# ID that you pass to GLthread.DelTimer() when you wish to stop the timer.
# When you register a timer, you specify a handler function to receive the
# timer notifcation.  The function must be of the following format:
#
#   def handler_fcn( self, timer_id, arg ):
#      pass
#
# Note that the unique timer ID is passed to you in your handler function when
# the timer expires.  You may register as many timers as you like and have a
# single handler function process one or more timers.
#
# Member | Description
# --- | ---
# _name      | Thread name
# _bRun      | Non-zero while GLthread is alive; Set to 0 in Shutdown()
# _cv        | threading.Event used to synchronize GLthread startup
# _tmrID     | Unique timer ID
# _tmrs      | Registered _GLtimer's - Relative and absolute
# _tmrMtx    | Mutex protecting _tmrs table
# _crons     | Registered _GLcronTimer's - cron
# _conns     | Registered GLconn's
# _tid       | GLthread.CurrentID()
# _sockMap   | Dictionary of asyncore.dispatcher's required to allow us to run asyncore.poll() in a thread-safe manner
# _evtQ      | The single GLeventQueue for this thread
# _tPoll0    | time.time() we last entered poll()
# _tPoll1    | time.time() we last exitted poll()
# _tExcept   | time.time() we caught last exception
# _tExp      | time.time() we last expired timers
# _tToExp    | 1st argument to asyncore.poll()
# _fcnExcpt  | Bind()'ed function list for EXCEPTION
#
class GLthread( threading.Thread ):
   ###########################
   # Constructor
   #
   # @param wx - wx.EvtHandler() if running wx GUI
   # @param name = Thread name
   ###########################
   def __init__( self, wx=None, name='' ):
      threading.Thread.__init__( self )
      self._name     = name
      self._bRun     = 1
      self._cv       = threading.Event()
      self._tmrID    = 0
      self._tmrs     = {}
      self._tmrMtx   = threading.Lock()
      self._crons    = {}
      self._conns    = []
      self._tid      = -1
      self._sockMap  = {}
      self._evtQ     = GLeventQueue.GLeventQueue( self, wx )
      self._tPoll0   = 0.0
      self._tPoll1   = 0.0
      self._tExcept  = 0.0
      self._tToExp   = 0.0
      self._tExp     = 0.0
      self._fcnExcpt = []
      _thrs.append( self )

   ########################
   # Returns the (single) GLeventQueue driving events into this thread
   #
   # @return The (single) GLeventQueue driving events into this thread
   ########################
   def EvtQ( self ):
      return self._evtQ

   ########################
   # Return True if called from this GLthread call stack
   #
   # @return True if called from this GLthread call stack
   ########################
   def IsOurThread( self ):
      return( self._tid == CurrentID() )

   ########################
   # Returns State as [ Time last entered poll(), Time of last exception ]
   #
   # @return State as [ Time last entered poll(), Time of last exception ]
   ########################
   def GetState( self ):
      return [ self._tPoll0, self._tExcept ]

   ########################
   # Set thread name
   #
   # @param pn - Thread name
   ########################
   def SetName( self, pn ):
      self._name = pn

   ########################
   # Add recurring, relative-based (every x.y seconds) timer
   #
   # @param tmout - Timer interval in seconds
   # @param fcn - Timer handler
   # @param arg - Optional timer handler argument
   # @return Unique timer ID
   # @see DelTimer()
   ########################
   def AddTimer( self, tmout, fcn, arg=None ):
      bAbs = False
      return self._AddAnyTimer( tmout, bAbs, fcn, arg )

   ########################
   # Add on-shot, absolute timer that expires at at the defined time.time().
   #
   # @param tmout - Absolute expiration time.time()
   # @param fcn - Timer handler
   # @param arg - Optional timer handler argument
   # @return Unique timer ID
   # @see DelTimer()
   ########################
   def AddAbsTimer( self, tmout, fcn, arg=None ):
      bAbs = True
      return self._AddAnyTimer( tmout, bAbs, fcn, arg )

   ########################
   # Add recurring, cron-based timer
   #
   # @param cTab - crontab-based expiration
   # @param fcn - Timer handler
   # @param arg - Optional timer handler argument
   # @return Unique timer ID
   # @see DelTimer()
   ########################
   def AddCronTimer( self, cTab, fcn, arg=None ):
      cdb     = self._crons
      id      = self._GetTimerID()
      cdb[id] = _GLcronTimer( self, cTab, id, fcn, arg, None )
      return id

   ########################
   # Delete existing timer by unique ID
   #
   # @param id - Unique timer ID
   # @see AddTimer()
   # @see AddAbsTimer()
   # @see AddCronTimer()
   ########################
   def DelTimer( self, id ):
      mtx = self._tmrMtx
      mtx.acquire()
      tdb = self._tmrs
      cdb = self._crons
      tmr = None
      if id in cdb:   tmr = cdb.pop( id )
      elif id in tdb: tmr = tdb.pop( id )
      mtx.release()
      if tmr:
         del tmr
      return 0

   ########################
   # Register a GLconn with this thread
   #
   # @param c - GLconn to register
   ########################
   def AddChannel( self, c ):
      self._conns.append( c )

   ########################
   # Cleanly shut down this thread
   #
   # Shut down as follows:
   # -# Remove from the master _thrs list
   # -# Set the _bRun flat to False
   # -# join() the thread at the operating system level
   #
   # You may safely call this method more than once for the same GLthread.
   ########################
   def Shutdown( self, tTmr=None ):
      try:
         bAlive = self.isAlive()
         _thrs.remove( self )
         self._evtQ.Stop()
         self._bRun = 0
         if tTmr:
            self.AddTimer( tTmr, None )
         if bAlive:
            self.join()
         self._evtQ.Close()
      except ValueError:
         pass

   ########################
   # Start thread and wait until ready to run
   #
   # @see start()
   # @see Ready()
   ########################
   def Start( self ):
      self.start()
      self.Ready()

   ########################
   # Wait until thread is ready to run
   ########################
   def Ready( self ):
      self._cv.wait()

   ########################
   # Register function to handle EXCEPTION event
   #
   # @param iEvt - GLthread.EXCEPTION
   # @param fcn - Handler for exception event
   ########################
   def Bind( self, iEvt, fcn ):
      if iEvt == EXCEPTION: self._fcnExcpt.append( fcn )

   ## \cond
   ########################
   # Timer Management
   ########################
   def _tmUpdate( self, tExp, tNow ):
      """GLthread._tmUpdate() is called once each asyncore.poll() loop to
decrement each registered timer by the amount of time thru the loop.  We also
determine the minimum time to expire of all available registered timers and
return that value."""
      rtn = 1.0
      tdb = self._GetTimerTable()
      for id in tdb:
         tmr  = tdb[id]
         try:
            tUpd = tmr._Update( tExp, tNow )
            rtn  = min( rtn, tUpd )
         except:
            pass
      return max( 0.0, rtn )

   def _tmExpire( self, tNow ):
      """GLthread._tmExpire() is called when asyncore.poll() shows the (single)
timer registered with the system has expired.  This method walks the list of
all registered timers, calling the timer handler functions of any timer that
has expired.

This method makes a copy of _tmrs and operates on the copy.  In this manner,
calls to AddTimer() and DelTimer() are allowed while we are iterating thru
the copy.
"""
      tdb = self._GetTimerTable()
      for id in tdb:
         tmr = tdb[id]
         try:
            if ( tmr._toExpire <= 0.0 ):
               tmr._Expire( tNow )
         except:
            pass
      pass

   ########################
   # Helpers
   ########################
   def _GetTimerTable( self ):
      """GLthread._GetTimerTable() returns a COPY of the _tmrs table.
"""
      mtx = self._tmrMtx
      mtx.acquire()
      rtn = self._tmrs.copy()
      mtx.release()
      return rtn

   def _GetTimerID( self ):
      """GLthread._GetTimerID() returns a unique ID used when creating a new
timer."""
#      self._tmrID += 1
      return GLcommon.GetIdx() # self._tmrID

   def _AddAnyTimer( self, tmout, bAbs, fcn, arg ):
      """GLthread._AddAnyTimer() is a worker function used by the AddXxxTimer()
methods to cleanly add a new timer into the thread."""
      tmr  = _GLtimer( self, tmout, bAbs, fcn, arg )
      id   = tmr._id
      mtx  = self._tmrMtx
      tdb  = self._tmrs
      mtx.acquire()
      if id in tdb:
         raise LookupError('Duplicate timer key %d' % id)
      tdb[id] = tmr
      mtx.release()
      return id

   ########################
   # Thread
   ########################
   def run( self ):
      """GLthread.run() is the main loop of the thread, called by the
threading.Thread framework."""
      #
      # Connect 'em all
      #
      for c in self._conns:
         c.Connect()

      # Ready to rock and roll

      self._tid = CurrentID()
      self._cv.set()

      # event loop: listen and multiplex until server process killed

      self._tPoll0 = time.time()
      moi          = self._name
      while self._bRun:
         if not _bCatch:
            self._Loop()
         else:
            try:
               self._Loop()
            except:
               self._tExcept = GLcommon.MicroTime()
               lst = []
               ex  = sys.exc_info()
               tb  = ex[2]
               stk = traceback.extract_stack( tb.tb_frame )
               lst.append( ex[0] )
               lst.append( ex[1] )
               for ( fname, nLine, name, line ) in stk[:-1]:
                  lst.append( '%s:%04d %s()' % ( fname, nLine, name ) )
               ( fname, nLine, name, line ) = stk[-1]
               lst.append( '%s:%04d %s()' % ( fname, tb.tb_lineno, name ) )
               for fcn in self._fcnExcpt:
                  fcn( self, lst )
         tl = time.time() - self._tPoll0

   def _Loop( self ):
      """GLthread.run() is one iteration thru the asyncore.poll() loop.
"""
      tNow = time.time()
      tExp = ( tNow - self._tPoll0 )
      tm   = self._tmUpdate( tExp, tNow )
      self._tToExp = tm
      self._tPoll0 = time.time()
      asyncore.poll( tm, self._sockMap )
      self._tPoll1 = time.time()
      if tm <= 0.0:
         self._tExp = self._tPoll1
         self._tmExpire( self._tPoll0 )

###############################
#                             #
#     _ G L t i m e r         #
#                             #
###############################
class _GLtimer( GLeventQueue.GLqueueReceivable ):
   """_GLtimer is an internal helper class that represents either a) a recurring
relative timer (e.g., every 60 seconds), or b) a one-shot absolute timer (e.g.,
expire at 08:30:00 on 1st Friday of the month.

The GLthread maintains a dictionary of _GLtimers, indexed by the _GLtimer._id
which the GLthread guarantees to be unique.  You may optionally specify an
opaque argument via GLthread.AddTimer() which is passed into your handler
function when the timer expires as follows:

   def handler_fcn( self, timer_id, arg ):
      pass
"""
   ########################
   # Constructor / Destructor
   ########################
   def __init__( self, thr, tmout, bAbs, fcn, arg ):
      self._thr      = thr
      self._id       = thr._GetTimerID()
      self._tmout    = tmout
      self._toExpire = tmout
      self._bAbs     = bAbs
      self._fcn      = fcn
      self._arg      = arg
      if bAbs:
         self._toExpire = tmout - time.time()

   def __del__( self ):
      thr = self._thr
      id  = self._id
      thr.DelTimer( self._id )

   ########################
   # Operations
   ########################
   def _Update( self, tExp, tNow ):
      """_GLtimer._Update() is a private helper method that we use to update
the internal _GLtimer._toExpire value.  We support both absolute and relative
timers in this method."""
      if self._bAbs:
         self._toExpire  = self._tmout - tNow
      else:
         self._toExpire -= tExp
      return self._toExpire

   def _Expire( self, tNow ):
      """_GLtimer._Expire() is a private helper method to expire the timer and
reset the internal _toExpire value for the next interval.  We support a user-
defined notification function in _GLtimer._fcn; The handle_timer() function is
deprecated."""

      # 1) Cancel one-shot abolute timer, else update relative expiration time

      thr = self._thr
      id  = self._id
      if self._bAbs:
         thr.DelTimer( id )
      else:
         self._toExpire += self._tmout

      # 2) Notify interested parties, in the thread of choosing ...

      self.OnEventQueue( None, self._fcn )

   ########################
   # GLqueueReceivable
   ########################
   def OnEventQueue( self, q, evt ):
      """_GLtimer.OnEventQueue() is called when the _GLtimer _Expire()'s and
the user has requested the notification in a user-defined thread."""
      if self._fcn:
         self._fcn( self._id, self._arg )

###############################
#                             #
#   _ G L c r o n T i m e r   #
#                             #
###############################
class _GLcronTimer( GLeventQueue.GLqueueReceivable ):
   """_GLcronTimer is an internal helper class that represents a recurring
timer based on crontab configuration:

    <minute> <hour> <mDay> <month> <weekDay>

Each of these patterns may be either an asterisk (all legal values), or a
list of elements separated by commas.  An element is either a number, or
two number separated by a minus sign (meaning an inclusive range).

The standard <weekDay> format is 0 = Sunday, 1 = Monday, etc.  We enhance this
to allow for specific weekDay's of the month by suffixing the <weekDay> with
an optional week.  Thus, 1st Friday of the month is 5:1 (5=Friday; 1=1st Friday).

Our implementation is 'inspired' by cron.py and sets a 60-second timer; Thus,
every minute we check to see if the cron fires.  If so, we call the UserFcn.
"""
   ########################
   # Constructor / Destructor
   ########################
   def __init__( self, thr, cTab, user_id, user_fcn, user_arg ):
      self._thr  = thr
      self._cTab = cTab
      self._uID  = user_id
      self._uFcn = user_fcn
      self._uArg = user_arg
      self._id   = thr.AddTimer( self._TopOfMinute(), self.CronChk )

   ########################
   # Timer Notification
   ########################
   def CronChk( self, id, name ):
      """_GLcronTimer.CronChk() is called every 60 seconds to see if any of our
crontabs should run."""
      thr      = self._thr
      self._id = thr.DelTimer( self._id )
      self._Check()
      self._id = thr.AddTimer( self._TopOfMinute(), self.CronChk )

   ########################
   # Helpers
   ########################
   def _TopOfMinute( self ):
      """_GLcronTimer._Tmout() calculates the time to next top-of-minute."""
      l    = time.localtime()
      tMin = l.tm_min + 1
      tHr  = l.tm_hour
      if tMin > 59:
         tMin = 0
         tHr += 1
      if tHr > 23:
         tHr = 0
      rtn = GLcommon.Time2Mark( tHr, tMin, 0 )
      return ( 1.0 * rtn ) / 1000.0

   def _Match( self, val, expr, bWkDy=False ):
      """_GLcronTimer._Match() determines if our crontab expression matches the
current value.  If it is a <weekDay> timer, the user may optionally specify a
specific week, for example 5:1 is 1st Friday in the month (5=Friday; 1=1st).
"""
      if expr == '*':
         return True
      ex1 = expr.split( ',' )
      for e in ex1:
         nWk = -1
         ex2 = e.split( '-' )
         ex3 = ex2[0].split( ':' )
         e0  = int( ex3[0] )
         e1  = e0
         if len( ex2 ) > 1:
            e1 = int( ex2[1] )
         if bWkDy:
            if len( ex3 ) > 1:
               nWk = int( ex3[1] )
         if ( e0 <= val ) and ( val <= e1 ):
            if nWk == -1:
               return True
            l     = time.localtime()
            mDay  = l[2]
            nWkDy = ( mDay-1 ) / 7
            if nWk == ( nWkDy+1 ):
               return True
      return False

   def _Check( self ):
      """_GLcronTimer._Check() determines if our crontab can run."""

      # 1) <minute> <hour> <mDay> <month> <weekDay>

      l        = time.localtime()
      toks     = string.split( self._cTab )
      bTime    = self._Match( l[4], toks[0] )           # minute
      bTime    = bTime and self._Match( l[3], toks[1] ) # hour
      bTime    = bTime and self._Match( l[2], toks[2] ) # day
      bTime    = bTime and self._Match( l[1], toks[3] ) # month

      # 2) Weekday (in crontab, 0 is Sunday; in localtime(), 0 is Monday)

      wkDy  = int( math.fmod( l[6]+1, 7 ) )
      bWkDy = self._Match( wkDy, toks[4], True )
      if not( bTime and bWkDy ):
         return

      # 3) Notify interested parties, in the thread of choosing ...

      self.OnEventQueue( None, self._uFcn )

   ########################
   # GLqueueReceivable
   ########################
   def OnEventQueue( self, q, evt ):
      """_GLcronTimer.OnEventQueue() is called when the _GLtimer _Expire()'s
and the user has requested the notification in a user-defined thread."""
      if self._uFcn:
         self._uFcn( self._uID, self._uArg )
   ## \endcond


def solution(clothes):
    # 해시 풀이
    answer = 1
    clothes_list = {}
    for c, t in clothes:
        if t not in clothes_list:
            clothes_list[t] = 2
        else:
            clothes_list[t] += 1
    for p in clothes_list.values():
        answer *= p
    return answer - 1

    # pythonic 풀이
    from collections import Counter
    from functools import reduce
    clothes_count = Counter([kind for cloth, kind in clothes])
    answer = reduce(lambda x, y : x * (y+1), clothes_count.values(), 1) - 1
    return answer


from typing import TYPE_CHECKING

from django.core.validators import MinValueValidator
from django.db import models
from django.db.models import QuerySet
from django.utils import timezone

import apps.core.return_calculation as rc
import apps.core.utils as utils
from apps.core.models import Depot as CoreDepot
from apps.users.models import StandardUser

class Depot(CoreDepot):
    user = models.ForeignKey(
        StandardUser,
        editable=False,
        related_name="alternative_depots",
        on_delete=models.CASCADE,
    )

    if TYPE_CHECKING:
        alternatives: QuerySet["Alternative"]

    def save(
        self, force_insert=False, force_update=False, using=None, update_fields=None
    ):
        super().save(
            force_insert=force_insert,
            force_update=force_update,
            using=using,
            update_fields=update_fields,
        )
        if self.is_active:
            self.user.alternative_depots.filter(is_active=True).exclude(
                pk=self.pk
            ).update(is_active=False)

    # getters
    def get_stats(self):
        return {"Value": self.get_value()}

    def get_value(self):
        if not hasattr(self, "value"):
            statement = """
            select sum(value) as value
            from alternative_value v
            left join alternative_alternative a on v.alternative_id = a.id
            where depot_id = {}
            and (date, alternative_id) in (
                select max(date) as date, v.alternative_id
                from alternative_value v
                group by v.alternative_id
            )
            """.format(
                self.pk
            )
            self.value = self.__get_number_from_database(statement)
        return self.value

    def __get_number_from_database(self, statement):
        assert str(self.pk) in statement
        return utils.get_number_from_database(statement)

    def get_value_df(self):
        if not hasattr(self, "value_df"):
            self.value_df = utils.sum_up_value_dfs_from_items(self.alternatives.all())
        return self.value_df

class Alternative(models.Model):
    name = models.CharField(max_length=200)
    depot = models.ForeignKey(
        Depot, on_delete=models.CASCADE, related_name="alternatives"
    )
    # query optimization
    invested_capital = models.FloatField(null=True)
    time_weighted_return = models.FloatField(null=True)
    internal_rate_of_return = models.FloatField(null=True)
    current_return = models.FloatField(null=True)
    profit = models.FloatField(null=True)

    if TYPE_CHECKING:
        values: QuerySet["Value"]
        flows: QuerySet["Flow"]

    class Meta:
        unique_together = ("depot", "name")

    def __str__(self):
        return "{}".format(self.name)

    # getters
    def get_stats(self):
        return {
            "Value": self.get_value(),
            "Invested Capital": self.get_invested_capital(),
            "Current Return": self.get_current_return(),
        }

    def get_value(self):
        value = Value.objects.filter(alternative=self).order_by("date").last()
        if value:
            return float(value.value)
        return None

    def get_profit(self):
        if self.profit is None:
            invested_capital = self.get_invested_capital()
            value = self.get_value()
            if invested_capital and value:
                self.profit = value - invested_capital
            else:
                self.profit = None
            self.save()
        return self.profit

    def get_invested_capital(self):
        if self.invested_capital is None:
            df = rc.get_current_return_df(self.get_flow_df(), self.get_value_df())
            self.invested_capital = rc.get_invested_capital(df)
            self.save()
        return self.invested_capital

    def get_current_return(self):
        if self.current_return is None:
            current_return_df = rc.get_current_return_df(
                self.get_flow_df(), self.get_value_df()
            )
            self.current_return = rc.get_current_return(current_return_df)
            self.save()
        return self.current_return

    def get_value_df(self):
        if not hasattr(self, "value_df"):
            statement = """
                select date(date) as date,
                       value      as value
                from alternative_value v
                where v.alternative_id = {}
                  and v.date in (
                    select max(date)
                    from alternative_value
                    where alternative_id = {}
                    group by date(date)
                )
            """.format(
                self.pk, self.pk
            )
            # get the flow df
            self.value_df = self.get_df_from_database(statement, ["date", "value"])
        return self.value_df

    def get_df_from_database(self, statement, columns):
        assert str(self.pk) in statement
        return utils.get_df_from_database(statement, columns)

    def get_flow_df(self):
        if not hasattr(self, "flow_df"):
            statement = """
                select
                    date(date) as date,
                    sum(flow) as flow
                from alternative_flow f
                join alternative_alternative a on f.alternative_id=a.id
                where a.id = {}
                group by date(date)
            """.format(
                self.pk
            )
            # get the flow df
            self.flow_df = self.get_df_from_database(statement, ["date", "flow"])
        return self.flow_df

    def get_flows_and_values(self):
        flows = list(self.flows.all().values("date", "flow", "pk"))
        values = list(self.values.all().values("date", "value", "pk"))
        flows_and_values = flows + values
        flows_and_values_sorted = sorted(flows_and_values, key=lambda k: k["date"])
        return flows_and_values_sorted

    # setters
    def reset(self):
        self.current_return = None
        self.internal_rate_of_return = None
        self.invested_capital = None
        self.time_weighted_return = None
        self.save()

class Value(models.Model):
    alternative = models.ForeignKey(
        Alternative, related_name="values", on_delete=models.CASCADE
    )
    date = models.DateTimeField()
    value = models.DecimalField(
        decimal_places=2, max_digits=15, validators=[MinValueValidator(0)]
    )

    class Meta:
        unique_together = ("alternative", "date")

    def __str__(self):
        return "{}: {} {}".format(self.alternative, self.get_date(), self.value)

    def save(self, *args, **kwargs):
        self.alternative.reset()
        super().save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        self.alternative.reset()
        super().delete(*args, **kwargs)

    # getters
    def get_date(self):
        return timezone.localtime(self.date).strftime("%d.%m.%y %H:%M")

class Flow(models.Model):
    alternative = models.ForeignKey(
        Alternative, related_name="flows", on_delete=models.CASCADE
    )
    date = models.DateTimeField()
    flow = models.DecimalField(decimal_places=2, max_digits=15)

    class Meta:
        unique_together = ("alternative", "date")

    def __str__(self):
        return "{}: {} {}".format(self.alternative, self.get_date(), self.flow)

    def save(self, *args, **kwargs):
        self.alternative.reset()
        super().save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        self.alternative.reset()
        super().delete(*args, **kwargs)

    # getters
    def get_date(self):
        return timezone.localtime(self.date).strftime("%d.%m.%y %H:%M")


"""first x

Revision ID: ab27394e52ad
Revises:
Create Date: 2020-06-06 21:28:23.609911

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = 'ab27394e52ad'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('user',
                    sa.Column('id', sa.String(), nullable=False),
                    sa.Column('email', sa.String(), nullable=False),
                    sa.Column('hashed_password', sa.String(), nullable=False),
                    sa.Column('is_active', sa.Boolean(), nullable=False),
                    sa.Column('is_superuser', sa.Boolean(), nullable=False),
                    sa.Column('name', sa.String(), nullable=True),
                    sa.PrimaryKeyConstraint('id'),
                    sa.UniqueConstraint('name')
                    )
    op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=True)
    op.create_table('posts',
                    sa.Column('post_id', sa.Integer(), nullable=False),
                    sa.Column('title', sa.String(), nullable=True),
                    sa.Column('text', sa.String(length=350), nullable=True),
                    sa.Column('date', sa.DateTime(timezone=True), server_default=sa.text('now()'),
                              nullable=True),
                    sa.Column('user_id', sa.String(), nullable=True),
                    sa.Column('parent_id', sa.Integer(), nullable=True),
                    sa.ForeignKeyConstraint(['parent_id'], ['posts.post_id'], ),
                    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
                    sa.PrimaryKeyConstraint('post_id')
                    )
    op.create_index(op.f('ix_posts_post_id'), 'posts', ['post_id'], unique=True)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_posts_post_id'), table_name='posts')
    op.drop_table('posts')
    op.drop_index(op.f('ix_user_email'), table_name='user')
    op.drop_table('user')
    # ### end Alembic commands ###