import requests
import re
import random
import configparser
from bs4 import BeautifulSoup
from flask import Flask, request, abort
from imgurpython import ImgurClient

from linebot import (
    LineBotApi, WebhookHandler
)
from linebot.exceptions import (
    InvalidSignatureError
)
from linebot.models import *

app = Flask(__name__)
config = configparser.ConfigParser()
config.read("config.ini")

line_bot_api = LineBotApi(config['line_bot']['Channel_Access_Token'])
handler = WebhookHandler(config['line_bot']['Channel_Secret'])
client_id = config['imgur_api']['Client_ID']
client_secret = config['imgur_api']['Client_Secret']
album_id = config['imgur_api']['Album_ID']
API_Get_Image = config['other_api']['API_Get_Image']

@app.route("/callback", methods=['POST'])
def callback():
    # get X-Line-Signature header value
    signature = request.headers['X-Line-Signature']

    # get request body as text
    body = request.get_data(as_text=True)
    # print("body:",body)
    app.logger.info("Request body: " + body)

    # handle webhook body
    try:
        handler.handle(body, signature)
    except InvalidSignatureError:
        abort(400)

    return 'ok'

def pattern_mega(text):
    patterns = [
        'mega', 'mg', 'mu', 'ＭＥＧＡ', 'ＭＥ', 'ＭＵ',
        'ｍｅ', 'ｍｕ', 'ｍｅｇａ', 'GD', 'MG', 'google',
    ]
    for pattern in patterns:
        if re.search(pattern, text, re.IGNORECASE):
            return True

def eyny_movie():
    target_url = 'http://www.eyny.com/forum-205-1.html'
    print('Start parsing eynyMovie....')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ''
    for titleURL in soup.select('.bm_c tbody .xst'):
        if pattern_mega(titleURL.text):
            title = titleURL.text
            if '11379780-1-3' in titleURL['href']:
                continue
            link = 'http://www.eyny.com/' + titleURL['href']
            data = '{}\n{}\n\n'.format(title, link)
            content += data
    return content

def apple_news():
    target_url = 'https://tw.appledaily.com/new/realtime'
    print('Start parsing appleNews....')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ""
    for index, data in enumerate(soup.select('.rtddt a'), 0):
        if index == 5:
            return content
        link = data['href']
        content += '{}\n\n'.format(link)
    return content

def get_page_number(content):
    start_index = content.find('index')
    end_index = content.find('.html')
    page_number = content[start_index + 5: end_index]
    return int(page_number) + 1

def craw_page(res, push_rate):
    soup_ = BeautifulSoup(res.text, 'html.parser')
    article_seq = []
    for r_ent in soup_.find_all(class_="r-ent"):
        try:
            # 先得到每篇文章的篇url
            link = r_ent.find('a')['href']
            if link:
                # 確定得到url再去抓 標題 以及 推文數
                title = r_ent.find(class_="title").text.strip()
                rate = r_ent.find(class_="nrec").text
                url = 'https://www.ptt.cc' + link
                if rate:
                    rate = 100 if rate.startswith('爆') else rate
                    rate = -1 * int(rate[1]) if rate.startswith('X') else rate
                else:
                    rate = 0
                # 比對推文數
                if int(rate) >= push_rate:
                    article_seq.append({
                        'title': title,
                        'url': url,
                        'rate': rate,
                    })
        except Exception as e:
            # print('crawPage function error:',r_ent.find(class_="title").text.strip())
            print('本文已被刪除', e)
    return article_seq

def crawl_page_gossiping(res):
    soup = BeautifulSoup(res.text, 'html.parser')
    article_gossiping_seq = []
    for r_ent in soup.find_all(class_="r-ent"):
        try:
            # 先得到每篇文章的篇url
            link = r_ent.find('a')['href']

            if link:
                # 確定得到url再去抓 標題 以及 推文數
                title = r_ent.find(class_="title").text.strip()
                url_link = 'https://www.ptt.cc' + link
                article_gossiping_seq.append({
                    'url_link': url_link,
                    'title': title
                })

        except Exception as e:
            # print u'crawPage function error:',r_ent.find(class_="title").text.strip()
            # print('本文已被刪除')
            print('delete', e)
    return article_gossiping_seq

def ptt_gossiping():
    rs = requests.session()
    load = {
        'from': '/bbs/Gossiping/index.html',
        'yes': 'yes'
    }
    res = rs.post('https://www.ptt.cc/ask/over18', verify=False, data=load)
    soup = BeautifulSoup(res.text, 'html.parser')
    all_page_url = soup.select('.btn.wide')[1]['href']
    start_page = get_page_number(all_page_url)
    index_list = []
    article_gossiping = []
    for page in range(start_page, start_page - 2, -1):
        page_url = 'https://www.ptt.cc/bbs/Gossiping/index{}.html'.format(page)
        index_list.append(page_url)

    # 抓取 文章標題 網址 推文數
    while index_list:
        index = index_list.pop(0)
        res = rs.get(index, verify=False)
        # 如網頁忙線中,則先將網頁加入 index_list 並休息1秒後再連接
        if res.status_code != 200:
            index_list.append(index)
            # print u'error_URL:',index
            # time.sleep(1)
        else:
            article_gossiping = crawl_page_gossiping(res)
            # print u'OK_URL:', index
            # time.sleep(0.05)
    content = ''
    for index, article in enumerate(article_gossiping, 0):
        if index == 15:
            return content
        data = '{}\n{}\n\n'.format(article.get('title', None), article.get('url_link', None))
        content += data
    return content

def ptt_beauty():
    rs = requests.session()
    res = rs.get('https://www.ptt.cc/bbs/Beauty/index.html', verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    all_page_url = soup.select('.btn.wide')[1]['href']
    start_page = get_page_number(all_page_url)
    page_term = 2  # crawler count
    push_rate = 10  # 推文
    index_list = []
    article_list = []
    for page in range(start_page, start_page - page_term, -1):
        page_url = 'https://www.ptt.cc/bbs/Beauty/index{}.html'.format(page)
        index_list.append(page_url)

    # 抓取 文章標題 網址 推文數
    while index_list:
        index = index_list.pop(0)
        res = rs.get(index, verify=False)
        # 如網頁忙線中,則先將網頁加入 index_list 並休息1秒後再連接
        if res.status_code != 200:
            index_list.append(index)
            # print u'error_URL:',index
            # time.sleep(1)
        else:
            article_list = craw_page(res, push_rate)
            # print u'OK_URL:', index
            # time.sleep(0.05)
    content = ''
    for article in article_list:
        data = '[{} push] {}\n{}\n\n'.format(article.get('rate', None), article.get('title', None),
                                             article.get('url', None))
        content += data
    return content

def ptt_hot():
    target_url = 'http://disp.cc/b/PttHot'
    print('Start parsing pttHot....')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ""
    for data in soup.select('#list div.row2 div span.listTitle'):
        title = data.text
        link = "http://disp.cc/b/" + data.find('a')['href']
        if data.find('a')['href'] == "796-59l9":
            break
        content += '{}\n{}\n\n'.format(title, link)
    return content

def movie():
    target_url = 'http://www.atmovies.com.tw/movie/next/0/'
    print('Start parsing movie ...')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ""
    for index, data in enumerate(soup.select('ul.filmNextListAll a')):
        if index == 20:
            return content
        title = data.text.replace('\t', '').replace('\r', '')
        link = "http://www.atmovies.com.tw" + data['href']
        content += '{}\n{}\n'.format(title, link)
    return content

def technews():
    target_url = 'https://technews.tw/'
    print('Start parsing movie ...')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ""

    for index, data in enumerate(soup.select('article div h1.entry-title a')):
        if index == 12:
            return content
        title = data.text
        link = data['href']
        content += '{}\n{}\n\n'.format(title, link)
    return content

def panx():
    target_url = 'https://panx.asia/'
    print('Start parsing ptt hot....')
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    content = ""
    for data in soup.select('div.container div.row div.desc_wrap h2 a'):
        title = data.text
        link = data['href']
        content += '{}\n{}\n\n'.format(title, link)
    return content

def oil_price():
    target_url = 'https://gas.goodlife.tw/'
    rs = requests.session()
    res = rs.get(target_url, verify=False)
    soup = BeautifulSoup(res.text, 'html.parser')
    title = soup.select('#main')[0].text.replace('\n', '').split('(')[0]
    gas_price = soup.select('#gas-price')[0].text.replace('\n\n\n', '').replace(' ', '')
    cpc = soup.select('#cpc')[0].text.replace(' ', '')
    content = '{}\n{}{}'.format(title, gas_price, cpc)
    return content

@handler.add(MessageEvent, message=TextMessage)
def handle_message(event):
    print("event.reply_token:", event.reply_token)
    print("event.message.text:", event.message.text)
    if event.message.text.lower() == "eyny":
        content = eyny_movie()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "蘋果即時新聞":
        content = apple_news()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "PTT 表特版 近期大於 10 推的文章":
        content = ptt_beauty()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "來張 imgur 正妹圖片":
        client = ImgurClient(client_id, client_secret)
        images = client.get_album_images(album_id)
        index = random.randint(0, len(images) - 1)
        url = images[index].link
        image_message = ImageSendMessage(
            original_content_url=url,
            preview_image_url=url
        )
        line_bot_api.reply_message(
            event.reply_token, image_message)
        return 0
    if event.message.text == "隨便來張正妹圖片":
        image = requests.get(API_Get_Image)
        url = image.json().get('Url')
        image_message = ImageSendMessage(
            original_content_url=url,
            preview_image_url=url
        )
        line_bot_api.reply_message(
            event.reply_token, image_message)
        return 0
    if event.message.text == "近期熱門廢文":
        content = ptt_hot()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "即時廢文":
        content = ptt_gossiping()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "近期上映電影":
        content = movie()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "觸電網-youtube":
        target_url = 'https://www.youtube.com/user/truemovie1/videos'
        rs = requests.session()
        res = rs.get(target_url, verify=False)
        soup = BeautifulSoup(res.text, 'html.parser')
        seqs = ['https://www.youtube.com{}'.format(data.find('a')['href']) for data in soup.select('.yt-lockup-title')]
        line_bot_api.reply_message(
            event.reply_token, [
                TextSendMessage(text=seqs[random.randint(0, len(seqs) - 1)]),
                TextSendMessage(text=seqs[random.randint(0, len(seqs) - 1)])
            ])
        return 0
    if event.message.text == "科技新報":
        content = technews()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "PanX泛科技":
        content = panx()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0
    if event.message.text == "開始玩":
        buttons_template = TemplateSendMessage(
            alt_text='開始玩 template',
            template=ButtonsTemplate(
                title='選擇服務',
                text='請選擇',
                thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                actions=[
                    MessageTemplateAction(
                        label='新聞',
                        text='新聞'
                    ),
                    MessageTemplateAction(
                        label='電影',
                        text='電影'
                    ),
                    MessageTemplateAction(
                        label='看廢文',
                        text='看廢文'
                    ),
                    MessageTemplateAction(
                        label='正妹',
                        text='正妹'
                    )
                ]
            )
        )
        line_bot_api.reply_message(event.reply_token, buttons_template)
        return 0
    if event.message.text == "新聞":
        buttons_template = TemplateSendMessage(
            alt_text='新聞 template',
            template=ButtonsTemplate(
                title='新聞類型',
                text='請選擇',
                thumbnail_image_url='https://i.imgur.com/vkqbLnz.png',
                actions=[
                    MessageTemplateAction(
                        label='蘋果即時新聞',
                        text='蘋果即時新聞'
                    ),
                    MessageTemplateAction(
                        label='科技新報',
                        text='科技新報'
                    ),
                    MessageTemplateAction(
                        label='PanX泛科技',
                        text='PanX泛科技'
                    )
                ]
            )
        )
        line_bot_api.reply_message(event.reply_token, buttons_template)
        return 0
    if event.message.text == "電影":
        buttons_template = TemplateSendMessage(
            alt_text='電影 template',
            template=ButtonsTemplate(
                title='服務類型',
                text='請選擇',
                thumbnail_image_url='https://i.imgur.com/sbOTJt4.png',
                actions=[
                    MessageTemplateAction(
                        label='近期上映電影',
                        text='近期上映電影'
                    ),
                    MessageTemplateAction(
                        label='eyny',
                        text='eyny'
                    ),
                    MessageTemplateAction(
                        label='觸電網-youtube',
                        text='觸電網-youtube'
                    )
                ]
            )
        )
        line_bot_api.reply_message(event.reply_token, buttons_template)
        return 0
    if event.message.text == "看廢文":
        buttons_template = TemplateSendMessage(
            alt_text='看廢文 template',
            template=ButtonsTemplate(
                title='你媽知道你在看廢文嗎',
                text='請選擇',
                thumbnail_image_url='https://i.imgur.com/ocmxAdS.jpg',
                actions=[
                    MessageTemplateAction(
                        label='近期熱門廢文',
                        text='近期熱門廢文'
                    ),
                    MessageTemplateAction(
                        label='即時廢文',
                        text='即時廢文'
                    )
                ]
            )
        )
        line_bot_api.reply_message(event.reply_token, buttons_template)
        return 0
    if event.message.text == "正妹":
        buttons_template = TemplateSendMessage(
            alt_text='正妹 template',
            template=ButtonsTemplate(
                title='選擇服務',
                text='請選擇',
                thumbnail_image_url='https://i.imgur.com/qKkE2bj.jpg',
                actions=[
                    MessageTemplateAction(
                        label='PTT 表特版 近期大於 10 推的文章',
                        text='PTT 表特版 近期大於 10 推的文章'
                    ),
                    MessageTemplateAction(
                        label='來張 imgur 正妹圖片',
                        text='來張 imgur 正妹圖片'
                    ),
                    MessageTemplateAction(
                        label='隨便來張正妹圖片',
                        text='隨便來張正妹圖片'
                    )
                ]
            )
        )
        line_bot_api.reply_message(event.reply_token, buttons_template)
        return 0
    if event.message.text == "imgur bot":
        carousel_template_message = TemplateSendMessage(
            alt_text='ImageCarousel template',
            template=ImageCarouselTemplate(
                columns=[
                    ImageCarouselColumn(
                        image_url='https://i.imgur.com/g8zAYMq.jpg',
                        action=URIAction(
                            label='加我好友試玩',
                            uri='https://line.me/R/ti/p/%40gmy1077x'
                        ),
                    ),
                ]
            )
        )
        line_bot_api.reply_message(
            event.reply_token,
            carousel_template_message)
        return 0
    if event.message.text == "油價查詢":
        content = oil_price()
        line_bot_api.reply_message(
            event.reply_token,
            TextSendMessage(text=content))
        return 0

    carousel_template_message = TemplateSendMessage(
        alt_text='目錄 template',
        template=CarouselTemplate(
            columns=[
                CarouselColumn(
                    thumbnail_image_url='https://i.imgur.com/kzi5kKy.jpg',
                    title='選擇服務',
                    text='請選擇',
                    actions=[
                        MessageAction(
                            label='開始玩',
                            text='開始玩'
                        ),
                        URIAction(
                            label='影片介紹 阿肥bot',
                            uri='https://youtu.be/1IxtWgWxtlE'
                        ),
                        URIAction(
                            label='如何建立自己的 Line Bot',
                            uri='https://github.com/twtrubiks/line-bot-tutorial'
                        )
                    ]
                ),
                CarouselColumn(
                    thumbnail_image_url='https://i.imgur.com/DrsmtKS.jpg',
                    title='選擇服務',
                    text='請選擇',
                    actions=[
                        MessageAction(
                            label='other bot',
                            text='imgur bot'
                        ),
                        MessageAction(
                            label='油價查詢',
                            text='油價查詢'
                        ),
                        URIAction(
                            label='聯絡作者',
                            uri='https://www.facebook.com/TWTRubiks?ref=bookmarks'
                        )
                    ]
                ),
                CarouselColumn(
                    thumbnail_image_url='https://i.imgur.com/h4UzRit.jpg',
                    title='選擇服務',
                    text='請選擇',
                    actions=[
                        URIAction(
                            label='分享 bot',
                            uri='https://line.me/R/nv/recommendOA/@vbi2716y'
                        ),
                        URIAction(
                            label='PTT正妹網',
                            uri='https://ptt-beauty-infinite-scroll.herokuapp.com/'
                        ),
                        URIAction(
                            label='youtube 程式教學分享頻道',
                            uri='https://www.youtube.com/channel/UCPhn2rCqhu0HdktsFjixahA'
                        )
                    ]
                )
            ]
        )
    )

    line_bot_api.reply_message(event.reply_token, carousel_template_message)

@handler.add(MessageEvent, message=StickerMessage)
def handle_sticker_message(event):
    print("package_id:", event.message.package_id)
    print("sticker_id:", event.message.sticker_id)
    # ref. https://developers.line.me/media/messaging-api/sticker_list.pdf
    sticker_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 21, 100, 101, 102, 103, 104, 105, 106,
                   107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125,
                   126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 401, 402]
    index_id = random.randint(0, len(sticker_ids) - 1)
    sticker_id = str(sticker_ids[index_id])
    print(index_id)
    sticker_message = StickerSendMessage(
        package_id='1',
        sticker_id=sticker_id
    )
    line_bot_api.reply_message(
        event.reply_token,
        sticker_message)

if __name__ == '__main__':
    app.run()


import os
import time
import logging
import random
import numpy as np
import tensorflow as tf
import math
from tensorflow.python.ops import variable_scope as vs
from matplotlib import pyplot as plt
from nets_classification import *

class DistrSystem(object):
	def __init__(self, FLAGS, labels, institutions):
		logging.basicConfig(filename=FLAGS.log, filemode='a', level=logging.INFO)
		self.FLAGS = FLAGS
		self.labels_dict = labels
		self.inst_dict = institutions

		self.images = tf.placeholder(tf.float32, shape=[None, FLAGS.img_height, FLAGS.img_width, FLAGS.img_channels])
		self.labels = tf.placeholder(tf.int32, shape=[None])
		self.loss_weights = tf.placeholder(tf.float32, shape=[None])
		self.lr = tf.placeholder(tf.float32)
		self.is_training = tf.placeholder(tf.uint8)
		self.keep_prob = tf.placeholder(tf.float32)

		if FLAGS.model == 'linear':
			self.setup_linear()
		elif FLAGS.model == 'googlenet':
			self.setup_googlenet()
		elif FLAGS.model == 'resnet':
			self.setup_resnet50()
		self.setup_loss()

		global_step = tf.Variable(0, trainable=False)
		learning_rate = tf.train.exponential_decay(self.lr, global_step,
			round(len(institutions)/self.FLAGS.batch_size/self.FLAGS.num_inst), self.FLAGS.decay)

		self.optimizer = tf.train.AdamOptimizer(learning_rate)
		self.updates = self.optimizer.minimize(self.loss)
		self.saver = tf.train.Saver()

	def setup_linear(self):
		x = tf.reshape(self.images, shape=[-1, self.FLAGS.img_height*self.FLAGS.img_width*self.FLAGS.img_channels])
		W = tf.get_variable('W', shape=[self.FLAGS.img_height*self.FLAGS.img_width*self.FLAGS.img_channels, self.FLAGS.num_classes],
			initializer=tf.contrib.layers.xavier_initializer())
		b = tf.get_variable('b', shape=[self.FLAGS.num_classes])
		self.predictions = tf.matmul(x, W) + b

	def setup_googlenet(self):
		self.predictions = GoogLe_Net(self.images, self.is_training, self.FLAGS.num_classes, self.FLAGS.batch_size, keep_prob=self.keep_prob)

	def setup_resnet50(self):
		self.predictions = Res_Net(self.images, self.is_training, self.FLAGS.num_classes, self.FLAGS.batch_size, keep_prob=self.keep_prob)

	def setup_loss(self):
		self.loss = tf.losses.sparse_softmax_cross_entropy(self.labels, self.predictions, weights=self.loss_weights)

	def accuracy(self, logits, truth):
		predictions = np.argmax(logits, axis=1)
		return np.mean(0.0 + (predictions == truth))

	def graph(self, train_iters, train_vals, val_iters, val_vals, y_label):
		fig = plt.figure()
		plt.xlabel('Epochs')
		plt.ylabel(y_label)
		plt.plot(train_iters / (len(self.inst_dict) / (self.FLAGS.num_inst * self.FLAGS.batch_size)), train_vals)
		plt.plot(val_iters / (len(self.inst_dict) / (self.FLAGS.num_inst * self.FLAGS.batch_size)), val_vals)
		if y_label == 'Loss':
			fig.savefig(self.FLAGS.loss_curve, bbox_inches='tight')
		else:
			fig.savefig(self.FLAGS.acc_curve, bbox_inches='tight')

	def augment(self, data_iter):
		matrix_size = data_iter.shape[0]
		roller = np.round(float(matrix_size/7))
		ox, oy = np.random.randint(-roller, roller+1, 2)
		# do_flip = np.random.randn() > 0
		# num_rot = np.random.choice(4)
		# pow_rand = np.clip(0.05*np.random.randn(), -.2, .2) + 1.0
		add_rand = np.clip(np.random.randn() * 0.1, -.4, .4)
		# Rolling
		data_iter = np.roll(np.roll(data_iter, ox, 0), oy, 1)

		# if do_flip:
		#     data_iter = np.fliplr(data_iter)

		# data_iter = np.rot90(data_iter, num_rot)

		#data_iter = data_iter ** pow_rand

		data_iter += add_rand
		return data_iter

	def optimize(self, session, sample, inst):
		y = [self.labels_dict[i] for i in sample]
		x = np.zeros([len(sample), self.FLAGS.img_height, self.FLAGS.img_width, self.FLAGS.img_channels], dtype=np.float32)
		for i in range(len(sample)):
			img = np.load(os.path.join(self.FLAGS.train, sample[i]))
			if self.FLAGS.augment:
				x[i] = self.augment(img)
			else:
				x[i] = img
		input_feed = {}
		input_feed[self.images] = x
		input_feed[self.labels] = y
		input_feed[self.lr] = self.inst_lr[inst]
		input_feed[self.loss_weights] = [self.loss_weights_dict[i] for i in sample]
		input_feed[self.keep_prob] = 1.-self.FLAGS.dropout
		input_feed[self.is_training] = 1
		output_feed = [self.updates, self.loss, self.predictions]
		outputs = session.run(output_feed, input_feed)
		return outputs[1], self.accuracy(outputs[2], np.asarray(y, dtype=int))

	def test(self, session, test_set, dataset):
		if dataset == 'val' or dataset == 'train':
			logging.info('VALIDATING')
		else:
			logging.info('TESTING')
		y = [self.labels_dict[i] for i in test_set]
		batch_indices = [0]
		index = 0
		while (True):
			index += 1
			if index == len(test_set):
				if batch_indices[-1] != index:
					batch_indices.append(index)
				break
			if index % self.FLAGS.batch_size == 0:
				batch_indices.append(index)
		num_minibatches = len(batch_indices) - 1
		losses = []
		accuracies = []
		for b_end in range(1, num_minibatches + 1):
			start = batch_indices[b_end-1]
			end = batch_indices[b_end]
			test_set_batch = test_set[start:end]
			y_batch = y[start:end]
			x = np.zeros([end-start, self.FLAGS.img_height, self.FLAGS.img_width, self.FLAGS.img_channels], dtype=np.float32)
			for i in range(len(test_set_batch)):
				img = None
				if dataset == 'train':
					img = np.load(os.path.join(self.FLAGS.train, test_set_batch[i]))
				elif dataset == 'val':
					img = np.load(os.path.join(self.FLAGS.val, test_set_batch[i]))
				else:
					img = np.load(os.path.join(self.FLAGS.test, test_set_batch[i]))
				x[i] = img

			input_feed = {}

			input_feed[self.images] = x
			input_feed[self.labels] = y_batch
			input_feed[self.loss_weights] = [1.0 for i in test_set_batch]
			input_feed[self.keep_prob] = 1.
			input_feed[self.is_training] = 0

			output_feed = [self.loss, self.predictions]

			outputs = session.run(output_feed, input_feed)
			losses += [outputs[0]]
			accuracies += [self.accuracy(outputs[1], np.asarray(y_batch, dtype=int))]

		return sum(losses)/len(losses), sum(accuracies)/len(accuracies)

	def get_loss_weights(self, train_sets):
		self.loss_weights_dict = {}
		if self.FLAGS.weighted_loss:
			for i in train_sets:
				inst_set = list(train_sets[i])
				inst_labels = [self.labels_dict[img] for img in inst_set]
				label_num = [0 for l in range(self.FLAGS.num_classes)]
				for l in inst_labels:
					label_num[l] += 1
				label_weights = [len(inst_set)/(label_num[l]*self.FLAGS.num_classes) for l in range(self.FLAGS.num_classes)]
				for l in range(len(inst_set)):
					self.loss_weights_dict[inst_set[l]] = label_weights[self.labels_dict[inst_set[l]]]
		else:
			self.loss_weights_dict = {img: 1.0 for img in self.inst_dict}

	def get_sample_weights(self, train_sets):
		self.sample_weights_dict = {}
		for i in train_sets:
			inst_set = list(train_sets[i])
			if self.FLAGS.weighted_samples:
				inst_labels = [self.labels_dict[img] for img in inst_set]
				label_num = [0 for l in range(self.FLAGS.num_classes)]
				for l in inst_labels:
					label_num[l] += 1
				sample_weights = [1./(label_num[l]*self.FLAGS.num_classes) for l in range(self.FLAGS.num_classes)]
				for l in range(len(inst_set)):
					self.sample_weights_dict[inst_set[l]] = sample_weights[self.labels_dict[inst_set[l]]]
			else:
				for img in inst_set:
					self.sample_weights_dict[img] = 1./len(inst_set)

	def get_num_iters_per_cycle(self, train_sets):
		self.inst_iters_per_cycle = [0 for i in range(self.FLAGS.num_inst)]
		if self.FLAGS.prop_iter:
			for i in train_sets:
				self.inst_iters_per_cycle[i] = round(len(train_sets[i])/self.FLAGS.batch_size*self.FLAGS.num_epochs)
		else:
			num_train_samples = len(self.inst_dict)
			self.inst_iters_per_cycle = [round(num_train_samples*self.FLAGS.num_epochs/(self.FLAGS.batch_size*self.FLAGS.num_inst)) for i in range(len(self.FLAGS.num_inst))]

	def get_lr(self, train_sets):
		self.inst_lr = [self.FLAGS.lr for i in range(self.FLAGS.num_inst)]
		if self.FLAGS.prop_lr:
			num_train_samples = len(self.inst_dict)
			for i in train_sets:
				self.inst_lr[i] = len(train_sets[i]) / num_train_samples * self.FLAGS.lr * self.FLAGS.num_inst

	def get_train_sample(self, train_set):
		train_list = list(train_set)
		weights = [self.sample_weights_dict[x] for x in train_set]
		sample = np.random.choice(train_list, self.FLAGS.batch_size, p=weights)
		return sample

	def train(self, session, train_sets, val_set):
		logging.info("training:")
		self.get_loss_weights(train_sets)
		self.get_sample_weights(train_sets)
		self.get_num_iters_per_cycle(train_sets)
		self.get_lr(train_sets)

		best_val_loss = np.inf
		train_iters = []
		train_losses = []
		train_accs = []
		val_iters = []
		val_losses = []
		val_accs = []

		tot_iters = 0
		for cycle in range(self.FLAGS.max_cycles):
			for inst in range(self.FLAGS.num_inst):
				inst_iters = self.inst_iters_per_cycle[inst]
				for i in range(inst_iters):
					sample = self.get_train_sample(train_sets[inst])
					train_loss, train_acc = self.optimize(session, sample, inst)
					logging.info('Iter: %s, Cycle: %s, Inst: %s, train loss: %.3f, train acc: %.3f' % (tot_iters, cycle, inst, train_loss, train_acc))
					tot_iters += 1
			if self.FLAGS.val is not None:
				if (cycle + 1) % self.FLAGS.val_freq == 0:
					# train_sample = np.random.choice(list(self.inst_dict.keys()), self.FLAGS.val_size)
					train_sample = list(self.inst_dict.keys())
					# val_sample = np.random.choice(list(val_set), self.FLAGS.val_size)
					val_sample = list(val_set)
					logging.info('='*90)
					train_loss, train_acc = self.test(session, train_sample, 'train')
					train_iters.append(tot_iters)
					train_losses.append(train_loss)
					train_accs.append(train_acc)
					val_loss, val_acc = self.test(session, val_sample, 'val')
					val_iters.append(tot_iters)
					val_losses.append(val_loss)
					val_accs.append(val_acc)
					if self.FLAGS.loss_curve is not None:
						self.graph(np.asarray(train_iters, dtype=float), train_losses, np.asarray(val_iters, dtype=float), val_losses, 'Loss')
					if self.FLAGS.acc_curve is not None:
						self.graph(np.asarray(train_iters, dtype=float), train_accs, np.asarray(val_iters, dtype=float), val_accs, 'Accuracy')
					logging.info('Cycle: %s, train loss: %.3f, train acc: %.3f' % (cycle, train_loss, train_acc))
					logging.info('Cycle: %s, val loss: %.3f, val acc: %.3f' % (cycle, val_loss, val_acc))
					if val_loss < best_val_loss:
						logging.info('NEW BEST VALIDATION LOSS: %s, SAVING' % (val_loss))
						best_val_loss = val_loss
						if self.FLAGS.save is not None:
							self.saver.save(session, os.path.join(self.FLAGS.save, 'model.weights'))
					logging.info('='*90)


"""
https://leetcode.com/problems/course-schedule/
"""

from typing import List
from collections import deque, defaultdict

class Solution:
    def canFinish(
            self, numCourses: int, prerequisites: List[List[int]]) -> bool:
        """
        Use BFS to perform topological sort.
        """
        todo, nodesWithPre = defaultdict(list), {}
        for curr, pre in prerequisites:  # Build from input.
            todo[pre].append(curr)
            nodesWithPre[curr] = nodesWithPre.get(curr, 0) + 1

        # Put all nodes which have empty pre nodes into the queue.
        queue = deque([i for i in range(numCourses) if i not in nodesWithPre])
        while queue:
            pre = queue.popleft()
            for curr in todo[pre]:
                nodesWithPre[curr] -= 1
                if not nodesWithPre[curr]:  # The current node has no pre.
                    queue.append(curr)

            todo.pop(pre)

        return not todo


import torch
import numpy as np
from torchvision import datasets, transforms
from base import BaseDataLoader

class MnistDataLoader(BaseDataLoader):
    """
    MNIST data loading demo using BaseDataLoader
    """
    def __init__(self, config):
        super(MnistDataLoader, self).__init__(config)
        self.data_dir = config['data_loader']['data_dir']
        self.data_loader = torch.utils.data.DataLoader(
            datasets.MNIST('../datasets', train=True, download=True,
                           transform=transforms.Compose([
                               transforms.ToTensor(),
                               transforms.Normalize((0.1307,), (0.3081,))
                           ])), batch_size=256, shuffle=False)
        self.x = []
        self.y = []
        for data, target in self.data_loader:
            self.x += [i for i in data.numpy()]
            self.y += [i for i in target.numpy()]
        self.x = np.array(self.x)
        self.y = np.array(self.y)

    def __next__(self):
        batch = super(MnistDataLoader, self).__next__()
        batch = [np.array(sample) for sample in batch]
        return batch

    def _pack_data(self):
        packed = list(zip(self.x, self.y))
        return packed

    def _unpack_data(self, packed):
        unpacked = list(zip(*packed))
        unpacked = [list(item) for item in unpacked]
        return unpacked

    def _update_data(self, unpacked):
        self.x, self.y = unpacked

    def _n_samples(self):
        return len(self.x)


import os
import time
import torch
from tqdm import tqdm

from experiment import Experiment
from data.data_loader import DistributedSampler
from apex import amp
from torch_npu.utils.profiler import Profile

def seed_everything(seed):
    os.environ['PYTHONHASHSEED'] = str(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

class Trainer:
    def __init__(self, experiment: Experiment):
        self.experiment = experiment
        self.init_device()
        if self.experiment.train.data_loader.seed is not None:
            seed_everything(self.experiment.train.data_loader.seed)
        self.structure = experiment.structure
        self.logger = experiment.logger
        self.model_saver = experiment.train.model_saver

        # FIXME: Hack the save model path into logger path
        self.model_saver.dir_path = self.logger.save_dir(
            self.model_saver.dir_path)
        self.current_lr = 0
        self.total = 0

    def init_device(self):
        if torch.npu.is_available():
            self.device = torch.device(f'npu:{self.experiment.device_id}')
        else:
            self.device = torch.device('cpu')

    def init_model(self):
        model = self.structure.builder.build(
            self.device, self.experiment.distributed, self.experiment.local_rank)
        return model

    def update_learning_rate(self, optimizer, epoch, step):
        lr = self.experiment.train.scheduler.learning_rate.get_learning_rate(
            epoch, step)

        for group in optimizer.param_groups:
            group['lr'] = lr
        self.current_lr = lr

    def train(self, profiling, collect_start, collect_end):
        self.logger.report_time('Start')
        self.logger.args(self.experiment)
        model = self.init_model()
        train_data_loader = self.experiment.train.data_loader.train_loader
        if self.experiment.validation:
            validation_loaders = self.experiment.validation.data_loaders
        self.steps = 0
        epoch = 0
        if self.experiment.train.checkpoint:
            self.experiment.train.checkpoint.restore_model(
                model, self.device, self.logger)
            epoch, iter_delta = self.experiment.train.checkpoint.restore_counter()
            self.steps = epoch * self.total + iter_delta

        # Init start epoch and iter
        optimizer = self.experiment.train.scheduler.create_optimizer(
            model.parameters())
        if self.experiment.amp:
            amp.register_float_function(torch, "sigmoid")
            model, optimizer = amp.initialize(model, optimizer, opt_level="O1", loss_scale=1024, combine_grad=True)
        self.logger.report_time('Init')
        batch_time = AverageMeter()
        data_time = AverageMeter()
        losses = AverageMeter()
        model.train()
        end = time.time()
        while True:
            self.logger.info('Training epoch ' + str(epoch))
            self.logger.epoch(epoch)
            self.total = len(train_data_loader)
            profiler = Profile(start_step=int(os.getenv("PROFILE_START_STEP", 10)),
                               profile_type=os.getenv("PROFILE_TYPE"))
            for batch in train_data_loader:
                data_time.update((time.time() - end) * 1000)
                self.update_learning_rate(optimizer, epoch, self.steps)
                self.logger.report_time("Data loading")
                if self.experiment.validation and\
                        self.steps % self.experiment.validation.interval == 0 and\
                        self.steps > self.experiment.validation.exempt:
                    self.validate(validation_loaders, model, epoch, self.steps)
                self.logger.report_time('Validating ')
                if self.logger.verbose:
                    torch.npu.synchronize()
                profiler.start()
                loss_item = self.train_step(model, optimizer, batch,
                                epoch=epoch, step=self.steps)
                profiler.end()
                losses.update(loss_item)
                if self.logger.verbose:
                    torch.npu.synchronize()
                self.logger.report_time('Forwarding ')
                self.model_saver.maybe_save_model(
                    model, epoch, self.steps, self.logger)
                self.steps += 1
                if self.steps == 9:
                    batch_time.reset()
                    data_time.reset()
                if self.total > 0 and self.steps % self.total == 2:
                    batch_time.reset()
                batch_time.update((time.time() - end) * 1000)
                if self.steps % self.experiment.logger.log_interval == 0:
                    fps = self.experiment.train.data_loader.batch_size * 1000 / batch_time.val
                    step_count = (self.steps - 1) % self.total + 1
                    msg = 'Epoch: [{0}][{1}/{2}]\t' \
                        'Time {batch_time.val:.3f}ms ({batch_time.avg:.3f}ms)\t' \
                        'Data {data_time.val:.3f}ms ({data_time.avg:.3f}ms)\t' \
                        'Fps {fps:.3f}\t' \
                        'Loss {loss.val:.5f} ({loss.avg:.5f})\t'.format(
                        epoch, step_count, self.total, batch_time=batch_time,
                        data_time=data_time, fps=fps, loss=losses)
                    self.logger.info(msg)
                end = time.time()
            self.logger.info('npu id: {device_no:1d}' \
                  ' * FPS@all {fps:.3f}'.format(device_no=self.device.index, fps=self.experiment.train.data_loader.batch_size * 1000 / batch_time.avg))

            epoch += 1
            if epoch > self.experiment.train.epochs:
                if self.experiment.local_rank in [-1, 0]:
                    self.model_saver.save_checkpoint(model, 'final')
                    if self.experiment.validation:
                        self.validate(validation_loaders, model, epoch, self.steps)
                    self.logger.info('Training done')
                break
            iter_delta = 0

    def train_step(self, model, optimizer, batch, epoch, step, **kwards):
        # Temporary solution for huge memory usage by topk.
        # This should be deleted in future version.
        if step == 0 and epoch == 0:
            tmp = torch.rand(16, 16, 640, 640, dtype=torch.float32).npu()
            top, _ = torch.topk(tmp.view(-1).half(), 4000000)
            top, _ = torch.topk(tmp.view(-1).half(), 6000000)
        optimizer.zero_grad()
        results = model.forward(batch, training=True)
        if len(results) == 2:
            l, pred = results
            metrics = {}
        elif len(results) == 3:
            l, pred, metrics = results

        if isinstance(l, dict):
            line = []
            loss = torch.tensor(0.).npu()
            for key, l_val in l.items():
                loss += l_val.mean()
                line.append('loss_{0}:{1:.4f}'.format(key, l_val.mean()))
        else:
            loss = l
        if self.experiment.amp:
            with amp.scale_loss(loss, optimizer) as scaled_loss:
                scaled_loss.backward()
        else:
            loss.backward()
        optimizer.step()
        return loss.item()

    def validate(self, validation_loaders, model, epoch, step):
        all_matircs = {}
        model.eval()
        for name, loader in validation_loaders.items():
            if self.experiment.validation.visualize:
                metrics, vis_images = self.validate_step(
                    loader, model, True)
                self.logger.images(
                    os.path.join('vis', name), vis_images, step)
            else:
                metrics, vis_images = self.validate_step(loader, model, False)
            for _key, metric in metrics.items():
                key = name + '/' + _key
                if key in all_matircs:
                    all_matircs[key].update(metric.val, metric.count)
                else:
                    all_matircs[key] = metric

        for key, metric in all_matircs.items():
            self.logger.info('%s : %f (%d)' % (key, metric.avg, metric.count))
        self.logger.metrics(epoch, self.steps, all_matircs)
        model.train()
        return all_matircs

    def validate_step(self, data_loader, model, visualize=False):
        raw_metrics = []
        vis_images = dict()
        for i, batch in tqdm(enumerate(data_loader), total=len(data_loader)):
            pred = model.forward(batch, training=False)
            output = self.structure.representer.represent(batch, pred)
            raw_metric, interested = self.structure.measurer.validate_measure(
                batch, output)
            raw_metrics.append(raw_metric)

            if visualize and self.structure.visualizer:
                vis_image = self.structure.visualizer.visualize(
                    batch, output, interested)
                vis_images.update(vis_image)
        metrics = self.structure.measurer.gather_measure(
            raw_metrics, self.logger)
        return metrics, vis_images

    def to_np(self, x):
        return x.cpu().data.numpy()


#!/usr/bin/env pypy3

import re
import sys
import time
from typing import List, Match, Optional
from copy import copy, deepcopy
from collections import defaultdict
from itertools import repeat
from functools import partial
from advent import sirange, srange, nddict
import pprint
import math
if type(__builtins__) is not dict or 'profile' not in __builtins__: profile=lambda f:f;
sys.setrecursionlimit(10000000)

ADVENTDAY="06"
test = False
debug = False
stdin = False
INFILENAME = f"inputs/{ADVENTDAY}.txt"
for arg in sys.argv:
    if arg == "--test":
        test = True
        INFILENAME = f"inputs/{ADVENTDAY}.test.txt"
    if arg == "--debug":
        debug = True
    if arg == "--stdin":
        stdin = True

NN = 2**16
print(f"NN = {NN}")
# Utilities
def rematch(pattern: str, s: str) -> Optional[Match]:
    return re.fullmatch(pattern, s)

def sirange(start: int, stop: int):
    stop += 1 if stop >= start else -1
    m = 1 if stop >= start else -1
    return range(start, stop, m)

def srange(start: int, stop: int):
    m = 1 if stop >= start else -1
    return range(start, stop, m)

def pp(stuff):
    print('-'*30)
    p = pprint.PrettyPrinter(indent=4, width=40, depth=3, compact=True)
    p.pprint(stuff)
    print('-'*30)

print(f"\n{'=' * 30}\n")

# Input parsing
input_start = time.time()
if stdin:
    lines: List[str] = [l.strip() for l in sys.stdin.readlines()]
else:
    with open(INFILENAME) as f:
        lines: List[str] = [l.strip() for l in f.readlines()]

input_end = time.time()

# Part 1
########################################################################################
print("Part 1:")

def part1() -> int:
    global ADVENTDAY, test, pp
    #G = defaultdict(int)
    G = []
    D = 6
    N = 1
    for f in [int(x) for x in lines[0].split(',')]:
        G.append(f)
    first = len(G)
    prev = len(G)
    S = []
    dd = 80
    if test:
        dd = 80
    else:
        dd = 256
    for d in range(0, dd, 2):
        print(f"day {d}")
        new = []
        for i,v in enumerate(G):
            if v <= 0:
                G[i] = 6 + abs(v)
                new.append(8+abs(v))
            else:
                G[i] -= 2
        G += new
    return len(G)

#print(part1())

D = defaultdict(int)

@profile
def c(f: int, d: int):
    global D
    if (f,d) in D:
        return D[(f,d)]
    if f > d or d == 0:
        D[(f,d)] = 1
        return 1
    if f == 0:
        D[(f,d)] = c(0, d - 7) + c(0, d - 9)
        return D[(f,d)]
    else:
        n = min(f,d)
        s = c(f-n,d-n)
        D[(f-n,d-n)] = s
        return s

@profile
def part2():
    global ADVENTDAY, test, pp, NN, D
    N = dict()
    GG = []
    for f in [int(x) for x in lines[0].split(',')]:
        GG.append(f)
    for f in range(6, 0, -1):
        N[f] = c(f, NN)
    T = 0
    for f in GG:
        T += N[f]
    print(len(D))
    rate = defaultdict(int)
    return T

def part22():
    global NN, D
    GG = defaultdict(int)
    for f in [int(x) for x in lines[0].split(',')]:
        GG[f] += 1
    for i in range(1,7):
        GG[0] += 1

print("Part 2:")
print(math.log(part2(),2))


import requests
from os import listdir

class YaUploader:
    def __init__(self, token: str):
        self.token = token

    def get_headers(self):
        return {
            'Authorization': f'OAuth {self.token}'
        }

    def _get_upload_url(self, file_path):
        upload_url = "https://cloud-api.yandex.net/v1/disk/resources/upload"
        headers = self.get_headers()
        params = {"path": file_path, "overwrite": "true"}
        response = requests.get(upload_url, headers=headers, params=params)
        response.raise_for_status()
        return response.json()

    def upload(self, path_to_dir, file_list):
        """Метод загружает файлы по списку file_list на яндекс диск"""
        for file_path in file_list:
            response_dict = self._get_upload_url(f'{path_to_dir}/{file_path}')
            upload_files_url = response_dict.get("href", "")
            with open(f'{path_to_dir}/{file_path}', 'rb') as file:
                response = requests.put(upload_files_url, files={"file": file})
                response.raise_for_status()
                if response.status_code == 201:
                    print('Status: OK')
        return 'Done'

def token_from_file():
    with open('token.txt') as t_file:
        token_f = t_file.read()
    return token_f

if __name__ == '__main__':
    path_to_dir = 'files'
    path_to_file = listdir(path_to_dir)
    token = token_from_file()
    uploader = YaUploader(token)
    result = uploader.upload(path_to_dir, path_to_file)
    print(result)


import numpy as np
import cv2
from matplotlib import pyplot as plt

## File path
## os.chdir("C:/Users/Sebastian/Documents/GitHub/Computer-Vision-and-Pattern-Recognition_Lab-Assignments/A0101856J_SebastianWong_Lab2/pic/pic")

# read an image in greyscale
img = cv2.imread("pic1.jpg", cv2.CV_LOAD_IMAGE_GRAYSCALE)
# Computes values of histogram and occurrences of input data
# that fall within each bin
hist,bins = np.histogram(img.flatten(),256,[0,256])
# Computing cummulative distribution function
cdf = hist.cumsum()
# Normalizing cummulative distribution function
cdf_normalized = cdf * hist.max()/ cdf.max()
plt.plot(cdf_normalized, color = 'g')
plt.hist(img.flatten(),256,[0,256], color = 'r')
plt.xlim([0,256])
plt.legend(('cdf','histogram'), loc = 'upper left')
plt.show()
cdf_m = np.ma.masked_equal(cdf,0)
cdf_m = (cdf_m - cdf_m.min())*255/(cdf_m.max()-cdf_m.min())
cdf = np.ma.filled(cdf_m,0).astype('uint8')
img1 = cdf[img]
# Computes new array values of histogram
hist,bins = np.histogram(img1.flatten(),256,[0,256])
cdfEqualized = hist.cumsum()
cdfEqualizedNormalised = cdfEqualized * hist.max()/cdfEqualized.max()
plt.plot(cdfEqualizedNormalised, color = 'g')
plt.hist(img1.flatten(),256,[0,256],color = 'r')
plt.xlim([0,256])
plt.legend(('cdf of equalized image', 'histogram after equalization'), loc = 'upper left')
plt.show()
cv2.imwrite("pic1_equalized.jpg",img1)


# 残差网络(ResNet)
# 适用于粒子分辨 train_type=particle

import torch
import torch.nn as nn
from torch.nn import functional as F

class Residual(nn.Module):
    def __init__(self,input_channels,num_channels,use_1x1conv=False,strides=1):
        super().__init__()
        self.conv1 = nn.Conv2d(input_channels, num_channels, kernel_size=3, padding=1, stride=strides)
        self.conv2 = nn.Conv2d(num_channels, num_channels, kernel_size=3, padding=1)
        if use_1x1conv:
            self.conv3 = nn.Conv2d(input_channels, num_channels, kernel_size=1, stride=strides)
        else:
            self.conv3 = None
        self.bn1 = nn.BatchNorm2d(num_channels)
        self.bn2 = nn.BatchNorm2d(num_channels)

    def forward(self,X):
        Y = F.relu(self.bn1(self.conv1(X)))
        Y = self.bn2(self.conv2(Y))
        if self.conv3:
            X = self.conv3(X)
        Y += X
        return F.relu(Y)

class ResNet_old(nn.Module):
    def __init__(self,input_channel,input_size_x,input_size_y,output_size,init_weights=False):
        super().__init__()
        def resnet_block(input_channels,num_channels,num_residuals,first_block=False):
            blk=[]
            for i in range(num_residuals):
                if i == 0 and not first_block:
                    blk.append(Residual(input_channels, num_channels, use_1x1conv=True, strides=2))
                else:
                    blk.append(Residual(num_channels, num_channels))
            return blk

        b1 = nn.Sequential(nn.Conv2d(input_channel, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(kernel_size=3, padding=1))
        b2 = nn.Sequential(*resnet_block(64, 64, 1, first_block=True))
        b3 = nn.Sequential(*resnet_block(64, 128, 1))
        b4 = nn.Sequential(*resnet_block(128, 256, 1))
        b5 = nn.Sequential(*resnet_block(256, 512, 1))

        self.features=nn.Sequential(
            b1,b2,b3,b4,b5,
            nn.AdaptiveAvgPool2d((1,1)),
            nn.Flatten()
        )
        x=torch.randn(1,input_channel,input_size_x,input_size_y)
        linear_input_size=self.features(x).size(-1)
        # print(self.features(x).shape)
        self.classifer=nn.Sequential(
            nn.Linear(linear_input_size,output_size)
        )
        if init_weights:
            self._initialize_weights()

    def forward(self,x):
        x=self.features(x)
        x=self.classifer(x)
        return x

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)


from typing import Any, Dict, Type, TypeVar, Union

import attr

from ..models.codat_assess_data_contracts_financials_metrics_metric_data_set_error_type import (
    CodatAssessDataContractsFinancialsMetricsMetricDataSetErrorType,
)
from ..types import UNSET, Unset

T = TypeVar("T", bound="CodatAssessDataContractsFinancialsMetricsMetricDataSetError")

@attr.s(auto_attribs=True)
class CodatAssessDataContractsFinancialsMetricsMetricDataSetError:
    """
    Attributes:
        type (Union[Unset, CodatAssessDataContractsFinancialsMetricsMetricDataSetErrorType]):
        message (Union[Unset, None, str]):
    """

    type: Union[Unset, CodatAssessDataContractsFinancialsMetricsMetricDataSetErrorType] = UNSET
    message: Union[Unset, None, str] = UNSET

    def to_dict(self) -> Dict[str, Any]:
        type: Union[Unset, str] = UNSET
        if not isinstance(self.type, Unset):
            type = self.type.value

        message = self.message

        field_dict: Dict[str, Any] = {}
        field_dict.update({})
        if type is not UNSET:
            field_dict["type"] = type
        if message is not UNSET:
            field_dict["message"] = message

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        _type = d.pop("type", UNSET)
        type: Union[Unset, CodatAssessDataContractsFinancialsMetricsMetricDataSetErrorType]
        if isinstance(_type, Unset):
            type = UNSET
        else:
            type = CodatAssessDataContractsFinancialsMetricsMetricDataSetErrorType(_type)

        message = d.pop("message", UNSET)

        codat_assess_data_contracts_financials_metrics_metric_data_set_error = cls(
            type=type,
            message=message,
        )

        return codat_assess_data_contracts_financials_metrics_metric_data_set_error


import time
import requests
from HTMLParser import HTMLParser
from collectors.lib.collectorbase import CollectorBase

SPARK_STANDALONE_MODE = 'spark_standalone_mode'
SPARK_YARN_MODE = 'spark_yarn_mode'
SPARK_MESOS_MODE = 'spark_mesos_mode'

REST_API = {
    "YARN_APPS_PATH": "ws/v1/cluster/apps",
    "SPARK_APPS_PATH": "api/v1/applications",
    "SPARK_MASTER_STATE_PATH": "json",
    "SPARK_MASTER_APP_PATH": "app",
    "MESOS_MASTER_APP_PATH": "frameworks"
}

# request :
# <track_url>/api/v1/applications/<is name , must return from 'api/v1/application' >/jobs

# spark Job Metrics
# [
#     {
#         "jobId": 0,
#         "name": "saveAsTextFile at NativeMethodAccessorImpl.java:-2",
#         "submissionTime": "2016-03-31T03:05:43.301GMT",
#         "completionTime": "2016-03-31T03:05:47.080GMT",
#         "stageIds": [
#             0,
#             1
#         ],
#         "status": "RUNNING",
#         "numTasks": 20,
#         "numActiveTasks": 30,
#         "numCompletedTasks": 40,
#         "numSkippedTasks": 50,
#         "numFailedTasks": 60,
#         "numActiveStages": 70,
#         "numCompletedStages": 80,
#         "numSkippedStages": 90,
#         "numFailedStages": 100
#     }
# ]
JOB_METRICS = [
    'numTasks',
    'numActiveTasks',
    'numCompletedTasks',
    'numSkippedTasks',
    'numFailedTasks',
    'numActiveStages',
    'numCompletedStages',
    'numSkippedStages',
    'numFailedStages'
]

# spark stage metric
# [
#     {
#         "status": "COMPLETE",
#         "stageId": 0,
#         "attemptId": 0,
#         "numActiveTasks": 100,
#         "numCompleteTasks": 101,
#         "numFailedTasks": 102,
#         "executorRunTime": 103,
#         "inputBytes": 104,
#         "inputRecords": 105,
#         "outputBytes": 106,
#         "outputRecords": 107,
#         "shuffleReadBytes": 108,
#         "shuffleReadRecords": 109,
#         "shuffleWriteBytes": 110,
#         "shuffleWriteRecords": 111,
#         "memoryBytesSpilled": 112,
#         "diskBytesSpilled": 113,
#         "name": "reduceByKey at <stdin>:1",
#         "details": "",
#         "schedulingPool": "default",
#         "accumulatorUpdates": []
#     }
# ]
STAGE_METRICS = [
    'numActiveTasks',
    'numCompleteTasks',
    'numFailedTasks',
    'executorRunTime',
    'inputBytes',
    'inputRecords',
    'outputBytes',
    'outputRecords',
    'shuffleReadBytes',
    'shuffleReadRecords',
    'shuffleWriteBytes',
    'shuffleWriteRecords',
    'memoryBytesSpilled',
    'diskBytesSpilled'
]

# spark executor metric
# [
#     {
#         "id": "driver",
#         "hostPort": "10.0.2.15:33870",
#         "rddBlocks": 99,
#         "memoryUsed": 98,
#         "diskUsed": 97,
#         "activeTasks": 96,
#         "failedTasks": 95,
#         "completedTasks": 94,
#         "totalTasks": 93,
#         "totalDuration": 92,
#         "totalInputBytes": 91,
#         "totalShuffleRead": 90,
#         "totalShuffleWrite": 89,
#         "maxMemory": 278019440,
#         "executorLogs": {}
#     }
# ]

EXECUTOR_METRICS = [
    'rddBlocks',
    'memoryUsed',
    'diskUsed',
    'activeTasks',
    'failedTasks',
    'completedTasks',
    'totalTasks',
    'totalDuration',
    'totalInputBytes',
    'totalShuffleRead',
    'totalShuffleWrite',
    'maxMemory'
]

# spacke rdd metric
# [
#     {
#         "id": 6,
#         "name": "PythonRDD",
#         "numPartitions": 2,
#         "numCachedPartitions": 2,
#         "storageLevel": "Memory Serialized 1x Replicated",
#         "memoryUsed": 284,
#         "diskUsed": 0
#     }
# ]
RDD_METRCIS = [
    'numPartitions',
    'numCachedPartitions',
    'memoryUsed',
    'diskUsed'
]

class Spark(CollectorBase):
    def __init__(self, config, logger, readq):
        super(Spark, self).__init__(config, logger, readq)

        self.spark_cluster_mode = self.get_config('spark_cluster_mode', SPARK_STANDALONE_MODE)
        self.host = self.get_config('spark_host', "localhost")
        self.port = self.get_config('spark_port', 8080)
        self.spark_url = 'http://%s:%s' % (self.host, self.port)

    def __call__(self):
        try:
            spark_apps = self._get_running_apps()
            self._spark_job_metrics(spark_apps)
            self._spark_stage_metrics(spark_apps)
            self._spark_executor_metrics(spark_apps)
            self._spark_rdd_metrics(spark_apps)
            self._readq.nput("spark.state %s %s" % (int(time.time()), '0'))
        except Exception as e:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception('exception collecting spark metrics %s' % e)

    def _get_running_apps(self):
        self.log_info("Choose Spark Cluster Mode : %s" % self.spark_cluster_mode)
        if self.spark_cluster_mode is None:
            raise Exception('spark_cluster_node must not be none')

        if self.spark_cluster_mode == SPARK_STANDALONE_MODE:
            return self._standalone_init()

        elif self.spark_cluster_mode == SPARK_MESOS_MODE:
            running_apps = self._mesos_init()
            return self._get_spark_app_ids(running_apps)

        elif self.spark_cluster_mode == SPARK_YARN_MODE:
            running_apps = self._yarn_init()
            return self._get_spark_app_ids(running_apps)

        else:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            raise Exception('Invalid setting for spark_cluster_mode. Received %s.' % (self.spark_cluster_mode))

    def _spark_job_metrics(self, running_apps):
        try:
            for app_id, (app_name, tracking_url) in running_apps.iteritems():
                i = 0
                ts = time.time()

                for job in self.request(url_join_4(tracking_url, REST_API["SPARK_APPS_PATH"], app_name, 'jobs')):
                    i = i + 1
                    for metric in JOB_METRICS:
                        self._readq.nput('spark.job.%s %d %d host=%s name=%s id=%s' % (metric, ts, job[metric], self.host, app_name, job['jobId']))

                self._readq.nput('spark.job.count %d %d host=%s' % (ts, i, self.host))
        except Exception as e:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception('exception collecting spark job metrics %s' % e)

    def _spark_stage_metrics(self, running_apps):
        try:
            for app_id, (app_name, tracking_url) in running_apps.iteritems():
                i = 0
                ts = time.time()

                for stage in self.request(url_join_4(tracking_url, REST_API["SPARK_APPS_PATH"], app_name, 'stages')):
                    i = i + 1
                    for metric in STAGE_METRICS:
                        self._readq.nput('spark.stage.%s %d %d host=%s name=%s id=%s' % (metric, ts, stage[metric], self.host, app_name, stage['stageId']))

                self._readq.nput('spark.stage.count %d %d host=%s' % (ts, i, self.host))
        except Exception as e:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception('exception collecting spark stage metrics %s' % e)

    def _spark_executor_metrics(self, running_apps):
        try:
            for app_id, (app_name, tracking_url) in running_apps.iteritems():
                i = 0
                ts = time.time()

                for executor in self.request(url_join_4(tracking_url, REST_API["SPARK_APPS_PATH"], app_name, 'executors')):
                    i = i + 1
                    for metric in EXECUTOR_METRICS:
                        self._readq.nput('spark.executor.%s %d %d host=%s name=%s id=%s' % (metric, ts, executor[metric], self.host, app_name, executor['id']))

                self._readq.nput('spark.executor.count %d %d host=%s' % (ts, i, self.host))
        except Exception as e:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception('exception collecting spark stage metrics %s' % e)

    def _spark_rdd_metrics(self, running_apps):
        try:
            for app_id, (app_name, tracking_url) in running_apps.iteritems():
                i = 0
                ts = time.time()

                for rdd in self.request(url_join_4(tracking_url, REST_API["SPARK_APPS_PATH"], app_name, 'storage/rdd')):
                    i = i + 1
                    for metric in RDD_METRCIS:
                        self._readq.nput('spark.rdd.%s %d %d host=%s name=%s id=%s' % (metric, ts, rdd[metric], self.host, app_name, rdd['id']))

                self._readq.nput('spark.rdd.count %d %d host=%s' % (ts, i, self.host))
        except Exception as e:
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception('exception collecting spark stage metrics %s' % e)

    def _standalone_init(self):
        # Return a dictionary of {app_id: (app_name, tracking_url)} for the running Spark applications
        jdata = self.request(url_join_2(self.spark_url, REST_API['SPARK_MASTER_STATE_PATH']))
        running_apps = {}

        if jdata.get('activeapps'):
            for app in jdata['activeapps']:
                app_id = app.get('id')
                app_name = app.get('name')

                # Parse through the HTML to grab the application driver's link
                app_url = self._get_standalone_app_url(app_id)

                if app_id and app_name and app_url:
                    running_apps[app_id] = (app_name, app_url)

        return running_apps

    def _yarn_init(self):
        running_apps = {}

        metrics_json = self.request(url_join_3(self.spark_url, REST_API['YARN_APPS_PATH'], "states=RUNNING&applicationTypes=SPARK"))
        if metrics_json.get('apps'):
            if metrics_json['apps'].get('app') is not None:

                for app_json in metrics_json['apps']['app']:
                    app_id = app_json.get('id')
                    tracking_url = app_json.get('trackingUrl')
                    app_name = app_json.get('name')

                    if app_id and tracking_url and app_name:
                        running_apps[app_id] = (app_name, tracking_url)

        return running_apps

    def _mesos_init(self):
        running_apps = {}

        metrics_json = self.request(url_join_2(self.spark_url, REST_API['MESOS_MASTER_APP_PATH']))
        if metrics_json.get('frameworks'):
            for app_json in metrics_json.get('frameworks'):
                app_id = app_json.get('id')
                tracking_url = app_json.get('webui_url')
                app_name = app_json.get('name')

                if app_id and tracking_url and app_name:
                    running_apps[app_id] = (app_name, tracking_url)

        return running_apps

    def _get_standalone_app_url(self, app_id):
        '''
        Return the application URL from the app info page on the Spark master.
        Due to a bug, we need to parse the HTML manually because we cannot
        fetch JSON data from HTTP interface.
        '''

        headers = {'Accept-Encoding': 'UTF-8'}
        try:
            url = url_join_3(self.spark_url, REST_API['SPARK_MASTER_APP_PATH'], "appId=" + app_id)
            app_page = requests.get(url , headers)
            # parser html and get app_detail_direct_link
            parser = SparkParser()
            parser.feed(app_page.text)
            parser.close()
            app_detail_ui_links = parser.href

            if app_detail_ui_links and len(app_detail_ui_links) == 1:
                return parser.href
            else :
                self.log_info ("Can not get any director link form spark standalone apps pages. Maybe parser would have problem: %s" % url)

        except Exception :
            self._readq.nput("spark.state %s %s" % (int(time.time()), '1'))
            self.log_exception("Can not access SPARK_MASTER_APP_PATH %s" % url)
            raise HTTPError(app_page)

    def _get_spark_app_ids(self, running_apps):
        '''
        Because appname in yarn doesn't as same as spark. so must be transform the name;
        Traverses the Spark application master in YARN to get a Spark application ID.
        Return a dictionary of {app_id: (app_name, tracking_url)} for Spark applications
        '''
        spark_apps = {}
        for app_id, (app_name, tracking_url) in running_apps.iteritems():
            response = self.request(url_join_2(tracking_url,REST_API["SPARK_APPS_PATH"]))

            for app in response:
                app_id = app.get('id')
                app_name = app.get('name')

                if app_id and app_name:
                    spark_apps[app_id] = (app_name, tracking_url)

        return spark_apps

    def request(self, url):
        headers = {'Content-Type': 'application/json'}
        resp = requests.get(url, headers=headers)
        if resp.status_code != 200:
            self.log_exception("spark collector can not access url : %s" % url)
            raise HTTPError(url)

        return resp.json()

# Get Url and Parse form Spark standalone page
#

#
Application Detail UI

#

class SparkParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.temp = None
        self.href = None

    def handle_starttag(self, tag, attrs):
        if tag == 'a':
            for name, value in attrs:
                if name == 'href':
                    self.temp = value

    def handle_data(self, data):
        if data == 'Application Detail UI':
            self.href = self.temp

class HTTPError(RuntimeError):
    def __init__(self, resp):
        RuntimeError.__init__(self, str(resp))
        self.resp = resp

def url_join_2(prefix, uri):
    return "%s%s" % (prefix, uri)

def url_join_3(prefix, uri, param):
    return "%s/%s?%s" % (prefix, uri, param)

def url_join_4(prefix, uri, appId, service):
    return "%s%s/%s/%s" % (prefix, uri, appId, service)


import pandas as pd
import streamlit as st
import numpy as np

st.title('Q-Week Results')

df = pd.read_csv('https://docs.google.com/spreadsheets/d/1UCrSZrIX45MlKz6EQfEVJ-9dHjLw7xeTpg77lujknEw/export?format=csv&gid=1779447167')

df = df[[col for col in df.columns if 'Unnamed' not in col and col != '1']]
df = df.dropna(axis = 'index', how = 'all')
df = df.dropna(axis = 'columns', how = 'all')

mapping = {
    "What is your quad name?": "Quad",
    "What is your kerb (no @mit.edu)?": "Kerb",
    "What event are you recording participation/submission for?": "Event",
    "If participation, how many people from the quad attended or participated?": "Participation",
    "If participation, how many people are in your quad?": "Quad Total"
}

df = df.rename(columns=mapping)

df['Quad Total'] = df['Quad Total'].fillna('4 ppl')
df['Participation'] = df['Participation'].fillna(3)

df['Quad Total'] = df['Quad Total'].apply(lambda x: x[0])

events = ['All'] + sorted(list(df['Event'].dropna().unique()))
events = st.multiselect('Event', options=events, default=['All'])

if 'All' not in events:
    df = df[df['Event'].isin(events)]

df['Kerb'] = 'bottom text'

quad_groups = df.groupby('Quad')
for quad, group in sorted(quad_groups, key=lambda x: str(x[1]['Quad'])):

    points_df = group.groupby('Event').agg('min')
    event_names = list(points_df.index.values)
    puntos = [int(i)/int(j)*400 for i, j in zip(points_df['Participation'].to_list(), points_df['Quad Total'].to_list())]
    five_hundred_puntos = ['Quad Intro Video', 'Photo Contest', 'Door Tag Contest', 'COVID Friends Design Contest', 'Meme Contest', 'Outfit Photo']
    for event_name in event_names:
        if event_name in five_hundred_puntos:
            puntos[event_names.index(event_name)] *= 5/4
        if event_name == '2/18: Wikipedia Game':
            puntos[event_names.index('2/18: Wikipedia Game')] = 400
        if quad == 'SITE SQU4D':
            try:
                puntos[event_names.index('2/18: Wikipedia Game')] = 600
            except ValueError:
                event_names.append('2/18: Wikipedia Game')
                puntos.append(600)
        if quad == 'Unicorms':
            puntos[event_names.index('2/18: Wikipedia Game')] = 600
        if quad == 'Baby Quad':
            puntos[event_names.index('2/18: Wikipedia Game')] = 600
    display_df = {'Event Name': event_names, 'Points Earned': puntos}

    f'### {quad}'
    f"**Total Points**: {format(sum(puntos),'.2f')}"

    display_df = pd.DataFrame(display_df)
    display_df = display_df.sort_values(by=['Points Earned'], ascending=False).reset_index(drop=True)
    st.table(display_df)


# %%
from __future__ import annotations

from pathlib import Path
from typing import Any

import geopandas as gpd
import topojson as tp
from traffic.data import aixm_airspaces

import numpy as np
import pandas as pd
from shapely.geometry import Polygon, mapping, shape
from shapely.ops import transform, unary_union

# %%

x = aixm_airspaces.data.loc[
    aixm_airspaces.data.designator == "BIRD",
    ["name", "type", "designator", "identifier"],
].iloc[0]

# Faroe issue in BIRD
aixm_airspaces.data.loc[
    aixm_airspaces.data.designator == "BIFAROER",
    ["name", "type", "designator", "identifier"],
] = x.values

# Jan Mayer issue in BIRD
aixm_airspaces.data.loc[
    aixm_airspaces.data.designator == "ENJA",
    ["name", "type", "designator", "identifier"],
] = x.values

aixm_airspaces.data.loc[
    aixm_airspaces.data["type"] == "UIR_P",
    ["name", "type", "designator"],
] = ["CANARIAS UIR", "UIR", "GCCC"]

for name in ["VCCF", "UTSD", "UTTR", "UHPP"]:
    aixm_airspaces.data.loc[
        aixm_airspaces.data.designator == name, "upper"
    ] = np.inf

# %%

def match180x(x: float) -> float:
    return 180 if x == 179.0 else (-180 if x == -179.0 else x)

def match90y(y: float) -> float:
    return 90 if y == 89.0 else (-90 if y == -89.0 else y)

def match180(x: Any, y: Any, z: None = None) -> tuple[Any, ...]:
    return tuple(
        filter(
            None,
            [
                tuple(match180x(x_) for x_ in x),
                tuple(match90y(y_) for y_ in y),
                z,
            ],
        )
    )

gdf = (
    aixm_airspaces.query(
        '(type == "FIR" or type == "UIR" or type == "NO_FIR" or '
        'designator == "BODO") and not (type == "FIR" and designator == "XXXX")'
    )
    .consolidate()
    .assign(
        geometry=lambda gdf: gdf.geometry.apply(
            lambda geom: transform(match180, geom)
        )
    )
    .drop(columns=["identifier"])
    .groupby(["designator", "type", "upper", "lower", "name"], as_index=False)
    .agg(dict(geometry=unary_union))
)
gdf.loc[
    (gdf["type"] != "UIR") & (gdf["type"] != "FIR"),
    ["designator", "type", "name"],
] = (None, "NO_FIR", "")
gdf.loc[gdf["upper"] == np.nan, "upper"] = None
gdf.loc[gdf.designator == "BIRD", "geometry"] = unary_union(
    gdf.query('designator == "BIRD"').geometry
)
gdf = gdf.query("not(designator == 'BIRD' and (upper == 200 or lower == 200))")
# weird hole in LOVV
gdf.loc[gdf.designator == "LOVV", "geometry"] = Polygon(
    gdf.loc[gdf.designator == "LOVV", "geometry"].item().exterior
)
# bug in NZZO around South Pole
debug_nzzo = mapping(gdf.query('designator == "NZZO"').geometry.item())
gdf.loc[gdf.designator == "NZZO", ["geometry"]] = gpd.GeoSeries(
    [
        shape(
            {
                "type": "MultiPolygon",
                "coordinates": (
                    (debug_nzzo["coordinates"][0][0],),
                    (
                        (
                            (-157.0, -30.0),
                            (-131.0, -30.0),
                            (-131.0, -90.0),
                            (-180.0, -90.0),
                            (-180.0, -38.69555555555556),
                            (-180.0, -38.45),
                            (-180.0, -38.270833333333336),
                            (-180.0, -25.0),
                            (-175.67527777777778, -15.545833333333333),
                            (-171.0, -5.0),
                            (-157.0, -5.0),
                            (-157.0, -30.0),
                        ),
                    ),
                ),
            }
        )
    ]
).values

def format_name(line: pd.Series) -> str:
    name: str = line["name"]
    if name == "":
        return ""
    if name == "V W A REGION":
        return "Hanoi FIR"
    name = (
        name.replace("FLIGHT INFORMATION REGION", "FIR")
        .replace("FIR / UIR", "FIR/UIR")
        .replace("FT WORTH", "FORT WORTH")
    )
    if name.startswith("UIR "):
        name = name[4:] + " UIR"
    if name.startswith("FIR "):
        name = name[4:] + " FIR"
    if not name.endswith("FIR") and not name.endswith("UIR"):
        name = name + " " + line["type"]

    return name.title().replace("Fir", "FIR").replace("Uir", "UIR")

gdf = gdf.assign(name=[format_name(line) for _, line in gdf.iterrows()])

# %%

t = tp.Topology(gdf)
Path("worldfirs.json").write_text(t.to_json().replace("Infinity", "null"))


import requests
from bs4 import BeautifulSoup
import studente
import mioTelegram
from ischedule import schedule, run_loop
import sys

URLS = ["https://docs.google.com/spreadsheets/d/e/2PACX-1vSxa-l5fqDBv5hOuNQ5q0kW19YOpPHAMe-kITWnBR567PIBhYkklOzpbjz-td77hY-jBw5_KGyz1fX7/pubhtml?gid=6878166&&range=A1:H&widget=false&chrome=false&headers=false&",
        "https://docs.google.com/spreadsheets/d/e/2PACX-1vSxa-l5fqDBv5hOuNQ5q0kW19YOpPHAMe-kITWnBR567PIBhYkklOzpbjz-td77hY-jBw5_KGyz1fX7/pubhtml?gid=695965261&range=A1:H&widget=false&chrome=false&headers=false&"]

IDS = ["6878166",
       "695965261"]

codici = []

#ogni volta devo svuotarla immagino
def riempiLista(s):
    index = 0
    limit = 30
    studenti=[]
    for riga in s:

        materia = riga.contents[1].string   #mhhhh, può creare problemi?
        if materia!=None and index!=0:  #questo index!= l'ho messo perchè l'albero html mi costringeva, non ricordo il motivo preciso
            materia = str(materia)
            #genere = riga.contents[2].string
            #eta = riga.contents[3].string
            #anno = riga.contents[4].string
            lezioni = str(riga.contents[5].string)

            #go = riga.contents[6].string

            info = riga.contents[7].get_text()
            ####
            numero = riga.contents[8].string.split()[1]
            walink = riga.contents[8].contents[0]['href']
            #print(walink)
            ####
            codici.append(numero)
            studenti.append(studente.student(materia, lezioni, info, numero, walink))

            """ print(type(materia))
            print(type(lezioni))
            print(type(info))
            print(type(numero))
            print(type(walink))
            print("-----------") """

        index +=1
        if index >= limit:
            return studenti
    return studenti #nel caso, non si sa mai che la tabella sia più corta di 30 righe

def leggiVecchiCodici(i):
    with open(f'codici{i}.txt', 'r', encoding="utf-8") as f:
        return f.read()

def scriviCodici(i):
    with open(f'codici{i}.txt', 'w', encoding="utf-8") as f:
        for c in codici:
            f.write(f"{c} ")

def mergeCodici(vecchi, studenti):
    for s in studenti:
        if s.numero not in vecchi:
            mioTelegram.inviaMessaggio(s)

def main():

    for i in range(2):
        r = requests.get(URLS[i])
        s = BeautifulSoup(r.content, 'html.parser')
        #print("dimensione studenti ",sys.getsizeof(soup))
        #print("prima riga")
        #mioTelegram.invio("inizio ricerca")
        #print("prima riga")
        s = s.find("div", {"id":IDS[i]}) #file di informatica, potrebbe cambiare nel tempo?
        s = s.contents[0].contents[0].contents[1]  #questo è il tbody

        vecchiCodici = leggiVecchiCodici(i)
        studenti = riempiLista(s)
        s.decompose()
        #print(studenti[0])
        mergeCodici(vecchiCodici, studenti)
        scriviCodici(i)

if __name__ == "__main__":
    main()
    schedule(main, interval=0.7*60)
    run_loop()


# 0815.py
import cv2
import numpy as np

#1
src = cv2.imread('./data/hand.jpg')
hsv = cv2.cvtColor(src, cv2.COLOR_BGR2HSV)
lowerb = (0, 40, 0)
upperb = (20, 180, 255)
bImage = cv2.inRange(hsv, lowerb, upperb)

mode   = cv2.RETR_EXTERNAL
method = cv2.CHAIN_APPROX_SIMPLE
image, contours, hierarchy = cv2.findContours(bImage, mode, method)

dst = src.copy()
##cv2.drawContours(dst, contours, -1, (255,0,0), 3)
cnt = contours[0]
cv2.drawContours(dst, [cnt], 0, (255,0,0), 2)

#2
dst2 = dst.copy()
rows,cols = dst2.shape[:2]
hull = cv2.convexHull(cnt, returnPoints = False)
hull_points = cnt[hull[:,0]]
cv2.drawContours(dst2, [hull_points], 0, (255,0,255), 6)

#3
T = 5 # 10
defects = cv2.convexityDefects(cnt, hull)
print('defects.shape=',defects.shape)
for i in range(defects.shape[0]):
    s,e,f,d = defects[i,0]
    dist = d/256
    start = tuple(cnt[s][0])
    end   = tuple(cnt[e][0])
    far   = tuple(cnt[f][0])
    if dist > T:
        cv2.line(dst2,start,end,[255,255,0],2)
        cv2.line(dst2,start,far,[0,255,0],1)
        cv2.line(dst2,end,  far,[0,255,0],1)

        cv2.circle(dst2,start,5,[0,255,255],-1)
        cv2.circle(dst2,end,  5,[0,128,255],-1)
        cv2.circle(dst2,far,  5,[0,0,255],-1)
cv2.imshow('dst2',  dst2)

cv2.waitKey()
cv2.destroyAllWindows()


import sys
from PyQt5 import QtWidgets, Qt
import tweepy
import pyperclip
from search_util import initalize, strip_id
from target_list import target_list

class MainLayout(QtWidgets.QWidget):
    def __init__(self, parent=None):
        # setup
        super().__init__(parent=parent)
        layout = QtWidgets.QVBoxLayout()
        self.setLayout(layout)

        # label
        label = QtWidgets.QLabel('Twitter救援ID取得ツール', self)
        layout.addWidget(label)

        self.info = QtWidgets.QLabel('ID: ', self)
        layout.addWidget(self.info)

        # combo
        self.combo = QtWidgets.QComboBox()
        self.combo.addItems(target_list)
        layout.addWidget(self.combo)

        # button
        # ID取得実行ボタン
        button1 = QtWidgets.QPushButton('ID取得', self)
        button1.clicked.connect(self.get_id)
        layout.addWidget(button1)

        # 終了ボタン
        button2 = QtWidgets.QPushButton('Quit', self)
        button2.clicked.connect(Qt.qApp.quit)
        layout.addWidget(button2)

    def get_id(self):
        api = initalize()
        word_string = '参加者募集！ ' + self.combo.currentText()
        word = [word_string]

        for tweet in tweepy.Cursor(api.search,count=100, q=word, tweet_mode='extend', lang='ja').items(2):
            if(tweet.source == 'グランブルー ファンタジー'):
                text = tweet.text.strip().replace('\n',' ')
                rescue_id = strip_id(text)
                # idをクリップボードにコピー
                pyperclip.copy(rescue_id)
                self.info.setText('ID: {0}'.format(rescue_id))
                break
            else:
                self.info.setText('ID: {0}'.format('id取得失敗'))

def main():

    # setup
    app = QtWidgets.QApplication(sys.argv)
    widget = MainLayout()

    # exec
    widget.show()
    app.exec_()

if __name__ == '__main__':
    main()


'''This program create a suitable timeseries data for calculating cross-correlation(xcorr() function in matplotlib)
and plots Window lag cross correlation on heat map'''

def WTLC_plot(file_x,file_y):
    from netCDF4 import Dataset as nc
    from WTLC_Correlation import WTLC_Corr
    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt
    from pylab import savefig

    df2 = nc('/incois_ncmrwfx/incois/hycom/ARVIND_SINGH/HYCOM_TS/BOX/'+file_y,'a')
    df1 = nc('/incois_ncmrwfx/incois/hycom/ARVIND_SINGH/HYCOM_TS/BOX/'+file_x,'a')
    dfx = df1.variables['DSWRF_AVG'][:]
    dfy = df2.variables['MLD_AVG'][:]
    sx=pd.Series(dfx)
    sy=pd.Series(dfy)
    datax=sx.interpolate(method='polynomial', order=2)
    datay=sy.interpolate(method='polynomial', order=2)
    for i in range(len(datax)):
        if(np.isnan(datax[i])==True):
           datax[i]=datax[i+1]
        if(np.isnan(datay[i])==True):
           datay[i]=datay[i+1]
        if(np.isnan(datax[len(datax)-1])==True):
           datax[i]=datax[i-1]
        if(np.isnan(datay[len(datay)-1])==True):
           datay[i]=datay[i-1]
    df2.close()
    df1.close()
    fig = plt.figure()
    Heat_map = WTLC_Corr(datax,datay)

    plt.xcorr(datax,datay,maxlags=10)
    dir='/incois_ncmrwfx/incois/hycom/ARVIND_SINGH/HYCOM_TS/IMG/'
    file_s=file_x.split('/')
    file_name=file_s[len(file_s)-1].split('_')[1]
    #X_corr = Corr.get_figure()
    Heat_map.savefig(dir+file_s[0]+'/'+file_name+'_WTLXCorr.png')


import json
import pandas as pd
import os
import sys
from urllib.parse import urlparse

def save2csv(con_list, fname):
    pf = pd.DataFrame(con_list,
                      columns=["ID", "URL", "Port", "IPs", "Title", "Status", "Banner", "Host", "Type", "Other"])
    pf.to_csv(fname, index=False, encoding="utf-8")
    print("Save file at  %s" % (con_list))

def parsejson(fpath):
    csv_data = []
    with open(fpath, "r", encoding="utf-8") as f:
        jsonfile = f.read()
        data = json.loads(jsonfile)
        pages = data["pages"]
    num = 0
    for i in pages:
        num += 1
        csv_line = {}
        url = pages[i]["url"]
        _url = urlparse(url)
        port = _url.port
        if port is None:
            port = 80 if _url.scheme == "http" else 443
        host = pages[i]["hostname"]
        if pages[i]["addrs"] is None:
            ips = 'None'
        else:
            ips = ",".join(pages[i]["addrs"])
        status = pages[i]["status"]
        status = status.strip()[:3]
        title = pages[i]["pageTitle"]
        tags = pages[i]["tags"]
        tag = []
        if not tags is None:
            for item in tags:
                tag.append(item.get("text"))
        banner = ",".join(tag)
        csv_line["ID"] = num
        csv_line["URL"] = url
        csv_line["Port"] = port
        csv_line["Host"] = host
        csv_line["IPs"] = ips
        csv_line["Status"] = status
        csv_line["Title"] = title
        csv_line["Banner"] = banner
        csv_line["Other"] = ""
        # print(url,port,host,ips,status,title,banner)
        csv_data.append(csv_line)
    return csv_data

if __name__ == "__main__":
    fpath = sys.argv[1]
    if not os.path.isabs(fpath):
        fpath = os.path.abspath(fpath)
    fdir, name = os.path.split(fpath)
    name, ext = os.path.splitext(name)
    con_list = parsejson(fpath)
    outpath = os.path.join(fdir, name + ".csv")
    save2csv(con_list, outpath)


"""Prometheus middleware."""

import time
from typing import Tuple

from opentelemetry import trace
from prometheus_client import Counter, Gauge, Histogram
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
from starlette.routing import Match
from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
from starlette.types import ASGIApp

from app.internal.pkg.middlewares.handle_http_exceptions import (
    handle_api_exceptions,
    handle_internal_exception,
)
from app.pkg.models.base import BaseAPIException
from app.pkg.models.types.fastapi import FastAPITypes

__all__ = ["PrometheusMiddleware"]

class PrometheusMiddleware(BaseHTTPMiddleware):
    app: FastAPITypes.FastAPIInstance

    __FILTER_UNHANDLED_PATHS: bool
    __app_name: str

    __INFO = Gauge("fastapi_app_info", "FastAPI application information.", ["app_name"])
    __REQUESTS = Counter(
        "fastapi_requests_total",
        "Total count of requests by method and path.",
        ["method", "path", "app_name"],
    )
    __RESPONSES = Counter(
        "fastapi_responses_total",
        "Total count of responses by method, path and status codes.",
        ["method", "path", "status_code", "app_name"],
    )
    __REQUESTS_PROCESSING_TIME = Histogram(
        "fastapi_requests_duration_seconds",
        "Histogram of requests processing time by path (in seconds)",
        ["method", "path", "app_name"],
    )
    __EXCEPTIONS = Counter(
        "fastapi_exceptions_total",
        "Total count of exceptions raised by path and exception type",
        ["method", "path", "exception_type", "app_name"],
    )
    __REQUESTS_IN_PROGRESS = Gauge(
        "fastapi_requests_in_progress",
        "Gauge of requests by method and path currently being processed",
        ["method", "path", "app_name"],
    )

    def __init__(
        self,
        app: ASGIApp,
        app_name: str = "api",
        filter_unhandled_paths: bool = True,
    ) -> None:
        super().__init__(app)
        self.__app_name = app_name
        self.__INFO.labels(app_name=self.__app_name).inc()
        self.__FILTER_UNHANDLED_PATHS = filter_unhandled_paths

    async def dispatch(
        self,
        request: Request,
        call_next: RequestResponseEndpoint,
    ) -> Response:
        method = request.method
        path_template, is_handled_path = self.__get_path_template(request)

        if self.__is_path_filtered(is_handled_path):
            return await call_next(request)

        self.__REQUESTS_IN_PROGRESS.labels(
            method=method,
            path=path_template,
            app_name=self.__app_name,
        ).inc()
        self.__REQUESTS.labels(
            method=method,
            path=path_template,
            app_name=self.__app_name,
        ).inc()
        before_time = time.perf_counter()
        status_code = HTTP_500_INTERNAL_SERVER_ERROR
        try:
            response = await call_next(request)
        except (BaseAPIException, Exception) as e:
            self.__EXCEPTIONS.labels(
                method=method,
                path=path_template,
                exception_type=type(e).__name__,
                app_name=self.__app_name,
            ).inc()
            if isinstance(e, BaseAPIException):
                return handle_api_exceptions(request=request, exc=e)
            return handle_internal_exception(request=request, exc=e)
        else:
            status_code = response.status_code
            after_time = time.perf_counter()
            span = trace.get_current_span()
            trace_id = trace.format_trace_id(span.get_span_context().trace_id)

            self.__REQUESTS_PROCESSING_TIME.labels(
                method=method,
                path=path_template,
                app_name=self.__app_name,
            ).observe(after_time - before_time, exemplar={"TraceID": trace_id})
        finally:
            self.__RESPONSES.labels(
                method=method,
                path=path_template,
                status_code=status_code,
                app_name=self.__app_name,
            ).inc()
            self.__REQUESTS_IN_PROGRESS.labels(
                method=method,
                path=path_template,
                app_name=self.__app_name,
            ).dec()

        return response

    @staticmethod
    def __get_path_template(request: Request) -> Tuple[str, bool]:
        for route in request.app.routes:
            match, child_scope = route.matches(request.scope)
            if match == Match.FULL:
                return route.path, True

        return request.url.path, False

    def __is_path_filtered(self, is_handled_path: bool) -> bool:
        return self.__FILTER_UNHANDLED_PATHS and not is_handled_path


#-*-coding:utf-8-*-
import pytest
from apis.calc import Calc

class TestCalc:
    def setup(self):
        self.calc = Calc()

    @pytest.mark.parametrize("a,b,expected", [
        (5, 5, 10),
        (-1, 0, -1),
        (0.5, 0.1, 0.6),
        (123456, 9455, 222222),
        ("a", 1, 'can only concatenate str (not "int") to str'),
        ([1], 1, 'can only concatenate list (not "int") to list'),
        (None, 1, "unsupported operand type(s) for +: 'NoneType' and 'int'")
    ])
    def test_add(self, a, b, expected):
        # a,b 为整数或浮点数
        if isinstance(a, (int, float)) and isinstance(b, (int, float)):
            result = self.calc.add(a, b)
            assert result == expected
        # a,b 为其他数据类型时
        else:
            with pytest.raises(TypeError):
                self.calc.add(a, b)

    @pytest.mark.parametrize("a,b,expected", [
        (10, 5, 2),
        (7, 3, 7/3),
        (-1, 1, -1),
        (2, 0.1, 20.0),
        (0, 1, 0),
        (0, 0, "division by zero")
    ])

    def test_div(self, a, b, expected):
        # 除数为0
        if b == 0:
            with pytest.raises(ZeroDivisionError):
                self.calc.div(a, b)
        # a,b 为整数或浮点数
        elif isinstance(a, (int, float)) and isinstance(b, (int, float)):
            result = self.calc.div(a, b)
            assert result == expected
        # a,b 为其他数据类型
        else:
            with pytest.raises(TypeError):
                self.calc.div(a, b)

if __name__ == '__main__':
    pytest.main(['-vs','test_calc.py'])


"""
A simple Python script to send messages to a sever over Bluetooth
using PyBluez (with Python 2).
"""

import bluetooth
import random
import time
import smbus

bus = smbus.SMBus(1)

ADC_ADDRESS = 0x4b

serverMACAddress = 'D8:3A:DD:3C:D9:91'
port = 4
s = bluetooth.BluetoothSocket(bluetooth.RFCOMM)
s.connect((serverMACAddress, port))

while True:
    data = bus.read_i2c_block_data(ADC_ADDRESS, 0)
    #data = [random.randrange(0,255)]
    print(data[0])
    s.send(str(data[0]))

    time.sleep(0.25)
#sock.close()


"""
This file defines the LIF class along with associated functionality.
"""

from typing import List, NamedTuple

import numpy as np

class LIFParams(NamedTuple):
    dt: float = 0.05 # msec
    c_m: float = 58.72 # pF
    g_l: float = 9.43 # 1/GOhm (nS)
    v_l: float = -78.85 # mV
    v_th: float = -51.68 # mV
    k_j: np.ndarray = np.array([0.003, 0.1]) # msec^-1
    f_j: np.ndarray = np.array([-9.18, -198.94]) # pA
    b_j: np.ndarray = np.array([1.0, 1.0]) # no unit

class LIF:
    """Initializes LIF neuron with parameters provided."""
    def __init__(self, params: LIFParams):
        self.dt = params.dt
        self.c_m = params.c_m
        self.g_l = params.g_l
        self.v_l = params.v_l
        self.v_th = params.v_th
        self.k_j = params.k_j
        self.b_j = params.b_j
        self.f_j = params.f_j

        # States to track
        self.v = self.v_l
        self.i_j = np.zeros(2)

    def forward(self, i_ext):
        """
        Computes response to external/applied current using following equations.

        If V > V_th:
        V(t) = V_l # Reset voltage
        I_j(t) = I_j(t-1) * f_j + b_j

        Else:
        V(t) = V(t) + (dt / C_m) * (I_ext(t) + sum(I_j(t-1)) - g_l * (V(t-1) - V_l)
        I_j(t) = I_j(t-1) * (1 - k_j * dt)

        Arguments:
        - i_ext: list
          list of externally applied currents (pA)
        - dt: float
          timestep duration to use in forward euler
        Returns:
        - s: list
          list of 1s and 0s pertaining to whether a neuron spiked
          in each timestep corresponding to the applied current
        """
        s = []
        for i_step in i_ext:
            if self.v > self.v_th:  # Spiking case
                self.v = self.v_l
                self.i_j = self.i_j * self.f_j + self.b_j
                s.append(1)
            else:  # Base case
                dv = (1 / self.c_m) * (i_step + np.sum(self.i_j) - self.g_l * (self.v - self.v_l))
                self.v += dv * self.dt
                self.i_j = self.i_j - self.k_j * self.i_j * self.dt
                s.append(0)
        return s


"""
This module provides classes for working with expectation values.

The relevant classes are:
    * :class:`ExpectationValueTrajectory`
    * :class:`ExpectationValueCollection`

The :class:`ExpectationValueTrajectory` stores **one** expectation value at
different points of time while the :class:`ExpectationValueCollection` stores
**several** expectation values at different points of time.
"""

import numpy
import utils

class ExpectationValueTrajectory(numpy.ndarray):
    r"""
    A class representing one expectation value at different points of time.

    *Usage*
        >>> T = numpy.linspace(0,10)
        >>> X = numpy.sin(T)
        >>> ev = ExpectationValueTrajectory(X, T, "<x>")

    *Arguments*
        * *data*
            A 1d array or similar data structure representing an expectation
            value at different points of time.

        * *time* (optional)
            A 1d array or list specifying the points of time.

        * *title* (optional)
            The name of the expectation value. (Can be any unicode string)

        * Any other argument that a numpy array can use for creation. E.g.
          ``copy = False`` can be used so that the ExpectationValueTrajecory
          shares the data storage with the given numpy array.
    """
    def __new__(cls, data, time=None, title=None, **kwargs):
        array = numpy.array(data, **kwargs).view(cls)
        if time is not None:
            array.time = time
        else:
            array.time = getattr(data, "time", None)
        if title is not None:
            array.title = title
        else:
            array.title = getattr(data, "title", None)
        return array

    def __array_finalize__(self, obj):
        self.time = getattr(obj, "time", None)
        self.title = getattr(obj, "title", None)

    def __str__(self):
        clsname = self.__class__.__name__
        if self.title is None:
            title="?"
        else:
            title = self.title
        return "%s('%s')" % (clsname, title)

class ExpectationValueCollection(numpy.ndarray):
    r"""
    A class representing several expectation values at different points of time.

    *Usage*
        >>> import numpy as np
        >>> T = np.linspace(0,10)
        >>> X = np.sin(T)
        >>> Y = np.cos(T)
        >>> ev = ExpectationValueTrajectoryCollection((X,Y), T, ("<x>", "<y>"))

    *Arguments*
        * *data*
            A list of 1D arrays or some similar data structure.

        * *time* (optional)
            A 1d array or list specifying the points of time.

        * *titles* (optional)
            A list of names for the expectation values. (The names can be any
            unicode string)

        * *subsystems* (optional)
            Dictionary specifying subsystems. E.g. ``{"Mode1" : (1,3)}``.

        * Any other argument that a numpy array can use for creation. E.g.
          ``copy = False`` can be used so that the
          ExpectationValueCollection shares the data storage with the given
          numpy array.
    """
    def __new__(cls, data, time=None, titles=None, subsystems=None, **kwargs):
        if isinstance(data, ExpectationValueTrajectory):
            array = numpy.asarray(data).reshape((1,-1)).view(cls)
            array.evtrajectories = (data,)
        else:
            array = numpy.array(data, **kwargs).view(cls)
            if titles is None:
                titles = []
            else:
                titles = list(titles)
            titles = titles + [None]*(len(data) - len(titles))
            traj = [None]*len(data)
            for i, col in enumerate(data):
                traj[i] = ExpectationValueTrajectory(col, time, titles[i])
            array.evtrajectories = tuple(traj)
        if time is not None:
            array.time = time
        else:
            array.time = getattr(data, "time", None)
        array.subsystems = utils.OrderedDict()
        if subsystems is not None:
            for key, value in subsystems.iteritems():
                array.subsystems[key] = cls(
                        array[value[0]:value[1]:], array.time,
                        array.titles[value[0]:value[1]:], copy=True)
        return array

    def __array_finalize__(self, obj):
        self.time = getattr(obj, "time", None)

    def titles(self):
        """
        Return a list of titles of the holded ExpectationValueTrajectories.

        Unknown titles are represented by "?".
        """
        titles = []
        for evt in self.evtrajectories:
            if evt.title is None:
                titles.append("?")
            else:
                titles.append(evt.title)
        return tuple(titles)

    titles = property(titles)

    def __str__(self):
        clsname = self.__class__.__name__
        return "%s('%s')" % (clsname, "', '".join(self.titles))