from django import forms
from . import models
from .models import Category

class CreateArticle(forms.ModelForm):

    class Meta:
        model = models.Article
        fields = ['title', 'body', 'thumb', 'category', 'slug']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['category'].empty_label = 'No Category'

class CommentForm(forms.ModelForm):
    class Meta:
        model = models.Comment
        fields = ['text']
        widgets = {
            'text': forms.Textarea(attrs={'rows': 4, 'cols': 15, 'placeholder': 'Write your comment here'}),
        }


from flask import Flask, request, jsonify
from flask_sock import Sock
import pyaudio

app = Flask(__name__)
sock = Sock(app)

# test websocket endpoint
@sock.route('/api/test', methods=['GET'])
def test(ws):
    while True:
        ws.send("hello world")

####################################################################

# detection endpoints
@app.route('/api/detection', methods=['GET'])
def detactions_options():
    return '***'

@sock.route('/api/detection/audio', methods=['GET'])
def get_audio(ws):
    audio_format = pyaudio.paInt16

    channels = 1
    sample_rate = 44100
    chunk_size = 512

    audio = pyaudio.PyAudio()

    stream = audio.open(format=audio_format, channels=channels, rate=sample_rate, input=True, frames_per_buffer=chunk_size, input_device_index=0)

    try:
        while True:
            data = stream.read(1024)
            print(data)
            print('_________________________________________')
            ws.send(data)

    except KeyboardInterrupt:
            pass
    stream.stop_stream()
    stream.close()
    audio.terminate()

@app.route('/api/detection/detectShot', methods=['POST'])
def detect_shot():
    if 'file' not in request.files:
        return 'No file part in the request'

    file = request.files['file']

    if file.filename == '':
        return 'No selected file'

    # send file to shot detector

    # get return and return

    resp = {
        'shot':True
    }

    return jsonify(resp)

@app.route('/api/detection/getLocation', methods=['POST'])
def get_location():
    if 'file' not in request.files:
        return 'No file part in the request'

    file = request.files['file']

    if file.filename == '':
        return 'No selected file'

    # noice redection

    # send file to audio madel

    # return value

    return 'File uploaded successfully'

# drone endpoints
@app.route('/api/drone', methods=['GET'])
def drone_options():
    return 'drone_options'

@app.route('/api/drone/chooseDrone', methods=['POST'])
def choose_drone():
    return 'choose_drone'

@app.route('/api/drone/sendDrone', methods=['POST'])
def send_drone():
    return 'send_drone'

# TOTO CV endpoints
@sock.route('/api/toto', methods=['GET'])
def toto_options(ws):
    while True:
        data = ws.receive()
        ws.send(data[::-1])
    return 'drone_options'

@app.route('/api/toto/getObjects', methods=['POST'])
def get_objects():
    return 'get_objects'

# surge endpoints
@app.route('/api/surge', methods=['GET'])
def surge_options():
    return 'surge_options'

# cloud endpoints

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-
"""
Created on Tue Nov  6 09:09:32 2018

@author: nivosco
"""

import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np

class LossHistory(tf.keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []
        self.val_losses = []
    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))
        self.val_losses.append(self.model.evaluate(self.validation_data[0], self.validation_data[1])[0])

class cnn_model:
    def __init__(self,input_shape, num_classes):
        self.model = tf.keras.models.Sequential()
        self.model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=input_shape))
        self.model.add(tf.keras.layers.Conv2D(64, (3, 3), activation='relu'))
        self.model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2)))
        self.model.add(tf.keras.layers.Dropout(0.25))
        self.model.add(tf.keras.layers.Flatten())
        self.model.add(tf.keras.layers.Dense(128, activation='relu'))
        self.model.add(tf.keras.layers.Dropout(0.5))
        self.model.add(tf.keras.layers.Dense(num_classes, activation='softmax'))
    def compile(self):
        self.model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    def train(self,x_train,y_train,batch_size,epochs,x_test,y_test):
        self.lossHistory = LossHistory()
        self.model.fit(x_train, y_train,batch_size=batch_size,epochs=epochs,verbose=1, validation_data=(x_test, y_test), callbacks=[self.lossHistory])

def lossPlot(cnn_model):
    plt.plot(np.arange(0,len(cnn_model.lossHistory.losses)),cnn_model.lossHistory.losses,'r',label='Train loss')
    plt.plot(np.arange(0,len(cnn_model.lossHistory.val_losses)),cnn_model.lossHistory.val_losses,'b',label='Validation loss')
    plt.title('Loss function convergence graph')
    plt.xlabel('Batch')
    plt.ylabel('Loss')
    plt.legend()
    plt.savefig('loss.png')
    plt.show()

## Parameters
batch_size = 128
num_classes = 10
epochs = 1
img_rows, img_cols = 28, 28

## Read the training set
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(-1,img_rows,img_cols,1)
x_test = x_test.reshape(-1,img_rows,img_cols,1)

## build, train and evaluate the models
cnn_model = cnn_model((img_rows,img_cols,1),num_classes)
cnn_model.compile()
cnn_model.train(x_train,y_train,batch_size,epochs,x_test,y_test)

## plot the loss function
lossPlot(cnn_model)


import requests
import pygal

from pygal.style import LightColorizedStyle as LCS, LightenStyle as LS

url = 'https://api.github.com/search/repositories?q=language:python&sort=stars'
r = requests.get(url)
print("Status code:", r.status_code)

response_dict = r.json()

print("Total Repositories:", response_dict['total_count'])

repo_dicts = response_dict['items']

names, plot_dicts = [], []
for repo_dict in repo_dicts:
    names.append(repo_dict['name'])
    plot_dicts.append({
        'value': repo_dict['stargazers_count'],
        'label': repo_dict['description'] if repo_dict['description'] is not None else '',
        'xlink': repo_dict['html_url']
    })

style = LS('#333366', base_style=LCS)

config = pygal.Config()
config.x_label_rotation = 45
config.show_legend = False
config.title_font_size = 24
config.label_font_size = 14
config.major_label_font_size = 18
config.truncate_label = 15
config.show_y_guides = False
config.width = 1000

chart = pygal.Bar(config, style=style)
chart.title = "Most Starred Python Projects - Github"
chart.x_labels = names

chart.add('', plot_dicts)

chart.render_to_file('python_repos.svg')


# !/usr/bin/python
# coding=utf-8

from datetime import datetime
from .. import rd
from .. import util

'''
1. 文章详细信息; 使用redis中的散列类型保存; key是'post:id'; 字段包括: title author time content category;
2. 文章总数; 保存于posts:count中; 只增不减;
3. 文章ID列表; 使用列表类型posts:list记录文章列表;
    a. 新文章发布使用LPUSH将文章加入到列表中;
    b. 删除文章 LREM
    c. 文章分页显示 LRANGE
4. 用户文章列表; 保存一个用户的所有文章; 用户名对应一个POST ID列表; posts:author
4. 删除文章列表; 使用列表类型保存 posts:del:list
'''

# 文章列表
POSTS_LIST = 'posts:list'
# 文章总数
POSTS_COUNT = 'posts:count'
# 文章删除列表
POSTS_DEL_LIST = 'posts:del_list'
# 每个作者文章列表
POST_AUTHOR_LIST = 'posts:author:'
# post infomation
POST_INFO = 'post:'

def publish_post(title, author_id, content, category):
    """
    publish post
    """
    post_id = rd.incr(POSTS_COUNT)
    pipe = rd.pipeline()
    pipe.multi()
    rd.hmset(POST_INFO + '{}'.format(post_id), {'post_id': post_id, 'title': title, 'author_id': author_id,
                                                'content': content, 'category': category, 'time': datetime.utcnow()})
    rd.lpush(POSTS_LIST, post_id)
    rd.lpush(POST_AUTHOR_LIST + '{}'.format(author_id), post_id)
    pipe.execute()

def update_post_content(post_id, content):
    """
     update post content
    """
    return rd.hset(POST_INFO + '{}'.format(post_id), 'content', content)

def delete_post(post_id):
    """
    delete post
    """
    pipe = rd.pipeline()
    pipe.multi()
    rd.lrem(POSTS_LIST, post_id, 0)
    rd.lpush(POSTS_DEL_LIST, post_id)
    pipe.execute()

def total_posts():
    """
    return total posts number
    """
    return rd.llen(POSTS_LIST)

def total_posts_by_author(author_id):
    """
    return total posts number of author
    """
    return rd.llen(POST_AUTHOR_LIST + '{}'.format(author_id))

def posts_by_page(page_id, per_page):
    """
    paging display
    page_id: start from 1
    post ID list key is str type
    """
    pages = int(rd.llen(POSTS_LIST) / per_page + 1)
    if 0 < page_id <= pages:
        return util.convert(rd.lrange(POSTS_LIST, (page_id - 1) * per_page, page_id * per_page - 1))
    print('invaild param page id[{0}, {1}]'.format(1, pages))

def get_post(post_id):
    """
    get post infomation
    return dict
    """
    return util.convert(rd.hgetall(POST_INFO + '{}'.format(post_id)))

def posts_by_author(author_id, page_id, per_page):
    """
    paging display
    get posts list by author
    """
    pages = int(rd.llen(POST_AUTHOR_LIST + '{}'.format(author_id)) / per_page + 1)
    if 0 < page_id <= pages:
        return util.convert(rd.lrange(POST_AUTHOR_LIST + '{}'.format(author_id), (page_id - 1) * per_page,
                                      page_id * per_page - 1))
    print('posts_by_author invaild param author {0} page id[{1}, {2}]: {3}'.format(author_id, 1, pages, page_id))


# -*- coding: utf-8 -*-
#harvest theses of university santacatrina
#JH: 2023-01-01

import re
from requests import Session
from time import sleep
from bs4 import BeautifulSoup
from typing import Union
import ejlmod3

publisher = 'Santa Catarina U.'
jnlfilename = 'THESES-SANTACATARINA-%s' % ejlmod3.stampoftoday()

rpp = 10
boring = ['Dissertação (Mestrado)', 'Dissertação (Mestrado profissional)',
          'Master Thesis']

def dspace_extract_links(content: str, prafix: str):
    soup = BeautifulSoup(content, 'lxml')
    article_links = soup.find_all('a', {'href': re.compile('/handle/\d{0,}/\d{0,}')})
    filtered = filter(lambda article: False if article.parent.get('class') is None
                        else True if 'artifact-title' in article.parent.get('class')
                        else False, article_links)

    return list(filtered)

def dspace_table_extract_information(titles: list, link: str, mapping: dict) -> Union[dict, int]:
    extracted_data: dict = {}

    with Session() as session:
        resp = session.get(link)

        if resp.status_code != 200:
            return -1

        resp_data = resp.content.decode('utf-8').replace('<?xml version="1.0" encoding="UTF-8"?>', '')
        soup = BeautifulSoup(resp_data, 'lxml')

        for title in titles:
            patter: str = '^%s$' % title.replace('.', '\\.')
            mapped_name: Union[str, dict] = mapping[title]

            if mapped_name is None:
                return -2

            fields = soup.find_all('td', text=re.compile(patter, re.MULTILINE))

            if len(fields) == 0:
                print("Can't find:", mapped_name)
                continue

            for field in fields:
                data_field: str = field.parent.find_all('td')[1].text

                datatype: int = mapping[title].get('datatype')

                if datatype == 1:
                    extracted_data[mapped_name.get('name')] = data_field
                elif datatype == 2:
                    if mapped_name.get('name') in extracted_data.keys():
                        extracted_data[mapped_name.get('name')].append(data_field)
                    else:
                        extracted_data[mapped_name.get('name')] = [data_field]
                elif datatype == 3:
                    if mapped_name.get('name') in extracted_data.keys():
                        extracted_data[mapped_name.get('name')].append([data_field])
                    else:
                        extracted_data[mapped_name.get('name')] = [[data_field]]

    # Get the pdf link
    pdf_section = soup.find_all('a', attrs={'href': re.compile('bitstream')})
    if len(pdf_section) == 2:
        rec['hidden'] = 'https://repositorio.ufsc.br%s' % pdf_section[0].get('href')

    return extracted_data

recs = []
alllinks = []
with Session() as session:

    for (dep, fc, pages) in [('76362', 'c', 2), ('75302', '', 2), ('77533', '', 2), ('123199', 'm', 1),
                             ('80823', 'mc', 1), ('103512', 'm', 2), ('74675', '', 0)]:
        for page in range(pages):
            to_curl = 'https://repositorio.ufsc.br/handle/123456789/' + dep + '/browse?rpp=' + str(
                rpp) + '&sort_by=2&type=dateissued&offset=' + str(rpp * page) + '&etal=-1&order=DESC'
            ejlmod3.printprogress('=', [[dep], [page+1, pages], [to_curl]])

            print('#### OPENING INDEX PAGE (%i/%i) OF %s:%s ####' % (page + 1, pages, dep, fc))
            index_resp = session.get(to_curl)

            if index_resp.status_code != 200:
                print('Error: Can\'t open the page\nSkipping this index page.')
                continue

            index_resp_data = index_resp.content.decode('utf-8').replace('<?xml version="1.0" encoding="UTF-8"?>', '')

            soup = BeautifulSoup(index_resp_data, 'lxml')
            article_links = dspace_extract_links(index_resp_data, 'https://repositorio.ufsc.br')
            rec = {}

            # Open the sub site
            for article in article_links:
                if ejlmod3.checkinterestingDOI(article.get('href')) and not article.get('href') in alllinks:
                    print('[%s] --> Harvesting data' % 'https://repositorio.ufsc.br%s?show=full' % article.get('href'))
                    alllinks.append(article.get('href'))
                else:
                    print('[%s]' % 'https://repositorio.ufsc.br%s?show=full' % article.get('href'))
                    continue
                try:
                    data_resp = session.get('https://repositorio.ufsc.br%s?show=full' % article.get('href'))
                except Exception:
                    print('[%s] --> Is not a real record. Skipping...' % 'https://repositorio.ufsc.br%s?show=full' %
                          article.get('href'))
                    continue

                if data_resp.status_code != 200:
                    print('[%s] Error: Can\'t open page!')
                    continue

                data_resp_data = data_resp.content.decode('utf-8').replace('<?xml version="1.0" encoding="UTF-8"?>', '')
                data_soup = BeautifulSoup(data_resp_data, 'lxml')

                mapping = {
                    'dc.contributor.advisor': {
                        'name': 'supervisor',
                        'datatype': 3
                    },
                    'dc.type': {
                        'name': 'note',
                        'datatype': 2
                    },
                    'dc.contributor.author': {
                        'name': 'autaff',
                        'datatype': 3
                    },
                    'dc.date.issued': {
                        'name': 'date',
                        'datatype': 1
                    },
                    'dc.identifier.uri': {
                        'name': 'link',
                        'datatype': 1
                    },
                    'dc.description.abstract': {
                        'name': 'abs',
                        'datatype': 1
                    },
                    'dc.subject.classification': {
                        'name': 'keyw',
                        'datatype': 2
                    },
                    'dc.title': {
                        'name': 'tit',
                        'datatype': 1
                    }
                }

                keepit = True
                rec = dspace_table_extract_information(mapping.keys(), 'https://repositorio.ufsc.br%s?show=full' % article.get('href'), mapping)
                rec['tc'] = 'T'
                rec['jnl'] = 'BOOK'
                if len(fc) != 0:
                    rec['fc'] = fc
                ejlmod3.metatagcheck(rec, data_soup, ['DCTERMS.extent', 'DC.language', 'DCTERMS.abstract', 'DC.type'])
                rec['autaff'][-1].append(publisher)
                if 'note' in rec:
                    for note in rec['note']:
                        if note in boring:
                            keepit = False
                            ejlmod3.adduninterestingDOI(article.get('href'))
                if keepit:
                    recs.append(rec)
                sleep(5)
            sleep(10)
        sleep(5)

ejlmod3.writenewXML(recs, publisher, jnlfilename)


"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

from setuptools import setup
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(

    name='deepstomata',
    version='1.0',
    description='iterative calculation of stomatal pores ',
    long_description=long_description,
    url='https://github.com/totti0223/',
    author='Yosuke Toda',
    author_email='tyosuke@aquaseerser.com',
    license='MIT',

    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 3 - Alpha',
        #'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.5',
        'Topic :: Scientific/Engineering :: Image Recognition'
    ],
    keywords='Facial Recognition Technology for Automated Stomatal Aperture Measurement',
    packages=['deepstomata'],
    install_requires=open('requirements.txt').read().splitlines(),
    package_data={
        "deepstomata": ["models/*", "config.ini"]
    }
)


# coding: utf-8
import os
import json

def load_actor_list():
    """
    将演员姓名存入列表
    :return: actor_list
    """
    actor_list = []
    with open("./actor.txt","r") as r:
        lines = r.readlines()
        for line in lines:
            line = line.strip('\n')
            actor_list.append(line)
    # print(actor_list)
    return actor_list

def load_totaloffice_attributes(actor_list):
    """
    计算演员经验值指标
    :param actor_list:
    :return: actor_experience
    """
    actor_experience = {}
    experience = {}
    for name in actor_list:
        filename = "./actor_office/" + name + ".json"
        if os.path.exists(filename):
            with open(filename,"r") as r:
                count = 0
                per_acotr = json.load(r)
                # print(per_acotr)
                for per_movie in per_acotr:
                    office = per_movie['office']
                    if office == 'null':
                        office = 0
                    else:
                        office = int(office.replace("万",""))
                    count += office
                actor_experience[name] = count
    # print(actor_experience)
    max_office,min_office = evaluate(actor_experience)
    for name in actor_experience.keys():
        sorce = normalized(actor_experience[name],max_office,min_office)
        experience[name] = sorce
    print("Empirical indicator: \n")
    print(experience)
    return experience

def load_time_attributes(star,end,actor_list):
    """
    计算时效性指标
    :param star:
    :param end:
    :param actor_list:
    :return:
    """
    # 获取该段时间内设计该演员的新闻数量
    actor_new = {}
    for name in actor_list:
        filename = "./new/" + name + ".txt"
        if os.path.exists(filename):
            count = 0
            with open(filename,"r") as r:
                news = r.readlines()
                for new in news:
                    new = new.strip('\n')
                    year = new.split('\t')[0]
                    num = new.split('\t')[1]
                    if int(year) >= star and int(year) <= end:
                        count += int(num)
                actor_new[name] = count

    # 获取该段时间内演员的票房数量
    actor_office = {}
    for name in  actor_list:
        filename = "./actor_office/" + name + ".json"
        if os.path.exists(filename):
            count = 0
            with open(filename,"r") as r:
                per_actor = json.load(r)
                for per_movie in per_actor:
                    if per_movie['year'] >= star and per_movie['year'] <= end:
                        office = per_movie['office']
                        if office == 'null':
                            office = 0
                        else:
                            office = int(office.replace("万", ""))
                        count += office
                actor_office[name] = count

    # 计算时效性指标
    max_office, min_office = evaluate(actor_office)
    max_new, min_new = evaluate(actor_new)
    time = {}
    for name in actor_list:
        if name in actor_office.keys() and name in actor_new.keys():
            sorce_office = normalized(actor_office[name],max_office,min_office)
            sorce_new = normalized(actor_new[name],max_new,min_new)
            time[name] = sorce_office + sorce_new
    print("Timeliness indicator: \n")
    print(time)
    return time

def load_actieve(actor_list,star,end):
    baidu = {}
    actieve = {}
    with open("./record.txt","r") as r:
        lines = r.readlines()
        for line in lines:
            line = line.strip('\n')
            flag = line.split(' ')[-1]
            names = line.split(' ')[0:-1]
            name = ''
            for i in names:
                name += i
            baidu[name] = flag
    # print(baidu)
    for name in actor_list:
        filename = "./演员活跃度/" + name + '.txt'
        if os.path.exists(filename) and name in baidu.keys():
            with open(filename, "r") as r:
                sorce = 0
                wei = 0
                aware = 0
                bai = 0
                lines = r.readlines()
                for line in lines:
                    line = line.strip('\n')
                    weibo = line.split(',')[0]

                    if weibo == "1":
                        wei = 1
                    awares = line.split(',')[1:]

                    for i in awares:
                        if int(i) >= star and int(i) <= end:
                            aware = 1
                    if baidu[name] == "1":
                        bai = 1
                sorce = (wei + bai + aware) * 0.3
                actieve[name] = sorce
    print("Character indicator: \n")
    print(actieve)
    return actieve

def evaluate(dict):
    Max = max(dict.values())
    Min = min(dict.values())
    return Max,Min

def normalized(value,Max,Min):
    return 0.5/(Max - Min)*(value - Min)

def ranking(actor_list,experience,time,actieve):
    total = {}
    for name in actor_list:
        if name in experience.keys() and name in time.keys() and name in actieve.keys():
            total[name] = experience[name] + time[name] + actieve[name]
    # with open("./actor_ranking_2015-2018.json","w") as w:
    #     json.dump(total,w)
    print(total)
    dict = sorted(total.items(), key=lambda d: d[1], reverse=True)
    for i in dict:
        print(str(i) + '\n')
if __name__ == '__main__':
    actor_list = load_actor_list()
    experience = load_totaloffice_attributes(actor_list)
    time = load_time_attributes(2015,2018,actor_list)
    actieve = load_actieve(actor_list,2011,2015)
    ranking(actor_list,experience,time,actieve)


#_*_ coding: utf-8 _*_
# 선형회귀분석 2.1

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

# 학습데이터 생성
num_points = 1000
vectors_set = []

for i in xrange(num_points):
    x1 = np.random.normal(0.0,0.55)
    y1 = x1 * 0.1 + 0.3 + np.random.normal(0.0,0.03)
    vectors_set.append([x1,y1])

x_data = [v[0] for v in vectors_set]
y_data = [v[1] for v in vectors_set]

# 그래프로 보여주는 부분
# plt.plot(x_data, y_data, 'ro')
# plt.show()

# 매개변수 지정(가설지정)
W = tf.Variable(tf.random_uniform([1],-1.0,-1.0))
b = tf.Variable(tf.zeros([1]))
y = x_data * W + b

# 경사 하강법에 의한 비용함수(오차함수) => 함수값을 최소화하는 알고리즘
# y값과 y_data값의 차이를 알기위해 그 사이의 거리를 제곱한 후 평균내어 계산함
loss = tf.reduce_mean(tf.square(y - y_data))

# 경사하강법은 초기 시작점에서 함수의 값이 최소화되는 방향으로 매개변수를 변경하는 것을 반복적으로 수행하는 알고리즘
# W(기울기)를 음의 방향쪽으로 진행하면서 반복적으로 최적화 수행
# 보통 양의 값을 만들기위해 거리값을 제곱, 기울기 계산을 위해 오차함수는 미분이 가능해야 함
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)

# 세션 초기화
init = tf.global_variables_initializer()

sess = tf.Session()
sess.run(init)

# 세션 반복수행(훈련)
# 경사 하강법 알고지즘을 수행할 때마다 평면의 한 지점에서 시작하여 더 작은 오차를 갖는 직선을 찾아 이동
# 오차함수의 기울기를 계산하기 위해 텐서플로는 오차함수를 미분함
# 반복이 일어날 때마다 움직힐 방향을 알아내기 위해 W,b에 대한 편미분방정식 계산이 필요함
for step in xrange(8):
    sess.run(train)
    # print step, sess.run(W),sess.run(b)
    plt.plot(x_data,y_data,'ro')
    plt.plot(x_data,sess.run(W)*x_data * sess.run(b))
    plt.xlabel('x')
    plt.ylabel('y')
    plt.show()
    print(step,sess.run(loss))


#import MoexParser
import sqlite3

'''

    2 БД:
        1) Одна таблица - SEC_ID и SHORTNAME.
        2) Много таблиц с названиями от SEC_ID по две колонки: дата и цена закрытия.

'''

class MoexDataWorker():

    def __init__(self):

        self.SECID_SHORTNAME_DATABASE_NAME = 'secid_shortname_data.db'
        self.SECID_SHORTNAME_TABLENAME = 'std_table'
        self.HISTORY_STOCKS_DATABASE_NAME = 'history_stocks_data.db'

    def create_table_ssdb(self):
        conn = sqlite3.connect(self.SECID_SHORTNAME_DATABASE_NAME)
        cursor = conn.cursor()

        sql = "CREATE TABLE %s\
                                      (secid text, shortname text)\
                                   " % str("'" + str(self.SECID_SHORTNAME_TABLENAME) + "'")

        cursor.execute(sql)

    def create_table_hsdb(self, sec_id):
        conn = sqlite3.connect(self.HISTORY_STOCKS_DATABASE_NAME)
        cursor = conn.cursor()

        sql = "CREATE TABLE %s\
                                      (date text, close_price text)\
                                   " % str("'" + str(sec_id) + "'")

        cursor.execute(sql)

    def get_last_date(self, sec_id, cursor):

        try:
            cursor.execute("SELECT date FROM '" + str(sec_id) + "'")
            date_arr = cursor.fetchall()
            return date_arr[len(date_arr) - 1][0]

        except Exception as e:
            return None

    def get_stock_info(self, sec_id):
        conn = sqlite3.connect(self.SECID_SHORTNAME_DATABASE_NAME)
        cursor = conn.cursor()

        cursor.execute("SELECT * FROM '" + str(self.SECID_SHORTNAME_TABLENAME) + "' WHERE secid LIKE '" + str(sec_id) + "'")

        ans = []

        ans.append(cursor.fetchall()[0])
        #ans.append(cursor.fetchall()[1])

        conn = sqlite3.connect(self.HISTORY_STOCKS_DATABASE_NAME)
        cursor = conn.cursor()

        cursor.execute("SELECT * FROM '" + str(sec_id) + "'")

        ans += cursor.fetchall()

        return ans

    def add_secid_and_shortname(self, sec_id, shortname, conn=None):
        if conn == None:
            conn = sqlite3.connect(self.SECID_SHORTNAME_DATABASE_NAME)

        cursor = conn.cursor()

        #cursor.execute("SELECT * FROM '" + self.SECID_SHORTNAME_TABLENAME + "' WHERE secid LIKE " + str(sec_id))
        #tmp = cursor.fetchall()
        tmp = None

        if tmp == None or len(tmp) == 0:
            cursor.execute("INSERT INTO '" + self.SECID_SHORTNAME_TABLENAME + "' VALUES (?,?) ", [sec_id, shortname])
            conn.commit()

    def add_history_stock_data(self, sec_id, data, conn=None):
        if conn == None:
            conn = sqlite3.connect(self.HISTORY_STOCKS_DATABASE_NAME)
        cursor = conn.cursor()

        last_date = self.make_arr_from_date(self.get_last_date(sec_id, cursor))
        new_date = self.make_arr_from_date(data[0])

        if not (last_date == None) and not (new_date == None):
            if last_date[0] > new_date[0] or (last_date[0] == new_date[0] and last_date[1] >= new_date[1]) or \
                    (last_date[0] == new_date[0] and last_date[1] == new_date[1] and last_date[2] >= new_date[2]):
                return

        try:
            cursor.execute("INSERT INTO '" + str(sec_id) + "' VALUES (?,?) ", [data[0], data[1]])
            conn.commit()
        except sqlite3.OperationalError as e:
            self.create_table_hsdb(sec_id)
            self.add_history_stock_data(sec_id, data, conn)

    def get_all_secid(self):
        conn = sqlite3.connect(self.SECID_SHORTNAME_DATABASE_NAME)
        cursor = conn.cursor()

        cursor.execute("SELECT secid FROM '" + str(self.SECID_SHORTNAME_TABLENAME) + "'")

        return cursor.fetchall()

    def make_arr_from_date(self, str_date):
        if str_date == None:
            return None
        return [int(i) for i in str_date.split('-')]


from fastapi import FastAPI
import helper
from models import Email, FacebookPost
import content

app = FastAPI()

@app.post("/extract")
async def extract(email: Email):

    content = email.content

    response = helper.extract_email(content)

    arguments = response.choices[0]["message"]["function_call"]["arguments"]

    name = eval(arguments).get("name")
    category = eval(arguments).get("category")
    summary = eval(arguments).get("summary")
    priority = eval(arguments).get("priority")

    return {"name": name, "category": category, "summary": summary, "priority": priority}

@app.post("/create")
async def create(fb: FacebookPost):

    niche = fb.niche
    company = fb.company
    website = fb.website
    topic = fb.topic
    title = fb.title

    response = content.create_fb_content(niche, company, website, topic)

    return {"content": response}


from scipy import io
import numpy as np
from matplotlib import pyplot as plt

def main():
    # 파일의 데이터 가져오기
    mat_file = io.loadmat("YaleB.mat")
    data_set = mat_file["fea"]
    # 사진 입력 받이서 float형태 행렬도 만들기
    all_faces_temp = np.ndarray.flatten(data_set)[: 1024 * 1774]
    test_faces_temp = np.ndarray.flatten(data_set)[1024 * 1774 :]
    all_faces_temp = all_faces_temp.astype(np.float)
    test_faces_temp = test_faces_temp.astype(np.float)

    # 데이터는 1774개로 정했고, 원본 데이터의 형식을 맞추기 위해 일단 이렇게 설정, 또 테스트용 50개의 vector를 만들기
    all_faces = all_faces_temp.reshape(1774, 1024)
    test_faces = test_faces_temp.reshape(640, 1024)

    # 입력 받은 사진이 옆으로 누워있어서 바로 세우기
    for i in range(0, 1774):
        all_faces[i] = all_faces[i].reshape(32, 32).transpose().flatten()
    for i in range(0, 640):
        test_faces[i] = test_faces[i].reshape(32, 32).transpose().flatten()

    # mean 구하기
    mean_arr = []
    for i in range(0, 1024):
        sum = 0
        for j in range(0, 1774):
            sum += all_faces[j][i]
        mean_arr.append(sum / 1774)
    mean_face = np.array(mean_arr)
    for i in range(0, 1774):
        all_faces[i] -= mean_face

    all_faces = all_faces.transpose()

    # 중앙값 얼굴 나타내기
    # mean_face = mean_face.reshape(32, 32)
    # img = plt.imshow(mean_face)
    # img.set_cmap("gray")
    # plt.show()

    # A^t@A 로 square matrix 만들기
    # all_faces_square = all_faces @ all_faces.transpose()
    # eigen_values, eigen_vectors = np.linalg.eig(all_faces_square)

    # Matrix를 svd 분할
    u, s, vt = np.linalg.svd(all_faces)
    # img = plt.imshow(u[:, 0].reshape(32, 32))
    # img.set_cmap("gray")
    # plt.show()

    # eigen value가 큰 eigen vector을 위해 idx를 리스트로 나타내려고 하는데 자동으로 정렬되는거같다.
    # eigen_idx = []
    # for i in range(0, 40):
    #     max_eigen = 0
    #     idx = -1
    #     for j in range(0, 1024):
    #         if max_eigen < s[j]:
    #             if j in eigen_idx:
    #                 continue
    #             else:
    #                 idx = j
    #                 max_eigen = s[j]
    #     eigen_idx.append(idx)

    # eigenVector img 보기
    # u = u.transpose()
    # for i in range(0, 40):
    #     img = plt.imshow(u[i].reshape(32, 32))
    #     img.set_cmap("gray")
    #     plt.show()

    # eigen vector들을 모아놓은 matrix를 계산하기 쉽도록 transpose 시킨다.
    u = u.transpose()
    all_faces = all_faces.transpose()
    weights_all = []
    for k in range(0, 10):
        for i in range(0, 5):
            weights = []
            # test_face = test_faces[i] - mean_face
            test_face = test_faces[k * 64 + i] - mean_face
            for j in range(0, 40):
                weights.append(u[j].transpose() @ test_face)
            # print("coefficients of ", i, " : ", weights)
            weights_all.append(weights)
            face = mean_face
            for j in range(0, 40):
                face += weights[j] * u[j]
            # img_origin = plt.imshow((test_faces[k * 64 + i]).reshape(32, 32))
            # img_origin.set_cmap("gray")
            # plt.show()

            # img = plt.imshow(face.reshape(32, 32))
            # img.set_cmap("gray")
            # plt.show()
    weights_all = np.array(weights_all)
    weights_all = weights_all.astype(np.float)
    weights_mean = []
    weights_variance = []
    for i in range(0, 10):
        temp_weight = []
        mean_weight = (
            weights_all[i * 5]
            + weights_all[i * 5 + 1]
            + weights_all[i * 5 + 2]
            + weights_all[i * 5 + 3]
            + weights_all[i * 5 + 4]
        )
        mean_weight /= 5
        weights_mean.append(mean_weight)

    all_variances = []
    for i in range(0, 10):
        variance = []
        for j in range(0, 40):
            var = 0
            for k in range(0, 5):
                var += (weights_all[i * 5 + k][j] - weights_mean[i][j]) * (
                    weights_all[i * 5 + k][j] - weights_mean[i][j]
                )
            var /= 5
            variance.append(np.sqrt(var))
        all_variances.append(variance)
    all_variances = np.array(all_variances)
    print(all_variances)

if __name__ == "__main__":
    main()


import discord
from discord.ext import commands
from wiki import Wiki

bot = commands.Bot(command_prefix="?")
wiki = Wiki()

@bot.event
async def on_ready():
    print("Logged in as")
    print(bot.user.name)
    print(bot.user.id)
    print("------")

@bot.command()
async def ping(ctx):
    await ctx.send(f"Pong! In {round(bot.latency * 1000)}ms")

@bot.command()
async def lookup(ctx, term):
    d = wiki.lookup(term)
    embed = discord.Embed(title=d["title"], url=d["url"], description=d["desc"])
    embed.set_thumbnail(url=d["image"])

    for k, v in d.items():
        if k not in ["title", "url", "desc", "image"]:
            embed.add_field(name=k, value=v, inline=False)

    await ctx.send(embed=embed)

bot.run(open("token.txt", "r").read())


import sys, os
import numpy as np
import scipy
from scipy.io.wavfile import read, write
from scipy.signal import lfilter
import scipy.signal
from sinsum import sinsum
import matplotlib.pyplot as plt

def lpc_ref(signal, order):
    """Compute the Linear Prediction Coefficients.

    Return the order + 1 LPC coefficients for the signal. c = lpc(x, k) will
    find the k+1 coefficients of a k order linear filter:

      xp[n] = -c[1] * x[n-2] - ... - c[k-1] * x[n-k-1]

    Such as the sum of the squared-error e[i] = xp[i] - x[i] is minimized.

    Parameters
    ----------
    signal: array_like
        input signal
    order : int
        LPC order (the output will have order + 1 items)

    Notes
    ----
    This is just for reference, as it is using the direct inversion of the
    toeplitz matrix, which is really slow"""
    if signal.ndim > 1:
        raise ValueError("Array of rank > 1 not supported yet")
    if order > signal.size:
        raise ValueError("Input signal must have a lenght >= lpc order")

    if order > 0:
        p = order + 1
        r = np.zeros(p, signal.dtype)
        # Number of non zero values in autocorrelation one needs for p LPC
        # coefficients
        nx = np.min([p, signal.size])
        x = np.correlate(signal, signal, 'full')
        r[:nx] = x[signal.size-1:signal.size+order]
        phi = np.dot(scipy.linalg.inv(scipy.linalg.toeplitz(r[:-1])), -r[1:])
        return np.concatenate(([1.], phi))
    else:
        return np.ones(1, dtype = signal.dtype)

def lpc_to_formants(lpc, sr):
    """Convert LPC to formants directly"""

    # extract roots, get angle and radius
    roots = np.roots(lpc)

    pos_roots = roots[np.imag(roots) >= 0]
    if len(pos_roots) < len(roots) // 2:
        pos_roots = list(pos_roots) + [0] * (len(roots) // 2 - len(pos_roots))
    if len(pos_roots) > len(roots) // 2:
        pos_roots = pos_roots[: len(roots) // 2]

    w = np.angle(pos_roots)
    a = np.abs(pos_roots)

    order = np.argsort(w)
    w = w[order]
    a = a[order]

    freqs = w * (sr / (2 * np.pi))
    bws = -0.5 * (sr / (2 * np.pi)) * np.log(a)

    # exclude DC and sr/2 frequencies
    return freqs, bws

def lpc_okno(okno_signala, red, fs):
    """Funkcija, ki poračuna frekvence in amplitude formantov v danem oknu
    signala z uporabo LPC analize."""
    # filtriranje vhodnega signala
    okno_signala = okno_signala * np.hanning(okno_signala.shape[0])
    okno_signala = lfilter([1], [1.0, 0.63], okno_signala)

    # določitev koeficientov LPC filtra z linearno regresijo
    A = lpc_ref(okno_signala, red)

    # Določitev frekvenc in pasovnih širin formantov
    formanti, pasovne_sirine = lpc_to_formants(A, fs)
    amplitude = np.exp(pasovne_sirine / 60)
    return formanti[1:5],  amplitude[1:5]


from decimal import Decimal

import pytest
from hypothesis import example, given, settings
from hypothesis import strategies as st

from vyper import ast as vy_ast
from vyper.exceptions import OverflowException, TypeMismatch, ZeroDivisionException

st_decimals = st.decimals(
    min_value=-(2**32), max_value=2**32, allow_nan=False, allow_infinity=False, places=10
)

@pytest.mark.fuzzing
@settings(max_examples=50)
@given(left=st_decimals, right=st_decimals)
@example(left=Decimal("0.9999999999"), right=Decimal("0.0000000001"))
@example(left=Decimal("0.0000000001"), right=Decimal("0.9999999999"))
@example(left=Decimal("0.9999999999"), right=Decimal("0.9999999999"))
@example(left=Decimal("0.0000000001"), right=Decimal("0.0000000001"))
@pytest.mark.parametrize("op", "+-*/%")
def test_binop_decimal(get_contract, assert_tx_failed, op, left, right):
    source = f"""
@external
def foo(a: decimal, b: decimal) -> decimal:
    return a {op} b
    """
    contract = get_contract(source)

    vyper_ast = vy_ast.parse_to_ast(f"{left} {op} {right}")
    old_node = vyper_ast.body[0].value
    try:
        new_node = old_node.evaluate()
        is_valid = True
    except ZeroDivisionException:
        is_valid = False

    if is_valid:
        assert contract.foo(left, right) == new_node.value
    else:
        assert_tx_failed(lambda: contract.foo(left, right))

def test_binop_pow():
    # raises because Vyper does not support decimal exponentiation
    vyper_ast = vy_ast.parse_to_ast("3.1337 ** 4.2")
    old_node = vyper_ast.body[0].value

    with pytest.raises(TypeMismatch):
        old_node.evaluate()

@pytest.mark.fuzzing
@settings(max_examples=50)
@given(
    values=st.lists(st_decimals, min_size=2, max_size=10),
    ops=st.lists(st.sampled_from("+-*/%"), min_size=11, max_size=11),
)
def test_nested(get_contract, assert_tx_failed, values, ops):
    variables = "abcdefghij"
    input_value = ",".join(f"{i}: decimal" for i in variables[: len(values)])
    return_value = " ".join(f"{a} {b}" for a, b in zip(variables[: len(values)], ops))
    return_value = return_value.rsplit(maxsplit=1)[0]
    source = f"""
@external
def foo({input_value}) -> decimal:
    return {return_value}
    """
    contract = get_contract(source)

    literal_op = " ".join(f"{a} {b}" for a, b in zip(values, ops))
    literal_op = literal_op.rsplit(maxsplit=1)[0]
    vyper_ast = vy_ast.parse_to_ast(literal_op)
    try:
        vy_ast.folding.replace_literal_ops(vyper_ast)
        expected = vyper_ast.body[0].value.value
        is_valid = -(2**127) <= expected < 2**127
    except (OverflowException, ZeroDivisionException):
        # for overflow or division/modulus by 0, expect the contract call to revert
        is_valid = False

    if is_valid:
        assert contract.foo(*values) == expected
    else:
        assert_tx_failed(lambda: contract.foo(*values))


# Standard Library
import traceback
import io

def email_function_failure_traceback(send_email_function):
    def decorator(original_func):
        def new_func(*args, **kwargs):
            try:
                function_value = original_func(*args, **kwargs)
            except Exception as e:
                exc_string = io.StringIO()
                traceback.print_exc(file=exc_string)
                message = exc_string.getvalue()
                message = message.replace('line ', '

line ')
                send_email_function(message)
                raise e
            return function_value
        return new_func
    return decorator


from django import forms

from MainScreen.models import Member, Due

choice = (
    (1, "Included"),
    (0, "Excluded"),
    (-1, "Not Applicable")
)

class NewDue(forms.Form):
    due_id = forms.CharField(label='Due ID')
    due_type = forms.CharField(label='Due Type')
    due_amount = forms.DecimalField(label='Due Amount')
    due_fineamt = forms.DecimalField(label='Fine Amount (Yearly)')
    is_head = forms.ChoiceField(choices=choice)
    is_pension = forms.ChoiceField(choices=choice)
    is_nri = forms.ChoiceField(choices=choice)
    is_govt = forms.ChoiceField(choices=choice)
    is_male = forms.ChoiceField(choices=choice)
    account_serial = forms.CharField(label='Account Serial')

ward_choices = (
    ("Moolavattom ward", "Moolavattom ward"),
    ("Masjid ward", "Masjid ward"),
    ("Sasthavattom ward", "Sasthavattom ward")
)

class NewMember(forms.Form):
    family_number = forms.IntegerField()
    jp_number = forms.DecimalField(max_digits=1000, decimal_places=2)
    name = forms.CharField()
    is_due_apply = forms.BooleanField(label='Is Due Apply?')
    is_pension = forms.BooleanField(required=False, label='Is Pension?')
    is_nri = forms.BooleanField(required=False, label='Is NRI?')
    is_govt = forms.BooleanField(required=False, label='Is Govt?')
    is_male = forms.BooleanField(required=False, label='Is Male?')
    is_alive = forms.BooleanField(required=False, label='Is Alive?')
    age = forms.IntegerField()
    mobile = forms.IntegerField(required=False)
    email = forms.CharField(required=False)
    description = forms.CharField(required=False)
    remarks = forms.CharField(required=False)

    def __init__(self, *args, **kwargs):
        super(NewMember, self).__init__(*args, **kwargs)
        self.fields['name'].widget.attrs['cols'] = 40

class NewFamily(forms.Form):
    family_no = forms.IntegerField()
    census_no = forms.IntegerField()
    house_name = forms.CharField()
    area = forms.ChoiceField(choices=ward_choices)
    postoffice = forms.CharField()
    address = forms.CharField()
    jp_number = forms.DecimalField(max_digits=1000, decimal_places=2)
    name = forms.CharField()
    is_due_apply = forms.BooleanField(label='Is Due Apply?')
    is_pension = forms.BooleanField(required=False, label='Is Pension?')
    is_nri = forms.BooleanField(required=False, label='Is NRI?')
    is_govt = forms.BooleanField(required=False, label='Is Govt?')
    is_male = forms.BooleanField(required=False, label='Is Male?')
    is_alive = forms.BooleanField(required=False, label='Is Alive?')
    age = forms.IntegerField()
    mobile = forms.IntegerField(required=False)
    email = forms.CharField(required=False)
    description = forms.CharField(required=False)
    remarks = forms.CharField(required=False)

class EditMember(forms.ModelForm):
    class Meta:
        model = Member
        exclude = ['serial', 'financial_year_serial', 'jp_number',
                   'description']
        widgets = {
            'area': forms.Select(choices=ward_choices),
        }

    def __init__(self, *args, **kwargs):
        super(EditMember, self).__init__(*args, **kwargs)
        self.fields['name'].widget.attrs['cols'] = 22
        self.fields['name'].widget.attrs['rows'] = 1
        self.fields['house_name'].widget.attrs['cols'] = 22
        self.fields['house_name'].widget.attrs['rows'] = 1
        self.fields['family_number'].widget.attrs['cols'] = 22
        self.fields['family_number'].widget.attrs['rows'] = 1
        self.fields['email'].widget.attrs['cols'] = 22
        self.fields['email'].widget.attrs['rows'] = 1
        self.fields['area'].widget.attrs['cols'] = 22
        self.fields['area'].widget.attrs['rows'] = 1
        self.fields['address'].widget.attrs['cols'] = 22
        self.fields['address'].widget.attrs['rows'] = 1
        self.fields['remarks'].widget.attrs['cols'] = 22
        self.fields['remarks'].widget.attrs['rows'] = 1
        self.fields['postoffice'].widget.attrs['cols'] = 22
        self.fields['postoffice'].widget.attrs['rows'] = 1

        self.validate = kwargs.pop('validate', False)
        self.fields['email'].required = False
        self.fields['house_name'].required = False
        self.fields['mobile'].required = False
        self.fields['remarks'].required = False
        self.fields['address'].required = False
        self.fields['area'].required = False
        self.fields['postoffice'].required = False

        self.fields['name'].label = "Name"
        self.fields['is_active'].label = "Is Active?"
        self.fields['is_head'].label = "Is Head?"
        self.fields['is_due_apply'].label = "Is Due Apply?"
        self.fields['is_pension'].label = "Is Pension?"
        self.fields['is_nri'].label = "Is NRI?"
        self.fields['is_govt'].label = "Is Govt"
        self.fields['is_male'].label = "Is Male?"
        self.fields['is_alive'].label = "Is Alive?"

class EditDues(forms.ModelForm):
    class Meta:
        model = Due
        exclude = ['account_serial', 'due_display_id', 'due_financial_year', 'due_type', 'applied']

    def __init__(self, *args, **kwargs):
        super(EditDues, self).__init__(*args, **kwargs)
        self.fields['due_amount'].label = "Due Amount"
        self.fields['due_fineamt'].label = "Due Fine Amount"
        self.fields['is_head'].label = "Is Head?"
        self.fields['is_pension'].label = "Is Pension?"
        self.fields['is_nri'].label = "Is NRI?"
        self.fields['is_govt'].label = "Is Govt"
        self.fields['is_male'].label = "Is Male?"
        self.fields['due_active'].label = "Due Active?"


'''launch dolphin, mainly for recording replays'''

from pathlib import Path
import subprocess
import json
from secrets import token_hex as rand_hex
from time import sleep
from os import rename as move_file
import os
import shutil
# import ubjson
import slippi
import video_tools

# dolphin.ini important settings:
# (no seekbar)
#
# MainWindowPosX = 535
# MainWindowPosY = 17
# MainWindowWidth = 600
# MainWindowHeight = 620
# __ didn't adjust these really
# RenderWindowXPos = 485
# RenderWindowYPos = 57
# RenderWindowWidth = 779
# RenderWindowHeight = 636
#
# => 584x480 (original 73:60 ratio)
#
# EmulationSpeed > 1.0 doesn't seem to work, so no gains there
#
# PauseMovie = True ? hopefully stops replay after n frames; EDIT - it doesn't
#
# make sure muted

# trick to record multiple replays faster: use multiple dolphin instances

def record_replay(replay_filepath, output_dir='../vid_out/', slowdown_factor=1.1):

    # paths
    replay_filepath = Path(replay_filepath)
    dolphin_folder_path = Path(r'../dolphin')
    executable_path = dolphin_folder_path / 'Slippi Dolphin.exe'
    iso_path = read_json('../settings.json')['iso_path']
    tmp_output_folder = Path('../tmp/dolphin_process_0')    # if multiple dolphins output to same folder, there would be default filename conflicts with "framedump0.avi"
    real_output_folder = Path(output_dir)

    DEFAULT_VIDEO_NAME = 'framedump0.avi'
    old_video_path = tmp_output_folder  / DEFAULT_VIDEO_NAME
    new_video_name = replay_filepath.stem + '.avi'
    new_video_path = real_output_folder / new_video_name

    # clear tmp (could have files if error on last run)
    shutil.rmtree(tmp_output_folder)
    os.mkdir(tmp_output_folder)

    # get how long it should take to finish playback
    game = slippi.Game(replay_filepath)
    num_frames = len(game.frames)
    FPS = 60
    duration_sec = num_frames / FPS

    # prep instructions for slippi dolphin to replay
    comm_filepath = generate_commfile(replay_filepath)

    # start dolphin, giving commfile (location) and other misc flags
    # for standard dolphin cmd line, see
    #  https://wiki.dolphin-emu.org/index.php?title=Help:Contents
    process = subprocess.Popen([
        str(executable_path),
        '-i', str(comm_filepath),
        '--hide-seekbar',
        '-e', str(iso_path),
        '-b',
        # '-l',
        # '-d',
        '--output-directory', str(tmp_output_folder),
    ])

    # I wish I could get a dolphin.on_start_playback and on_finish
    #   but this crude dircheck and sleep() will have to suffice

    # wait until video file is created, signaling start
    # print('waiting on dolphin start..')
    while not os.listdir(tmp_output_folder):
        pass
    # print('dolphin has started playing')

    # wait in realtime for replay to finish
    # 4:02 (+123 f) of game duration only made 3:41 of video with slowdown; guess my computer is not fast enough for realtime
    # 244 s made 221 s, so multiply by factor of 244/221 ~= 1.104
    sleep(slowdown_factor * duration_sec) # overshoot with extra time as buffer

    # close dolphin, which will finish recording
    process.terminate()

    # give it time to close so video file can be modified
    sleep(1)

    # abort if video is too short (ie didn't finish)
    if video_tools.num_frames(str(old_video_path)) < num_frames:
        raise Exception('Video did not finish recording; trying again with slowdown_factor + 0.1 ?')
        # return record_replay(replay_filepath, output_dir, slowdown_factor+0.1)
        return -1

    # prep new location
    if new_video_path.exists():
        os.remove(new_video_path)
    # move_file(old_video_path, new_video_path)

    # trim excess "waiting for game.." frames and move to final location
    video_tools.trim(0, num_frames, str(old_video_path), str(new_video_path))

    # alternative idea to implement:
    # use SlippstreamClient (libmelee)
    # read replay file and use libmelee to set up game
    #     (characters, costumes, stage, ports, [tags?])
    # listen for gameevent or menuevent in slippstream client or something
    #     to signal game start and game end
    #
    #     could I start and stop recording on demand?

def play_replay(replay_filepath):
    # mainly just for testing
    replay_filepath = Path(replay_filepath)
    dolphin_folder_path = Path(r'../dolphin')
    executable_path = dolphin_folder_path / 'Slippi Dolphin.exe'
    iso_path = read_json('../settings.json')['iso_path']

    comm_filepath = generate_commfile(replay_filepath)

    return subprocess.Popen([
        str(executable_path),
        '-i', str(comm_filepath),
        # '-m', str(replay_filepath),
        '--hide-seekbar',
        '-e', str(iso_path),
        '-b',
        # '-C', '<System>.Movie.DumpFrames=False'
        # '-C', 'Movie.DumpFrames=False'
        # '-l',
        # '-d',
        # '--output-directory', str(tmp_output_folder),
    ])

def generate_commfile(replay_filepath):
    # comm file is used to tell slippi-dolhpin how to play a replay
    # https://github.com/project-slippi/slippi-wiki/blob/master/COMM_SPEC.md
    # I feel these should be turned into command line flags for simplicity/consistency with slippi dolphin cmd line

    unique_id = rand_hex(3*4)   # n bytes
    comm_filepath = Path(f'../tmp/commfile_{unique_id}.json')
    write_json(comm_filepath, {
        # 'mode': 'normal',   # default normal, or mirror or spectate
        'replay': str(replay_filepath),
        'isRealTimeMode': True,       # seems to prevent slowdowns?
        'commandId': unique_id,
        'startFrame': -123,
        # 'endFrame': num_frames,
    })
    return comm_filepath

# def enhanced_slippi_dolphin_argparse():
#     # remove need for callers to create commfile (-i)
#     # add dolphin configurability
#
#     custom_cmds = {
#         '-r', 'replay filepath'
#         '--config', '[section].[key]=[value]'
#     }
#     args = argparse.args()
#
#     # if there are custom commands, do them
#     for cmd in args:
#         if cmd in custom_cmds:
#             perform(cmd)
#             args.remove(cmd)
#
#     # do the rest of normal slippi dolphin cmdline
#     return subprocess.Popen(args)

def write_json(fp, data):
    with open(fp, 'w') as file:
        json.dump(data, file)

def read_json(fp):
    with open(fp, 'r') as file:
        return json.load(file)

if __name__ == '__main__':
    # record_replay(r'../test/Game_20210408T225110.slp', slowdown_factor=1.25)
    play_replay(r'../test/Game_20210408T225110.slp')


from os import walk
from os.path import join
from os.path import isdir
import imghdr

def getPicturesByFolder(mainPath, folder):

  imageExtensionAccepted = ['jpeg', 'png']
  filesInFolder = []
  imagesInFolder = []

  fullFolder = join(mainPath,folder)
  for (dirpath, dirnames, filenames) in walk(fullFolder):
    #filesInFolder.extend([join(dirpath,f) for f in filenames])
    for filename in filenames:
      filesInFolder.append({'dir':dirpath, 'filename': filename})
    break # we stop the list at the first level of the directory (we don't look in subfolders)

  for myfile in filesInFolder:
    if imghdr.what(join(myfile['dir'],myfile['filename'])) in imageExtensionAccepted:
      imagesInFolder.append(myfile)

  return imagesInFolder

def checkFolderExistence(mainPath,innerFolder):
  fullFolder = join(mainPath,innerFolder)
  return isdir(fullFolder)


import importlib
import os
import re
import sys

from cpt.packager import ConanMultiPackager

spec = importlib.util.spec_from_file_location('comm', 'scripts/common.py')
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)

if __name__ == "__main__":
    projname = os.getenv('CONAN_PACKAGE_NAME')
    if not projname:
        raise Exception('CONAN_PACKAGE_NAME environment variable not defined')
    username = os.getenv('CONAN_USERNAME')
    if not username:
        raise Exception('CONAN_USERNAME environment variable not defined')
    branch = os.getenv('TRAVIS_BRANCH')
    if not branch:
        raise Exception('TRAVIS_BRANCH environment variable not defined (are you not releasing on Travis?)')

    matched, projver = mod.is_dev_branch()
    channel = 'nightly'
    if not matched:
        matched, projver = mod.is_rel_branch()
        channel = 'stable'

    if matched:
        reference = f'{projname}/{projver}@{username}/{channel}'

        builder = ConanMultiPackager(reference=reference)

        builder.add({}, {}, {}, {})
        builder.run()
    else:
        print('Not release or development branch, ignoring release!')


import json

import requests

from sdcclient._common import _SdcCommon

class EventsClientV2(_SdcCommon):
    def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True, custom_headers=None):
        super().__init__(token, sdc_url, ssl_verify, custom_headers)
        self.product = "SDC"

    def get_events(self, name=None, category=None, direction='before', status=None, limit=100, pivot=None):
        '''**Description**
            Returns the list of Sysdig Monitor events.

        **Arguments**
            - **name**: filter events by name. Default: None.
            - **category**: filter events by category. Default: ['alert', 'custom', 'docker', 'containerd', 'kubernetes'].
            - **direction**: orders the list of events. Valid values: "before", "after". Default: "before".
            - **status**: status of the event as list. Default: ['triggered', 'resolved', 'acknowledged', 'unacknowledged']
            - **limit**: max number of events to retrieve. Default: 100.
            - **pivot**: event id to use as pivot. Default: None.

        **Success Return Value**
            A dictionary containing the list of events.

        **Example**
            `examples/list_events.py <https://github.com/draios/python-sdc-client/blob/master/examples/list_events.py>`_
        '''
        valid_categories = ['alert', 'custom', 'docker', 'containerd', 'kubernetes']

        if category is None:
            category = valid_categories

        for c in category:
            if c not in valid_categories:
                return False, "Invalid category '{}'".format(c)

        valid_status = ["triggered", "resolved", "acknowledged", "unacknowledged"]
        if status is None:
            status = valid_status

        for s in status:
            if s not in valid_status:
                return False, "Invalid status '{}'".format(s)

        if direction not in ["before", "after"]:
            return False, "Invalid direction '{}', must be either 'before' or 'after'".format(direction)

        options = {
            'alertStatus': status,
            'category': ','.join(category),
            'dir': direction,
            'feed': 'true',
            'include_pivot': 'true',
            'include_total': 'true',
            'limit': str(limit),
            'pivot': pivot,
            'filter': name,
        }
        params = {k: v for k, v in options.items() if v is not None}
        res = requests.get(self.url + '/api/v2/events/', headers=self.hdrs, params=params, verify=self.ssl_verify)
        return self._request_result(res)

    def delete_event(self, event):
        '''**Description**
            Deletes an event.

        **Arguments**
            - **event**: the event object as returned by :func:`~SdcClient.get_events`.

        **Success Return Value**
            `None`.

        **Example**
            `examples/delete_event.py <https://github.com/draios/python-sdc-client/blob/master/examples/delete_event.py>`_
        '''
        if 'id' not in event:
            return [False, "Invalid event format"]

        res = requests.delete(self.url + '/api/v2/events/' + str(event['id']), headers=self.hdrs,
                              verify=self.ssl_verify)
        if not self._checkResponse(res):
            return [False, self.lasterr]
        return [True, None]

    def post_event(self, name, description=None, severity=None, event_filter=None, tags=None):
        '''**Description**
            Send an event to Sysdig Monitor. The events you post are available in the Events tab in the Sysdig Monitor UI and can be overlied to charts.

        **Arguments**
            - **name**: the name of the new event.
            - **description**: a longer description offering detailed information about the event.
            - **severity**: syslog style from 0 (high) to 7 (low).
            - **event_filter**: metadata, in Sysdig Monitor format, of nodes to associate with the event, e.g. ``host.hostName = 'ip-10-1-1-1' and container.name = 'foo'``.
            - **tags**: a list of key-value dictionaries that can be used to tag the event. Can be used for filtering/segmenting purposes in Sysdig Monitor.

        **Success Return Value**
            A dictionary describing the new event.

        **Examples**
            - `examples/post_event_simple.py <https://github.com/draios/python-sdc-client/blob/master/examples/post_event_simple.py>`_
            - `examples/post_event.py <https://github.com/draios/python-sdc-client/blob/master/examples/post_event.py>`_
        '''
        options = {
            'name': name,
            'description': description,
            'severity': severity,
            'filter': event_filter,
            'tags': tags
        }
        edata = {
            'event': {k: v for k, v in options.items() if v is not None}
        }
        res = requests.post(self.url + '/api/v2/events/', headers=self.hdrs, data=json.dumps(edata),
                            verify=self.ssl_verify)
        return self._request_result(res)


import argparse
import json
import logging
import os
import sys
from pathlib import Path
from typing import Dict

import mlflow
import pandas as pd
from omegaconf import DictConfig
from sklearn.model_selection import train_test_split

sys.path.append("..")
from common.custom_logger import CustomLogger  # noqa: E402
from common.omegaconf_manager import OmegaconfManager  # noqa: E402
from common.utils import get_mlflow_tag_from_input_parameters, split_input_parameters_str  # noqa: E402
from preprocess.src.extract_data import get_test_data_files, get_train_data_files  # noqa: E402
from preprocess.src.extract_dummy_data import get_dummy_data_files, get_meta_test_info  # noqa: E402

logging.basicConfig(
    level=logging.INFO,
)
logger = CustomLogger("Preprocess_Logger")

def main(cfg: DictConfig):
    """The main process of `preprocess` step.

    In `preprocess` step, split all datasets into 3 parts.
        1. Train (`meta_train.json`): The training dataset files with `train_dataset.csv`.
        2. Validation (`meta_valid.json`): The validation dataset files with `train_dataset.csv`.
        3. Test (`meta_test.json`): The test dataset files with `test_dataset.json`.

    Then, the 3 meta files are saved and logged as mlflow artifacts.
    """
    input_parameters = split_input_parameters_str(cfg.input_parameters)
    mlflow.set_tag("mlflow.runName", get_mlflow_tag_from_input_parameters(input_parameters) + "_preprcess")

    downstream_dir_path: str = cfg.preprocess.downstream_dir_path
    os.makedirs(downstream_dir_path, exist_ok=True)

    time_step_minutes = cfg.preprocess.time_step_minutes
    time_slides_delta = cfg.preprocess.time_slides_delta

    if cfg.use_dummy_data is True:
        logger.warning("... Using dummy data ...")
        data_files = get_dummy_data_files(
            input_parameters=input_parameters,
            time_step_minutes=time_step_minutes,
            downstream_dir_path=downstream_dir_path,
            dataset_length=100,
            input_seq_length=cfg.input_seq_length,
            label_seq_length=cfg.label_seq_length,
        )

        data_file_length = len(data_files)
        train_data_size = data_file_length // 2
        valid_test_data_size = data_file_length // 4
        train_data_files, valid_data_files, test_data_files = (
            data_files[:train_data_size],
            data_files[train_data_size : train_data_size + valid_test_data_size],  # noqa: E203
            data_files[
                train_data_size + valid_test_data_size : train_data_size + valid_test_data_size + 5
            ],  # noqa: E203
        )

    else:
        # train_dataset.csv comes from https://github.com/tsugumi-sys/poteka_data_analysis/blob/main/EDA/rain/rain_durations.ipynb
        current_dir = os.getcwd()
        train_list_df = pd.read_csv(os.path.join(current_dir, "src/train_dataset.csv"))
        train_data_files = get_train_data_files(
            project_root_dir_path=Path(cfg.pipeline_root_dir_path).parent,
            train_list_df=train_list_df,
            input_parameters=input_parameters,
            time_step_minutes=time_step_minutes,
            time_slides_delta=time_slides_delta,
            input_seq_length=cfg.input_seq_length,
            label_seq_length=cfg.label_seq_length,
        )
        train_data_files, valid_data_files = train_test_split(train_data_files, test_size=0.2, random_state=11)

        # test_dataset.json comes from https://github.com/tsugumi-sys/poteka_data_analysis/blob/main/EDA/rain/select_test_dataset.ipynb
        with open(os.path.join(current_dir, "src/test_dataset.json")) as f:
            test_data_list = json.load(f)
        test_data_files = get_test_data_files(
            project_root_dir_path=Path(cfg.pipeline_root_dir_path).parent,
            test_data_list=test_data_list,
            input_parameters=input_parameters,
            time_step_minutes=time_step_minutes,
            input_seq_length=cfg.input_seq_length,
            label_seq_length=cfg.label_seq_length,
        )

    meta_train = {"file_paths": train_data_files}
    meta_valid = {"file_paths": valid_data_files}
    meta_test = {
        "file_paths": test_data_files
        if isinstance(test_data_files, Dict)
        else get_meta_test_info(test_data_files, cfg.label_seq_length)
    }

    meta_train_filepath = os.path.join(
        downstream_dir_path,
        "meta_train.json",
    )
    meta_valid_filepath = os.path.join(
        downstream_dir_path,
        "meta_valid.json",
    )
    meta_test_filepath = os.path.join(
        downstream_dir_path,
        "meta_test.json",
    )

    with open(meta_train_filepath, "w") as f:
        json.dump(meta_train, f)
    with open(meta_valid_filepath, "w") as f:
        json.dump(meta_valid, f)
    with open(meta_test_filepath, "w") as f:
        json.dump(meta_test, f)

    for path in [meta_train_filepath, meta_valid_filepath, meta_test_filepath]:
        mlflow.log_artifact(path)
    logger.info("meta info files have saved")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Preprocess arguments")
    parser.add_argument("--hydra_file_path", type=str, help="Hydra configuration file saved in main.py.")
    args = parser.parse_args()
    omegaconf_manager = OmegaconfManager()
    hydra_conf = omegaconf_manager.load(args.hydra_file_path)
    main(hydra_conf)


import keras
from keras.models import *
from keras.layers import *
import tensorflow as tf
from .config import IMAGE_ORDERING

if IMAGE_ORDERING == 'channels_first':
    MERGE_AXIS = 1
elif IMAGE_ORDERING == 'channels_last':
    MERGE_AXIS = -1

def BR(input):
    output = BatchNormalization()(input)
    output = PReLU()(output)
    return output

def CDilated(input, filters, kernel, stride, dilate):
    output = Conv2D(filters= filters, kernel_size = kernel, strides = stride, dilation_rate = dilate, padding = 'same', data_format = IMAGE_ORDERING)(input)
    return output

def get_custom_encoder_v1(input_height=512,  input_width=512):

    if IMAGE_ORDERING == 'channels_first':
        img_input = Input(shape=(3, input_height, input_width))
    elif IMAGE_ORDERING == 'channels_last':
        img_input = Input(shape=(input_height, input_width, 3))

    levels = []
    x = Conv2D(filters = 32, kernel_size = 3, strides = 2, padding = 'same', data_format = IMAGE_ORDERING)(img_input)
    #print(f'Conv1_1: {x.shape}')
    x = CDilated(x, 32, 3, 1, 2)
    x = BR(x)
    #print(f'Conv1_2: {x.shape}')
    pool1 = AveragePooling2D(pool_size=(3,3), strides=2, padding='same')(img_input)
    #print(f'Pool1: {pool1.shape}')
    x = Concatenate(axis=MERGE_AXIS)([x, pool1])
    #print(f'Concat1: {x.shape}')
    levels.append(x)
    x = Conv2D(filters = 64, kernel_size = 3, strides = 2, padding = 'same', data_format = IMAGE_ORDERING)(x)
    #print(f'Conv2_1: {x.shape}')
    x = CDilated(x, 64, 3, 1, 4)
    x = BR(x)
    #print(f'Conv2_2: {x.shape}')
    pool2 = AveragePooling2D(pool_size=(3,3), strides=2, padding='same')(pool1)
    #print(f'Pool2: {pool2.shape}')
    x = Concatenate(axis=MERGE_AXIS)([x, pool2])
    #print(f'Concat2: {x.shape}')
    levels.append(x)
    x = Conv2D(filters = 128, kernel_size = 3, strides = 2, padding = 'same', data_format = IMAGE_ORDERING)(x)
    #print(f'Conv3_1: {x.shape}')
    x = CDilated(x, 128, 3, 1, 8)
    x = BR(x)
    #print(f'Conv3_2: {x.shape}')
    pool3 = AveragePooling2D(pool_size=(3,3), strides=2, padding='same')(pool2)
    #print(f'Pool3: {pool2.shape}')
    x = Concatenate(axis=MERGE_AXIS)([x, pool3])
    x = BR(x)
    #print(f'Concat3: {x.shape}')
    levels.append(x)
    x = Conv2D(filters = 256, kernel_size = 3, strides = 2, padding='same',data_format = IMAGE_ORDERING)(x)
    x = BR(x)
    #print(f'Conv4: {x.shape}')
    levels.append(x)
    return img_input, levels


import datetime
import sqlite3

from matplotlib import dates
from matplotlib import ticker
from matplotlib.backends.backend_gtk3cairo import FigureCanvasGTK3Cairo as FigureCanvas
from matplotlib.dates import MonthLocator, DateFormatter
from matplotlib.figure import Figure
import gi

from .preferencesTabacchi import prefs
from . import utility

gi.require_version('Gtk', '3.0')
from gi.repository import Gtk  # noqa: E402

# Classe minimale per la visualizzazione delle statistiche
class SimpleStatsDialog(utility.GladeWindow):
    def __init__(self, parent):
        super().__init__(parent, "statsDialog.glade")

        self.statsDialog = self.builder.get_object("statsDialog")
        self.statsNotebook = self.builder.get_object("statsNotebook")
        self.consumiScrolledWindow = self.builder.get_object("consumiScrolledWindow")
        self.pesiScrolledWindow = self.builder.get_object("pesiScrolledWindow")
        self.quantitaScrolledWindow = self.builder.get_object("quantitaScrolledWindow")
        self.piuAcquistatiTreeview = self.builder.get_object("piuAcquistatiTreeview")

        self.statsDialog.set_transient_for(parent)

        self.builder.connect_signals({"on_statsDialog_delete_event": self.close,
                                      "on_statsDialog_destroy_event": self.close,
                                      "on_closeButton_clicked": self.close
                                      })
        now = datetime.datetime.now().replace(microsecond=0)
        self.dataInizioEntry = utility.DataEntry(
            self.builder.get_object("dataInizioButton"),
            now - datetime.timedelta(days=365 * 2),
            " %d %B %Y ", self.refreshData)
        self.dataFineEntry = utility.DataEntry(self.builder.get_object("dataFineButton"), now, " %d %B %Y ", self.refreshData)

        self.figureCanvas = None

    def refreshData(self):
        self.loadData()
        self.showData()

    def initData(self):
        pass

    def loadData(self):
        pass

    def showData(self):
        pass

    def showPlot(self, scrolledWindow, date, valori, unitaMin=None):
        figure = Figure(figsize=None, dpi=None)
        ax = figure.add_subplot(111)

        ax.plot_date(dates.date2num(date), valori, 'o-')

        ax.xaxis.set_major_locator(MonthLocator())
        ax.xaxis.set_major_formatter(DateFormatter('%b %y'))
        if unitaMin:
            ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
            ax.yaxis.set_minor_locator(ticker.MultipleLocator(unitaMin))
        figure.autofmt_xdate()
        ax.autoscale_view()
        ax.grid(True)

        figureCanvas = FigureCanvas(figure)

        oldCanvas = scrolledWindow.get_child()
        if oldCanvas:
            scrolledWindow.remove(oldCanvas)

        scrolledWindow.add_with_viewport(figureCanvas)
        figureCanvas.show()

    def run(self):
        self.result = self.statsDialog.run()
        return self.result

    def close(self, widget, other=None):
        self.result = Gtk.ResponseType.CANCEL
        self.statsDialog.destroy()

class StatsDialog(SimpleStatsDialog):
    def __init__(self, parent, idPar):
        SimpleStatsDialog.__init__(self, parent)
        self.statsNotebook.remove_page(3)
        self.id = idPar
        self.initData()
        self.loadData()

        self.statsDialog.set_title(self.descrizione)
        self.showData()

    # Inizializzazione
    def initData(self):
        self.date = []
        self.consumi = []
        self.giacenze = []
        self.ordini = []
        self.unitaMin = None
        self.descrizione = None

    # Legge i dati
    def loadData(self):
        cursor = None
        conn = None
        try:
            conn = prefs.getConn()
            cursor = prefs.getCursor(conn)
            cursor.execute("SELECT t.Descrizione, t.UnitaMin FROM tabacchi t WHERE t.ID = ?", (self.id,))
            row = cursor.fetchone()
            self.descrizione = row['Descrizione']
            self.unitaMin = row['UnitaMin']
            dataFine = self.dataFineEntry.data.strftime("%Y-%m-%d")
            dataInizio = self.dataInizioEntry.data.strftime("%Y-%m-%d")
            cursor.execute(
                "SELECT r.Consumo, r.Giacenza, r.Ordine, o.Data as 'Data [timestamp]' FROM rigaOrdineTabacchi r, ordineTabacchi o where (r.ID_Ordine = o.ID) and (r.ID = ?) and (o.Data < ?) and (o.Data > ?) order by o.Data",
                (self.id, dataFine, dataInizio))
            result_set = cursor.fetchall()
            self.date[:] = []
            self.consumi[:] = []
            self.giacenze[:] = []
            self.ordini[:] = []
            oldOrdine = 0
            for row in result_set:
                data = row['Data']
                consumo = round(row['Consumo'], 3)
                ordine = round(row['Ordine'], 3)
                giacenza = round(row['Giacenza'], 3) - oldOrdine
                self.date.append(data)
                self.consumi.append(consumo if (consumo > 0) else 0)
                self.ordini.append(ordine if (ordine > 0) else 0)
                self.giacenze.append(giacenza if (giacenza > 0) else 0)
                oldOrdine = ordine
        except sqlite3.Error as e:
            utility.gtkErrorMsg(e, self.statsDialog)
        finally:
            if cursor:
                cursor.close()
            if conn:
                conn.close()

    def showData(self):
        self.showPlot(self.consumiScrolledWindow, self.date, self.consumi, self.unitaMin)
        self.showPlot(self.pesiScrolledWindow, self.date, self.ordini, self.unitaMin)
        self.showPlot(self.quantitaScrolledWindow, self.date, self.giacenze, self.unitaMin)

class GlobalStatsDialog(SimpleStatsDialog):
    modelInfoList = [("+Descrizione", "str"), ("^Totale acquisti", "float"), ("^Data ultimo acquisto", "date")]

    def __init__(self, parent):
        SimpleStatsDialog.__init__(self, parent)

        utility.ExtTreeView(self.modelInfoList, self.piuAcquistatiTreeview, formats={1: "%.3f kg"})
        self.piuAcquistatiModel = self.piuAcquistatiTreeview.get_model()
        self.piuAcquistatiTreeview.set_model(None)
        self.initData()
        self.loadData()
        self.piuAcquistatiTreeview.set_model(self.piuAcquistatiModel)

        self.statsDialog.set_title("Statistiche generali")

        self.showData()

    # Inizializzazione
    def initData(self):
        self.date = []
        self.consumi = []
        self.giacenze = []
        self.ordini = []

    # Legge i dati

    def loadData(self):
        cursor = None
        conn = None
        try:
            conn = prefs.getConn()
            cursor = prefs.getCursor(conn)
            dataFine = self.dataFineEntry.data.strftime("%Y-%m-%d")
            dataInizio = self.dataInizioEntry.data.strftime("%Y-%m-%d")
            cursor.execute(
                "SELECT t.Descrizione, SUM(r.Ordine) totPeso, MAX(o.Data) as 'maxData [timestamp]' FROM rigaOrdineTabacchi r, ordineTabacchi o, tabacchi t where (r.ID = t.ID) and t.InMagazzino and (r.ID_Ordine = o.ID) and (r.Ordine > 0) and (o.Data < ?) and (o.Data > ?) group by r.ID order by totPeso desc",
                (dataFine, dataInizio))
            result_set = cursor.fetchall()
            self.piuAcquistatiModel.clear()
            for row in result_set:
                self.piuAcquistatiModel.append([row["Descrizione"], row["totPeso"], row["maxData"]])
            cursor.execute(
                "SELECT SUM(r.Consumo) as Consumo, SUM(r.Giacenza) as Giacenza, SUM(r.Ordine) as Ordine, o.Data as 'Data [timestamp]' FROM rigaOrdineTabacchi r, ordineTabacchi o where (r.ID_Ordine = o.ID) and ((r.Consumo > 0) or (r.Giacenza > 0) or (r.Ordine > 0)) and (o.Data < ?) and (o.Data > ?) group by o.ID order by o.Data",
                (dataFine, dataInizio))
            result_set = cursor.fetchall()

            self.date[:] = []
            self.consumi[:] = []
            self.giacenze[:] = []
            self.ordini[:] = []
            # oldOrdine = 0
            for row in result_set:
                data = row['Data']
                consumo = round(row['Consumo'], 3)
                ordine = round(row['Ordine'], 3)
                esistenza = round(row['Giacenza'], 3)
                giacenza = esistenza  # - oldOrdine

                self.date.append(data)
                self.consumi.append(consumo if (consumo > 0) else 0)
                self.ordini.append(ordine if (ordine > 0) else 0)
                self.giacenze.append(giacenza if (giacenza > 0) else 0)
                # oldOrdine = ordine
        except sqlite3.Error as e:
            utility.gtkErrorMsg(e, self.statsDialog)
        finally:
            if cursor:
                cursor.close()
            if conn:
                conn.close()

    def showData(self):
        self.showPlot(self.consumiScrolledWindow, self.date, self.consumi)
        self.showPlot(self.pesiScrolledWindow, self.date, self.ordini)
        self.showPlot(self.quantitaScrolledWindow, self.date, self.giacenze)


import inspect

import numpy as np

from hypothesis import given, example, settings

from pytest import raises, warns

from ..ndindex import ndindex, asshape
from ..integer import Integer
from ..ellipsis import ellipsis
from ..integerarray import IntegerArray
from ..tuple import Tuple
from .helpers import ndindices, check_same, assert_equal

@given(ndindices)
def test_eq(idx):
    index = ndindex(idx)
    new = type(index)(*index.args)
    assert (new == index) is True
    try:
        if isinstance(new.raw, np.ndarray):
            raise ValueError
        assert (new.raw == index.raw) is True
        assert (index.raw == index) is True
    except ValueError:
        np.testing.assert_equal(new.raw, index.raw)
        # Sadly, there is now way to bypass array.__eq__ from producing an
        # array.
    assert hash(new) == hash(index)
    assert (index == index.raw) is True
    assert (index == 'a') is False
    assert ('a' == index) is False
    assert (index != 'a') is True
    assert ('a' != index) is True

def test_eq_explicit():
    assert Integer(0) != False
    assert Integer(1) != True
    assert Integer(0) != IntegerArray(0)

@example((np.array([1, 2]), 0))
@example([1, 2, 3])
@given(ndindices)
def test_ndindex(idx):
    index = ndindex(idx)
    assert index == idx
    def test_raw_eq(idx, index):
        if isinstance(idx, np.ndarray):
            assert_equal(index.raw, idx)
        elif isinstance(idx, list):
            assert index.dtype in [np.intp, np.bool_]
            assert_equal(index.raw, np.asarray(idx, dtype=index.dtype))
        elif isinstance(idx, tuple):
            assert type(index.raw) == type(idx)
            assert len(index.raw) == len(idx)
            assert index.args == index.raw
            for i, j in zip(idx, index.args):
                test_raw_eq(i, j)
        else:
            assert index.raw == idx
    test_raw_eq(idx, index)
    assert ndindex(index.raw) == index

def test_ndindex_invalid():
    a = np.arange(10)
    for idx in [1.0, [1.0], np.array([1.0]), np.array([1], dtype=object),
                np.array([])]:
        check_same(a, idx)

    # This index is allowed by NumPy, but gives a deprecation warnings. We are
    # not going to allow indices that give deprecation warnings in ndindex.
    with warns(None) as r: # Make sure no warnings are emitted from ndindex()
        raises(IndexError, lambda: ndindex([1, []]))
    assert not r

def test_ndindex_ellipsis():
    raises(IndexError, lambda: ndindex(ellipsis))

def test_signature():
    sig = inspect.signature(Integer)
    assert sig.parameters.keys() == {'idx'}

@example(([0, 1],))
@example((IntegerArray([], (0, 1)),))
@example(IntegerArray([], (0, 1)))
@example((1, ..., slice(1, 2)))
# eval can sometimes be slower than the default deadline of 200ms for large
# array indices
@settings(deadline=None)
@given(ndindices)
def test_repr_str(idx):
    # The repr form should be re-creatable
    index = ndindex(idx)
    d = {}
    exec("from ndindex import *", d)
    assert eval(repr(index), d) == idx

    # Str may not be re-creatable. Just test that it doesn't give an exception.
    str(index)

def test_asshape():
    assert asshape(1) == (1,)
    assert asshape(np.int64(2)) == (2,)
    assert type(asshape(np.int64(2))[0]) == int
    assert asshape((1, 2)) == (1, 2)
    assert asshape([1, 2]) == (1, 2)
    assert asshape((np.int64(1), np.int64(2))) == (1, 2)
    assert type(asshape((np.int64(1), np.int64(2)))[0]) == int
    assert type(asshape((np.int64(1), np.int64(2)))[1]) == int

    raises(TypeError, lambda: asshape(1.0))
    raises(TypeError, lambda: asshape((1.0,)))
    raises(ValueError, lambda: asshape(-1))
    raises(ValueError, lambda: asshape((1, -1)))
    raises(TypeError, lambda: asshape(...))
    raises(TypeError, lambda: asshape(Integer(1)))
    raises(TypeError, lambda: asshape(Tuple(1, 2)))
    raises(TypeError, lambda: asshape((True,)))


import pymysql
from datetime import datetime
from db import Db, WriteErrorToLog, log_file
from db_create import QueryCreateRecipe, QueryCreateMethodItems

####################### CLASS FOR SQL QUERY TO DELETE RECIPE BASED ON RECIPE ID #########################

class QueryDeleteRecipe():

    def __init__(self, recipe_id, table):
        self.recipe_id = recipe_id
        self.table = table

    def delete(self):
        try:
            with Db(commit=True) as cursor:
                cursor.execute(""" DELETE FROM %s
                                    WHERE RecipeId = %s;
                                """ % (self.table, self.recipe_id))
        except pymysql.err.OperationalError as e:
            message = " FAILED: delete method in db_read.QueryDeleteRecipes."
            log = WriteErrorToLog(str(e), message, log_file, str(datetime.now()))
            log.write_to_doc()
        else:
            print("delete completed")

    def delete_saved_recipe(self, user_id):
        try:
            with Db(commit=True) as cursor:
                cursor.execute(""" DELETE FROM %s
                                    WHERE UserId = %s
                                    AND RecipeId = %s;""" % (self.table, user_id, self.recipe_id))
        except pymysql.err.OperationalError as e:
            message = " FAILED: delete_saved_recipe in db_read.QueryDeleteRecipes."
            log = WriteErrorToLog(str(e), message, log_file, str(datetime.now()))
            log.write_to_doc()
        else:
            print("delete_saved_recipe completed")

############################### CLASS FOR SQL UPDATE QUERIES ############################################
#########################################################################################################

class UpdateQuery():

    recipe_query = """ UPDATE Recipe SET
                        RecipeTitle = %s,
                        RecipeDescription = %s,
                        CookingTimeMins = %s,
                        MakePublic = %s
                        WHERE RecipeId = %s;
                    """

    def single_column_query(self, table, column, column_value, recipe_id):
        query = """ UPDATE %s SET
                    %s = '%s'
                    WHERE RecipeId = %s;
                """ % (table, column, column_value, recipe_id)
        return query

###################### CLASS FOR EXECUTING UPDATE QUERIES TO RECIPE TABLE AND STATS TABLES ###############

class QueryUpdateRecipe(QueryCreateRecipe):

    def __init__(self, recipe, user_id, recipe_id):
        super().__init__(recipe, user_id)
        self.recipe_id = recipe_id

    def update_recipe(self):
        recipe_values = (self.recipe_title, self.recipe_description,
                            self.cooking_time, self.make_public, self.recipe_id)
        try:
            with Db(commit=True) as cursor:
                cursor.execute(UpdateQuery.recipe_query, recipe_values)
        except pymysql.err.OperationalError as e:
            message = " FAILED: update_recipe in db_read.QueryUpdateRecipes."
            log = WriteErrorToLog(str(e), message, log_file, str(datetime.now()))
            log.write_to_doc()
        else:
            print("update_recipe completed")

    def update_stats(self):
        stat_queries = [[UpdateQuery.single_column_query(self,'Cuisine','CuisineName',
                                                        self.cuisine_name, self.recipe_id)],
                    [UpdateQuery.single_column_query(self,'Course','CourseName',
                                                        self.course_name, self.recipe_id)],
                    [UpdateQuery.single_column_query(self,'Health','Calories',
                                                        self.calories, self.recipe_id)],
                    [UpdateQuery.single_column_query(self,'Cost','Price',
                                                         self.cost, self.recipe_id)],
                    [UpdateQuery.single_column_query(self,'Servings','Servings',
                                                        self.servings, self.recipe_id)]]
        try:
            with Db(commit=True) as cursor:
                for stat in stat_queries:
                    cursor.execute(stat[0])
        except pymysql.err.OperationalError as e:
            message = " FAILED: update_recipe in db_read.QueryUpdateRecipes."
            log = WriteErrorToLog(str(e), message, log_file, str(datetime.now()))
            log.write_to_doc()
        else:
            print("update_recipe completed")


import asyncio
import json
import logging
import platform
import threading
import time
import urllib.parse
from contextlib import contextmanager
from functools import lru_cache
from typing import Any
from typing import Dict
from typing import List

import aiohttp
import pydantic
import requests
from lib.core.exceptions import AppException
from lib.core.service_types import ServiceResponse
from lib.core.serviceproviders import BaseClient
from requests.adapters import HTTPAdapter
from requests.adapters import Retry
from superannotate import __version__

logger = logging.getLogger("sa")

class PydanticEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, pydantic.BaseModel):
            return json.loads(obj.json(exclude_none=True))
        return json.JSONEncoder.default(self, obj)

class HttpClient(BaseClient):
    AUTH_TYPE = "sdk"

    def __init__(self, api_url: str, token: str, verify_ssl: bool = True):
        super().__init__(api_url, token)
        self._verify_ssl = verify_ssl

    @lru_cache(maxsize=32)
    def _get_session(self, thread_id, ttl=None):  # noqa
        del ttl
        del thread_id
        retries = Retry(total=3, backoff_factor=0.1, status_forcelist=[502, 503, 504])

        session = requests.Session()
        session.mount("http://", HTTPAdapter(max_retries=retries))  # noqa
        session.mount("https://", HTTPAdapter(max_retries=retries))
        session.headers.update(self.default_headers)
        return session

    def get_session(self):
        return self._get_session(
            thread_id=threading.get_ident(), ttl=round(time.time() / 360)
        )

    @property
    def default_headers(self):
        return {
            "Authorization": self._token,
            "authtype": self.AUTH_TYPE,
            "Content-Type": "application/json",
            "User-Agent": f"Python-SDK-Version: {__version__}; Python: {platform.python_version()};"
            f"OS: {platform.system()}; Team: {self.team_id}",
        }

    @property
    def safe_api(self):
        """
        Context manager which will handle requests calls.
        """

        @contextmanager
        def safe_api():
            """
            Context manager which handles Requests error.
            """
            try:
                yield None
            except (requests.RequestException, ConnectionError) as exc:
                raise AppException(f"Unknown exception: {exc}.")

        return safe_api

    def _request(self, url, method, session, retried=0, **kwargs):

        with self.safe_api():
            req = requests.Request(
                method=method,
                url=url,
                **kwargs,
            )
            prepared = session.prepare_request(req)
            response = session.send(request=prepared, verify=self._verify_ssl)

        if response.status_code == 404 and retried < 3:
            time.sleep(retried * 0.1)
            return self._request(
                url, method=method, session=session, retried=retried + 1, **kwargs
            )
        if response.status_code > 299:
            logger.debug(
                f"Got {response.status_code} response from backend: {response.text}"
            )
        return response

    def _get_url(self, url):
        if not url.startswith("htt"):
            return urllib.parse.urljoin(self._api_url, url)
        return url

    def request(
        self,
        url,
        method="get",
        data=None,
        headers=None,
        params=None,
        retried=0,
        content_type=ServiceResponse,
        files=None,
        dispatcher: str = None,
    ) -> ServiceResponse:
        _url = self._get_url(url)
        kwargs = {"params": {"team_id": self.team_id}}
        if data:
            kwargs["data"] = json.dumps(data, cls=PydanticEncoder)
        if params:
            kwargs["params"].update(params)
        session = self.get_session()
        if files and session.headers.get("Content-Type"):
            del session.headers["Content-Type"]
        session.headers.update(headers if headers else {})
        response = self._request(_url, method, session=session, retried=0, **kwargs)
        if files:
            session.headers.update(self.default_headers)
        return self.serialize_response(response, content_type, dispatcher)

    def paginate(
        self,
        url: str,
        item_type: Any = None,
        chunk_size: int = 2000,
        query_params: Dict[str, Any] = None,
    ) -> ServiceResponse:
        offset = 0
        total = []
        splitter = "&" if "?" in url else "?"

        while True:
            _url = f"{url}{splitter}offset={offset}"
            _response = self.request(
                _url, method="get", params=query_params, dispatcher="data"
            )
            if _response.ok:
                if _response.data:
                    total.extend(_response.data)
                else:
                    break
                data_len = len(_response.data)
                offset += data_len
                if data_len < chunk_size or _response.count - offset < 0:
                    break
            else:
                break

        if item_type:
            response = ServiceResponse(
                status=_response.status,
                res_data=pydantic.parse_obj_as(List[item_type], total),
            )
        else:
            response = ServiceResponse(
                status=_response.status,
                res_data=total,
            )
        if not _response.ok:
            response.set_error(_response.error)
            response.status = _response.status
        return response

    @staticmethod
    def serialize_response(
        response: requests.Response, content_type, dispatcher: str = None
    ) -> ServiceResponse:
        data = {
            "status": response.status_code,
        }
        try:
            if not response.ok:
                if response.status_code in (502, 504):
                    data[
                        "_error"
                    ] = "Our service is currently unavailable, please try again later."
                    return content_type(**data)
                else:
                    data_json = response.json()
                    data["res_error"] = data_json.get(
                        "error", data_json.get("errors", "Unknown Error")
                    )
                    return content_type(**data)
            data_json = response.json()
            if dispatcher:
                if dispatcher in data_json:
                    data["res_data"] = data_json.pop(dispatcher)
                else:
                    data["res_data"] = data_json
                    data_json = {}
                data.update(data_json)
            else:
                data["res_data"] = data_json
            return content_type(**data)
        except json.decoder.JSONDecodeError:
            data["res_error"] = response.content
            data["reason"] = response.reason
            return content_type(**data)

class AIOHttpSession(aiohttp.ClientSession):
    RETRY_STATUS_CODES = [401, 403, 502, 503, 504]
    RETRY_LIMIT = 3
    BACKOFF_FACTOR = 0.3

    async def request(self, *args, **kwargs) -> aiohttp.ClientResponse:
        attempts = self.RETRY_LIMIT
        delay = 0
        for _ in range(attempts):
            delay += self.BACKOFF_FACTOR
            attempts -= 1
            response = await super()._request(*args, **kwargs)
            if response.status not in self.RETRY_STATUS_CODES or not attempts:
                return response
            await asyncio.sleep(delay)


from __future__ import annotations

from typing import cast, Union, TypeVar, Optional, Protocol
from collections.abc import Iterable

from time import perf_counter
import logging

from .utils import isclose

ObjectiveBound = Union[float, int]

Objective = TypeVar('Objective', bound=ObjectiveBound, covariant=True)
LocalMove = TypeVar('LocalMove')

class SolutionProtocol(Protocol[Objective, LocalMove]):
    def objective(self) -> Objective: ...
    def copy(self: Solution) -> Solution: ...
    def random_local_moves_wor(self) -> Iterable[LocalMove]: ...
    def objective_incr_local(self, lmove: LocalMove) -> Optional[Objective]: ...
    def step(self, lmove: LocalMove) -> None: ...
    def perturb(self, ks: int) -> None: ...

Solution = TypeVar('Solution', bound=SolutionProtocol)

def ils(solution: Solution, budget: float, ks: int = 3) -> Solution:
    start = perf_counter()
    best = solution.copy()
    best_obj = cast(ObjectiveBound, best.objective())
    while perf_counter() - start < budget:
        # for move in solution.enumLocalMove():
        for move in solution.random_local_moves_wor():
            incr = cast(ObjectiveBound, solution.objective_incr_local(move))
            if incr < 0 and not isclose(incr, 0):
                solution.step(move)
                break
            if perf_counter() - start >= budget:
                obj = cast(ObjectiveBound, solution.objective())
                if obj < best_obj:
                    return solution
                else:
                    return best
        else:
            # Local optimum found
            obj = cast(ObjectiveBound, solution.objective())
            if obj < best_obj or isclose(obj, best_obj):
                best = solution.copy()
                best_obj = obj
                logging.info(f"New best solution found: {best_obj}")
            else:
                solution = best.copy()
            solution.perturb(ks)
    obj = cast(ObjectiveBound, solution.objective())
    if obj < best_obj:
        return solution
    else:
        return best


"""Split domain IDs into training/validation/test subsets."""

import os
import logging

from tfold_se3.utils import tfold_init
from tfold_se3.utils import get_md5sum

def export_chain_ids(chain_ids, path):
    """Export chain IDs to a plain-text file."""

    os.makedirs(os.path.dirname(os.path.realpath(path)), exist_ok=True)
    with open(path, 'w') as o_file:
        o_file.write('\n'.join(sorted(chain_ids)) + '\n')

def main():
    """Main entry."""

    # load configurations from the YAML file
    curr_dir = os.path.dirname(os.path.realpath(__file__))
    cfg_fpath = os.path.join(curr_dir, 'config_gpu241.yaml')
    config = tfold_init(cfg_fpath)

    # get chain IDs
    with open(config['cid_fpath_all'], 'r') as i_file:
        chain_ids = [i_line.strip() for i_line in i_file]

    # split domain IDs into training/validation/test subsets
    n_chains = len(chain_ids)
    chain_ids.sort(key=get_md5sum)
    n_chains_trn = int(n_chains * config['ratio_trn'])
    n_chains_val = int(n_chains * config['ratio_val'])
    n_chains_tst = n_chains - n_chains_trn - n_chains_val
    logging.info('# of chains: %d (trn) / %d (val) / %d (tst)',
                 n_chains_trn, n_chains_val, n_chains_tst)
    chain_ids_trn = chain_ids[:n_chains_trn]
    chain_ids_val = chain_ids[n_chains_trn:-n_chains_tst]
    chain_ids_tst = chain_ids[-n_chains_tst:]

    # export domain IDs belonging to each subset to a plain-text file
    export_chain_ids(chain_ids_trn, config['cid_fpath_trn'])
    export_chain_ids(chain_ids_val, config['cid_fpath_val'])
    export_chain_ids(chain_ids_tst, config['cid_fpath_tst'])

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
# author: zhouwei
# description: 仅适用于通过key连接，如果需要密码连接，需要修改本脚本的内容
import paramiko as paramiko

class SSHClient:
    def __init__(self, host, port, user, pkey):
        self.ssh_host = host
        self.ssh_port = port
        self.ssh_user = user
        self.private_key = paramiko.RSAKey.from_private_key_file(pkey)
        self.ssh = None
        self._connect()

    def _connect(self):
        self.ssh = paramiko.SSHClient()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        try:
            self.ssh.connect(hostname=self.ssh_host, port=self.ssh_port, username=self.ssh_user, pkey=self.private_key,
                             timeout=10)
        except:
            return 'ssh connect fail'

    def execute_command(self, command):
        stdin, stdout, stderr = self.ssh.exec_command(command)
        out = stdout.read()
        err = stderr.read()
        return out, err

    def close(self):
        self.ssh.close()