from __future__ import annotations

from unittest import mock

from alibabacloud_adb20211201.models import (
    GetSparkAppLogResponse,
    GetSparkAppLogResponseBody,
    GetSparkAppLogResponseBodyData,
    GetSparkAppStateResponse,
    GetSparkAppStateResponseBody,
    GetSparkAppStateResponseBodyData,
    GetSparkAppWebUiAddressResponse,
    GetSparkAppWebUiAddressResponseBody,
    GetSparkAppWebUiAddressResponseBodyData,
    KillSparkAppResponse,
    SubmitSparkAppResponse,
)

from airflow.providers.alibaba.cloud.hooks.analyticdb_spark import AnalyticDBSparkHook
from tests.providers.alibaba.cloud.utils.analyticdb_spark_mock import mock_adb_spark_hook_default_project_id

ADB_SPARK_STRING = "airflow.providers.alibaba.cloud.hooks.analyticdb_spark.{}"
MOCK_ADB_SPARK_CONN_ID = "mock_id"
MOCK_ADB_CLUSTER_ID = "mock_adb_cluster_id"
MOCK_ADB_RG_NAME = "mock_adb_rg_name"
MOCK_ADB_SPARK_ID = "mock_adb_spark_id"

class TestAnalyticDBSparkHook:
    def setup_method(self):
        with mock.patch(
            ADB_SPARK_STRING.format("AnalyticDBSparkHook.__init__"),
            new=mock_adb_spark_hook_default_project_id,
        ):
            self.hook = AnalyticDBSparkHook(adb_spark_conn_id=MOCK_ADB_SPARK_CONN_ID)

    def test_build_submit_app_data(self):
        """Test build submit application data for analyticDB spark as expected."""
        res_data = self.hook.build_submit_app_data(
            file="oss://test_file",
            class_name="com.aliyun.spark.SparkPi",
            args=[1000, "test-args"],
            conf={"spark.executor.instances": 1, "spark.eventLog.enabled": "true"},
            jars=["oss://1.jar", "oss://2.jar"],
            py_files=["oss://1.py", "oss://2.py"],
            files=["oss://1.file", "oss://2.file"],
            driver_resource_spec="medium",
            executor_resource_spec="medium",
            num_executors=2,
            archives=["oss://1.zip", "oss://2.zip"],
            name="test",
        )
        except_data = {
            "file": "oss://test_file",
            "className": "com.aliyun.spark.SparkPi",
            "args": ["1000", "test-args"],
            "conf": {
                "spark.executor.instances": 1,
                "spark.eventLog.enabled": "true",
                "spark.driver.resourceSpec": "medium",
                "spark.executor.resourceSpec": "medium",
            },
            "jars": ["oss://1.jar", "oss://2.jar"],
            "pyFiles": ["oss://1.py", "oss://2.py"],
            "files": ["oss://1.file", "oss://2.file"],
            "archives": ["oss://1.zip", "oss://2.zip"],
            "name": "test",
        }
        assert res_data == except_data

    def test_build_submit_sql_data(self):
        """Test build submit sql data for analyticDB spark as expected."""
        res_data = self.hook.build_submit_sql_data(
            sql="""
            set spark.executor.instances=1;
            show databases;
            """,
            conf={"spark.executor.instances": 2},
            driver_resource_spec="medium",
            executor_resource_spec="medium",
            num_executors=3,
            name="test",
        )
        except_data = (
            "set spark.driver.resourceSpec = medium;set spark.executor.resourceSpec = medium;set "
            "spark.executor.instances = 2;set spark.app.name = test;\n            set "
            "spark.executor.instances=1;\n            show databases;"
        )
        assert res_data == except_data

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_submit_spark_app(self, mock_service):
        """Test submit_spark_app function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.submit_spark_app
        exists_method.return_value = SubmitSparkAppResponse(status_code=200)

        # When
        res = self.hook.submit_spark_app(MOCK_ADB_CLUSTER_ID, MOCK_ADB_RG_NAME, "oss://test.py")

        # Then
        assert isinstance(res, SubmitSparkAppResponse)
        mock_service.assert_called_once_with()

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_submit_spark_sql(self, mock_service):
        """Test submit_spark_app function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.submit_spark_app
        exists_method.return_value = SubmitSparkAppResponse(status_code=200)

        # When
        res = self.hook.submit_spark_sql(MOCK_ADB_CLUSTER_ID, MOCK_ADB_RG_NAME, "SELECT 1")

        # Then
        assert isinstance(res, SubmitSparkAppResponse)
        mock_service.assert_called_once_with()

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_get_spark_state(self, mock_service):
        """Test get_spark_state function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.get_spark_app_state
        exists_method.return_value = GetSparkAppStateResponse(
            body=GetSparkAppStateResponseBody(data=GetSparkAppStateResponseBodyData(state="RUNNING"))
        )

        # When
        res = self.hook.get_spark_state(MOCK_ADB_SPARK_ID)

        # Then
        assert res == "RUNNING"
        mock_service.assert_called_once_with()

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_get_spark_web_ui_address(self, mock_service):
        """Test get_spark_web_ui_address function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.get_spark_app_web_ui_address
        exists_method.return_value = GetSparkAppWebUiAddressResponse(
            body=GetSparkAppWebUiAddressResponseBody(
                data=GetSparkAppWebUiAddressResponseBodyData(web_ui_address="https://mock-web-ui-address.com")
            )
        )

        # When
        res = self.hook.get_spark_web_ui_address(MOCK_ADB_SPARK_ID)

        # Then
        assert res == "https://mock-web-ui-address.com"
        mock_service.assert_called_once_with()

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_get_spark_log(self, mock_service):
        """Test get_spark_log function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.get_spark_app_log
        exists_method.return_value = GetSparkAppLogResponse(
            body=GetSparkAppLogResponseBody(data=GetSparkAppLogResponseBodyData(log_content="Pi is 3.14"))
        )

        # When
        res = self.hook.get_spark_log(MOCK_ADB_SPARK_ID)

        # Then
        assert res == "Pi is 3.14"
        mock_service.assert_called_once_with()

    @mock.patch(ADB_SPARK_STRING.format("AnalyticDBSparkHook.get_adb_spark_client"))
    def test_kill_spark_app(self, mock_service):
        """Test kill_spark_app function works as expected."""
        # Given
        mock_client = mock_service.return_value
        exists_method = mock_client.kill_spark_app
        exists_method.return_value = KillSparkAppResponse()

        # When
        self.hook.kill_spark_app(MOCK_ADB_SPARK_ID)

        # Then
        mock_service.assert_called_once_with()


from flask import Flask, redirect, url_for, request, render_template, jsonify
from pymongo import MongoClient

import numpy as np
import json

app = Flask(__name__)

client = MongoClient()
db = client.hackerton

PROFESSOR_NUMBER = 258
WORD_NUMBER = 1000

# main 화면을 보여준 친구
@app.route('/')
def index():
    return render_template('index.html')

@app.route('/result', methods=['GET','POST'])
def result():
    # 검색, 쿼리를 보내고
    query = request.args.get('query')
    # 검색 화면에 따라서 달라지는 교수님인가 키워드인가
    type = request.args.get('type')
    # render_template 페이지화면을 호출한다, 쿼리와 타입을 포함하여
    return render_template('result.html', query=query, type=type)

# 검색 쿼리에서 get으로 얻은 이름을, 몽고디비에서 조회시킨다. 교수님을 클릭할때, 이 친구들이 호출된다.
@app.route('/professor')
def professor():
    name = request.args.get('name')
    doc = db.finaldb.find_one({'Name':name})
    doc.pop('_id')
    return jsonify(doc)

# 시각화 데이터를 가져온다. 검색하면 쿼리랑 타입을 가져와서, result matrix를 호출한다.
@app.route('/data')
def get_data():
    query = request.args.get('query')
    type = request.args.get('type')
    # 메트릭스 코드
    res_mat = np.random.rand(PROFESSOR_NUMBER+WORD_NUMBER,PROFESSOR_NUMBER+WORD_NUMBER)

    return jsonify(getJSON(query, type, res_mat))

# 메트릭스 코드를 이제 json으로 변환시킨다, threshold=0.92 이상만 보여준다.
def getJSON(query, query_type, res_mat, threshold=0.92):

    # 1200개 짜리 label이다.
    label_map = db.finaldb.distinct("Name") + db.wordname.distinct("wordname")[:1000]
    for i in range(PROFESSOR_NUMBER):
        label_map.append('professor'+str(i))
    for i in range(WORD_NUMBER):
        label_map.append('word'+str(i))

    # node 교수님, link similarity 값
    res_json={
        "nodes": [],
        "links": []
    }

    # 학과별로 컬러를 맵핑시킨다.
    color_map = {'Graduate School of Technology and Innovation Management(기술경영전문대학원)':'#cd7fb3', 'School of Business Administration(경영학부)':'#cd7fb3', 'School of Electrical and Computer Engineering(전기전자컴퓨터공학부)':'#ea4f6e', 'School of Energy and Chemical Engineering(에너지및화학공학부)':'#52a6b1', 'School of Management Engineering(경영공학부)':'#cd7fb3', 'School of Materials Science and Engineering(신소재공학부)':'#52a6b1', 'School of Mechanical, Aerospace and Nuclear Engineering(기계항공및원자력공학부)':'#52a6b1', 'Graduate School of Creative Design Engineering(디자인-공학 융합전문대학원)':'#f6c431', 'School of Design and Human Engineering(디자인 및 인간공학부)':'#f6c431', 'School of Life Sciences(생명과학부)':'#afcb05', 'School of Natural Science(자연과학부)':'#c4c4c4', 'School of Urban and Environmental Engineering(도시환경공학부)':'#00963f'}
    # 만약에 교수님 쿼리가 들어온다면
    print(res_mat)
    print(query_type)

    if query_type == "professor":
        # 매트릭스를 조회하야
        for idx, i in enumerate(res_mat):
            print(label_map[idx], query)
            # label_map[idx] 이름들의 배열에서, 쿼리와 같은걸 찾는다 "검색한 교수님이름과 같은 배열을 찾는다"
            if label_map[idx] == query:
                # 타겟 : 프로페서들 similarity만을 조회한다.
                target = i[:PROFESSOR_NUMBER]
                # 프로페서중에서 유사도가 top 50인 교수님들을 뽑는다, 그리고 자기자신을 맨 마지막으로 보낸다. 맨뒤에각 맨위로 오더라
                top_50 = sorted(range(len(target)), key=lambda k: target[k])[-49:]
                print(top_50)
                # 50개 중에 자기자신이 없으면 자기자신을 넣는다
                if idx not in top_50:
                    top_50.append(idx)
                # 자기자신이 있으면 뻇다가 마지막에 넣는다
                else:
                    top_50.append(top_50.pop(top_50.index(idx)))

        # 50명의 유사도가 높은 교수님의 배열에서 위치값을 저장한게 top_50
        for jdx, j in enumerate(top_50):

            # 확인차 한번 뽑자
            print(db.finaldb.find_one({"Name":label_map[j]})['department'])

            # node에 해당하는 json itme을 넣는다, id, name은 교수님 이름, chapter : 1, color : 학부
            res_json["nodes"].append({
                "id":label_map[j],
                "chapters": [
                    "1"
                ],
                "name":label_map[j],
                "color":color_map[db.finaldb.find_one({"Name":label_map[j]})['department']]
            })

        # 50개의 유사한 교수님들 위치값을 호출해서
        for i in top_50:
            # 모든 교수님과 키워드에 대한 similarity값을 호출해서
            target = res_mat[i][:PROFESSOR_NUMBER]
            # 각자의 simliarity값을 source랑 target에 맞춰서 넣어준다.
            for jdx, j in enumerate(target):
                if j > threshold and jdx in top_50:
                    res_json["links"].append({
                          "source":label_map[i],
                          "target":label_map[jdx],
                          "similarity":j
                        })
    # keyword로 쿼리가 들어간다면
    else:
        for idx, i in enumerate(res_mat):
            if label_map[idx] == query:
                print(i)
                target = i[:PROFESSOR_NUMBER]
                top_50 = sorted(range(len(target)), key=lambda k: target[k])[-49:]
                if idx not in top_50:
                    top_50.append(idx)
                else:
                    top_50.append(top_50.pop(top_50.index(idx)))

        for jdx, j in enumerate(top_50):
            print(db.finaldb.find_one({"Name":label_map[j]})['department'])
            res_json["nodes"].append({
                "id":label_map[j],
                "chapters": [
                    "1"
                ],
                "name":label_map[j],
                "color":color_map[db.finaldb.find_one({"Name":label_map[j]})['department']]
            })

        for i in top_50:
            target = res_mat[i][:PROFESSOR_NUMBER]
            for jdx, j in enumerate(target):
                if j > threshold and jdx in top_50:
                    res_json["links"].append({
                          "source":label_map[i],
                          "target":label_map[jdx],
                          "similarity":j
                        })
    return res_json

# 플라스크 서버세팅
if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
import re

import scrapy
from PyBots.items import JobItem

class ShixianSpider(scrapy.Spider):
    name = "shixian"
    domain = 'https://www.shixian.com'
    headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Encoding':'gzip, deflate, sdch, br',
            'Accept-Language':'zh,en-US;q=0.8,en;q=0.6,zh-CN;q=0.4,zh-TW;q=0.2',
            'Cache-Control':'max-age=0',
            'Connection':'keep-alive',
            'Host':'shixian.com',
            'Upgrade-Insecure-Requests':'1',
    }

    def start_requests(self):
        urls = ['https://shixian.com/jobs']
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        urls = self.parse_index(response)
        for url in urls:
            yield scrapy.Request(
                url=url,
                callback=self.parse_detail,
                headers=self.headers
            )

    def parse_index(self, response):
        urls = []
        job_list = response.xpath('//div[@class="job-list"]/div/div/a')
        for job in job_list:
            url =  'https://shixian.com%s'%job.xpath('./@href').extract_first()
            urls.append(url)
        return urls

    def parse_detail(self, response):
        item = JobItem()
        #大约 19 小时前发布 1 天前发布 大约 1 个月前发布 大约 1 年前发布
        item['platform'] = u'实现网'
        item['title'] = response.xpath('//article[@class="job-show"]/h1[@class="title"]/text()').extract_first().strip()
        # print('title ==>', title)
        item['url'] = response.url
        item['body'] = ''.join(response.xpath('//div[@class="content"]/p//text()').extract()).strip()
        item['checksum'] = '{}_{}'.format( 'shixian_com', response.url.split('/')[-1] )
        item['city'] = response.xpath('//ol[@class="breadcrumb jobs md-no-padding"]/li/a/text()').extract_first().strip()
        item['price'] = int(response.xpath('//strong[@class="price"]/text()').re('\d+')[0])
        release_date = response.xpath('//small[@class="time"]/text()').extract_first().strip()
        item['release_date'] = self.str2date(release_date)
        expire_date = response.xpath('//section[@class="info clearfix"]/dl/dd/span/text()')[-1].extract().strip()
        item['expire_date'] = self.str2date(expire_date)
        #item['view_count'] = int(response.xpath('//div[@class="pull-right text-muted"]/text()').re('\d+')[0])
        item['categories'] = response.xpath('//section[@class="info clearfix"]/dl/dd/span/text()')[0].extract().strip().split('/')
        item['roles'] = [response.xpath('//ol[@class="breadcrumb jobs md-no-padding"]/li/a/text()')[1].extract().strip()]
        item['skills'] = response.xpath('//section[@class="skill-tags clearfix"]/dl/dd/text()').extract()
        yield item

    def str2date(self, date_str):
        if re.search('\d{4}-\d{2}-\d{2}',date_str):
            _d = re.search('\d{4}-\d{2}-\d{2}',date_str).group()
            dt = datetime.strptime(_d, '%Y-%m-%d')
            return dt.isoformat('T') + 'Z'

        if re.search('\d+',date_str):
            num = int(re.search('\d+',date_str).group())
        now = datetime.now()
        if u'分钟' in date_str:
            dt = now - timedelta(minutes=num)
        elif u'小时' in date_str:
            dt = now - timedelta(hours=num)
        elif u'天' in date_str:
            dt = now - timedelta(days=num)
        elif u'月' in date_str:
            dt = now - timedelta(days=num*30)
        elif u'年' in date_str:
            dt = now - timedelta(days=num*365)
        else:
            dt = now

        return dt.isoformat('T') + 'Z'


import pickle

import numpy as np
from PIL import Image
from torch.utils.data.dataset import Dataset
N_ATTRIBUTES = 112
import torch

class CUBDataset(Dataset):
    """
    Returns a compatible Torch Dataset object customized for the CUB dataset
    """

    def __init__(self, pkl_file_paths, use_attr, no_img, uncertain_label, image_dir, n_class_attr, transform=None,
                 train=True, perc_supervised=100):
        """
        Arguments:
        pkl_file_paths: list of full path to all the pkl data
        use_attr: whether to load the attributes (e.g. False for simple finetune)
        no_img: whether to load the images (e.g. False for A -> Y model)
        uncertain_label: if True, use 'uncertain_attribute_label' field (i.e. label weighted by uncertainty score, e.g. 1 & 3(probably) -> 0.75)
        image_dir: default = 'images'. Will be append to the parent dir
        n_class_attr: number of classes to predict for each attribute. If 3, then make a separate class for not visible
        transform: whether to apply any special transformation. Default = None, i.e. use standard ImageNet preprocessing
        """
        self.data = []
        self.data.extend(pickle.load(open(pkl_file_paths, 'rb')))
        self.transform = transform
        self.use_attr = use_attr
        self.no_img = no_img
        self.uncertain_label = uncertain_label
        self.image_dir = image_dir
        self.n_class_attr = n_class_attr
        self.is_train = train
        self.data = np.array(self.data)
        self.targets = [x["class_label"] for x in self.data]
        self.attributes = [x["attribute_label"] for x in self.data]
        self.perc_supervised = perc_supervised
        if self.uncertain_label:
            self.attributes = [x["uncertain_attribute_label"]
                               for x in self.data]
        else:
            self.attributes = [x["attribute_label"] for x in self.data]
        self.data = [x["img_path"] for x in self.data]
        tot_imgs = len(self.targets)
        images_supervised = int(tot_imgs*self.perc_supervised/100)
        print(images_supervised)
        self.flagvector = np.zeros(len(self.data))
        print(self.flagvector.shape)
        self.flagvector[images_supervised:] = 0
        self.flagvector[:images_supervised] = 1
        self.flagvector = self.flagvector[torch.randperm(len(self.data))]

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        img_path, class_label, attr, is_supervised = self.data[idx], self.targets[
            idx], self.attributes[idx], self.flagvector[idx]
        # Trim unnecessary paths
        #img_path=img_path.replace("/juice/scr/scr102/scr/thaonguyen/CUB_supervision/datasets/", "/homes/gbontempo/ExAIProject/data/")

        img_path = img_path.replace(
            "/juice/scr/scr102/scr/thaonguyen/CUB_supervision/datasets/CUB_200_2011/images", self.image_dir)
        #img_path=img_path.replace("/home/emanuele/Programmi/Machine_Learning/ExAIProject/", "/data/emarcona/")
        img = Image.open(img_path).convert('RGB')

        if self.transform:
            img = self.transform(img)

        return img, class_label, attr, is_supervised


# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['pycoinglass']

package_data = \
{'': ['*']}

install_requires = \
['pandas>=1.4.2,<2.0.0', 'requests>=2.27.1,<3.0.0']

setup_kwargs = {
    'name': 'pycoinglass',
    'version': '0.1.0',
    'description': '',
    'long_description': None,
    'author': 'ko0hi',
    'author_email': 'ko0hi.4731@gmail.com',
    'maintainer': None,
    'maintainer_email': None,
    'url': None,
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'python_requires': '>=3.9.0,<4.0.0',
}

setup(**setup_kwargs)


from statistics import mean
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
import random

style.use('fivethirtyeight')

xs =np.array([1,2,3,4,5,6],dtype=np.float64)
ys =np.array([5,4,6,5,6,7],dtype=np.float64)

def create_dataset(hm,variance,step=2,correlation=False):
	val = 1
	ys=[]
	for i in range(hm):
		y=val + random.randrange(-variance,variance)
		ys.append(y)
		if correlation and correlation=='pos':
			val+=step
		elif correlation and correlation=='neg':
			val-=step
	xs = [i for i in range(len(ys))]
	return np.array(xs,dtype=np.float64),np.array(ys,dtype=np.float64)
def best_fit_slope_and_intercept(xs,ys):
	m = ((mean(xs)*mean(ys)) -(mean(xs*ys)))/(mean(xs)**2-mean(xs**2))
	b = mean(ys)-m*mean(xs)
	return m,b

def squared_error(ys_orig, ys_line):
	return sum((ys_line-ys_orig)**2)

def coefficient_of_determination(ys_orig,ys_line):
	y_mean_line = [mean(ys_orig) for y in ys_orig]
	squared_error_regr = squared_error(ys_orig,ys_line)
	squared_error_y_mean = squared_error(ys_orig,y_mean_line)
	return 1 - (squared_error_regr/squared_error_y_mean)

xs,ys = create_dataset(40, 40, 2, correlation='pos')
m,b=best_fit_slope_and_intercept(xs,ys)

predict_x = 8
predict_y = (m*predict_x)+b
regression_line = [(m*x)+b for x in xs]
r_squared= coefficient_of_determination(ys, regression_line)
print(r_squared)

plt.scatter(xs,ys)
plt.scatter(predict_x,predict_y,s=100,color="green")
plt.plot(xs,regression_line)
plt.show()


import os
import sys
import time
import logging
import platform
import traceback
import uuid
from html import escape
from threading import Thread
from pprint import pformat

from tempfile import mkstemp
from collections import OrderedDict
from urllib.parse import urljoin, urlencode
from urllib.request import pathname2url, urlopen, build_opener
from unittest.mock import patch

import pkg_resources

from AnyQt.QtCore import pyqtSlot, QSettings, Qt
from AnyQt.QtGui import QDesktopServices, QFont
from AnyQt.QtWidgets import (
    QApplication,
    QCheckBox,
    QDialog,
    QHBoxLayout,
    QLabel,
    QMessageBox,
    QPushButton,
    QStyle,
    QTextBrowser,
    QVBoxLayout,
    QWidget,
)

from Orange.util import try_

try:
    from Orange.widgets.widget import OWWidget
    from Orange.version import full_version as VERSION_STR
except ImportError:
    # OWWidget (etc.) is not available because this is not Orange
    class OWWidget:
        pass

    VERSION_STR = "???"

REPORT_POST_URL = "https://qa.orange.biolab.si/error_report/v1/"

log = logging.getLogger()

def get_installed_distributions():
    for dist in pkg_resources.working_set:  # type: pkg_resources.Distribution
        name = dist.project_name
        try:
            version = dist.version
        except ValueError:
            # PKG-INFO/METADATA is not available or parsable.
            version = "Unknown"
        yield "{name}=={version}".format(name=name, version=version)

class ErrorReporting(QDialog):
    _cache = set()  # For errors already handled during one session

    class DataField:
        EXCEPTION = "Exception"
        MODULE = "Module"
        WIDGET_NAME = "Widget Name"
        WIDGET_MODULE = "Widget Module"
        VERSION = "Version"
        ENVIRONMENT = "Environment"
        INSTALLED_PACKAGES = "Installed Packages"
        MACHINE_ID = "Machine ID"
        WIDGET_SCHEME = "Widget Scheme"
        STACK_TRACE = "Stack Trace"
        LOCALS = "Local Variables"

    def __init__(self, data):
        icon = QApplication.style().standardIcon(QStyle.SP_MessageBoxWarning)
        F = self.DataField

        def _finished(
            *,
            key=(data.get(F.MODULE), data.get(F.WIDGET_MODULE)),
            filename=data.get(F.WIDGET_SCHEME)
        ):
            self._cache.add(key)
            try:
                os.remove(filename)
            except Exception:
                pass

        super().__init__(
            None,
            Qt.Window,
            modal=True,
            sizeGripEnabled=True,
            windowIcon=icon,
            windowTitle="Unexpected Error",
            finished=_finished,
        )
        self._data = data

        layout = QVBoxLayout(self)
        self.setLayout(layout)
        labels = QWidget(self)
        labels_layout = QHBoxLayout(self)
        labels.setLayout(labels_layout)
        labels_layout.addWidget(QLabel(pixmap=icon.pixmap(50, 50)))
        labels_layout.addWidget(
            QLabel(
                "The program encountered an unexpected error. Please
"
                "report it anonymously to the developers.

"
                "The following data will be reported:"
            )
        )
        labels_layout.addStretch(1)
        layout.addWidget(labels)
        font = QFont("Monospace", 10)
        font.setStyleHint(QFont.Monospace)
        font.setFixedPitch(True)
        textbrowser = QTextBrowser(
            self,
            font=font,
            openLinks=False,
            lineWrapMode=QTextBrowser.NoWrap,
            anchorClicked=QDesktopServices.openUrl,
        )
        layout.addWidget(textbrowser)

        def _reload_text():
            add_scheme = cb.isChecked()
            settings.setValue("error-reporting/add-scheme", add_scheme)
            lines = ["
"]
            for k, v in data.items():
                if k.startswith("_"):
                    continue
                _v, v = v, escape(str(v))
                if k == F.WIDGET_SCHEME:
                    if not add_scheme:
                        continue
                    v = '{}'.format(
                        urljoin("file:", pathname2url(_v)), v
                    )
                if k in (F.STACK_TRACE, F.LOCALS):
                    v = v.replace("\n", "
").replace(" ", " ")
                lines.append(
                    '
{}:	{}
'.format(k, v)
                )
            lines.append("
")
            textbrowser.setHtml("".join(lines))

        settings = QSettings()
        cb = QCheckBox(
            "Include workflow (data will NOT be transmitted)",
            self,
            checked=settings.value("error-reporting/add-scheme", True, type=bool),
        )
        cb.stateChanged.connect(_reload_text)
        _reload_text()

        layout.addWidget(cb)
        buttons = QWidget(self)
        buttons_layout = QHBoxLayout(self)
        buttons.setLayout(buttons_layout)
        buttons_layout.addWidget(
            QPushButton("Send Report (Thanks!)", default=True, clicked=self.accept)
        )
        buttons_layout.addWidget(
            QPushButton("Don't Send", default=False, clicked=self.reject)
        )
        layout.addWidget(buttons)

    def accept(self):
        super().accept()
        F = self.DataField
        data = self._data.copy()

        if QSettings().value("error-reporting/add-scheme", True, type=bool):
            data[F.WIDGET_SCHEME] = data["_" + F.WIDGET_SCHEME]
        else:
            data.pop(F.WIDGET_SCHEME, None)
        del data["_" + F.WIDGET_SCHEME]

        def _post_report(data):
            MAX_RETRIES = 2
            for _retry in range(MAX_RETRIES):
                try:
                    opener = build_opener()
                    u = opener.open(REPORT_POST_URL)
                    url = u.geturl()
                    urlopen(url, timeout=10, data=urlencode(data).encode("utf8"))
                except Exception as e:
                    if _retry == MAX_RETRIES - 1:
                        e.__context__ = None
                        log.exception("Error reporting failed", exc_info=e)
                    time.sleep(10)
                    continue
                break

        Thread(target=_post_report, args=(data,)).start()

    @classmethod
    @patch("sys.excepthook", sys.__excepthook__)  # Prevent recursion
    @pyqtSlot(object)
    def handle_exception(cls, exc):
        (etype, evalue, tb), canvas = exc
        exception = traceback.format_exception_only(etype, evalue)[-1].strip()
        stacktrace = "".join(traceback.format_exception(etype, evalue, tb))

        def _find_last_frame(tb):
            if not tb:
                return None
            while tb.tb_next:
                tb = tb.tb_next
            return tb

        err_locals, err_module, frame = None, None, _find_last_frame(tb)
        if frame:
            err_module = "{}:{}".format(
                frame.tb_frame.f_globals.get(
                    "__name__", frame.tb_frame.f_code.co_filename
                ),
                frame.tb_lineno,
            )
            err_locals = OrderedDict(sorted(frame.tb_frame.f_locals.items()))
            err_locals = try_(
                lambda: pformat(err_locals), try_(lambda: str(err_locals))
            )

        def _find_widget_frame(tb):
            while tb:
                if isinstance(tb.tb_frame.f_locals.get("self"), OWWidget):
                    return tb
                tb = tb.tb_next

        widget_module, widget, frame = None, None, _find_widget_frame(tb)
        if frame:
            widget = frame.tb_frame.f_locals["self"].__class__
            widget_module = "{}:{}".format(widget.__module__, frame.tb_lineno)

        packages = ", ".join(sorted(get_installed_distributions()))

        machine_id = QSettings().value("error-reporting/machine-id", "", type=str)

        # If this exact error was already reported in this session,
        # just warn about it
        if (err_module, widget_module) in cls._cache:
            QMessageBox(
                QMessageBox.Warning,
                "Error Encountered",
                "Error encountered{}:

<tt>{}</tt>".format(
                    " in widget {}".format(widget.name) if widget else "",
                    stacktrace.replace("\n", "
").replace(" ", " "),
                ),
                QMessageBox.Ignore,
            ).exec()
            return

        F = cls.DataField
        data = OrderedDict()
        data[F.EXCEPTION] = exception
        data[F.MODULE] = err_module
        if widget:
            data[F.WIDGET_NAME] = widget.name
            data[F.WIDGET_MODULE] = widget_module
        if canvas:
            fd, filename = mkstemp(prefix="ows-", suffix=".ows.xml")
            os.close(fd)
            # Prevent excepthook printing the same exception when
            # canvas tries to instantiate the broken widget again
            with patch("sys.excepthook", lambda *_: None), open(filename, "wb") as f:
                scheme = canvas.current_document().scheme()
                try:
                    scheme.save_to(f, pretty=True, pickle_fallback=True)
                except Exception:
                    pass
            data[F.WIDGET_SCHEME] = filename
            with open(filename, encoding="utf-8") as f:
                data["_" + F.WIDGET_SCHEME] = f.read()
        data[F.VERSION] = VERSION_STR
        data[F.ENVIRONMENT] = "Python {} on {} {} {} {}".format(
            platform.python_version(),
            platform.system(),
            platform.release(),
            platform.version(),
            platform.machine(),
        )
        data[F.INSTALLED_PACKAGES] = packages
        data[F.MACHINE_ID] = machine_id or str(uuid.getnode())
        data[F.STACK_TRACE] = stacktrace
        if err_locals:
            data[F.LOCALS] = err_locals

        cls(data=data).exec()

def handle_exception(exc):
    return ErrorReporting.handle_exception(exc)


#!/usr/bin/env python3

from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from logging import DEBUG, INFO, basicConfig, info
from os.path import isfile
from socketserver import ThreadingTCPServer
from sys import exit

from mysqlite import __version__
from mysqlite.server import Server

def main():
  parser = ArgumentParser(prog="mysqlite", add_help=False, allow_abbrev=False, formatter_class=ArgumentDefaultsHelpFormatter)
  parser.add_argument("--filename", help="Filename of the SQLite database")
  parser.add_argument("--address", default="localhost", help="IP address to bind to")
  parser.add_argument("--port", default=3306, type=int, help="Port number to use for connections")
  parser.add_argument("--debug", action="store_true", help="Print packets payload")
  parser.add_argument("--version", action="store_true", help="Version information")
  parser.add_argument("--help", action="store_true", help="This help")
  args = parser.parse_args()

  if args.version:
    print(f"MySQLite {__version__}")
    exit()
  elif args.help or args.filename is None or len(args.filename) == 0:
    parser.print_help()
    exit()
  elif not isfile(args.filename):
    raise FileNotFoundError(args.filename)

  level = DEBUG if args.debug else INFO
  basicConfig(format="%(threadName)s %(levelname).3s %(message)s", level=level)

  with ThreadingTCPServer((args.address, args.port), Server, False) as server:
    try:
      info(f"MySQLite {__version__}")
      info("STARTING...")
      server.allow_reuse_address = True
      server.server_bind()
      server.server_activate()
      server.filename = args.filename

      server.serve_forever()
    except KeyboardInterrupt:
      info("STOPPING...")
      server.shutdown()

if __name__ == "__main__":
  main()


import numpy as np
import tensorflow as tf
import keras
from keras.layers import *
from keras.activations import relu
from keras.models import Model, Sequential
from keras.models import load_model
import keras.backend as Ki
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn
import time
import sys
sys.path.append('./utils')
sys.path.append('./networks')
import logging
import logging_utils
from parameters import load_params
import plots
import tboard
from SpecNormLayers import ConvSN2D, ConvSN2DTranspose, DenseSN

_SNlayertypes = [type(ConvSN2D(filters=2, kernel_size=2)), type(ConvSN2DTranspose(5, 5)), type(DenseSN(1))]

class DCGAN:

    def __init__(self, configtag, expDir):

        # Load hyperparmeters
        self.configtag = configtag
        logging.info('Parameters:')
        self.init_params(configtag)
        self.expDir = expDir
        self.inits = {'dense':keras.initializers.RandomNormal(mean=0.0, stddev=0.02),
                      'conv':keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02),
                      'tconv':keras.initializers.RandomNormal(mean=0.0, stddev=0.02)}

        # Import slices
        self.real_imgs = np.load('./data/'+self.dataname+'_train.npy')
        self.val_imgs = np.load('./data/'+self.valname+'_val.npy')
        self.n_imgs = self.real_imgs.shape[0]
        if self.datafmt == 'channels_last':
            self.real_imgs = np.moveaxis(self.real_imgs, 1, -1)
            self.val_imgs = np.moveaxis(self.val_imgs, 1, -1)

        # Build networks
        self.discrim, self.genrtor = self.load_networks()

        # Compile discriminator so it can be trained separately
        loss_fns = self._get_loss()
        def mean_prob(y_true, y_pred):
            # metric to measure mean probability of D predictions (0=fake, 1=real)
            return K.mean(K.sigmoid(y_pred))

        self.discrim.compile(loss=loss_fns['D'],
                             optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5),
                             metrics=[mean_prob])

        # Stack generator and discriminator networks together and compile
        z = Input(shape=(1,self.noise_vect_len))
        genimg = self.genrtor(z)
        self.discrim.trainable = False
        decision = self.discrim(genimg)
        self.stacked = Model(z, decision)
        self.stacked.compile(loss=loss_fns['G'], optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5))

        # Setup tensorboard stuff
        self.TB_genimg = tboard.TboardImg('genimg')
        self.TB_pixhist = tboard.TboardImg('pixhist')
        self.TB_scalars = tboard.TboardScalars()
        if self.weight_hists:
            self.TB_Dwts = tboard.TboardHists('D')
            self.TB_Gwts = tboard.TboardHists('G')
        if self.grad_hists:
            self.TB_Ggrads = tboard.TboardHists('Ggrad')
        if self.specnorm and self.sigma_plot:
            self.TB_Dsigmas = tboard.TboardSigmas('Discriminator')
            self.TB_Gsigmas = tboard.TboardSigmas('Generator')

    def load_networks(self):
        if self.resume:
            logging.info("Resuming D: %s, G: %s"%(self.resume['D'], self.resume['G']))
            custom_lyrs = {'DenseSN':DenseSN, 'ConvSN2D':ConvSN2D, 'ConvSN2DTranspose':ConvSN2DTranspose}
            return ( load_model(self.resume['D'], custom_objects=custom_lyrs),
                     load_model(self.resume['G'], custom_objects=custom_lyrs) )
        else:
            return ( self.build_discriminator(), self.build_generator() )

    def _get_loss(self):

        Ldict = {}
        if self.loss == 'binary_crossentropy':
            def my_crossentropy(y_true, y_pred):
                return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True), axis=-1)

            Ldict['D'] = my_crossentropy
            Ldict['G'] = my_crossentropy
        elif self.loss == 'hinge': # hinge loss is untested
            def Ghinge(ytrue, ypred):
                return -K.mean(ypred, axis=-1)
            Ldict['D'] = keras.losses.hinge
            Ldict['G'] = Ghinge
        return Ldict

    def build_discriminator(self):

        dmodel = Sequential()
        for lyrIdx in range(self.nlayers):
            if self.doubleconv:
                dmodel.add(self._conv_layer_type(1, lyrIdx))
            dmodel.add(self._conv_layer_type(self.convstride, lyrIdx))
            dmodel.add(BatchNormalization(epsilon=1e-5, momentum=0.9, axis=self.bn_axis))
            dmodel.add(LeakyReLU(alpha=self.alpha))
        dmodel.add(Flatten(data_format=self.datafmt))
        dmodel.add(self._dense_layer_type(1, shape_spec=False))
        if self.loss == 'hinge': # hinge loss is untested
            act = 'tanh'
        else:
            act = 'sigmoid'
        #dmodel.add(Activation(act)) # remove sigmoid activation to avoid bug w/ default Keras loss
        dmodel.summary(print_fn=logging_utils.print_func)
        img = Input(shape=self.imshape)
        return Model(img, dmodel(img))

    def build_generator(self):
        fmapsize = self.img_dim//int(2**self.nlayers)
        gmodel = Sequential()
        gmodel.add(self._dense_layer_type(fmapsize*fmapsize*(2*self.ndeconvfilters[0]), shape_spec=(1,self.noise_vect_len)))
        gmodel.add(BatchNormalization(epsilon=1e-5, momentum=0.9, axis=self.bn_axis))
        gmodel.add(ReLU())
        if self.datafmt == 'channels_last':
            fmapshape = (fmapsize, fmapsize, 2*self.ndeconvfilters[0])
        else:
            fmapshape = (2*self.ndeconvfilters[0], fmapsize, fmapsize)
        gmodel.add(Reshape(fmapshape))
        for lyrIdx in range(self.nlayers):
            if self.doubleconv:
                gmodel.add(self._deconv_layer_type(1, lyrIdx))
            gmodel.add(self._deconv_layer_type(self.convstride, lyrIdx))
            if lyrIdx == self.nlayers - 1:
                gmodel.add(Activation('tanh')) # last layer is deconv+tanh
            else:
                gmodel.add(BatchNormalization(epsilon=1e-5, momentum=0.9, axis=self.bn_axis)) # hidden layers are deconv+batchnorm+relu
                gmodel.add(ReLU())
        gmodel.summary(print_fn=logging_utils.print_func)
        noise = Input(shape=(1,self.noise_vect_len))
        return Model(noise, gmodel(noise))

    def _dense_layer_type(self, num, shape_spec=False):

        lyr = None
        if shape_spec:
            if self.specnorm:
                # Use DenseSN if you want spectral normalization in G
                lyr = Dense(num, input_shape=shape_spec, kernel_initializer=self.inits['dense'])
            else:
                lyr = Dense(num, input_shape=shape_spec, kernel_initializer=self.inits['dense'])
        else:
            if self.specnorm:
                # Use DenseSN if you want spectral normalization in D
                lyr = DenseSN(num, kernel_initializer=self.inits['dense'])
            else:
                lyr = Dense(num, kernel_initializer=self.inits['dense'])
        return lyr

    def _conv_layer_type(self, stride, lyrIdx):

        lyr = None
        if lyrIdx == 0:
            if self.specnorm:
                # Use ConvSN2D if you want spectral normalization in D
                lyr = ConvSN2D(filters=self.nconvfilters[lyrIdx], kernel_size=self.convkern, strides=stride,
                             padding='same', input_shape=self.imshape, data_format=self.datafmt,
                             kernel_initializer=self.inits['conv'])
            else:
                lyr = Conv2D(filters=self.nconvfilters[lyrIdx], kernel_size=self.convkern, strides=stride,
                             padding='same', input_shape=self.imshape, data_format=self.datafmt,
                             kernel_initializer=self.inits['conv'])
        else:
            if self.specnorm:
                # Use ConvSN2D if you want spectral normalization in D
                lyr = ConvSN2D(filters=self.nconvfilters[lyrIdx], kernel_size=self.convkern, strides=stride,
                             padding='same', data_format=self.datafmt, kernel_initializer=self.inits['conv'])
            else:
                lyr = Conv2D(filters=self.nconvfilters[lyrIdx], kernel_size=self.convkern, strides=stride,
                             padding='same', data_format=self.datafmt, kernel_initializer=self.inits['conv'])
        return lyr

    def _deconv_layer_type(self, stride, lyrIdx):

        lyr = None
        if self.specnorm:
            # Use ConvSN2DTranspose if you want spectral normalization in G
            lyr = Conv2DTranspose(self.ndeconvfilters[lyrIdx], self.convkern, strides=stride,
                                    padding='same', data_format=self.datafmt, kernel_initializer=self.inits['tconv'])
        else:
            lyr = Conv2DTranspose(self.ndeconvfilters[lyrIdx], self.convkern, strides=stride,
                                  padding='same', data_format=self.datafmt, kernel_initializer=self.inits['tconv'])
        return lyr

    def init_params(self,configtag):

        params = load_params('./config.yaml', configtag)
        logging.info(str(params))
        self.dataname = params['dataname']
        self.valname = params['valname']
        self.img_dim = params['img_dim']
        self.noise_vect_len = params['noise_vect_len']
        self.nlayers = params['nlayers']
        self.convkern = params['convkern']
        self.convstride = params['convstride']
        self.nconvfilters = params['nconvfilters']
        self.ndeconvfilters = params['ndeconvfilters']
        self.specnorm = params['specnorm']
        self.label_flip = params['label_flip']
        self.batchsize = params['batchsize']
        self.print_batch = params['print_batch']
        self.checkpt_batch = params['checkpt_batch']
        self.cscale = params['cscale']
        self.datascale = params['datascale']
        self.G_lr, self.D_lr = params['learn_rate']
        self.DG_update_ratio = params['DG_update_ratio']
        self.resume = params['resume']
        self.Nepochs = params['Nepochs']
        self.loss = params['loss']
        self.weight_hists = params['weight_hists']
        self.grad_hists = params['grad_hists']
        self.sigma_plot = params['sigma_plot']
        self.doubleconv = params['doubleconv']
        self.datafmt = params['datafmt']
        self.alpha = 0.2
        self.start = 0
        self.bestchi = np.inf
        if self.datafmt == 'channels_last':
            self.bn_axis = -1
            self.imshape = (self.img_dim, self.img_dim, 1)
        else:
            self.bn_axis = 1
            self.imshape = (1, self.img_dim, self.img_dim)
        self.real = 1
        if self.loss == 'hinge': # hinge loss is untested
            self.fake = -1
        else:
            self.fake = 0

    def train_epoch(self, shuffler, num_batches, epochIdx):

        d_losses = []
        d_real_losses = []
        d_fake_losses = []
        g_losses = []

        for batch in range(num_batches):
            iternum = (epochIdx*num_batches + batch)
            t1 = time.time()

            # generate fakes
            real_img_batch = self.real_imgs[shuffler[batch*self.batchsize:(batch+1)*self.batchsize]]
            noise_vects = np.random.normal(loc=0.0, size=(self.batchsize, 1, self.noise_vect_len))
            fake_img_batch = self.genrtor.predict(noise_vects)

            reals = self.real*np.ones((self.batchsize,1))
            fakes = self.fake*np.ones((self.batchsize,1))
            labelflip = self.label_flip
            for i in range(reals.shape[0]):
                if np.random.uniform(low=0., high=1.0) < labelflip:
                    reals[i,0] = self.fake
                    fakes[i,0] = self.real

            # train discriminator
            for iters in range(self.DG_update_ratio//2):
                discr_real_loss = self.discrim.train_on_batch(real_img_batch, reals)
                discr_fake_loss = self.discrim.train_on_batch(fake_img_batch, fakes)
                discr_loss = 0.5*(discr_real_loss[0]+discr_fake_loss[0])
                d_losses.append(discr_loss)
                d_real_losses.append(discr_real_loss[0])
                d_fake_losses.append(discr_fake_loss[0])

            # train generator via stacked model
            genrtr_loss = self.stacked.train_on_batch(noise_vects, self.real*np.ones((self.batchsize,1)))
            t2 = time.time()

            g_losses.append(genrtr_loss)

            t2 = time.time()

            if batch%self.print_batch == 0:
                logging.info("| --- batch %d of %d --- |"%(batch + 1, num_batches))
                logging.info("|Discr real pred=%f, fake pred=%f"%(discr_real_loss[1], discr_fake_loss[1]))
                logging.info("|Discriminator: loss=%f"%(discr_loss))
                logging.info("|Generator: loss=%f"%(genrtr_loss))
                logging.info("|Time: %f"%(t2-t1))
            if iternum%self.checkpt_batch == 0:
                # Tensorboard monitoring
                iternum = iternum/self.checkpt_batch
                self.TB_genimg.on_epoch_end(iternum, self, logs={})
                chisq = self.TB_pixhist.on_epoch_end(iternum, self, logs={})
                scalars = {'d_loss':np.mean(d_losses), 'd_real_loss':np.mean(d_real_losses),
                           'd_fake_loss':np.mean(d_fake_losses), 'g_loss':np.mean(g_losses), 'chisq':chisq}
                self.TB_scalars.on_epoch_end(self, iternum, scalars)

                sess = K.get_session()

                if self.weight_hists:
                    # Monitor histogram of weights for conv and dense layers
                    Dlayerdict = {layer.name:layer.get_weights()[0] for layer in self.discrim.layers[1].layers \
                                                                    if 'conv' in layer.name or 'dense' in layer.name}
                    Glayerdict = {layer.name:layer.get_weights()[0] for layer in self.genrtor.layers[1].layers \
                                                                    if 'conv' in layer.name or 'dense' in layer.name}

                    self.TB_Dwts.on_epoch_end(self, iternum, Dlayerdict)
                    self.TB_Gwts.on_epoch_end(self, iternum, Glayerdict)

                if self.grad_hists:
                    # Monitor histogram of gradients -- VERY slow and crashes when using big network
                    ggrad_tensors = self._get_grad_tensors(self.stacked, tf.ones((self.batchsize, 1)), \
                                                           tf.random.normal((self.batchsize, 1, self.noise_vect_len)), stacked=True)
                    ggrads = sess.run(ggrad_tensors)
                    self.TB_Ggrads.on_epoch_end(self, iternum, ggrads)
                if self.specnorm and self.sigma_plot:
                    # Monitor spectral norm of normalized weight matrices
                    dsigmas = [layer.sigma.eval(session=sess)[0,0] for layer in self.discrim.layers[1].layers\
                                                                   if type(layer) in _SNlayertypes]
                    gsigmas = [layer.sigma.eval(session=sess)[0,0] for layer in self.genrtor.layers[1].layers\
                                                                   if type(layer) in _SNlayertypes]

                    dsiglabels = ['conv'+str(idx+1) for idx in range(self.nlayers)] + ['dense_d']
                    gsiglabels = ['dense_g'] + ['tconv'+str(idx+1) for idx in range(self.nlayers)]

                    dsiglog = dict(zip(dsiglabels, dsigmas))
                    gsiglog = dict(zip(gsiglabels, gsigmas))
                    self.TB_Dsigmas.on_epoch_end(self, iternum, dsiglog)
                    self.TB_Gsigmas.on_epoch_end(self, iternum, gsiglog)

                d_losses = []
                d_real_losses = []
                d_fake_losses = []
                g_losses = []

                if chisq<self.bestchi and iternum>50:
                    # update best chi-square score and save
                    self.bestchi = chisq
                    self.genrtor.save(self.expDir+'models/g_cosmo_best.h5')
                    self.discrim.save(self.expDir+'models/d_cosmo_best.h5')
                    logging.info("BEST saved at %d, chi=%f"%(iternum, chisq))

    def _get_grad_tensors(self, model, labels, data, stacked=False):

        loss = keras.losses.get(self.loss)
        if stacked:
            learn_rate = self.G_lr
            weights = [[layer.name, layer.trainable_weights[0]] for layer in model.layers[1].layers[1].layers \
                                                                if 'conv' in layer.name or 'dense' in layer.name]
        else:
            learn_rate = self.D_lr
            weights = [[layer.name, layer.trainable_weights[0]] for layer in model.layers[1].layers \
                                                                if 'conv' in layer.name or 'dense' in layer.name]
        optim = keras.optimizers.Adam(learn_rate, beta_1 = 0.5)
        grads = optim.get_gradients(loss(labels, model(data)), [wvars for _ , wvars in weights])
        gradsdict = dict(zip([name for name, _ in weights], grads))
        return gradsdict


import http.server
import socketserver

class SimpleHTMLHandler(http.server.SimpleHTTPRequestHandler):
    def do_GET(self):
        # Set the response code to 200 (OK)
        self.send_response(200)

        # Set the Content-type header to indicate HTML content
        self.send_header('Content-type', 'text/html')
        self.end_headers()

        # Read the content of the HTML file and send it as the response
        with open('page1.html', 'rb') as file:
            self.wfile.write(file.read())

# Set the port number you want to run the server on
port = 8080

# Create the HTTP server and bind it to the specified port
with socketserver.TCPServer(("127.0.0.1", port), SimpleHTMLHandler) as httpd:
    print(f"Server running on port {port}")

    # Start serving indefinitely until interrupted
    httpd.serve_forever()


from typing import List, NewType

import torch
from injector import Module, inject, provider, singleton

from mcp.config.optimizer import OptimizerType, _OptimizerConfig
from mcp.config.parser import ExperimentConfig
from mcp.config.scheduler import SchedulerType, _SchedulerConfig
from mcp.context.task import TasksTrain, TasksValid, TaskTest
from mcp.model.base import Model

OptimizerTrain = NewType("OptimizerTrain", torch.optim.Optimizer)
OptimizerValid = NewType("OptimizerValid", torch.optim.Optimizer)
OptimizerTest = NewType("OptimizerTest", torch.optim.Optimizer)

SchedulerTrain = NewType("SchedulerTrain", torch.optim.lr_scheduler._LRScheduler)
SchedulerValid = NewType("SchedulerValid", torch.optim.lr_scheduler._LRScheduler)
SchedulerTest = NewType("SchedulerTest", torch.optim.lr_scheduler._LRScheduler)

class OptimizerModule(Module):
    def __init__(self, config: ExperimentConfig, output_dir: str, device: torch.device):
        self.config = config
        self.output_dir = output_dir
        self.device = device

    @provider
    @inject
    @singleton
    def provide_optimizer_train(
        self, model: Model, tasks_train: TasksTrain, tasks_valid: TasksValid
    ) -> OptimizerTrain:
        modules = [model] + tasks_train
        parameters = self._merge_param(modules)
        return self._create_optimizer(self.config.optimizer.train, parameters)

    @provider
    @inject
    @singleton
    def provide_optimizer_test(self, model: Model, task: TaskTest) -> OptimizerTest:
        return self._create_optimizer(self.config.optimizer.support, task.parameters())  # type: ignore

    @provider
    @inject
    @singleton
    def provide_optimizer_valid(
        self, model: Model, tasks_valid: TasksValid
    ) -> OptimizerValid:
        parameters = self._merge_param(tasks_valid)
        return self._create_optimizer(self.config.optimizer.support, parameters)

    def _create_optimizer(self, config: _OptimizerConfig, parameters):
        if config.type == OptimizerType.SGD:
            return torch.optim.SGD(  # type: ignore
                parameters,
                lr=config.learning_rate,
                weight_decay=config.weight_decay,
                momentum=config.sgd.momentum,
            )
        elif config.type == OptimizerType.ADAM:
            return torch.optim.Adam(  # type: ignore
                parameters, lr=config.learning_rate, weight_decay=config.weight_decay,
            )
        else:
            raise ValueError(f"Optimizer not yet supported {config.type}")

    def _merge_param(self, modules: List[torch.nn.Module]):
        for module in modules:
            for parameter in module.parameters():
                yield parameter

class SchedulerModule(Module):
    def __init__(self, config: ExperimentConfig, output_dir: str, device: torch.device):
        self.config = config
        self.output_dir = output_dir
        self.device = device

    @provider
    @inject
    @singleton
    def provide_scheduler_train(self, optimizer: OptimizerTrain) -> SchedulerTrain:
        return self._create_scheduler(optimizer, self.config.scheduler.train)

    @provider
    @inject
    @singleton
    def provide_scheduler_valid(self, optimizer: OptimizerValid) -> SchedulerValid:
        return self._create_scheduler(optimizer, self.config.scheduler.support)

    @provider
    @inject
    @singleton
    def provide_scheduler_test(self, optimizer: OptimizerTest) -> SchedulerTest:
        return self._create_scheduler(optimizer, self.config.scheduler.support)

    def _create_scheduler(
        self, optimizer: torch.optim.Optimizer, config: _SchedulerConfig
    ):
        if config.type == SchedulerType.MULTI_STEP:
            return torch.optim.lr_scheduler.MultiStepLR(
                optimizer,
                milestones=config.multistep.milestones,
                gamma=config.multistep.gamma,
            )
        elif config.type == SchedulerType.CONSTANT:
            return torch.optim.lr_scheduler.MultiplicativeLR(optimizer, lr_lambda=lambda _: 1.0)  # type: ignore
        else:
            raise ValueError(f"Scheduler not yet supported {config.type}")


from sqlalchemy import desc, and_
from sqlalchemy.sql import func
from app.core.service import BaseService
from app.models.medicine_in import MedicineIn
from app.models.medicine_out import MedicineOut
from app.schemas.medicine_in import MedicineInCreate, MedicineInFilter, MedicineInUpdateRequest
from fastapi import HTTPException, status
from datetime import datetime, timedelta
from app.models.specification import Specifications
from app.models.unit import Units

class MedicineInsService(BaseService):

    def get_by_id(self, id: int):
        medicine_in = self.db.query(MedicineIn).filter(MedicineIn.id == id).first()

        if not medicine_in:
            raise HTTPException(
                    status.HTTP_422_UNPROCESSABLE_ENTITY,
                    [{
                            'error' : True,
                            'message': f"Medicine with id: {id} does not exist",
                            'code': 'ER0041'
                        }]
                )
        return medicine_in

    def create(self, medicine_in_data: MedicineInCreate):

        last_record = self.db.query(MedicineIn).filter(MedicineIn.medicine_id == medicine_in_data.medicine_id)\
                            .filter(MedicineIn.adopt_area_id == medicine_in_data.adopt_area_id)\
                            .filter(MedicineIn.in_date == medicine_in_data.in_date)\
                            .order_by(desc(MedicineIn.id)).first()
        date_code = medicine_in_data.in_date.strftime("%d-%m-%y").replace('-', '')
        prefix = f'{"TH-"}{date_code}{"-"}'
        if last_record is None:
            code = f'{prefix}{0}{1}'
        else:
            temp = last_record.batch_code[:10]
            index = last_record.batch_code.replace(temp,'')
            index = int(index) + 1
            if index < 10:
                code = f'{prefix}{0}{index}'
            else:
                code = f'{prefix}{index}'

        specification = self.db.query(Specifications).filter(and_(Specifications.id == medicine_in_data.specification_id, Specifications.deleted_at.is_(None))).first()

        if not specification:
            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
                                detail=f"Qui cách không tồn tại")

        quantity = medicine_in_data.amount * specification.amount

        new_medicine_in = MedicineIn(
            batch_code=code,
            medicine_id=medicine_in_data.medicine_id,
            adopt_area_id=medicine_in_data.adopt_area_id,
            quantity=quantity,
            inventory=quantity,
            unit_id=medicine_in_data.unit_id,
            in_date=medicine_in_data.in_date,
            mfg_date=medicine_in_data.mfg_date,
            exp_date=medicine_in_data.exp_date,
            specification_id=medicine_in_data.specification_id
        )
        self.db.add(new_medicine_in)
        self.db.commit()
        self.db.refresh(new_medicine_in)

        return new_medicine_in

    def update(self, id, medicine_in_data: MedicineInUpdateRequest):
        medicine_in = self.get_by_id(id)
        medicine_in_query = self.db.query(MedicineIn).filter(MedicineIn.id == id)
        medicine_in_quantity = medicine_in_query.join(MedicineIn.specification).first()

        if medicine_in_data.amount and medicine_in_quantity:
            medicine_in_quantity_value = medicine_in_quantity.specification.amount * medicine_in_data.amount

            total_quantity_medicine_out = self.db.query(func.sum(MedicineOut.quantity).label("total_score")).filter(MedicineOut.medicine_in_id ==id).scalar()

            if total_quantity_medicine_out:
                if medicine_in_quantity_value < total_quantity_medicine_out:
                    raise HTTPException(
                        status_code=status.HTTP_403_FORBIDDEN,
                        detail=f"{total_quantity_medicine_out}"
                    )
                inventory = medicine_in_quantity_value - total_quantity_medicine_out
                medicine_in_query.update({"inventory": inventory}, synchronize_session=False)
                self.db.query(MedicineOut).filter(MedicineOut.medicine_in_id == id).update({"inventory": inventory}, synchronize_session=False)
            else:
                medicine_in_query.update({"inventory": medicine_in_quantity_value}, synchronize_session=False)

            medicine_in_query.update({"quantity": medicine_in_quantity_value}, synchronize_session=False)

        if medicine_in_data.unit_id:
            unit_deleted = self.db.query(Units).filter(and_(Units.id == medicine_in_data.unit_id, Units.deleted_at.is_not(None))).first()

            if unit_deleted:
                raise HTTPException(
                         status.HTTP_422_UNPROCESSABLE_ENTITY,
                        [{
                            'toast': f"Đơn vị đã bị xóa."
                        }]
                    )

            medicine_in_query.update({"unit_id": medicine_in_data.unit_id}, synchronize_session=False)

        if medicine_in_data.mfg_date:
            medicine_in_query.update({"mfg_date": medicine_in_data.mfg_date}, synchronize_session=False)

        if medicine_in_data.exp_date:
            medicine_in_query.update({"exp_date": medicine_in_data.exp_date}, synchronize_session=False)

        if medicine_in_data.in_date:
            if medicine_in_data.in_date != medicine_in.in_date:
                last_record = self.db.query(MedicineIn).filter(MedicineIn.medicine_id == medicine_in.medicine_id)\
                                .filter(MedicineIn.adopt_area_id == medicine_in.adopt_area_id)\
                                .filter(MedicineIn.in_date == medicine_in_data.in_date)\
                                .order_by(desc(MedicineIn.id)).first()

                date_code = medicine_in_data.in_date.strftime("%d-%m-%y").replace('-', '')
                prefix = f'{"TH-"}{date_code}{"-"}'

                if last_record is None:
                    code = f'{prefix}{0}{1}'
                else:
                    temp = last_record.batch_code[:10]
                    index = last_record.batch_code.replace(temp,'')
                    index = int(index) + 1
                    if index < 10:
                        code = f'{prefix}{0}{index}'
                    else:
                        code = f'{prefix}{index}'

                medicine_in_query.update({"in_date": medicine_in_data.in_date}, synchronize_session=False)
                medicine_in_query.update({"batch_code": code}, synchronize_session=False)

        self.db.commit()

        return medicine_in

    def get_all(self, medicine_in_filter: MedicineInFilter, all = False):
        query = self.db.query(MedicineIn)

        if medicine_in_filter.adopt_area_id:
            query = query.filter(MedicineIn.adopt_area_id == medicine_in_filter.adopt_area_id)

        if medicine_in_filter.medicine_id:
            query = query.filter(MedicineIn.medicine_id == medicine_in_filter.medicine_id)

        if medicine_in_filter.from_date:
            query = query.filter(MedicineIn.created_at >= medicine_in_filter.from_date.date())

        if medicine_in_filter.to_date:
            to_date = medicine_in_filter.to_date + timedelta(days=1)
            query = query.filter(MedicineIn.created_at <= to_date.date())

        if medicine_in_filter.out_of_inventory:
            query = query.filter(MedicineIn.inventory != 0)

        query = query.order_by(desc(MedicineIn.created_at))

        if all == False:
            if medicine_in_filter.limit:
                query = query.limit(medicine_in_filter.limit)

            if medicine_in_filter.offset:
                query = query.offset(medicine_in_filter.offset)

        return query.all()


import pandas as pd
import numpy as np
import pandas as pd
import joblib
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF
import csv
# Загрузка данных из CSV без разделительных столбцов "|||"
data = pd.read_csv('../metrics/updated_data1.csv', sep=',', usecols=lambda col: '|||' not in col)
# print(data)

# Загрузка данных из CSV без разделительных столбцов "|||"
data = pd.read_csv('../metrics/updated_data1.csv', sep=',', usecols=lambda col: '|||' not in col)
# print(data)

X = data.iloc[:, :-5]
y = data.iloc[:, -2]

# print(X)
# print(y)

X = X.values
y = y.values
# print(y)
X = data.iloc[:, :-5]
y = data.iloc[:, -2]
X = X.values
y = y.values
kernel = RBF()
gpr = GaussianProcessRegressor(kernel=kernel)

gpr.fit(X, y)
# Обучить модель на данных
model_name = 'gpr_trained_model.joblib'
joblib.dump(gpr, model_name)
print("Saved model to disk")
# Обучить модель на данных
model_name = 'gpr_trained_model.joblib'
joblib.dump(gpr, model_name)
print("Saved model to disk")
# Вывести гиперпараметры модели
with open("output.csv", mode="w", encoding="utf-8", newline='') as w_file:
    file_writer = csv.writer(w_file, delimiter=",")
    file_writer.writerow(["Hyperparameters"])
    for hyperparameter in gpr.kernel_.get_params():
        file_writer.writerow([str(hyperparameter)])

    file_writer.writerow([])
    file_writer.writerow(["Covariance Function"])
    covariance_matrix = gpr.kernel_(X)
    for i in range(len(covariance_matrix)):
        file_writer.writerow(covariance_matrix[i])

    file_writer.writerow([])
    file_writer.writerow(["Predictions"])
    predictions, std = gpr.predict(X, return_std=True)
    for i in range(len(X)):
        file_writer.writerow([predictions[i], std[i]])
print("Done")


from django.db.models import Q, QuerySet

from rest_framework.generics import get_object_or_404

from app import models
from app import utils

def get_wallet_report_queryset(wallet_id, data) -> QuerySet:
    get_object_or_404(queryset=models.Wallet.objects.all(), id=wallet_id)

    res = models.StatusChange.objects.all().select_related('operation')
    res = res.filter(Q(operation__wallet_from=wallet_id) | Q(operation__wallet_to=wallet_id))

    date_from = utils.parse_user_date(data.get('date_from'))
    if date_from:
        res = res.filter(datetime__gte=date_from)

    date_to = utils.parse_user_date(data.get('date_to'))
    if date_to:
        res = res.filter(datetime__lte=date_to)

    res = res.order_by('-datetime')
    return res

def get_username_report_queryset(username, data) -> QuerySet:
    user = get_object_or_404(queryset=models.MyUser.objects.all(), username=username)
    return get_wallet_report_queryset(user.wallet.id, data)


# coding:utf-8

import os
import argparse
import numpy as np
import json
import cv2
import torch
import copy
from typing import Any, Iterator, List, Union
from detectron2.structures import (
    Boxes,
    PolygonMasks,
    BoxMode,
    polygons_to_bitmask
)
from mask_encoding import DctMaskEncoding

class IOUMetric(object):
    """
    Class to calculate mean-iou using fast_hist method
    """

    def __init__(self, num_classes):
        self.num_classes = num_classes
        self.hist = np.zeros((num_classes, num_classes))

    def _fast_hist(self, label_pred, label_true):
        mask = (label_true >= 0) & (label_true < self.num_classes)
        hist = np.bincount(
            self.num_classes * label_true[mask].astype(int) +
            label_pred[mask], minlength=self.num_classes ** 2).reshape(self.num_classes, self.num_classes)
        return hist

    def add_batch(self, predictions, gts):
        for lp, lt in zip(predictions, gts):
            self.hist += self._fast_hist(lp.flatten(), lt.flatten())

    def evaluate(self):
        acc = np.diag(self.hist).sum() / self.hist.sum()
        acc_cls = np.diag(self.hist) / self.hist.sum(axis=1)
        acc_cls = np.nanmean(acc_cls)
        iu = np.diag(self.hist) / (self.hist.sum(axis=1) + self.hist.sum(axis=0) - np.diag(self.hist))
        mean_iu = np.nanmean(iu)
        freq = self.hist.sum(axis=1) / self.hist.sum()
        fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
        return acc, acc_cls, iu, mean_iu, fwavacc

def rasterize_polygons_within_box_for_arbitrary_shape(
        polygons: List[np.ndarray], box: np.ndarray, mask_size_h: int, mask_size_w: int
) -> torch.Tensor:
    """
    New rasterize. Rasterize polygons within box with specific size.
    Args:
        polygons (list[ndarray[float]]): a list of polygons, which represents an instance.
        box: 4-element numpy array
        mask_size_h (int):
        mask_size_w (int):

    Returns:
        Tensor: BoolTensor of shape (mask_size, mask_size)
    """
    # 1. Shift the polygons w.r.t the boxes
    w, h = box[2] - box[0], box[3] - box[1]

    polygons = copy.deepcopy(polygons)
    for p in polygons:
        p[0::2] = p[0::2] - box[0]
        p[1::2] = p[1::2] - box[1]

    # 2. Rescale the polygons to the new box size
    # max() to avoid division by small number
    ratio_h = mask_size_h / max(h, 0.1)
    ratio_w = mask_size_w / max(w, 0.1)

    if ratio_h == ratio_w:
        for p in polygons:
            p *= ratio_h
    else:
        for p in polygons:
            p[0::2] *= ratio_w
            p[1::2] *= ratio_h

    # 3. Rasterize the polygons with coco api
    mask = polygons_to_bitmask(polygons, mask_size_h, mask_size_w)
    mask = torch.from_numpy(mask)
    return mask

def valid_dct_source(coco, dct_dim, mask_size):
    dct_mask_encoding = DctMaskEncoding(dct_dim, mask_size)
    mIoU = []
    Number = 0
    for ann in coco:
        Number += 1
        bbox = np.array([ann["bbox"]])  # xmin, ymin, w, h
        w, h = bbox[0][2], bbox[0][3]
        w, h = round(w), round(h)
        bbox = BoxMode.convert(bbox, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)  # x1y1x2y2
        bbox = Boxes(bbox)

        # mask transform.
        mask = PolygonMasks([ann["segmentation"]])
        mask_source = rasterize_polygons_within_box_for_arbitrary_shape(mask.polygons[0], bbox.tensor[0].numpy(), h, w)
        mask_source = mask_source.numpy()  # numpy [h,w] binary

        mask_k = mask.crop_and_resize(bbox.tensor, mask_size).float()  # tensor [1,28,28],all 0 or 1
        mask_k = mask_k.view([mask_size, mask_size])
        dct_code = dct_mask_encoding.encode(mask_k)
        mask_re = dct_mask_encoding.decode(dct_code).numpy().squeeze()
        res = cv2.resize(mask_re.astype('float'),
                         dsize=(mask_source.shape[1], mask_source.shape[0]),
                         interpolation=cv2.INTER_LINEAR)

        res = np.where(res >= 0.5, 1, 0)
        res = np.reshape(res, [1, -1])
        mask_source = np.reshape(mask_source, [1, -1])
        res = res.astype(int)

        IoUevaluate = IOUMetric(2)
        IoUevaluate.add_batch(res, mask_source)

        _, _, _, mean_iu, _ = IoUevaluate.evaluate()
        mIoU.append(mean_iu)
        if Number % 1000 == 1:
            print(np.mean(mIoU))
    return np.mean(mIoU)

DATASETS = {
    "coco_2017_train": {
        "img_dir": "coco/train2017",
        "ann_file": "coco/annotations/instances_train2017.json"
    },
    "coco_2017_val": {
        "img_dir": "coco/val2017",
        "ann_file": "coco/annotations/instances_val2017.json"
    }
}

def parse_args():
    parser = argparse.ArgumentParser(description='Evaluation for DCT-Mask Encoding.')
    parser.add_argument('--root', default='datasets', type=str)
    parser.add_argument('--dataset', default='coco_2017_val', type=str)
    # mask encoding params.
    parser.add_argument('--mask_size', default=128, type=int)
    parser.add_argument('--dim', default=300, type=int)
    args = parser.parse_args()
    return args

if __name__ == "__main__":
    args = parse_args()
    mask_size = args.mask_size
    dim = args.dim

    dataset_root = args.root

    data_info = DATASETS[args.dataset]
    img_dir, ann_file = data_info['img_dir'], data_info['ann_file']
    img_dir = os.path.join(dataset_root, img_dir)  # actually we do not use it.
    ann_file = os.path.join(dataset_root, ann_file)

    with open(ann_file, 'r') as f:
        anns = json.load(f)
    anns = anns['annotations']
    coco = list()
    for ann in anns:
        if ann.get('iscrowd', 0) == 0:
            coco.append(ann)
    print("Removed {} images with no usable annotations. {} images left.".format(
        len(anns) - len(coco), len(coco)))

    mean_iou = valid_dct_source(coco, dct_dim=dim, mask_size=mask_size)
    print('mask_size: {}, dim: {}, mIoU: {}'.format(mask_size, dim, mean_iou))


import tempfile
import shutil
import numpy as np

import Bio
from Bio import pairwise2, AlignIO
#from Bio.SubsMat import MatrixInfo as matlist

from comutils.tool_wrapper import *
from comutils import files_management as fm
from comutils.global_variables import ALPHABET
from comutils.pseudocounts import *
q = len(ALPHABET)

import ccmpred.weighting
import ccmpred.io
import ccmpred.counts

def code(c):
    """ gives number code in [0,21] for letter @c"""
    return ALPHABET.find(c.upper())

def code_whole_seq(sequence):
    return [code(c) for c in sequence]

def get_reordered_v(v, alphabet_to, alphabet_from=ALPHABET):
    """ reorders all vi for a given alphabet """
    q = v.shape[1]
    idx = [alphabet_from[:q].find(a) for a in alphabet_to[:q]]
    return v[:,idx]

def get_reordered_wij(wij, alphabet_to, alphabet_from=ALPHABET):
    """ reorders all wij for a given alphabet """
    q = wij.shape[0]
    idx = [alphabet_from[:q].find(a) for a in alphabet_to[:q]]
    new_wij = np.zeros_like(wij)
    for a in range(len(alphabet_to)):
        for b in range(len(alphabet_to)):
            new_wij[a][b] = wij[idx[a]][idx[b]]
    return new_wij

def get_reordered_w(w, alphabet_to, alphabet_from=ALPHABET):
    new_w = np.zeros_like(w)
    L = w.shape[0]
    for i in range(L-1):
        for j in range(i+1,L):
            new_w[i,j] = get_reordered_wij(w[i,j], alphabet_to=alphabet_to, alphabet_from=alphabet_from)
            new_w[j,i] = np.transpose(new_w[i,j])
    return new_w

def euclidean_norm(vector):
    return np.linalg.norm(vector)

def scalar_product(v1, v2):
    return np.dot(v1.flatten(), v2.flatten())

def compute_w_norms(w):
    L = w.shape[0]
    w_norms = np.zeros((L,L))
    for i in range(0, L-1):
        for j in range(i+1, L):
            w_norms[i][j] = euclidean_norm(w[i,j])
            w_norms[j][i] = w_norms[i][j]
    return w_norms

def sign_ind(x):
    """ returns 1 if @x>=0, -1 otherwise """
    return 2*(x>=0)-1

def seq_identity(seq1, seq2):
    """ sequence identity between unaligned sequences @seq1 et @seq2 """
    alignment = pairwise2.align.globalxx(seq1, seq2, one_alignment_only=True)[0]
    nb_matches = alignment[2]
    length_alignment = len(alignment[0])
    #score = (nb_matches/max(len(seq1),len(seq2)))
    score = nb_matches/length_alignment
    return score

def is_gap_column(i, msa):
    """ returns True iff the column @i of @msa contains only gaps """
    gap = True
    s=0
    while ( (gap==True) and (s<len(msa)) ):
        seq = msa[s].seq
        if (seq[i]!="-"):
            gap = False
        s+=1
    return gap

def get_trimmed_sequence_for_msa(msa_file, seq):
    """ aligns sequence @seq to MSA in @msa_file and removes all inserted positions in @seq so that it can be aligned to the MSA """
    temp_folder = pathlib.Path(tempfile.mkdtemp())
    tempseq_file = temp_folder/("seq.fasta")
    seq_name = fm.create_seq_fasta(seq, tempseq_file)
    combined_aln_file = temp_folder/"aln.fasta"
    call_muscle_profile(msa_file,tempseq_file,combined_aln_file)
    comb_msa = AlignIO.read(str(combined_aln_file), "fasta")
    new = comb_msa[len(comb_msa)-1].seq
    trimmed=""
    n = len(comb_msa[0].seq)
    for i in range(n):
        if not is_gap_column(i, comb_msa[0:len(comb_msa)-1]): # si pas insertion
            trimmed+=new[i]
    shutil.rmtree(temp_folder)
    return trimmed

def get_pos_first_seq_to_second_seq(first_seq, second_seq):
    """ returns dictionary d[pos_in_first_seq] = pos_in_second_seq """
    gap_char='.'
    alns = pairwise2.align.globalxx(first_seq, second_seq, gap_char=gap_char)
    top_aln = alns[0]
    aln_first, aln_second, score, begin, end = top_aln
    first_pos = 0
    second_pos = 0
    pos_dict_first_seq_to_second_seq = [None for k in range(len(first_seq))]
    for i in range(len(aln_first)):
        if (aln_first[i]==gap_char) and (aln_second[i]!=gap_char):
            second_pos+=1
        elif (aln_first[i]!=gap_char) and (aln_second[i]==gap_char):
            pos_dict_first_seq_to_second_seq[first_pos] = None
            first_pos+=1
        elif (aln_first[i]!=gap_char) and (aln_second[i]!=gap_char):
            pos_dict_first_seq_to_second_seq[first_pos] = second_pos
            first_pos+=1
            second_pos+=1
    return pos_dict_first_seq_to_second_seq

def f_to_v_star(f):
    """ input: single frequencies (Lxq) array
        output: fields of an independent-site Potts model"""
    lf = np.log(f)
    v_star = lf - np.tile(np.mean(lf, axis=1), (f.shape[1],1)).T
    return v_star

def compute_v_star(msa_file, wt_cutoff, pc_single_count, q=21):
    msa_ccmpred = ccmpred.io.read_msa(msa_file, 'fasta')
    weights = ccmpred.weighting.weights_simple(msa_ccmpred, cutoff=wt_cutoff)
    single_counts, double_counts = ccmpred.counts.both_counts(msa_ccmpred, weights)
    neff = np.sum(weights)
    single_freqs = single_counts/neff
    tau = pc_single_count/(neff+pc_single_count)
    uniform_pc = np.zeros_like(single_freqs)
    uniform_pc.fill(1. / single_freqs.shape[1])
    single_freqs = (1-tau)*single_freqs+tau*uniform_pc
    lsingle_freqs = np.log(single_freqs)
    v_star = lsingle_freqs - np.mean(lsingle_freqs[:,:q], axis=1)[:, np.newaxis]
    return v_star[:,:q]


import sys
from collections import deque

N = int(sys.stdin.readline())
li = [int(sys.stdin.readline()) for _ in range(N)]
ans = 0

li.sort()
deq = deque(li)

while len(deq) > 1 and deq[0] < 0 and deq[1] < 0:  # 둘 다 음수인 경우
    ans += deq[0] * deq[1] if deq[0] * deq[1] > deq[0] + deq[1] else deq[0] + deq[1]
    deq.popleft()
    deq.popleft()

while len(deq) > 1 and deq[-2] > 0 and deq[-1] > 0:# 둘 다 양수
    ans += deq[-2]*deq[-1] if deq[-2] * deq[-1] > deq[-2] + deq[-1] else deq[-2] + deq[-1]
    deq.pop()
    deq.pop()

#현재 음수 0 ~ 1개 0 0~여러개 양수 0 ~ 1개 남아있는 상황

while len(deq) > 1:
    ans += deq[0] * deq[1] if deq[0] * deq[1] > deq[0] + deq[1] else deq[0] + deq[1]
    deq.popleft()
    deq.popleft()

if deq:
    ans += deq[0]

print(ans)


import os
import subprocess
import time
import logging.handlers
# import multiprocessing

def measure_time(func):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        elapsed_time = end_time - start_time

        print(f"Function '{func.__name__}' took:")
        print(f"Seconds: {elapsed_time:.2f}")
        print(f"Minutes: {elapsed_time / 60:.2f}")
        print(f"Hours: {elapsed_time / 3600:.2f}")

        return result

    return wrapper

def run_command(command):
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
                         shell=True)
    return iter(p.stdout.readline, b'')

def run_bash_command(command):
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT, shell=True)

    out, err = p.communicate()
    return out

@measure_time
def build_index(filename, currentTime, baseDir):
    """
    Parse the trectext file given, and create an index.
    """
    pathToFolder = baseDir + 'Collections/IndriIndices/'
    if not os.path.exists(pathToFolder):
        os.makedirs(pathToFolder)

    INDRI_BUILD_INDEX = '/lv_local/home/niv.b/indri/bin/IndriBuildIndex'
    CORPUS_PATH = filename
    CORPUS_CLASS = 'trectext'
    MEMORY = '1G'
    INDEX = pathToFolder + currentTime
    STEMMER = 'krovetz'
    if os.path.exists(INDEX):
        run_bash_command('rm -r ' + INDEX)
    command = INDRI_BUILD_INDEX + ' -corpus.path=' + CORPUS_PATH + ' -corpus.class=' + CORPUS_CLASS + ' -index=' + INDEX + ' -memory=' + MEMORY + ' -stemmer.name=' + STEMMER
    out = run_bash_command(command)
    return INDEX

@measure_time
def merge_indices(asrIndex, baseDir, currentTime):
    """
    Merge indices of ASR and ClueWeb09. If MergedIndx exists, it will be deleted.
    """

    INDRI_DUMP_INDEX = '/lv_local/home/niv.b/indri/bin/dumpindex'
    CLUEWEB = f'/lv_local/home/niv.b/cluewebindex'
    pathToFolder = baseDir + 'Collections/'
    MERGED_INDEX = pathToFolder + f'/mergedindex_{currentTime}'
    run_bash_command('rm -r ' + MERGED_INDEX)
    command = INDRI_DUMP_INDEX + ' ' + MERGED_INDEX + ' merge ' + CLUEWEB + ' ' + asrIndex
    out = run_bash_command(command)
    return MERGED_INDEX

@measure_time
def run_ranking_model(mergedIndex, workingSet, currentTime, baseDir):
    """
    workingSet - a file in trec format that dictates which population to work on
    format is: <qid> Q0 <docid> <rank> <score> <experiment name>\n - rank and score arguments can be filled
    arbitrarily they are simply for the desired format

    """
    pathToFolder = baseDir + 'Results/'
    if not os.path.exists(pathToFolder):
        os.makedirs(pathToFolder)
    INDEX = mergedIndex
    WORKING_SET_FILE = workingSet
    # MODEL_DIR = baseDir+"Code/Models/"
    # MODEL_FILE = MODEL_DIR+"model"
    MODEL_FILE = '/lv_local/home/niv.b/content_modification_code-master/rank_models/model_lambdatamart'
    # QUERIES_FILE = '/lv_local/home/niv.b/content_modification_code-master/data/queries_bot_modified_sorted_1.xml'
    QUERIES_FILE = f'/lv_local/home/niv.b/content_modification_code-master/data/query_files/queries_{current_time}.xml'

    FEATURES_DIR = pathToFolder + '/Features/' + currentTime
    if not os.path.exists(FEATURES_DIR):
        os.makedirs(FEATURES_DIR)
    FEATURES_FILE = 'features'

    if not os.listdir(FEATURES_DIR):

        command = baseDir + 'scripts/LTRFeatures ' + QUERIES_FILE + ' -stream=doc -index=' + INDEX + ' -repository=' + INDEX + ' -useWorkingSet=true -workingSetFile=' + WORKING_SET_FILE + ' -workingSetFormat=trec'
        print(command)
        out = run_bash_command(command)
        print(out)
        out = run_command('mv doc*_* ' + FEATURES_DIR)
        print(out)

        if any(file.startswith("doc") for file in os.listdir('/lv_local/home/niv.b/content_modification_code-master')):
            command = f"find /lv_local/home/niv.b/content_modification_code-master -maxdepth 1 -name 'doc*' -exec mv {{}} /lv_local/home/niv.b/content_modification_code-master/Results/Features/{currentTime}/ \\;"
            output = run_bash_command(command)
            print(output.decode('utf-8'))
        else:
            print("No files starting with 'doc' found in the specified directory.")

    command = 'perl ' + baseDir + 'scripts/generate.pl ' + FEATURES_DIR + ' ' + WORKING_SET_FILE
    print(command)
    out = run_bash_command(command)
    print(out)
    command = '/lv_local/home/niv.b/opt/java/jdk1.8.0/bin/java -jar ' + baseDir + 'scripts/RankLib.jar -load ' + MODEL_FILE + ' -rank ' + FEATURES_FILE + ' -score predictions.tmp'
    print(command)
    out = run_bash_command(command)
    print(out)
    command = 'cut -f3 predictions.tmp > predictions'
    print(command)
    out = run_bash_command(command)
    print(out)
    run_bash_command('rm predictions.tmp')
    RANKED_LIST_DIR = pathToFolder + 'RankedLists/'
    if not os.path.exists(RANKED_LIST_DIR):
        os.makedirs(RANKED_LIST_DIR)
    PREDICTIONS_FILE = 'predictions'
    command = 'perl ' + baseDir + '/scripts/order.pl ' + RANKED_LIST_DIR + '/LambdaMART' + currentTime + ' ' + FEATURES_FILE + ' ' + PREDICTIONS_FILE
    print(command)
    out = run_bash_command(command)
    print(out)
    return RANKED_LIST_DIR + '/LambdaMART' + currentTime

# if __name__ == '__main__':
#     workingSet = '/lv_local/home/niv.b/content_modification_code-master/trecs/working_set.trectext'
#     baseDir = '/lv_local/home/niv.b/content_modification_code-master/'
#     currentTime = "1"
#     documents = '/lv_local/home/niv.b/content_modification_code-master/trecs/bot_followup.trectext'
#
#     asrIndex = build_index(documents, currentTime, baseDir)
#     print("build_index done...")
#     mergedIndex = merge_indices(asrIndex, baseDir)
#     # mergedIndex = '/lv_local/home/niv.b/content_modification_code-master/Collections/mergedindex'
#     print("merge_indices done...")
#     res = run_ranking_model(mergedIndex, workingSet, currentTime, baseDir)
#     print("run_ranking_model done...")
#     print(res)
#
#     # res = run_ranking_model('/lv_local/home/niv.b/cluewebindex', workingSet, currentTime, baseDir)
#     x = 1

def main_task(currentTime):
    logger.info("Starting...")
    baseDir = '/lv_local/home/niv.b/content_modification_code-master/'
    documents = f'/lv_local/home/niv.b/content_modification_code-master/trecs/bot_followup_{currentTime}.trectext'
    workingSet = f'/lv_local/home/niv.b/content_modification_code-master/trecs/working_set_{currentTime}.trectext'

    # asrIndex = build_index(documents, currentTime, baseDir)
    # print("build_index done...")
    # logger.info("build_index done...")
    #
    # mergedIndex = merge_indices(asrIndex, baseDir, currentTime)
    # print("merge_indices done...")
    # logger.info("merge_indices done...")
    # time.sleep(10)

    mergedIndex = baseDir + 'Collections/' + f'/mergedindex_{currentTime}' # in case of running only the ranking model
    res = run_ranking_model(mergedIndex, workingSet, currentTime, baseDir)
    print("run_ranking_model done...")
    logger.info("run_ranking_model done...")

    print(res)
    logger.info(f'{res}')

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
    logger = logging.getLogger('ranking_logger')
    logger.setLevel(logging.DEBUG)
    # num_cores = multiprocessing.cpu_count()
    # pool = multiprocessing.Pool(processes=num_cores)
    # current_times = ["g1"]
    # print(f"running {current_times[0]} model...")
    # pool.map(main_task, current_times)
    # pool.close()
    # pool.join()
    current_time = "llama13btrain"
    print(f'Starting version {current_time}...')
    logger.info(f'Starting version {current_time}...')
    main_task(current_time)


import os
import sys
import random
from tqdm import tqdm
import torch
import numpy as np
import scipy

this_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(this_path)
transformemnn_path = this_path.strip("data") + "/training/"
# sys.path.append(transformemnn_path + "/db_dataloader")
droidlet_path = this_path.strip("data") + "/fairo/"
sys.path.append(droidlet_path)
sys.path.append(transformemnn_path + "/utils/")
sys.path.append(transformemnn_path)
from droidlet.lowlevel.minecraft.small_scenes_with_shapes import build_shape_scene
from droidlet.shared_data_structs import ErrorWithResponse
from query_builder import QAZoo
from utils.memory_utils import build_memory, get_memory_text
from utils.nsm_utils import make_world_args_from_config
from db_dataloader.db_encoder import DBEncoder
from config_args import get_opts, read_configs
from active_agent_world import EpisodeRunner, WorldBuilder
from utils.log_data import maybe_make_save_dir, update_data_info, save_files
from utils.nsm_utils import NULL_MEMID
from transformers import GPT2Tokenizer

tokenizer = GPT2Tokenizer.from_pretrained("gpt2")

def print_num_choices(opts):
    num_choices = 0
    for i in range(opts.props_per_name):
        num_choices += scipy.special.comb(opts.num_props, i + 1)
    possible_samples = ((num_choices) ** opts.num_names) * (
        opts.num_names + opts.num_props
    )
    print("number of possible samples: {:.0f}\n".format(possible_samples))

def increment_clause_counts(clause_types, clause_type_counts):
    for clause_type in clause_types:
        clause_type_counts[clause_type] = clause_type_counts.get(clause_type, 0) + 1
        clause_type_counts["total_clauses"] += 1

def increment_conjunction_counts(conj_type, conj_type_counts):
    if conj_type:
        conj_type_counts[conj_type] += 1
    else:
        conj_type_counts["NONE"] += 1
    conj_type_counts["total_samples"] += 1

def increment_null_counts(is_null, null_counts):
    if is_null:
        null_counts["null_samples"] += 1
    null_counts["total_samples"] += 1

def check_update_proportions(
    args,
    configs,
    memids,
    query_type,
    clause_types,
    clause_type_counts,
    conj_type,
    conj_type_counts,
    null_counts,
    max_null_percent,
):
    """
    Checks query clause types to see if they are within the target distributions
    Returns reject_sample True/False
    """
    # Clause types
    if query_type == "FiltersQA":
        for clause_type in clause_types:
            target_percentage = configs[query_type]["clause_types"][clause_type]
            actual_percentage = (clause_type_counts.get(clause_type, 0) + 1) / (
                clause_type_counts["total_clauses"] + 1
            )
            if actual_percentage > target_percentage:
                return False

    # AND and NONE conjunctions
    if (args.num_clauses > 1) and (configs[query_type].get("operator_probs", False)):
        target_percentage = configs[query_type]["operator_probs"][conj_type]
        actual_percentage = (conj_type_counts.get(conj_type, 0) + 1) / (
            conj_type_counts["total_samples"] + 1
        )
        if actual_percentage > target_percentage:
            return False

    # NULL answers
    is_null = False
    if (not memids) or (memids[0] == NULL_MEMID):
        is_null = True
        actual_percentage = (null_counts["null_samples"] + 1) / (
            null_counts["total_samples"] + 1
        )
        if actual_percentage > max_null_percent:
            return False

    # TODO: sometimes, nearly all the refobjs are true. we should reject some of these
    # is_all = (len(memids[0]) >= len(db_dump['context']['reference_objects'])-2)
    # if is_all:

    # if it gets here, it passed all of the percentage checks and we will use this query
    increment_clause_counts(clause_types, clause_type_counts)
    increment_conjunction_counts(conj_type, conj_type_counts)
    increment_null_counts(is_null, null_counts)
    return True

def find_valid_query(
    args,
    Z,
    memory,
    snapshots,
    encoder,
    configs,
    clause_type_counts,
    conj_type_counts,
    null_counts,
    max_null_percent,
    verbose,
):
    """
    Given a single context memory, this searches for a query that doesn't doesn't go over the target proportions
    """
    search_counter = 0
    while search_counter < 10:  # search 10 different queries
        search_counter += 1
        try:
            db_dump = Z.get_qa(memory=memory, snapshots=snapshots)
        except (ErrorWithResponse, KeyError) as e:
            # print("ERROR: {}".format(e))
            # keep on truckin
            continue

        if not db_dump:
            if verbose:
                print("REJECTED: inactive clause")
            continue

        keep_sample = check_update_proportions(
            args,
            configs["QA"],
            db_dump["memids_and_vals"][0],
            db_dump["sample_query_type"],
            db_dump["sample_clause_types"],
            clause_type_counts,
            db_dump["sample_conjunction_type"],
            conj_type_counts,
            null_counts,
            max_null_percent,
        )

        if not keep_sample:
            if verbose:
                print("REJECTED: incorrect proportions")
            continue

        db_dump["context"]["text"] = get_memory_text(
            memory, db_dump
        )  # move to get_qa ?
        encoded_sample = encoder.encode_all(db_dump)

        return encoded_sample

    # couldn't find a valid query. Move on and generate a new scene
    return False

def main(opts):
    configs = opts.configs

    random.seed(opts.seed)
    torch.manual_seed(opts.seed)
    np.random.seed(opts.seed)

    save_dir, metadata_file_name = maybe_make_save_dir(opts)

    if not opts.tokenizer_path:
        opts.tokenizer_path = save_dir

    if save_dir:
        print(save_dir)

    if not configs.get("WORLD"):
        print_num_choices(opts)
    else:
        world_opts = make_world_args_from_config(configs)
        opts.SL = world_opts.SL
        opts.H = world_opts.H
        print(
            'WARNING using config["WORLD"], ignoring opts.num_names, opts.num_props, opts.SL, etc.'
        )

    encoder = DBEncoder(
        query_encode=opts.query_encode,
        SL=opts.SL,
        COORD_SHIFT=(-opts.SL // 2, -opts.SL // 2, -opts.SL // 2),
        tokenizer=opts.tokenizer,
        tokenizer_path=opts.tokenizer_path,
        remove_instseg_locations=True,
    )
    examples = []
    info = {}

    num_iterations = opts.num_examples * 20

    pbar = None
    if opts.active_world_steps > 0:
        W = WorldBuilder()
    if not opts.disable_tqdm:
        pbar = tqdm(total=opts.num_examples, ncols=50)

    Z = QAZoo(opts, configs)

    clause_type_counts = {"total_clauses": 0}
    conj_type_counts = {"total_samples": 0, "AND": 0, "OR": 0, "NONE": 0}
    null_counts = {"total_samples": 0, "null_samples": 0}
    for i in range(num_iterations):
        world_spec = None
        if configs.get("WORLD"):
            world_opts = make_world_args_from_config(configs)
            world_spec = build_shape_scene(world_opts)

        if opts.active_world_steps > 0:
            a, w, lf, chat = W.instantiate_world_from_spec(configs, opts)
            runner = EpisodeRunner(a, snapshot_freq=50, keep_on_truckin=True)
            N = opts.active_world_steps
            if lf is not None:
                runner.run_episode(
                    min_steps=N, max_steps=N, logical_form=lf, chatstr=chat
                )
            else:
                runner.run_episode(min_steps=N, max_steps=N, no_agent_step=True)
            snapshots = runner.snapshots
            memory = a.memory
        else:
            snapshots = None
            memory = build_memory(opts, world_spec=world_spec)

        Z = QAZoo(opts, configs)

        encoded_sample = find_valid_query(
            opts,
            Z,
            memory,
            snapshots,
            encoder,
            configs,
            clause_type_counts,
            conj_type_counts,
            null_counts,
            opts.max_percent_null,
            opts.verbose,
        )

        if encoded_sample:
            encoded_sample["world_configs"] = {"SL": opts.SL}
            if pbar:
                pbar.update(1)

            examples.append(encoded_sample)
            info = update_data_info(info, encoded_sample)
            if info["num_samples"] == opts.num_examples:
                break

    assert info["num_samples"] == opts.num_examples, "num_samples != opts.num_examples"

    print("{}\n{}\n{}".format(clause_type_counts, conj_type_counts, null_counts))

    if save_dir:
        save_files(examples, save_dir, encoder, opts, info, metadata_file_name)

if __name__ == "__main__":
    parser = get_opts()

    opts = parser.parse_args()
    opts.configs = read_configs(opts.config_file)

    main(opts)