import sqlite3
import random
from PyQt5 import uic
from PyQt5.QtWidgets import QMainWindow, QLineEdit
from PyQt5.QtCore import QTimer
from PyQt5.QtGui import QPixmap
from ClickedLabel import ClickedLabel

class Game(QMainWindow):
    def __init__(self):
        super().__init__()
        uic.loadUi("resources/ui/game.ui", self)
        self.setFixedSize(1280, 720)
        self.move(0, 0)

        self.userName = None

        self.setupUi()

        self.errors = 0
        self.mainMenu = None
        self.winWindow = None
        self.firstTime = True

        self.sec = 0
        self.timer = QTimer()
        self.setTime()
        self.timer.timeout.connect(self.counter)

        self.randomPic()
        self.makeAllOff()
        self.makeAllDefault()

        self.startButton.clicked.connect(self.start)
        self.stopButton.clicked.connect(self.stop)
        self.exitButton.clicked.connect(self.exit)

        for el in self.labels:
            el.clicked.connect(self.nothing)

    def setupUi(self):
        self.picPart_1 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_1, 0, 0, 1, 1)
        self.picPart_2 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_2, 0, 1, 1, 1)
        self.picPart_3 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_3, 0, 2, 1, 1)
        self.picPart_4 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_4, 0, 3, 1, 1)
        self.picPart_5 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_5, 1, 0, 1, 1)
        self.picPart_6 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_6, 1, 1, 1, 1)
        self.picPart_7 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_7, 1, 2, 1, 1)
        self.picPart_8 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_8, 1, 3, 1, 1)
        self.picPart_9 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_9, 2, 0, 1, 1)
        self.picPart_10 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_10, 2, 1, 1, 1)
        self.picPart_11 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_11, 2, 2, 1, 1)
        self.picPart_12 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_12, 2, 3, 1, 1)
        self.picPart_13 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_13, 3, 0, 1, 1)
        self.picPart_14 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_14, 3, 1, 1, 1)
        self.picPart_15 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_15, 3, 2, 1, 1)
        self.picPart_16 = ClickedLabel()
        self.gridLayout.addWidget(self.picPart_16, 3, 3, 1, 1)
        self.picPart_1.setName("1")
        self.picPart_2.setName("2")
        self.picPart_3.setName("3")
        self.picPart_4.setName("4")
        self.picPart_5.setName("5")
        self.picPart_6.setName("6")
        self.picPart_7.setName("7")
        self.picPart_8.setName("8")
        self.picPart_9.setName("9")
        self.picPart_10.setName("10")
        self.picPart_11.setName("11")
        self.picPart_12.setName("12")
        self.picPart_13.setName("13")
        self.picPart_14.setName("14")
        self.picPart_15.setName("15")
        self.picPart_16.setName("16")

        self.pics = ["resources/pics/auto1.jpeg", "resources/pics/auto2.jpeg", "resources/pics/auto3.jpeg", "resources/pics/auto4.jpeg",
                     "resources/pics/auto5.jpeg", "resources/pics/auto6.jpeg", "resources/pics/auto7.jpeg", "resources/pics/auto8.jpeg",
                     "resources/pics/auto1.jpeg", "resources/pics/auto2.jpeg", "resources/pics/auto3.jpeg", "resources/pics/auto4.jpeg",
                     "resources/pics/auto5.jpeg", "resources/pics/auto6.jpeg", "resources/pics/auto7.jpeg", "resources/pics/auto8.jpeg"]

        self.labels = [self.picPart_1, self.picPart_2, self.picPart_3, self.picPart_4,
                       self.picPart_5, self.picPart_6, self.picPart_7, self.picPart_8,
                       self.picPart_9, self.picPart_10, self.picPart_11, self.picPart_12,
                       self.picPart_13, self.picPart_14, self.picPart_15, self.picPart_16]

        for el in self.labels:
            el.hide()

    # сделал так, так как при setEnabled(False) виджеты окрашивются в серый, а замена на функцию nothing() это исправляет
    def reconnect(self, signal, newhandler=None, oldhandler=None):
        while True:
            try:
                if oldhandler is not None:
                    signal.disconnect(oldhandler)
                else:
                    signal.disconnect()
            except TypeError:
                break
        if newhandler is not None:
            signal.connect(newhandler)

    # пустая функция чтобы реализовать аналог setEnabled(False)
    def nothing(self):
        pass

    def addWinWindow(self, winWindow):
        self.winWindow = winWindow

    # перемешивание картинок
    def randomPic(self):
        random.shuffle(self.pics)

    # замена всех picLabel на работающую функцию
    def makeAllOn(self):
        for el in self.labels:
            self.reconnect(el.clicked, self.picClicked)

    # замена всех picLabel на пустую функцию
    def makeAllOff(self):
        for el in self.labels:
            self.reconnect(el.clicked, self.nothing)

    # замена всех картинок picLabel на дефолтную
    def makeAllDefault(self):
        for el in self.labels:
            el.setPixmap(QPixmap("resources/pics/default.jpeg"))

    # замена всех picLabel на их картинки
    def showAll(self):
        for el in self.labels:
            el.setPixmap(QPixmap(self.pics[int(el.getName()) - 1]))

    # проверка все ли картинки были найдены
    def checkAll(self):
        for el in self.labels:
            if not el.outOfGame:
                return False
        return True

    # показ 2 нажатых картинок и смена их на дефолтные
    def make2Default(self, el1, el2):
        def func():
            for el in self.labels:
                self.reconnect(el.clicked, self.picClicked)
                if el.outOfGame:
                    self.reconnect(el.clicked, self.nothing)
                    el.setPixmap(QPixmap("resources/pics/outofgame.jpeg"))
            if not el1.outOfGame:
                el1.setPixmap(QPixmap("resources/pics/default.jpeg"))
            if not el2.outOfGame:
                el2.setPixmap(QPixmap("resources/pics/default.jpeg"))
        return func

    # функция выполняется если нажата picLabel(картинка)
    def picClicked(self):
        sender = self.sender()
        if sender.wasClicked:
            return
        sender.wasClicked = True
        sender.setPixmap(QPixmap(self.pics[int(sender.getName()) - 1]))
        for el in self.labels:
            if sender == el:
                continue
            if el.wasClicked:
                if self.pics[int(sender.getName()) - 1] == self.pics[int(el.getName()) - 1]:
                    self.makeAllOff()
                    self.timerScreen = QTimer()
                    self.timerScreen.setSingleShot(True)
                    self.timerScreen.timeout.connect(
                        self.make2Default(el, sender))
                    self.timerScreen.start(1500)
                    sender.wasClicked = False
                    el.wasClicked = False
                    sender.outOfGame = True
                    el.outOfGame = True
                else:
                    self.makeAllOff()
                    self.timerScreen = QTimer()
                    self.timerScreen.setSingleShot(True)
                    self.timerScreen.timeout.connect(
                        self.make2Default(el, sender))
                    self.timerScreen.start(1500)
                    sender.wasClicked = False
                    el.wasClicked = False
                    self.errors += 1
                    self.wrongsNumber.display(self.errors)

        # если все пары были найдены
        if self.checkAll():
            self.userName = self.mainMenu.autorization.getUserName()
            self.winWindow.show()
            self.setEnabled(False)
            hour = self.sec / 3600
            minut = (self.sec % 3600) / 60
            sec = (self.sec % 3600) % 60
            self.winWindow.timeLabel.setText(
                "%02d:%02d:%02d" % (hour, minut, sec))
            self.winWindow.errorsLabel.setText(str(self.errors))

            db = sqlite3.connect('MemoryTrainingDB.db')
            sql = db.cursor()
            sql.execute(
                f"SELECT time FROM users WHERE login = '{self.userName}'")
            bestTime = sql.fetchone()
            bestTime = int(list(bestTime)[0])
            sql.execute(
                f"SELECT mistakes FROM users WHERE login = '{self.userName}'")
            bestErrors = sql.fetchone()
            bestErrors = int(list(bestErrors)[0])
            if 1000 - bestTime - bestErrors * 5 < 1000 - self.sec - self.errors * 5:
                sql.execute(
                    f"UPDATE users SET time = {self.sec} WHERE login = '{self.userName}'")
                db.commit()
                sql.execute(
                    f"UPDATE users SET mistakes = {self.errors} WHERE login = '{self.userName}'")
                db.commit()
            self.stop()

    # кнопка старта
    def start(self):
        # если кнопка была нажата впервый раз
        if self.firstTime:
            self.firstTime = False
            for el in self.labels:
                el.show()
            self.timerScreen = QTimer()
            self.timerScreen.setSingleShot(True)
            self.timerScreen.timeout.connect(self.makeAllDefault)
            self.timerScreen.start(5000)
            self.timerScreen1 = QTimer()
            self.timerScreen1.setSingleShot(True)
            self.timerScreen1.timeout.connect(self.makeAllOn)
            self.timerScreen1.start(5000)
            self.wrongsNumber.display(self.errors)
            self.showAll()
        else:
            self.makeAllOn()
            for el in self.labels:
                if el.outOfGame:
                    self.reconnect(el.clicked, self.nothing)
                    el.setPixmap(QPixmap("resources/pics/outofgame.jpeg"))
        self.timer.start(1000)

    def stop(self):
        for el in self.labels:
            el.wasClicked = False
        self.timer.stop()
        self.makeAllDefault()
        self.makeAllOff()

    def counter(self):
        self.sec += 1
        self.setTime()

    def setTime(self):
        hour = self.sec / 3600
        minut = (self.sec % 3600) / 60
        sec = (self.sec % 3600) % 60
        self.timeLabel.setText("%02d:%02d:%02d" % (hour, minut, sec))

    def exit(self):
        self.stop()
        for el in self.labels:
            el.hide()
            el.outOfGame = False
            el.wasClicked = False
        self.firstTime = True
        self.randomPic()
        self.mainMenu.show()
        self.timer.stop()
        self.sec = 0
        self.errors = 0
        self.setTime()
        self.randomPic()
        self.makeAllDefault()
        self.wrongsNumber.display(self.errors)
        self.makeAllOff()
        self.hide()

    def addMenu(self, mainMenu):
        self.mainMenu = mainMenu


import os
import tempfile
import boto3
from .localfile import deletefile
import controllers.configs as cfg

class S3EventsImages:
    def __init__(self):
        self.bucket = cfg.BUCKET
        self.client = boto3.client('s3')

    def download(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            _, tmpfolder = os.path.split(tempfile.mkdtemp())
            tmpfolder = cfg.IMAGE_FILE_MOUNTPOINT + tmpfolder
            os.mkdir(tmpfolder)
            tmpfile = os.path.join(tmpfolder, event_id+"."+image_id)
            with open(tmpfile, 'wb') as f:
                self.client.download_fileobj(self.bucket, fileobj, f)
        except Exception as ex:
            deletefile(tmpfile)
            raise
        return tmpfile

    def delete(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            if not self.__find(event_id, image_id):
                raise
            self.client.delete_object(Bucket=self.bucket, Key=fileobj)
        except Exception as ex:
            raise

    def upload(self, imagefile, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            with open(imagefile, 'rb') as f:
                self.client.upload_fileobj(f, self.bucket, fileobj)
        except Exception as ex:
            raise

    def __find(self, event_id, image_id):
        try:
            fileobj = '%s/%s/%s.jpg' % (cfg.AWS_IMAGE_FOLDER_PREFIX, event_id, image_id)
            get_folder_objects = self.client.list_objects_v2(
                Bucket=self.bucket,
                Delimiter='',
                EncodingType='url',
                MaxKeys=1000,
                Prefix=fileobj,
                FetchOwner=False,
                StartAfter=''
            )
            if not get_folder_objects.get('Contents'):
                return False
        except Exception as ex:
            raise
        return True


import requests
from bs4 import BeautifulSoup
from urllib.request import urlretrieve

def run():
    for i in range (1,6):
        response = requests.get('https://xkcd.com/{}/'.format(i), verify = True)
        soup = BeautifulSoup(response.content,'html.parser')
        image_container = soup.find(id='comic')

        image_url = image_container.find('img')['src']
        image_name = image_url.split('/')[-1]
        print('Descargando la imagen {}'.format(image_name))
        urlretrieve('https:{}'.format(image_url),image_name)

if __name__ == '__main__':
    run()


# -*- coding: utf-8 -*-

import codecs
import json
import os
import sys
import time
import urllib2
from urllib import urlencode

from config import read_config
from downloader import Downloader
from memoize import Memoize
from messages import MessageWriter
from reporter import Reporter
import vk_auth

def _api(method, params, token):
    params.append(("access_token", token))
    url = "https://api.vk.com/method/%s?%s" % (method, urlencode(params))
    payload = urllib2.urlopen(url).read()
    payload_json = json.loads(payload)
    if not "response" in payload_json:
        sys.exit("Request failed:\nURL     = %s\nPAYLOAD = %s" % (url, payload))
    return payload_json["response"]

config = read_config()
reporter = Reporter.std_reporter()
downloader = Downloader(reporter, directory=config["export"]["output_directory"])

# auth to get token

try:
    reporter.progress("Authenticating as %s" % config["auth"]["username"], pad=True)
    token, user_id = vk_auth.auth(config["auth"]["username"], config["auth"]["password"], config["app"]["id"], 'messages')
    reporter.line("OK")
except RuntimeError:
    reporter.line("FAILED")
    sys.exit("Cannot authenticate, please check your credentials in .auth.ini")

# get some information about chat

selector = "chat_id" if config["export"]["is_group_chat"] else "uid"
messages = _api("messages.getHistory", [(selector, config["export"]["chat_id"])], token)

# prepare output

if config["export"]["output_directory"] is not None:
    if not os.path.exists(config["export"]["output_directory"]):
        os.makedirs(config["export"]["output_directory"])
output_filename = 'vk_exported_dialogue_%s%s.txt' % ('ui' if not config["export"]["is_group_chat"] else 'c', config["export"]["chat_id"])
output_path = Downloader.resolve_path(config["export"]["output_directory"], output_filename)
out = codecs.open(output_path, "w+", "utf-8")

def resolve_uid_details(uid):
    return _api("users.get", [("user_ids", uid)], token)[0]

resolve_uid_details = Memoize(resolve_uid_details)

message_writer = MessageWriter(out, downloader, lambda uid: resolve_uid_details(uid), save_photos=config["export"]["save_photos"])

mess = 0
max_part = 200  # Due to vk.api

cnt = messages[0]
reporter.line("Message count: %s" % cnt)

request_num = 0

while mess != cnt:
    # Try to retrieve info anyway

    if request_num % 5 == 0:
        time.sleep(1)
    request_num += 1

    while True:
        try:
            message_part = _api(
                "messages.getHistory",
                [(selector, config["export"]["chat_id"]), ("offset", mess), ("count", max_part), ("rev", 1)],
                token
            )
        except Exception as e:
            reporter.error_line('Got error %s, continue...' % e)
            continue
        break

    try:
        for i in range(1, 201):
            message_writer.write(message_part[i]["uid"], message_part[i])
    except IndexError:
        break

    result = mess + max_part
    if result > cnt:
        result = (mess - cnt) + mess
    mess = result
    reporter.line("Exported %s messages of %s" % (mess, cnt))

out.close()
reporter.line('Export done!')


# -*- coding: utf-8 -*-

from core import *
from functools import reduce
import random
import colorsys
import sys
import os
import json
import datetime
from itertools import product

import numpy as np

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib import pyplot as plt
from matplotlib_venn import venn3, venn3_circles

matplotlib.use('TkAgg')

def protocol6(results, repair_tools, disposition=110):
    fig = plt.figure(figsize=(15, 12))

    axs = []
    for tool, index in zip(repair_tools, range(len(repair_tools))):
        ax = fig.add_subplot(disposition + index + 1)
        axs.append(ax)
        protocol6_subplot(results, tool, ax, not index)

    return fig

def mix_colors(*colors):
    color_avg = [0] * 3
    colors_len = len(colors)
    for color in colors:
        rgb = hex_to_rgb(color)
        for i in range(len(rgb)):
            color_avg[i] += rgb[i] / colors_len
    print(color_avg)
    return color_avg

def gen_get_colors_venn(labels, colors):
    def get_colors_venn(group_id):
        result = []
        for index, value in enumerate(group_id):
            if value == '1':
                result.append(colors[labels[index]])
        return tuple(result)
    return get_colors_venn

def venn(data):
    labels = tuple(data.keys())
    v = venn3(data.values(), labels)
    colors = {
        'Codebuff': codebuff_color,
        'Naturalize': naturalize_color,
        'Styler': styler_color
    }
    get_colors_venn = gen_get_colors_venn(labels, colors)
    alpha = 2/3
    for id in [''.join(elements) for elements in product('01', repeat=3)]:
        if id != '000':
            v.get_patch_by_id(id).set_color(mix_colors(*get_colors_venn(id)))
            v.get_patch_by_id(id).set_alpha(alpha)

    plt.show()

def protocol6_subplot(results, repair_tool, ax, y_axis=True):
    error_type_repair = dict()
    for result in results:
        file_with_cs_errors = result["file_with_cs_errors_{}".format(repair_tool)]
        for file, liste in result["file_with_cs_errors_ugly"].items():
            for modification in liste:
                errors = list(set([error["source"].split(".")[-1] for error in modification["errors"]]))
                errors_after_repair = []
                if file in file_with_cs_errors:
                    modification_repaired = list(filter(lambda x: x["type"] == modification["type"] and x["modification_id"] == modification["modification_id"], file_with_cs_errors[file]))
                    if len(modification_repaired):
                        if len( modification_repaired[0]["errors"]):
                            errors_after_repair = list(set([error["source"].split(".")[-1] for error in modification_repaired[0]["errors"]]))
                        else:
                            errors_after_repair = errors

                for error in errors:
                    if error not in error_type_repair:
                        error_type_repair[error] = {"repaired": 0, "errors_remaining": 0, "not_repaired": 0}
                    if error in errors_after_repair:
                        error_type_repair[error]["not_repaired"] += 1
                    else:
                        if len(errors_after_repair):
                            error_type_repair[error]["errors_remaining"] += 1
                        else:
                            error_type_repair[error]["repaired"] += 1
    def f(x):
        s = sum(x.values())
        obj = {key: (value / s) for key, value in x.items()}
        obj["sum"] = s
        return obj
    error_type_repair = {key[:-5]:f(value) for key, value in error_type_repair.items()}

    objects = error_type_repair.keys()
    y_pos = np.arange(len(objects))
    types = ("repaired", "errors_remaining", "not_repaired")
    colors = {"repaired": "#2ecc71", "errors_remaining": "#f39c12", "not_repaired": "#e74c3c"}
    sum_left = [0] * len(error_type_repair)
    for type in types:
        performance = [item[type] for item in error_type_repair.values()]
        ax.barh(y_pos, performance, align='center', color=colors[type], left=sum_left, label=type)
        sum_left = list(map(lambda x, y: x+y, sum_left, performance))

    def with_percentage(error_type_repair, y_pos):
        for error, pos in zip(error_type_repair.values(), y_pos):
            value = error["sum"]
            plt.text(1.02, pos - 0.2, '%d' % int(value), ha='center', va='bottom')
    with_percentage(error_type_repair, y_pos)
    if y_axis:
        plt.yticks(y_pos, objects, rotation=0)
    else:
        plt.yticks(y_pos, [""]*len(objects), rotation=0)
    plt.xlabel('')
    plt.title('Percentage of repaired checkstyle errors by {}.'.format(repair_tool))
    plt.legend()

def plot_repaired_files(results):

    counts = ('naturalize', 'codebuff', 'both_sniper')

    barWidth = 1. / (len(counts) + 1)
    bars = [[] for i in range(len(counts)) ]

    labels = []

    for result in results:
        # labels.append( exp.corpus.name + "(" + str(exp.corpus.get_number_of_files()) + " files)" )
        with_errors = result["number_of_injections"] * result["corrupted_files_ratio_ugly"]
        labels.append("{}, \n /{} injections".format(result["name"], int(with_errors)))
        for count, i in zip(counts, range(len(counts))):
            if count == "both_sniper":
                file_with_cs_errors_codebuff_sniper = result["file_with_cs_errors_codebuff_sniper"]
                file_with_cs_errors_naturalize_sniper = result["file_with_cs_errors_naturalize_sniper"]
                c = 0
                for file, modifications in file_with_cs_errors_codebuff_sniper.items():
                    if file in file_with_cs_errors_naturalize_sniper:
                        errors = [m["type"]+str(m["modification_id"]) for m in file_with_cs_errors_naturalize_sniper[file]]
                        for modification in modifications:
                            if modification["type"]+str(modification["modification_id"]) in errors:
                                c += 1
                bars[i].append( 1 - c / with_errors )
            else:
                prop = result["corrupted_files_ratio_" + count]
                bars[i].append( 1 - ( result["number_of_injections"] * result["corrupted_files_ratio_" + count]) / with_errors )
            # bars[i].append( result["corrupted_files_ratio_" + count] )

    # Set position of bar on X axis
    r = []
    r.append(np.arange(len(labels)))
    for i in range(1,len(counts)):
        r.append([x + barWidth for x in r[i-1]])

    def with_percentage(bars):
        for bar in bars:
            height = bar.get_height()
            plt.text(bar.get_x() + bar.get_width()/2., 1*height, '%d' % int(height*100) + "%", ha='center', va='bottom')
    # Make the plot
    for count, i in zip(counts, range(len(counts))):
        with_percentage(plt.bar(r[i], bars[i], width=barWidth, edgecolor='white', label=count))

    modifications = (2,2,2,2,2)

    # Add xticks on the middle of the group bars
    plt.xlabel('Proportion of files with errors (m=' + str(modifications) + ')', fontweight='bold')
    plt.xticks([r + barWidth * (len(counts)-1) / 2 for r in range(len(results))], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(bottom=0.20)
    # Create legend & Show graphic
    plt.legend()

def avg(array_list):
    return sum(array_list)/len(array_list)

def hex_to_rgb(hex_color):
    hex = hex_color.lstrip('#')
    return tuple(int(hex[i:i+2], 16)/255. for i in (0, 2 ,4))

def n_bar_plot(plot_data):
    counts = plot_data['labels']
    colors = plot_data['colors']

    plot_average = True

    horizontal = True

    barWidth = 1. / (len(counts) + 1)
    bars = [ # Transpose
        [ line[collumn] for line in plot_data['data'].values() ]
        for collumn in range(len(plot_data['labels']))
    ]

    labels = list(plot_data['data'].keys())

    if plot_average:
        print(bars)
        bars = [
            numbers + [avg(numbers)]
            for numbers in bars
        ]
        labels += ['Average']
        print(labels)

    r = []
    r.append(list(np.arange(len(labels))))
    if plot_average:
        r[0][-1] += 0.5
    for i in range(1,len(counts)):
        r.append([x + barWidth for x in r[i-1]])

    def with_percentage(bars, bar_color='#FFFFFF'):
        if horizontal:
            for bar in bars:
                width = bar.get_width()
                ha = 'right' if width > 0.2 else 'left'
                color = '#000000' if ha == 'left' or colorsys.rgb_to_hls(*hex_to_rgb(bar_color))[1] > 0.45 else '#FFFFFF'
                plt.text(1*width, bar.get_y() + bar.get_height()*1.25/2. , f'{(width*100):.1f}%', ha=ha, va='center', fontsize=12, color=color)
        else:
            for bar in bars:
                height = bar.get_height()
                plt.text(bar.get_x() + bar.get_width()/2., 1*height, f'{(height*100):.1f}%', ha='center', va='bottom')
    # Make the plot
    for i, count in enumerate(counts):
        if horizontal:
            with_percentage(plt.barh(r[i], bars[i], height=barWidth * 0.95, edgecolor='white', label=count, color=colors[count]), bar_color=colors[count])
        else:
            with_percentage(plt.bar(r[i], bars[i], width=barWidth * 0.95, edgecolor='white', label=count, color=colors[count]))

    # Add xticks on the middle of the group bars
    plt.xlabel(plot_data.get('x_label', ''), fontsize=15)
    plt.ylabel(plot_data.get('y_label', ''), fontsize=15)
    if horizontal:
        plt.yticks([r + barWidth * (len(counts)-1) / 2 for r in r[0]], labels, fontsize=15)
    else:
        plt.xticks([r + barWidth * (len(counts)-1) / 2 for r in r[0]], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(bottom=0.05, left=0.17, right=0.99, top=0.99)

    if horizontal:
        plt.gca().invert_yaxis()

    # Create legend & Show graphic
    plt.legend()
    plt.show()

def plot_diffs(results):
    modifications = (2,2,2,2,2)

    counts = ('naturalize', 'naturalize_sniper', 'codebuff', 'codebuff_sniper')

    barWidth = 1. / (len(counts) + 1)
    bars = [[] for i in range(len(counts)) ]

    labels = []

    for result in results:
        # labels.append( exp.corpus.name + "(" + str(exp.corpus.get_number_of_files()) + " files)" )
        with_errors = result["number_of_injections"] * result["corrupted_files_ratio_ugly"]
        labels.append("{}, \n /{} injections".format(result["name"], int(with_errors)))
        for count, i in zip(counts, range(len(counts))):
            bars[i].append( result["diffs_avg_" + count] )
            # bars[i].append( result["corrupted_files_ratio_" + count] )

    # Set position of bar on X axis
    r = []
    r.append(np.arange(len(labels)))
    for i in range(1,len(counts)):
        r.append([x + barWidth for x in r[i-1]])

    # Make the plot
    for count, i in zip(counts, range(len(counts))):
        plt.bar(r[i], bars[i], width=barWidth, edgecolor='white', label=count)

    # Add xticks on the middle of the group bars
    plt.xlabel('Proportion of files with errors (m=' + str(modifications) + ')', fontweight='bold')
    plt.xticks([r + barWidth * (len(counts)-1) / 2 for r in range(len(results))], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(bottom=0.30)
    # Create legend & Show graphic
    plt.legend()

def plot_percentage_of_errors(results):
    modifications = (5, 5)

    barWidth = 0.25
    bars1 = []
    naturalize_res = []
    codebuff_res = []
    labels = []

    for result in results:
        # labels.append( exp.corpus.name + "(" + str(exp.corpus.get_number_of_files()) + " files)" )
        labels.append(result["name"])
        bars1.append(result["corrupted_files_ratio_ugly"] )
        naturalize_res.append( result["corrupted_files_ratio_naturalize"] )
        codebuff_res.append( result["corrupted_files_ratio_codebuff"] )

    # Set position of bar on X axis
    r1 = np.arange(len(bars1))
    r2 = [x + barWidth for x in r1]
    r3 = [x + barWidth for x in r2]

    # Make the plot
    plt.bar(r1, bars1, color='#3498db', width=barWidth, edgecolor='white', label='Error injection')
    plt.bar(r2, naturalize_res, color='#f1c40f', width=barWidth, edgecolor='white', label='Naturalize')
    plt.bar(r3, codebuff_res, color='#1abc9c', width=barWidth, edgecolor='white', label='Codebuff')

    # Add xticks on the middle of the group bars
    plt.xlabel('Number of fully patched files (m=' + str(modifications) + ')', fontweight='bold')
    plt.xticks([r + barWidth for r in range(len(bars1))], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(bottom=0.30)
    # Create legend & Show graphic
    plt.legend()

def plot_errors_types(results, counts): # protocol1
    modifications = (2, 2, 2, 2, 2)

    labels = []

    errors_labels = set()

    for result in results:
        # labels.append( exp.corpus.name + "(" + str(exp.corpus.get_number_of_files()) + " files)" )
        labels.append("{} ({})".format(result["name"], result["number_of_injections"]))
        for count in counts:
            errors_labels = errors_labels | result[count].keys()

    total_error_count = dict()
    for error in errors_labels:
        total_error_count[error] = 0
    for result in results:
        for count in counts:
            for error, n in result[count].items():
                total_error_count[error] += n
    sum_total_error_count = sum(total_error_count.values())

    n_errors_labels = len(errors_labels)
    colors = []
    if n_errors_labels > 1:
        for i in range( 0, n_errors_labels ):
            colors.append('#%02x%02x%02x' % tuple(map(lambda x: int( x*256 ), colorsys.hls_to_rgb( 1 / (n_errors_labels-1) * i * 0.9 , random.uniform(0.4, 0.6), random.uniform(0.4, 0.6)))))
        random.shuffle(colors)
    else :
        colors.append('#ff00ff')

    lables_colors = dict()
    i = 0
    for error_label in errors_labels:
        lables_colors[error_label] = colors[i]
        i += 1

    def compute_errors_layer(errors_count_name):
        layers = dict()
        for result in results:
            errors = result[errors_count_name]
            for error_label in errors_labels:
                if ( error_label not in layers):
                    layers[error_label] = []
                if ( error_label in errors ):
                    layers[error_label].append(errors[error_label])
                else:
                    layers[error_label].append(0)
        return layers

    layers = dict()
    for count in counts:
        layers[count] = compute_errors_layer(count)

    barWidth = 1. / (len(counts) + 1)
    # Set position of bar on X axis
    r = []
    r.append(np.arange(len(labels)))
    for i in range(1, len(counts)):
        r.append([x + barWidth for x in r[i-1]])

    def add_layers_to_the_graph(layers, position):
        sum = [0] * len(labels)
        for key, values in layers.items():
            plt.bar(position, values, width=barWidth, color=lables_colors[key], bottom=sum, edgecolor='white')
            sum = list(map( lambda x, y: x + y, sum, values))
        return sum
    # plt.bar(r2, naturalize_res, color='#f1c40f', width=barWidth, edgecolor='white', label='Naturalize')
    # plt.bar(r3, codebuff_res, color='#1abc9c', width=barWidth, edgecolor='white', label='Codebuff')
    i = 0
    sums = []
    for count, i in zip(counts, range(len(counts))):
        sums.append(add_layers_to_the_graph(layers[count], r[i]))

    # Add xticks on the middle of the group bars
    plt.xlabel('Number of errors (m={}) \n {}'.format(modifications, counts), fontweight='bold')
    plt.xticks([r + barWidth * (len(counts)-1) / 2 for r in range(len(results))], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(top=0.80)

    plt.subplots_adjust(bottom=0.30)
    # Create legend & Show graphic
    patches = [ mpatches.Patch(color=c, label="{} ({:.2f}%)".format(l.split(".")[-1], total_error_count[l] / sum_total_error_count * 100)) for l, c in lables_colors.items()]
    plt.legend(handles = patches, loc='upper center', ncol=3, fancybox=True, bbox_to_anchor=(0.5, 1.4))

def cumulatives_bars(plot_data):

    errors_labels = set()
    labels = tuple(plot_data['data'].keys())

    compute_avg = plot_data.get('avg', True)

    label_sum = {}

    for count in plot_data['data'].values():
        errors_labels = errors_labels | set(count.keys())

    data = plot_data['data']

    layers = {}

    for label in errors_labels:
        layers[label] = []
        label_sum[label] = 0
        for name in labels:
            layers[label].append(data[name].get(label,0))
            label_sum[label] += data[name].get(label,0)

    total_sum = sum(label_sum.values())
    avg = {label: (value/total_sum*100.) for label, value in label_sum.items()}
    print(avg)

    n_errors_labels = len(errors_labels)
    colors = []
    if ( n_errors_labels > 1):
        for i in range( 0, n_errors_labels ):
            colors.append('#%02x%02x%02x' % tuple(map(lambda x: int( x*256 ), colorsys.hls_to_rgb( 1 / (n_errors_labels-1) * i * 0.9 , random.uniform(0.3, 0.7), random.uniform(0.3, 0.7)))))
        random.shuffle(colors)
    else :
        colors.append('#ff00ff')

    lables_colors = dict()
    for i, error_label in enumerate(errors_labels):
        lables_colors[error_label] = colors[i]

    bar_width = 0.5
    # Set position of bar on X axis
    r = list()
    r.append(np.arange(len(data.keys())))
    # for i in range(1,len(counts)):
    #     r.append([x + bar_width for x in r[i-1]])

    def add_layers_to_the_graph(layers, position):
        sum = [0] * len(labels)
        for key, values in layers.items():
            plt.bar(position, values, width=bar_width, color=lables_colors[key], bottom=sum, edgecolor='white')
            sum = list(map( lambda x, y: x + y, sum, values))
        return sum
    # plt.bar(r2, naturalize_res, color='#f1c40f', width=bar_width, edgecolor='white', label='Naturalize')
    # plt.bar(r3, codebuff_res, color='#1abc9c', width=bar_width, edgecolor='white', label='Codebuff')
    i = 0
    sums = []
    sums.append(add_layers_to_the_graph(layers, r[0]))

    # Add xticks on the middle of the group bars
    plt.xlabel(plot_data['title'], fontweight='bold')
    plt.xticks( range(len(labels)), labels, rotation=45, fontsize=8)
    plt.subplots_adjust(top=0.75)
    plt.subplots_adjust(bottom=0.20)
    # Create legend & Show graphic
    patches = [ mpatches.Patch(color=c, label=f'{l} ({avg[l]:.2f}%)') for l, c in lables_colors.items()]
    plt.legend(handles = patches, loc='upper center', ncol=3, fancybox=True, bbox_to_anchor=(0.5, 1.4))
    plt.show()

def dict_to_list(dict, order):
    return [dict[key] for key in order]

def violin_plot(plot_data):
    data = plot_data['data']
    colors = plot_data['colors']
    order = tuple(colors.keys())
    print(order)

    fig, axes = plt.subplots()

    parts = axes.violinplot([list(filter(lambda a: a<50, points)) for points in dict_to_list(data, order)], range(len(data)), points=1000, vert=False, widths=0.7,
                          showmeans=False, showextrema=False, showmedians=False,
                          bw_method='silverman')
    for pc, label in zip(parts['bodies'], order) :
        # print(pc)
        pc.set_facecolor(colors[label])
        pc.set_alpha(0.8)
    medianprops = dict(linestyle='-.', linewidth=3.5, color='#000000')
    axes.boxplot(dict_to_list(data, order), whis=[5, 95], positions=range(len(data)), vert=False, medianprops=medianprops)

    patches = [ mpatches.Patch(color=c, label=l) for l, c in list(colors.items())[::-1]]
    plt.legend(handles = patches, loc='upper right', ncol=3, fancybox=True, fontsize=15)
    # plt.yticks( range(len(order)), order, fontsize=15)
    plt.yticks( [1], ('all \nprojects',), fontsize=15)
    plt.xlabel(plot_data.get('x_label', ''), fontsize=15)
    plt.ylabel(plot_data.get('y_label', ''), fontsize=15)
    plt.xlim(0,40)
    plt.show()

def boxplot(plot_data):
    labels = list(plot_data['data'].keys())
    sub_labels = plot_data['sub_labels']
    vert = plot_data.get('vert', False)
    colors = plot_data['colors']

    boxWidth = 1. / (len(sub_labels) + 1)

    show_all = plot_data.get('show_all', True)

    if show_all:
        labels.append('all')

    r = []
    r.append(list(np.arange(len(labels))))
    if show_all:
        r[0][-1] += 0.5

    for i in range(1,len(sub_labels)):
        r.append([x + boxWidth for x in r[i-1]])
    r = sorted(reduce(list.__add__,r))

    data = []
    for data_list in plot_data['data'].values():
        for label in sub_labels:
            data += [data_list[label]]
    print(len(data[0]))
    if show_all:
        all = [ reduce(list.__add__, data[i::len(sub_labels)]) for i in range(len(sub_labels)) ]
        print(all)
        data += all

    fig7, ax7 = plt.subplots()
    medianprops = dict(linestyle='-.', linewidth=3.5, color='#000000')
    bplot = ax7.boxplot(data, whis=[5, 95], positions=r, widths=boxWidth*0.8, vert=vert, patch_artist=True, labels=sub_labels*len(labels), medianprops=medianprops)
    for patch, color in zip(bplot['boxes'], colors  * len(labels)):
        patch.set_facecolor(color)
    patches = [ mpatches.Patch(color=c, label=l) for l, c in zip(sub_labels, colors)]
    plt.legend(handles = patches, loc='upper right', ncol=3, fancybox=True, fontsize=15)
    if vert:
        plt.xticks([pos + boxWidth * (len(sub_labels)-1) / 2 for pos in r[::len(sub_labels)]], labels, rotation=45, fontsize=15)
    else:
        plt.yticks([pos + boxWidth * (len(sub_labels)-1) / 2 for pos in r[::len(sub_labels)]], labels, fontsize=15)
    plt.xlabel(plot_data.get('x_label', ''), fontsize=15)
    plt.ylabel(plot_data.get('y_label', ''), fontsize=15)
    plt.xlim(0,40)
    plt.subplots_adjust(bottom=0.06, left=0.11, right=0.95, top=0.95)
    plt.gca().invert_yaxis()
    plt.show()

def plot_errors_types_per_injection_type(results):
    modifications = (2, 2, 2, 2, 2)

    counts = ("insertions-newline", "insertions-space", "insertions-tab", "deletions-newline", "deletions-space")

    labels = []
    errors_labels = set()

    for result in results:
        # labels.append( exp.corpus.name + "(" + str(exp.corpus.get_number_of_files()) + " files)" )
        labels.append("{} ({})".format(result["name"], result["number_of_injections"]))
        errors_labels = errors_labels | result["checkstyle_errors_count_ugly"].keys()

    n_errors_labels = len(errors_labels)
    colors = []
    if n_errors_labels > 1:
        for i in range(0, n_errors_labels):
            colors.append('#%02x%02x%02x' % tuple(map(lambda x: int( x*256 ), colorsys.hls_to_rgb( 1 / (n_errors_labels-1) * i * 0.9, random.uniform(0.4, 0.6), random.uniform(0.4, 0.6)))))
        random.shuffle(colors)
    else:
        colors.append('#ff00ff')

    lables_colors = dict()
    i = 0
    for error_label in errors_labels:
        lables_colors[error_label] = colors[i]
        i += 1

    def compute_error_origines(result):
        result["errors_origine"] = dict()
        for file_with_cs_errors in result["file_with_cs_errors_ugly"].values():
            for file_modification in file_with_cs_errors:
                type = file_modification["type"]
                if (type not in result["errors_origine"]):
                    result["errors_origine"][type] = dict()
                for error in file_modification["errors"]:
                    if (error["source"] not in result["errors_origine"][type]):
                        result["errors_origine"][type][error["source"]] = 0
                    result["errors_origine"][type][error["source"]] += 1

    for result in results:
        compute_error_origines(result)

    def compute_errors_layer(injection_type):
        layers = dict()
        for result in results:
            if injection_type in result["errors_origine"]:
                errors = result["errors_origine"][injection_type]
            else:
                errors = []
            for error_label in errors_labels:
                if ( error_label not in layers):
                    layers[error_label] = []
                if ( error_label in errors ):
                    layers[error_label].append(errors[error_label])
                else:
                    layers[error_label].append(0)
        return layers

    layers = dict()
    for count in counts:
        layers[count] = compute_errors_layer(count)

    barWidth = 1. / (len(counts) + 1)
    # Set position of bar on X axis
    r = []
    r.append(np.arange(len(labels)))
    for i in range(1,len(counts)):
        r.append([x + barWidth for x in r[i-1]])

    def add_layers_to_the_graph(layers, position):
        sum = [0] * len(labels)
        for key, values in layers.items():
            plt.bar(position, values, width=barWidth, color=lables_colors[key], bottom=sum, edgecolor='white')
            sum = list(map( lambda x, y: x + y, sum, values))
        return sum
    # plt.bar(r2, naturalize_res, color='#f1c40f', width=barWidth, edgecolor='white', label='Naturalize')
    # plt.bar(r3, codebuff_res, color='#1abc9c', width=barWidth, edgecolor='white', label='Codebuff')
    i = 0
    sums = []
    for count, i in zip(counts, range(len(counts))):
        sums.append(add_layers_to_the_graph(layers[count], r[i]))

    # Add xticks on the middle of the group bars
    plt.xlabel('Number of errors (m={}) \n {}'.format(modifications, counts), fontweight='bold')
    plt.xticks([r + barWidth * (len(counts)-1) / 2 for r in range(len(results))], labels, rotation=45, fontsize=8)
    plt.subplots_adjust(top=0.80)

    plt.subplots_adjust(bottom=0.30)
    # Create legend & Show graphic
    patches = [ mpatches.Patch(color=c, label="{}".format(l.split(".")[-1])) for l, c in lables_colors.items()]
    plt.legend(handles = patches, loc='upper center', ncol=3, fancybox=True, bbox_to_anchor=(0.5, 1.4))

def dist_from_modification(results):
    distances = []
    for result in results:
        for id, modifications in result["file_with_cs_errors_ugly"].items():
            for modification in modifications:
                errors_pos = list(set([(int(error["line"]), int(error.get("column", 0))) for error in modification["errors"]]))
                modifications_pos = result["modifications"][id][modification["type"]][modification["modification_id"]]
                if (errors_pos[0][0] != modifications_pos[0][0] or errors_pos[0][1] != modifications_pos[0][1] ):
                    print(errors_pos, modifications_pos)
                    distances.append(1)
                else:
                    distances.append(0)
    print(sum(distances)/len(distances))

def load_results(dir):
    data = {}
    with open(os.path.join(dir, "results.json")) as f:
        data = json.load(f)
    return data


import glob
import time
import sys
from dataclasses import dataclass

import torch
import torchvision
import numpy as np
import torch.utils.data
from torch import optim
from torch.nn import functional as F
from torch.utils.tensorboard import SummaryWriter

from src.preprocess.image_loader import ImgDataset, ImageTransform
from src.vae.vae_linear import VAE

"""
To do : データのpathがおかしい -> configの作業dirが間違っていた
"""

@dataclass
class Config:
    lr: float = 1e-4
    beta1: float = 0.9
    beta2: float = 0.9
    input_dim: int = 16384
    num_epoch: int = 1000
    num_stopping: int = 50
    batch_size: int = 64
    z_dim: int = 32
    save_path: str = '../../model/vae.pt'
    log_path: str = '../../log/vae_linear/vae_lr_8e3_nonorma'

def loss_function(recon_x, x, mu, logvar, config):
    bce = F.binary_cross_entropy_with_logits(recon_x, x.view(-1, config.input_dim), reduction='sum')
    kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    return bce + kld

def train(train_dataloader, eval_dataloader, model, config):

    # check GPU
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print("Use device：", device)

    model = model.to(device)
    optimizer = optim.Adam(model.parameters(), lr=config.lr, betas=[config.beta1, config.beta2])

    writer = SummaryWriter(log_dir=config.log_path)

    models = []
    eval_loss = []
    generated_images = []
    for epoch in range(config.num_epoch):
        t_epoch_start = time.time()

        print('-------------')
        print('Epoch {}/{}'.format(epoch, config.num_epoch))
        print('-------------')

        # ---------------------------------------------------------------------------------

        model.train()
        train_epoch_loss = 0
        n_t = 0
        for images in train_dataloader:

            images = images.to(device)

            # For liner vae
            images = images.view(-1, config.input_dim)

            pred, mu, logvar = model(images)

            loss = loss_function(pred, images, mu, logvar, config)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            train_epoch_loss += loss.item()
            n_t += 1

        print('Train epoch loss:{:.4f}'.format(train_epoch_loss / train_dataloader.batch_size))

        # -------------------------------------------------------------------------------

        model.eval()
        eval_epoch_loss = 0
        n_e = 0
        for images in eval_dataloader:

            images = images.to(device)

            # For liner vae
            images = images.view(-1, config.input_dim)

            pred, mu, logvar = model(images)

            loss = loss_function(pred, images, mu, logvar, config)

            eval_epoch_loss += loss.item()
            n_e += 1

        models.append(model)

        # Early stopping -------------------------------------------------------

        eval_loss.append(eval_epoch_loss/n_e)

        # To tensor board
        writer.add_scalar('Train/loss', train_epoch_loss / n_t, epoch)
        writer.add_scalar('Eval/loss', eval_epoch_loss / n_e, epoch)

        if epoch >= config.num_stopping:
            if epoch == config.num_stopping:
                low_loss = np.min(eval_loss)
                low_index = np.argmin(eval_loss)
                if low_index == 0:
                    print('-------------------------------------------------------------------------------------------')
                    print("Early stopping")
                    print('Best Iteration:{}'.format(low_index+1))
                    print('Best evaluation loss:{}'.format(low_loss))
                    break

            elif epoch == low_index + config.num_stopping:
                low_loss_new = np.min(eval_loss[low_index:])
                low_index_new = np.argmin(eval_loss[low_index:])+low_index

                if low_loss <= low_loss_new:
                    print('-------------------------------------------------------------------------------------------')
                    print("Early stopping")
                    print('Best Iteration:{}'.format(low_index + 1))
                    print('Best evaluation loss:{}'.format(low_loss))
                    break
                else:
                    low_loss = low_loss_new
                    low_index = low_index_new
        else:
            pass

        t_epoch_finish = time.time()
        print('Eval_Epoch_Loss:{:.4f}'.format(eval_epoch_loss / n_e))
        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))

        # check generated image ---------------------------------------------------
        if epoch % 20 == 0:
            pred = pred.reshape(images.shape[0], 1, 128, 128)
            images = images.reshape(images.shape[0], 1, 128, 128)

            _generated = torchvision.utils.make_grid(pred[:10], nrow=5)
            _true = torchvision.utils.make_grid(images[:10], nrow=5)
            writer.add_image('Eval/generated', _generated, epoch)
            writer.add_image('Eval/true', _true, epoch)

    return models[low_index + 1], generated_images

def process(train_dir_path, eval_dir_path, config):

    # params of normalization
    _mean = 0
    _std = 1

    # read file path
    train_path_list = glob.glob(train_dir_path)
    eval_path_list = glob.glob(eval_dir_path)

    if train_path_list == [] or eval_path_list == []:
        print('FileNotFoundError: No such file or directory: ', file=sys.stderr)
        sys.exit(1)

    # mk dataloader
    train_dataset = ImgDataset(file_list=train_path_list, transform=ImageTransform(_mean, _std))
    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)

    eval_dataset = ImgDataset(file_list=eval_path_list, transform=ImageTransform(_mean, _std))
    eval_dataloader = torch.utils.data.DataLoader(eval_dataset, batch_size=config.batch_size, shuffle=True)

    vae = VAE(config.input_dim, config.z_dim)

    # train model
    model, generated = train(train_dataloader, eval_dataloader, vae, config)

    # save model
    torch.save(model.state_dict(), config.save_path)

    return generated

if __name__ == '__main__':
    t_dir_path = rf'..\..\figure\spectrogram_png\train\*.png'
    e_dir_path = rf'..\..\figure\spectrogram_png\test\*.png'
    process(t_dir_path, e_dir_path, Config())


import numpy as np
import torch
from tqdm import tqdm
from cheml.datasets import load_qm7
from sklearn import linear_model, model_selection
from scipy.spatial.distance import pdist
from scatwave.scattering import SolidHarmonicScattering
from scatwave.utils import compute_integrals

def evaluate_bilinear_regression(scat_0, scat_1, scat_2, target):
    x = np.concatenate([scat_0, scat_1, scat_2], axis=1)
    x = (x - x.mean()) / x.std()
    y_factor = (target.max() - target.min())
    y = (target - target.min()) / y_factor
    x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y)

    x_train = torch.autograd.Variable(torch.from_numpy(x_train), requires_grad=False)
    x_test = torch.autograd.Variable(torch.from_numpy(x_test), requires_grad=False)
    y_train = torch.autograd.Variable(torch.from_numpy(y_train), requires_grad=False)
    y_test = torch.autograd.Variable(torch.from_numpy(y_test), requires_grad=False)

    n_x = x_train.size(1)
    bilinear = torch.nn.Bilinear(n_x, n_x, 1, bias=False)
    linear = torch.nn.Linear(n_x, 1)
    loss_fn = torch.nn.MSELoss(size_average=True)
    params = list(bilinear.parameters()) + list(linear.parameters())

    learning_rate = 1e-3
    optimizer = torch.optim.Adam(params, lr=learning_rate)

    for t in range(20000):
        y_train_pred = bilinear(x_train, x_train) + linear(x_train)
        loss = loss_fn(y_train_pred, y_train)
        if (t % 500) == 0:
            print(t, np.sqrt(loss.data[0])*y_factor)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    print('train RMSE : ', np.sqrt(loss.data[0])*y_factor)

    y_test_pred = bilinear(x_test, x_test) + linear(x_test)
    loss = loss_fn(y_test_pred, y_test)
    print('test RMSE: ', np.sqrt(loss.data[0])*y_factor)

def evaluate_linear_regression(scat_0, scat_1, scat_2, target):
    x_1 = np.concatenate([scat_0, scat_1], axis=1)
    x_1_2 = np.concatenate([x_1, scat_2], axis=1)
    lin_regressor = linear_model.LinearRegression()
    scat_1_prediction = model_selection.cross_val_predict(lin_regressor, x_1, target)
    scat_1_MAE = np.mean(np.abs(scat_1_prediction - target))
    scat_1_RMSE = np.sqrt(np.mean((scat_1_prediction - target)**2))
    print('''scattering order 1, linear regression,
          MAE: {}, RMSE: {} (kcal.mol-1)'''.format(scat_1_MAE, scat_1_RMSE))

    scat_1_2_prediction = model_selection.cross_val_predict(lin_regressor, x_1_2, target)
    scat_1_2_MAE = np.mean(np.abs(scat_1_2_prediction - target))
    scat_1_2_RMSE = np.sqrt(np.mean((scat_1_2_prediction - target)**2))
    print('''scattering order 1 and 2, linear regression,
          MAE: {}, RMSE: {} (kcal.mol-1)'''.format(scat_1_2_MAE, scat_1_2_RMSE))

def get_valence(charges):
    return (
        charges * (charges <= 2) +
        (charges - 2) * np.logical_and(charges > 2, charges <= 10) +
        (charges - 10) * np.logical_and(charges > 8, charges <= 18))

def renormalize(positions, charges, sigma, overlapping_precision=1e-1):
    # x_min, x_max = positions[:,:,0].min(), positions[:,:,0].max()
    # y_min, y_max = positions[:,:,1].min(), positions[:,:,1].max()
    # z_min, z_max = positions[:,:,2].min(), positions[:,:,2].max()

    # positions[:,:,0] -= 0.5*(x_max + x_min)
    # positions[:,:,1] -= 0.5*(y_max + y_min)
    # positions[:,:,2] -= 0.5*(z_max + z_min)

    min_dist = np.inf
    for i in range(positions.shape[0]):
        n_atoms = np.sum(charges[i] != 0)
        pos = positions[i, :n_atoms, :]
        min_dist = min(min_dist, pdist(pos).min())

    delta = sigma * np.sqrt(-8 * np.log(overlapping_precision))

    return positions * delta / min_dist

def get_qm7_positions_energies_and_charges(M, N, O, J, L, sigma):
    qm7 = load_qm7(align=True)
    positions = qm7.R
    charges = qm7.Z
    energies = qm7.T.transpose()
    valence_charges = get_valence(charges)

    positions = renormalize(positions, charges, sigma)

    return torch.from_numpy(positions), torch.from_numpy(energies), torch.from_numpy(charges), torch.from_numpy(valence_charges)

def generate_weighted_sum_of_gaussians(grid, positions, weights, sigma, cuda=False):
    _, M, N, O = grid.size()
    if cuda:
        signals = torch.cuda.FloatTensor(positions.size(0), M, N, O).fill_(0)
    else:
        signals = torch.FloatTensor(positions.size(0), M, N, O).fill_(0)

    for i_signal in range(positions.size(0)):
        n_points = positions[i_signal].size(0)
        for i_point in range(n_points):
            if weights[i_signal, i_point] == 0:
                break
            weight = weights[i_signal, i_point]
            center = positions[i_signal, i_point]
            signals[i_signal] += weight * torch.exp(
                -0.5 * ((grid[0]-center[0])**2 + (grid[1]-center[1])**2 + (grid[2]-center[2])**2) / sigma**2)
    return signals / ((2 * np.pi)**1.5 * sigma**3)

def main():
    """Trains a simple linear regression model with solid harmonic
    scattering coefficients on the atomisation energies of the QM7
    database.

    Achieves a MAE of ... kcal.mol-1
    """
    cuda = torch.cuda.is_available()
    batch_size = 32
    M, N, O = 192, 128, 96
    grid = torch.from_numpy(
            np.fft.ifftshift(np.mgrid[-M//2:-M//2+M, -N//2:-N//2+N, -O//2:-O//2+O].astype('float32'), axes=(1,2,3)))
    if cuda:
        grid = grid.cuda()
    sigma = 2.
    J, L = 2, 3
    integral_powers = [0.5, 1., 2., 3.]
    args = {'integral_powers': integral_powers}
    pos, energies, full_charges, valence_charges = get_qm7_positions_energies_and_charges(
            M, N, O, J, L, sigma)
    n_molecules = pos.size(0)
    n_batches = np.ceil(n_molecules / batch_size).astype(int)
    n_batches = 2

    scat = SolidHarmonicScattering(M=M, N=N, O=O, J=J, L=L, sigma_0=sigma)

    scat_0, scat_1, scat_2 = [], [], []
    print('Computing solid harmonic scattering coefficients of molecules \
            of QM7 database on {}'.format('GPU' if cuda else 'CPU'))
    print('L: {}, J: {}, integral powers: {}'.format(L, J, integral_powers))
    for i in tqdm(range(n_batches)):
        start, end = i*batch_size, min((i+1)*batch_size, n_molecules)
        pos_batch = pos[start:end]

        full_batch = full_charges[start:end]
        full_density_batch = generate_weighted_sum_of_gaussians(
                grid, pos_batch, full_batch, sigma, cuda=cuda)

        full_scat_0 = compute_integrals(full_density_batch, integral_powers)
        full_scat_1, full_scat_2 = scat(full_density_batch, True, 'integral', args)

        val_batch = valence_charges[start:end]
        val_density_batch = generate_weighted_sum_of_gaussians(
                grid, pos_batch, val_batch, sigma, cuda=cuda)
        val_scat_0 = compute_integrals(val_density_batch, integral_powers)
        val_scat_1, val_scat_2 = scat(val_density_batch, True, 'integral', args)

        core_density_batch = full_density_batch - val_density_batch
        core_scat_0 = compute_integrals(core_density_batch, integral_powers)
        core_scat_1, core_scat_2 = scat(core_density_batch, True, 'integral', args)

        scat_0.append(torch.stack([full_scat_0, val_scat_0, core_scat_0], dim=-1))
        scat_1.append(torch.stack([full_scat_1, val_scat_1, core_scat_1], dim=-1))
        scat_2.append(torch.stack([full_scat_2, val_scat_2, core_scat_2], dim=-1))

    n = batch_size * n_batches

    scat_0 = torch.cat(scat_0, dim=0)
    scat_1 = torch.cat(scat_1, dim=0)
    scat_2 = torch.cat(scat_2, dim=0)

    np_scat_0 = scat_0.numpy().reshape((n, -1))
    np_scat_1 = scat_1.numpy().reshape((n, -1))
    np_scat_2 = scat_2.numpy().reshape((n, -1))

    print('order 1 : {} coef, order 2 : {} coefs'.format(np_scat_1.shape[1], np_scat_2.shape[1]))
    target = energies.numpy()[:n]
    evaluate_linear_regression(np_scat_0, np_scat_1, np_scat_2, target)
    evaluate_bilinear_regression(np_scat_0, np_scat_1, np_scat_2, target)

    basename = 'qm7_L_{}_J_{}_sigma_{}_MNO_{}_powers_{}.npy'.format(
            L, J, sigma, (M, N, O), integral_powers)
    np.save('scat_0_'+basename, np_scat_0)
    np.save('scat_1_'+basename, np_scat_1)
    np.save('scat_2_'+basename, np_scat_2)

if __name__ == '__main__':
    main()


from selenium import webdriver
import os
import time
'''
используем   - это headless webkit браузер для селениум
update - отложено. надо разобраться как заходить на мобильную версию
'''

class Driver():
	def __init__(self):
		options = webdriver.ChromeOptions()
		options.add_argument('--headless')
		self.browser = webdriver.Chrome()#chrome_options=options)

		self.number = None

	def navigate(self, url):
		self.browser.set_window_size(450, 300)
		self.browser.get(url)
		ActionChains(browser).key_down(Keys.LEFT_CONTROL).key_down(Keys.LEFT_SHIFT).send_keys('i').perform()
		self.browser.get(url)
		self.browser.implicitly_wait(5)
		self.browser.find_element_by_class_name('BPWk2').click()
		self.number = self.browser.find_element_by_class_name('_3Ryy-').text
		return self.number

def main():
	pass

if __name__ == '__main__':
	main()


# collect-links-cursor.py
# MCW - 2/11/2021 / mperry - 3/10/2021
import sys
import json
import tweepy

# use coronavirus as default search term unless one provided
search_term = "coronavirus"
if len(sys.argv) > 1:
     search_term = str(sys.argv[1])

# number of links to collect
MAX_COUNT = 1200
count = 0

# load Twitter API keys from a file so they're not hard-coded
with open("secrets.json", "r") as secretsFile:
    secrets = json.load(secretsFile)
# print(secrets['consumer_key'])
# print(secrets['consumer_secret'])

# setup Twitter API with OAuth2 procedure
consumer_key = secrets['consumer_key']
consumer_secret = secrets['consumer_secret']
auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
api = tweepy.API(auth)

try:
  for page in tweepy.Cursor(api.search, q=search_term, tweet_mode='extended', lang='en').pages():
    for tweet in page:
         for link in tweet.entities["urls"]:
             print("%s" % link['expanded_url'])
             count = count + 1
    if count > MAX_COUNT:
         break
except tweepy.TweepError as e:
  print ("Tweepy Error: %s" % str(e))


import os
import cv2
import numpy as np

from PIL import Image, ImageEnhance

def read_img(img_path):
    """
    读取图像,将RGB转为3通道的灰度图
    :param img_path:
    :return:
    """
    img = cv2.imread(img_path)
    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    gray_3channel_img = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
    return gray_3channel_img

def enhance_brightness(img, brightness):
    """
    亮度增强
    :param img:
    :param brightness:
    :return:
    """
    img = Image.fromarray(img)
    enh_bri = ImageEnhance.Brightness(img)
    image_brightened = enh_bri.enhance(brightness)
    return np.array(image_brightened)

def enhance_color(img, color):
    """
    色度增强
    :param img:
    :param color:
    :return:
    """
    img = Image.fromarray(img)
    enh_col = ImageEnhance.Color(img)
    image_colored = enh_col.enhance(color)
    return np.array(image_colored)

def enhance_contrast(img, contrast):
    """
    对比度增强
    :param img:
    :param contrast:
    :return:
    """
    img = Image.fromarray(img)
    enh_con = ImageEnhance.Contrast(img)
    image_contrasted = enh_con.enhance(contrast)
    return np.array(image_contrasted)

def enhance_sharpness(img, sharpness):
    """
    锐度增强
    :param img:
    :param sharpness:
    :return:
    """
    img = Image.fromarray(img)
    enh_sha = ImageEnhance.Sharpness(img)
    image_sharped = enh_sha.enhance(sharpness)
    return np.array(image_sharped)

if __name__ == "__main__":

    #img = Image.open('0.jpg')
    img = read_img('/home/tony/Pictures/ocr/rgb_original/test2.jpg')

    img_enhanced = enhance_contrast(img, 5)
    print(img_enhanced)
    num = np.array(img_enhanced)
    print(num)
    cv2.imshow('dwad',num)
    cv2.waitKey()
    # for i in range(8):
    #     img = Image.open(str(i)+'.jpg')
    #     img_enhanced = enhance_contrast(img, 5)
    #
    #     img_enhanced.save('/home/tony/Pictures/ocr/gray_contrast_original/'+str(i)+'_contrast.jpg')


from SearchEngine.spiders import models
from nltk.corpus import wordnet
import csv
import pandas as pd
import search_pb2

def clearCsvFile():
    f = open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', "w+")
    f.close()
    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'a') as result:
        fieldnames = ['link', 'score']
        writer = csv.DictWriter(result, fieldnames=fieldnames)
        writer.writeheader()

def sortCSVFile():
    df = pd.read_csv('result.csv')
    df = df.sort_values(by=('score'), ascending=False)
    df.to_csv('result.csv', index=False)

def writeResult(link, score):
    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'a') as result:
        fieldnames = ['link', 'score']
        writer = csv.DictWriter(result, fieldnames=fieldnames)
        #writer.writeheader()
        writer.writerow({'link': link, 'score': score})
        sortCSVFile()

def doSearch(query):
    clearCsvFile()
    splitQ = query.split(':')
    shouldBe = splitQ[0].split(',')
    notBe = splitQ[1].split(',')
    mayBe = splitQ[2].split(',')
    for i in models.TopWords.select():
        l1 = []
        for j in eval(i.topWords):
            for k in shouldBe:
                wordFromList1 = wordnet.synsets(j[0])
                wordFromList2 = wordnet.synsets(k)
                if wordFromList1 and wordFromList2:
                    s = wordFromList1[0].wup_similarity(wordFromList2[0])
                    if s != None and s > 0.1:
                        l1.append(s)
        print(query)
        try:
            writeResult(i.link, str(max(l1)))
        except:
            pass

def RetrieveMessagesValuesFromFile():
    searchResponse = search_pb2.SearchResponse()

    with open('/Users/sajjadaboutalebi/Documents/SearchEngine/Server/result.csv', 'r') as result:
        reader = csv.DictReader(result)
        for row in reader:
            item = searchResponse.bars.add()
            item.link = row['link']
            item.score = row['score']
            #print(item.link)
    return searchResponse.SerializeToString()

#doSearch('love')
#a = RetrieveMessagesValuesFromFile()
#print(a)
#s = search_pb2.SearchResponse()
#s.ParseFromString(a)
#print(len(s.bars))


#!/usr/bin/env python
import nltk
import re
import string
import json

class FileReader:
    @staticmethod
    def read_file(name):
        """ func to read from file """
        with open(name) as fl:
            return fl.readlines()

class JsonWriter:
    @staticmethod
    def write_json(name, jsn):
        with open(name, 'w') as file:                   # serialization
            json.dump(jsn, file)

class RemoverUnwantedWords:
    @staticmethod
    def remove_words(expr, line):
        """ func for cleaning by regex """
        return re.sub(expr, '', line)

class CheckerForMeaning:
    @staticmethod
    def find_trash(clean_line, inf_dict):
        """ find contextless word """
        tokens = clean_line.split(' ')

        clean_tokens = []
        for i in range(len(tokens)):                    # removing punctuation and " "
            if len(tokens[i]) < 2:
                continue
            while True:
                if tokens[i][-1] in string.punctuation:
                    tokens[i] = tokens[i][:-1]
                else:
                    clean_tokens.append(tokens[i])
                    break

        clean_tokens = set(clean_tokens)
        for i in clean_tokens:                          # check for meaning
            if nltk.wsd.lesk(tokens, i) is None:
                inf_dict['orphan_tokens'].append(i)
        return inf_dict

class UrlFinder:
    @staticmethod
    def find_urls(clean_line, raw_line, inf_dict):
        """ find URL """
        urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_line)
        funk_for_clean = RemoverUnwantedWords.remove_words
        clean_line = funk_for_clean('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
                                    clean_line)

        if len(urls):
            inf_dict["metadata"].extend(urls)
        return clean_line, inf_dict

class TweetLineHandler:
    """ class for performance extraction useful inf from tweet"""

    def __init__(self, line):
        self.line = line

    def explore(self):
        inf_dict = {'body': '', "metadata": [], 'body_tags': [], 'orphan_tokens': []}

        dog_index = self.line.rfind('@') + 1
        inf_dict["metadata"].append(self.line[dog_index:-3])                        # find [@...]

        line = self.line[:dog_index - 4]
        inf_dict['body'] = line
        clean_line = RemoverUnwantedWords.remove_words(r'[\$]\w+', line)            # remove $-words

        inf_dict['body_tags'] = re.findall(r'[\#\@](\w+)', clean_line)
        clean_line = RemoverUnwantedWords.remove_words(r'[\#\@](\w+)', clean_line)  # clean line

        clean_line, inf_dict = UrlFinder.find_urls(clean_line, self.line, inf_dict)

        inf_dict = CheckerForMeaning.find_trash(clean_line, inf_dict)

        return inf_dict

if __name__ == "__main__":

    data = FileReader.read_file('input.txt')  # reading from file

    result = {'records': []}

    for i in data:                            # tweet handling
        hndlr = TweetLineHandler(i)
        result['records'].append(hndlr.explore)

    JsonWriter.write_json('output.json', result)


import os
import re
import cv2
from torch import nn
from torch.nn import init
# from random import shuffle
import time
import numpy as np
from skimage import feature
from skimage.morphology import remove_small_holes, remove_small_objects

# ---------- make dir ----------
def make_dir(path):
    if not os.path.exists(path):
        os.makedirs(path)

# ---------- config ----------
def conf2dict(key_values):
    kv_dict = {}
    for kv in key_values:
        key = kv[0]
        value = get_value(kv[1])
        kv_dict[key] = value
    return kv_dict

def get_value(key_value):
    value = key_value
    float_p = re.compile(r"^[-+]?\d+\.\d+")
    float_ep = re.compile(r"\d+e[-+]?\d+")
    tuple_p = re.compile(r"\(\d+, *\d+\)")
    if value.isdigit():
        value = int(value)
    elif value.lower() == 'true':
        value = True
    elif value.lower() == 'false':
        value = False
    elif value.lower() == 'none':
        value = None
    elif float_p.match(value):
        value = float_p.match(value).group()
        value = float(value)
    elif float_ep.match(value):
        value = float_ep.match(value).group()
        value = float(value)
    elif tuple_p.findall(value):
        vlist = []
        for v in tuple_p.findall(value):
            value_tu = tuple(eval(v))
            value_tu = (int(value_tu[0]), int(value_tu[1]))
            vlist.append(value_tu)
        value = vlist
    return value

def print_config(config, save_path):
    mode = config.get('base_options', 'mode')

    sections = config.sections()
    for sec in sections:
        if (sec not in mode) & ((sec == 'train') | (sec == 'val') | (sec == 'test')):
            continue
        print('----------- ' + sec + ' ----------')
        for items in config.items(sec):
            print(items[0] + ' = ' + str(items[1]))
        print()
    config.write(open(save_path + 'config.conf', 'w'))

# ---------- name list ----------
def default_split(data_set):
    if data_set == 'ARAI':
        train_num = 143
    elif data_set == 'STARE':
        train_num = 5 * 2
    elif data_set == 'DRIVE':
        train_num = 20
    elif data_set == 'CHASE_DB1':
        train_num = 7 * 2
    else:
        train_num = 5 * 3
    return train_num

def get_name_list(data_set, img_dir=None, name_list=None, train_num=0, shuf=False):
    if name_list is None:
        name_list = []
        for _, _, f_n in os.walk(img_dir):
            name_list.extend(f_n)
        if data_set == 'CHASE_DB1':
            f_n = [k for k in name_list if '.jpg' in k]
            name_list = f_n
        # name_list.sort()
    val_num = len(name_list) - train_num
    # if shuf:
    #     shuffle(name_list)
    if val_num < train_num:
        split_num = val_num
        small_list = 'val_list'
    else:
        split_num = train_num
        small_list = 'train_list'

    val_list, train_list = split_nameList(name_list=name_list, split_num=split_num,
                                          data_set=data_set, small_list=small_list)
    if len(train_list) > 0:
        if type(train_list[0]) != list:
            train_list = [train_list]
    if len(val_list) > 0:
        if type(val_list[0]) != list:
            val_list = [val_list]

    return val_list, train_list

def split_nameList(name_list, split_num=0, data_set=None, small_list='val_list', shuf=False):
    if data_set == 'HRF':
        val_list = []
        train_list = []
        split_num = int(split_num / 3)
        healthy = [j for j in name_list if '_h' in j]
        glaucomstous = [j for j in name_list if '_g' in j]
        diabetic = [j for j in name_list if '_dr' in j]

        h_valList, h_trainList = split_list(healthy, split_num, small_list=small_list)
        g_valList, g_trainList = split_list(glaucomstous, split_num, small_list=small_list)
        dr_valList, dr_trainList = split_list(diabetic, split_num, small_list=small_list)

        if split_num == 0:
            if small_list == 'val_list':
                train_l = h_trainList + g_trainList + dr_trainList
                train_list.append(train_l)
            else:
                val_l = h_valList + g_valList + dr_valList
                val_list.append(val_l)
        else:
            for i in range(len(h_valList)):
                val_l = h_valList[i] + g_valList[i] + dr_valList[i]
                train_l = h_trainList[i] + g_trainList[i] + dr_trainList[i]
                val_list.append(val_l)
                train_list.append(train_l)

    elif data_set == 'STARE':
        # split_num = int(split_num / 2)
        h = ['im0077.ppm', 'im0081.ppm', 'im0082.ppm', 'im0162.ppm', 'im0163.ppm',
             'im0235.ppm', 'im0236.ppm', 'im0239.ppm', 'im0240.ppm', 'im0255.ppm']
        p = ['im0001.ppm', 'im0002.ppm', 'im0003.ppm', 'im0004.ppm', 'im0005.ppm',
             'im0044.ppm', 'im0139.ppm', 'im0291.ppm', 'im0319.ppm', 'im0324.ppm']
        # if shuf:
        #     shuffle(h)
        #     shuffle(p)
        total_names = [h, p]
        pic_names = [total_names[i][j] for j in range(len(h))
                     for i in range(len(total_names))]
        val_list, train_list = split_list(pic_names, split_num, small_list=small_list)
    else:

        val_list, train_list = split_list(name_list, split_num, small_list=small_list)

    return val_list, train_list

def split_list(name_list, split_num, small_list='val_list'):
    val_list = []
    train_list = []
    n = len(name_list)
    nameList_array = np.array(range(n))
    if small_list == 'val_list':
        print(n - split_num, 'for train, ', split_num, 'for validation')
        if split_num == 0:
            train_list.extend(name_list)
        else:
            for i in range(int(n / split_num) - 1, -1, -1):
                temp = nameList_array[i * split_num:(i + 1) * split_num]
                val_l = [name_list[j] for j in temp]
                train_l = [name_list[j] for j in range(n) if j not in temp]
                val_list.append(val_l)
                train_list.append(train_l)

    else:
        print(split_num, 'for train, ', n - split_num, 'for validation')
        if split_num == 0:
            val_list.extend(name_list)
        else:
            for i in range(int(n / split_num)):
                temp = nameList_array[i * split_num:(i + 1) * split_num]
                train_l = [name_list[j] for j in temp]
                val_l = [name_list[j] for j in range(n) if j not in temp]
                val_list.append(val_l)
                train_list.append(train_l)

    return val_list, train_list

# ---------- image information ----------
def get_imgInfo(mode, data_set, data_path):
    img_path = None
    gt_path = None
    mask_path = None
    img_num = 0
    img_size = (0, 0, 0)
    if data_set == 'DRIVE':
        if mode == 'train':
            img_path = 'DRIVE/training/images/'
            gt_path = 'DRIVE/training/1st_manual/'
            mask_path = 'DRIVE/training/mask/'
            img_num = 20
            img_size = (584, 565)
        elif mode == 'test':
            img_path = 'DRIVE/test/images/'
            gt_path = 'DRIVE/test/1st_manual/'
            mask_path = 'DRIVE/test/mask/'
            img_size = (584, 565)
            img_num = 20
    elif data_set == 'ARIA':
        img_path = 'ARIA/ARIA_all/'
        gt_path = 'ARIA/ARIA_all_GT/'
        img_num = 143
        img_size = (576, 768)
    elif data_set == 'CHASE_DB1':
        img_path = 'CHASE_DB1/'
        gt_path = img_path
        img_num = 28
        img_size = (960, 999)
    elif data_set == 'HRF':
        img_path = 'HRF/all/images/'
        gt_path = 'HRF/all/manual1/'
        mask_path = 'HRF/all/mask/'
        img_num = 45
        img_size = (2336, 3504)
    elif data_set == 'STARE':
        img_path = 'STARE/stare-images/'
        gt_path = 'STARE/labels-ah/'
        img_num = 20
        img_size = (605, 700)
    img_path = data_path + img_path
    if gt_path:
        gt_path = data_path + gt_path
    if mask_path:
        mask_path = data_path + mask_path
    return img_path, gt_path, mask_path, img_num, img_size

def get_name(dataSet, name, mode='train'):
    dSet = dataSet
    imgName = None
    gtName = None
    maskName = None

    if dSet == 'DRIVE':
        if mode == "test":
            imgNum = name.replace('_test.tif', '')
        else:
            imgNum = name.replace('_training.tif', '')
        imgName = imgNum
        gtName = imgNum + "_manual1.gif"
        maskName = name.replace('.tif', '_mask.gif')

    if dSet == "ARIA":
        imgNum = name[-6:-4]
        imgNum = imgNum.replace('_', '')
        gtName = name.replace('.tif', '')
        gtName = gtName + '_BDP.tif'
        imgName = imgNum

    if dSet == 'CHASE_DB1':
        imgName = name.replace('.jpg', '')
        gtName = imgName + '_1stHO.png'

    if dSet == 'HRF':
        imgName = name.replace('.jpg', '')
        imgName = imgName.replace('.JPG', '')
        gtName = imgName + '.tif'
        maskName = imgName + '_mask.tif'

    if dSet == 'STARE':
        imgName = filter(str.isdigit, name)
        gtName = name.replace('.ppm', '')
        gtName = gtName + '.ah.ppm'

    return gtName, maskName, imgName

def get_gt(gt):
    if gt.max() > 1:
        gt = gt / gt.max()
    gt_min = (gt == gt.min()) * 1
    gt_max = (gt == gt.max()) * 1
    if gt_min.sum() > gt_max.sum():
        return gt_max
    else:
        return gt_min

# get mask
def get_mask(name, img, dataSet):
    th = None
    fovTH = None
    before_FOV = None
    if dataSet == 'CHASE_DB1':
        fovTH = 300  # chase_DB1:300, stare: 430
        th = 10  # chase_db1: 10, stare_50
    elif dataSet == 'STARE':
        fovTH = 430  # chase_DB1:300, stare: 430
        th = 50  # chase_db1: 10, stare_50

    feature_sig = 3
    if name == 'im0004.ppm':
        fovTH = 1153
        feature_sig = 3

    img_lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)
    L = img_lab[:, :, 0]
    before = 0
    while True:
        inFOV = (L > th) * 1
        after = inFOV.sum()
        aaa = after - before
        if name == 'im0004.ppm':
            if aaa == 147:
                break
        if aaa <= fovTH:
            break
        before = after
        before_FOV = inFOV
        th = th - 1
    R = img[:, :, 0]
    edges = feature.canny(R, sigma=feature_sig)
    bbb = before_FOV.copy()
    bbb = (bbb == 1)
    bbb = (edges | bbb) * 1
    bbb = remove_small_holes(bbb, min_size=6000, connectivity=1)
    bbb = remove_small_objects(bbb, min_size=50)
    # bbb = bwareaopen(bbb, 1000)
    mask = bbb * 1
    return mask

# ---------- tiles ----------
def get_tile_size(length, ratio, num_down, pyr_level):
    max_down = max(num_down, pyr_level)
    tile_len = length * ratio
    tile_len = int(np.ceil(tile_len / (2 ** max_down)) * (2 ** max_down))
    return tile_len

def test_tiles(img, tile_size=(128, 128), stride=(64, 64), l_max=3):
    start = time.time()
    if not stride:
        stride = (int(tile_size[0] / 2), int(tile_size[1] / 2))

    dims = img.ndim
    length = img.shape[0]
    width = img.shape[1]
    tile_len = tile_size[0]
    tile_wid = tile_size[1]
    x_stride = stride[0]
    y_stride = stride[1]

    x_tNum = int(np.ceil((length - tile_len) * 1.0 / x_stride))
    y_tNum = int(np.ceil((width - tile_wid) * 1.0 / y_stride))
    tile_num = (x_tNum + 1) * (y_tNum + 1)

    new_len = x_stride * x_tNum + tile_len
    new_wid = y_stride * y_tNum + tile_wid

    pad_l = new_len - length + tile_len * (2 ** (l_max - 1) - 1)
    pl2 = pad_l / 2
    pad_w = new_wid - width + tile_wid * (2 ** (l_max - 1) - 1)
    pw2 = pad_w / 2

    x_padL = int(pl2)
    x_padR = int(pad_l - x_padL)
    y_padT = int(pw2)
    y_padB = int(pad_w - y_padT)

    print('pre:', time.time() - start)

    start = time.time()
    if dims == 3:
        channels = img.shape[2]
        img_padded = np.lib.pad(img, ((x_padL, x_padR), (y_padT, y_padB), (0, 0)), 'symmetric')
        tiles = np.zeros((tile_num * l_max, tile_len, tile_wid, channels))
    else:
        img_padded = np.lib.pad(img, ((x_padL, x_padR), (y_padT, y_padB)), 'symmetric')
        tiles = np.zeros((tile_num * l_max, tile_len, tile_wid))
    print('pad:', time.time() - start)

    for level in range(l_max):
        start = time.time()
        tiles[level::l_max] = make_test_tiles(img_padded, x_tNum, y_tNum, tile_len,
                                              tile_wid, level, l_max, x_stride, y_stride)
        print('make tile:', time.time() - start)

    return tiles, x_tNum + 1, y_tNum + 1
    # return tiles

def tile_combine(tiles, img_size, tile_size=(128, 128), x_stride=64, y_stride=64):
    t_len = tile_size[0]
    t_wid = tile_size[1]

    length = img_size[0]
    width = img_size[1]

    x_tNum = int(np.ceil((length - t_len) * 1.0 / x_stride))
    y_tNum = int(np.ceil((width - t_wid) * 1.0 / y_stride))

    new_len = x_stride * x_tNum + t_len
    new_wid = y_stride * y_tNum + t_wid

    pad_l = new_len - length
    pl2 = int(pad_l / 2)
    pad_w = new_wid - width
    pw2 = int(pad_w / 2)

    pred_padded = np.zeros((new_len, new_wid))
    count_padded = np.zeros((new_len, new_wid))
    for i in range(x_tNum + 1):
        for j in range(y_tNum + 1):
            pred_padded[i * x_stride:i * x_stride + t_len, j * y_stride:j * y_stride + t_wid] += tiles[
                i * (y_tNum + 1) + j]
            count_padded[i * x_stride:i * x_stride + t_len, j * y_stride:j * y_stride + t_wid] += 1

    pred_cols = pred_padded[pl2:new_len - pad_l + pl2]
    count_cols = count_padded[pl2:new_len - pad_l + pl2]
    pad_index = range(pl2)
    pred_cols[list(reversed(pad_index))] += pred_padded[pad_index]
    count_cols[list(reversed(pad_index))] += count_padded[pad_index]

    pad_index1 = range(length - pad_l + pl2, length)
    pad_index2 = range(new_len - pad_l + pl2, new_len)
    pred_cols[pad_index1] += pred_padded[list(reversed(pad_index2))]
    count_cols[pad_index1] += count_padded[list(reversed(pad_index2))]

    pred = pred_cols[:, pw2:new_wid - pad_w + pw2]
    count = count_cols[:, pw2:new_wid - pad_w + pw2]
    pad_index = range(pw2)
    pred[:, list(reversed(pad_index))] += pred_cols[:, pad_index]
    count[:, list(reversed(pad_index))] += count_cols[:, pad_index]

    pad_index1 = range(width - pad_w + pw2, width)
    pad_index2 = range(new_wid - pad_w + pw2, new_wid)
    pred[:, pad_index1] += pred_cols[:, list(reversed(pad_index2))]
    count[:, pad_index1] += count_cols[:, list(reversed(pad_index2))]

    pred = pred / count

    return pred

def make_test_tiles(img_padded, x_tNum, y_tNum, tile_len, tile_wid, level, l_max,
                    x_stride, y_stride):
    dims = img_padded.ndim
    x_s = int(x_stride / (2 ** level))
    y_s = int(y_stride / (2 ** level))

    x_pad = tile_len * (2 ** (l_max - 1) - 1) / 2
    y_pad = tile_wid * (2 ** (l_max - 1) - 1) / 2

    x_start = int(x_pad - tile_len * (2 ** level - 1) / 2)
    y_start = int(y_pad - tile_wid * (2 ** level - 1) / 2)

    if dims == 3:
        channels = img_padded.shape[2]
        new_img = img_padded[x_start::2 ** level][:, y_start::2 ** level, :]
        tiles = np.zeros(((x_tNum + 1) * (y_tNum + 1), tile_len, tile_wid, channels))
        for i in range(x_tNum + 1):
            for j in range(y_tNum + 1):
                tiles[i * (y_tNum + 1) + j] = new_img[i * x_s:i * x_s + tile_len, j * y_s:j * y_s + tile_wid, :]
    else:
        new_img = img_padded[x_start::2 ** level][:, y_start::2 ** level]
        tiles = np.zeros(((x_tNum + 1) * (y_tNum + 1), tile_len, tile_wid))
        for i in range(x_tNum + 1):
            for j in range(y_tNum + 1):
                tiles[i * (y_tNum + 1) + j] = new_img[i * x_s:i * x_s + tile_len, j * y_s:j * y_s + tile_wid]

    return tiles

# initilize weight
def init_weight(net):
    if isinstance(net, nn.Conv2d):
        init.xavier_normal(net.weight)
        init.constant(net.bias, 0)
    elif isinstance(net, nn.ConvTranspose2d):
        init.xavier_normal(net.weight)
        init.constant(net.bias, 0)
    else:
        for m in net.modules():
            if isinstance(m, nn.Conv2d):
                init.xavier_normal(m.weight)
                init.constant(m.bias, 0)
            elif isinstance(m, nn.ConvTranspose2d):
                init.xavier_normal(m.weight)
                init.constant(m.bias, 0)

def adjust_learning_rate(optimizer, decay_rate=0.1):
    for param_group in optimizer.param_groups:
        param_group['lr'] = param_group['lr'] * decay_rate


import os
import numpy as np
from Base import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger

def getTrain(arrayTrainUser, arrayTrainMovie, arrayTrainRate,
             arrayValidUser, arrayValidMovie, arrayValidRate,
             epochs, batch_size,
             intUserSize, intMovieSize, intEmbedding, floatDropout, floatL2regularizer,
             boolBias, boolDeep,
             strProjectFolder, strOutputPath):
    if boolDeep:
        model = Model.MatrixFactorizationWithDeep(intUserSize, intMovieSize, intEmbedding, floatDropout,
                                                  floatL2regularizer)
    else:
        model = Model.MatrixFactorizationWithBias(intUserSize, intMovieSize, intEmbedding, floatDropout,
                                                  floatL2regularizer, boolBias)

    callbacks = [EarlyStopping("val_loss", patience=100, restore_best_weights=True)
        , ModelCheckpoint(os.path.join(strProjectFolder, strOutputPath + "model.h5"), save_best_only=True)
        , CSVLogger(os.path.join(strProjectFolder, strOutputPath + "training-log.csv"), separator=",", append=False)]

    History = model.fit(x=[arrayTrainUser, arrayTrainMovie], y=arrayTrainRate,
                        epochs=epochs, batch_size=batch_size, verbose=1,
                        validation_data=([arrayValidUser, arrayValidMovie], arrayValidRate),
                        callbacks=callbacks)

    model.save(os.path.join(strProjectFolder, strOutputPath + "model.h5"))

    training_loss, training_accuracy = model.evaluate(x=[arrayTrainUser, arrayTrainMovie], y=arrayTrainRate)

    return model, History, training_loss, training_accuracy

Week08（HW5） - Predict movie ratings/Base/Train.py


# ! /usr/bin/env python
# -*- coding: utf-8 -*-
import clearData,json,pymongo
"""
┏┓　　　┏┓
  ┏┛┻━━━┛┻┓
┃　　　　　　　┃
┃　　　━　　　┃
┃　┳┛　┗┳　┃
┃　　　　　　　┃
┃　　　┻　　　┃
┃　　　　　　　┃
┗━┓　　　┏━┛
┃　　　┃
┃　　　┃
┃　　　┗━━━┓
┃　　　　　　　┣┓
┃　　　　　　　┏┛
┗┓┓┏━┳┓┏┛
┃┫┫　┃┫┫
┗┻┛　┗┻┛
神兽保证代码没bug
"""
class data:

    def save_data_json(self,Dict,file_position='',way=''):

        '''
        将数据通过json的方式保存在txt中。
        :param Dict:以字典形式存放的数据
        :param file_position:如果不存在那么就是默认保存在file文件夹下面，否则是保存在其他路径下.
        :param way:以json格式进行保存
        :return:
        '''

        Dict = clearData.clear_dict_space(Dict)
        if way == '':
            way = 'a'
        f = open(file_position,way)
        Dict = json.dumps(Dict)
        f.write(Dict)
        f.write('\n')
        f.close()

    def save_data_mongodb(self,Dict,mongodbUrl,databaseName,collectionName):

        '''
        :param Dict:以字典形式存放的数据
        :param mongodbUrl:mongodb的地址
        :param databaseName:数据库的名字
        :param collectionName:集合的名字
        :return:
        '''

        Dict = clearData.clear_dict_space(Dict)
        client = pymongo.MongoClient(mongodbUrl)
        db = client[databaseName]
        posts = db[collectionName]
        posts.insert(Dict)
        client.close()


'''
Created by Samvel Khalatyan, May 23, 2013
Copyright 2013, All rights reserved
'''

from django.conf.urls import patterns, url

from authors import views

urlpatterns = patterns('',
    # get list of authors
    url(r'^$',
        views.AuthorsIndexView.as_view(),
        name='index'),

    # get author details
    url(r'^(?P<author_id>\d+)/$',
        views.AuthorDetailView.as_view(),
        name='detail'),

    # -- AJAX requests
    # get list of authors
    url(r'^ajax/$',
        views.AjaxAuthorsIndexView.as_view(),
        name='ajax-index')
)


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from api.routes.todo import todo
from api.routes.user import user

app = FastAPI()
origins = ['https://localhost:3000']
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials = True,
    allow_methods=['*'],
    allow_headers=['*'],
)
app.include_router(user)
app.include_router(todo)


from sklearn import neural_network
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler

from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from sklearn.metrics import r2_score, mean_squared_error
import pickle
import numpy as np
import sys
import pandas as pd
import datetime
import csv

class Pred_Table():
    def __init__(self):
        self.pred_table={}
        self.time_model = None
        self.targetNum_model = None
        self.target_std = None
        self.df = None

    def parse_name(self,video_name):
        info = video_name.split('_')
        machine = info[1]
        date=info[2].split('-')
        weekday = str(datetime.datetime(int(date[0]),int(date[1]),int(date[2])).strftime("%w"))
        m = info[3].split(':')
        min_ = int(m[0])*60+int(m[1])

        return machine,weekday,min_

    def get_pred_targetNum_GT(self, video_name,frames_num,sample_rate):

        machine,weekday,min_ = self.parse_name(video_name)
        x_df1 = pd.get_dummies(pd.DataFrame({
              'machine':[str(machine)],
              'weekday':[str(weekday)],
              'minute':[min_],
              'frames':[frames_num],
              'sample_rate':[sample_rate]
             }))

        if self.df is not None:
            x = x_df1.reindex(columns = self.df.columns,fill_value=0).values.astype(float)
        else:
            raise Exception('dataFrame has not been set')

        if self.target_std is not None:
            x[:,:3] = self.target_std.transform(x[:,:3])
        else:
            raise Exception('stdizer has not been set')

        y = self.targetNum_model.predict(x)
        if y[0]<=0:
            return 0.001
        else:
            return y[0]

    def target_regression(self, dataSet_path='dataSet/dataSet_8.csv', check_score=True):
        # reshape the target

        raw_data = pd.read_csv(dataSet_path)

        names = raw_data.drop(['frames','sample_rate','targetNum','time'], axis = 1).values
        names=np.reshape(names,(names.shape[0],))

        # split the data which is needed for regression
        with open('dataSet/processed_data.csv','w',newline='') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['machine','weekday','minute','frames','sample_rate','targetNum'])
            for i,video_name in enumerate(names):
                machine,weekday,min_ = self.parse_name(video_name)
                row = [
                        machine,weekday,
                        min_,
                        raw_data['frames'][i],
                        raw_data['sample_rate'][i],
                        raw_data['targetNum'][i]
                      ]
                writer.writerow(row)

        # fit the model
        df = pd.read_csv('dataSet/processed_data.csv')
        X = df.drop(['targetNum'], axis = 1)
        y = df['targetNum'].values
        cat_feature=['machine','weekday']

        X = pd.get_dummies(X,columns=cat_feature)

        self.df = X.iloc[[0]] # save dataFrame for new input one-hot coding

        X = X.values.astype(float)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,random_state = 0)

        # Standardize is important to fit this model
        self.target_std = StandardScaler() # save the std scaler (before inferencing the data, we need to std it first)
        self.target_std.fit(X_train[:,:3])

        X_train[:,:3]=self.target_std.transform(X_train[:,:3])
        X_test[:,:3]=self.target_std.transform(X_test[:,:3])

        # start to train regression model
        mlp = neural_network.MLPRegressor(hidden_layer_sizes=(20,10), activation="relu",
                 solver='adam', alpha=0.0001,
                 batch_size=10, learning_rate="constant",
                 learning_rate_init=0.01,
                 power_t=0.5, max_iter=200,tol=1e-4)

        mlp.fit(X_train,y_train)

        self.targetNum_model = mlp # save the model

        if check_score:
            y_train_pred = mlp.predict(X_train)
            y_test_pred = mlp.predict(X_test)

            print('------predicting target model performance-----')
            print('(MSE) train: %.2f, test: %.2f'%(mean_squared_error(y_train,y_train_pred),
                                                   mean_squared_error(y_test,y_test_pred)))
            print('(R^2) train: %.2f, test: %.2f'%(r2_score(y_train,y_train_pred),
                                                   r2_score(y_test,y_test_pred)))

    def get_pred_time(self,sample_rate):
        try:
            if self.time_model is not None:
                y_test_pred = self.time_model.predict(np.asarray(sample_rate).reshape([-1,1]))
        except ModelError:
            print("Not yet set model !")

        return y_test_pred[0]

    def	time_regression(self,dataSet_path='dataSet/dataSet_8.csv',check_score=True):

        df = pd.read_csv(dataSet_path)
        df = shuffle(df)
        X = df.drop(['video_name','time','frames'], axis = 1).values

        y = df['time'].values
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,random_state = 0)

        # start to train regression model
        mlp = neural_network.MLPRegressor(hidden_layer_sizes=(10), activation="relu",
                 solver='adam', alpha=0.0001,
                 batch_size=10, learning_rate="constant",
                 learning_rate_init=0.01,
                 power_t=0.5, max_iter=200,tol=1e-4)
        mlp.fit(X_train,y_train)

        self.time_model = mlp
        if check_score:
            y_train_pred = mlp.predict(X_train)
            y_test_pred = mlp.predict(X_test)

            print('-----predicting time model performance-----')
            print('(MSE) train: %.2f, test: %.2f'%(mean_squared_error(y_train,y_train_pred),
                                                   mean_squared_error(y_test,y_test_pred)))
            print('(R^2) train: %.2f, test: %.2f'%(r2_score(y_train,y_train_pred),
                                                   r2_score(y_test,y_test_pred)))
    def save_table(self):
        with open("dataSet/pred_table.pickle", "wb") as file_:
            pickle.dump(self, file_, -1)

pred = Pred_Table()
pred.time_regression()
pred.target_regression()
pred.save_table()


import prism_metadata
import setup_logger
import logging

logger = logging.getLogger(setup_logger.LOGGER_NAME)

if __name__ == "__main__":
    setup_logger.setup(verbose=True)

    assay_plates = prism_metadata.read_assay_plate_from_file(
        "functional_tests/test_functional_prism_metadata_perturbagen/REP Master Plate Key_All Pools.txt", "prism_pipeline.cfg")
    replicates_det_plates = {"PREP001_P7_X1", "PREP001_P8_X1"}
    assay_plates = [x for x in assay_plates if x.det_plate in replicates_det_plates]
    logger.info("assay_plates:  {}".format(assay_plates))

    assay_plate_barcodes = set([x.assay_plate_barcode for x in assay_plates])

    perts = prism_metadata.build_perturbagens_from_file(
        "functional_tests/test_functional_prism_metadata_perturbagen/7159-03-A02-01-01_02-29-16_12.20.32.txt",
        prism_metadata.plate_map_type_CM,
        "prism_pipeline.cfg")

    replicate_perts = [x for x in perts if x.assay_plate_barcode in assay_plate_barcodes]

    well_pert_map = prism_metadata.validate_perturbagens(replicate_perts)

    logger.info("len(well_part_map):  {}".format(len(well_pert_map)))
    assert len(well_pert_map) == 384, len(well_pert_map)


from SparkContext import sc
from pyspark.mllib.clustering.KMeansModel import KMeans

def main():
    data = sc.textFile('Kmeans_Data.txt')
    k = 4 #假设总共聚成4类

    model = KMeans.train( data, k, maxIterations = 60, runs = 5, initializationMode = 'random') #模型并行训练

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Feb  4 13:05:32 2020

@author: knoch
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as tick

plt.close('all') #closes all figures

pi = np.pi
x = np.arange(0, 2*np.pi, 2*np.pi/100.0)
ll_y = -2
ul_y = 2
y = np.sin(x)

yp= np.diff(y)/np.diff(x)
xp = x[:-1]+np.diff(x)/2

fig1 = plt.figure(1)
ax1 = fig1.gca()
ax1.set_ylim(ll_y, ul_y)
ax1.yaxis.set_major_formatter(tick.StrMethodFormatter('{x:5.1f}'))
ax1.grid(True)
ax1.plot(x, y,color='green',marker='.',linestyle='None')

ax1.plot(xp, yp,color='red',marker='.',linestyle='None')