# -*- coding: utf-8 -*-
"""
Created on Sun Jan 16 20:24:40 2022

@author: MC
"""

import pandas as pd
import ELO_Functions as elo
import streamlit as st
import altair as alt

header = st.container()
inputs = st.container()
graph = st.container()
footer = st.container()

with header:
    st.title('The Vaz Brothers Fifa Rankings!')
    st.text('')
    st.text('This project uses a custom ELO formula that takes into account team ratings')
    st.text('This custom formula uses a different k-factor depending on team ratings')
    st.text('The purpose is to determine which brother is the best at FIFA')
    st.text('')
    st.text('Instructions:')
    st.text('1. Select the Winner')
    st.text('2. Select the Loser')
    st.text('3. Team Rankings (stars)')
    st.text('4. Hit the "Calculate" button')

with inputs:

    st.title('Inputs for calculation')
    col1, col2, col3, col4 = st.columns((3,1,3,1))

    Players_List = ['MC','Austin','Liam','Donovan']
    Ratings_List = [5,4.5,4,3.5,3,2.5,2,1.5,1,0.5]

    with col1:
        Won = st.selectbox('Winner', Players_List)
        WTR = st.selectbox('Winner Team Rating',Ratings_List)
    with col3:
        Lost = st.selectbox('Loser', Players_List)
        LTR = st.selectbox('Loser Team Rating',Ratings_List)

    calculate = st.button('Calculate')

    Player_Data = pd.read_csv('ELO_Rankings')

    melted_df = Player_Data.melt(id_vars = ['Date'],
                                 value_vars = ['Austin','MC','Donovan','Liam'],
                                 var_name = 'Player',
                                 value_name = 'ELO')

    Chart = (alt.Chart(melted_df, width =700, height = 600)
             .mark_line()
             .configure_title(fontSize = 24)
             .configure_axis(titleFontSize = 20)
             .encode(x=alt.X("Date:T", title = 'Date'),
                     y= alt.Y("ELO:Q",title = 'ELO Rating',scale = alt.Scale(domain=(1200, 1700))),
                     color = 'Player')
             .properties(title = "Ranking over Time"))

if calculate:

    result = elo.play_a_game(Won, Lost, WTR, LTR, Player_Data)

    result.to_csv('ELO_Rankings')

    with graph:

        st.dataframe(result.drop(columns = 'Date').tail(1))

        #st.text('This graph shows our ELO scores over time!')

        #st.altair_chart(Chart)


### 메인 함수 있는 곳

# --------------------------------------------

import streamlit as st

# 다른 파이썬 파일 임포트
from app_image import run_app_image
from app_csv import run_app_csv
from app_about import run_app_about
# ----------------------------------------------

def main():
    st.title('내 앱 대시보드')

    ### 누르면 왼쪽에 메뉴 나오게 하기
    menu = ['이미지업로드', 'csv업로드', 'About']
    # ('메뉴'-> 유저에게 보여지는거, menu -> 여기서 가져와라)
    # st.sidebar.selectbox('메뉴', menu)  # 화면에 보여주기만 하는것

    ## 선택한 메뉴 페이지로 이동하기
    choice = st.sidebar.selectbox('메뉴', menu)
    # print(choice)
    # if choice == '이미지업로드': # 라고 해도 되지만 메뉴 이름 자동변환위해

    if choice == menu[0] :
        run_app_image()

    elif choice == menu[1] :  # ------------------------------------
        run_app_csv()

    else : # -------------------------------------------------------
        run_app_about()

if __name__ == '__main__':
    main()


from bs4 import BeautifulSoup as bs
import requests
import cssutils
import pandas as pd

def scrape():

    mars_data = {}

    #retrieve new mars news
    news = {}
    mars_news = requests.get("https://mars.nasa.gov/news/")
    soup = bs(mars_news.text, 'html.parser')
    topNews = soup.find("div", class_="image_and_description_container")

    news_p = topNews.find("div", class_="rollover_description_inner").text
    news_title = topNews.find_all("img")[1]
    news_title = news_title.get("alt","")

    mars_data["news_title"] = news_title
    mars_data["news_p"] = news_p

    #retrieve jpl featured image
    jpl = requests.get("https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars")
    jplsoup = bs(jpl.text, 'html.parser')
    featured = jplsoup.find("div", id="page")
    featured = featured.find("article")
    featured = featured.get("style")
    style = cssutils.parseStyle(featured)
    url = style['background-image']
    url = url.replace('url(', '').replace(')', '')
    featured_image_url = "https://www.jpl.nasa.gov" + url

    mars_data["featured_image"] = featured_image_url

    #retrieve most recent mars weather
    weather = requests.get("https://twitter.com/marswxreport?lang=en")
    wsoup = bs(weather.text, "html.parser")

    latest = wsoup.find('div', {'class': 'ProfileTimeline'})
    latest = latest.find("p", {'class': "TweetTextSize TweetTextSize--normal js-tweet-text tweet-text"})
    mars_weather = latest.text

    mars_data["weather"] = mars_weather

    #retrieve mars fact table
    facts = requests.get("https://space-facts.com/mars/")
    fsoup = bs(facts.text, "html.parser")

    def souptable(table):
        for row in table.find_all('tr'):
            yield [col.text for col in row.find_all('td')]

    table = fsoup.find("table")
    mf_df = pd.DataFrame(souptable(table))
    mf_df.columns = ['','value']
    facts = pd.DataFrame.to_html(mf_df)

    mars_data["facts"] = facts

    #retrieve mars hemisphere enhanced photos
    hemi = requests.get("https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars")
    hsoup = bs(hemi.text, "html.parser")

    picsource = hsoup.find_all("div", class_="item")

    hemi_dicts = []
    baseurl = "https://astrogeology.usgs.gov"

    for i in range(len(picsource)):
        img_info = {}
        title = picsource[i]("h3")[0].text
        imagehunt = picsource[i].find("a").get("href", "")
        temp = requests.get(baseurl + imagehunt)
        tempsoup = bs(temp.text, "html.parser")
        image_ext = tempsoup.find("img", class_="wide-image").get("src","")
        img_url = baseurl + image_ext
        img_info["title"] = title
        img_info["img_url"] = img_url
        hemi_dicts.append(img_info)

    mars_data["hemispheres"] = hemi_dicts

    print("SCRAPE COMPLETE")
    #print(mars_data)

    return mars_data

#scrape()


# USAGE
# python object_detector.py --conf config/config.json --image florida_trip.png

# import the necessary packages

from config.conf import Conf
from yolo_parser.parseyolooutput import ParseYOLOOutput
from datetime import datetime
from imutils import paths
import numpy as np
import argparse
import imutils
import cv2
import os

#imagePaths = list(paths.list_images(args["dataset"]))

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-c", "--conf", required=True,
	help="Path to the input configuration file")
ap.add_argument("-i", "--image", required=True,
	help="Path of the image")
args = vars(ap.parse_args())

# load the image configuration file
conf = Conf(args["conf"])
image = cv2.imread(args["image"])

# load the COCO class labels our YOLO model was trained on
labelsPath = os.path.sep.join([conf["yolo_path"], "coco.names"])
LABELS = open(labelsPath).read().strip().split("\n")

# initialize a list of colors to represent each possible class label
np.random.seed(42)
COLORS = np.random.randint(0, 255, size=(len(LABELS), 3),
	dtype="uint8")

# derive the paths to the YOLO weights and model configuration
weightsPath = os.path.sep.join([conf["yolo_path"], "yolov3.weights"])
configPath = os.path.sep.join([conf["yolo_path"], "yolov3.cfg"])

# load our YOLO object detector trained on COCO dataset (80 classes)
# and determine only the *output* layer names that we need from YOLO
print("[INFO] loading YOLO from disk...")
net = cv2.dnn.readNetFromDarknet(configPath, weightsPath)
ln = net.getLayerNames()
ln = [ln[i[0] - 1] for i in net.getUnconnectedOutLayers()]

# initialize the YOLO output parsing object
pyo = ParseYOLOOutput(conf)

# resize the image, convert it to grayscale, and blur it
image = imutils.resize(image, width=400)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)

# construct a blob from the input image and then perform
# a forward pass of the YOLO object detector, giving us
# our bounding boxes and associated probabilities
blob = cv2.dnn.blobFromImage(image, 1 / 255.0,
            (416, 416), swapRB=True, crop=False)
net.setInput(blob)
layerOutputs = net.forward(ln)

(H, W) = image.shape[:2]

# parse YOLOv3 output
(boxes, confidences, classIDs) = pyo.parse(layerOutputs, LABELS, H, W)

# apply non-maxima suppression to suppress weak,
# overlapping bounding boxes
idxs = cv2.dnn.NMSBoxes(boxes, confidences,
				conf["confidence"], conf["threshold"])

# ensure at least one detection exists
if len(idxs) > 0:
    # loop over the indexes we are keeping
    for i in idxs.flatten():
        # extract the bounding box coordinates
        (x, y) = (boxes[i][0], boxes[i][1])
        (w, h) = (boxes[i][2], boxes[i][3])

        # draw a bounding box rectangle and label on the frame
        color = [int(c) for c in COLORS[classIDs[i]]]
        cv2.rectangle(image, (x, y), (x + w, y + h), color, 2)
        text = "{}: {:.4f}".format(LABELS[classIDs[i]],confidences[i])
        y = (y - 15) if (y - 15) > 0 else h - 15
        cv2.putText(image, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
        # show the frame
        cv2.imshow("show", image)
        cv2.waitKey(0)

# do a bit of cleanup
cv2.destroyAllWindows()


from collections import namedtuple

from django.conf import ImproperlyConfigured, settings
from django.contrib.admindocs.views import simplify_regex
from django.core.cache import cache
from django.urls.resolvers import RegexPattern, RoutePattern, URLPattern, URLResolver
from django.utils.module_loading import import_string

from . import defaults

def _get_setting(name):
    return getattr(settings, name, getattr(defaults, name))

def _get_blacklist(name):
    blacklist = _get_setting('PERMISSIONS_AUDITOR_BLACKLIST')

    # Fall back to the defauls if the user does not provide the specific
    # blacklist in their settings.
    if name not in blacklist:
        blacklist = getattr(defaults, 'PERMISSIONS_AUDITOR_BLACKLIST')

    return blacklist[name]

NAMESPACE_BLACKLIST = tuple(_get_blacklist('namespaces'))
VIEW_BLACKLIST = tuple(_get_blacklist('view_names'))
MODULE_BLACKLIST = tuple(_get_blacklist('modules'))

ViewDetails = namedtuple('ViewDetails', [
    'module', 'name', 'url', 'permissions', 'login_required', 'docstring'
])

class ViewParser:

    def __init__(self):
        self.load_processors()

    def load_processors(self):
        self._processors = []

        for processor_path in _get_setting('PERMISSIONS_AUDITOR_PROCESSORS'):

            try:
                processor = import_string(processor_path)
                self._processors.append(processor())
            except (ImportError, TypeError) as ex:
                raise ImproperlyConfigured(
                    '{} is not a valid permissions processor.'.format(processor_path)
                ) from ex

    def parse(self, view):
        """
        Process a view.

        Returns a tuple containing:
        permissions (list), login_required (boolean or None), docstring (str)
        """
        permissions = []
        login_required = False
        docstrings = []

        for processor in self._processors:
            if processor.can_process(view):
                permissions.extend(processor.get_permission_required(view))

                login_required_result = processor.get_login_required(view)
                if login_required_result is None and not login_required:
                    login_required = login_required_result
                else:
                    login_required = login_required_result or login_required

                docstrings.append(processor.get_docstring(view))

        return (
            sorted(set(permissions)),
            login_required,
            '\n'.join(list(set(filter(None, docstrings))))
        )

def _get_views(urlpatterns=None, base_url=''):
    """
    Recursively fetch all views in the specified urlpatterns.

    If urlpatterns is not specified, uses the `PERMISSIONS_AUDITOR_ROOT_URLCONF`
    setting, which by default is the value of `ROOT_URLCONF` in your project settings.

    Returns a list of `View` namedtuples.
    """
    views = []

    if urlpatterns is None:
        root_urlconf = import_string(_get_setting('PERMISSIONS_AUDITOR_ROOT_URLCONF'))
        urlpatterns = root_urlconf.urlpatterns

    parser = ViewParser()

    for pattern in urlpatterns:
        if isinstance(pattern, RoutePattern) or isinstance(pattern, URLResolver):

            if pattern.namespace in NAMESPACE_BLACKLIST:
                continue

            # Recursively fetch patterns
            views.extend(_get_views(pattern.url_patterns, base_url + str(pattern.pattern)))

        elif isinstance(pattern, URLPattern) or isinstance(pattern, RegexPattern):
            view = pattern.callback

            # If this is a CBV, use the actual class instead of the as_view() classmethod.
            view = getattr(view, 'view_class', view)
            view = getattr(view, 'cls', view)

            full_view_path = '{}.{}'.format(view.__module__, view.__name__)
            if full_view_path in VIEW_BLACKLIST or view.__module__ in MODULE_BLACKLIST:
                continue

            permissions, login_required, docstring = parser.parse(view)

            views.append(ViewDetails._make([
                view.__module__,
                view.__name__,
                simplify_regex(base_url + str(pattern.pattern)),
                permissions,
                login_required,
                docstring
            ]))

    return views

def get_views(urlpatterns=None, base_url=''):
    """
    Wrapper for caching _get_views().
    """
    cache_key = _get_setting('PERMISSIONS_AUDITOR_CACHE_KEY')
    cache_base_url_key = cache_key + '_BASE_URL'
    cache_timeout = _get_setting('PERMISSIONS_AUDITOR_CACHE_TIMEOUT')

    cache_content = cache.get_many([cache_key, cache_base_url_key])
    views = cache_content.get(cache_key, None)
    cached_base_url = cache_content.get(cache_base_url_key, None)

    if views is None or cached_base_url != base_url:
        views = _get_views(urlpatterns, base_url)
        cache.set_many({
            cache_key: views,
            cache_base_url_key: base_url
        }, timeout=cache_timeout)

    return views


from django.conf.urls import url
from django.contrib import admin

from .views import (
	olymps_list,
	olymp_create,
	olymp_detail,
	olymp_update,
	olymp_delete,
	)

app_name = 'Enigmath'
urlpatterns = [
	url(r'^$', olymps_list, name='list'),
    url(r'^create/$', olymp_create, name='create'),
    url(r'^(?P<slug>[\w-]+)/$', olymp_detail, name='detail'),
    url(r'^(?P<slug>[\w-]+)/edit/$', olymp_update, name='update'),
    url(r'^(?P<slug>[\w-]+)/delete/$', olymp_delete, name='delete'),
]


import argparse

try:
    from waymo_open_dataset import dataset_pb2
except ImportError:
    raise ImportError(
        'Please run "pip install waymo-open-dataset-tf-2-1-0==1.2.0" '
        'to install the official devkit first.')

from glob import glob
from os.path import join

import mmcv
import tensorflow as tf
from waymo_open_dataset.protos import metrics_pb2

def save_label(frame, objects, version='multi-view', cam_sync=True):
    """Parse and save gt bin file for camera-only 3D detection on Waymo.

    Args:
        frame (:obj:`Frame`): Open dataset frame proto.
        objects (:obj:`Object`): Ground truths in waymo dataset Object proto.
        version (str): Version of gt bin file. Choices include 'multi-view'
            and 'front-view'. Defaults to 'multi-view'.
        cam_sync (bool): Whether to generate camera synced gt bin. Defaults to
            True.
    """
    id_to_bbox = dict()
    id_to_name = dict()
    for labels in frame.projected_lidar_labels:
        name = labels.name  # 0 unknown, 1-5 corresponds to 5 cameras
        for label in labels.labels:
            # TODO: need a workaround as bbox may not belong to front cam
            bbox = [
                label.box.center_x - label.box.length / 2,
                label.box.center_y - label.box.width / 2,
                label.box.center_x + label.box.length / 2,
                label.box.center_y + label.box.width / 2
            ]
            # object id in one frame
            id_to_bbox[label.id] = bbox
            id_to_name[label.id] = name - 1
    if version == 'multi-view':
        cam_list = [
            '_FRONT', '_FRONT_LEFT', '_FRONT_RIGHT', '_SIDE_LEFT',
            '_SIDE_RIGHT'
        ]
    elif version == 'front-view':
        cam_list = ['_FRONT']
    else:
        raise NotImplementedError
    for obj in frame.laser_labels:
        bounding_box = None
        name = None
        id = obj.id
        for cam in cam_list:
            if id + cam in id_to_bbox:
                bounding_box = id_to_bbox.get(id + cam)
                name = str(id_to_name.get(id + cam))
                break
        num_pts = obj.num_lidar_points_in_box

        if cam_sync:
            if obj.most_visible_camera_name:
                box3d = obj.camera_synced_box
            else:
                continue
        else:
            box3d = obj.box

        if bounding_box is not None and obj.type > 0 and num_pts >= 1:
            o = metrics_pb2.Object()
            o.context_name = frame.context.name
            o.frame_timestamp_micros = frame.timestamp_micros
            o.score = 0.5
            o.object.CopyFrom(obj)
            o.object.box.CopyFrom(box3d)
            objects.objects.append(o)

parser = argparse.ArgumentParser(description='Waymo GT Generator arg parser')
parser.add_argument(
    '--root-path',
    type=str,
    default='data/waymo/waymo_format/validation',
    help='specify the root path of dataset')
parser.add_argument(
    '--version',
    type=str,
    default='multi-view',
    required=False,
    help='specify the gt version')
parser.add_argument(
    '--out-dir',
    type=str,
    default='data/waymo/waymo_format/',
    required=False,
    help='name of info pkl')
parser.add_argument(
    '--backend',
    type=str,
    default='disk',
    required=False,
    help='file backend for data loading')
args = parser.parse_args()

if __name__ == '__main__':
    load_dir = args.root_path
    version = args.version
    out_dir = args.out_dir
    if args.backend == 'disk':
        file_client_args = dict(backend='disk')
    elif args.backend == 'petrel':
        file_client_args = dict(
            backend='petrel',
            path_mapping=dict({
                './data/waymo/':
                's3://openmmlab/datasets/detection3d/waymo-v1.3.1/',
                'data/waymo/':
                's3://openmmlab/datasets/detection3d/waymo-v1.3.1/'
            }))
    tfrecord_pathnames = glob(join(load_dir, '*.tfrecord'))
    if file_client_args['backend'] == 'disk':
        tfrecord_pathnames = sorted(glob(join(load_dir, '*.tfrecord')))
    else:
        file_client = mmcv.FileClient(**file_client_args)
        load_dir = file_client.client._map_path(load_dir)
        from petrel_client.client import Client
        client = Client()
        contents = client.list(load_dir)
        tfrecord_pathnames = list()
        for content in sorted(list(contents)):
            if content.endswith('tfrecord'):
                tfrecord_pathnames.append(join(load_dir, content))

    objects = metrics_pb2.Objects()
    progress_bar = mmcv.ProgressBar(len(tfrecord_pathnames))
    for i in range(len(tfrecord_pathnames)):
        pathname = tfrecord_pathnames[i]
        segment_filename = pathname.split('/')[-1]
        context_name = segment_filename[8:-28]
        dataset = tf.data.TFRecordDataset(pathname, compression_type='')
        for frame_idx, data in enumerate(dataset):
            frame = dataset_pb2.Frame()
            frame.ParseFromString(bytearray(data.numpy()))
            save_label(frame, objects, version)
        progress_bar.update()

    # Write objects to a file.
    f = open(join(out_dir, 'cam_gt.bin'), 'wb')
    f.write(objects.SerializeToString())
    f.close()


import os
import numpy as np
from flask import Flask, render_template, request
import pickle

os.system('python predictor.py')
fileobj = 'Dataset.pkl'
classifier= pickle.load(open(fileobj, 'rb'))
with open(fileobj, 'rb') as fs:
    classifier = pickle.load(fs)

app = Flask(__name__)

@app.route('/')
def home():
	return render_template('index.html')

@app.route('/predicting', methods=['POST'])
def predicting():
    a=0
    if request.method == 'POST':
        age = int(request.form['age'])
        dpf = float(request.form['dpf'])
        bmi = float(request.form['bmi'])
        insulin = int(request.form['insulin'])
        st = int(request.form['skinthickness'])
        bp = int(request.form['bloodpressure'])
        glucose = int(request.form['glucose'])
        preg = int(request.form['pregnancies'])
        data = np.array([[preg, glucose, bp, st, insulin, bmi, dpf, age]])
        result = classifier.predict(data)
        accuracies=pickle.load( open('Accuracies.pkl', 'rb'))
        predicting.farray=[]
        for i in accuracies:
            predicting.farray.append(i[1])
        for i in accuracies:
            if i[1]==max(predicting.farray):
                maximum=a

            a=a+1

        return render_template('result.html', result=result,accuracies=accuracies,maximum=accuracies[maximum])

if __name__ == '__main__':
	app.run(debug=True)


import os
import pickle
import chess
from chess.pgn import read_game
from copy import deepcopy
import numpy as np

piece_map = {
    'p' : [1,0,0,0,0,0,0,0,0,0,0,0,0],
    'P' : [0,0,0,0,0,0,1,0,0,0,0,0,0],
    'n' : [0,1,0,0,0,0,0,0,0,0,0,0,0],
    'N' : [0,0,0,0,0,0,0,1,0,0,0,0,0],
    'b' : [0,0,1,0,0,0,0,0,0,0,0,0,0],
    'B' : [0,0,0,0,0,0,0,0,1,0,0,0,0],
    'r' : [0,0,0,1,0,0,0,0,0,0,0,0,0],
    'R' : [0,0,0,0,0,0,0,0,0,1,0,0,0],
    'q' : [0,0,0,0,1,0,0,0,0,0,0,0,0],
    'Q' : [0,0,0,0,0,0,0,0,0,0,1,0,0],
    'k' : [0,0,0,0,0,1,0,0,0,0,0,0,0],
    'K' : [0,0,0,0,0,0,0,0,0,0,0,1,0],
    '.' : [0,0,0,0,0,0,0,0,0,0,0,0,1],
}

column_map = {
    'a' : [1,0,0,0,0,0,0,0,0,0,0,0,0],
    'b' : [0,1,0,0,0,0,0,0,0,0,0,0,0],
    'c' : [0,0,1,0,0,0,0,0,0,0,0,0,0],
    'd' : [0,0,0,1,0,0,0,0,0,0,0,0,0],
    'e' : [0,0,0,0,1,0,0,0,0,0,0,0,0],
    'f' : [0,0,0,0,0,1,0,0,0,0,0,0,0],
    'g' : [0,0,0,0,0,0,1,0,0,0,0,0,0],
    'h' : [0,0,0,0,0,0,0,1,0,0,0,0,0],
}

row_map = {
    '1' : [1,0,0,0,0,0,0,0,0,0,0,0,0],
    '2' : [0,1,0,0,0,0,0,0,0,0,0,0,0],
    '3' : [0,0,1,0,0,0,0,0,0,0,0,0,0],
    '4' : [0,0,0,1,0,0,0,0,0,0,0,0,0],
    '5' : [0,0,0,0,1,0,0,0,0,0,0,0,0],
    '6' : [0,0,0,0,0,1,0,0,0,0,0,0,0],
    '7' : [0,0,0,0,0,0,1,0,0,0,0,0,0],
    '8' : [0,0,0,0,0,0,0,1,0,0,0,0,0],
}

# this character is appended to the end of the move if a piece is being promoted to the piece denoted by the letter
promotion_map = {
    'n' : [0,0,0,0,0,0,0,0,1,0,0,0,0],
    'b' : [0,0,0,0,0,0,0,0,0,1,0,0,0],
    'r' : [0,0,0,0,0,0,0,0,0,0,1,0,0],
    'q' : [0,0,0,0,0,0,0,0,0,0,0,1,0],
    '.' : [0,0,0,0,0,0,0,0,0,0,0,0,1],
}

turn_map = {
    chess.WHITE: [1,1,1,1,1,1,0,0,0,0,0,0,0],
    chess.BLACK: [0,0,0,0,0,0,1,1,1,1,1,1,0],
    'b': [0,0,0,0,0,0,1,1,1,1,1,1,0],
    'w': [1,1,1,1,1,1,0,0,0,0,0,0,0],
}

class DataOrganization:
    MAX_MOVES = 100
    GAME_COUNT = 200
    def __init__(self, path=''):
        self.path = path
        self.x1 = []
        self.x2 = []
        self.y = []

    def create_pickle(self, name, obj):
        out = open(os.path.join(os.getcwd(), name), 'wb')
        pickle.dump(obj, out)
        out.close()
        return obj

    def depickle(self, name):
        with open(os.path.join(os.getcwd(), name), 'rb') as pick:
            tree = pickle.load(pick)
        return tree

    def gather_data(self):
        for file in os.listdir(self.path):
            print('reading ' + file)
            games = 0
            if (file.endswith('pgn')):
                pgn = open(os.path.join(self.path, file))
                game = read_game(pgn)
                while game is not None and games < self.GAME_COUNT:
                    games += 1
                    result = game.headers['Result'].strip()
                    if result == '1-0':
                        self.get_game_data_v2(game, True, games)
                    elif result == '0-1':
                        self.get_game_data_v2(game, False, games)
                    else:
                        self.get_game_data_v2(game, False, games)
                        self.get_game_data_v2(game, True, games)
                    # the chess package can sometimes be finicky about which games it wants to read. This prevents any unnecessary breakdowns
                    try:
                        game = read_game(pgn)
                    except:
                        game = read_game(pgn)
        self.x1 = np.asarray(self.x1)
        self.x2 = np.asarray(self.x2)
        self.y = np.asarray(self.y)
        print(self.x2.shape)
        print(self.x1.shape)
        print(self.y.shape)

    def check(self):
        for i in range(len(self.x1)):
            print(self.x1.shape)
            print(self.get_board_from_binary(self.x1[i]))
            print(self.convert_binary_to_move(self.x1[i]))
            print('white' if self.find_key(self.x1[i][7][8], turn_map) == chess.WHITE else 'black')
            print(self.y[i])

    def check_input(self, x):
        print(self.get_board_from_binary(x[0]))
        print(self.convert_binary_to_move(x[0]))
        print('white' if self.find_key(x[0][7][8], turn_map) == chess.WHITE else 'black')

    def get_game_data(self, game, is_white, games):
        iter = 0
        board = chess.Board()
        for move in game.mainline_moves():
            # we dont want white's moves if we're black and vice versa and we dont want to overload the model with opening knowledge
            if (iter % 2 != (0 if is_white else 1)) or (games % 5 != 0 and iter < 15):
                iter += 1
                board.push(move)
                continue
            move_uci = chess.Move.uci(move)
            input_board = self.get_binary_board(board, is_white)
            legal_moves = board.legal_moves
            num_moves_taken = np.random.randint(1, 5)

            if legal_moves.count() < num_moves_taken:
                board.push(move)
                iter += 1
                continue
            # we want the model to see more bad moves than good moves but we also dont want it to adapt to a certain probability
            # (i.e. just guessing that 3/4 of moves will be 0 and 1/4 will be one and outputting the same value for everything)
            # even if this skews its binary accuracy to be low
            # in practice, it will see all legal moves and only a handful will be good
            legal_moves = np.random.choice(list(legal_moves), num_moves_taken)
            legal_moves[num_moves_taken - 1] = move
            for legal_move in legal_moves:
                legal_move_uci = chess.Move.uci(legal_move)
                board_copy = self.append_binary_move(deepcopy(input_board), legal_move_uci)
                self.y.append(1 if move_uci == legal_move_uci else 0)
                self.x1.append(board_copy)
            board.push(move)
            iter += 1

    def append_binary_move(self, board, move):
        binary_move = self.get_binary_move(move)
        for i in range(5):
            board[i][8] = binary_move[i]
        return board

    def get_game_data_v2(self, game, is_white, games):
        board = chess.Board()
        for iterations, move in enumerate(game.mainline_moves()):
            # we dont want white's moves if we're black and vice versa and we dont want to overload the model with opening knowledge
            if (iterations % 2 != (0 if is_white else 1)) or (games % 10 != 0 and iterations < 15):
                board.push(move)
                continue
            move_uci = chess.Move.uci(move)
            x1_i = self.get_binary_board_v2(board)

            x2_i, y_i = self.get_x2_y(board, move_uci)
            x2_i[self.MAX_MOVES - 1] = np.ones(shape=(5, 13)) if is_white else  x2_i[self.MAX_MOVES - 1]
            self.x1.append(x1_i)
            self.x2.append(x2_i)
            self.y.append(y_i)
            board.push(move)

    def get_binary_board(self, board: chess.Board, is_single_batch = False):
        map = board.piece_map()
        output_board = np.zeros(shape=(8, 9, 13))
        iter = 0
        for i in range(8):
            row = np.zeros(shape=(9, 13))
            for j in range(8):
                row[j] = piece_map['.'] if iter not in map else piece_map[map[iter].symbol()]
                iter += 1
            output_board[i] = row
        output_board[7][8] = turn_map[board.turn]
        return output_board if not is_single_batch else np.asarray([output_board])

    '''
    this somewhat non pythonic method does what the above does without the middle man of the chess package.
    much faster when caller only has fen string to work with
    '''
    def get_binary_board_from_fen(self, fen: str):
        output_board = np.zeros(shape=(8, 9, 13))
        fen_idx = 0
        for i in range(8):
            row = np.zeros(shape=(9, 13))
            j = 0
            while j < 8:
                if fen[fen_idx] in piece_map:
                    row[j] = piece_map[fen[fen_idx]]
                    j += 1
                else:
                    for k in range(int(fen[fen_idx])):
                        row[j] = piece_map['.']
                        j += 1
                fen_idx += 1
            fen_idx += 1 # for / character
            output_board[7-i] = row
        output_board[7][8] = turn_map[fen[fen_idx]]
        return output_board

    def get_binary_board_v2(self, board):
        map = board.piece_map()
        output_board = np.zeros(shape=(8, 8, 13))
        iter = 0
        for i in range(8):
            row = np.zeros(shape=(8, 13))
            for j in range(8):
                row[j] = piece_map['.'] if iter not in map else piece_map[map[iter].symbol()]
                iter += 1
            output_board[7-i] = row
        return output_board

    def get_x2_y(self, board: chess.Board, move):
        y_i = np.zeros(shape=(self.MAX_MOVES))
        x2_i = np.zeros(shape=(self.MAX_MOVES, 5, 13))
        for idx, legal_move in enumerate(board.legal_moves()):
            legal_move_uci = chess.Move.uci(legal_move)
            y_i[idx] = 1 if move == legal_move_uci else 0
            x2_i[idx] = self.get_binary_move(legal_move_uci)
        return x2_i, y_i

    '''
    Converts moves to binary representations in constant time using the global maps
    This can either be done to create probability vector training outputs or inputs when assigning probabilities to legal moves
    '''
    def get_binary_move(self, move):
        arr = np.zeros(shape=(5, 13))
        arr[0] = column_map[move[0]]
        arr[1] = row_map[move[1]]
        arr[2] = column_map[move[2]]
        arr[3] = row_map[move[3]]

        arr[4] = promotion_map[move[4]] if len(move) == 5 else promotion_map['.']
        return arr

    def prepare_input(self, board: chess.Board, move: str, is_batch=False):
        board = self.get_binary_board(board, board.turn)
        a = self.append_binary_move(board, move)
        return np.asarray([self.append_binary_move(board, move)] if not is_batch else self.append_binary_move(board, move))

    def prepare_input_v2(self, board: chess.Board, is_batch=False):
        binary_moves, uci_moves = self.get_move_input_v2(board)
        return self.get_binary_board_v2(board), binary_moves, uci_moves

    def prepare_graph_train_data(self, board):
        return self.get_binary_board_v2(board),

    def get_move_input_v2(self, board: chess.Board):
        binary_moves = np.zeros(shape=(self.MAX_MOVES, 5, 13))
        uci_moves = []
        idx = 0
        for idx, move in enumerate(board.generate_legal_moves()):
            uci = chess.Move.uci(move)
            binary_moves[idx] = self.get_binary_move(uci)
            uci_moves.append(uci)

        if board.turn:
            binary_moves[idx + 1] = np.ones(shape=(5, 13))
        return binary_moves, uci_moves
    '''
    Mostly for debug, used to ensure binary representaitons of moves look the way they're supposed to
    '''
    def convert_binary_to_move(self, move):
        try:
            toRet = self.find_key(move[0], column_map)
            toRet += self.find_key(move[1], row_map)
            toRet += self.find_key(move[2], column_map)
            toRet += self.find_key(move[3], row_map)
            toRet += self.find_key(move[4], promotion_map)
        except:
            print(np.zeros(shape=(5, 13)))
            print(move)

        return toRet

    '''
    Mostly for debug, used to ensure boards look the way they're supposed to by converting binary boards into human readable character representations
    This is very slow, I would not recommend using this for large batches of data
    '''
    def get_board_from_binary(self, board):
        to_ret = np.zeros(shape=(8, 8), dtype=str)
        for i in range(8):
            for j in range(8):
                for piece in piece_map:
                    if np.allclose(np.asarray(piece_map[piece]), np.asarray(board[i][j])):
                        to_ret[i][j] = piece
                        break
        return to_ret

    def find_key(self, val, dict):
        for i in dict.keys():
            if np.allclose(np.asarray(dict[i]), val):
                return i

    '''
    it's necessary to 'clean' the fen because we don't want the same position represented multiple times through the move counters at the end of the fen
    '''
    @staticmethod
    def clean_fen(fen: str, start=''):
        if fen is not None and ' ' in fen:
            for i in (fen.split(' ')[0:4]):
                start += i
            return start
        return fen

    @staticmethod
    def dirty_fen(clean_fen: str):
        out_fen = ''
        board_idx = 0
        for idx, c in enumerate(clean_fen):
            if board_idx == 64:
                out_fen += ' ' + c + ' '
            elif board_idx > 64 and c not in ['K', 'Q', 'k', 'q']:
                if c == '-':
                    out_fen += c + ' ' if out_fen[len(out_fen)-1] == ' ' else ' ' + c + ' '
                else:
                    out_fen += ' ' + clean_fen[idx:len(clean_fen)] + ' '
                    break
            else:
                out_fen += c
            board_idx += int(c) if c.isnumeric() else 1 if c != '/' else 0

        return out_fen + '20 21'


#
# In-memory DNA sequence database.
#
# Future enhancements:
# 1. Since a DNA sequence database can contain hundreds of thousands of sequences (and more), sharding can be introduced
# to improve its performance.
# For examlpe, there could be different shards based on the first base in the sequence (so 4 shards), and the sequence ID
# would start with the base itself to identify which shard it belongs to.
# The "insert", "get" and "overlap" methods would directly access a particular shard based on the sequence or the sequence ID.
# The "find" method would have to search through all shards and combine together the results. But since each shard has
# independent sequences, these "finds" can be performed concurrently and the results merged together when each "find worker"
# has completed their search. This sharding can easily be extended to any number of bases as the shard identifier,
# like having 16 shards based on a shard identifier of 2 bases ("AA", "AC", etc...).
#
# 2. To permit concurrency, the sequence IDs could be UUIDs to ensure that no clashes will occur between different threads
# accessing the database.
#
# 3. The "overlap" method could return the found overlap sequence, and indicate if it corresponds to the prefix or the suffix
# (or both) of the sequence.

from enum import Enum

from dna_utilities import (
    DNA_BASES,
    is_valid_sequence,
    overlap_prefix,
    overlap_suffix
)

from exceptions.invalid_sample_ex import InvalidSample
from exceptions.invalid_sequence_ex import InvalidSequence
from exceptions.invalid_sequence_id_ex import InvalidSequenceId

# Indicates the insertion result. Only two possibilities for now (which could be replaced by True/False),
# but keeping the door open in case other situations are added.
class InsertResult(Enum):
    INSERTED = "Inserted"
    ALREADY_PRESENT = "Already present"

# The in-memory DNA sequence database.
# This database will permit to insert DNA sequences, retrieve these sequences,
# search for sequences containing a sample pattern and verify if a sample pattern
# overlaps a particular sequence in the database.
class SequenceDb:

    def __init__(self):
        # This could establish a connection to an actual database.
        # But for now, just create an empty dictionary that will acts as a database.
        # The sequences that are stored will be associated with an id.
        self.database = {}

        # For now, a simple integer will serve as an ID, but to be scalable
        # to concurrent access, a UUID should be used.
        self.sequence_id = 0

    # Get the next ID t be used for a sequence to be stored.
    def _get_next_id(self):
        self.sequence_id += 1
        return f"{self.sequence_id}"

    # Get the size of the sequence database.
    def __len__(self):
        return len(self.database)

    # Search the database for the presence of an exact sequence.
    #
    # Params:
    # - sequence: the sequence to search for
    # Returns the sequence ID if found, None otherwise
    def _is_present(self, sequence):
        for id in self.database.keys():
            if self.database[id] == sequence:
                return id
        return None

    # Insert a sequence into the database.
    # A sequence will be inserted if it is valid and not already present in the database.
    # A tuple is returned indicating the insertion result (<InsertResult>, <sequence_id>):
    # Params:
    # - sequence: the sequence to insert
    # Returns:
    # - (INSERTED, sequence_id) if the sequence is new, or
    # - (ALREADY_PRESENT, sequence_id) if the sequence was already present.
    # Raises:
    # - InvalidSequence if the sequence is not a valid DNA sequence
    def insert(self, sequence):
        if not is_valid_sequence(sequence):
            raise InvalidSequence(sequence)
        else:
            # Is the sequence already in the database?
            upper_sequence = sequence.upper()
            present_id = self._is_present(upper_sequence)
            if present_id:
                return (InsertResult.ALREADY_PRESENT, present_id)

            # Sequence not already there, insert it.
            sequence_id = self._get_next_id()
            self.database[sequence_id] = upper_sequence
            return (InsertResult.INSERTED, sequence_id)

    # Get the sequence associated with a sequence ID.
    #
    # Params:
    # - sequence_id: the ID of the sequence to retrieve
    # Returns the DNA sequence associated with the sequence ID.
    # Raises:
    # - InvalidSequenceId if the sequence ID is not found in the database.
    def get(self, sequence_id):
        if sequence_id in self.database:
            return self.database[sequence_id]

        raise InvalidSequenceId(sequence_id)

    # Find all sequences in the database that contains a sampel sequence.
    # Params:
    # - sample: the sampel DNA sequence to match
    # Returns a list of sequence IDs for all matching sequences in the database.
    # Raises:
    # - InvalidSample if the sample sequence is not a valid DNA sequence.
    def find(self, sample):
        if not is_valid_sequence(sample):
            raise InvalidSample(sample)
        upper_sample = sample.upper()
        return [id for (id, seq) in self.database.items() if upper_sample in seq]

    # Validate if a sample sequence overlaps a sequence in the database.
    # The sample overlap could be with the sequence's prefix or its suffix (or both).
    # Params:
    # - sample: the sample sequence to validate the overlap
    # - sequence_id: the sequence ID in the database
    # - minimum_overlap: - the minimum lenght of an accepted overlap sequence
    # Returns True is the sample overlaps the sequence prefix or suffix (or both), False otherwise.
    def overlap(self, sample, sequence_id, minimum_overlap=2):
        if not is_valid_sequence(sample):
            raise InvalidSample(sample)
        else:
            upper_sample = sample.upper()
            sequence = self.get(sequence_id)
            prefix_overlap = overlap_prefix(upper_sample, sequence)
            suffix_overlap = overlap_suffix(upper_sample, sequence)
            # Future enhancement: could return the obtained prefix/suffix overlap sequences instead of just True/False.
            return prefix_overlap is not None or suffix_overlap is not None


from datetime import datetime
from flask import render_template, session, redirect, url_for, request, abort, current_app, flash,make_response
from flask_login import current_user
from . import main
from .forms import PostForm, EditProfileForm,EditProfileAdminForm, CommentForm
from .. import db, photos
from ..models import User,Role,Post,Comment
from ..email import send_email
from ..decorators import admin_required, permission_required
from ..models import Permission
from flask_login import login_required

@main.route("/", methods=["GET","POST"])
def index():
    form = PostForm()
    page = request.args.get('page', 1, type=int)
    show_followed = False
    show_yours = False
    if current_user.is_authenticated:
        show_followed = bool(request.cookies.get('show_followed', ''))
        show_yours = bool(request.cookies.get('show_yours', ''))
    if show_followed:
        query = current_user.followed_posts
    elif show_yours:
        query = Post.query.filter_by(author=current_user)
    else:
        query = Post.query
    pagination = query.order_by(Post.timestamp.desc()).paginate(page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'], error_out=False)
    posts = pagination.items
    return render_template('index.html', form=form, posts=posts,show_followed=show_followed, pagination=pagination,show_yours=show_yours)

@main.route('/admin', methods=["GET","POST"])
@login_required
@admin_required
def for_admins_only():
    return "For administrators!"

@main.route('/moderator')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def for_moderators_only():
    return "For comment moderators!"

@main.route("/user/<username>",methods=["GET","POST"])
def user(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        abort(404)
    page = request.args.get("page",1,type=int)
    pagination = user.posts.order_by(Post.timestamp.desc()).paginate(page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],error_out=False)
    posts = pagination.items
    return render_template("user.html", user=user, posts=posts,pagination=pagination)

@main.route("/editProfile",methods=["GET","POST"])
@login_required
def edit_user():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.location = form.location.data
        current_user.about_me = form.about_me.data
        if form.photo.data:
            current_user.image_filename = photos.save(form.photo.data,name="user/" + current_user.username + ".")
            current_user.image_url = photos.url(current_user.image_filename)
        db.session.add(current_user)
        return redirect(url_for("main.user",username=current_user.username))
    form.location.data=current_user.location
    form.about_me.data=current_user.about_me
    return render_template("editProfile.html", form=form, image_url=current_user.image_url)

@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])
@login_required
@admin_required
def edit_profile_admin(id):
    user = User.query.get_or_404(id)
    form = EditProfileAdminForm(user=user)
    if form.validate_on_submit():
        user.email = form.email.data
        user.username = form.username.data
        user.confirmed = form.confirmed.data
        user.role = Role.query.get(form.role.data)
        user.location = form.location.data
        user.about_me = form.about_me.data
        if form.photo.data:
            user.image_filename = photos.save(form.photo.data, name="user/" + user.username+".")
            user.image_url = photos.url(user.image_filename)
        db.session.add(user)
        return redirect(url_for('.user', username=user.username))
    form.email.data = user.email
    form.username.data = user.username
    form.confirmed.data = user.confirmed
    form.role.data = user.role_id
    form.location.data = user.location
    form.about_me.data = user.about_me
    return render_template('editProfile.html', form=form, user=user,image_url=user.image_url)

@main.route("/post/<int:id>",methods=['GET', 'POST'])
def post(id):
    show_more = False
    post = Post.query.get_or_404(id)
    form = CommentForm()
    if current_user.can(Permission.COMMENT) and form.validate_on_submit():
        comment = Comment(body=form.body.data,author=current_user._get_current_object(),post=post)
        db.session.add(comment)
        return redirect(url_for('main.post',id=id))
    page = request.args.get('page', 1, type=int)
    pagination = Comment.query.filter_by(post_id=id,disable=False).order_by(Comment.timestamp.desc()).paginate(page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'], error_out=False)
    comments = pagination.items
    return render_template("post.html", form=form,posts=[post],comments=comments,pagination=pagination,show_more=show_more)

@main.route("/createPost", methods=["GET","POST"])
@login_required
def create_post():
    form = PostForm()
    if current_user.can(Permission.WRITE_ARTICLES) and form.validate_on_submit():
        post = Post(title=form.title.data,body=form.body.data,author=current_user._get_current_object())
        db.session.add(post)
        db.session.commit()
        return redirect(url_for('main.post',id=post.id))
    return render_template("editPost.html",form=form,createPostFlag=True)

@main.route("/editPost/<int:id>", methods=['GET', 'POST'])
@login_required
def edit_post(id):
    post = Post.query.get_or_404(id)
    if current_user != post.author and not current_user.can(Permission.ADMINISTER):
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.body = form.body.data
        post.title = form.title.data
        db.session.add(post)
        return redirect(url_for("main.post",id=post.id))
    form.body.data = post.body
    form.title.data = post.title
    return render_template("editPost.html",form=form)

@main.route('/follow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('main.index'))
    if current_user.is_following(user):
        flash('You are already following this user.')
        return redirect(url_for('main.user', username=username))
    current_user.follow(user)
    return redirect(url_for('main.user', username=username))

@main.route('/unfollow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if not current_user.is_following(user):
        flash('You are not following this user.')
        return redirect(url_for('.user', username=username))
    current_user.unfollow(user)
    return redirect(url_for('.user', username=username))

@main.route('/followers/<username>')
def followers(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followers.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.follower, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followers of",
                           endpoint='.followers', pagination=pagination,
                           follows=follows)

@main.route('/followed-by/<username>')
def followed_by(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followed.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.followed, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followed by",
                           endpoint='.followed_by', pagination=pagination,
                           follows=follows)

@main.route('/all')
@login_required
def show_all():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '', max_age=30*24*60*60)
    resp.set_cookie('show_yours', '', max_age=30*24*60*60)

    return resp

@main.route('/yours')
@login_required
def show_yours():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_yours', '1', max_age=30*24*60*60)
    resp.set_cookie('show_followed', '', max_age=30*24*60*60)
    return resp

@main.route('/followed')
@login_required
def show_followed():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '1', max_age=30*24*60*60)
    resp.set_cookie('show_yours', '', max_age=30*24*60*60)
    return resp

@main.route("/moderate-comments", methods=["GET","POST"])
@permission_required(Permission.MODERATE_COMMENTS)
@login_required
def moderate_comments():
    page = request.args.get('page', 1, type=int)
    pagination = Comment.query.order_by(Comment.timestamp.desc()).paginate(page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'], error_out=False)
    comments = pagination.items
    return render_template('comments.html', pagination=pagination,comments=comments)

@main.route("/disable-comment/<int:id>", methods=["GET","POST"])
@permission_required(Permission.MODERATE_COMMENTS)
@login_required
def disable_comment(id):
    comment = Comment.query.get_or_404(id)
    comment.disable = True
    db.session.add(comment)
    return redirect(url_for("main.moderate_comments", page=request.args.get('page', 1, type=int)))

@main.route("/enable-comment/<int:id>", methods=["GET","POST"])
@permission_required(Permission.MODERATE_COMMENTS)
@login_required
def enable_comment(id):
    comment = Comment.query.get_or_404(id)
    comment.disable = False
    db.session.add(comment)
    return redirect(url_for("main.moderate_comments", page=request.args.get('page', 1, type=int)))


# Необходимо собрать информацию о вакансиях на вводимую должность (используем input или через аргументы) с сайтов
# Superjob и HH. Приложение должно анализировать несколько страниц сайта (также вводим через input или аргументы).
# Получившийся список должен содержать в себе минимум:
# Наименование вакансии.
# Предлагаемую зарплату (отдельно минимальную и максимальную).
# Ссылку на саму вакансию.
# Сайт, откуда собрана вакансия.
# По желанию можно добавить ещё параметры вакансии
# (например, работодателя и расположение).
# Структура должна быть одинаковая для вакансий с обоих сайтов.
# Общий результат можно вывести с помощью dataFrame через pandas.

from bs4 import BeautifulSoup as bs
import requests
# from fake_headers import Headers # падал с ошибкой
# ERROR: Could not find a version that satisfies the requirement Headers (from versions: none)
# Не смогла зарезолвить
import re

def parse_hh(address, vacancy, pages):
    headers = {'User-agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 '
                             'Safari/537.36'}
    html = requests.get(
        address + '/search/vacancy?clusters=true&enable_snippets=true&text=' + vacancy + '&showClusters=true',
        headers=headers).text
    parsed_html = bs(html, 'lxml')

    vacancies = []
    for i in range(pages):
        jobs_block = parsed_html.find('div', {'class': 'vacancy-serp'})
        jobs_list = jobs_block.findChildren(recursive=False)
        for job in jobs_list:
            job_data = {}
            req = job.find('span', {'class': 'g-user-content'})
            if req is not None:
                main_info = req.findChild()
                job_name = main_info.getText()
                job_link = main_info['href']
                salary = job.find('div', {'class': 'vacancy-serp-item__compensation'})
                if not salary:
                    salary_min = None
                    salary_max = None
                else:
                    salary = salary.getText().replace(u'\xa0', u'')
                    salaries = salary.split('-')
                    salaries[0] = re.sub(r'[^0-9]', '', salaries[0])
                    salary_min = int(salaries[0])
                    if len(salaries) > 1:
                        salaries[1] = re.sub(r'[^0-9]', '', salaries[1])
                        salary_max = int(salaries[1])
                    else:
                        salary_max = None
                job_data['name'] = job_name
                job_data['salary_min'] = salary_min
                job_data['salary_max'] = salary_max
                job_data['link'] = job_link
                job_data['site'] = address
                vacancies.append(job_data)

        next_btn_block = parsed_html.find('a', {'class': 'bloko-button'})
        next_btn_link = next_btn_block['href']
        html = requests.get(address + next_btn_link, headers=headers).text
        parsed_html = bs(html, 'lxml')

    print(vacancies)
    return vacancies

parse_hh('https://nn.hh.ru', 'Data Scientist', 2)

def parse_sj(address, vacancy, pages):
    headers = {'User-agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 '
                             'Safari/537.36'}
    base_url = address + '/vacancy/search/?keywords=' + vacancy
    vacancies = []
    session = requests.Session()
    for i in range(pages):
        request = session.get(base_url, headers=headers)
        if request.status_code == 200:
            soup = bs(request.content, 'lxml')
            divs = soup.find_all('div', {'class': '_31XDP iJCa5 f-test-vacancy-item _1fma_ _2nteL'})
            for div in divs:
                common = div.find('div', {'class': '_1h3Zg _2rfUm _2hCDz _21a7u'})
                title = common.text
                href = common.contents[0].attrs['href']
                salary = div.find('span',
                                  {'class': '_1OuF_ _1qw9T f-test-text-company-item-salary'}).text
                salary = salary.replace(u'\xa0', u'')
                if '—' in salary:
                    salary_min = salary.split('—')[0]
                    salary_min = re.sub(r'[^0-9]', '', salary_min)
                    salary_max = salary.split('—')[1]
                    salary_max = re.sub(r'[^0-9]', '', salary_max)
                    salary_min = int(salary_min)
                    salary_max = int(salary_max)
                elif 'от' in salary:
                    salary_min = salary[2:]
                    salary_min = re.sub(r'[^0-9]', '', salary_min)
                    salary_min = int(salary_min)
                    salary_max = None
                elif 'договорённости' in salary:
                    salary_min = None
                    salary_max = None
                elif 'до' in salary:
                    salary_min = None
                    salary_max = salary[2:]
                    salary_max = re.sub(r'[^0-9]', '', salary_max)
                    salary_max = int(salary_max)
                else:
                    salary_min = int(re.sub(r'[^0-9]', '', salary))
                    salary_max = int(re.sub(r'[^0-9]', '', salary))

                vacancies.append({
                    'title': title,
                    'href': 'https://www.superjob.ru' + href,
                    'salary_min': salary_min,
                    'salary_max': salary_max,
                    'link': address
                })
            base_url = address + \
                       soup.find('a', {'class': 'icMQ_ bs_sM _3ze9n _3fOgw f-test-button-dalshe f-test-link-Dalshe'})['href']
        else:
            print('Ошибка')

    print(vacancies)
    return vacancies

parse_sj('https://russia.superjob.ru', 'Python', 2)


# -*- coding=utf-8 -*-
from apscheduler.scheduler import Scheduler
from web.webtoon.model import WEBTOON, WEBTOON_DETAIL, db

sched = Scheduler()

# identify no  refined_detail[i][0]
@sched.interval_scheduler(days=1)
def update_data():
    refind_data, refined_detail = refined_data()
    for identify in refined_detail:
        webtoon = WEBTOON_DETAIL.query.filter(
            WEBTOON_DETAIL.chapter == identify[2],
            WEBTOON_DETAIL.identify_no == identify[1]
        )

        # if not webtoon:
        #     detail = WEBTOON_DETAIL(
        #         identify_no=identify[1],
        #         detail_no=identify[0],
        #         chapter=identify[2],
        #         list_title_url=identify[3],
        #         detail_url=identify[4]
        #     )
        #     db.session.add(detail)
        #     try:
        #         db.session.commit()
        #     except:
        #         db.session.rollback()


from flask import request , make_response
from models.blog_models import blog
import json
import bson.json_util as json_util

def Blog():
    body = json.loads(request.data)
    title = body['title']
    author = body['author']
    content = body['content']
    user_ID = body['user_ID']

    blogs = blog(title = title , author=author , content= content , user_ID= user_ID)

    if not blogs.title and blogs.content:
        return make_response ({"message":"auhtor and content cannot be empty"} , 401)

    saved_blog = blogs.save_blog()
    json_version = json_util.dumps(saved_blog)

    return make_response({'message':"Your blog has been posted successfully" , "blog": json_version} , 201)

def GetAllBLogs():
    blogs = blog.get_blogs()
    json_Version = json_util.dumps(blogs)
    return make_response(json_Version , 201)


import sqlite3
import sqlalchemy.exc
from flask import Blueprint, render_template, request, flash, jsonify, url_for, redirect
from flask_api import status
import flask_sqlalchemy
from flask_login import login_required, current_user
from .models import User, Country, UserCountry
from . import db
import json
from datetime import datetime
import os
from website.questionHandler import *
import traceback

views = Blueprint('views', __name__)

@views.route("/usercountries", methods=["DELETE"])
@login_required
def delete_country():
    country = json.loads(request.data)
    countryCode = country.get('countryCode')
    print(countryCode)
    country = UserCountry.query.get((current_user.id, countryCode))
    print(country)
    if country:
        if country.user_id == current_user.id:
            db.session.delete(country)
            db.session.commit()
            print("Hello")

    return jsonify({})

@views.route('/', methods=['GET', 'POST'])
@login_required
def home():
    print(os.getcwd())
    return render_template("home.html", user=current_user)

@views.route("/travelID", methods=["GET"])
@login_required
def newTravel():
    x = select(UserTravelScore) \
        .where(UserTravelScore.user_id == current_user.id)
    result = db.session.connection().execute(x)
    result = result.fetchall()
    print("THIS IS RESULT", result)

    travelIDs = []
    try:
        for travel in result:
            travelID = travel[1]
            travelIDs.append(travelID)

        newTravelID = max(travelIDs) + 1
    except ValueError:
        # When the user hasn't travelled yet
        newTravelID = 1

    return jsonify(newTravelID)

@views.route("/journeys", methods=["GET"])
@login_required
def journey():
    countries = Country.query.all()
    AllCountries = {}
    for country in countries:
        AllCountries[country.country_code] = country.country_name

    x = select(UserTravelScore)\
            .where(UserTravelScore.user_id == current_user.id)
    result = db.session.connection().execute(x)
    result = result.fetchall()
    travel_sessions = []
    for travel in result:
        num_country_scores = 0
        travelID = travel[1]
        dateAdded = travel[2]
        dateString = dateAdded.strftime("%d/%m/%Y")
        # To find if the current journey is complete, check...
        # if all 197 countries have been added as that is the only guaranteed common factor for all completed travels
        # Cannot use questions answered as number of questions each user gets may be different
        countries_X = select(UserCountryScore)\
            .where(UserCountryScore.user_id == current_user.id, UserCountryScore.travel_id == travelID)
        result_countries = db.session.connection().execute(countries_X)
        result_travels = result_countries.fetchall()
        if result_travels == []:
            # This will run when the user has not completed the questionnaire
            travel_sessions.append((travelID, dateString, {'Status': 'Incomplete'}))
        else:
            for country_score in result_travels:
                num_country_scores += 1
            # If there are 197 records / 197 country scores as there are 197 countries in the database
            # consider this a complete travel
            if num_country_scores == 197:
                # Selects all countries with the user id and travel id matching
                # and orders by descending so the first one is the optimal country
                country_scores_x = select(UserCountryScore)\
                    .where(UserCountryScore.user_id == current_user.id, UserCountryScore.travel_id == travelID)\
                    .order_by(UserCountryScore.total_score.desc())
                result_best_countries = db.session.connection().execute(country_scores_x)
                result_all_countries = result_best_countries.fetchall()
                # As result_all_countries returns a list of tuples
                # And each tuple is in the same structure as the database table UserCountryScore
                # The third column (index 2) is the country code
                # So result_top_country will be the first tuple's country code
                result_top_country = result_all_countries[0][2]

                country_codes_to_filter = []
                for result in result_all_countries:
                    country_codes_to_filter.append(result[2])

                # Uses the function 'filterPrevCountries' to avoid showing the top country which is one they visited
                filtered_country_codes = filterPrevCountries(country_codes_to_filter, travelID)
                top_filtered_country_code = filtered_country_codes[0]
                top_country_name = AllCountries[top_filtered_country_code]

                # print(travelID, top_country_name)

                travel_sessions.append((travelID, dateString, {'Top Country': top_country_name}))
            else:
                # This else will only run if there is at least one country score, otherwise the first if will catch it
                # As the code that adds countries only ever adds all countries, there has likely been a database error
                print("Incomplete - possible server error")
                travel_sessions.append((travelID, dateString, {'Status': 'Incomplete'}))

    return render_template("journeys.html", user=current_user, journeys=travel_sessions)

@views.route("/noTravel", methods=["GET"])
@login_required
def noTravel():
    return render_template("no_travel.html", user=current_user)

@views.route("/questions/<travelID>", methods=["GET"])
@login_required
def questionsPage(travelID):
    return render_template("questions.html", user=current_user, travelID=travelID)

@views.route("/suggestions/<travelID>", methods=["GET"])
@login_required
def suggestions(travelID):
    countries = Country.query.all()
    AllCountries = {}
    for country in countries:
        AllCountries[country.country_code] = country.country_name

    user_travel_details = []
    try:
        ranked_countries = userCountryScore(travelID, AllCountries)
        # map function is used to replace country codes with country names for user convenience
        # x[0] is the country code
        # the first part of the tuple (x[0]) will be replaced with the the country name
        # By the country with country code index of x[0]
        # x[1] is the original second part of the tuple and x[2] is the 3rd part
        ranked_countries_UF = list(map(lambda x: (AllCountries[x[0]], x[1], x[2]), ranked_countries))
        current_travel = UserTravelScore.query.get((current_user.id, travelID))
        num_travellers = current_travel.num_travellers
        travelling_time = current_travel.travelling_time
        user_travel_details.append(travelID)
        user_travel_details.append(num_travellers)
        user_travel_details.append(travelling_time)
        #print("THIS IS ALL COUNTRIES", AllCountries)
        #print("This is rankedCountriesUF", ranked_countries_UF)
        return render_template("suggestions.html",
                               user=current_user,
                               best_countries=ranked_countries_UF,
                               user_travel=user_travel_details)

    except (AttributeError):
        print("ERROR", traceback.format_exc())
        user_travel_details.append(0)
        user_travel_details.append(1)
        return redirect(url_for('views.noTravel'))

@views.route("/userQuestionAnswer", methods=["POST"])
@login_required
def userAnswer():
    userAnswerResponse = json.loads(request.data)
    questionID = userAnswerResponse.get("questionID")
    answerID = userAnswerResponse.get("answerID")
    travelID = userAnswerResponse.get("travelID")

    userQuestionAnswer(questionID, answerID, travelID)

    return jsonify({}), status.HTTP_200_OK

@views.route("/api/questions/", methods=["GET"])
@login_required
def AllQuestions():
    data = getQuestions()

    return jsonify(data)

@views.route("/api/questions/nextQuestion/<travelID>", methods=["GET"])
@login_required
def nextQuestion(travelID):
    questionID = nextQuestionID(travelID)
    question = getQuestion(questionID)

    if question == None:
        return f"A question with questionID: {questionID} was not found", status.HTTP_406_NOT_ACCEPTABLE
    else:
        return question, status.HTTP_200_OK

@views.route("/api/questions/<questionID>", methods=["GET"])
@login_required
def questions(questionID):
    question = getQuestion(questionID)
    if question == None:
        return f"A question with questionID: {questionID} was not found", status.HTTP_404_NOT_FOUND

    else:
        return question, status.HTTP_200_OK

@views.route('/countries', methods=['GET', 'POST'])
@login_required
def countries():
    countries = Country.query.all()
    countriess = []
    for country in countries:
        countriess.append({country.country_code: country.country_name})

    print(countriess)

    return render_template("countries.html", user=current_user, countries=countriess)

@views.route('/usercountries', methods=['GET', 'POST'])
@login_required
def addCountry():
    if request.method == "POST":
        try:
            now = datetime.now()
            country = json.loads(request.data)
            country_code = country.get("countryCode")
            new_user_country = UserCountry(user_id=current_user.id,
                                           country_code=country_code,
                                           date_added=datetime.date(now),
                                           rating=0)
            db.session.add(new_user_country)
            # the commit will confirm that the changes are added together
            # ensuring consistency
            db.session.commit()
            print(country_code)
        except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
            print("Country already Added")
            # return f"This user has already added this country {country_code}", 500
            return f"This user has already added this country {country_code}", status.HTTP_400_BAD_REQUEST

    return render_template("countries.html", user=current_user)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Visualization functions
"""

from itaqa.visualization.defs import get_geometry_from_subplots_amount, get_pollutant_color
from pandas import Timestamp as pd_ts
from plotly.subplots import make_subplots

import plotly.graph_objects as go

def AQS_plot(AQS, pollutant=None):
    """Create a single plot, with all the pollutants on the same graph area"""
    data_lines = get_data_lines(AQS)
    fig = go.Figure(data=data_lines)
    fig.layout.template = 'none'
    fig.update_layout(title=f'{AQS.name}', xaxis_title='Time', plot_bgcolor='rgba(0,0,0,0.04)')
    fig.show()

def AQS_multiplot(AQS):
    """Create multiple subplots (1 for pollutant), automatically choosing the layout"""
    data_lines = get_data_lines(AQS)
    # Determine rows and cols depending on the subplots to visualize
    if len(data_lines) <= 3:
        rows = len(data_lines)
        cols = 1
    else:
        rows, cols = get_geometry_from_subplots_amount(len(data_lines))

    fig = make_subplots(rows=rows,
                        cols=cols,
                        horizontal_spacing=0.05 / cols,
                        vertical_spacing=0.15 / rows,
                        shared_xaxes=True,
                        subplot_titles=([col for col in AQS.data.columns if col != 'Timestamp']))
    for idx, scatter in enumerate(data_lines):
        quo, rem = divmod(idx, rows)
        fig.add_trace(scatter, row=rem + 1, col=quo + 1)
    fig.update_layout(title=f'{AQS.name}',
                      title_font_size=30,
                      title_x=0.5,
                      plot_bgcolor='rgba(0,0,0,0.04)',
                      showlegend=False)

    lockdown_21_feb = pd_ts(year=2020, month=2, day=21)
    lockdown_8_mar = pd_ts(year=2020, month=3, day=8)
    lockdown_21_mar = pd_ts(year=2020, month=3, day=21)

    #for i in range(1, len(data_lines):
    #    import ipdb; ipdb.set_trace()

    #fig.update_layout(shapes=[
    #                            {
    #                                'type': 'line',
    #                                'x0': lockdown_21_feb,
    #                                'y0': 0,
    #                                'x1': lockdown_21_feb,
    #                                'y1': 85,
    #                                'line': {
    #                                    'color': 'black',
    #                                    'width': 2
    #                                }
    #                            }
    #                        ])

    # Tweak: link all x-axes when multiple columns are visualized
    for i in range(1, len(data_lines) + 1):
        fig['layout'][f'xaxis{str(i)}']['matches'] = 'x'
    fig.show()

def get_data_lines(AQS):
    """Obtain scatter lines for plotting from the data stored in the AQS"""
    data_lines = list()
    for pollutant in AQS.data.columns:
        if pollutant != 'Timestamp':
            data_line = go.Scattergl(x=AQS.data['Timestamp'],
                                     y=AQS.data[f'{pollutant}'],
                                     name=f'{pollutant}',
                                     mode='lines+markers',
                                     marker=dict(color='Black', size=2),
                                     line=dict(color=get_pollutant_color(pollutant), width=2))
            data_lines.append(data_line)
    return data_lines


from matplotlib import pyplot as plt
import numpy as np
from colour import Color
from SOM import SOM
import os
import math
import random

fInput = 'input10classes/VisualInputTrainingSet.csv'
N = 1000
lenExample = 2048
NumXClass = 10

def printToFileCSV(prototipi,file):
  """
    print of the prototypes in file.csv
    prototipi: dictionary of the prototypes to print
  """

  f = open(file,'w')

  # stampa su file
  for k in prototipi.keys():
    st = k+','
    for v in prototipi[k]:
      st += str(v)+','
    st = st[0:-1]
    f.write(st+'\n')

  f.close()

def showSom(som,inputs,nameInputs,count,title):
  """
    build of the map with the color associated to the different classes
  """
  print('costruzione mappa '+title)
  mapped = som.map_vects(inputs)
  image_grid = np.zeros(shape=(20,30,3))
  plt.figure(count)
  plt.imshow(image_grid)
  plt.title(title)
  inputClass = nameInputs[0]

  # color generation
  classColor = list()
  ## for 100 classes
  # for i in range(100):
  #   print(i)
  #   c = Color(rgb=(random.random(), random.random(), random.random()))
  #   classColor.append(str(c))
  ## for 10 classes:
  classColor = ['white','red','blue','cyan','yellow','green','gray','brown','orange','magenta']

  iColor = 0

  lenExample = len(inputs[0])
  print(lenExample)

  print(inputClass+' -- '+classColor[iColor])

  for i, m in enumerate(mapped):
    if nameInputs[i] != inputClass:
      inputClass = nameInputs[i]
      iColor = iColor + 1
      print(inputClass+' -- '+classColor[iColor])

    plt.text(m[1], m[0], str('____'), ha='center', va='center', color=classColor[iColor], alpha=0.5,
          bbox=dict(facecolor=classColor[iColor], alpha=0.6, lw=0, boxstyle='round4'))

  ## draw of the prototypes on the map
  # for k in prototipi.keys():
  #     [BMUi, BMUpos] = som.get_BMU(prototipi[k])
  #     plt.text(BMUpos[1], BMUpos[0], str(k), ha='center', va='center',
  #             bbox=dict(facecolor='white', alpha=0.9, lw=0))
  plt.draw()
  plt.show()
  return plt

def classPrototype(inputs,nameInputs):
  #build the prototypes of the different classes
  protClass = dict()
  nameS = list(set(nameInputs))
  temp = np.array(inputs)

  i = 0
  for name in nameS:
    protClass[name] = np.mean(temp[i:i+NumXClass][:],axis=0)
    i = i + NumXClass

  #printToFileCSV(protClass,'prototipi.csv')
  return protClass

if __name__ == '__main__':
  #read the inputs from the file fInput and show the SOM with the BMUs of each input

  inputs = np.zeros(shape=(N,lenExample))
  nameInputs = list()

  # read the inputs
  with open(fInput, 'r') as inp:
      i = 0
      for line in inp:
        if len(line)>2:
          inputs[i] = (np.array(line.split(',')[1:])).astype(np.float)
          nameInputs.append((line.split(',')[0]).split('/')[6])
          i = i+1

  prototipi = classPrototype(inputs,nameInputs)

  #get the 20x30 SOM or train a new one (if the folder does not contain the model)
  som = SOM(20, 30, lenExample, checkpoint_dir= './VisualModel10classes/', n_iterations=20,sigma=4.0)

  loaded = som.restore_trained()
  if not loaded:
    som.train(inputs)

  for k in range(len(nameInputs)):
    nameInputs[k] = nameInputs[k].split('_')[0]

  #shows the SOM
  showSom(som,inputs,nameInputs,1,'Visual map')


from datetime import datetime
from typing import Dict, Generator, List, Optional, Tuple, Union
from sqlalchemy.future import select
from sqlalchemy.sql.expression import delete, update
from sqlalchemy.sql.functions import func
from zavod.logs import get_logger
from followthemoney.types import registry
from nomenklatura.resolver import Resolver
from nomenklatura.statement import Statement

from opensanctions.core.db import stmt_table
from opensanctions.core.db import Conn, upsert_func
from opensanctions.core.dataset import Dataset

log = get_logger(__name__)

def save_statements(conn: Conn, statements: List[Statement]) -> None:
    # unique = {s["id"]: s for s in values}
    # values = list(unique.values())
    if not len(statements):
        return None
    # log.debug("Saving statements", size=len(statements))
    values = [s.to_dict() for s in statements]
    istmt = upsert_func(stmt_table).values(values)
    stmt = istmt.on_conflict_do_update(
        index_elements=["id"],
        set_=dict(
            canonical_id=istmt.excluded.canonical_id,
            schema=istmt.excluded.schema,
            prop_type=istmt.excluded.prop_type,
            target=istmt.excluded.target,
            lang=istmt.excluded.lang,
            original_value=istmt.excluded.original_value,
            last_seen=istmt.excluded.last_seen,
        ),
    )
    conn.execute(stmt)
    return None

def all_statements(
    conn: Conn, dataset: Dataset = None, external: bool = False
) -> Generator[Statement, None, None]:
    q = select(stmt_table)
    if dataset is not None and dataset.name != Dataset.ALL:
        q = q.filter(stmt_table.c.dataset.in_(dataset.scope_names))
    if external is False:
        q = q.filter(stmt_table.c.external == False)
    # q = q.order_by(stmt_table.c.canonical_id.asc())
    conn = conn.execution_options(stream_results=True)
    cursor = conn.execute(q)
    while True:
        rows = cursor.fetchmany(20000)
        if not rows:
            break
        for row in rows:
            yield Statement.from_db_row(row)
    # for row in result.yield_per(20000):
    #     yield Statement.from_db_row(row)

def count_entities(
    conn: Conn,
    dataset: Optional[Dataset] = None,
    target: Optional[bool] = None,
    schemata: Optional[List[str]] = None,
) -> int:
    q = select(func.count(func.distinct(stmt_table.c.canonical_id)))
    q = q.filter(stmt_table.c.prop_type == Statement.BASE)
    q = q.filter(stmt_table.c.external == False)  # noqa
    if target is not None:
        q = q.filter(stmt_table.c.target == target)
    if dataset is not None:
        q = q.filter(stmt_table.c.dataset.in_(dataset.scope_names))
    if schemata is not None and len(schemata):
        q = q.filter(stmt_table.c.schema.in_(schemata))
    return conn.scalar(q)

def entities_datasets(
    conn: Conn, dataset: Optional[Dataset] = None
) -> Generator[Tuple[str, str], None, None]:
    """Return all entity IDs with the dataset they belong to."""
    q = select(stmt_table.c.entity_id, stmt_table.c.dataset)
    q = q.filter(stmt_table.c.prop_type == Statement.BASE)
    if dataset is not None:
        q = q.filter(stmt_table.c.dataset.in_(dataset.scope_names))
    q = q.distinct()
    result = conn.execution_options(stream_results=True).execute(q)
    for row in result:
        entity_id, scope = row
        yield (entity_id, scope)

def cleanup_dataset(conn: Conn, dataset: Dataset):
    # remove non-current statements (in the future we may want to keep them?)
    q = select(func.max(stmt_table.c.last_seen))
    q = q.filter(stmt_table.c.prop == Statement.BASE)
    # q = q.filter(stmt_table.c.external == False)  # noqa
    q = q.filter(stmt_table.c.dataset == dataset.name)
    q = q.group_by(stmt_table.c.dataset)
    cursor = conn.execute(q)
    last_seen = cursor.scalar()
    if last_seen is not None:
        pq = delete(stmt_table)
        pq = pq.where(stmt_table.c.dataset == dataset.name)
        pq = pq.where(stmt_table.c.last_seen < last_seen)
        conn.execute(pq)

def resolve_all_canonical(conn: Conn, resolver: Resolver):
    log.info("Getting all canonical value pairs...", resolver=resolver)
    q = select(stmt_table.c.entity_id, stmt_table.c.canonical_id)
    q = q.distinct()
    # updated = 0
    updates: Dict[str, str] = {}
    cursor = conn.execute(q)
    while True:
        pairs = cursor.fetchmany(5000)
        if not pairs:
            break
        for (entity_id, current_id) in pairs:
            canonical_id = resolver.get_canonical(entity_id)
            if canonical_id != current_id:
                log.info(
                    "Resolve",
                    entity_id=entity_id,
                    canonical_id=canonical_id,
                    current_id=current_id,
                )
                updates[entity_id] = canonical_id

    log.info("Collected updates", count=len(updates))
    for entity_id, canonical_id in updates.items():
        uq = update(stmt_table)
        uq = uq.where(stmt_table.c.entity_id == entity_id)
        uq = uq.where(stmt_table.c.canonical_id != canonical_id)
        uq = uq.values({stmt_table.c.canonical_id: canonical_id})
        conn.execute(uq)

def resolve_canonical(conn: Conn, resolver: Resolver, canonical_id: str):
    referents = resolver.get_referents(canonical_id)
    log.debug(
        "Resolving",
        canonical=canonical_id,
        referents=referents,
        resolver=resolver,
    )
    q = update(stmt_table)
    q = q.where(stmt_table.c.entity_id.in_(referents))
    q = q.values({stmt_table.c.canonical_id: canonical_id})
    conn.execute(q)

def clear_statements(
    conn: Conn,
    dataset: Optional[Dataset] = None,
    external: Optional[bool] = None,
):
    q = delete(stmt_table)
    if external is not None:
        q = q.filter(stmt_table.c.external == external)
    # TODO: should this do collections?
    if dataset is not None:
        q = q.filter(stmt_table.c.dataset == dataset.name)
    conn.execute(q)


import json
import os.path
import pathlib

import falcon

current_directory = os.path.dirname(os.path.abspath(__file__))

def get_files(root_path):
    files = {}
    root_path = pathlib.Path(root_path)
    for entry in root_path.iterdir():
        if entry.is_file():
            files[entry.name] = entry.read_text()
        else:
            files[entry.name] = get_files(entry)
    return files

def get_components_js(map_name, path):
    path = pathlib.Path(path)
    js = map_name +" = {"
    for component_name, component_files in get_files(path).items():
        js += component_name + ": {"
        for file_name, file_contents in component_files.items():
            file_name = pathlib.Path(file_name)
            js += "\"" + file_name.name + "\": "
            if file_name.suffix == ".html":
                js += json.dumps(file_contents)
            else:
                js += file_contents
            js += ","
        js += "},"
    js += "};\n"
    return js

def load_framework_js():
    with open(os.path.join(current_directory, "sandals.js")) as app_file_obj:
        framework_js = app_file_obj.read()
    return framework_js

def load_index_js(app_dir):
    with open(os.path.join(app_dir, "index.js")) as app_file_obj:
        app_js = app_file_obj.read()
    app_js += "\n" + get_components_js("pages", os.path.join(app_dir, "pages")) + "\n"
    app_js += get_components_js("components", os.path.join(app_dir, "components"))
    return app_js

def load_index_html(app_dir):
    with open(os.path.join(app_dir, "index.html")) as file_obj:
        index_html = file_obj.read()
    return index_html

class FrameworkJSResource:
    def __init__(self):
        self.framework_js = load_framework_js()

    def on_get(self, req, resp):
        resp.body = self.framework_js
        resp.content_type = falcon.MEDIA_JS

class IndexJSResource:
    def __init__(self, app_path, reload_disk=False):
        self.app_path = app_path
        self.index_js = load_index_js(self.app_path)
        self.reload_disk = reload_disk

    def on_get(self, req, resp):
        if self.reload_disk:
            self.index_js = load_index_js(self.app_path)
        resp.body = self.index_js
        resp.content_type = falcon.MEDIA_JS

class IndexHTMLResource:
    def __init__(self, app_path, reload_disk=False):
        self.app_path = app_path
        self.index_html = load_index_html(self.app_path)
        self.reload_disk = reload_disk

    def on_get(self, req, resp):
        if self.reload_disk:
            self.index_html = load_index_html(self.app_path)
        resp.body = self.index_html
        resp.content_type = falcon.MEDIA_HTML


import time
import pygame
import numpy

cor_background = (10, 10, 10)
cor_tabuleiro = (40, 40, 40)
cor_cel_morta = (68, 4, 142)
cor_cel_viva = (216, 162, 247)

pygame.init()
pygame.display.set_caption("joguinho da vida")

def update(tela, cells, progresso=False):
    tamanho = 10
    updated_cells = numpy.zeros((cells.shape[0], cells.shape[1]))

    for row, col in numpy.ndindex(cells.shape):
        alive = numpy.sum(cells[row-1:row+2, col-1:col+2]) - cells[row, col]
        color = cor_background if cells[row, col] == 0 else cor_cel_viva

        if cells[row, col] == 1:
            if alive < 2 or alive > 3:
                if progresso:
                    color = cor_cel_morta
            elif 2 <= alive <= 3:
                updated_cells[row, col] = 1
                if progresso:
                    color = cor_cel_viva
        else:
            if alive == 3:
                updated_cells[row, col] = 1
                if progresso:
                    color = cor_cel_viva

        pygame.draw.rect(tela, color, (col * tamanho, row * tamanho, tamanho - 1, tamanho - 1))

    return updated_cells

def main():
    pygame.init()
    tela = pygame.display.set_mode((800, 600))

    cells = numpy.zeros((60, 80))
    tela.fill(cor_tabuleiro)
    update(tela, cells)

    pygame.display.flip()
    pygame.display.update()

    running = False

    while True:
        for saida in pygame.event.get():
            if saida.type == pygame.QUIT:
                pygame.quit()
                return
            elif saida.type == pygame.KEYDOWN:
                if saida.key == pygame.K_SPACE:
                    running = not running
                    update(tela, cells, 10)
                    pygame.display.update()
            if pygame.mouse.get_pressed()[0]:
                pos = pygame.mouse.get_pos()
                cells[pos[1] // 10, pos[0] // 10] = 1
                update(tela, cells)
                pygame.display.update()

        tela.fill(cor_tabuleiro)

        if running:
            cells = update(tela, cells, progresso=True)
            pygame.display.update()

        time.sleep(0.1)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

import json

from ..base.multi_downloader import MultiDownloader

class FastixRu(MultiDownloader):
    __name__ = "FastixRu"
    __type__ = "downloader"
    __version__ = "0.22"
    __status__ = "testing"

    __pattern__ = r"http://(?:www\.)?fastix\.(ru|it)/file/\w{24}"
    __config__ = [
        ("enabled", "bool", "Activated", True),
        ("use_premium", "bool", "Use premium account if available", True),
        ("fallback", "bool", "Fallback to free download if premium fails", False),
        ("chk_filesize", "bool", "Check file size", True),
        ("max_wait", "int", "Reconnect if waiting time is greater than minutes", 10),
        ("revert_failed", "bool", "Revert to standard download if fails", True),
    ]

    __description__ = """Fastix multi-downloader plugin"""
    __license__ = "GPLv3"
    __authors__ = [("Massimo Rosamilia", "max@spiritix.eu")]

    def setup(self):
        self.chunk_limit = 3

    def handle_premium(self, pyfile):
        self.data = self.load(
            "http://fastix.ru/api_v2/",
            get={
                "apikey": self.account.get_data("apikey"),
                "sub": "getdirectlink",
                "link": pyfile.url,
            },
        )
        data = json.loads(self.data)

        self.log_debug("Json data", data)

        if 'error":true' in self.data:
            self.offline()
        else:
            self.link = data["downloadlink"]