# -*- coding: UTF-8 -*-
"""
title: 开密码锁
一个密码锁由 4 个环形拨轮组成，每个拨轮都有 10 个数字： '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' 。每个拨轮可以自由旋转：例如把 '9' 变为 '0'，'0' 变为 '9' 。每次旋转都只能旋转一个拨轮的一位数字。
锁的初始数字为 '0000' ，一个代表四个拨轮的数字的字符串。
列表 deadends 包含了一组死亡数字，一旦拨轮的数字和列表里的任何一个元素相同，这个锁将会被永久锁定，无法再被旋转。
字符串 target 代表可以解锁的数字，请给出解锁需要的最小旋转次数，如果无论如何不能解锁，返回 -1 。

示例 1:
输入：deadends = ["0201","0101","0102","1212","2002"], target = "0202"
输出：6
解释：
可能的移动序列为 "0000" -> "1000" -> "1100" -> "1200" -> "1201" -> "1202" -> "0202"。
注意 "0000" -> "0001" -> "0002" -> "0102" -> "0202" 这样的序列是不能解锁的，因为当拨动到 "0102" 时这个锁就会被锁定。

示例 2:
输入: deadends = ["8888"], target = "0009"
输出：1
解释：
把最后一位反向旋转一次即可 "0000" -> "0009"。

示例 3:
输入: deadends = ["8887","8889","8878","8898","8788","8988","7888","9888"], target = "8888"
输出：-1
解释：
无法旋转到目标数字且不被锁定。

示例 4:
输入: deadends = ["0000"], target = "8888"
输出：-1

提示：
1 <= deadends.length <= 500
deadends[i].length == 4
target.length == 4
target 不在 deadends 之中
target 和 deadends[i] 仅由若干位数字组成
"""
from collections import deque
from typing import List

class Solution:
    def openLock(self, deadends: List[str], target: str) -> int:
        """BFS"""
        start = '0000'
        if start == target:
            return 0
        dead_set = set(deadends)
        if start in dead_set:
            return -1
        queue = deque([(start, 0)])
        visited = {start}
        while queue:
            item, step = queue.popleft()
            for i in range(4):
                ch1 = str((int(item[i]) + 1) % 10)
                # 在Python中，-1 % 10 == 9
                ch2 = str((int(item[i]) - 1) % 10)
                for ch in [ch1, ch2]:
                    next_item = item[:i] + ch + item[i + 1:]
                    if next_item not in dead_set and next_item not in visited:
                        if next_item == target:
                            return step + 1
                        queue.append((next_item, step + 1))
                        visited.add(next_item)
        return -1

    def openLock_2(self, deadends: List[str], target: str) -> int:
        """双向BFS"""
        start = '0000'
        if start == target:
            return 0
        dead_set = set(deadends)
        if start in dead_set:
            return -1
        forward_queue, backward_queue = deque([start]), deque([target])
        forward_visited, backward_visited = {start: 0}, {target: 0}
        while forward_queue:
            if len(forward_queue) > len(backward_queue):
                forward_queue, backward_queue = backward_queue, forward_queue
                forward_visited, backward_visited = backward_visited, forward_visited
            for _ in range(len(forward_queue)):
                item = forward_queue.popleft()
                for i in range(4):
                    ch1 = str((int(item[i]) + 1) % 10)
                    # 在Python中，-1 % 10 == 9
                    ch2 = str((int(item[i]) - 1) % 10)
                    for ch in [ch1, ch2]:
                        next_item = item[:i] + ch + item[i + 1:]
                        if next_item not in dead_set and next_item not in forward_visited:
                            if next_item in backward_visited:
                                # forward_visited、backward_visited 中的层级均从0开始
                                return forward_visited[item] + backward_visited[next_item] + 1
                            forward_queue.append(next_item)
                            forward_visited[next_item] = forward_visited[item] + 1
        return -1

if __name__ == '__main__':
    print(Solution().openLock_2(deadends=["0201", "0101", "0102", "1212", "2002"], target="0202"))


import requests
import pandas as pd
from bs4 import BeautifulSoup

filmdata = []
# Set the website URL
year = ['00','01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22']
for i in year:
    url = (f"https://en.wikipedia.org/wiki/List_of_American_films_of_20{i}")

    # Make a GET request to the website
    response = requests.get(url)

    # Parse the HTML content of the website
    soup = BeautifulSoup(response.content, "html.parser")
    # print(soup)
    movie_titles = []
    # Find all elements with the class "movie-title"
    rows = soup.find_all('tr')
    for row in rows:
        title = row.find('a')
        if title:
            movie_titles.append(title.text)
    print(movie_titles)
    movie_titles = movie_titles[29:]
    movie_titles = movie_titles[:len(movie_titles) - 30]
    print(movie_titles)
    for i in movie_titles:
        filmdata.append(i)

# print(movie_titles)
# Print the list of movie titles
# print(cleaned_titles)

df = pd.DataFrame(filmdata)
df.to_csv("MovieTitles2000-2021_American.csv", index=False)


import requests
import argparse
import sys
from bulk_insert import BulkInserter

def main(args):
    args.n_gt = 1
    args.bulk_size = 5000
    if args.scenario == 'tiny':
        args.n_pt = 10
        args.n_iov = 10
    elif args.scenario == 'tiny-moderate':
        args.n_pt = 10
        args.n_iov = 200
    elif args.scenario == 'moderate':
        args.n_pt = 100
        args.n_iov = 200
    elif args.scenario == 'heavy-usage':
        args.n_pt = 100
        args.n_iov = 5000
    elif args.scenario == 'worst-case':
        args.n_pt = 200
        args.n_iov = 26000
    else:
        sys.exit(f"invalid scenario: '{args.scenario}'. chose from ['tiny', 'tiny-moderate', 'moderate', 'heavy-usage', 'worst-case']")
    bulk_inserter = BulkInserter(args)
    bulk_inserter.prepare()
    bulk_inserter.insert()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--hostname", type=str, default="test111.apps.usatlas.bnl.gov")
    parser.add_argument("--scenario", type=str, default="tiny")
    args = parser.parse_args()
    main(args)


import csv
import glob
import shelve
import copy
import os
import argparse
import tempfile
import sys

from glancesync_image import GlanceSyncImage

"""This module contains all the code that interacts directly with the glance
implementation. It isolates the main code from the glance interaction.

Therefore, this module may be changed if the API is upgraded or it is
invoked in a different way, without affecting the main module.

This is a mock implementation, used for testing.
"""

import logging

class ServersFacade(object):
    images_dir = '/var/lib/glance/images'
    images = dict()
    # Put this property to False to use this file as a mock in a unittest
    # when use_persistence is true, image information is preserved in disk.
    use_persistence = False
    dir_persist = './.glancesync_persist'

    def __init__(self, target):
        self.target = target

    def get_regions(self):
        """It returns the list of regions on the specified target.
        :return: a list of region names.
        """
        all_regions = ServersFacade.images.keys()
        target_name = self.target['target_name']
        regions_list = list()
        for region in all_regions:
            parts = region.split(':')
            if target_name == 'master':
                if len(parts) != 1:
                    continue
                regions_list.append(region)
            else:
                if len(parts) != 2:
                    continue
                if parts[0] != target_name:
                    continue
                regions_list.append(parts[1])
        return regions_list

    def get_imagelist(self, regionobj):
        """return a image list from the glance of the specified region

        :param regionobj: The GlanceSyncRegion object of the region to list
        :return: a list of GlanceSyncImage objects
        """
        # clone the object: otherwise modifying the returned object
        # modify the object in the images.
        return copy.deepcopy(ServersFacade.images[regionobj.fullname].values())

    def update_metadata(self, regionobj, image):
        """ update the metadata of the image in the specified region
        See GlanceSync.update_metadata_image for more details.

        :param regionobj: region where it is the image to update
        :param image: the image with the metadata to update
        :return: this function doesn't return anything.
        """
        images = ServersFacade.images[regionobj.fullname]
        updatedimage = images[image.id]
        updatedimage.is_public = image.is_public
        updatedimage.name = image.name
        # updatedimage.owner = image.owner
        updatedimage.user_properties = dict(image.user_properties)
        if ServersFacade.use_persistence:
            images[image.id] = updatedimage
            images.sync()

    def upload_image(self, regionobj, image):
        """Upload the image to the glance server on the specified region.

        :param regionobj: GlanceSyncRegion object; the region where the image
          will be upload.
        :param image: GlanceSyncImage object; the image to be uploaded.
        :return: The UUID of the new image.
        """
        count = 1
        if regionobj.fullname not in ServersFacade.images:
            ServersFacade.images[regionobj.fullname] = dict()
        imageid = '1$' + image.name
        while imageid in ServersFacade.images[regionobj.fullname]:
            count += 1
            imageid = str(count) + '$' + image.name
        owner = regionobj.target['tenant'] + 'id'
        new_image = GlanceSyncImage(
            image.name, imageid, regionobj.fullname, owner, image.is_public,
            image.checksum, image.size, image.status,
            dict(image.user_properties))

        ServersFacade.images[regionobj.fullname][imageid] = new_image
        if ServersFacade.use_persistence:
            ServersFacade.images[regionobj.fullname].sync()

        return imageid

    def delete_image(self, regionobj, id, confirm=True):
        """delete a image on the specified region.

        Be careful, this action cannot be reverted and for this reason by
        default requires confirmation!

        :param regionobj: the GlanceSyncRegion object
        :param id: the UUID of the image to delete
        :param confirm: ask for confirmation
        :return: true if image was deleted, false if it was canceled by user
        """
        if regionobj.fullname not in ServersFacade.images:
            return False
        images = ServersFacade.images[regionobj.fullname]
        if id not in images:
            return False
        del images[id]

        if ServersFacade.use_persistence:
            ServersFacade.images[regionobj.fullname].sync()
        return True

    def get_tenant_id(self):
        """It returns the tenant id corresponding to the target. It is
        necessary to use the tenant_id instead of the tenant_name because the
        first is used as the owner of the images.

        :return: the tenant id
        """
        if 'tenant_id' in self.target:
            return self.target['tenant_id']
        else:
            return self.target['tenant'] + 'id'

    @staticmethod
    def init_persistence(dir=None, clean=False):
        """Function to start using persistence: load the data from the lass
        session if it exists
        :param dir: path of the directory where the persistence files go.
        Default dir is ./.glancesync_persist
        :param clean: if path exists, discard all existing content
        :return:
        """
        if dir:
            ServersFacade.dir_persist = dir
        ServersFacade.use_persistence = True
        ServersFacade.images = dict()
        if os.path.exists(dir):
            for name in glob.glob(dir + '/_persist_*'):
                if clean:
                    os.unlink(name)
                else:
                    region = os.path.basename(name)[9:]
                    ServersFacade.images[region] = shelve.open(name)
        else:
            os.mkdir(ServersFacade.dir_persist)

    @staticmethod
    def add_image_to_mock(image):
        """Add the image to the mock
        :param image: The image to add. If can be a GlanceSyncImage or a list
        :return: This method does not return nothing.
        """
        if type(image) == list:
            image = GlanceSyncImage.from_field_list(image)
        else:
            image = copy.deepcopy(image)

        if image.region not in ServersFacade.images:
            if ServersFacade.use_persistence:
                ServersFacade.images[image.region] =\
                    shelve.open(ServersFacade.dir_persist + '/_persist_' +
                                image.region)
            else:
                ServersFacade.images[image.region] = dict()
        ServersFacade.images[image.region][image.id] = image
        if ServersFacade.use_persistence:
            ServersFacade.images[image.region].sync()

    @staticmethod
    def add_emptyregion_to_mock(region):
        """Add empty region to mock
        :param image: The image region (e.g. other:Madrid)
        :return: This method does not return nothing.
        """
        if ServersFacade.use_persistence:
            ServersFacade.images[region] = shelve.open(
                ServersFacade.dir_persist + '/_persist_' + region)
        else:
            ServersFacade.images[region] = dict()

    @staticmethod
    def clear_mock():
        """clear all the non-persistent content of the mock"""
        ServersFacade.images = dict()
        # if using persintence, deleting _persist_ file is responsability of
        # the caller.

    @staticmethod
    def add_images_from_csv_to_mock(path):
        """Add images to the mock, reading the csv files saved by the backup
         tool.
        :param path: The directory where the csv files are.
        :return: This method does not return nothing.
        Each file in path has this pattern: backup_<regionname>.csv.
        """
        for file in glob.glob(path + '/*.csv'):
            region_name = os.path.basename(file)[7:-4]
            if region_name not in ServersFacade.images:
                if ServersFacade.use_persistence:
                    ServersFacade.images[region_name] =\
                        shelve.open(ServersFacade.dir_persist + '/_persist_' +
                                    region_name)
                else:
                    ServersFacade.images[region_name] = dict()
            with open(file) as f:
                for row in csv.reader(f):
                    # ignore blank lines
                    if len(row) == 0:
                        continue
                    image = GlanceSyncImage.from_field_list(row)
                    ServersFacade.images[region_name][image.id] = image
            if ServersFacade.use_persistence:
                ServersFacade.images[region_name].sync()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Start a clean persistent session'
    )
    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--path',  default='~/.glancesync_persist/',
                       help='path where the persistent objects are created')
    group.add_argument('--random', action='store_true',
                       help='create a random path')
    parser.add_argument(
        'initial_load',
        help='directory with initial load, with files (backup_<region>.csv)')
    parser.add_argument(
        '--confirm', action='store_true',
        help='If path exists and it is not empty, this option is required')

    meta = parser.parse_args()
    meta.initial_load = os.path.normpath(os.path.expanduser(meta.initial_load))
    if not os.path.exists(meta.initial_load):
        logging.error('The directory "%s" with the initial load must exist' %
                      meta.initial_load)
        sys.exit(-1)

    if meta.random:
        meta.path = tempfile.mkdtemp(prefix='glancesync_tmp')
    else:
        meta.path = os.path.normpath(os.path.expanduser(meta.path))
        m = 'The directory "%s" is not empty. If you are sure, pass --confirm'
        if os.path.exists(meta.path) and not meta.confirm \
                and len(glob.glob(meta.path + '/_persist_*')) != 0:

                logging.error(m % meta.path)
                sys.exit(-1)

    facade = ServersFacade(dict())
    facade.init_persistence(meta.path, True)
    facade.add_images_from_csv_to_mock(meta.initial_load)

    print('export GLANCESYNC_MOCKPERSISTENT_PATH=' + meta.path)


from typing import List

"""
expression = "2*3-4*5
"""

class Solution:
    OP_SET = ("+", "-", "*")

    def diffWaysToCompute(self, expression: str) -> List[int]:
        if expression.isdigit():
            return [int(expression)]

        res = []
        for i, char in enumerate(expression):
            if char in self.OP_SET:
                left = self.diffWaysToCompute(expression[0:i])
                right = self.diffWaysToCompute(expression[i+1:])

                for x in left:
                    for y in right:
                        if char == "+":
                            res.append(x+y)
                        elif char == "-":
                            res.append(x-y)
                        else:
                            res.append(x*y)
        return res

    # def solve(self, expression: List, ans: List):
    #     print("exp", expression)
    #     if len(expression) == 1:
    #         ans.append(int(expression[0]))
    #     for i in range(len(expression)):
    #         if expression[i] in self.OP_SET:
    #             c = self.cal(expression[i-1], expression[i+1], expression[i])
    #             self.solve(expression[:i-1] + [str(c)] + expression[i+2:], ans)

if __name__ == "__main__":
    solution = Solution()
    print(solution.diffWaysToCompute("2*3-4*5"))
    print(solution.diffWaysToCompute("2-1-1"))

241. 为运算表达式设计优先级.py

241. 为运算表达式设计优先级.py


import queue
import sys
import time
import matplotlib.pyplot as plt
from fastqueue.prototypes import *
from fastqueue import *

queue_size = 1000000
queue_structures = {
    "Python queue": (queue.Queue(), queue.Queue.put, queue.Queue.get),
    "Python Simple queue": (
        queue.SimpleQueue(),
        queue.SimpleQueue.put,
        queue.SimpleQueue.get,
    ),
    "Matt's Linked List queue": (LLQueue(), LLQueue.enqueue, LLQueue.dequeue),
    "Matt's Contiguous queue": (ContQueue(), ContQueue.enqueue, ContQueue.dequeue),
    "Matt's C queue": (Queue(), Queue.enqueue, Queue.dequeue),
    "Matt's C Contiguous queue": (QueueC(), QueueC.enqueue, QueueC.dequeue),
}
queue_types = queue_structures.keys()
queue_times = []

for q in queue_types:
    enqueue = queue_structures[q][1]
    dequeue = queue_structures[q][2]
    start = time.time()
    for j in range(queue_size):
        enqueue(queue_structures[q][0], j)
    assert [dequeue(queue_structures[q][0]) for _ in range(queue_size)] == list(
        range(queue_size)
    )
    queue_times.append(time.time() - start)
queue_types, queue_times = zip(
    *sorted(zip(queue_types, queue_times), key=lambda x: x[1], reverse=True)
)
fig = plt.figure(figsize=(15, 7))
plt.bar(queue_types, queue_times, color=["y", "r", "g", "c", "m", "b"], width=0.5)
for i, v in enumerate(queue_times):
    plt.text(i, v, str(round(v, 5)), ha="center", family="sans-serif")
plt.xlabel("Queue Type")
plt.ylabel("Time")
plt.title(f"Queue Analysis : {sys.platform}")
plt.savefig(f"Queue_types_{sys.platform}.png")
print("Queue times Graphed")


"""
Authors: Spotlight Pathology Ltd
License: Apache License V.2
"""

import torch
from train.segmentation.segmentation_model import Unet
from torch.optim import Adam, SGD, lr_scheduler
from warmup_scheduler import GradualWarmupScheduler

DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def setup_training(args, label_ignore):
    """
    Setup model, optimizers and losses for Unet multiclass.
    """
    # setup model
    # ---------------------------------------------------------------------
    model = Unet(
        args.backbone,
        num_classes=args.n_classes,
        in_channels=3,
        model_size=args.tile,
        freeze_encoder=(args.freeze_encoder == 'y')
    )

    # move model to CUDA before constructing optimizers
    # https://pytorch.org/docs/stable/optim.html
    model.to(DEVICE)

    # setup optimizer
    # ---------------------------------------------------------------------
    params = filter(lambda p: p.requires_grad, model.parameters())
    if args.optimizer == 'Adam':
        optimizer = Adam(params, lr=args.lr, weight_decay=args.weight_decay)
    elif args.optimizer == 'SGD':
        optimizer = SGD(
            params,
            lr=args.lr,
            momentum=0.9,
            weight_decay=args.weight_decay
        )
    else:
        raise ValueError('Parser arg optimizer only supports SGD, Adam.')

    # setup loss
    # ---------------------------------------------------------------------
    criterion = torch.nn.CrossEntropyLoss(
        weight=torch.Tensor(args.clw).to(DEVICE),
        ignore_index=label_ignore  # ignored/ does not contribute to gradient
    )

    # setup lr scheduler
    # ---------------------------------------------------------------------
    scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
    if args.warmup == 'y':
        scheduler = GradualWarmupScheduler(
            optimizer, multiplier=1, total_epoch=5, after_scheduler=scheduler
        )

    return model, optimizer, criterion, scheduler


import os
import sys
import pandas as pd
import matplotlib.pyplot as plt

# creaa un piccolo dataframe con i dati con una descrizione e pochi dati
df = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                     'b': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'c': [1, 2, 3, 4, 5, 6, 7, 8, 9],

                        'd': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'e': [1, 2, 3, 4, 5, 6, 7, 8, 9],
                        'f': [1, 2, 3, 4, 5, 6, 7, 8, 9],

                        'g': [1, 2, 3, 4, 5, 6, 7, 8, 9]})

# crea un grafico con i dati del dataframe
df.plot()
plt.show()

df = pd.read_csv('Analisi_trade.csv')

# estraggo il secondo campo di simbolo
df['Scadenza'] = df['Simbolo'].str.split(' ').str[1]

# faccio un ripilogo dei dati per scadenza  e con solo il realizzato totale e lo memorizzo in un df
df2 = df.groupby(['Scadenza'])['Realizzato Totale'].sum().to_frame()

df2.plot()
plt.show()


import streamlit as st
import pandas as pd
import numpy as np
import plotly.express as px
import seaborn as sns
import json

def main():
    st.title("Análise da quantidade de produtos conforme suas caracterizações")

    analysis_options = ["Material", "Número de features", "Categoria"]
    st.sidebar.header("Visualizações")
    analysis = st.sidebar.selectbox("Escolha a característa para contagem:", analysis_options)

    st.header(f"Distribuição da categoria \"{analysis}\"")

    #load dataframe
    data_path = r"/data/products_clean.json"
    with open(data_path) as json_file:
        products = json.load(json_file)
        df = pd.json_normalize(products["products"], max_level=0)
    df["features"] = df["features"][df["features"].notnull()].apply(lambda x: len(x))

    if analysis == "Material":
        count = df["material"].value_counts()

    if analysis == "Número de features":
        count = df["features"].value_counts()

    if analysis == "Categoria":
        count = df["product_category"].value_counts()

    sorted_count = count.sort_values(ascending=False)
    results = sorted_count.iloc[:8].copy()
    try:
        results["Other"] = sorted_count.iloc[8:].sum()
    except KeyError:
        pass

    fig = px.pie(results, values=results.values, names=results.index)
    st.plotly_chart(fig)

if __name__ == "__main__":
    main()


import statistics

with open("in.txt") as f:
	crabs = [int(x) for x in f.readline().rstrip().split(',')]

med = int(statistics.median(crabs))
avg = int(statistics.mean(crabs))

print("part1: fuel spent: {}".format(sum([abs(crabs[i]-med) for i in range(len(crabs))])))

# the sum of 1+2+3+4+..+n is n(n+1)/2
tot = 0
for i in range(len(crabs)):
	tot += abs(crabs[i]-avg)*(abs(crabs[i]-avg)+1)/2

print("part2: fuel spent: {}".format(int(tot)))


import httplib2
import json
import sys
import mysql.connector
import os
from ftplib import FTP
import gzip
from scythepkg.helpers.fastahelper import FastaParser
import os

def specInfo():
    pass
    http = httplib2.Http(".cache")
    server = "http://rest.ensembl.org"
    ext = "/info/species"
    resp, content = http.request(server+ext, method="GET", headers={"Content-Type":"application/json"})
    data = json.loads(content.decode("utf8"))
    return data

def getSequencesFromFTP(outdir, release, specieslist=[]):
    path=outdir+os.sep+"fa"
    print(outdir, release, specieslist)
    if len(release)==1:
        release = release[0]
        dirlist = []
        if not os.path.isdir(path):
                os.makedirs(path)
        if (specieslist==[]):
            print("Warning: No set of species selected")
            exit(0)
        ftp = FTP('ftp.ensembl.org')
        ftp.login()
        ftp.cwd('pub') #ftp://ftp.ensembl.org/pub/
        ftp.retrlines('LIST', callback=dirlist.append)           # list directory contents
        dirlist = [r for r in dirlist if "release-"+str(release)  in r and "fasta" in r]
        print(dirlist)
        if(len(dirlist )==0):
            print("Nothing available for release-"+str(release))
            exit(1)
        ftprelhome = dirlist[0].split(" ")[-1]
        ftp.cwd(ftprelhome)
        ftp.retrlines('LIST', callback=dirlist.append)

        for s in specieslist:
            tmp = [d for d in dirlist if s in d][0]
            spec = tmp.split(" ")[-1]
            ftp.cwd(spec)
            ftp.cwd('pep')
            falist = []
            ftp.retrlines('LIST', callback=falist.append)
            falist = [f for f in falist if "all.fa" in f][0]
            fafile = falist.split(" ")[-1]
            outfaname = spec+".fa.gz"
            outfa = open(outfaname,'wb')
            ftp.retrbinary("RETR "+fafile,outfa.write)
            outfa.close()
            xtract(outfaname, path)
            ftp.cwd("/pub/"+ftprelhome)
       #nucleotides
        for s in specieslist:
            tmp = [d for d in dirlist if s in d][0]
            spec = tmp.split(" ")[-1]
            ftp.cwd(spec)
            ftp.cwd('cds')
            falist = []
            ftp.retrlines('LIST', callback=falist.append)
            falist = [f for f in falist if "all.fa" in f][0]
            fafile = falist.split(" ")[-1]
            outfaname = spec+".cds.all.fa.gz"
            print(outfaname, "cds\n")
            print(falist, "falist\n")
            outfa = open(outfaname,'wb')
            ftp.retrbinary("RETR "+fafile,outfa.write)
            outfa.close()
            if not os.path.exists(path+"_cds"):
                    os.makedirs(path+"_cds")
            xtract(outfaname, path+"_cds")
            ftp.cwd("/pub/"+ftprelhome)
        ftp.quit()

    else:
        print("Releases:")
        for a in zip(release,specieslist):
            print(a)
        for a in zip(release,specieslist):
            getSequencesFromFTP(outdir, release=[a[0]], specieslist=[a[1]])

def xtract(cfile, outpath = "."):

    if cfile.endswith('.gz'):
        dzf = ".".join(cfile.split(".")[:-1])
        #gzip
        gzf = gzip.open(cfile,'rb')
        content = gzf.read()
        out = open(outpath+os.sep+dzf,'wb')
        out.write(content)
        out.close
        gzf.close()
    else:
        print("Can't extract "+cfile)

###########################
class Pep(object):
    def __init__(self,pep,gene,length ):
        self.gene = gene
        self.pep  = pep
        self.length = length
        self.isLongest = None

def prepareLocFromFasta(fasta, outpath, specname):
    genes = {}
    longest = {}
    print(fasta, outpath)
    if not os.path.isdir(outpath):
        os.makedirs(outpath)

    if not outpath:
        out = fasta+".loc"
    else:
        out = outpath+os.sep+specname+".loc"
    fp = FastaParser()
    out = open(out,'w')
    print(out)
    for i in fp.read_fasta(fasta):
        tmp = i[0].split(" ")
        geneid = [t for t in tmp if "gene:" in t]
        geneid = geneid[0].split("gene:")[-1]
        proteinid = tmp[0]
        protlen = len(i[1])
        peptide = Pep(proteinid,geneid,protlen)
        try:
            genes[geneid].append(peptide)
            if peptide.length > longest[geneid].length:
                longest[geneid].isLongest =False
                longest[geneid]=peptide
                peptide.isLongest = True
        except KeyError as e:
            genes[geneid] = [peptide]
            longest[geneid] = peptide
            peptide.isLongest=True
    for g in genes:
        print(g)
        firstcol = [w for w in genes[g] if w.isLongest==True ]
        restcol = [w for w in genes[g] if w.isLongest!=True ]
        s = firstcol[0].gene+"\t"+firstcol[0].pep+"\t"
        s+="\t".join([v.pep for v in restcol])
        s+="\n"
        out.write(s)
        print(s)
    out.close()
    print("done")
#######################


import numpy as np
import sawyer_control.envs.sawyer_reaching as sawyer_reaching

from multiworld.core.multitask_env import MultitaskEnv
from multiworld.core.serializable import Serializable
from gym.spaces import Dict

class SawyerReachXYZEnv(sawyer_reaching.SawyerReachXYZEnv, MultitaskEnv):
    def __init__(self,
                 **kwargs
                 ):
        Serializable.quick_init(self, locals())
        sawyer_reaching.SawyerReachXYZEnv.__init__(self, **kwargs)
        self.observation_space = Dict([
            ('observation', self.observation_space),
            ('desired_goal', self.goal_space),
            ('achieved_goal', self.goal_space),
            ('state_observation', self.observation_space),
            ('state_desired_goal', self.goal_space),
            ('state_achieved_goal', self.goal_space),
        ])

    def step(self, action):
        self._act(action)
        observation = self._get_obs()
        reward = MultitaskEnv.compute_reward(self, action, observation)
        info = self._get_info()
        done = False
        return observation, reward, done, info

    def compute_rewards(self, actions, obs):
        achieved_goals = obs['state_achieved_goal']
        desired_goals = obs['state_desired_goal']
        hand_pos = achieved_goals
        goals = desired_goals
        distances = np.linalg.norm(hand_pos - goals, axis=1)
        if self.reward_type == 'hand_distance':
            r = -distances
        elif self.reward_type == 'hand_success':
            r = -(distances > self.indicator_threshold).astype(float)
        else:
            raise NotImplementedError("Invalid/no reward type.")
        return r

    def _get_obs(self):
        ee_pos = self._get_endeffector_pose()
        state_obs = super()._get_obs()
        return dict(
            observation=state_obs,
            desired_goal=self._state_goal,
            achieved_goal=ee_pos,

            state_observation=state_obs,
            state_desired_goal=self._state_goal,
            state_achieved_goal=ee_pos,
        )

    def reset(self):
        if self.action_mode == "position":
            self._position_act(self.reset_pos - self._get_endeffector_pose(), in_reset=True)
        else:
            self._reset_robot()
        goal = self.sample_goal()
        self._state_goal = goal['state_desired_goal']
        return self._get_obs()

    """
    Multitask functions
    """

    def get_goal(self):
        return {
            'desired_goal': self._state_goal,
            'state_desired_goal': self._state_goal,
        }

    def sample_goal(self):
        return MultitaskEnv.sample_goal(self)

    def sample_goals(self, batch_size):
        goals = super().sample_goals(batch_size)
        return {
            'desired_goal': goals,
            'state_desired_goal': goals,
        }

    def set_to_goal(self, goal):
        goal = goal['state_desired_goal']
        super().set_to_goal(goal)

if __name__=="__main__":
    env = SawyerReachXYZEnv()
    env.reset()


import cv2
from glob import glob

imgs1 = glob('../dataset/GetChu_1-75/*')
imgs2 = glob('../dataset/GetChu_76-118/*')
imgs = imgs1+imgs2
print(len(imgs))

from tqdm import tqdm

def detect_face(imgs):
    face_cascade = cv2.CascadeClassifier('lbpcascade_animeface.xml')
    for img in tqdm(imgs):
        img_temp = cv2.imread(img)
        gray = cv2.cvtColor(img_temp, cv2.COLOR_BGR2GRAY)
        gray = cv2.equalizeHist(gray)
        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(24, 24))
        for i, face in enumerate(faces):
            (stx, sty, w, h) = face
            stx -= (w * 0.3)
            sty -= (h * 0.3)
            w *= 1.5
            h *= 1.45
            stx = max(0, stx)
            sty = max(0, sty)
            name = img.split('/')[-1].split('.')[0] + '_face' + str(i) + '.jpg'
            img_new = img_temp[int(sty):int(sty + h), int(stx):int(stx + w), :]
            cv2.imwrite('../dataset/GetChu_aligned/' + name, img_new)

from multiprocessing import Pool
# 分割path
split_num=8
temp_len=len(imgs)//split_num
base_paths=[]
for i in range(split_num):
    if i != split_num-1:
        base_paths.append(imgs[i*temp_len:(i+1)*temp_len])
    else:
        base_paths.append(imgs[i*temp_len:])

pool=Pool(split_num)
result=[]
for i in base_paths:
    result.append(pool.apply_async(detect_face, kwds={'imgs':i}))
pool.close()
pool.join()


import sys
import pygame
from bullet import Bullet
from alien import Alien
from time import sleep

def check_evens(ship, bullets, settings, screen):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_event(event, ship, settings, screen, bullets)
        elif event.type == pygame.KEYUP:
            check_keyup_event(event, ship)

def check_keydown_event(event, ship, settings, screen, bullets):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = True
    elif event.key == pygame.K_LEFT:
        ship.moving_left = True
    elif event.key == pygame.K_SPACE:
        fire_bullets(ship, bullets, settings, screen)
    elif event.key == pygame.K_q:
        sys.exit()

def fire_bullets(ship, bullets, settings, screen):
    if len(bullets) < settings.bullet_allowed:
        new_bullets = Bullet(settings, screen, ship)
        bullets.add(new_bullets)

def check_keyup_event(event, ship):
    if event.key == pygame.K_RIGHT:
        ship.moving_right = False
    elif event.key == pygame.K_LEFT:
        ship.moving_left = False

def update_screen(ship, bullets, aliens, settings, screen):
    screen.fill(settings.bg_color)
    for bullet in bullets:
        bullet.draw_bullet()

    ship.blitme()
    aliens.draw(screen)
    pygame.display.flip()

def update_bullets(bullets, aliens, ship, settings, screen):
    for bullet in bullets.copy():
        if bullet.rect.bottom <= 0:
            bullets.remove(bullet)
    check_bullet_aliens_collisions(aliens, bullets, ship, settings, screen)

def create_fleet(settings, screen, aliens, ship):
    """Создает флот пришельцев."""
    alien = Alien(settings, screen)
    number_aliens_x = get_number_aliens_x(settings, alien.rect.width)
    number_rows = get_number_rows(settings, alien.rect.height, ship.rect.height)
    for alien_row in range(number_rows):
        for alien_number in range(number_aliens_x):
            create_alien(settings, screen, aliens, alien_number, alien_row)

def get_number_aliens_x(settings, alien_width) -> int:
    """Вычисляет количество пришельцев в ряду."""
    avalible_space = settings.screen_width - alien_width * 2
    return int(avalible_space / (alien_width * 2))

def create_alien(settings, screen, aliens, alien_number, row_number):
    """Создает пришельца и размещает его в ряду."""
    new_alien = Alien(settings, screen)
    alien_width = new_alien.rect.width
    new_alien.x = alien_width + alien_width * 2 * alien_number
    new_alien.rect.x = new_alien.x
    new_alien.rect.y = new_alien.rect.height + new_alien.rect.height * 2 * row_number
    aliens.add(new_alien)

def get_number_rows(settings, alien_height, ship_height) -> int:
    """Определяет количество рядов, помещающихся на экране."""
    avalible_space_y = settings.screen_height - (alien_height * 3) - ship_height
    return int(avalible_space_y / (alien_height * 2))

def update_aliens(settings, stats, aliens, ship, bullets, screen):
    """
    Проверяет, достиг ли флот края экрана,
    после чего обновляет позиции всех пришельцев во флоте.
    """
    check_fleet_edges(settings, aliens)
    aliens.update()

    if pygame.sprite.spritecollideany(ship, aliens):
        ship_hit(stats, aliens, bullets, ship, settings, screen)

def change_fleet_directions(settings, aliens):
    """Опускает весь флот и меняет направление флота."""
    for alien in aliens.sprites():
        alien.rect.y += settings.aliens_fleet_drop_speed
    settings.fleet_direction *= -1

def check_fleet_edges(settings, aliens):
    """Реагирует на достижение пришельцем края экрана."""
    for alien in aliens.sprites():
        if alien.check_edges():
            change_fleet_directions(settings, aliens)
            break

def check_bullet_aliens_collisions(aliens, bullets, ship, settings, screen):
    """Проверка попаданий в пришельцев."""
    collisions = pygame.sprite.groupcollide(bullets, aliens, True, True)
    if len(aliens) == 0:
        bullets.empty()
        create_fleet(settings, screen, aliens, ship)

def ship_hit(stats, aliens, bullets, ship, settings, screen):
    """Обрабатывает столкновение корабля с пришельцем."""
    stats.ship_left -= 1

    aliens.empty()
    bullets.empty()

    create_fleet(settings, screen, aliens, ship)
    ship.center_ship()

    sleep(0.5)


# -*- coding: utf-8 -*-

from datetime import datetime
import os
from pathlib import Path  # Python 3.6+ only
import backtrader as bt
import pandas as pd
from pandas_datareader import data as pdr
import matplotlib.pyplot as plt
from dotenv import load_dotenv

__updated__ = "2021-08-09"

# proxy servers for internet connection
env_path = f"{Path(__file__).parent}/.env"
load_dotenv(dotenv_path=env_path, verbose=True)

# 可能需要proxy
import requests
proxies = {
    'http': os.getenv("httpproxy", "127.0.0.1:8087"),
    'https': os.getenv("socks5", "socks5://127.0.0.1:1081"),
    'socks5': 'socks5://127.0.0.1:1081',
}
# proxies = {'http': '127.0.0.1:8087'}
headers = {"Accept": "application/json",
           'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
           "Accept-Encoding": "none",
           "Accept-Language": "en-US,en;q = 0.8",
           "Connection": "keep-alive",
           "Referer": "https://cssspritegenerator.com",
           "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML,like Gecko) Chrome/23.0.1271.64 Safari/537.11"
           }

session = requests.session()
session.proxies = {}
session.proxies['http'] = 'socks5://localhost:1081'
session.proxies['https'] = 'socks5://localhost:1081'
session.proxies['socks5'] = 'socks5://localhost:1081'

# with requests.Session() as s:
#     s.headers = headers
#     s.proxies.update(proxies)
# data = pdr.DataReader('AAPL', 'yahoo',
#                       start=datetime(2018, 8, 13),
#                       end=datetime(2020, 8, 14), session=session)
data = pdr.get_data_yahoo('AAPL',
                          start=datetime(2018, 8, 13),
                          end=datetime(2020, 8, 14), session=session)

data.columns = ['high', 'low', 'open', 'close', 'volume', 'adj_close']
data['pct'] = data.close.pct_change(1) * 100
data['pct2'] = data.close.pct_change(2) * 100
data['pct3'] = data.close.pct_change(3) * 100

ax = plt.gca()
data.plot(kind='line', y='close', use_index=True, color='blue', ax=ax)
data.plot(kind='line', y='pct', use_index=True, color='red', ax=ax)
data.plot(kind='line', y='pct2', use_index=True, color='green', ax=ax)
data.plot(kind='line', y='pct3', use_index=True, color='orange', ax=ax)
#  plt.show()

class PandasData(bt.feeds.PandasData):
    lines = ('adj_close', 'pct', 'pct2', 'pct3')
    params = (
        ('datetime', None),
        ('open', 'open'),
        ('high', 'high'),
        ('low', 'low'),
        ('close', 'close'),
        ('volume', 'volume'),
        ('openinterest', None),
        ('adj_close', 'adj_close'),
        ('pct', 'pct'),
        ('pct2', 'pct2'),
        ('pct3', 'pct3'),
    )

class MyStrategy(bt.Strategy):
    def __init__(self):
        self.sma = bt.ind.SMA(period=10)

    def next(self):
        if self.sma > self.data.close and self.data.pct > 0 or self.data.pct2 > 0:
            print('BUY CREATE, %.2f' % self.data.close[0])
        elif self.sma < self.data.close and self.data.pct < 0 or self.data.pct2 < 0:
            print('SELL CREATE, %.2f' % self.data.close[0])
            self.order = self.sell()

def runstrat():
    cerebro = bt.Cerebro()
    cerebro.addstrategy(MyStrategy)

    #data = bt.feeds.YahooFinanceData(dataname='MSFT', fromdate=datetime(2011, 1, 1),todate=datetime(2012, 12, 31))
    df = PandasData(dataname=data)
    cerebro.adddata(df)
    startcash = 100000.0
    cerebro.broker.setcash(startcash)

    cerebro.addsizer(bt.sizers.FixedSize, stake=1)

    cerebro.broker.setcommission(commission=0.05)

    cerebro.addanalyzer(bt.analyzers.SharpeRatio, _name='mysharpe')
    cerebro.addanalyzer(bt.analyzers.DrawDown, _namge='DrawDown')

    print('Starting Balance: %.2f' % cerebro.broker.getvalue())

    strats = cerebro.run(stdstats=False)
    strat = strats[0]
    cerebro.addobserver(bt.observers.Value)

    print('Sharpe Ratio:', strat.analyzers.mysharpe.get_analysis())
    print('DrawDown:', strat.analyzers.DrawDown.get_analysis())
    print('Final Balance: %.2f' % cerebro.broker.getvalue())
    cerebro.run()
    print(f"Done.")
    # Get final portfolio Value
    portvalue = cerebro.broker.getvalue()
    pnl = portvalue - startcash

    # Print out the final result
    print('Final Portfolio Value: ${}'.format(portvalue))
    print('P/L: ${}'.format(pnl))

    # Finally plot the end results
    # plt.rcParams['font.sans-serif']=['SimHei']
    plt.rcParams['axes.unicode_minus'] = False
    plt.rcParams['figure.figsize'] = [18, 16]
    plt.rcParams['figure.dpi'] = 300
    plt.rcParams['figure.facecolor'] = 'w'
    plt.rcParams['figure.edgecolor'] = 'k'
    cerebro.plot(style='candlestick')

if __name__ == '__main__':
    runstrat()


import json
import requests
import time

from headers_generation import get_headers

def get_html(url: str) -> str:
    rq = requests.get(url, headers=get_headers())
    print('Getting HTML-code from ', url)
    return rq.text

def is_empty(html: str, vacancy_list_attribute: str) -> bool:
    data_dict = json.loads(html)

    if not data_dict.get(vacancy_list_attribute):
        return True

    return False

def get_offers_links(html, vacancy_list_attribute: str, url_attribute: str):
    links = []

    data_dict = json.loads(html)
    vacancy = data_dict[vacancy_list_attribute]

    for item in vacancy:
        links.append(item[url_attribute])
    return links

class HabrCrawler:
    def __init__(self, query: str = '', areas: list = None, per_page: int = 100, specializations: list = None,
                 time_wait: float = 0.5):
        self.query = query
        self.per_page = per_page
        self.areas = areas
        self.specializations = specializations
        self.root_url = 'https://career.habr.com/api/frontend/vacancies?'
        self.time_wait = time_wait

    def get_vacancies_urls(self, limit: int = None):
        all_links = []
        page = 0

        vacancy_root_url = 'https://career.habr.com'

        while True:
            url = f"{self.root_url}page={page}&per_page={self.per_page}"
            if self.areas:
                for area in self.areas:
                    url += f"&area={area}"
            if self.specializations:
                for specialization in self.specializations:
                    pass
            time.sleep(self.time_wait)
            html = get_html(url)
            if is_empty(html, vacancy_list_attribute='list'):
                break

            all_links += [f"{vacancy_root_url}{x}" for x in get_offers_links(html,
                                                                             vacancy_list_attribute='list',
                                                                             url_attribute='href')]
            page += 1

            if len(all_links) > limit:
                all_links = all_links[:limit]
                break

        return list(set(all_links))

class HHCrawler:
    def __init__(self, query: str = '', areas: list = None, per_page: int = 100, specializations: list = None,
                 time_wait: float = 0.0):
        self.query = query

        if areas is None:
            self.areas = [113]

        self.per_page = per_page

        if specializations is None:
            self.specializations = [160,
                                    10,
                                    12,
                                    150,
                                    25,
                                    165,
                                    34,
                                    36,
                                    73,
                                    155,
                                    96,
                                    164,
                                    104,
                                    157,
                                    107,
                                    112,
                                    113,
                                    148,
                                    114,
                                    116,
                                    121,
                                    124,
                                    125,
                                    126]

            self.root_url = 'https://api.hh.ru/vacancies?'
            self.time_wait = time_wait

    def get_vacancies_urls(self, limit_per_specialization: int = None):
        all_links = []

        for specialization in self.specializations:
            page = 0

            links_to_add = []

            while True:
                url = f"{self.root_url}text={self.query}&page={page}&per_page={self.per_page}" \
                      f"&professional_role={specialization}"
                for area in self.areas:
                    url += f"&area={area}"
                time.sleep(self.time_wait)
                html = get_html(url)
                if is_empty(html, vacancy_list_attribute='items'):
                    break

                links_to_add += get_offers_links(html, vacancy_list_attribute='items', url_attribute='url')
                page += 1

                if len(links_to_add) > limit_per_specialization:
                    links_to_add = links_to_add[:limit_per_specialization]
                    break

            all_links.extend(links_to_add)

        return list(set(all_links))


import pickle as pkl
import os
import cv2
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

with open('./vrd/test.pkl', 'rb') as infile:
	data_test = pkl.load(infile)

with open('./vrd/train.pkl', 'rb') as infile:
	data_train = pkl.load(infile)

with open('objects-dict-2.pkl', 'rb') as infile:
	objects = pkl.load(infile)

l = []
d = dict()
d_classes = dict()
counter = 0
for img in os.listdir("./test_model_images"):
	print (img)
	test_image = cv2.imread('./test_model_images/' + str(img))

	for i in data_test:
		if i!=None:
			if i['img_path'] == '../data/sg_dataset/sg_test_images/' + str(img):
				data = i
				break

	for i in data_train:
		if i!=None:
			if i['img_path'] == '../data/sg_dataset/sg_train_images/' + str(img):
				data = i
				break

	#print (data)
	for val, name in zip(data['boxes'], data['classes']):

		print (val, objects[name])
		cv2.rectangle(test_image, (val[0], val[1]), (val[0]+val[2], val[1]+val[3]), (255, 0, 0), 2)
		x = val[0]
		h = val[1]
		cv2.putText(test_image, objects[name], (x, h), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255))

		l.append(counter)
		d[counter] = {'boxes': val, 'classes': objects[name]}
		d_classes[counter] = str(objects[name])
		counter = counter + 1

	cv2.imshow('im', test_image)
	cv2.imwrite('annotated_image.png', test_image)
	cv2.waitKey(0)
	cv2.destroyAllWindows()

	G = nx.complete_graph(len(l))
	G.add_nodes_from(l)
	nx.set_node_attributes(G, name='label', values=d)
	for d in (G.node.data()):
		print (d)

	print (len(G.edges()))

	print (d_classes)
	print ("Graph created and saved")
	counter = 0

	H = nx.relabel_nodes(G, d_classes, copy = True)
	nx.draw(H, with_labels = True)
	plt.savefig('scene_graph_beginning.png')
	plt.show()


from flask import Flask
from api import make_api

app = Flask('App')

api = make_api()

app.register_blueprint(api, url_prefix='/test')

if __name__ == "__main__":
    app.run('127.0.0.1', '5000', debug=True)


#1. Zadatak
from sklearn import datasets
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn import linear_model
import numpy as np
from sklearn.metrics import mean_squared_error,r2_score
#a. Učitajte California housing skup podataka iz Scikit-learn dataset modula.
housing_x,housing_y=datasets.fetch_california_housing(return_X_y=True, as_frame=True)
#print(housing_x.shape)
#b. Eksplorativnom analizom podataka istražite odnose među varijablama.
def feature_relations(x_values,y_values):
    nrows,ncolumns=2,4
    fig=plt.figure()
    fig.suptitle("Feature relations")
    for i,column_name in enumerate(x_values):
        ax=fig.add_subplot(nrows,ncolumns,i+1)
        ax.scatter(x_values[column_name],y_values)
        ax.set_title(column_name)

    plt.show()

feature_relations(housing_x,housing_y)
#c. Odaberite jednu ulaznu veličinu i podijelite podatke na trening i testni skup u
#odnosu 70%-30%.
#d. Izgradite linearni model.
#e. Vrjednovanjem modela odredite srednju kvadratnu pogrešku i koeficijent
#determinacije
def fir_and_visualize_model(hosuing_X,housing_y):
    housing_y=housing_y.values
    nrows,ncols=2,4
    fig=plt.figure()
    fig.suptitle("Linear regression results")
    for i,feature_name in enumerate(hosuing_X):
        hosuing_X_feature=hosuing_X[feature_name].values
        hosuing_X_feature=np.expand_dims(hosuing_X_feature,axis=1)
        housing_x_train,\
        housing_x_test,\
        housing_y_train,\
        housing_y_test=train_test_split(hosuing_X_feature,housing_y,test_size=0.3,random_state=1)
        lm=linear_model.LinearRegression()
        lm.fit(housing_x_train,housing_y_train)
        housing_y_pred=lm.predict(housing_x_test)
        mse=mean_squared_error(housing_y_test,housing_y_pred)
        r2_result=r2_score(housing_y_test,housing_y_pred)

        ax=fig.add_subplot(nrows,ncols,i+1)
        ax.set_title("{}\nmse: {:.2f}\nr2: {:.2f}".format(feature_name, mse, r2_result))
        ax.scatter(housing_x_test,housing_y_test,color='black',label='real values')
        ax.plot(housing_x_test, housing_y_pred, color="blue", marker="o", linewidth=3, label="predicted values")
    plt.show()

fir_and_visualize_model(housing_x,housing_y)

#f. Eksperimentirajte s različitim ulaznim veličinama. Koja veličina je najprikladnija?


#-*- coding:utf-8 -*-
'''
DOMAIN ADAPTATION OF LINEAR CLASSIFIERS (aka DALC)
See: http://arxiv.org/abs/1506.04573

Kernel class and KernelClassifier class

@author: Pascal Germain -- http://researchers.lille.inria.fr/pgermain/
'''
import numpy as np
from scipy.spatial.distance import cdist
from scipy.sparse import issparse

# kernel functions
def linear_kernel(point_1, point_2):
    return np.dot(point_1, point_2)

def linear_matrix(X1, X2):
    return np.dot(X1, X2.T)

def rbf_kernel(point_1, point_2, gamma):
    diff = point_1 - point_2
    return np.exp(-gamma * np.dot(diff, diff))

def rbf_matrix(X1, X2, gamma):
    return np.exp(-gamma * cdist(X1, X2, 'sqeuclidean') )

def precomputed_matrix(X1, X2):
    return X2.T

class Kernel:
    """
    Wrapper around a kernel function.
    """
    def __init__(self, kernel_str='linear', kernel_func=None, matrix_func=None, **kernel_args):
        """Create a kernel
        kernel_str: kernel name ('linear', 'rbf', 'precomputed', 'custom')
        kernel_func: if kernel_str='custom', function to compute the kernel between two example vectors
        matrix_func: if kernel_str='custom', function to compute the kernel between two example matrices
        kernel_args: extra kernel arguments (e.g., gamma for the 'rbf' kernel)"""
        self.name = kernel_str
        self.args = kernel_args

        if self.name == 'custom':
            self.kernel_func = kernel_func
            self.matrix_func = matrix_func

        elif self.name == 'rbf':
            self.args.setdefault('gamma', 1.0)
            self.kernel_func = rbf_kernel
            self.matrix_func = rbf_matrix

        elif self.name == 'linear':
            self.kernel_func = linear_kernel
            self.matrix_func = linear_matrix

        elif self.name == 'precomputed':
            self.kernel_func = None
            self.matrix_func = precomputed_matrix

        else:
            raise Exception("Unknown kernel.")

    def create_matrix(self, X1, X2=None):
        """Compute the kernel matrix between two example matrices (X1 and X2)
        If X2 is not specified, then X2=X1"""
        if self.matrix_func is not None:
            if X2 is None: X2 = X1
            kernel_matrix = self.matrix_func(X1, X2, **self.args)

        elif self.kernel_func is not None:
            if X2 is not None:
                dim1 = np.size(X1,0)
                dim2 = np.size(X2,0)
                kernel_matrix = np.zeros( (dim1,dim2) )

                for i in range(dim1):
                    for j in range(dim2):
                        kernel_matrix[i,j] = self.kernel_func(X1[i], X2[j], **self.args)
            else:
                dim1 = np.size(X1,0)
                kernel_matrix = np.zeros( (dim1,dim1) )

                for i in range(dim1):
                    for j in range(i+1):
                        value = self.kernel_func(X1[i], X1[j], **self.args)
                        kernel_matrix[i,j] = value
                        kernel_matrix[j,i] = value

        else:
            raise Exception('kernel function unspecified.')

        if issparse(kernel_matrix):
            return kernel_matrix.toarray()
        else:
            return kernel_matrix

class KernelClassifier:
    """
    A linear classifier in the kernel space.
    """
    def __init__(self, kernel, X, alpha_vector=None):
        """Create a kernel classifier.
        kernel: the kernel object
        X: training examples matrix
        alpha_vector: weight vector (optional) """
        self.kernel = kernel
        self.X1 = X
        self.X1_shape = np.shape(X) if X is not None else (0,0)

        if alpha_vector is None:
            self.alpha_vector = np.zeros(self.X1_shape[0])
        else:
            self.alpha_vector = alpha_vector

    def create_matrix(self, X2):
        """Create a kernel matrix between the training matrix and X2"""
        return self.kernel.create_matrix(self.X1, X2)

    def create_self_matrix(self):
        """Create a kernel matrix between the training matrix and itself"""
        return self.kernel.create_matrix(self.X1)

    def predict(self, X=None, kernel_matrix=None):
        """Classifies examples (i.e., return a prediction vector)"""
        if kernel_matrix is None:
            if X is not None:
                kernel_matrix = self.create_matrix(X)
            else:
                raise Exception('Nothing to classify!')

        return np.dot(kernel_matrix.T, self.alpha_vector)

    def calc_risk(self, Y, X=None, kernel_matrix=None, predictions=None):
        """Compute the classification risk on a datatset."""
        if predictions is None:
            predictions = self.predict(X, kernel_matrix)
        return np.mean( predictions * Y <= 0.0 )

    def write_to_file(self, filename):
        """Write the weight vector into a file."""
        self.alpha_vector.tofile(filename, '\n')


### This script will be run from the command line to get cross-session registration
import sys
import os
from pathlib import Path
import click
import json
import subprocess

from helper_fx import Preprocess, setup_logger

@click.command()
@click.option("--config-file", type=str, default="config.json", help="A file containing config options. If not present are parsed correctly the script will exit.")
@click.option("--get-metafile", "-m", type=bool, is_flag=True, help="If selected attempts to download metafile from Azure")
@click.option("--animals", "-a", type=str, default="", help="List of animals to be processed")
@click.option("--dates", "-d", type=str, default="", help="List of dates to be processed")
@click.option("--use-fast-dir", "-f", type=bool, is_flag=True, help="Path to fast directory, important when using with a VM and file share to speed up")
@click.option("--overwrite", type=bool, is_flag=True, help="Choose if you want the option to overwrite files")
@click.option("--delete_intermediates", "-X", type=bool, is_flag=True, help="When selected, will delete raw data and imageJ files")
def run_processing(config_file, get_metafile, animals, dates, use_fast_dir, overwrite, delete_intermediates):
    print("run")

        # finds and opens config file
    print(f"The config file is {config_file}")
    f = open(config_file)
    config_data = json.load(f)

    # Initializes class
    preprocess = Preprocess(config_data, use_fast_dir, overwrite, delete_intermediates)

    # Sets project_dir (and create if does not exist)
    preprocess.set_project_dir()

    # Sets up logger and make directory if needed
    preprocess.set_logger()

    # Gets metafile if requested
    if get_metafile:
        preprocess.get_metafile()

        # Reads metafile in
    preprocess.read_metafile()

    # Parses animals and check to make sure only one is given
    preprocess.parse_animals(animals)
    if len(preprocess.animals) > 1:
        preprocess.logger.info("Exiting as more than one animal given. This script can only process one at a time")
        sys.exit(2)

    # Parses dates and checks to make sure >1 given
    preprocess.parse_dates(dates)
    if len(preprocess.dates) < 2:
        preprocess.logger.info("Exiting as only one date given. Script is designed to look at 2 or more sessions")
        sys.exit(2)

    # Sets up paths and directories
    preprocess.define_root()
    preprocess.define_nwb_paths()
    preprocess.define_multisession_paths() # need to define multiple paths for rawdata and imagej data using dates

# need to decide on file structure
# function to create
# then look for existing registered tiffs

if __name__ == "__main__":
    print("processing stuff")
    run_processing()


import asyncio
import random
from datetime import datetime
from io import BytesIO

import chess
import discord
from cairosvg import svg2png
from chess import pgn, svg
from discord.ext import commands
from discord.ext.commands import BadArgument, MemberConverter

class Games(commands.Cog):
    def __init__(self, bot: commands.Bot):
        self.bot = bot

    @commands.command()
    async def chess(self, ctx: commands.Context, *args: str):
        """
        `!chess` __`Play chess`__

        **Usage:** !chess <user>

        **Examples:**
        `!chess abc#1234` starts a game of chess with abc#1234

        **Note:**
        Moves are in standard algebraic notation (e4, Nxf7, etc).
        """

        try:
            user = await MemberConverter().convert(ctx, " ".join(args))
        except BadArgument:
            return await ctx.send("That user doesn't exist.", delete_after=5)

        board = chess.Board()
        game = pgn.Game()
        node = None
        players = [ctx.author, user]
        random.shuffle(players)
        turn = chess.WHITE
        game.headers["Date"] = datetime.today().strftime("%Y.%m.%d")
        game.headers["White"] = players[chess.WHITE].display_name
        game.headers["Black"] = players[chess.BLACK].display_name

        def render_board(board: chess.Board) -> BytesIO:
            boardimg = chess.svg.board(board=board, lastmove=board.peek() if board.move_stack else None, check=board.king(turn) if board.is_check() or board.is_checkmate() else None, flipped=board.turn == chess.BLACK)
            res = BytesIO()
            svg2png(bytestring=boardimg, write_to=res)
            res.seek(0)
            return res

        while True:
            res = render_board(board)

            if board.outcome() is not None:
                game.headers["Result"] = board.outcome().result()

                if board.is_checkmate():
                    return await ctx.send(f"Checkmate!\n\n{players[not turn].mention} wins.\n{str(game)}", file=discord.File(res, "file.png"))
                else:
                    return await ctx.send(f"Draw!\n{str(game)}", file=discord.File(res, "file.png"))

            game_msg = await ctx.send(f"{players[turn].mention}, its your turn." + ("\n\nCheck!" if board.is_check() else ""), file=discord.File(res, "file.png"))

            try:
                msg = await self.bot.wait_for("message", timeout=600, check=lambda msg: msg.channel == ctx.channel and msg.author.id == players[turn].id)
            except asyncio.TimeoutError:
                return await ctx.send("Timed out.", delete_after=5)

            await msg.delete()
            await game_msg.delete()

            if msg.content == "exit" or msg.content == "resign":
                res = render_board(board)
                game.headers["Result"] = "0-1" if turn else "1-0"
                await game_msg.delete()
                return await ctx.send(f"{players[turn].mention} resigned. {players[not turn].mention} wins.\n{str(game)}", file=discord.File(res, "file.png"))

            try:
                move = board.push_san(msg.content)
            except ValueError:
                continue

            if not node:
                node = game.add_variation(move)
            else:
                node = node.add_variation(move)

            turn = not turn

def setup(bot: commands.Bot) -> None:
    bot.add_cog(Games(bot))


import sys
import datetime, time
from pyspark.sql import functions as F
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

def get_null_perc(spark, df, null_cols):
    """ Get null/empty percentage for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        null_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with null check analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("NullPercentage",StringType(),True)
    ])
    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    df_count = df.count()
    for x in null_cols:
        if x.upper() in (name.upper() for name in df.columns):
            df_null_count = df.select(F.col(x)).filter(F.col(x).isNull() | (F.col(x) == '')).count()
            df_null = spark.createDataFrame([[x, str(df_null_count*100.0/df_count) + '%' ]],schema=schema)
            resultdf = resultdf.union(df_null)
    return resultdf

def get_summary_numeric(df, numeric_cols):
    """ Get Summary for numeric columns

    Args:
        df (DataFrame): dataframe to perform analysis on
        numeric_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with summary analysis
    """
    for x in numeric_cols:
        if x.upper() not in (name.upper() for name in df.columns):
            numeric_cols.remove(x)
    return df.select(numeric_cols).summary()

def get_distinct_counts(spark, df, aggregate_cols):
    """ Get distinct count for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform distinct count analysis on
        aggregate_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with distinct count analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("DistinctCount",StringType(),True)
    ])

    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    for x in aggregate_cols:
        if x.upper() in (name.upper() for name in df.columns):
            df_distinct_count = df.select(F.col(x)).distinct().count()
            df_distinct = spark.createDataFrame([[x, str(df_distinct_count)]],schema=schema)
            resultdf = resultdf.union(df_distinct)

    return resultdf

def get_distribution_counts(spark, df, aggregate_cols):
    """ Get Distribution Counts for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        aggregate_cols (List): list of columns that need to be considered for analysis

    Returns:
        Array: Array of objects with dataframes
    """
    result = []
    for x in aggregate_cols:
        if x.upper() in (name.upper() for name in df.columns):
            result.append(df.groupby(F.col(x)).count().sort(F.col("count").desc()))
    ###

    return result

def get_mismatch_perc(spark, df, data_quality_cols_regex):
    """ Get Mismatch Percentage for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        data_quality_cols_regex (Dictionary): Dictionary of columns/regex-expression for data quality analysis

    Returns:
        DataFrame: DataFrame with data quality analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("MismatchPercentage",StringType(),True)
    ])

    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    df_count = df.count()
    for key, value in data_quality_cols_regex.items():
        if key.upper() in (name.upper() for name in df.columns):
            df_regex_not_like_count = df.select(F.col(key)).filter(~F.col(key).rlike(value)).count()
            df_regex_not_like = spark.createDataFrame([[key, str(df_regex_not_like_count*100.0/df_count) + '%']],schema=schema)
            resultdf = resultdf.union(df_regex_not_like)

    return resultdf


"""add_explorer_field_to_dataset

Revision ID: 16_b65dad1f2a7e
Revises: 15_78103925d379
Create Date: 2021-06-02 14:52:17.931804

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "16_b65dad1f2a7e"
down_revision = "15_78103925d379"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column("dataset", sa.Column("explorer_url", sa.String(), nullable=True))
    op.create_index(op.f("ix_dataset_explorer_url"), "dataset", ["explorer_url"], unique=False)
    op.execute(
        "UPDATE dataset SET explorer_url=deployment_directory.url FROM deployment_directory WHERE "
        "deployment_directory.dataset_id=dataset.id;"
    )  # noqa E501
    op.drop_constraint("deployment_directory_dataset_id_fkey", "deployment_directory")
    op.drop_table("deployment_directory")
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "deployment_directory",
        sa.Column("id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("dataset_id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("url", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column(
            "created_at", postgresql.TIMESTAMP(), server_default=sa.text("now()"), autoincrement=False, nullable=False
        ),
        sa.Column(
            "updated_at", postgresql.TIMESTAMP(), server_default=sa.text("now()"), autoincrement=False, nullable=False
        ),
        sa.ForeignKeyConstraint(["dataset_id"], ["dataset.id"], name="deployment_directory_dataset_id_fkey"),
        sa.PrimaryKeyConstraint("id", name="deployment_directory_pkey"),
    )
    op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
    op.execute(
        "INSERT into deployment_directory (id, dataset_id, url, created_at, updated_at) SELECT uuid_generate_v4(), id, "
        "explorer_url, NOW(), NOW() FROM dataset WHERE dataset.explorer_url is not null;"
    )  # noqa E501
    op.drop_index(op.f("ix_dataset_explorer_url"), table_name="dataset")
    op.drop_column("dataset", "explorer_url")
    # ### end Alembic commands ###


import ast
import difflib
import importlib.util
import re
from collections import Counter, defaultdict
from importlib import import_module
from pathlib import Path
from typing import Dict, List, Optional, Pattern, Tuple

from robot.api import Token

try:
    from robot.api.parsing import Variable
except ImportError:
    from robot.parsing.model.statements import Variable

from robot.variables.search import VariableIterator
from robot.version import VERSION as RF_VERSION

from robocop.exceptions import InvalidExternalCheckerError
from robocop.utils.version_matching import Version
from robocop.version import __version__

ROBOT_VERSION = Version(RF_VERSION)
ROBOT_WITH_LANG = Version("6.0")

def rf_supports_lang():
    return ROBOT_VERSION >= ROBOT_WITH_LANG

def modules_in_current_dir(path, module_name):
    """Yield modules inside `path` parent directory"""
    yield from modules_from_path(Path(path).parent, module_name)

def modules_from_paths(paths):
    for path in paths:
        path_object = Path(path)
        if path_object.exists():
            if path_object.is_dir():
                yield from modules_from_paths(
                    [file for file in path_object.iterdir() if "__pycache__" not in str(file)]
                )
            else:
                spec = importlib.util.spec_from_file_location(path_object.stem, path_object)
                mod = importlib.util.module_from_spec(spec)

                spec.loader.exec_module(mod)
                yield mod
        else:
            # if it's not physical path, try to import from installed modules
            try:
                parent_name, *lib_name = path.rsplit(".", 1)
                if lib_name:
                    parent = __import__(parent_name, fromlist=lib_name)
                    mod = getattr(parent, "".join(lib_name))
                else:
                    mod = __import__(parent_name, None)
                yield mod
            except ImportError:
                raise InvalidExternalCheckerError(path) from None

def modules_from_path(path, module_name=None, relative="."):
    """Traverse current directory and yield python files imported as module"""
    if path.is_file():
        yield import_module(relative + path.stem, module_name)
    elif path.is_dir():
        for file in path.iterdir():
            if "__pycache__" in str(file):
                continue
            if file.suffix == ".py" and file.stem != "__init__":
                yield from modules_from_path(file, module_name, relative)

def normalize_robot_name(name: str, remove_prefix: Optional[str] = None) -> str:
    name = name.replace(" ", "").replace("_", "").lower() if name else ""
    if remove_prefix:
        return name[name.startswith(remove_prefix) and len(remove_prefix) :]
    return name

def normalize_robot_var_name(name: str) -> str:
    return normalize_robot_name(name)[2:-1] if name else ""

def remove_nested_variables(var_name):
    for prefix, match, suffix in VariableIterator(var_name, ignore_errors=True):
        if match:  # if nested variable exists
            # take what surrounds it and run the check again
            var_name = remove_nested_variables(prefix + suffix)
            break
    return var_name.strip()

def keyword_col(node) -> int:
    return token_col(node, Token.KEYWORD)

def token_col(node, *token_type) -> int:
    if ROBOT_VERSION.major == 3:
        for tok_type in token_type:
            token = node.get_token(tok_type)
            if token is not None:
                break
        else:
            return 1
    else:
        token = node.get_token(*token_type)

    if token is None:
        return 1
    return token.col_offset + 1

def issues_to_lsp_diagnostic(issues) -> List[Dict]:
    return [
        {
            "range": {
                "start": {
                    "line": max(0, issue.line - 1),
                    "character": max(0, issue.col - 1),
                },
                "end": {
                    "line": max(0, issue.end_line - 1),
                    "character": max(0, issue.end_col - 1),
                },
            },
            "severity": issue.severity.diag_severity(),
            "code": issue.rule_id,
            "source": "robocop",
            "message": issue.desc,
            "codeDescription": {"href": f"https://robocop.readthedocs.io/en/{__version__}/rules.html#{issue.name}"},
        }
        for issue in issues
    ]

def str2bool(v):
    if isinstance(v, bool):
        return v
    return v.lower() in ("yes", "true", "1")

class AssignmentTypeDetector(ast.NodeVisitor):
    """Visitor for counting number and type of assignments"""

    def __init__(self):
        self.keyword_sign_counter = Counter()
        self.keyword_most_common = None
        self.variables_sign_counter = Counter()
        self.variables_most_common = None

    def visit_File(self, node):  # noqa
        self.generic_visit(node)
        if len(self.keyword_sign_counter) >= 2:
            self.keyword_most_common = self.keyword_sign_counter.most_common(1)[0][0]
        if len(self.variables_sign_counter) >= 2:
            self.variables_most_common = self.variables_sign_counter.most_common(1)[0][0]

    def visit_KeywordCall(self, node):  # noqa
        if node.assign:  # if keyword returns any value
            sign = self.get_assignment_sign(node.assign[-1])
            self.keyword_sign_counter[sign] += 1

    def visit_VariableSection(self, node):  # noqa
        for child in node.body:
            if not isinstance(child, Variable):
                continue
            var_token = child.get_token(Token.VARIABLE)
            sign = self.get_assignment_sign(var_token.value)
            self.variables_sign_counter[sign] += 1
        return node

    @staticmethod
    def get_assignment_sign(token_value):
        return token_value[token_value.find("}") + 1 :]

def parse_assignment_sign_type(value: str) -> str:
    types = {
        "none": "",
        "equal_sign": "=",
        "space_and_equal_sign": " =",
        "autodetect": "autodetect",
    }
    if value not in types:
        raise ValueError(
            f"Expected one of ('none', 'equal_sign', 'space_and_equal_sign', 'autodetect') but got '{value}' instead"
        )
    return types[value]

class RecommendationFinder:
    def find_similar(self, name, candidates):
        norm_name = self.normalize(name)
        norm_cand = self.get_normalized_candidates(candidates)
        matches = []
        for norm in norm_name:
            matches += self.find(norm, norm_cand.keys())
        if not matches:
            return ""
        matches = self.get_original_candidates(matches, norm_cand)
        suggestion = "Did you mean:\n"
        suggestion += "\n".join(f"    {match}" for match in matches)
        return suggestion

    def find(self, name, candidates, max_matches=5):
        """Return a list of close matches to `name` from `candidates`."""
        if not name or not candidates:
            return []
        cutoff = self._calculate_cutoff(name)
        return difflib.get_close_matches(name, candidates, n=max_matches, cutoff=cutoff)

    @staticmethod
    def _calculate_cutoff(string, min_cutoff=0.5, max_cutoff=0.85, step=0.03):
        """The longer the string the bigger required cutoff."""
        cutoff = min_cutoff + len(string) * step
        return min(cutoff, max_cutoff)

    @staticmethod
    def normalize(name):
        """
        Return tuple where first element is string created from sorted words in name,
        and second element is name without `-` and `_`.
        """
        norm = re.split("[-_ ]+", name)
        return " ".join(sorted(norm)), name.replace("-", "").replace("_", "")

    @staticmethod
    def get_original_candidates(candidates, norm_candidates):
        """Map found normalized candidates to unique original candidates."""
        return sorted(list(set(c for cand in candidates for c in norm_candidates[cand])))

    def get_normalized_candidates(self, candidates):
        """
        Thanks for normalizing and sorting we can find cases like this-is, thisis, this-is1 instead of is-this.
        Normalized names form dictionary that point to original names - we're using list because several names can
        have one common normalized name.
        Different normalization methods try to imitate possible mistakes done when typing name - different order,
        missing `-` etc.
        """
        norm = defaultdict(list)
        for cand in candidates:
            for norm_cand in self.normalize(cand):
                norm[norm_cand].append(cand)
        return norm

class TestTemplateFinder(ast.NodeVisitor):
    def __init__(self):
        self.templated = False

    def visit_TestTemplate(self, node):  # noqa
        self.templated = bool(node.value)

def is_suite_templated(model):
    finder = TestTemplateFinder()
    finder.visit(model)
    return finder.templated

def next_char_is(string: str, i: int, char: str) -> bool:
    if not i < len(string) - 1:
        return False
    return string[i + 1] == char

def remove_robot_vars(name: str) -> str:
    var_start = set("$@%&")
    brackets = 0
    open_bracket, close_bracket = "", ""
    replaced = ""
    index = 0
    while index < len(name):
        if brackets:
            if name[index] == open_bracket:
                brackets += 1
            elif name[index] == close_bracket:
                brackets -= 1
            # check if next chars are not ['key']
            if not brackets and next_char_is(name, index, "["):
                brackets += 1
                index += 1
                open_bracket, close_bracket = "[", "]"
        # it looks for $ (or other var starter) and then check if next char is { and previous is not escape \
        elif name[index] in var_start and next_char_is(name, index, "{") and not (index and name[index - 1] == "\\"):
            open_bracket = "{"
            close_bracket = "}"
            brackets += 1
            index += 1
        else:
            replaced += name[index]
        index += 1
    return replaced

def find_robot_vars(name: str) -> List[Tuple[int, int]]:
    """return list of tuples with (start, end) pos of vars in name"""
    var_start = set("$@%&")
    brackets = 0
    index = 0
    start = -1
    variables = []
    while index < len(name):
        if brackets:
            if name[index] == "{":
                brackets += 1
            elif name[index] == "}":
                brackets -= 1
                if not brackets:
                    variables.append((start, index + 1))
        # it looks for $ (or other var starter) and then check if next char is { and previous is not escape \
        elif name[index] in var_start and next_char_is(name, index, "{") and not (index and name[index - 1] == "\\"):
            brackets += 1
            start = index
            index += 1
        index += 1
    return variables

def pattern_type(value: str) -> Pattern:
    try:
        pattern = re.compile(value)
    except re.error as err:
        raise ValueError(f"Invalid regex pattern: {err}")
    return pattern

def get_section_name(node):
    if not node.header:
        #  Lines before first section are considered to be in `*** Comments ***` section even if header name is missing
        return "*** Comments ***"
    for token in node.header.data_tokens:
        if ROBOT_VERSION.major > 3:
            if token.type in node.header.handles_types:
                return token.value
        elif "HEADER" in token.type:
            return token.value
    return ""

def get_errors(node):
    if ROBOT_VERSION.major == 3:
        return [node.error] if node.error else []
    return node.errors


'''
Agent-based model based on War and Reason, BDM & Lalman, 1992.

The model consists of agents characterized by Capabilities and a risk acceptance
coefficient. Agents are connected via a network of alliances, which determines
their utility of challenging one another, as well as who will come to their aid.

'''
import random
import numpy as np
from scipy.stats import kendalltau
import pandas as pd
import networkx as nx

from consul.meta_model import MetaModel

from consul.model_types.utilities.game_tree import Tree, TreeNode
from consul.model_types.extensive_form import ExtensiveGameModel
from consul.model_types.extensive_form import ExtensiveGameAgent

from consul.agent_types.spe_agent import SPEAgent

class WarAgent(ExtensiveGameAgent):
    '''
    Base class.
    '''

    def __init__(self, capabilities, R, *args, **kwargs):
        '''
        New WarAgent
        '''
        self.capabilities = capabilities
        self.risk = (1-(R/3))/(1+(R/3))
        self.p_win = 0
        self.k_ab = 0
        self.total_utility = 0
        super().__init__(*args, **kwargs)

    def start_conflict(self, world, rival):
        '''
        Called at the beginning of a sub-model run to compute the utility and
        percieved probability of winning for the agent.
        '''

        # Convert network
        adj_mat = nx.to_numpy_matrix(world.network)
        # Build position-to-actor dictionaries
        pos_to_name = {}
        name_to_pos = {}
        for i, node in enumerate(world.network.nodes()):
            pos_to_name[i] = node
            name_to_pos[node] = i

        def get_tau(ccode1, ccode2):
            '''
            Closure to find K-Tau between two actors.
            '''
            x = np.array(adj_mat[name_to_pos[ccode1]])[0]
            y = np.array(adj_mat[name_to_pos[ccode2]])[0]
            return kendalltau(x, y)[0]

        # Find K-tau, similarity between alliance portfolios of self and rival
        k_ab = get_tau(self.name, rival.name)

        # Estimate probability of victory
        top = 0
        bottom = 0
        for actor in world.agents:
            k_ka = get_tau(self.name, actor.name)
            k_kb = get_tau(rival.name, actor.name)
            p = (k_ka - k_kb) / 2
            p *= np.exp(self.risk * (k_ka - k_kb))
            contribution = actor.capabilities * p
            if k_ka > k_kb:
                top += contribution
            bottom += abs(contribution)

        p_win = top / bottom

        # Set the variables at the agent level for utility calculations
        self.k_ab = k_ab
        self.p_win = p_win

class WarModel(ExtensiveGameModel):
    '''
    A single, two-player conflict game as described in BDM & Lalman, 1992.

    '''
    tree_file = "international_interaction_game.json"

    def __init__(self, agents):
        super().__init__(agents, tree=Tree.load_tree(self.tree_file))
        agent1, agent2 = agents

        # Calculate utilities
        # TODO: Mirror-imaging
        self.utilities = {node: {agent1.name: 0, agent2.name: 0}
                          for node in self.tree.get_final_nodes()}
        # Agent 1 utilities
        for agent in agents:
            u_sq = 2 - 4 * (0.5**agent.risk)
            ua_da = 2 - 4*((2 - (1 - agent.k_ab))/4)**agent.risk
            ua_db = 2 - 4*((2 - (agent.k_ab - 1))/4)**agent.risk
            u_acq_i = ua_da # This agent wins
            u_acq_j = ua_db # The other agent wins
            u_nego = agent.p_win * ua_da + (1-agent.p_win)*ua_db
            u_cap_j = ua_da - agent.p_win # The other agent surrenders
            u_cap_i = ua_db - (1 - agent.p_win)
            u_war = agent.p_win * (ua_da - agent.p_win - (1-agent.p_win))
            u_war += (1-agent.p_win) * (ua_db - agent.p_win - (1-agent.p_win))

            self.utilities["StatusQuo"][agent.name] = u_sq
            self.utilities["Negotiate_1"][agent.name] = u_nego
            self.utilities["Negotiate_2"][agent.name] = u_nego
            for label in ["War_A1", "War_B1", "War_A2", "War_B2"]:
                self.utilities[label][agent.name] = u_war
            prefix = "A" if agent is agent1 else "B"
            # The other guy winning
            self.utilities["Capitulate_"+prefix+"1"][agent.name] = u_acq_j
            self.utilities["Capitulate_"+prefix+"2"][agent.name] = u_acq_j
            self.utilities["Acquiesce_"+prefix][agent.name] = u_acq_j
            # This agent winning
            prefix = "B" if prefix=="A" else "A"
            self.utilities["Capitulate_"+prefix+"1"][agent.name] = u_acq_i
            self.utilities["Capitulate_"+prefix+"2"][agent.name] = u_acq_i
            self.utilities["Acquiesce_"+prefix][agent.name] = u_acq_i

    @classmethod
    def conflict_from_eugene(cls, eugene_row, agent_class, agent_args={},
                             read_utilities=False):
        '''
        Generate a conflict scenario from a row of EUGene data

        Args:
            eugene_row: A pandas row from EUGene
            agent_class: Class of agent to generate
            agent_args: Dictionary of global arguments to pass to all agents
                        (e.g. learning rate)

        '''

        agents = []
        for i in ['1', '2']:
            j = '2' if i == '1' else '1'
            # Create agents
            name = eugene_row['ccode' + i]
            nmc = eugene_row['cap_' + i]
            risk_aversion = eugene_row['riskT' + i]
            new_agent_args = dict(name=name,
                              capabilities=nmc,
                              R=risk_aversion)
            for arg, val in agent_args.items():
                new_agent_args[arg] = val
            a = agent_class(**new_agent_args)
            a.p_win = eugene_row["wrTp"+i+"win"]
            a.k_ab = eugene_row["tau_glob"]
            agents.append(a)
        model = cls(agents)
        if not read_utilities:
            return model
        # Fill in utilities

        util_keys = {"StatusQuo": "vsq",
                     "Negotiate_1": "neg",
                     "Negotiate_2": "neg",
                     "War_A1": "wr1",
                     "War_A2": "wr1",
                     "War_B1": "wr2",
                     "War_B2": "wr2",
                     "Capitulate_A1": "cp1",
                     "Capitulate_A2": "cp1",
                     "Capitulate_B1": "cp2",
                     "Capitulate_B2": "cp2",
                     "Acquiesce_A": "ac1",
                     "Acquiesce_B": "ac2" }

        for c, i in enumerate(['1', '2']):
            j = '2' if i == '1' else '1'
            name = agents[c].name
            prefix = "wrTu" + i # War and Reason U(...) using Tau
            for node, col in util_keys.items():
                model.utilities[node][name] = eugene_row[prefix+col]
        return model

class WarWorld(MetaModel):
    '''
    Basic MetaModel for creating a simulated world following the BDM & Lalman
    conflict decision methodology.
    '''

    model_class = WarModel
    agent_class = WarAgent
    agent_count = 20
    density = 0.2
    agent_args = {}
    network = nx.Graph()

    def __init__(self, agent_class, agent_count, density, agent_args={},
                 seed=None):
        '''
        Instantiate a new WarWorld model.

        Args:
            ...
        '''
        self.agent_class = agent_class
        self.agent_count = agent_count
        self.agent_args = agent_args
        self.density = density
        super().__init__(self.model_class, agents_per_model=2, seed=seed)
        self.log = []

    def make_agents(self):
        '''
        Create the agents randomly.
        '''
        self.agents = []
        for i in range(self.agent_count):
            nmc = np.random.lognormal(-2.5, 0.2) # Seems to work.
            risk_aversion = np.random.random() * 2 - 1 # Good enough?
            agent_args = dict(name=i,
                              capabilities=nmc,
                              R=risk_aversion)
            for arg, val in self.agent_args.items():
                agent_args[arg] = val
            a = self.agent_class(**agent_args)
            self.agents.append(a)

        # Create the network
        self.network = nx.erdos_renyi_graph(self.agent_count, self.density)

    def step(self):
        '''
        Pick two agents, and have them play the sub-model to completion.
        '''
        pass

class EUGeneWorld(MetaModel):
    '''
    MetaModel that runs a simulation using agents and utilities from the
    EUGene data.
    '''

    model_class = WarModel
    current_row = None

    def __init__(self, agent_class, eugene_data, agent_args={},
                 seed=None, copy_data=False, shuffle_rows=False):
        '''

        '''
        self.agent_class = agent_class
        if not copy_data:
            self.eugene_data = eugene_data
        else:
            self.eugene_data = eugene_data.copy()
        self.agent_args = agent_args
        super().__init__(self.model_class, agents_per_model=2, seed=seed)

    def _shuffle_data(self):
        '''
        Shuffle the order of rows in the EUGene data, preserving year ordering.
        '''
        new_data = []
        for group in self.eugene_data.groupby("year"):
            new_order = list(group[1].index)
            random.shuffle(new_order)
            temp = group[1].ix[new_order]
            new_data.append(temp)
        self.eugene_data = pd.concat(new_data)

    def make_agents(self):
        '''
        Create one agent per CCode appearing in the dataset.
        '''
        self.agents = {}
        all_ccodes = set(list(self.eugene_data.ccode1.unique()) +
                         list(self.eugene_data.ccode2.unique()))
        for ccode in all_ccodes:
            agent_args = dict(name=ccode,
                              capabilities=-9,
                              R=-9)
            for arg, val in self.agent_args.items():
                agent_args[arg] = val
            a = self.agent_class(**agent_args)
            self.agents[ccode] = a

    def step(self, row, return_result=False, training=False):
        '''
        Given a EUGene row, create a war game for it and run to completion.
        '''
        self.current_row = row
        ccode1 = row['ccode1']
        ccode2 = row['ccode2']

        agent1 = self.agents[ccode1]
        agent2 = self.agents[ccode2]

        # Reset best_moves
        agent1.start_conflict(self, agent2)
        agent2.start_conflict(self, agent1)

        model = self.model_class([agent1, agent2])

        util_keys = {"StatusQuo": "vsq",
                     "Negotiate_1": "neg",
                     "Negotiate_2": "neg",
                     "War_A1": "wr1",
                     "War_A2": "wr1",
                     "War_B1": "wr2",
                     "War_B2": "wr2",
                     "Capitulate_A1": "cp1",
                     "Capitulate_A2": "cp1",
                     "Capitulate_B1": "cp2",
                     "Capitulate_B2": "cp2",
                     "Acquiesce_A": "ac1",
                     "Acquiesce_B": "ac2" }

        for c, i in enumerate(['1', '2']):
            name = model.agents[c].name
            prefix = "wrTu" + i # War and Reason U(...) using Tau
            for node, col in util_keys.items():
                model.utilities[node][name] = row[prefix+col]

        if not training:
            model.run()
            if return_result:
                return model.current_node
            else:
                return model
        else:
            node_keys = {"eqTsq": ["StatusQuo"],
                         "eqTnego": ["Negotiate_1", "Negotiate_2"],
                         "eqTacqa": ["Acquiesce_A"],
                         "eqTacqb": ["Acquiesce_B"],
                         "eqTwara": ["War_A1"],
                         "eqTwarb": ["War_B1"]
                         }
            target_node = random.choice(node_keys[row["Equilibrium"]])
            for agent in model.agents:
                agent.train_to_outcome(model, target_node)

    def run(self, report_every=100000, model_assessors={}):
        '''
        Run a step for each row of the data.

        Args:
            output_each: Print current step once every this-many steps
                        (TODO: or TimeDelta)
            model_assessments: DataCollector-style names and functions to run
                               on each model object.
        '''

        model_assessments = {name: [] for name in model_assessors}
        count = 0
        for i, row in self.eugene_data.iterrows():
            if report_every is not None and count % report_every == 0:
                print(count)
            m = self.step(row, return_result=False)
            for name, assessor in model_assessors.items():
                r = assessor(m)
                model_assessments[name].append(r)
            count += 1
        for name, data in model_assessments.items():
            self.eugene_data[name] = data


import time

import logging
import os
print(os.getcwd())
logging.basicConfig(filename='example.log',level=logging.DEBUG)
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')

i = 0
while True:
	print(i)
	logging.info(i)
	i+=1
	print("This prints once 1s.")
	time.sleep(1)