import functools
import json
import logging
from tornado.web import MissingArgumentError
from torndsession.sessionhandler import SessionBaseHandler
from models.dao import Dao
from models.admin import FFAdmin

__author__ = 'Siglud'

class BaseHandler(SessionBaseHandler):
    def data_received(self, chunk):
        pass

    def initialize(self):
        """初始化数据库连接"""
        self.__angular_body_argument = None
        Dao.create_mysql_session()

    def on_finish(self):
        """关闭数据库连接"""
        Dao.close_mysql_session()

    def get_current_user(self):
        uid = self.session.get('uid', '')
        return FFAdmin(user_id=uid)

    def prepare(self):
        """请求前的操作，检查是否有登录"""
        if not self.get_current_user().exists and self.request.uri != '/login/':
            self.redirect('/login/')
            return

    def get_angular_argument(self, name):
        """获取angular传过来的参数"""
        if not self.request.body:
            raise MissingArgumentError(name)
        if not self.__angular_body_argument:
            try:
                self.__angular_body_argument = json.loads(self.request.body.decode('utf-8'))
            except (ValueError, TypeError):
                raise MissingArgumentError(name)
        return self.__angular_body_argument.get(name)

    def on_success(self, data=None):
        """正确的返回"""
        self.write(dict(
            ret=0,
            msg='ok',
            data=data if data else ''
        ))

    def on_error(self, code, msg):
        """错误的返回"""
        self.write(dict(
            ret=code,
            msg=msg,
        ))

    def logger(self):
        return logging.getLogger()


# encoding: utf-8

import tushare as ts
import pickle

# 获取股票列表
stock_basics = ts.get_stock_basics()

# 循环获取股票历史行情
stock_hist_data = {}
start_dt = '2015-01-01'
end_dt = '2018-06-30'

counter = 0
for code in stock_basics.index.tolist():
    counter += 1
    print(counter)
    stock_hist_data[code] = ts.get_hist_data(code, start=start_dt, end=end_dt)

with open('./stock_hist_data.pkl', 'wb') as fw:
    pickle.dump(stock_hist_data, file=fw)

with open('./stock_basics.pkl', 'wb') as fw:
    pickle.dump(stock_basics, file=fw)

# file_to_read = open('./stock_hist_data.pkl', 'rb')
# res = pickle.load(file_to_read)


# coding: utf-8

# # Preprocessing

# In[279]:

from string import punctuation
import numpy as np
def clean_text(data_list):
    lines = list()
    for i in range(2000):
        line = str(data_list[i]).split()
        table = str.maketrans('', '', punctuation)
        line = [w.translate(table) for w in line]
        line  = [word for word in line if word.isalpha() and len(word) > 1]
        lines.append(' '.join(line))
    return lines

# In[280]:

import sklearn.datasets

# load text files and process
docs_to_train = sklearn.datasets.load_files("./txt_sentoken",shuffle = False)
data = clean_text(docs_to_train.data)

# In[281]:

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split

# select words with min occurance >=5
# and does not appear in more than 80% of the files
# filter stop words
vectorizer = CountVectorizer(min_df = 5,max_df= 0.8,stop_words='english')
X = vectorizer.fit_transform(data)
y = docs_to_train.target

# train test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42,shuffle = True)

# # Uncertainty Sampling

# In[272]:

from sklearn.metrics import accuracy_score
from sklearn.svm import LinearSVC
K = 10
T = X_train.todense()[0:100]
T_y = y_train[0:100]
T_prime = np.delete(X_train.todense(),list(range(100)),axis = 0)
T_prime_y = y_train[100:1600]
T_len = len(T_prime_y)
test_errors = []
train_size = []

while T_len >= 0:
    clf = LinearSVC()
    clf.fit(T,T_y)
    if (len(T_prime_y) > K):
        # selecting examples closest to the hyperplane
        idx = np.argpartition(np.abs(clf.decision_function(T_prime)), K)[0:K]
    else:
        idx = list(range(K))

    y_pred = clf.predict(X_test.todense())
    error = 1- accuracy_score(y_test, y_pred)
    test_errors.append(error)
    train_size.append(len(T))

    if len(T_prime_y) > 0:
        for i in idx:
            T = np.vstack([T,T_prime[i]])
            T_y = np.append(T_y, T_prime_y[i])
        T_prime = np.delete(T_prime,[i for i in idx],axis = 0)
        T_prime_y = np.delete(T_prime_y,[i for i in idx])
    T_len -= K

# In[271]:

import matplotlib.pyplot as plt
fig, ax = plt.subplots()
ax.plot(train_size,test_errors)

ax.set(xlabel='training set size', ylabel='test error',
        title='Uncertainty Sampling')

# helper function to display point with lowest y value
def annot_min(x,y, ax=None):
    xmin = x[np.argmin(y)]
    ymin = min(y)
    text= "x={:.3f}, y={:.3f}".format(xmin, ymin)
    if not ax:
        ax=plt.gca()
        bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
        arrowprops=dict(arrowstyle="->",connectionstyle="angle,angleA=0,angleB=60")
        kw = dict(xycoords='data',textcoords="axes fraction",
                  arrowprops=arrowprops, bbox=bbox_props, ha="right", va="top")
        ax.annotate(text, xy=(xmin, ymin), xytext=(0.94,0.96), **kw)

annot_min(train_size,test_errors)

plt.show()

# # Query by Committee

# In[ ]:

from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import tree

K= 10
T = X_train.todense()[0:100]
T_y = y_train[0:100]
T_prime = np.delete(X_train.todense(),list(range(100)),axis = 0)
T_prime_y = y_train[100:1600]
T_len = len(T_prime_y)
test_errors = []
train_size = []
while T_len >= 0:
    knn_clf = KNeighborsClassifier(n_neighbors = 1)
    knn_clf.fit(T,T_y)
    nb_clf = MultinomialNB()
    nb_clf.fit(T,T_y)
    tree_clf = tree.DecisionTreeClassifier(max_depth = 30)
    tree_clf.fit(T,T_y)

    knn_pred = knn_clf.predict(X_test.todense())
    nb_pred = nb_clf.predict(X_test.todense())
    tree_pred = tree_clf.predict(X_test.todense())

    train_size.append(len(T))
    print(len(T))

    majority_pred = []
    idx = []

    for i in range(len(y_test)):
        votes = np.array([knn_pred[i],nb_pred[i],tree_pred[i]])
        majority_pred.append(np.argmax(np.bincount(votes)))

        #vote entropy
        if len(idx) < K:
            if np.count_nonzero(votes == 0) == 2 or np.count_nonzero(votes == 1) == 2:
                idx.append(i)

    error = 1- accuracy_score(y_test, majority_pred)
    print(error)
    test_errors.append(error)
    if len(T_prime_y) > K:
        for i in idx:
            T = np.vstack([T,T_prime[i]])
            T_y = np.append(T_y, T_prime_y[i])
        T_prime = np.delete(T_prime,[i for i in idx],axis = 0)
        T_prime_y = np.delete(T_prime_y,[i for i in idx])

        while len(idx) < K:
            T = np.vstack([T,T_prime[0]])
            T_y = np.append(T_y, T_prime_y[0])
            T_prime = np.delete(T_prime,[0],axis = 0)
            T_prime_y = np.delete(T_prime_y,[0])
            idx.append(0)

    T_len -= K

# perfrom last QBC operation on the entire training dataset
knn_clf = KNeighborsClassifier(n_neighbors = 1)
knn_clf.fit(X_train.todense(),y_train)
nb_clf = MultinomialNB()
nb_clf.fit(X_train.todense(),y_train)
tree_clf = tree.DecisionTreeClassifier(max_depth = 30)
tree_clf.fit(X_train.todense(),y_train)

knn_pred = knn_clf.predict(X_test.todense())
nb_pred = nb_clf.predict(X_test.todense())
tree_pred = tree_clf.predict(X_test.todense())

train_size.append(len(X_train.todense()))
print(len(X_train.todense()))

majority_pred = []

for i in range(len(y_test)):
    votes = np.array([knn_pred[i],nb_pred[i],tree_pred[i]])
    majority_pred.append(np.argmax(np.bincount(votes)))

error = 1- accuracy_score(y_test, majority_pred)
test_errors.append(error)

# In[324]:

fig, ax = plt.subplots()
ax.plot(train_size,test_errors)

ax.set(xlabel='training set size', ylabel='test error',
        title='QBC')
annot_min(train_size,test_errors)

plt.show()


import os
import json
from copy import deepcopy
from mechanics import get_damage
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import pickle

VERSION = 3
_TYPE = 0

def main(config, name, ret_dist=False, sim_size=250, dur_dist=None):
    config["sim_size"] = sim_size
    values = get_damage(config, ret_dist=ret_dist, dur_dist=dur_dist)
    return values

def load_config(cfn):
    config_file = os.path.join("../config/", cfn)
    with open(config_file, 'rt') as fid:
        config = json.load(fid)

    return config

def regress(encounter):
    config = load_config(encounter + ".json")

    sim_size = 10000
    values = main(config, encounter, sim_size=sim_size)
    print(encounter, values)

if __name__ == '__main__':
    for rdx in range(4):
       regress(f"regression{rdx + 1:d}")


# Local import
import dbus
from dbusitem import Dbusitem
import tracing

class DbusDevice(object):
	## The constructor processes the tree of dbus-items.
	# @param bus Session/System bus object
	# @param name the dbus-service-name.
	def __init__(self, bus, name, eventCallback):
		self._dbus_name = name
		self._dbus_conn = bus
		self._items = []
		self._eventCallback = eventCallback
		self._getChildren(bus, name)
		self._service_id = self._dbus_conn.get_name_owner(name)

	def __del__(self):
		tracing.log.debug('__del__ %s' % self)
		self._dbus_name = None
		self._value = None
		self._eventCallback = None

	def _getChildren(self, bus, service):
		data = self._dbus_conn.call_blocking(service, '/', None, 'GetValue', '', [])
		for child in data:
			name = "/" + child
			self._items.append(name)
		self._dbus_conn.add_signal_receiver(self._on_dbus_value_changed,
			dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path_keyword='path',
			sender_keyword='service_id')

	def _on_dbus_value_changed(self, changes, path=None, service_id=None):
		if service_id != None and service_id == self._service_id:
			self._eventCallback(self._dbus_name, path, changes)

	## Returns the dbus-service-name which represents the Victron-device.
	def __str__(self):
		return "DbusDevice=%s" % self._dbus_name

	def getBusName(self):
		return self._dbus_name

	def getValues(self):
		values = {}
		for i in self._items:
			properties = {}
			properties['Value'] = self._dbus_conn.call_blocking(self._dbus_name, i, None, 'GetValue', '', [])
			properties['Valid'] = bool(properties['Value'] != dbus.Array([]))
			properties['Text'] = str(self._dbus_conn.call_blocking(self._dbus_name, i, None, 'GetText', '', []))
			values[i] = properties
		return values

# convert to python type.
# pickle doesn't handle dbus-types.
def dbusTypeToPythonType(obj_path, dbusValue):
	pythonValue = dbusValue
	dbusType = type(dbusValue).__name__
	if dbusType == dbus.UInt16 or dbusType == dbus.UInt32 or dbusType == dbus.UInt64:
		pythonValue = int(dbusValue)
	elif dbusType == dbus.Byte or dbusType == dbus.Int16 or dbusType == dbus.Int32 or dbusType == dbus.Int64:
		pythonValue = int(dbusValue)
	elif dbusType == 'Double':
		pythonValue == float(dbusValue)
	elif dbusType == dbus.String:
		pythonValue == str(dbusValue)
	if pythonValue == -1:
		if obj_path == '/Ac/ActiveIn/L1/V':
			tracing.log.info("unknown %s %s %s" % (obj_path, dbusType, dbusValue))
	return pythonValue


# coding: utf-8

# public items
__all__ = [
    "chunk",
    "xarrayfunc",
]

# standard library
from concurrent.futures import ProcessPoolExecutor as Pool
from functools import wraps
from inspect import Parameter, signature, stack
from multiprocessing import cpu_count
from sys import _getframe as getframe

# dependent packages
import fmflow as fm
import numpy as np
import xarray as xr

# module constants
DEFAULT_N_CHUNKS = 1
try:
    MAX_WORKERS = cpu_count() - 1
except:
    MAX_WORKERS = 1

# decorators
def xarrayfunc(func):
    """Make a function compatible with xarray.DataArray.

    This function is intended to be used as a decorator like::

        >>> @fm.xarrayfunc
        >>> def func(array):
        ...     # do something
        ...     return newarray
        >>>
        >>> result = func(array)

    Args:
        func (function): A function to be wrapped. The first argument
            of the function must be an array to be processed.

    Returns:
        wrapper (function): A wrapped function.

    """

    @wraps(func)
    def wrapper(*args, **kwargs):
        if any(isinstance(arg, xr.DataArray) for arg in args):
            newargs = []
            for arg in args:
                if isinstance(arg, xr.DataArray):
                    newargs.append(arg.values)
                else:
                    newargs.append(arg)

            return fm.full_like(args[0], func(*newargs, **kwargs))
        else:
            return func(*args, **kwargs)

    return wrapper

def chunk(*argnames, concatfunc=None):
    """Make a function compatible with multicore chunk processing.

    This function is intended to be used as a decorator like::

        >>> @fm.chunk('array')
        >>> def func(array):
        ...     # do something
        ...     return newarray
        >>>
        >>> result = func(array, timechunk=10)

    or you can set a global chunk parameter outside the function::

        >>> timechunk = 10
        >>> result = func(array)

    """

    def _chunk(func):
        depth = [s.function for s in stack()].index("<module>")
        f_globals = getframe(depth).f_globals

        # original (unwrapped) function
        orgname = "_original_" + func.__name__
        orgfunc = fm.utils.copy_function(func, orgname)
        f_globals[orgname] = orgfunc

        @wraps(func)
        def wrapper(*args, **kwargs):
            depth = [s.function for s in stack()].index("<module>")
            f_globals = getframe(depth).f_globals

            # parse args and kwargs
            params = signature(func).parameters
            for i, (key, val) in enumerate(params.items()):
                if not val.kind == Parameter.POSITIONAL_OR_KEYWORD:
                    break

                try:
                    kwargs.update({key: args[i]})
                except IndexError:
                    kwargs.setdefault(key, val.default)

            # n_chunks and n_processes
            n_chunks = DEFAULT_N_CHUNKS
            n_processes = MAX_WORKERS
            multiprocess = True

            if argnames:
                length = len(kwargs[argnames[0]])

                if "numchunk" in kwargs:
                    n_chunks = kwargs.pop("numchunk")
                elif "timechunk" in kwargs:
                    n_chunks = round(length / kwargs.pop("timechunk"))
                elif "numchunk" in f_globals:
                    n_chunks = f_globals["numchunk"]
                elif "timechunk" in f_globals:
                    n_chunks = round(length / f_globals["timechunk"])

                if "n_processes" in kwargs:
                    n_processes = kwargs.pop("n_processes")
                elif "n_processes" in f_globals:
                    n_processes = f_globals["n_processes"]

                if "multiprocess" in kwargs:
                    multiprocess = kwargs.pop("multiprocess")
                elif "multiprocess" in f_globals:
                    multiprocess = f_globals["multiprocess"]

            # make chunked args
            chunks = {}
            for name in argnames:
                arg = kwargs.pop(name)
                try:
                    chunks.update({name: np.array_split(arg, n_chunks)})
                except TypeError:
                    chunks.update({name: np.tile(arg, n_chunks)})

            # run the function
            futures = []
            results = []

            if multiprocess:
                with fm.utils.one_thread_per_process(), Pool(n_processes) as p:
                    for i in range(n_chunks):
                        chunk = {key: val[i] for key, val in chunks.items()}
                        futures.append(p.submit(orgfunc, **{**chunk, **kwargs}))

                    for future in futures:
                        results.append(future.result())
            else:
                for i in range(n_chunks):
                    chunk = {key: val[i] for key, val in chunks.items()}
                    results.append(orgfunc(**{**chunk, **kwargs}))

            # make an output
            if concatfunc is not None:
                return concatfunc(results)

            try:
                return xr.concat(results, "t")
            except TypeError:
                return np.concatenate(results, 0)

        return wrapper

    return _chunk


import database_util
import unittest
from gradescope_utils.autograder_utils.decorators import weight, visibility
from HiddenPrints import HiddenPrints
with HiddenPrints():
    import SQLLab
import sqlite3 as sqlite
import os

class q3(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        try:
            os.remove('database.db')
        except:
            pass

    def setUp(self):
        self.con = sqlite.connect('database.db')
        self.cur = self.con.cursor()
        database_util.initialize_tables()

    '''
    @weight(1)
    def test_empty(self):
        """Q5: Function called on an empty table"""
        self.assertEqual(len(SQLLab.max_salary_title()), 0)
    '''

    @weight(4)
    def test_q5(self):
        """Q5: Main function"""
        database_util.testDatabase()
        self.assertListEqual(SQLLab.max_salary_title(),
            [(4, 'John', 'Cena', 'Sales', 100.01),
            (2, 'Rachel', 'Green', 'HR', 50.0),
            (3, 'Ross', 'Geller', 'IT', 49.99),
            (7, 'John', 'Watson', 'Investments', 39.0),
            (0, 'Harry', 'Truman', 'Engineering', 17.1)])

    def tearDown(self):
        self.con.close()
        os.remove('database.db')

if __name__ == '__main__':
    unittest.main()


'''
Plot the results of a Grid Search
'''

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import os
import localConfig as cfg

def findParam(inA,word,outA):
    for i in range(4,len(inA)):
        if inA[i].find(word) != -1:
            outA.append(float(inA[i+1].replace(',','').replace('}','')))

filepath = cfg.lgbk+"Searches"
os.chdir(filepath)

with open("gS:2017-09-06_00:10.txt") as f:
    data = f.read()

data = data.split("\n")

accuracy = []
std_accuracy = []
layers = []
learn_rate = []
dropout_rate = []
batch_size = []
epochs = []
neurons = []

for line in range(5,len(data)-1):
    l = data[line].split(" ")
    accuracy.append(float(l[0]))
    std_accuracy.append(float(l[2].replace('(','').replace(')','')))

    findParam(l,'layers',layers)
    findParam(l,'learn_rate',learn_rate)
    findParam(l,'dropout_rate',dropout_rate)
    findParam(l,'batch_size',batch_size)
    findParam(l,'epochs',epochs)
    findParam(l,'neurons',neurons)

di = dropout_rate.index(0.5)

fig = plt.figure(figsize=plt.figaspect(0.5))
ax = fig.add_subplot(121, projection='3d')
x = neurons[:di]
y = layers[:di]
z = accuracy[:di]
ax.plot(x, y, z, c='r', marker='o')
ax.set_title('Dropout-rate: 0')
ax.set_xlabel('Neurons')
ax.set_ylabel('Layers')
ax.set_zlabel('Accuracy')

ax = fig.add_subplot(122, projection='3d')
x = neurons[di:]
y = layers[di:]
z = accuracy[di:]
ax.scatter(x, y, z, c='b', marker='^')
ax.set_title('Dropout-rate: 0.5')
ax.set_xlabel('Neurons')
ax.set_ylabel('Layers')
ax.set_zlabel('Accuracy')

plt.show()

'''
print accuracy
print "\n"
print std_accuracy
print "\n"
print layers
print "\n"
print learn_rate
print "\n"
print dropout_rate
print "\n"
print batch_size
print "\n"
print epochs
print "\n"
print neurons
'''


# coding=utf-8
# --------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
#
# Code generated by Microsoft (R) AutoRest Code Generator.
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------

from msrest.serialization import Model

class DataTableResponseObject(Model):
    """Data Table which defines columns and raw row values.

    :param table_name: Name of the table
    :type table_name: str
    :param columns: List of columns with data types
    :type columns: list[~azure.mgmt.web.models.DataTableResponseColumn]
    :param rows: Raw row values
    :type rows: list[list[str]]
    """

    _attribute_map = {
        'table_name': {'key': 'tableName', 'type': 'str'},
        'columns': {'key': 'columns', 'type': '[DataTableResponseColumn]'},
        'rows': {'key': 'rows', 'type': '[[str]]'},
    }

    def __init__(self, *, table_name: str=None, columns=None, rows=None, **kwargs) -> None:
        super(DataTableResponseObject, self).__init__(**kwargs)
        self.table_name = table_name
        self.columns = columns
        self.rows = rows


from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import *
from django.db import IntegrityError
from django.db import connection
from django.core.cache import cache
from django.db.models import Count
from marimobackend.models import *
from marimobackend.serializers import *
from optparse import make_option
from bs4 import BeautifulSoup
from multiprocessing import Pool
from cfscrape import cfscrape
import requests
import string
import re
import datetime
import mechanicalsoup

BASE_URL = "http://kissanime.com"
ANIME_LIST_URL = "http://kissanime.com/AnimeList"
UPDATE_URL = "http://kissanime.com/AnimeList/LatestUpdate"
LOGIN_URL = "http://kissanime.com/Login"

class Command(BaseCommand):
    help = 'Scrapes all of KissAnime.'
    option_list = BaseCommand.option_list + (
        make_option('--update',
                    action='store_true',
                    dest='update',
                    default=False,
                    help='Scrapes the latest updates from KissAnime.'),
        make_option('--cache',
                    action='store_true',
                    dest='cache',
                    default=False,
                    help='Cache data after scraping.')
        )

    def handle(self, *args, **options):
        scraper = Scraper(should_update=options['update'],
                          should_cache=options['cache'])
        scraper.scrape()

class Scraper:
    def __init__(self, should_update, should_cache):
        self.should_update = should_update
        self.should_cache = should_cache
        self.session = None
        self.urls = None

    def scrape(self):
        self.login()
        self.update_potential_categories()
        self.scrape_anime_list()
        for url in self.urls:
            self.scrape_anime(url)
        if self.should_cache:
            self.cache_anime()

    def login(self):
        """
        Login in order to bypass restrictions.
        """
        print('Logging in...')
        try:
            cloudflare_scraper = cfscrape.create_scraper()
            cloudflare_scraper.get(LOGIN_URL)
            browser = mechanicalsoup.Browser(session=cloudflare_scraper)
        except RuntimeError:
            browser = mechanicalsoup.Browser()

        login_page = browser.get(LOGIN_URL)
        login_form = login_page.soup.select("#formLogin")[0]
        login_form.select("input#username")[0]['value'] = 'hey123123'
        login_form.select("input#password")[0]['value'] = '123123123'
        browser.submit(login_form, login_page.url)
        self.session = browser.session

    def update_potential_categories(self):
        """
        Store the categories in advance to prevent race conditions.
        """
        page = self.session.get(ANIME_LIST_URL)
        soup = BeautifulSoup(page.text)
        categories = soup.select('a[href^=/Genre/]')
        for category in categories:
            text = category.text.lstrip()
            try:
                cat = Category.objects.get(name=text)
            except Category.DoesNotExist:
                cat = Category(name=text)
                cat.save()

    def cache_anime(self):
        """
        Cache the newly scraped anime.
        """
        updated_anime = Anime.objects.filter().annotate(num_episodes=Count('episode')).filter(num_episodes__gt=0)
        serialized_anime = AnimeSerializer(updated_anime, many=True)
        data = serialized_anime.data
        cache.set('anime_list', data, None)

    def scrape_anime_list(self):
        """
        Recursively scrapes urls and names of every anime series in the anime list.
        """
        def get_last_page():
            """
            Finds the page number of the last page in the anime list so we know
            when the list is over.
            """
            page = self.session.get(ANIME_LIST_URL)
            soup = BeautifulSoup(page.text)
            last_page_num = soup.select('a[href^=/AnimeList?page=]')[-1]['page']
            return int(last_page_num)

        def scrape_page(page_num, anime_list=[]):
            """
            Scrapes each page in the anime list.
            """
            page = self.session.get(ANIME_LIST_URL + "?page=" + str(page_num))
            soup = BeautifulSoup(page.text)
            for anime in soup.select('a[href^=/Anime/]'):
                anime_list.append(anime)
            if page_num == last_page:
                return anime_list
            else:
                return scrape_page(page_num + 1, anime_list)

        def filter_anime(anime_data):
            """
            Filters out unwanted scraped urls.
            """
            anime_list = []
            for anime in anime_data:
                text = anime.text.lstrip()
                if not re.match("Episode", text) \
                and not re.match("^OVA", text) \
                and not re.match("^OAD", text) \
                and not re.match("_Preview", text) \
                and not re.match("^_Ending Preview", text) \
                and not re.match("13 - Shield and", text) \
                and not re.match('_CM', text):
                    anime_list.append(BASE_URL + anime['href'])
            return anime_list

        def scrape_updates():
            """
            Scrapes a list of only the latest updates.
            """
            page = self.session.get(UPDATE_URL)
            soup = BeautifulSoup(page.text)
            anime_list = []
            for anime in soup.select('a[href^=/Anime/]'):
                anime_list.append(anime)
            return anime_list

        if self.should_update:
            raw_data = scrape_updates()
            self.urls = filter_anime(raw_data)
        else:
            last_page = get_last_page()
            raw_data = scrape_page(0)
            self.urls = filter_anime(raw_data)

    def scrape_anime(self, url):
        """
        Scrapes all information and episode urls for a given anime series.
        i.e: http://kissanime.com/Anime/Psycho-Pass-2
        >>> x = scrape_anime('http://kissanime.com/Anime/Psycho-Pass-2')
        >>> x.episodes[0].date
        '12/18/2014'
        """
        connection.close()
        page = self.session.get(url)
        soup = BeautifulSoup(page.text)

        def get_anime_info():
            info = {}
            title = soup.find('a', class_='bigChar').text
            print("Scraping Anime: " + title)
            image = soup.select('div.barContent > div > img')[0]['src']
            for x in soup.select('div.barContent > div > p > span.info'):
                value = x.next_siblings.__next__().strip()
                if x.text.strip() == 'Date aired:':
                    info['date'] = value
                elif x.text.strip() == 'Views:':
                    info['views'] = int(value.replace(',', ''))
                elif x.text.strip() == 'Status:':
                    info['status'] = value

            summary = ''
            try:
                for s in soup.select('div.barContent > div > p ')[-1].contents[3].contents:
                    try:
                        summary += s + ' '
                    except:
                        continue
            except:
                summary += soup.select('div.barContent > div > p ')[-1].next_sibling.strip()

            info['title'] = title
            info['image'] = image
            info['summary'] = summary
            return info

        def get_anime():
            info = get_anime_info()
            try:
                anime = Anime.objects.get(url=url)
                anime.title = info['title']
                anime.cover_image = info['image']
                anime.date_aired = info.get('date', None)
                anime.status = info['status']
                anime.views = info['views']
                anime.origin = "KissAnime"
                anime.save()

            except Anime.DoesNotExist:
                anime = Anime(title=info['title'],
                          cover_image=info['image'],
                          date_aired=info.get('date', None),
                          status=info['status'],
                          views=info['views'],
                          origin="KissAnime",
                          summary=info['summary'],
                          url=url)
                anime.save()
            except IntegrityError:
                    return
            return anime

        def get_categories():
            genres = soup.select('div.barContent > div > p > a[href^=/Genre]')
            for category in genres:
                cat = Category.objects.get(name=category.text)
                anime.categories.add(cat)
                cat.animes.add(anime)

        def get_alternate_names():
            other_names = soup.select('div.barContent > div > p > a[href^=/Anime]')
            for alt in other_names:
                try:
                    AlternateName.objects.get(name=alt.text, anime=anime)
                except AlternateName.DoesNotExist:
                    name = AlternateName(name=alt.text, anime=anime)
                    anime.alternatename_set.add(name)

        def get_episodes():
            episode_base = url.replace(BASE_URL, '')
            episode_base = episode_base.replace(' ', '-')
            episode_data = soup.select('a[href^=' + episode_base + '/Episode]')
            if episode_data:
                episode_data = episode_data[:-1] #Get rid of unwanted fake episode
            # Note: If the `episodes` are actually movies or OVAs, this will not detect them, so we have to try movies as well.
            if not episode_data:
                episode_data = soup.select('a[href^=' + episode_base + '/Movie]')
            if not episode_data:
                episode_data = soup.select('a[href^=' + episode_base + '/OVA]')
            for raw_episode in episode_data:
                episode_title = raw_episode.text.lstrip()
                date_data = raw_episode.parent.next_siblings
                for s in date_data:
                    if s != '\n':
                        date = s.text.strip()
                        try:
                            date = datetime.datetime.strptime(date, '%m/%d/%Y').date()
                        except:
                            date = datetime.datetime.now().strftime('%m/%d/%Y')
                try:
                    episode = anime.episode_set.get(title=episode_title)
                except Episode.DoesNotExist:
                    print('Scraping new episode:' + episode_title)
                    episode = Episode(title=episode_title, date=date, anime=anime)
                    episode.save()
                    anime.episode_set.add(episode)
                except IntegrityError:
                    pass
                if episode.video_set.count() <= 0:
                    print("Scraping videos...")
                    get_videos(BASE_URL + raw_episode['href'], episode)
            refresh_for_video_updates(episode_data)
            anime.save()

        def get_videos(episode_url, episode):
            """
            Scrapes the different quality video URLs for a given episode.
            i.e: "http://kissanime.com/Anime/Fate-stay-night-2014/Episode-007?id=82557"
            """
            page = self.session.get(episode_url)
            soup = BeautifulSoup(page.text)
            video_data =  soup.select('#divDownload')
            if video_data:
                for video in video_data[0].find_all('a'):
                    try:
                        Video.objects.get(quality=video.text, episode=episode)
                    except Video.DoesNotExist:
                        vid = Video(quality=video.text, link=video['href'], episode=episode)
                        vid.save()
                        episode.video_set.add(vid)
                    except IntegrityError:
                        connection.close()

        def refresh_for_video_updates(episode_data):
            """
            Check the latest episode again to make sure no additional quality videos have been added.
            """
            try:
                episode_title = episode_data[0].text.lstrip()
                episode = anime.episode_set.get(title=episode_title)
                if episode.video_set.count() <= 3:
                    print('Checking for video updates...')
                    get_videos(BASE_URL+ episode_data[0]['href'], episode)
            except:
                pass

        anime = get_anime()
        get_categories()
        get_alternate_names()
        get_episodes()

        if self.should_cache:
            if cache.get(str(anime.pk)):
                cache.delete(anime.pk)


# coding: utf-8
import json
import os
from xml.sax.saxutils import escape
from nhentai.constant import LANGUAGE_ISO

def serialize_json(doujinshi, output_dir):
    metadata = {'title': doujinshi.name,
                'subtitle': doujinshi.info.subtitle}
    if doujinshi.info.date:
        metadata['upload_date'] = doujinshi.info.date
    if doujinshi.info.parodies:
        metadata['parody'] = [i.strip() for i in doujinshi.info.parodies.split(',')]
    if doujinshi.info.characters:
        metadata['character'] = [i.strip() for i in doujinshi.info.characters.split(',')]
    if doujinshi.info.tags:
        metadata['tag'] = [i.strip() for i in doujinshi.info.tags.split(',')]
    if doujinshi.info.artists:
        metadata['artist'] = [i.strip() for i in doujinshi.info.artists.split(',')]
    if doujinshi.info.groups:
        metadata['group'] = [i.strip() for i in doujinshi.info.groups.split(',')]
    if doujinshi.info.languages:
        metadata['language'] = [i.strip() for i in doujinshi.info.languages.split(',')]
    metadata['category'] = doujinshi.info.categories
    metadata['URL'] = doujinshi.url
    metadata['Pages'] = doujinshi.pages

    with open(os.path.join(output_dir, 'metadata.json'), 'w') as f:
        json.dump(metadata, f, separators=(',', ':'))

def serialize_comic_xml(doujinshi, output_dir):
    from iso8601 import parse_date
    with open(os.path.join(output_dir, 'ComicInfo.xml'), 'w', encoding="utf-8") as f:
        f.write('<?xml version="1.0" encoding="utf-8"?>\n')
        f.write('<ComicInfo xmlns:xsd="http://www.w3.org/2001/XMLSchema" '
                'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n')

        xml_write_simple_tag(f, 'Manga', 'Yes')

        xml_write_simple_tag(f, 'Title', doujinshi.name)
        xml_write_simple_tag(f, 'Summary', doujinshi.info.subtitle)
        xml_write_simple_tag(f, 'PageCount', doujinshi.pages)
        xml_write_simple_tag(f, 'URL', doujinshi.url)
        xml_write_simple_tag(f, 'NhentaiId', doujinshi.id)
        xml_write_simple_tag(f, 'Genre', doujinshi.info.categories)

        xml_write_simple_tag(f, 'BlackAndWhite', 'No' if doujinshi.info.tags and
                             'full color' in doujinshi.info.tags else 'Yes')

        if doujinshi.info.date:
            dt = parse_date(doujinshi.info.date)
            xml_write_simple_tag(f, 'Year', dt.year)
            xml_write_simple_tag(f, 'Month', dt.month)
            xml_write_simple_tag(f, 'Day', dt.day)
        if doujinshi.info.parodies:
            xml_write_simple_tag(f, 'Series', doujinshi.info.parodies)
        if doujinshi.info.characters:
            xml_write_simple_tag(f, 'Characters', doujinshi.info.characters)
        if doujinshi.info.tags:
            xml_write_simple_tag(f, 'Tags', doujinshi.info.tags)
        if doujinshi.info.artists:
            xml_write_simple_tag(f, 'Writer', ' & '.join([i.strip() for i in
                                                          doujinshi.info.artists.split(',')]))

        if doujinshi.info.languages:
            languages = [i.strip() for i in doujinshi.info.languages.split(',')]
            xml_write_simple_tag(f, 'Translated', 'Yes' if 'translated' in languages else 'No')
            [xml_write_simple_tag(f, 'LanguageISO', LANGUAGE_ISO[i]) for i in languages
             if (i != 'translated' and i in LANGUAGE_ISO)]

        f.write('</ComicInfo>')

def xml_write_simple_tag(f, name, val, indent=1):
    f.write(f'{" "*indent}<{name}>{escape(str(val))}</{name}>\n')

def merge_json():
    lst = []
    output_dir = "./"
    os.chdir(output_dir)
    doujinshi_dirs = next(os.walk('.'))[1]
    for folder in doujinshi_dirs:
        files = os.listdir(folder)
        if 'metadata.json' not in files:
            continue
        data_folder = output_dir + folder + '/' + 'metadata.json'
        json_file = open(data_folder, 'r')
        json_dict = json.load(json_file)
        json_dict['Folder'] = folder
        lst.append(json_dict)
    return lst

def serialize_unique(lst):
    dictionary = {}
    parody = []
    character = []
    tag = []
    artist = []
    group = []
    for dic in lst:
        if 'parody' in dic:
            parody.extend([i for i in dic['parody']])
        if 'character' in dic:
            character.extend([i for i in dic['character']])
        if 'tag' in dic:
            tag.extend([i for i in dic['tag']])
        if 'artist' in dic:
            artist.extend([i for i in dic['artist']])
        if 'group' in dic:
            group.extend([i for i in dic['group']])
    dictionary['parody'] = list(set(parody))
    dictionary['character'] = list(set(character))
    dictionary['tag'] = list(set(tag))
    dictionary['artist'] = list(set(artist))
    dictionary['group'] = list(set(group))
    return dictionary

def set_js_database():
    with open('data.js', 'w') as f:
        indexed_json = merge_json()
        unique_json = json.dumps(serialize_unique(indexed_json), separators=(',', ':'))
        indexed_json = json.dumps(indexed_json, separators=(',', ':'))
        f.write('var data = ' + indexed_json)
        f.write(';\nvar tags = ' + unique_json)


from .renderer import Renderer
from flask import Response, render_template
import urllib.request, urllib.parse, urllib.error
import database
from modules.ldapi import LDAPI
from rdflib import Graph, URIRef, Literal, Namespace

class ReportRenderer(Renderer):
    def __init__(self, uri, endpoints):
        Renderer.__init__(self, uri, endpoints)

        self.uri_encoded = urllib.parse.quote_plus(uri)
        self.label = None
        self.rt = None
        self.rt_label = None
        self.nid = None
        self.gat = None
        self.rs = None
        self.rs_encoded = None
        self.rs_label = None
        self.sa = None
        self.sa_label = None
        self.ea = None
        self.ea_label = None
        self.script = None

    def render(self, view, mimetype):
        if view == 'neighbours':
            # no work to be done as we have already loaded the triples
            if mimetype in LDAPI.get_rdf_mimetypes_list():
                return self._neighbours_rdf(mimetype)
            elif mimetype == 'text/html':
                self._get_details()
                return self._neighbours_html()
        elif view == 'prov':
            if mimetype in LDAPI.get_rdf_mimetypes_list():
                return Response(
                    self._prov_rdf().serialize(format=LDAPI.get_rdf_parser_for_mimetype(mimetype)),
                    status=200,
                    mimetype=mimetype
                )
            elif mimetype == 'text/html':
                self._get_details()
                return self._prov_html()

    def _neighbours_rdf(self, mimetype):
        query = '''
                  SELECT * WHERE {
                     <%(uri)s>  ?p ?o .
                  }
          ''' % {'uri': self.uri}
        g = Graph()
        g.bind('prov', Namespace('http://www.w3.org/ns/prov#'))
        for r in database.query(query)['results']['bindings']:
            if r['o']['type'] == 'literal':
                g.add((URIRef(self.uri), URIRef(r['p']['value']), Literal(r['o']['value'])))
            else:
                g.add((URIRef(self.uri), URIRef(r['p']['value']), URIRef(r['o']['value'])))

        query2 = '''
                  SELECT * WHERE {
                     ?s ?p <%(uri)s> .
                  }
          ''' % {'uri': self.uri}
        for r in database.query(query2)['results']['bindings']:
            g.add((URIRef(r['s']['value']), URIRef(r['p']['value']), URIRef(self.uri)))

        return Response(
            g.serialize(format=LDAPI.get_rdf_parser_for_mimetype(mimetype)),
            status=200,
            mimetype=mimetype
        )

    def _neighbours_html(self):
        """Returns a simple dict of Activity properties for use by a Jinja template"""
        self._make_svg_script()

        ret = {
            'rt_label': self.rt_label,
            'uri': self.uri,
            'uri_encoded': self.uri_encoded,
            'label': self.label,
            'nid': self.nid,
            'gat': self.gat,
            'rs_encoded': self.rs_encoded,
            'rs_label': self.rs_label,
            'sa': self.sa,
            'ea': self.ea,
            'script': self.script
        }

        return render_template(
            'class_report.html',
            report=ret
        )

    def _prov_rdf(self):
        query = '''
                 SELECT * WHERE {
                    <%(uri)s>  ?p ?o .
                 }
         ''' % {'uri': self.uri}
        g = Graph()
        g.bind('prov', Namespace('http://www.w3.org/ns/prov#'))
        for r in database.query(query)['results']['bindings']:
            if r['o']['type'] == 'literal':
                g.add((URIRef(self.uri), URIRef(r['p']['value']), Literal(r['o']['value'])))
            else:
                g.add((URIRef(self.uri), URIRef(r['p']['value']), URIRef(r['o']['value'])))

        query2 = '''
                 SELECT * WHERE {
                    ?s ?p <%(uri)s> .
                 }
         ''' % {'uri': self.uri}
        for r in database.query(query2)['results']['bindings']:
            g.add((URIRef(r['s']['value']), URIRef(r['p']['value']), URIRef(self.uri)))

        return g

    def _prov_html(self):
        """Returns a simple dict of Entity properties for use by a Jinja template"""
        ret = {
            'rt_label': self.rt_label,
            'uri': self.uri,
            'uri_encoded': self.uri_encoded,
            'label': self.label,
            'nid': self.nid,
            'gat': self.gat,
            'rs_encoded': self.rs_encoded,
            'rs_label': self.rs_label,
            'sa': self.sa,
            'ea': self.ea
        }

        prov_data = self._prov_rdf().serialize(format='turtle')

        return render_template(
            'class_report_prov.html',
            report=ret,
            prov_data=prov_data
        )

    def _get_details(self):
        """ Get the details for a Report from an RDF triplestore"""
        # formulate the query
        query = '''
            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
            PREFIX proms: <http://promsns.org/def/proms#>
            PREFIX prov: <http://www.w3.org/ns/prov#>
            SELECT *
            WHERE {
                GRAPH ?g {
                    <%(uri)s>
                        a ?rt ;
                        rdfs:label ?label ;
                        proms:nativeId ?nid ;
                        prov:generatedAtTime ?gat ;
                        proms:wasReportedBy ?rs .
                    OPTIONAL {
                       ?rs rdfs:label ?rs_label .
                    }
                    OPTIONAL {
                        <%(uri)s>
                            proms:startingActivity ?sa .
                            ?sa rdfs:label ?sa_label .
                    }
                    OPTIONAL {
                        <%(uri)s>
                            proms:endingActivity ?ea .
                            ?ea rdfs:label ?ea_label .
                    } .
                }
            }
        ''' % {'uri': self.uri}

        # run the query
        report_details = database.query(query)

        # extract results into instance vars
        if report_details and 'results' in report_details:
            if len(report_details['results']['bindings']) > 0:
                ret = report_details['results']['bindings'][0]
                self.rt = ret['rt']['value']
                if 'Basic' in self.rt:
                    self.rt_label = 'Basic'
                elif 'Internal' in self.rt:
                    self.rt_label = 'Internal'
                elif 'External' in self.rt:
                    self.rt_label = 'External'
                self.label = ret['label']['value']
                self.nid = ret['nid']['value']
                self.gat = ret['gat']['value']
                self.rs = ret['rs']['value']
                self.rs_encoded = urllib.parse.quote_plus(self.rs)
                self.rs_label = ret['rs_label']['value'] if 'rs_label' in ret else self.rs
                if 'sa' in ret:
                    self.sa = ret['sa']['value']
                    self.sa_label = ret['sa_label']['value']
                if 'ea' in ret:
                    self.ea = ret['ea']['value']
                    self.ea_label = ret['ea_label']['value']

    def _make_svg_script(self):
        """ Construct the SVG code for a Report's Neighbours view"""
        self.script = '''
            var rLabel = "%(label)s";
            var report = addReport(350, 200, rLabel, "");
        ''' % {'label': self.label}

        self.script += '''
            var rsUri = "%(instance_endpoint)s?_uri=%(uri_encoded)s";
            var rsLabel = "%(label)s";
            var repSystem = addReportingSystem(350, 20, rsLabel, rsUri);
            addLink(report, repSystem, "proms:reportingSystem", RIGHT);
        ''' % {
            'instance_endpoint': self.endpoints['instance'],
            'uri_encoded': self.rs_encoded,
            'label': self.rs_label
        }

        if self.sa is not None and self.ea is not None:
            if self.sa == self.ea:
                # External Report -- single Activity
                self.script += '''
                    var uri = "%(instance_endpoint)s?_uri=%(uri_encoded)s";
                    var label = "%(label)s";
                    var activity = addActivity(50, 200, label, uri);
                    addLink(report, activity, "proms:startingActivity", TOP);
                    addLink(report, activity, "proms:endingActivity", BOTTOM);
                ''' % {
                    'instance_endpoint': self.endpoints['instance'],
                    'uri_encoded': urllib.parse.quote(self.sa),
                    'label': self.sa_label
                }
            else:
                # Internal Report -- 2 Activities
                self.script += '''
                    var saUri = "%(instance_endpoint)s?_uri=%(uri_encoded)s";
                    var saLabel = "%(label)s";
                    var sacActivity = addActivity(50, 120, sacLabel, sacUri);
                    addLink(report, sacActivity, "proms:startingActivity", TOP);
                ''' % {
                    'instance_endpoint': self.endpoints['instance'],
                    'uri_encoded': urllib.parse.quote(self.sa),
                    'label': self.sa_label
                }

                self.script += '''
                    var eacUri = "%(instance_endpoint)s?_uri=%(uri_encoded)s";
                    var eacLabel = "%(label)s";
                    var eacActivity = addActivity(50, 280, eacLabel, eacUri);
                    addLink(report, eacActivity, "proms:endingActivity", BOTTOM);
                ''' % {
                    'instance_endpoint': self.endpoints['instance'],
                    'uri_encoded': urllib.parse.quote(self.ea),
                    'label': self.ea_label
                }
        else:
            # Basic Report -- no Activities
            pass


# -*- coding: utf-8 -*-

import os
import sys
import oss2

access_key_id = ""
access_key_secret = ""
bucket_name = "pi-miner"
painull_path = "tools/painull/v%s"

def upload_file(file, version):
    auth = oss2.Auth(access_key_id, access_key_secret)
    endpoint = "http://oss-cn-beijing.aliyuncs.com"
    bucket = oss2.Bucket(auth, endpoint, bucket_name)
    # info = bucket.get_bucket_info()

    oss_path = os.path.join(painull_path % version, os.path.basename(file))
    for i in range(5):
        try:
            bucket.put_object_from_file(oss_path, file)
        except Exception as e:
            print("Upload file error: %s" % e)
        else:
            print("Upload file completely")
            break

if __name__ == '__main__':
    if len(sys.argv) >= 3:
        file_version = sys.argv[1]
        file_name = sys.argv[2]
        print("Upload file: %s" % file_name)
        print("Upload file version: %s" % file_version)
        upload_file(file_name, file_version)
    else:
        print("Input parameter error")


import os
from pathlib import Path
from unittest.mock import MagicMock, Mock

import pytest
from click import FileError, NoSuchOption

from robotidy.cli import read_config, validate_regex
from robotidy.files import find_project_root, read_pyproject_config, get_paths, DEFAULT_EXCLUDES
from robotidy.transformers import load_transformers
from robotidy.transformers.AlignSettingsSection import AlignSettingsSection
from robotidy.transformers.ReplaceRunKeywordIf import ReplaceRunKeywordIf
from robotidy.transformers.SmartSortKeywords import SmartSortKeywords
from robotidy.utils import node_within_lines
from robotidy.version import __version__
from .utils import run_tidy

@pytest.fixture
def test_data_dir():
    return Path(__file__).parent / 'testdata'

class TestCli:
    @pytest.mark.parametrize('name, similar', [
        ('NotExisting', ''),
        ('AlignSettings', ' Did you mean:\n    AlignSettingsSection'),
        ('align', ' Did you mean:\n    AlignSettingsSection\n    AlignVariablesSection'),
        ('splittoolongline', ' Did you mean:\n    SplitTooLongLine'),
        ('AssignmentNormalizer', ' Did you mean:\n    NormalizeAssignments')
    ])
    def test_not_existing_transformer(self, name, similar):
        expected_output = f"Importing transformer '{name}' failed. " \
                          f"Verify if correct name or configuration was provided.{similar}"
        args = f'--transform {name} --transform MissingTransformer --transform DiscardEmptySections -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.exception)

    def test_transformer_order(self):
        order_1 = ['NormalizeSeparators', 'OrderSettings']
        order_2 = ['OrderSettings', 'NormalizeSeparators']
        transformers_1 = load_transformers([(transf, []) for transf in order_1], {})
        transformers_2 = load_transformers([(transf, []) for transf in order_2], {})
        assert all(t1.__class__.__name__ == t2.__class__.__name__ for t1, t2 in zip(transformers_1, transformers_2))

    def test_transformer_force_order(self):
        # default_order = ['NormalizeSeparators', 'OrderSettings']
        custom_order = ['OrderSettings', 'NormalizeSeparators']
        transformers = load_transformers([(transf, []) for transf in custom_order], {}, force_order=True)
        assert all(t1.__class__.__name__ == t2 for t1, t2 in zip(transformers, custom_order))

    # TODO: raise exception if kwarg does not match
    # def test_not_existing_configurable(self):
    #     expected_output = "Usage: cli [OPTIONS] [PATH(S)]\n\n" \
    #                       "Error: Invalid configurable name: 'missing_configurable' for transformer: " \
    #                       "'DiscardEmptySections'\n"
    #
    #     args = '--transform DiscardEmptySections:allow_only_commentss=True -'.split()
    #     result = run_tidy(args, exit_code=2)
    #     assert expected_output == result.output

    def test_invalid_configurable_usage(self):
        expected_output = "Importing transformer 'DiscardEmptySections=allow_only_comments=False' failed. " \
                          "Verify if correct name or configuration was provided"
        args = '--transform DiscardEmptySections=allow_only_comments=False -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.exception)

    def test_too_many_arguments_for_transform(self):
        expected_output = "not enough values to unpack (expected 2, got 1)"
        args = '--transform DiscardEmptySections:allow_only_comments:False -'.split()
        result = run_tidy(args, exit_code=1)
        assert str(result.exception) == expected_output

    # def test_invalid_argument_type_for_transform(self):
    #     expected_output = "Importing 'robotidy.transformers.DiscardEmptySections' failed:  'DicardEmptySection'"
    #     args = '--transform DiscardEmptySections:allow_only_comments=true'.split()
    #     result = run_tidy(args, exit_code=1)
    #     assert expected_output == str(result.exception)

    def test_find_project_root_from_src(self, test_data_dir):
        src = test_data_dir / 'nested' / 'test.robot'
        path = find_project_root((src,))
        assert path == test_data_dir / 'nested'

    def test_read_robotidy_config(self, test_data_dir):
        """ robotidy.toml follows the same format as pyproject starting from 1.2.0 """
        expected_config = {
            'overwrite': False,
            'diff': False,
            'spacecount': 4,
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        config = read_pyproject_config(config_path)
        assert config == expected_config

    def test_read_pyproject_config(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': False,
            'diff': False,
            'startline': 10,
            'endline': 20,
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'SplitTooLongLine'
            ],
            'configure': [
                'DiscardEmptySections:allow_only_comments=False',
                'OrderSettings: keyword_before = documentation,tags,timeout,arguments'
            ]
        }
        config_path = str(test_data_dir / 'only_pyproject' / 'pyproject.toml')
        config = read_pyproject_config(config_path)
        assert config == expected_parsed_config

    def test_read_pyproject_config_e2e(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'startline': '10',
            'endline': '20',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'SplitTooLongLine'
            ],
            'configure': [
                'DiscardEmptySections:allow_only_comments=False',
                'OrderSettings: keyword_before = documentation,tags,timeout,arguments'
            ]
        }
        config_path = str(test_data_dir / 'only_pyproject')
        ctx_mock = MagicMock()
        ctx_mock.params = {'src': (config_path,)}
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, value=None)
        assert ctx_mock.default_map == expected_parsed_config

    def test_read_invalid_config(self, test_data_dir):
        config_path = str(test_data_dir / 'invalid_pyproject' / 'pyproject.toml')
        with pytest.raises(FileError) as err:
            read_pyproject_config(config_path)
        assert 'Error reading configuration file: ' in str(err)

    @pytest.mark.parametrize('option, correct', [
        ('confgure', 'configure'),
        ('idontexist', None)
    ])
    def test_read_invalid_option_config(self, option, correct, test_data_dir):
        config_path = str(test_data_dir / 'invalid_options_config' / f'pyproject_{option}.toml')
        ctx_mock = MagicMock()
        param_mock = MagicMock()
        with pytest.raises(NoSuchOption) as err:
            read_config(ctx_mock, param_mock, config_path)
        similar = '' if correct is None else f' Did you mean {correct}'
        assert f'no such option: {option}{similar}'

    def test_read_config_from_param(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'spacecount': '4',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        ctx_mock = MagicMock()
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, config_path)
        assert ctx_mock.default_map == expected_parsed_config

    def test_read_config_without_param(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'spacecount': '4',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        ctx_mock = MagicMock()
        ctx_mock.params = {'src': (config_path,)}
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, value=None)
        assert ctx_mock.default_map == expected_parsed_config

    @pytest.mark.parametrize('node_start, node_end, start_line, end_line, expected', [
        (15, 30, 15, None, True),
        (15, 30, 15, 30, True),
        (14, 30, 15, 30, False),
        (15, 31, 15, 30, False),
        (15, 30, None, 30, True),
        (15, 30, None, None, True)
    ])
    def test_skip_node_start_end_line_setting(self, node_start, node_end, start_line, end_line, expected):
        assert node_within_lines(node_start, node_end, start_line, end_line) == expected

    @pytest.mark.parametrize('flag', ['--list', '-l'])
    def test_list_transformers(self, flag):
        result = run_tidy([flag])
        assert 'To see detailed docs run --desc <transformer_name> or --desc all. Transformers with (disabled) ' \
               'tag \nare executed only when selected explicitly with --transform or configured with param ' \
               '`enabled=True`.\n' \
               'Available transformers:\n'\
               in result.output
        assert 'ReplaceRunKeywordIf\n' in result.output
        assert 'SmartSortKeywords (disabled)\n' in result.output  # this transformer is disabled by default
        assert 'Available transformers:\n\nAddMissingEnd\n' in result.output  # assert order

    @pytest.mark.parametrize('flag', ['--desc', '-d'])
    @pytest.mark.parametrize('name, expected_doc', [
        ('ReplaceRunKeywordIf', ReplaceRunKeywordIf.__doc__.replace('::', ':').replace("``", "'")),
        ('SmartSortKeywords', SmartSortKeywords.__doc__.replace('::', ':').replace("``", "'"))
    ])
    def test_describe_transformer(self, flag, name, expected_doc):
        not_expected_doc = AlignSettingsSection.__doc__.replace('::', ':').replace("``", "'")
        result = run_tidy([flag, name])
        assert expected_doc in result.output
        assert not_expected_doc not in result.output

    def test_describe_transformer_all(self):
        expected_doc = ReplaceRunKeywordIf.__doc__.replace('::', ':').replace("``", "'")
        expected_doc2 = AlignSettingsSection.__doc__.replace('::', ':').replace("``", "'")
        result = run_tidy(['--desc', 'all'])
        assert expected_doc in result.output
        assert expected_doc2 in result.output

    @pytest.mark.parametrize('name, similar', [
        ('NotExisting', ''),
        ('AlignSettings', ' Did you mean:\n    AlignSettingsSection'),
        ('align', ' Did you mean:\n    AlignSettingsSection\n    AlignVariablesSection'),
        ('splittoolongline', ' Did you mean:\n    SplitTooLongLine'),
        ('AssignmentNormalizer', ' Did you mean:\n    NormalizeAssignments')
    ])
    def test_describe_invalid_transformer(self, name, similar):
        expected_output = f"Transformer with the name '{name}' does not exist.{similar}"
        args = f'--desc {name} -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.output)

    @pytest.mark.parametrize('flag', ['--help', '-h'])
    def test_help(self, flag):
        result = run_tidy([flag])
        assert f'Version: {__version__}' in result.output

    @pytest.mark.parametrize('source, return_status', [
        ('golden.robot', 0),
        ('not_golden.robot', 1)
    ])
    def test_check(self, source, return_status, test_data_dir):
        source = test_data_dir / 'check' / source
        run_tidy(
            ['--check', '--overwrite', '--transform', 'NormalizeSectionHeaderName', str(source)],
            exit_code=return_status
        )

    def test_diff(self, test_data_dir):
        source = test_data_dir / 'check' / 'not_golden.robot'
        result = run_tidy(['--diff', '--no-overwrite', '--transform', 'NormalizeSectionHeaderName', str(source)])
        assert "*** settings ***" in result.output
        assert "*** Settings ***" in result.output

    def test_disabled_transformer(self):
        transformers = load_transformers(None, {})
        assert all(transformer.__class__.__name__ != 'SmartSortKeywords' for transformer in transformers)

    def test_enable_disable_transformer(self):
        transformers = load_transformers([('SmartSortKeywords', [])], {})
        assert transformers[0].__class__.__name__ == 'SmartSortKeywords'

    def test_configure_transformer(self):
        transformers = load_transformers(
            None,
            {'AlignVariablesSection': ['up_to_column=4']}
        )
        transformers_not_configured = load_transformers(None, {})
        assert len(transformers) == len(transformers_not_configured)
        for transformer in transformers:
            if transformer.__class__.__name__ == 'AlignVariablesSection':
                assert transformer.up_to_column + 1 == 4

    def test_configure_transformer_overwrite(self):
        transformers = load_transformers(
            [('AlignVariablesSection', ['up_to_column=3'])],
            {'AlignVariablesSection': ['up_to_column=4']}
        )
        assert transformers[0].up_to_column + 1 == 4

    @pytest.mark.parametrize('line_sep', ['unix', 'windows', 'native', None])
    def test_line_sep(self, line_sep, test_data_dir):
        source = test_data_dir / 'line_sep' / 'test.robot'
        expected = test_data_dir / 'line_sep' / 'expected.robot'
        actual = test_data_dir.parent / 'actual' / 'test.robot'
        if line_sep is not None:
            run_tidy(['--lineseparator', line_sep, str(source)], output='test.robot')
        else:
            run_tidy([str(source)], output='test.robot')
        line_end = {
            'unix': '\n',
            'windows': '\r\n',
            'native': os.linesep,
            None: os.linesep
        }[line_sep]
        with open(str(expected)) as f:
            expected_str = f.read()
        expected_str = expected_str.replace('\n', line_end)
        with open(str(actual), newline='') as f:
            actual_str = f.read()
        assert actual_str == expected_str, 'Line endings does not match'

    @pytest.mark.parametrize('force_order', [True, False])
    @pytest.mark.parametrize('allow_disabled', [True, False])
    @pytest.mark.parametrize('transformers, configure, present, test_for', [
        # robotidy .
        (None, {}, True, 'AlignVariablesSection'),
        # robotidy -c AlignVariablesSection:enabled=True .
        (None, {'AlignVariablesSection': ['enabled=True']}, True, 'AlignVariablesSection'),
        # robotidy -c AlignVariablesSection:enabled=false .
        (None, {'AlignVariablesSection': ['enabled=false']}, False, 'AlignVariablesSection'),
        # robotidy -c SmartSortKeywords:enabled=True .
        (None, {'SmartSortKeywords': ['enabled=True']}, True, 'SmartSortKeywords'),  # disabled by default
        # robotidy -c SmartSortKeywords:enabled=False .
        (None, {'SmartSortKeywords': ['enabled=False']}, False, 'SmartSortKeywords'),
        # robotidy --transform SmartSortKeywords:enabled=True .
        ([('SmartSortKeywords', ['enabled=True'])], {}, True, 'SmartSortKeywords'),
        # robotidy --transform NormalizeAssignments .
        ([('NormalizeAssignments', [])], {}, False, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', [])], {}, True, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])], {}, True,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4:enabled=True .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4', 'enabled=True'])], {}, True,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4:enabled=False .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4', 'enabled=False'])], {}, False,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 -c
        # AlignVariablesSection:enabled=True .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])],
         {'AlignVariablesSection': ['enabled=True']}, True, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 -c
        # AlignVariablesSection:enabled=False .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])],
         {'AlignVariablesSection': ['enabled=False']}, False, 'AlignVariablesSection')
    ])
    def test_disable_transformers(self, transformers, configure, present, force_order, allow_disabled, test_for):
        if force_order and not transformers:
            present = False
        loaded_transformers = load_transformers(transformers, configure, allow_disabled=allow_disabled,
                                                force_order=force_order)
        if present:
            assert any(transformer.__class__.__name__ == test_for for transformer in loaded_transformers)
        else:
            assert all(transformer.__class__.__name__ != test_for for transformer in loaded_transformers)

    @pytest.mark.parametrize('exclude, extend_exclude, allowed', [
        (DEFAULT_EXCLUDES, None, ['nested/test.robot', 'test.resource', 'test.robot']),
        ('test.resource', None, ['test.robot', 'nested/test.robot']),
        (DEFAULT_EXCLUDES, 'test.resource', ['test.robot', 'nested/test.robot']),
        ('test.resource', 'nested/*', ['test.robot'])
    ])
    def test_exclude_gitignore(self, exclude, extend_exclude, allowed, test_data_dir):
        source = test_data_dir / 'gitignore'
        allowed_paths = {Path(source, path) for path in allowed}
        paths = get_paths((str(source),), exclude=validate_regex(exclude),
                          extend_exclude=validate_regex(extend_exclude))
        assert paths == allowed_paths

    @pytest.mark.parametrize('source, should_parse', [
        (None, ['test.robot', 'resources/test.robot']),  # calls: robotidy
        ('test3.robot', ['test3.robot']),  # calls: robotidy test3.robot
        ('test.robot', ['test.robot']),
        ('.', ['test.robot', 'test3.robot', 'resources/test.robot']),
    ])
    def test_src_in_configuration(self, source, should_parse, test_data_dir):
        source_dir = test_data_dir / 'pyproject_with_src'
        os.chdir(source_dir)
        if source is not None:
            source = source_dir / source
            result = run_tidy([str(source)])
        else:
            result = run_tidy()
        expected = [f"Loaded configuration from {source_dir / 'pyproject.toml'}"]
        for file in should_parse:
            path = source_dir / file
            expected.append(f"Transforming {path} file")
        actual = sorted(line for line in result.output.split('\n') if line.strip())
        assert actual == sorted(expected)

    @pytest.mark.parametrize('source', [1, 2])
    def test_empty_configuration(self, source, test_data_dir):
        config_dir = test_data_dir / f'empty_pyproject{source}'
        os.chdir(config_dir)
        result = run_tidy(exit_code=1)
        assert "Loaded configuration from" not in result.output

    def test_loading_from_stdin(self):
        input_file = '*** Settings ***\nLibrary  SomeLib\n\n\n' \
                     '*** Variables ***\n\n\n\n' \
                     '*** Keywords ***\nKeyword\n    Keyword1 ${arg}\n'
        expected_output = '*** Settings ***\nLibrary  SomeLib\n\n\n' \
                          '*** Keywords ***\nKeyword\n    Keyword1 ${arg}\n\n'
        args = '--transform DiscardEmptySections -'.split()
        result = run_tidy(args, std_in=input_file)
        assert result.output == expected_output