import datetime
import io
import os
import json
import time
import typing

import iso8601
import nestedcontext
import requests
from cloud_blobstore import BlobNotFoundError, BlobStore, BlobStoreTimeoutError
from flask import jsonify, request

from dss.storage.identifiers import TombstoneID, BundleFQID, FileFQID
from dss.storage.blobstore import test_object_exists, ObjectTest
from dss.storage.bundles import get_bundle
from dss.util.version import datetime_to_version_format
from dss import DSSException, dss_handler
from dss.config import Config, Replica
from dss.storage.hcablobstore import BundleFileMetadata, BundleMetadata, FileMetadata

PUT_TIME_ALLOWANCE_SECONDS = 10
"""This is the minimum amount of time remaining on the lambda for us to retry on a PUT /bundles request."""

ADMIN_USER_EMAILS = set(os.environ['ADMIN_USER_EMAILS'].split(','))

@dss_handler
def get(uuid: str,
        replica: str,
        version: str=None,
        # TODO: (ttung) once we can run the endpoints from each cloud, we should default to the local cloud.
        directurls: bool=False):
    return get_bundle(uuid, Replica[replica], version, directurls)

@dss_handler
def list_versions(uuid: str):
    return ["2014-10-23T00:35:14.800221Z"]

@dss_handler
def post():
    pass

@dss_handler
def put(uuid: str, replica: str, json_request_body: dict, version: str = None):
    uuid = uuid.lower()
    if version is not None:
        # convert it to date-time so we can format exactly as the system requires (with microsecond precision)
        timestamp = iso8601.parse_date(version)
    else:
        timestamp = datetime.datetime.utcnow()
    version = datetime_to_version_format(timestamp)

    handle = Config.get_blobstore_handle(Replica[replica])
    bucket = Replica[replica].bucket

    # what's the target object name for the bundle manifest?
    bundle_manifest_key = BundleFQID(uuid=uuid, version=version).to_key()

    # decode the list of files.
    files = [{'user_supplied_metadata': file} for file in json_request_body['files']]

    time_left = nestedcontext.inject("time_left")

    while True:  # each time through the outer while-loop, we try to gather up all the file metadata.
        for file in files:
            user_supplied_metadata = file['user_supplied_metadata']
            metadata_key = FileFQID(
                uuid=user_supplied_metadata['uuid'],
                version=user_supplied_metadata['version'],
            ).to_key()
            if 'file_metadata' not in file:
                try:
                    file_metadata = handle.get(bucket, metadata_key)
                except BlobNotFoundError:
                    continue
                file['file_metadata'] = json.loads(file_metadata)

        # check to see if any file metadata is still not yet loaded.
        for file in files:
            if 'file_metadata' not in file:
                missing_file_user_metadata = file['user_supplied_metadata']
                break
        else:
            break

        # if we're out of time, give up.
        if time_left() > PUT_TIME_ALLOWANCE_SECONDS:
            time.sleep(1)
            continue

        raise DSSException(
            requests.codes.conflict,
            "file_missing",
            f"Could not find file {missing_file_user_metadata['uuid']}/{missing_file_user_metadata['version']}."
        )

    # build a manifest consisting of all the files.
    bundle_metadata = {
        BundleMetadata.FORMAT: BundleMetadata.FILE_FORMAT_VERSION,
        BundleMetadata.VERSION: version,
        BundleMetadata.FILES: [
            {
                BundleFileMetadata.NAME: file['user_supplied_metadata']['name'],
                BundleFileMetadata.UUID: file['user_supplied_metadata']['uuid'],
                BundleFileMetadata.VERSION: file['user_supplied_metadata']['version'],
                BundleFileMetadata.CONTENT_TYPE: file['file_metadata'][FileMetadata.CONTENT_TYPE],
                BundleFileMetadata.SIZE: file['file_metadata'][FileMetadata.SIZE],
                BundleFileMetadata.INDEXED: file['user_supplied_metadata']['indexed'],
                BundleFileMetadata.CRC32C: file['file_metadata'][FileMetadata.CRC32C],
                BundleFileMetadata.S3_ETAG: file['file_metadata'][FileMetadata.S3_ETAG],
                BundleFileMetadata.SHA1: file['file_metadata'][FileMetadata.SHA1],
                BundleFileMetadata.SHA256: file['file_metadata'][FileMetadata.SHA256],
            }
            for file in files
        ],
        BundleMetadata.CREATOR_UID: json_request_body['creator_uid'],
    }

    try:
        created, idempotent = _idempotent_save(
            handle,
            bucket,
            bundle_manifest_key,
            bundle_metadata,
        )
    except BlobStoreTimeoutError:
        raise DSSException(
            requests.codes.unavailable,
            "service_unavailable",
            f"Service unavailable due to unusually high load/latency"
        )

    if not idempotent:
        raise DSSException(
            requests.codes.conflict,
            "bundle_already_exists",
            f"bundle with UUID {uuid} and version {version} already exists"
        )
    status_code = requests.codes.created if created else requests.codes.ok

    return jsonify(dict(version=version)), status_code

@dss_handler
def delete(uuid: str, replica: str, json_request_body: dict, version: str=None):
    email = request.token_info['email']

    if email not in ADMIN_USER_EMAILS:
        raise DSSException(
            requests.codes.forbidden,
            "forbidden",
            f"You can't delete bundles with these credentials!",
        )

    uuid = uuid.lower()
    version = datetime_to_version_format(iso8601.parse_date(version)) if version else None

    tombstone_id = TombstoneID(uuid=uuid, version=version)
    bundle_prefix = tombstone_id.to_key_prefix()
    tombstone_object_data = _create_tombstone_data(
        email=email,
        reason=json_request_body.get('reason'),
        version=version,
    )

    handle = Config.get_blobstore_handle(Replica[replica])
    bucket = Replica[replica].bucket

    if test_object_exists(handle, bucket, bundle_prefix, test_type=ObjectTest.PREFIX):
        created, idempotent = _idempotent_save(
            handle,
            bucket,
            tombstone_id.to_key(),
            tombstone_object_data
        )
        if not idempotent:
            raise DSSException(
                requests.codes.conflict,
                f"bundle_tombstone_already_exists",
                f"bundle tombstone with UUID {uuid} and version {version} already exists",
            )
        status_code = requests.codes.ok
        response_body = dict()  # type: dict
    else:
        status_code = requests.codes.not_found
        response_body = dict(title="bundle not found")

    return jsonify(response_body), status_code

def _idempotent_save(blobstore: BlobStore, bucket: str, key: str, data: dict) -> typing.Tuple[bool, bool]:
    """
    _idempotent_save attempts to save an object to the BlobStore. Its return values indicate whether the save was made
    successfully and whether the operation could be completed idempotently. If the data in the blobstore does not match
    the data parameter, the data in the blobstore is _not_ overwritten.

    :param blobstore: the blobstore to save the data to
    :param bucket: the bucket in the blobstore to save the data to
    :param key: the key of the object to save
    :param data: the data to save
    :return: a tuple of booleans (was the data saved?, was the save idempotent?)
    """
    if test_object_exists(blobstore, bucket, key):
        # fetch the file metadata, compare it to what we have.
        existing_data = json.loads(blobstore.get(bucket, key).decode("utf-8"))
        return False, existing_data == data
    else:
        # write manifest to persistent store
        blobstore.upload_file_handle(
            bucket,
            key,
            io.BytesIO(json.dumps(data).encode("utf-8")),
        )
        return True, True

def _create_tombstone_data(email: str, reason: str, version: typing.Optional[str]) -> dict:
    # Future-proofing the case in which garbage collection is added
    data = dict(
        email=email,
        reason=reason,
        admin_deleted=True,
    )
    # optional params
    if version is not None:
        data.update(version=version)
    return data


import networkx as nx
import matplotlib.pyplot as plt

def plot_single_neuron(neuron, input):
    G = nx.DiGraph()
    middle_pos = -len(neuron.w)//2
    G.add_node("Σ", pos = (0, middle_pos) )

    for i in range(0, len(neuron.w)):
        input_label = "x"+ str(i+1) +" = " +str(round(input[i], 2))
        G.add_node(input_label, pos = (-1,-i-1))
        G.add_edges_from([(input_label, "Σ")])

    u = neuron.linear_combination(input)
    u_label = "u = "+ str(round(u,2))
    G.add_node(u_label, pos=(1, middle_pos))

    y = neuron.activation_function(u)
    y_label = "y = " + str(round(y,2))

    G.add_node("f", pos=(2, middle_pos))
    G.add_node(y_label, pos=(3, middle_pos))

    G.add_edges_from([
            ("Σ",u_label),
            (u_label, "f"),
            ("f", y_label),
            ])

    node_colors = []
    for node in G.nodes:
        if( node in ["Σ", "u", "f", "y"]):
            node_colors.append('lightblue')

        else:
            node_colors.append('pink')

    # # Add a neuron node
    # G.add_node("Neuron", pos=(0, 0))

    # # Add some synapses (connections)
    # G.add_node("Synapse 1", pos=(1, 1))
    # G.add_node("Synapse 2", pos=(-1, 1))
    # G.add_edges_from([("Neuron", "Synapse 1"), ("Neuron", "Synapse 2")])

    # Draw the graph
    pos = nx.get_node_attributes(G, 'pos')
    nx.draw(G, pos, with_labels=True, node_size=3000, node_color=node_colors, font_size=10, font_color='black', arrows = True)
    plt.show()


#!/usr/bin/env python

'''Fine-tune a language model for translation direction classification'''

import sys
import random as python_random
import argparse
import numpy as np
import torch
from sklearn.metrics import precision_recall_fscore_support, accuracy_score
from transformers import AutoTokenizer, AutoConfig, AutoModelForSequenceClassification, \
                         Trainer, TrainingArguments, EarlyStoppingCallback
sys.stdin.reconfigure(encoding='utf-8')
sys.stdout.reconfigure(encoding='utf-8')

def create_arg_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--train_file", required=True, type=str,
                        help="Input file to learn from")
    parser.add_argument("-d", "--dev_file", type=str, required=True,
                        help="Separate dev set to evaluate on")
    parser.add_argument("-of", "--output_folder", type=str, required=True,
                        help="Write output(s) to this folder")
    parser.add_argument("-c", "--cont", type=str, default='',
                        help="Continue training from this model folder")
    parser.add_argument("-lm", "--lm_ident", type=str, default="xlm-roberta-base",
                        help="Language model identifier (default XLM-R")
    parser.add_argument("-l", "--limit_train", default=0, type=int,
                        help="Limit training set to this many instances (default 0 means no limit)")
    # Arguments for training a model
    parser.add_argument("-pa", "--padding", default="max_length", type=str,
                        help="How to do the padding: max_length (default) or longest")
    parser.add_argument("-ml", "--max_length", default=None,
                        help="Max length of the inputs, usually not necessary to specify")
    parser.add_argument("-o", "--output_dir", type=str, required=True,
                        help="Where we save models and output files")
    parser.add_argument("-str", "--strategy", type=str, choices=["no", "steps", "epoch"],
                        default="epoch", help="Strategy for evaluating/saving/logging")
    parser.add_argument("-ss", "--save_strategy", type=str, choices=["no", "steps", "epoch"],
                        default="no", help="Strategy for saving models")
    parser.add_argument("-bs", "--batch_size", default=12, type=int,
                        help="Batch size per device")
    parser.add_argument("-p", "--patience", default=1, type=int,
                        help="Patience of earlystopping (default 1)")
    parser.add_argument("-lr", "--learning_rate", default=1e-5, type=float,
                        help="Learning rate")
    parser.add_argument("-wd", "--weight_decay", default=0, type=float,
                        help="Weight decay")
    parser.add_argument("-mgn", "--max_grad_norm", default=1, type=float,
                        help="Max grad norm")
    parser.add_argument("-ne", "--num_train_epochs", default=2, type=int,
                        help="Training epochs")
    parser.add_argument("-wr", "--warmup_ratio", default=0.1, type=float,
                        help="Warmup ratio (0 to 1)")
    parser.add_argument("-af", "--adafactor", action="store_true",
                        help="Use Adafactor instead of AdamW")
    parser.add_argument("-id", "--ignore_data_skip", action="store_true",
                        help="Ignore data skip when continuing training")
    parser.add_argument("-ls", "--label_smoothing", default=0.1, type=float,
                        help="Label smoothing percentage, 0-1")
    parser.add_argument("-es", "--eval_steps", default=500, type=int,
                        help="Evaluate after how many steps? Only works with -str steps")
    parser.add_argument("-eas", "--eval_accumulation_steps", default=250, type=int,
                        help="Number of steps to accumulate during evaluation")
    parser.add_argument("-gc", "--grad_check", action="store_true",
                        help="Use gradient checkpointing (slower, but more memory efficient)")
    parser.add_argument("-dr", "--dropout", default=0.1, type=float,
                        help="Dropout applied to the classifier layer")
    # Random seed is cmd line arg
    parser.add_argument("-s", "--seed", default=1234, type=int,
                        help="Random seed")
    args = parser.parse_args()
    # Make reproducible as much as possible
    np.random.seed(args.seed)
    python_random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    return args

def get_training_arguments(args, model_dir):
    '''Load all training arguments here. There are a lot more not specified, check:
    https://github.com/huggingface/transformers/blob/master/src/transformers/training_args.py#L72'''
    return TrainingArguments(model_dir, evaluation_strategy=args.strategy,
           eval_steps=args.eval_steps, per_device_train_batch_size=args.batch_size,
           per_device_eval_batch_size=args.batch_size, learning_rate=args.learning_rate,
           weight_decay=args.weight_decay, max_grad_norm=args.max_grad_norm,
           num_train_epochs=args.num_train_epochs, warmup_ratio=args.warmup_ratio,
           logging_strategy=args.strategy, save_strategy=args.save_strategy, seed=args.seed,
           load_best_model_at_end=True, label_smoothing_factor=args.label_smoothing,
           adafactor=args.adafactor, gradient_checkpointing=args.grad_check,
           eval_accumulation_steps=args.eval_accumulation_steps, log_level="debug",
           metric_for_best_model='accuracy', save_total_limit=1, ignore_data_skip=args.ignore_data_skip)

class DirectionDataset(torch.utils.data.Dataset):
    '''Dataset for using Transformers'''
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

def read_data(corpus_file, limit_train):
    '''Read in document and label (tab-separated)'''
    documents, labels = [], []
    for line in open(corpus_file, 'r', encoding="utf-8"):
        l = line.strip().split('\t')
        if len(l) != 3:
            raise ValueError(f"Data lines should contain 3 tabs, this has:\n {line.strip()}")
        documents.append([l[0], l[1]])
        labels.append(l[2])
    if limit_train:
        documents = documents[0:limit_train]
        labels = labels[0:limit_train]
    return documents, labels

def read_test_data(test_file):
    '''Read in test data: might not have labels, but could'''
    X_test, Y_test = [], []
    for idx, line in enumerate(open(test_file, 'r', encoding="utf-8")):
        # Check for first line if there are labels
        if idx == 0:
            has_labels = len(line.split('\t')) == 3
        if has_labels:
            text1, text2, label = line.split('\t')
            X_test.append([text1.strip(), text2.strip()])
            Y_test.append(label.strip())
        else:
            text1, text2 = line.split('\t')
            X_test.append([text1.strip(), text2.strip()])
    return X_test, Y_test

def compute_metrics(pred):
    '''Compute the metrics we are interested in'''
    labels = pred.label_ids
    # Sometimes the output is a tuple, take first argument then
    if isinstance(pred.predictions, tuple):
        pred = pred.predictions[0]
    else:
        pred = pred.predictions
    preds = pred.argmax(-1)
    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
    acc = accuracy_score(labels, preds)
    return {'accuracy': acc, 'f1': f1, 'precision': precision, 'recall': recall}

def get_uniq_labels(in_list):
    '''Get consistent list of labels from input'''
    keep = []
    for item in in_list:
        if item not in keep:
            keep.append(item)
    return keep

def process_data(in_file, limit_train, lm_ident, padding, max_length, uniq_labels):
    '''Process a data set to get ready for training/testing a model'''
    # Read in data
    if not uniq_labels:
        X_data, Y_data = read_data(in_file, limit_train)
    else:
        X_data, Y_data = read_test_data(in_file)

    # Tokenize the data here
    tokenizer = AutoTokenizer.from_pretrained(lm_ident)
    if max_length is not None:
        max_length = int(max_length)
    data_inputs = tokenizer(X_data, max_length=max_length, padding=padding, truncation=True)

    # Log the maximum input length
    print ("Maximum input length found: {0}\n".format(max([len(val) for val in data_inputs["input_ids"]])))

    # Determine the labels in an order we can get back if needed
    if not uniq_labels:
        uniq_labels = get_uniq_labels(Y_data)
        print (f"Labels:\n{', '.join(uniq_labels)}\n")

    # Convert labels to numbers to avoid errors
    Y_data = [uniq_labels.index(x) for x in Y_data]
    # Make it a dummy data set for the test set to avoid errors
    Y_data_use = [0 for _ in X_data] if not Y_data else Y_data

    # Make it a Dataset (necessary)
    data = DirectionDataset(data_inputs, Y_data_use)
    print (f"Loaded a dataset of {len(Y_data_use)} instances")
    return data, Y_data, uniq_labels

def main():
    '''Main function to train and test neural network given cmd line arguments'''
    args = create_arg_parser()
    # Read in data for training
    train_data, Y_train, uniq_labels = process_data(args.train_file, args.limit_train,
                                       args.lm_ident, args.padding, args.max_length, [])
    # Read in dev data
    dev_data, _, _ = process_data(args.dev_file, args.limit_train, args.lm_ident,
                                  args.padding, args.max_length, uniq_labels)

    # Setup variables
    num_labels = len(set(Y_train))
    model_dir = args.output_dir

    # Select model we will use
    config = AutoConfig.from_pretrained(args.lm_ident, classifier_dropout=args.dropout, num_labels=num_labels)
    model = AutoModelForSequenceClassification.from_pretrained(args.lm_ident, config=config)

    # Setup arguments training the model
    training_args = get_training_arguments(args, model_dir)

    # For logging purposes
    print("Generated by command:\npython", " ".join(sys.argv))
    print("Logging training settings\n", training_args)

    # Set EarlyStopping with patience of 1, since we evaluate each epoch, with a lot of data
    callbacks = [EarlyStoppingCallback(early_stopping_patience=args.patience)]
    trainer = Trainer(model=model, args=training_args, train_dataset=train_data,
                      eval_dataset=dev_data, compute_metrics=compute_metrics, callbacks=callbacks)

    # Actually train here, best model gets saved automatically
    if args.cont:
        print (f"Continue training from {args.cont}")
        mets = trainer.train(args.cont)
    else:
        mets = trainer.train()

    # Log the training info and we're done
    print("\nTraining info:\n", mets,"\n")

if __name__ == '__main__':
    main()


from barbados.factories.base import BaseFactory
from barbados.serializers import ObjectSerializer
from barbados.objects.resolution.summary import RecipeResolutionSummary
from uuid import uuid4, UUID
from barbados.objects.resolution.component import ComponentResolution
from barbados.factories.citation import CitationFactory
from barbados.factories.component import ComponentFactory
from barbados.models.reciperesolutionsummary import RecipeResolutionSummaryModel
from barbados.services.database import DatabaseService
from barbados.indexes.reciperesolutionindex import RecipeResolutionIndex
from barbados.factories.parser import FactoryParser

class RecipeResolutionFactory(BaseFactory):
    _model = RecipeResolutionSummaryModel
    _validator = None
    _index = RecipeResolutionIndex

    required_keys = {
        'alpha': str(),
        'citations': list(),
        'cocktail_slug': str(),
        'components': list(),
        'construction_slug': str(),
        'garnish': list(),
        'spec_slug': str(),
        'status_count': dict(),
    }

    unwanted_keys = [
        'component_count',
        'id',
        'status_count'
    ]

    @classmethod
    def raw_to_obj(cls, raw):
        raw_srs = cls.sanitize_raw(raw_input=raw, required_keys=cls.required_keys, unwanted_keys=cls.unwanted_keys)

        # Parse the fields
        raw_srs = cls._parse_inventory_id(raw_srs)
        raw_srs = cls._parse_components(raw_srs)
        raw_srs = cls._parse_garnish(raw_srs)
        raw_srs = cls._parse_citations(raw_srs)

        return RecipeResolutionSummary(**raw_srs)

    @staticmethod
    def _parse_inventory_id(raw_input):
        key = 'inventory_id'

        if type(raw_input.get(key)) is not UUID:
            raw_id = raw_input.get(key)
            raw_input.update({key: UUID(raw_id)})

        return raw_input

    @staticmethod
    def _parse_components(raw_input):
        key = 'components'

        objs = []
        for raw_component in raw_input.get(key):
            r = ComponentResolution(**raw_component)
            objs.append(r)
        raw_input.update({key: objs})

        return raw_input

    @staticmethod
    def _parse_garnish(raw_input):
        key = 'garnish'

        objs = ComponentFactory.raw_list_to_obj(raw_input.get(key))
        raw_input.update({key: objs})

        return raw_input

    @staticmethod
    def _parse_citations(raw_input):
        key = 'citations'

        objs = CitationFactory.raw_list_to_obj(raw_input.get(key))
        raw_input.update({key: objs})

        return raw_input

    @classmethod
    def obj_to_index(cls, obj, format='dict'):
        """
        Serialize an object into a standard form suitable
        for indexing.
        :param obj: Serializable object.
        :param format: Format of the data to pass to the serializer.
        :return: Instance of the Index class.
        """
        return cls._index(meta={'id': obj.id}, **ObjectSerializer.serialize(obj, format))

    @classmethod
    def index_to_obj(cls, indexable):
        raw_indexable = indexable.to_dict()
        return cls.raw_to_obj(raw_indexable)

    @classmethod
    def obj_to_model(cls, obj):
        raw_dict = ObjectSerializer.serialize(obj, 'dict')
        raw_dict.update({'id': obj.id})
        return cls._model(**raw_dict)

    @classmethod
    def from_objects(cls, inventory, cocktail, spec):
        return RecipeResolutionSummary(inventory_id=inventory.id, cocktail_slug=cocktail.slug, spec_slug=spec.slug,
                                       alpha=cocktail.alpha, citations=spec.citations + cocktail.citations,
                                       components=[], construction_slug=spec.construction.slug, garnish=spec.garnish)

    @classmethod
    def produce_all_objs_from_inventory(cls, inventory_id):
        """
        :return:
        """
        with DatabaseService.get_session() as session:
            results = session.query(RecipeResolutionSummaryModel).filter(RecipeResolutionSummaryModel.inventory_id==inventory_id)

            objects = []
            for result in results:
                obj = cls.model_to_obj(result)
                objects.append(obj)

        return objects


from time import sleep
from appium.webdriver.common.appiumby import AppiumBy
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException

class PageUtils():

    def __init__(self, driver) -> None:
        self.driver = driver

    def get_element(self, locator):
        method = locator[0]
        values = locator[1]

        if type(values) is str:
            return self.get_element_by_type(method, values)
        elif type(values) is list:
            for value in values:
                try:
                    return self.get_element_by_type(method, value)
                except NoSuchElementException:
                    pass
            raise NoSuchElementException

    def get_element_by_type(self, method, value):
        if method == "accessibility_id":
            return self.driver.find_element(AppiumBy.ACCESSIBILITY_ID, value)
        elif method == "android":
            return self.driver.find_element(AppiumBy.ANDROID_UIAUTOMATOR, 'new UiSelector().%s' % value)
        elif method == "ios":
            return self.driver.find_element(AppiumBy.IOS_UIAUTOMATION, value)
        elif method == "class_name":
            return self.driver.find_element(AppiumBy.CLASS_NAME, value)
        elif method == "id":
            return self.driver.find_element(AppiumBy.ID, value)
        elif method == "xpath":
            return self.driver.find_element(AppiumBy.XPATH, value)
        elif method == "name":
            return self.driver.find_element(AppiumBy.NAME, value)
        else:
            raise Exception('Invalid locator method.')

    def is_visible(self, locator):
        try:
            self.get_element(locator).is_displayed()
            return True
        except NoSuchElementException:
            return False

    def wait_visible(self, locator, timeout=10):
        i = 0
        while i != timeout:
            try:
                self.is_visible(locator)
                return self.get_element(locator)
            except NoSuchElementException:
                sleep(1)
                i += 1
        raise Exception('Element never became visible: %s (%s)' % (locator[0], locator[1]))

    def click(self, locator):
        element = self.wait_visible(locator)
        element.click()

    def tap(self, locator):
        element = self.wait_visible(locator)
        element.tap()

    def wait(self, locator, seconds=25):
        WebDriverWait(self.driver, seconds).until(EC.presence_of_element_located(locator))

    def get_text(self, locator):
        element = self.wait_visible(locator)
        return element.text


import logging

import numpy as np

from . import webservices

logger = logging.getLogger(__name__)

def get_polygons(**kwargs):
    """Get the location of the Waterboards as a Polygon GeoDataFrame."""
    url = "https://services.arcgis.com/nSZVuSZjHpEZZbRo/arcgis/rest/services/Waterschapsgrenzen/FeatureServer"
    layer = 0
    ws = webservices.arcrest(url, layer, **kwargs)
    # remove different prefixes
    ws["waterschap"] = ws["waterschap"].str.replace("HH van ", "")
    ws["waterschap"] = ws["waterschap"].str.replace("HHS van ", "")
    ws["waterschap"] = ws["waterschap"].str.replace("HH ", "")
    ws["waterschap"] = ws["waterschap"].str.replace("Waterschap ", "")
    ws["waterschap"] = ws["waterschap"].str.replace("Wetterskip ", "")
    ws = ws.set_index("waterschap")

    return ws

def get_configuration():
    """Get the configuration of of the data sources of the Waterboards."""
    config = {}

    config["Aa en Maas"] = {
        "bgt_code": "W0654",
        "watercourses": {
            "url": "https://gisservices.aaenmaas.nl/arcgis/rest/services/EXTERN/Oppervlaktewater_L/MapServer",
            "layer": 8,
            "bottom_width": "BODEMBREEDTE",
            "bottom_height": [["BODEMHOOGTE_BOS", "BODEMHOOGTE_BES"]],
        },
        "level_areas": {
            # "server_kind": "wfs",
            # "url": "https://maps.aaenmaas.nl/services/DAMO_S/wfs?",
            # "layer": "WS_PEILGEBIED",
            "url": "https://gisservices.aaenmaas.nl/arcgis/rest/services/EXTERN/Oppervlaktewater_B/MapServer",
            "layer": 5,
            "summer_stage": "ZOMERPEIL",
            "winter_stage": "WINTERPEIL",
        },
    }

    config["Amstel, Gooi en Vecht"] = {
        "bgt_code": "W0155",
        "watercourses": {
            "url": "https://maps.waternet.nl/arcgis/rest/services/AGV_Legger/AGV_Onderh_Secundaire_Watergangen/MapServer",
            "layer": 40,
            "bottom_width": "BODEMBREEDTE",
            "bottom_height": "BODEMHOOGTE",
            "water_depth": "WATERDIEPTE",
        },
        "level_areas": {
            "url": "https://maps.waternet.nl/arcgis/rest/services/AGV_Legger/Vastgestelde_Waterpeilen/MapServer",
            "layer": 0,
            "index": "GPGIDENT",
            "summer_stage": [
                "GPGZMRPL",
                "IWS_BOVENGRENS_ZOMERPEIL",
                "IWS_GPGVASTP",
            ],
            "winter_stage": [
                "GPGWNTPL",
                "OPVAFWWP",
                "IWS_GPGVASTP",
            ],
        },
    }

    config["Brabantse Delta"] = {
        "bgt_code": "W0652",
        "watercourses": {
            # legger
            "url": "https://geoservices.brabantsedelta.nl/arcgis/rest/services/EXTERN/WEB_Vastgestelde_Legger_Oppervlaktewaterlichamen/FeatureServer",
            "layer": 11,  # categorie A
            "bottom_width": "WS_BODEMBREEDTE_L",
            "bottom_height": [["WS_BH_BENEDENSTROOMS_L", "WS_BH_BOVENSTROOMS_L"]],
            # "layer": 12,  # categorie B
            # beheer
            # "url": "https://geoservices.brabantsedelta.nl/arcgis/rest/services/EXTERN/WEB_Beheerregister_Waterlopen_en_Kunstwerken/FeatureServer",
            # "layer": 13,  # categorie A
            # "layer": 14,  # categorie B
            # "layer": 15,  # categorie C
        },
        "level_areas": {
            "url": "https://geoservices.brabantsedelta.nl/arcgis/rest/services/EXTERN/WEB_Peilbesluiten/MapServer",
            "layer": 0,  # Peilgebied vigerend
            # "layer": 1, # Peilgebied praktijk
            "summer_stage": [
                "WS_ZOMERPEIL",
                "WS_VAST_PEIL",
                "WS_MAXIMUM",
            ],
            "winter_stage": [
                "WS_WINTERPEIL",
                "WS_VAST_PEIL",
                "WS_MINIMUM",
            ],
        },
    }

    config["De Dommel"] = {
        "bgt_code": "W0539",
        "watercourses": {
            "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/arcgis/rest/services/LW_2021_20211110/FeatureServer",
            "layer": 9,  # LOW_2021_A_Water
            "index": "LOKAALID",
            # "layer": 10,  # LOW_2021_A_Water_Afw_Afv
            # "layer": 11,  # LOW_2021_B_Water
            # "layer": 2,  # LOW_2021_Profielpunt
            # "layer": 13,  # LOW_2021_Profiellijn
            # "index": "WS_PROFIELID",
        },
    }

    config["De Stichtse Rijnlanden"] = {
        "bgt_code": "W0636",
        "watercourses": {
            "url": "https://services1.arcgis.com/1lWKHMyUIR3eKHKD/ArcGIS/rest/services/Keur_2020/FeatureServer",
            "layer": 39,  # Leggervak
            # "layer": 43, # Leggervak droge sloot
        },
        "level_areas": {
            "url": "https://geoservices.hdsr.nl/arcgis/rest/services/Extern/PeilbesluitenExtern/FeatureServer",
            "layer": 1,
            "index": "WS_PGID",
            "summer_stage": ["WS_ZP", "WS_BP", "WS_OP", "WS_VP"],
            "winter_stage": ["WS_WP", "WS_OP", "WS_BP", "WS_VP"],
        },
    }

    config["Delfland"] = {
        "bgt_code": "W0372",
        "watercourses": {
            "url": "https://services.arcgis.com/f6rHQPZpXXOzhDXU/arcgis/rest/services/Leggerkaart_Delfland_definitief/FeatureServer",
            "layer": 39,  # primair
            # "layer": 40,  # secundair
        },
        "level_areas": {
            "url": "https://services.arcgis.com/f6rHQPZpXXOzhDXU/arcgis/rest/services/Peilbesluiten2/FeatureServer",
            "summer_stage": "WS_HOOGPEIL",
            "winter_stage": "WS_LAAGPEIL",
        },
        "level_deviations": {
            "url": "https://services.arcgis.com/f6rHQPZpXXOzhDXU/arcgis/rest/services/Peilbesluiten2/FeatureServer",
            "layer": 2,
        },
    }

    config["Drents Overijsselse Delta"] = {
        "bgt_code": "W0664",
        "watercourses": {
            "url": "https://services6.arcgis.com/BZiPrSbS4NknjGsQ/arcgis/rest/services/Primaire_watergang_20_3_2018/FeatureServer",
            "index": "OVKIDENT",
        },
        "level_areas": {
            "url": "https://services6.arcgis.com/BZiPrSbS4NknjGsQ/arcgis/rest/services/Peilgebieden_opendata/FeatureServer",
            "index": "GPGIDENT",
            "summer_stage": "GPGZMRPL",
            "winter_stage": "GPGWNTPL",
        },
    }

    config["Fryslân"] = {
        "bgt_code": "W0653",
        "watercourses": {
            "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/BeheerregisterWaterlopen/MapServer",
            "layer": 0,  # # Wateren (primair, secundair)
            "index": "OVKIDENT",
            "bottom_height": "AVVBODH",
            "water_depth": "AVVDIEPT",
            # "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Legger_vastgesteld__2019/MapServer",
            # "layer": 604,  # Wateren legger
            # "index": "BLAEU.LEG_VL_GW_OVK.OVKIDENT",
            # "bottom_height": "BLAEU.LEG_VL_GW_OVK.AVVBODH",
        },
        "level_areas": {
            # "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Peilbelsuit_Friese_boezem/MapServer",
            # "index": "BLAEU_WFG_GPG_BEHEER_PBHIDENT",
            "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Peilen/MapServer",
            "layer": 1,  # PeilenPeilenbeheerkaart - Peilen
            "index": "PBHIDENT",
            # "layer": 4,  # Peilbesluitenkaart
            # "index": "GPGIDENT",
            "summer_stage": "HOOGPEIL",
            "winter_stage": "LAAGPEIL",
        },
    }

    config["Hollands Noorderkwartier"] = {
        "bgt_code": "W0651",
        "watercourses": {
            "url": "https://kaarten.hhnk.nl/arcgis/rest/services/od_legger/od_legger_wateren_2022_oppervlaktewateren_ti/MapServer",
            "bottom_height": "WS_BODEMHOOGTE",
        },
        "level_areas": {
            "url": "https://kaarten.hhnk.nl/arcgis/rest/services/NHFLO/Peilgebied_beheerregister/MapServer",
            "summer_stage": [
                "ZOMER",
                "STREEFPEIL_ZOMER",
                "BOVENGRENS_JAARROND",
                "ONDERGRENS_JAARROND",
                "VAST",
                "STREEFPEIL_JAARROND",
            ],
            "winter_stage": [
                "WINTER",
                "STREEFPEIL_WINTER",
                "ONDERGRENS_JAARROND",
                "BOVENGRENS_JAARROND",
                "VAST",
                "STREEFPEIL_JAARROND",
            ],
        },
        "level_deviations": {
            "url": "https://kaarten.hhnk.nl/arcgis/rest/services/NHFLO/Peilafwijking_gebied/MapServer"
        },
    }

    config["Hollandse Delta"] = {
        "bgt_code": "W0655",
        "watercourses": {
            # "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Geoportaal/Legger2014waterbeheersing_F_transparant/FeatureServer",
            "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Watersysteem/MapServer",
            "layer": 15,  # Waterlopen
            "bottom_height": [
                ["BODEMHOOGTE_BOVENSTROOMS", "BODEMHOOGTE_BENEDENSTROOMS"]
            ],
            "bottom_width": "BODEMBREEDTE",
        },
        "level_areas": {
            "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Peilgebieden/MapServer",
            "layer": 31,
            "f": "json",
            "summer_stage": [
                "REKENPEIL_ZOMER",
                "BOVENGRENS_BEHEERMARGE_ZOMER",
                "ONDERGRENS_BEHEERMARGE_ZOMER",
            ],
            "winter_stage": [
                "REKENPEIL_WINTER",
                "BOVENGRENS_BEHEERMARGE_WINTER",
                "ONDERGRENS_BEHEERMARGE_WINTER",
            ],
        },
    }

    config["Hunze en Aa's"] = {
        "bgt_code": "W0646",
    }

    config["Limburg"] = {
        "bgt_code": "W0665",
        "watercourses": {
            # "url": "https://maps.waterschaplimburg.nl/arcgis/rest/services/Legger/Leggerwfs/MapServer",
            # "layer": 1,  # primair
            # "layer": 2, # secundair
            "url": "https://maps.waterschaplimburg.nl/arcgis/rest/services/Legger/Legger/MapServer",
            "layer": 22,  # primair
            # "layer": 23,  # secunair
            # "layer": 24,  # Waterplas
        },
    }

    config["Noorderzijlvest"] = {
        "bgt_code": "W0647",
        "watercourses": {
            "url": "https://arcgis.noorderzijlvest.nl/server/rest/services/Legger/Legger_Watergangen_2012/MapServer",
            "index": "OVKIDENT",
        },
        "level_areas": {
            "url": "https://arcgis.noorderzijlvest.nl/server/rest/services/Peilbeheer/Peilgebieden/MapServer",
            "layer": 3,
            "index": "GPGIDENT",
            "summer_stage": "OPVAFWZP",
            "winter_stage": "OPVAFWWP",
        },
    }

    config["Rijn en IJssel"] = {
        "bgt_code": "W0152",
        "watercourses": {
            "url": "https://opengeo.wrij.nl/arcgis/rest/services/VigerendeLegger/MapServer",
            # "layer": 12,
            # "index": "OWAIDENT",
            # "layer": 11,
            # "index": "OBJECTID",
            "layer": 10,
            "index": "OVKIDENT",
            # "f": "json",
            "bottom_height": [["IWS_AVVHOBOS_L", "IWS_AVVHOBES_L"]],
            "bottom_width": "AVVBODDR",
        },
        "culverts": {
            "url": "https://opengeo.wrij.nl/arcgis/rest/services/VigerendeLegger/MapServer",
            "layer": 7,
            "index": "KDUIDENT",
            "bottom_height": [["KDUBOKBO_L", "KDUBOKBE_L"]],
            "bottom_width": "KDUBREED_L",
        },
        "weirs": {
            "url": "https://opengeo.wrij.nl/arcgis/rest/services/VigerendeLegger/MapServer",
            "layer": 5,
            "index": "KSTIDENT",
            "summer_stage": "IWS_KSTZMRPL",
            "winter_stage": "IWS_KSTWTPL",
        },
    }

    config["Rijnland"] = {
        "bgt_code": "W0616",
        "watercourses": {
            "url": "https://rijnland.enl-mcs.nl/arcgis/rest/services/Leggers/Legger_Oppervlaktewater_Vigerend/MapServer",
            "layer": 1,
            "water_depth": "WATERDIEPTE",
        },
        "level_areas": {
            "url": "https://rijnland.enl-mcs.nl/arcgis/rest/services/Peilgebied_vigerend_besluit/MapServer",
            "summer_stage": [
                "ZOMERPEIL",
                "VASTPEIL",
                "FLEXZOMERPEILBOVENGRENS",
            ],
            "winter_stage": [
                "WINTERPEIL",
                "VASTPEIL",
                "FLEXWINTERPEILBOVENGRENS",
            ],
        },
        "level_deviations": {
            "url": "https://rijnland.enl-mcs.nl/arcgis/rest/services/Peilafwijking_praktijk/MapServer"
        },
    }

    config["Rivierenland"] = {
        "bgt_code": "W0621",
        "watercourses": {
            "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/WatersysteemLeggerVastgesteld/MapServer",
            # "layer": 13,  # profiellijn
            "layer": 14,  # waterloop
            "index": "code",
        },
        "level_areas": {
            "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/Peilgebieden_praktijk/FeatureServer",
            # "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/Peilgebieden_vigerend/FeatureServer",
            "summer_stage": [
                "ZOMERPEIL",
                "MIN_PEIL",
                "STREEFPEIL",
                "VASTPEIL",
            ],
            "winter_stage": [
                "WINTERPEIL",
                "MAX_PEIL",
                "STREEFPEIL",
                "VASTPEIL",
            ],
        },
    }

    config["Scheldestromen"] = {
        "bgt_code": "W0661",
        "watercourses": {
            "url": "http://geo.scheldestromen.nl/arcgis/rest/services/Extern/EXT_WB_Legger_Oppervlaktewaterlichamen_Vastgesteld/MapServer",
            "layer": 6,
            "index": "OAFIDENT",
            "bottom_height": "OAFBODHG",
            "bottom_width": "OAFBODBR",
        },
        "level_areas": {
            "url": "http://geo.scheldestromen.nl/arcgis/rest/services/Extern/EXT_WB_Waterbeheer/FeatureServer",
            "layer": 14,  # Peilgebieden (praktijk)
            "index": "GPGIDENT",
            # "layer": 15,  # Peilgebieden (juridisch)
            # "index": "GJPIDENT",
            "f": "json",  # geojson does not return GPGZP and GPGWP
            "summer_stage": "GPGZP",
            "winter_stage": "GPGWP",
            "nan_values": [-99, 99],
        },
    }

    config["Schieland en de Krimpenerwaard"] = {
        "bgt_code": "W0656",
        "watercourses": {
            "url": "https://services.arcgis.com/OnnVX2wGkBfflKqu/arcgis/rest/services/HHSK_Legger_Watersysteem/FeatureServer",
            "layer": 11,  # Hoofdwatergang
            # "layer": 12,  # Overig Water
            "water_depth": "DIEPTE",
        },
        "level_areas": {
            # "url": "https://services.arcgis.com/OnnVX2wGkBfflKqu/ArcGIS/rest/services/Peilbesluiten/FeatureServer",
            "url": "https://services.arcgis.com/OnnVX2wGkBfflKqu/ArcGIS/rest/services/VigerendePeilgebiedenEnPeilafwijkingen_HHSK/FeatureServer",
            "summer_stage": ["BOVENPEIL", "VASTPEIL"],
            "winter_stage": ["ONDERPEIL", "VASTPEIL"],
            "nan_values": 9999,
        },
        "level_deviations": {
            "url": "https://services.arcgis.com/OnnVX2wGkBfflKqu/ArcGIS/rest/services/VigerendePeilgebiedenEnPeilafwijkingen_HHSK/FeatureServer",
            "layer": 1,
        },
    }

    config["Vallei & Veluwe"] = {
        "bgt_code": "W0662",
        "watercourses": {
            "url": "https://services1.arcgis.com/ug8NBKcLHVNmdmdt/ArcGIS/rest/services/Legger_Watersysteem/FeatureServer",
            "layer": 16,  # A-water
            # "layer": 17,  # B-water
            # "layer": 18, # A-water
        },
        "level_areas": {
            "url": "https://services1.arcgis.com/ug8NBKcLHVNmdmdt/arcgis/rest/services/Peilvakken/FeatureServer",
            "summer_stage": "WS_MAX_PEIL",
            "winter_stage": "WS_MIN_PEIL",
            "nan_values": 999,
        },
    }

    config["Vechtstromen"] = {
        "bgt_code": "W0663",
        "watercourses": {
            "url": "https://services1.arcgis.com/3RkP6F5u2r7jKHC9/arcgis/rest/services/Legger_publiek_Vastgesteld_Openbaar/FeatureServer",
            "layer": 11,
            "index": "OVKIDENT",
            "bottom_height": [["IWS_AVVHOBES_L", "IWS_AVVHOBOS_L"]],
            "bottom_width": "AVVBODDR",
        },
        "level_areas": {
            "url": "https://services1.arcgis.com/3RkP6F5u2r7jKHC9/arcgis/rest/services/WBP_Peilen/FeatureServer",
            "layer": 0,  # Peilgebieden voormalig Velt en Vecht
            "index": "GPG_ID",
            "summer_stage": "GPGZMRPL",
            "winter_stage": "GPGWNTPL",
            "nan_values": 0,
            # "layer": 1,  # Peilregister voormalig Regge en Dinkel
            # "index": None,
        },
        "weirs": {
            "url": "https://services1.arcgis.com/3RkP6F5u2r7jKHC9/arcgis/rest/services/Legger_publiek_Vastgesteld_Openbaar/FeatureServer",
            "layer": 5,
            "index": "KSTIDENT",
            "summer_stage": "IWS_KSTZMRPL",
            "winter_stage": "IWS_KSTWTPL",
        },
        "culverts": {
            "url": "https://services1.arcgis.com/3RkP6F5u2r7jKHC9/arcgis/rest/services/Legger_publiek_Vastgesteld_Openbaar/FeatureServer",
            "layer": 8,
            "index": "KDUIDENT",
            "bottom_height": [["KDUBHBO", "KDUBHBE"]],
            "bottom_width": "KDUBREED_L",
        },
    }

    config["Zuiderzeeland"] = {
        "bgt_code": "W0650",
        "watercourses": {
            # "url": "https://services.arcgis.com/84oM5NriBghHdQ3Z/ArcGIS/rest/services/leggerkavelsloten/FeatureServer",
            "url": "https://services.arcgis.com/84oM5NriBghHdQ3Z/arcgis/rest/services/legger_concept/FeatureServer",
            "layer": 12,  # Profiel (lijnen)
            # "layer": 13,  # Oppervlaktewater (vlakken)
            "index": "IDENT",
        },
        "level_areas": {
            "url": "https://services.arcgis.com/84oM5NriBghHdQ3Z/arcgis/rest/services/zzl_Peilgebieden/FeatureServer",
            "index": "GPGIDENT",
            "summer_stage": "GPGZMRPL",
            "winter_stage": "GPGWNTPL",
            "nan_values": -999,
        },
    }

    return config

def get_data(wb, data_kind, extent=None, max_record_count=None, config=None, **kwargs):
    """Get the data for a Waterboard and a specific data_kind.

    Parameters
    ----------
    ws : str
        The name of the waterboard.
    data_kind : str
        The kind of data you like to download. Possible values are
        'watercourses', 'level_areas' and 'level_deviations'
    extent : tuple or list of length 4, optional
        THe extent of the data you like to donload: (xmin, xmax, ymin, ymax).
        Download everything when extent is None. The default is None.
    max_record_count : int, optional
        THe maximum number of records that are downloaded in each call to the
        webservice. When max_record_count is None, the maximum is set equal to
        the maximum of the server. The default is None.
    config : dict, optional
        A dictionary with properties of the data sources of the Waterboards.
        When None, the configuration is retreived from the method
        get_configuration(). The default is None.
    **kwargs : dict
        Optional arguments which are passed onto arcrest() or wfs().

    Raises
    ------

        DESCRIPTION.

    Returns
    -------
    gdf : GeoDataFrame
        A GeoDataFrame containing data from the waterboard (polygons for
        level_areas/level_deviations and lines for watercourses).
    """
    if config is None:
        config = get_configuration()
    # some default values
    layer = 0
    index = "CODE"
    server_kind = "arcrest"
    f = "geojson"

    if wb not in config:
        raise (Exception(f"No configuration available for {wb}"))
    if data_kind not in config[wb]:
        raise (Exception(f"{data_kind} not available for {wb}"))
    conf = config[wb][data_kind]
    url = conf["url"]
    if "layer" in conf:
        layer = conf["layer"]
    if "index" in conf:
        index = conf["index"]
    if "server_kind" in conf:
        server_kind = conf["server_kind"]
    if "f" in conf:
        f = conf["f"]

    # % download and plot data
    if server_kind == "arcrest":
        gdf = webservices.arcrest(
            url,
            layer,
            extent,
            f=f,
            max_record_count=max_record_count,
            **kwargs,
        )
    elif server_kind == "wfs":
        gdf = webservices.wfs(
            url, layer, extent, max_record_count=max_record_count, **kwargs
        )
    else:
        raise (Exception("Unknown server-kind: {server_kind}"))
    if len(gdf) == 0:
        return gdf

    # set index
    if index is not None:
        if index not in gdf:
            logger.warning(f"Cannot find {index} in {data_kind} of {wb}")
        else:
            gdf = gdf.set_index(index)

    # set a value for summer_stage and winter_stage or bottom_height and water_depth
    if data_kind in ["level_areas", "weirs"]:
        set_columns = ["summer_stage", "winter_stage"]
    elif data_kind in ["watercourses", "culverts"]:
        set_columns = ["bottom_height", "water_depth"]
    else:
        set_columns = []
    nan_values = None
    if "nan_values" in conf:
        nan_values = conf["nan_values"]

    for set_column in set_columns:
        from_columns = conf[set_column] if set_column in conf else []
        gdf = _set_column_from_columns(
            gdf, set_column, from_columns, nan_values=nan_values
        )
    return gdf

def _set_column_from_columns(gdf, set_column, from_columns, nan_values=None):
    """Retrieve values from one or more Geo)DataFrame-columns and set these
    values as another column."""
    if set_column in gdf.columns:
        raise (Exception(f"Column {set_column} allready exists"))
    gdf[set_column] = np.NaN
    if from_columns is None:
        return gdf
    if isinstance(from_columns, str):
        from_columns = [from_columns]
    for from_column in from_columns:
        msg = "Cannot find column {} as source for {}"
        if isinstance(from_column, list):
            mask = [x not in gdf for x in from_column]
            if np.any(mask):
                for i in np.where(mask)[0]:
                    logger.warning(msg.format(from_column[i], set_column))
                from_column = list(np.array(from_column)[~np.array(mask)])
                if len(from_column) == 0:
                    continue
        else:
            if from_column not in gdf:
                logger.warning(msg.format(from_column, set_column))
                continue
        mask = gdf[set_column].isna()
        if not mask.any():
            break

        if isinstance(from_column, list):
            mask = mask & ~gdf[from_column].isna().all(1)
            if not mask.any():
                continue
            gdf.loc[mask, set_column] = gdf.loc[mask, from_column].mean(1)
        else:
            mask = mask & ~gdf[from_column].isna()
            if not mask.any():
                continue
            gdf.loc[mask, set_column] = gdf.loc[mask, from_column]
        if nan_values is not None:
            if isinstance(nan_values, (float, int)):
                nan_values = [nan_values]
            gdf.loc[gdf[set_column].isin(nan_values), set_column] = np.NaN
    return gdf


from pymongo import MongoClient
from functools import reduce
from collections import defaultdict
import time
import json

def get_negative_recall(round_id, winner):
    actions = list(db['actions'].find({'round_id': round_id, 'player_name': winner}))
    edge_set = set()
    cog_itr = db['cogs'].find({'round_id': round_id})
    cog_list = []
    recall_count = 0
    total_count = 0
    actions.sort(key=lambda dd: dd['time'])
    for a_iter, a in enumerate(actions):
        if not a_iter % 10:
            print(a_iter)
        for e, d in a['links_size'].items():
            if d['size'] > 0:
                edge_set.add(e)
            else:
                if e in edge_set:
                    edge_set.remove(e)
        if not a['is_hint']:
            continue
        for cog in cog_itr:
            cog_list.append(cog)
            if cog['time'] > a['time']:
                break
        cog = cog_list[-2]
        error_set = set()
        for e, d in cog['edges_saved'].items():
            wp, wn = float(d['wp']), float(d['wn'])
            if not (wp + wn) or wp / (wp + wn) > 0.5:
                continue
            error_set.add(e)
        for e in edge_set:
            l, r = e.split('-')
            x, tag, y = int(l[:-1]), 'LR' if r[0] == 'R' else 'TB', int(r[1:])
            if tag == 'LR':
                if not (y % 10 and x + 1 == y):
                    total_count += 1
                    if e in error_set:
                        recall_count += 1
            else:
                if not (x + 10 == y):
                    total_count += 1
                    if e in error_set:
                        recall_count += 1
    return recall_count, total_count, (recall_count / total_count) if total_count else 1

def get_negative_precision(round_id, winner):
    actions = list(db['actions'].find({'round_id': round_id, 'player_name': winner}))
    edge_set = set()
    cog_itr = db['cogs'].find({'round_id': round_id})
    cog_list = []
    feedback_count = 0
    correct_count = 0
    actions.sort(key=lambda dd: dd['time'])
    for a in actions:
        for e, d in a['links_size'].items():
            if d['size'] > 0:
                edge_set.add(e)
            else:
                if e in edge_set:
                    edge_set.remove(e)
        if not a['is_hint']:
            continue
        for cog in cog_itr:
            cog_list.append(cog)
            if cog['time'] > a['time']:
                break
        cog = cog_list[-2]
        for e, d in cog['edges_saved'].items():
            wp, wn = float(d['wp']), float(d['wn'])
            if not (wp + wn) or wp / (wp + wn) > 0.5:
                continue
            if e in edge_set:
                feedback_count += 1
                l, r = e.split('-')
                x, tag, y = int(l[:-1]), 'LR' if r[0] == 'R' else 'TB', int(r[1:])
                if tag == 'LR':
                    if y % 10 and x + 1 == y:
                        correct_count += 1
                else:
                    if x + 10 == y:
                        correct_count += 1
    return correct_count, feedback_count, ((feedback_count - correct_count) / feedback_count) if feedback_count else 1

def parse_time(time_str, join_time, start_time, end_time):
    join_time = int(time.mktime(time.strptime(':'.join(join_time.split(':')[:-1]), '%Y-%m-%d %H:%M:%S'))) \
        if join_time else None
    start_time = int(start_time) // 1000 if start_time else None
    end_time = int(time.mktime(time.strptime(':'.join(end_time.split(':')[:-1]), '%Y-%m-%d %H:%M:%S')))
    return min(reduce(lambda t1, t2: 60 * t1 + t2, map(int, time_str.split(':'))),
               end_time - start_time if start_time else (end_time - join_time if join_time else None))

def get_solve_time(record):
    return parse_time(record['time'], record['join_time'] if 'join_time' in record else None,record['start_time'],
                      record['end_time'])

mongo_ip = "162.105.89.173"
mongo_port = 27017

client = MongoClient(mongo_ip, mongo_port)
db = client.CrowdJigsaw

official_rounds = list(db["rounds"].find({
    'round_id': {'$lt': 500},
    'official': True,
    'algorithm': 'central',
    'solved_players': {'$gt': 1},
}))
print([r['round_id'] for r in official_rounds])

with open('negative_feedback.csv', 'w+') as f:
    f.write("img,round_id,players_num,solve_time,hit_error,total_error,recall\n")
    ga_rounds = dict()
    for r in official_rounds:
        round_id = r['round_id']
        tpr, tpc = int(r['tilesPerRow']), int(r['tilesPerColumn'])
        if tpr < 10 and tpc < 10:
            continue
        img = r['image'].split('/')[-1].split('_')[0]
        records = list(db['records'].find({'round_id': round_id, 'end_time': {'$ne': '-1'}}))
        if not records:
            continue
        has_genetic = False
        for rr in records:
            if rr['username'].startswith('GeneticAlgorithm'):
                has_genetic = True
            rr['solve_time'] = get_solve_time(rr)
        if not has_genetic:
            continue
        records.sort(key=lambda r: r['solve_time'])
        print(round_id, records[0]['username'], records[1]['username'] if len(records) > 1 else '')
        winner = records[1] if len(records) > 1 and records[0]['username'].startswith('GeneticAlgorithm') else records[0]
        players_num = len(set(record['username'] for record in records if int(record['steps']) > 10))
        r['winner'] = winner
        ga_rounds[round_id] = r
        correct, total, precision = get_negative_precision(round_id, winner['username'])
        output = '%s,%s,%s,%s,%d,%d,%f' % (img, round_id, players_num, winner['solve_time'],
                                             correct, total, precision)
        print(output)
        f.write('%s\n' % output)


from pydantic import BaseModel
from typing import List, Optional, Union
from nodes.models.operation import AddOp, RemoveOp, ModifyOp
from enum import Enum

class BaseQuery(BaseModel):
    page_id: str

class GetPageVersionRequest(BaseQuery):
    pass

class GetPageVersionResponse(BaseQuery):
    version: Optional[str] = None

class Meta(BaseModel):
    page_id: str
    page_name: str

class UpdatePageRequest(BaseQuery):
    operations: List[Union[AddOp, RemoveOp, ModifyOp]]
    root_hash: str
    meta: Meta
    prev_version: Optional[str] = None

class Status(Enum):
    OK = 'ok'
    ERROR = 'error'

class UpdatePageResponse(BaseModel):
    status: Status


from django.contrib import admin
from django.urls import path
from Home import views

urlpatterns = [

    path('',views.home,name='home'),
    path('Home',views.home,name='home'),
    path('Registration',views.Registration,name='registration'),
    path('About',views.about,name='about'),
    path('Players',views.players,name='players'),
    path('Playerlist',views.playerlist,name='playerlist'),
    path('News',views.News,name='news'),
]


from collections.abc import Mapping, Sequence
from enum import Enum
from http import HTTPStatus
from typing import Any

from apiclient import discovery
from backoff import on_exception, expo
from google.oauth2 import service_account
from googleapiclient.errors import HttpError
from token_bucket import MemoryStorage

from .decorator import token_bucket
from .limiter import BurstLimiter

class GoogleSheetsClient:
    SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]

    class ValueInputOption(Enum):
        RAW = "RAW"
        USER_ENTERED = "USER_ENTERED"

    def __init__(self, googleCredentials: Mapping[str, str]) -> None:
        self.service = discovery.build(
            "sheets",
            "v4",
            credentials=service_account.Credentials.from_service_account_info(
                googleCredentials, scopes=GoogleSheetsClient.SCOPES
            ),
        ).spreadsheets()

    @on_exception(
        expo,
        exception=HttpError,
        giveup=lambda e: not isinstance(e, HttpError)
        or e.status_code != HTTPStatus.TOO_MANY_REQUESTS,
        max_tries=10,
    )
    def get_values(self, spreadsheetId: str, range: str = ""):
        return (
            self.service.values()
            .get(spreadsheetId=spreadsheetId, range=range)
            .execute()["values"]
        )

    @on_exception(
        expo,
        exception=HttpError,
        giveup=lambda e: not isinstance(e, HttpError)
        or e.status_code != HTTPStatus.TOO_MANY_REQUESTS,
        max_tries=10,
    )
    def get(self, spreadsheetId: str, ranges: Sequence[str] = [], fields: str = ""):
        return self.service.get(
            spreadsheetId=spreadsheetId, ranges=ranges, fields=fields
        ).execute()

    @on_exception(
        expo,
        exception=HttpError,
        giveup=lambda e: not isinstance(e, HttpError)
        or e.status_code != HTTPStatus.TOO_MANY_REQUESTS,
        max_tries=10,
    )
    def append_values(
        self,
        spreadsheetId: str,
        range: str = "",
        values: Sequence[Sequence[str]] = [],
        valueInputOption: ValueInputOption = ValueInputOption.RAW,
    ):
        self.service.values().append(
            spreadsheetId=spreadsheetId,
            range=range,
            body={
                "values": values,
            },
            valueInputOption=valueInputOption.name,
        ).execute()

    @on_exception(
        expo,
        exception=HttpError,
        giveup=lambda e: not isinstance(e, HttpError)
        or e.status_code != HTTPStatus.TOO_MANY_REQUESTS,
        max_tries=10,
    )
    def batch_update(self, spreadsheetId: str, requests: Sequence[Mapping[str, Any]]):
        self.service.batchUpdate(
            spreadsheetId=spreadsheetId, body={"requests": requests}
        ).execute()

    def get_row(self, cellRef: str) -> int:
        for i in range(len(cellRef)):
            try:
                return int(cellRef[i:])
            except ValueError:
                continue

        return -1

    def get_row_idx(self, cellRef: str) -> int:
        row = self.get_row(cellRef)
        if row == -1:
            return -1

        return row - 1

    def get_column(self, cellRef: str) -> str:
        row = self.get_row(cellRef)
        if row == -1:
            return cellRef

        return cellRef[: len(cellRef) - len(str(row))]

    def get_column_idx(self, cellRef: str) -> int:
        col = self.get_column(cellRef)
        ret = -1
        for i, c in enumerate(col[::-1]):
            cVal = ord(c) - ord("A") + 1
            ret += cVal * (26**i)

        return ret

    def color_to_hex(self, color: Mapping[str, int]) -> str:
        r, g, b = map(
            lambda colorComponent: round(color[colorComponent] * 255)
            if colorComponent in color
            else 0,
            ["red", "green", "blue"],
        )
        return f"{r:02x}{g:02x}{b:02x}"

    def is_white(self, color: Mapping[str, int]) -> bool:
        return self.color_to_hex(color).upper() == "FFFFFF"

class RateLimitedGoogleSheetsClient(GoogleSheetsClient):
    def __init__(self, googleCredentials: Mapping[str, str]) -> None:
        super().__init__(googleCredentials)

        self.bucket = BurstLimiter(
            rate=1, capacity=1, initialCapacity=60, storage=MemoryStorage()
        )

    @token_bucket("read", 1)
    def get_values(self, spreadsheetId: str, range: str = ""):
        return super().get_values(spreadsheetId, range)

    @token_bucket("read", 1)
    def get(self, spreadsheetId: str, ranges: Sequence[str] = [], fields: str = ""):
        return super().get(spreadsheetId, ranges, fields)

    @token_bucket("write", 1)
    def append_values(
        self,
        spreadsheetId: str,
        range: str = "",
        values: Sequence[Sequence[str]] = [],
        valueInputOption: GoogleSheetsClient.ValueInputOption = GoogleSheetsClient.ValueInputOption.RAW,
    ):
        return super().append_values(spreadsheetId, range, values, valueInputOption)

    @token_bucket("write", 1)
    def batch_update(self, spreadsheetId: str, requests: Sequence[Mapping[str, Any]]):
        return super().batch_update(spreadsheetId, requests)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config.from_pyfile('config.py') #import configs
db = SQLAlchemy(app) #comandos pra db
app.config["SESSION_PERMANENT"] = False
app.config["SESSION_TYPE"] = "filesystem"
app.config['SESSION_COOKIE_HTTPONLY'] = False

from views import * #importa views

# trecho da app
if __name__ == '__main__':
    app.run(host='0.0.0.0', debug = True)


from .models import *
from screening.models import ScreeningStatus, PubmedImportedArticle
from django.shortcuts import get_object_or_404, render, redirect
from django.utils.safestring import mark_safe
from django.urls import reverse
from random import randint
from screening.utils.highlighter import LitrevHighlighter
import numpy as np
from .forms import LandmarkPaperForm

from base.utils import build_csv_response, build_mendeley_xml_response

def __highlight_text(text, highlights, css_class):
    # keywords = list()
    # [keywords.append(h.text) for h in highlights]
    # query = ' '.join(highlights)
    highlighter = LitrevHighlighter(highlights, css_class=css_class, max_length=10000)
    return highlighter.highlight(text)

def index(request, id=-1):
    if id is not -1:
        article = ScreeningStatus.objects.get(article_id=id)
    else:
        included_articles = ScreeningStatus.objects.filter(decision='Y', article__tagged=False)

        if len(included_articles) <= 0:
            return render(request, 'tagging/empty.html')

        idx = randint(0, len(included_articles)-1)
        article = included_articles[idx]

    highlights = Highlight.objects.all()
    for h in highlights.iterator():
        article.article.abstract = __highlight_text(article.article.abstract, h.keyword, h.css_class)
        article.article.title = __highlight_text(article.article.title, h.keyword, h.css_class)

    article.article.abstract = article.article.abstract[1:]
    article.article.abstract = article.article.abstract[:-1]

    tags = Tag.objects.all()
    tag_group = TagGroup.objects.filter(enabled=True)

    available_tags = list()

    for tg in tag_group:
        tags_in_group = tg.tag_set.all()
        group_tags = {
            'group_name': tg.group_name,
            'tags': list()
        }
        for t in tags_in_group:
            article_tags = ArticleTag.objects.filter(article_id = article.article_id, tag_id = t.id)
            group_tags['tags'].append({
                'tag_id': t.id,
                'tag_name': t.tag_name,
                'tagged': len(article_tags) > 0
            })
        available_tags.append(group_tags)

    tagged_articles = ArticleTag.objects.values_list('article_id')

    num_tagged_articles = np.count_nonzero(np.unique(np.array(tagged_articles)))
    num_articles = ScreeningStatus.objects.filter(decision='Y').count()
    progress = int((num_tagged_articles / num_articles) * 100)

    data = {
        'article': article.article,
        'tags': tags,
        'available_tags': available_tags,
        'progress': progress,
        'articles_total': num_articles,
        'articles_tagged': num_tagged_articles
    }

    return render(request, 'tagging/index.html', data)

def view_tags(request, article_id=0):
    # tagged_articles = PubmedImportedArticle.objects.filter(tagged=True).order_by('pmid')
    # article = tagged_articles[article_id]
    article = PubmedImportedArticle.objects.get(pmid=article_id)

    tags = ArticleTag.objects.filter(article_id = article.id)

    data = {
        'article': article,
        'tags': tags,
        'next_idx': article_id + 1,
        'prev_idx': article_id - 1
        # 'available_tags': available_tags,
        # 'progress': progress,
        # 'articles_total': num_articles,
        # 'articles_tagged': num_tagged_articles
    }

    return render(request, 'tagging/view_tags.html', data)

def landmark_papers(request):
    tagged_articles = PubmedImportedArticle.objects.filter(landmark=True).order_by('pmid')

    if request.method == 'POST':
        form = LandmarkPaperForm(request.POST)
        if form.is_valid():
            pmid = form.cleaned_data['pmid']
            print(pmid)
            article = PubmedImportedArticle.objects.get(pmid=pmid)
            print(article)
            article.landmark = True
            article.save()

    form = LandmarkPaperForm()

    data = {
        'articles': tagged_articles,
        'form': form
    }

    return render(request, 'tagging/landmark_papers.html', data)

def tag_article(request, article_id=0, tag_id=0):
    article = PubmedImportedArticle.objects.get(pk=article_id)
    tag = Tag.objects.get(pk=tag_id)

    article_tags = ArticleTag.objects.filter(article_id = article_id, tag_id = tag_id)
    if len(article_tags) == 1:
        article_tags.delete()
    else:
        article_tag = ArticleTag()
        article_tag.article = article
        article_tag.tag = tag
        article_tag.save()

    article_tags = ArticleTag.objects.filter(article_id = article_id)
    if len(article_tags) > 0:
        article.tagged = True
        article.save()

    return redirect('tagging:index', id=article_id)

def mark_irrelevant(request, article_id=0):
    article_status = ScreeningStatus.objects.get(article_id=article_id)
    article_status.decision = 'N'
    article_status.save()

    return redirect('tagging:index')

def export(request):
    return render(request, 'tagging/export.html')

def export_csv(request):
    tagged_articles = PubmedImportedArticle.objects.filter(tagged=True)

    col_names = ['PMID', 'authors', 'year', 'month', 'title', 'tags']
    data = []
    for ta in tagged_articles:
        tags = []
        [tags.append(t.tag.tag_name) for t in ta.articletag_set.all()]
        tags_str = ','.join(tags)
        data.append([ta.pmid, ta.authors, ta.pub_date.year, ta.pub_date.month, ta.title, tags_str])
    return build_csv_response(col_names, data)

def export_landmarks_csv(request):
    tagged_articles = PubmedImportedArticle.objects.filter(landmark=True)

    col_names = ['PMID', 'authors', 'authors_etal', 'year', 'month', 'title', 'tags']
    data = []
    for ta in tagged_articles:
        tags = []
        [tags.append(t.tag.tag_name) for t in ta.articletag_set.all()]
        tags_str = ','.join(tags)
        authors_etal = ta.authors.split()[0] + " et al."
        data.append([ta.pmid, ta.authors, authors_etal, ta.pub_date.year, ta.pub_date.month, ta.title, tags_str])
    return build_csv_response(col_names, data, filename='soa-timeline.csv')

def export_mendeley(request):
    tagged_articles = PubmedImportedArticle.objects.filter(tagged=True)

    data = []
    for ta in tagged_articles:
        tags = []
        [tags.append(t.tag.tag_name) for t in ta.articletag_set.all()]
        tags_str = ';'.join(tags)
        data.append({
            'pmid': ta.pmid,
            'title': ta.title,
            'tags': tags_str
        })
    return build_mendeley_xml_response(data)


from matplotlib import pyplot as plt
import numpy as np
import os
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

from MiniFramework.EnumDef_6_0 import *
from MiniFramework.ActivationLayer import *
from MiniFramework.ClassificationLayer import *
from MiniFramework.LossFunction_1_1 import *
from MiniFramework.TrainingHistory_3_0 import *
from MiniFramework.HyperParameters_4_3 import *
from MiniFramework.WeightsBias_2_1 import *
from ExtendedDataReader.NameDataReader import *

def load_data():
    dr = NameDataReader()
    dr.ReadData()
    dr.GenerateValidationSet(1000)
    return dr

class timestep(object):
    # for the first cell, prev_s should be zero
    def forward(self, x, U, V, W, prev_s, isFirst, isLast):
        self.U = U
        self.V = V
        self.W = W
        self.x = x

        if (isFirst):
            # 公式1
            self.h = np.dot(x, U)
        else:
            # 公式2
            self.h = np.dot(x, U) + np.dot(prev_s, W)
        # 公式3
        self.s = Tanh().forward(self.h)
        if (isLast):
            # 公式4
            self.z = np.dot(self.s, V)
            # 公式5
            self.a = Softmax().forward(self.z)

    # for the first cell, prev_s should be zero
    # for the last cell, next_dh should be zero
    def backward(self, y, prev_s, next_dh, isFirst, isLast):
        if (isLast):
            # 公式7
            self.dz = (self.a - y)
        else:
            self.dz = np.zeros_like(y)
        # end if
        if (isLast):
            # 公式8
            self.dh = np.dot(self.dz, self.V.T) * Tanh().backward(self.s)
        else:
            # 公式9
            self.dh = np.dot(next_dh, self.W.T) * Tanh().backward(self.s)
        # end if
        if (isLast):
            # 公式10
            self.dV = np.dot(self.s.T, self.dz)
        else:
            self.dV = np.zeros_like(self.V)
        # 公式11
        self.dU = np.dot(self.x.T, self.dh)
        if (isFirst):
            self.dW = np.zeros_like(self.W)
        else:
            # 公式12
            self.dW = np.dot(prev_s.T, self.dh)
        # end if

class net(object):
    def __init__(self, hp, model_name):
        self.hp = hp
        self.model_name = model_name
        self.subfolder = os.getcwd() + "/" + self.__create_subfolder()
        print(self.subfolder)

        if (self.load_init_value() == False):
            self.U,_ = WeightsBias_2_1.InitialParameters(self.hp.num_input, self.hp.num_hidden, InitialMethod.Normal)
            self.V,_ = WeightsBias_2_1.InitialParameters(self.hp.num_hidden, self.hp.num_output, InitialMethod.Normal)
            self.W,_ = WeightsBias_2_1.InitialParameters(self.hp.num_hidden, self.hp.num_hidden, InitialMethod.Normal)
            self.save_init_value()
        #end if

        self.zero_state = np.zeros((self.hp.batch_size, self.hp.num_hidden))
        self.loss_fun = LossFunction_1_1(self.hp.net_type)
        self.loss_trace = TrainingHistory_3_0()
        self.ts_list = []
        for i in range(self.hp.num_step+1):
            ts = timestep()
            self.ts_list.append(ts)
        #end for

    def __create_subfolder(self):
        if self.model_name != None:
            path = self.model_name.strip()
            path = path.rstrip("/")
            isExists = os.path.exists(path)
            if not isExists:
                os.makedirs(path)
            return path

    def forward(self,X):
        self.x = X
        self.batch = self.x.shape[0]
        self.ts = self.x.shape[1]
        for i in range(0, self.ts):
            if (i == 0):
                self.ts_list[i].forward(X[:,i], self.U, self.V, self.W, None, True, False)
            elif (i == self.ts - 1):
                self.ts_list[i].forward(X[:,i], self.U, self.V, self.W, self.ts_list[i-1].s[0:self.batch], False, True)
            else:
                self.ts_list[i].forward(X[:,i], self.U, self.V, self.W, self.ts_list[i-1].s[0:self.batch], False, False)
        #end for
        return self.ts_list[self.ts-1].a

    def backward(self,Y):
        for i in range(self.ts-1, -1, -1):
            if (i == 0):
                self.ts_list[i].backward(Y, None, self.ts_list[i+1].dh[0:self.batch], True, False)
            elif (i == self.ts - 1):
                self.ts_list[i].backward(Y, self.ts_list[i-1].s[0:self.batch], None, False, True)
            else:
                self.ts_list[i].backward(Y, self.ts_list[i-1].s[0:self.batch], self.ts_list[i+1].dh[0:self.batch], False, False)
        #end for

    def update(self):
        du = np.zeros_like(self.U)
        dv = np.zeros_like(self.V)
        dw = np.zeros_like(self.W)
        for i in range(self.ts):
            du += self.ts_list[i].dU
            dv += self.ts_list[i].dV
            dw += self.ts_list[i].dW
        #end for
        self.U = self.U - du * self.hp.eta / self.batch
        self.V = self.V - dv * self.hp.eta / self.batch
        self.W = self.W - dw * self.hp.eta / self.batch

    def save_init_value(self):
        self.init_file_name = str.format("{0}/init.npz", self.subfolder)
        np.savez(self.init_file_name, U=self.U, V=self.V, W=self.W)

    def load_init_value(self):
        self.init_file_name = str.format("{0}/init.npz", self.subfolder)
        w_file = Path(self.init_file_name)
        if w_file.exists():
            data = np.load(self.init_file_name)
            self.U = data["U"]
            self.V = data["V"]
            self.W = data["W"]
            return True
        else:
            return False

    def save_parameters(self):
        print("save best parameters...")
        self.result_file_name = str.format("{0}/result.npz", self.subfolder)
        np.savez(self.result_file_name, U=self.U, V=self.V, W=self.W)

    def load_parameters(self):
        print("load best parameters...")
        self.result_file_name = str.format("{0}/result.npz", self.subfolder)
        data = np.load(self.result_file_name)
        self.U = data["U"]
        self.V = data["V"]
        self.W = data["W"]

    def check_loss(self,X,Y):
        LOSS = 0
        ACC = 0
        for i in range(self.dataReader.num_dev):
            a = self.forward(X[i])
            loss,acc = self.loss_fun.CheckLoss(a, Y[i])
            LOSS += loss
            ACC += acc
        return LOSS/self.dataReader.num_dev, ACC/self.dataReader.num_dev

    def train(self, dataReader, checkpoint=0.1):
        self.dataReader = dataReader
        min_loss = 10
        total_iter = 0
        for epoch in range(self.hp.max_epoch):
            dataReader.Shuffle()
            while(True):
                # get data
                batch_x, batch_y = dataReader.GetBatchTrainSamples(self.hp.batch_size)
                if (batch_x is None):
                    break
                # forward
                self.forward(batch_x)
                # backward
                self.backward(batch_y)
                # update
                self.update()
                total_iter += 1
            #enf while
            # check loss
            X,Y = dataReader.GetValidationSet()
            loss,acc = self.check_loss(X,Y)
            self.loss_trace.Add(epoch, total_iter, None, None, loss, acc, None)
            print(str.format("{0}:{1}:{2} loss={3:6f}, acc={4:6f}", epoch, total_iter, self.hp.eta, loss, acc))
            if (loss < min_loss):
                min_loss = loss
                self.save_parameters()
            #endif
        #end for
        self.loss_trace.ShowLossHistory("Loss and Accuracy", XCoordinate.Epoch)

    def test(self, dataReader):
        confusion_matrix = np.zeros((dataReader.num_category, dataReader.num_category))
        correct = 0
        for i in range(dataReader.num_train):
            x,y = dataReader.GetBatchTrainSamples(1)
            output = self.forward(x)
            pred = np.argmax(output)
            label = np.argmax(y)
            confusion_matrix[label, pred] += 1
            if (pred == label):
                correct += 1
        #end for
        print(str.format("correctness={0}/{1}={2}", correct, dataReader.num_train, correct / dataReader.num_train))
        self.draw_confusion_matrix(dataReader, confusion_matrix)

    def draw_confusion_matrix(self, dataReader, confusion_matrix):
        for i in range(dataReader.num_category):
            confusion_matrix[i] = confusion_matrix[i] / confusion_matrix[i].sum()

        # Set up plot
        fig = plt.figure()
        ax = fig.add_subplot(111)
        cax = ax.matshow(confusion_matrix)
        fig.colorbar(cax)

        # Set up axes
        ax.set_xticklabels([''] + dataReader.language_list, rotation=90)
        ax.set_yticklabels([''] + dataReader.language_list)

        # Force label at every tick
        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))

        # sphinx_gallery_thumbnail_number = 2
        plt.show()

if __name__=='__main__':
    dataReader = load_data()
    eta = 0.02 # 0.02
    max_epoch = 100 # 100
    batch_size = 8 # 8
    num_input = dataReader.num_feature
    num_hidden = 16 # 16
    num_output = dataReader.num_category
    model = str.format("Level5_{0}_{1}_{2}_{3}_{4}_{5}", max_epoch, batch_size, num_input, num_hidden, num_output, eta)
    hp = HyperParameters_4_3(
        eta, max_epoch, batch_size,
        dataReader.max_step, num_input, num_hidden, num_output,
        OutputType.LastStep, NetType.MultipleClassifier)
    n = net(hp, model)
    n.train(dataReader, checkpoint=1)

    # last
    n.test(dataReader)
    # best
    n.load_parameters()
    dataReader.ResetPointer()
    n.test(dataReader)


import random

import discord
from discord.ext import commands

class Mention_Reply(commands.Cog):

    def __init__(self, bot):
        self.bot = bot

    async def cog_check(self, ctx):
        if ctx.channel.type == discord.ChannelType.private:
            return True
        enabled = await self.bot.pg_conn.fetchval("""
            SELECT enabled FROM cogs_data
            WHERE guild_id = $1
            """, ctx.guild.id)
        if f"Bot.cogs.{self.qualified_name}" in enabled:
            return True
        return False

    @commands.Cog.listener()
    async def on_message_create(self, message: discord.Message):
        try:
            prefix = random.choice(await self.bot.get_prefix(message))
            if message.channel.type == discord.ChannelType.private:
                if message.content.startswith('<@') and message.content.endswith('>'):
                    for mention in message.mentions:
                        if mention == self.bot.user:
                            await message.channel.send(f"I am {self.bot.user.name}. To see my prefix do `{prefix}prefix`. ")
                            return
        except AttributeError:
            pass

def setup(bot):
    bot.add_cog(Mention_Reply(bot))


from string import ascii_lowercase
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os
import sys

from selenium.common.exceptions import TimeoutException

PATH = '.\\msedgedriver.exe'
driver = webdriver.Edge(PATH)
name = sys.arg[1]

for con in ascii_lowercase:
    driver.get(f"https://codeforces.com/problemset/problem/{name}/{con}")
    try:
        checker = driver.find_element_by_class_name("message").is_displayed()
        if checker == True:
            break
        dirloc = ".\\"+name+"\\"+con
        os.makedirs(dirloc)
        driver.screenshot(dirloc + "\\problem.png")
        inputs = WebDriverWait(driver, 10).until(
            EC.presence_of_all_elements_located(By.CLASS_NAME, "input"))
        outputs = WebDriverWait(driver, 10).until(
            EC.presence_of_all_elements_located(By.CLASS_NAME, "output"))
        for i in range(0, len(inputs)):
            file = open(dirloc+"\\input"+str(i+1)+".txt", 'w')
            file.write(inputs[i].find_element_by_tag_name("pre").text)
            file.close()

            file = open(dirloc+"\\output"+str(i+1)+".txt", 'w')
            file.write(
                outputs[i].find_element_by_tag_name("pre").text)
            file.close()
    except:
        pass


from unittest.mock import patch

import pytest

from brain_brew.constants.deckpart_keys import NoteFlagKeys
from brain_brew.constants.global_config_keys import ConfigKeys

from brain_brew.representation.configuration.global_config import GlobalConfig
from tests.test_files import TestFiles

class TestSingletonConstructor:
    def test_runs(self):
        fm = GlobalConfig.get_instance()
        assert isinstance(fm, GlobalConfig)

    def test_returns_existing_singleton(self):
        fm = GlobalConfig.get_instance()
        fm.known_files_dict = {'test': None}
        fm2 = GlobalConfig.get_instance()

        assert fm2.known_files_dict == {'test': None}
        assert fm2 == fm

    def test_raises_error(self):
        with pytest.raises(Exception):
            GlobalConfig({})
            GlobalConfig({})

@pytest.fixture()
def global_config():
    GlobalConfig.clear_instance()
    return GlobalConfig({
        ConfigKeys.DECK_PARTS.value: {
            ConfigKeys.HEADERS.value: TestFiles.Headers.LOC,
            ConfigKeys.NOTE_MODELS.value: TestFiles.NoteModels.LOC,
            ConfigKeys.NOTES.value: TestFiles.NoteFiles.LOC,
            ConfigKeys.MEDIA_FILES.value: TestFiles.MediaFiles.LOC,

            ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value: {
                NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: False,
                NoteFlagKeys.EXTRACT_SHARED_TAGS.value: False
            }
        },
        ConfigKeys.FLAGS.value: {

        }
    })


import fs_database
import time
import datetime
import json
from dicts import *

class geojson():

    def __init__(self, cursor, seconds):
        self.GEOJSON = GEO.JSON
        self.seconds = seconds
        now = time.time()
        self.start = now-self.seconds
        self.cursor = cursor
        self.longList = []

    def getData(self):
        self.now = time.time()
        if self.longList == []:
            self.cursor.execute("SELECT * FROM LOCATION WHERE UNIX BETWEEN {} AND {}".format(self.start, self.now))
        else:
            try:
                self.last = self.result[-1][0]
                self.cursor.execute("SELECT * FROM LOCATION WHERE UNIX BETWEEN {} AND {}".format(self.last, self.now))
            except:
                pass
        self.result = self.cursor.fetchall()
        for i in self.result:
            self.lon = i[2]
            self.lat = i[1]
            self.lonLat = [self.lon,self.lat]
            self.longList.append(self.lonLat)
        print(len(self.longList))
        self.GEOJSON['geometry']['coordinates'] = self.longList
        j = json.dumps(self.GEOJSON)
        return(j)

if __name__ == '__main__':
    cursor = fs_database.logging().c
    geoline = geojson(cursor, 600)
    while True:
        geojson = geoline.getData()
        time.sleep(1)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import json
import texttk
import preprocessfile as ppf

def out_json():
    tweets_data_path = 'data.json'

    tweets_data = []
    tweets_file = open(tweets_data_path, "r")
    for line in tweets_file:
        #try:
            #tweets_data.append(tweet['id'])
            if(len(line)<2):
                continue;
            else:
                tweet = json.loads(line)
                tweets_data.append(tweet)
        #except:
        #    continue
    return tweets_data

def addTweet(tweet, list):

    # a dict to contain information about single tweet
    tweet_information={}

    # text of tweet
    tweet_information['text']= ppf.preprocess(tweet['text'])

    # date and time at which tweet was created
    tweet_information['created_at']=tweet['created_at'] #.strftime("%Y-%m-%d %H:%M:%S")

    # id of this tweet
    tweet_information['id']=tweet['id']

    # retweet count
    tweet_information['retweet_count']=tweet['retweet_count']

    # user information in user dictionery
    user_dictionery=tweet['user']

    # screename of the person who tweeted this
    tweet_information['user_id']=user_dictionery['id']
    tweet_information['screen_name']={}
    tweet_information['screen_name']['posted_by']=user_dictionery['screen_name']
    i=1
    while(len(tweet['entities']['user_mentions'])>0 and len(tweet['entities']['user_mentions'])>=i):
        tweet_information['screen_name']['mentioned_{}'.format(i)]=tweet['entities']['user_mentions'][i-1]['screen_name']
        i+=1

    # add this tweet to the tweet_list
    list.append(tweet_information)

def store_tweets(alltweets,file='tweets.json'):

    # a list of all formatted tweets
    tweet_list=[]
    tweet_dict={}
    for tweet in alltweets:

        if('limit' in tweet.keys()):
            continue

        # check if retweeted tweet
        if('retweeted_status' in tweet.keys()):
            tweet = tweet['retweeted_status']

        addTweet(tweet, tweet_list)

    tweet_dict["json"] = tweet_list

    # open file desc to output file with write permissions
    file_des=open(file,'wb')

    # dump tweets to the file
    json.dump(tweet_dict,file_des,indent=4,sort_keys=True)

    # close the file_des
    file_des.close()

if __name__ == '__main__':

    # pass in the username of the account you want to download
    alltweets=out_json()
    #print alltweets
    # store the data into json file
#    if len(sys.argv[2])>0:
    store_tweets(alltweets,"final_output.json")


# %%
import lxml.html
# %%
tree = lxml.html.parse("dp.html")
html = tree.getroot()

# %%
print(html)

# %%
html.make_links_absolute('https://gihyo.jp/')

# %%
for a in html.cssselect('#listBook > li > a[itemprop="url"]'):
    # a要素のhref属性から書籍のURLを取得する。
    url = a.get('href')

    # 書籍のタイトルは itemprop="name" という属性を持つp要素から取得する。
    p = a.cssselect('p[itemprop="name"]')[0]
    title = p.text_content()  # wbr要素などが含まれるのでtextではなくtext_content()を使う。

    # 書籍のURLとタイトルを出力する。
    print(url, title)

# %%


from discord.ext import commands
from utils.asset import Assets

class error(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.Cog.listener()
    async def on_command_error(self, ctx, error):
        if hasattr(ctx.command, 'on_error'):
            return
        if isinstance(error, commands.MissingPermissions):
            await ctx.send(f"{Assets.red_tick} **You don't have the permission to execute this command.**")
        elif isinstance(error, commands.CommandNotFound):
            return
        else: await ctx.send(f"{Assets.red_tick} **`{error}` ({ctx.command})**")
        print(error)

async def setup(client):
    await client.add_cog(error(client))


from redminelib import Redmine
from secretkey import *
import unicodedata

def get_char_width(c):
    data = unicodedata.east_asian_width(c)
    if data == 'Na' or data == 'H':
        return 1
    return 2

def get_string_width(string):
    width = 0
    for c in string:
        width += get_char_width(c)
    return width

def multibyte_str_align(string, length):
    num = get_string_width(string)
    if num>=length:
        return string
    else:
        return (' ' * (length-num)) + string

redmine = Redmine(REDMINE_URL, key=API_KEY)
issue_itr = redmine.issue.filter(status='完了')
ticket_str = '#{id:0>4} | {status} | {subject}\n'
output = 'ID    | status | subject\n'

for issue in issue_itr:
    output += ticket_str.format(id=issue.id, status=multibyte_str_align(str(issue.status), 6), subject=issue.subject)

print(output)


import numpy
import pygad
import pygad.nn
import pygad.gann
def fitness_func(solution, sol_idx):
    global GANN_instance, data_inputs, data_outputs
    predictions = pygad.nn.predict(last_layer=GANN_instance.population_networks[sol_idx],
                                   data_inputs=data_inputs)
    correct_predictions = numpy.where(predictions == data_outputs)[0].size
    solution_fitness = (correct_predictions/data_outputs.size)*100
    return solution_fitness
def callback_generation(ga_instance):
    global GANN_instance
    population_matrices = pygad.gann.population_as_matrices(population_networks=GANN_instance.population_networks,
                                                            population_vectors=ga_instance.population)
    GANN_instance.update_population_trained_weights(population_trained_weights=population_matrices)
    print("Generation = {generation}".format(generation=ga_instance.generations_completed))
    print("Accuracy   = {fitness}".format(fitness=ga_instance.best_solution()[1]))
data_inputs = numpy.array([[1, 1],
                           [1, 0],
                           [0, 1],
                           [0, 0]])
data_outputs = numpy.array([0,
                            1,
                            1,
                            0])
GANN_instance = pygad.gann.GANN(num_solutions=5,
                                num_neurons_input=2,
                                num_neurons_hidden_layers=[2],
                                num_neurons_output=2,
                                hidden_activations=["relu"],
                                output_activation="softmax")
population_vectors = pygad.gann.population_as_vectors(population_networks=GANN_instance.population_networks)
ga_instance = pygad.GA(num_generations=50,
                       num_parents_mating=3,
                       initial_population=population_vectors.copy(),
                       fitness_func=fitness_func,
                       mutation_percent_genes=5,
                       callback_generation=callback_generation)
ga_instance.run()
ga_instance.plot_result()
solution, solution_fitness, solution_idx = ga_instance.best_solution()
print(solution)
print(solution_fitness)
print(solution_idx)


import unittest
from selenium import webdriver
from pageObjects import MessageFeature

class MessagingTest(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Firefox()
        self.Message = MessageFeature(self.driver)

        # Navigate to https://staging.joordev.com
        self.driver.implicitly_wait(30)
        self.driver.get('https://staging.joordev.com')

        # Click on person login icon in top right corner
        self.Message.clickLoginPerson()

        # Add email and password credentials and click sign in
        self.Message.addEmail('qatest1')
        self.Message.addPassword('qatest1')
        self.Message.clickSignIn()

    def testSentMessage(self):
        self.driver.implicitly_wait(30)
        self.Message.hoverMessage()  # Hover over Messages tab
        self.Message.clickSendMessage() # Click Send a Message
        self.Message.clickSelectConnections() # Click select connections
        self.Message.addSelectConnections('JOOR Regress') # Add connection
        self.Message.clickJoorReg() # Clicks on JOOR Regress connection
        self.Message.addSubject('Message Subject') # Fill out subject of message feature
        self.Message.addMessage('Message Body') # Fill out body of message feature
        self.Message.clickSend() # Click send on message feature
        self.assertEqual(self.Message.msgSentVerification(), 'Your message has been sent')

    def tearDown(self):
        self.driver.quit()

if __name__ == "__main__":
    unittest.main()


from sklearn.metrics import classification_report, r2_score, mean_squared_error

from src.loader import load_sklearn_data
from src.components.model import Model

def evaluate_model(preds, y_true, problem_type, model_title=None):
    """
    Evaluates the predictions of the model
    Parameters
    ----------
    preds: np.array of integers
        A numpy array of integer predictions. Must be 1-dimensional (shape [N,])

    y_true: np.array of integers
        A numpy array of integer true values. Must be 1-dimensional (shape [N,])

    problem_type: str
        "classification" or "regression"

    model_title: str
        An optional argument that delineates the name of the model

    """
    if problem_type == "classification":
        print(
            f"Evaluating Model:"
            if model_title is None
            else f"Evaluating {model_title}:"
        )
        print(classification_report(y_true, preds))
    elif problem_type == "regression":
        print("Mean Squared Error: %.2f" % mean_squared_error(y_true, preds))
        print("Coefficient of Determination: [0 | %.2f | 1]" % r2_score(y_true, preds))

def test_model_implementation(
    model: Model, problem_type="classification", **class_params
):
    """
    The purpose of this function is to test the implementation of a class that inherits the Model object

    Parameters
    ----------
    model: object
        An instantiation of a class that inherits the master Model class

    class_params: dict
        dictionary of optional arguments for the make_classification function

        E.g.:

        class_params = {
            "n_samples": 100,
            "n_features": 20,
            "n_classes": 2,
            "class_sep": 4,
            "random_state": 88
        }

    """
    (
        X_train,
        X_valid,
        X_test,
        y_train,
        y_valid,
        y_test,
    ) = load_sklearn_data(problem_type, valid=True, split=[70, 10, 20], **class_params)

    model.fit(X_train, y_train)
    preds = model.predict(X_test)

    evaluate_model(preds, y_test, problem_type)