# -*- coding: utf-8 -*-
"""
this file implements the core functionality for collectors and analyzers
"""

__author__ = "Lukas Reiter"
__license__ = "GPL v3.0"
__copyright__ = """Copyright 2018 Lukas Reiter

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
__version__ = 0.1

import logging
import argparse
from queue import Queue
from threading import Lock
from threading import Thread
from database.core import Engine
from database.model import Path
from database.model import File
from database.model import MatchRule
from database.model import SearchLocation
from database.model import FileRelevance
from config.config import FileHunter as FileHunterConfig

logger = logging.getLogger('analyzer')

class BaseAnalyzer(Thread):
    """
    This class implements all base functionalities for collectors and analyzers
    """

    DB_OPERATION_MUTEX = Lock()

    def __init__(self,
                 engine: Engine,
                 args: argparse.Namespace,
                 config: FileHunterConfig,
                 file_queue: Queue,
                 daemon: bool = False):
        super().__init__(daemon=daemon)
        self.engine = engine
        self.file_queue = file_queue
        self._args = args
        self.workspace = args.workspace
        self.config = config

    def is_file_size_below_threshold(self, path: Path, size: int) -> bool:
        """
        This method determines if the given file size in bytes is below the configured threshold.
        """
        return self.config.is_below_threshold(path, size)

    def add_content(self, path: Path, rule: MatchRule = None, file: File = None):
        """
        This method adds the rule matching result to the database.
        """
        if rule and file:
            raise ValueError("parameters rule and file are mutual exclusive")
        elif not rule and not file:
            raise ValueError("either parameter rule or file must be given")
        with BaseAnalyzer.DB_OPERATION_MUTEX:
            with self.engine.session_scope() as session:
                workspace = self.engine.get_workspace(session, name=self.workspace)
                host = self.engine.add_host(session=session,
                                            workspace=workspace,
                                            address=path.service.host.address)
                service = self.engine.add_service(session=session,
                                                  port=path.service.port,
                                                  name=path.service.name,
                                                  host=host)
                if rule:
                    match_file = self.engine.add_match_rule(session=session,
                                                            search_location=rule.search_location,
                                                            search_pattern=rule.search_pattern,
                                                            relevance=rule.relevance,
                                                            accuracy=rule.accuracy,
                                                            category=rule.category)
                    file = self.engine.add_file(session=session,
                                                workspace=workspace,
                                                file=path.file)
                    file.add_match_rule(match_file)
                self.engine.add_path(session=session,
                                     service=service,
                                     full_path=path.full_path,
                                     share=path.share,
                                     file=file,
                                     access_time=path.access_time,
                                     modified_time=path.modified_time,
                                     creation_time=path.creation_time)

    def _analyze_content(self, path: Path) -> FileRelevance:
        """
        This method analyzes the file's content for interesting information.
        :param path: The path object whose content shall be analyzed.
        :return: True if file is of relevance
        """
        result = None
        for rule in self.config.matching_rules[SearchLocation.file_content.name]:
            if rule.is_match(path):
                logger.info("Match: {} ({})".format(str(path), rule.get_text(not self._args.nocolor)))
                result = rule.relevance
                self.add_content(path=path, rule=rule)
                break
        return result

    def _analyze_path_name(self, path: Path) -> FileRelevance:
        """
        This method analyzes the file's name for interesting information.
        :param path: The path object whose name shall be analyzed.
        :return: True if file is of relevance
        """
        result = None
        # First we search the full path
        for rule in self.config.matching_rules[SearchLocation.full_path.name]:
            if rule.is_match(path):
                logger.info("Match: {} ({})".format(str(path), rule.get_text(not self._args.nocolor)))
                result = rule.relevance
                self.add_content(rule=rule, path=path)
                break
        else:
            # If nothing is found, then we search the file name
            for rule in self.config.matching_rules[SearchLocation.file_name.name]:
                if rule.is_match(path):
                    logger.info("Match: {} ({})".format(str(path), rule.get_text(not self._args.nocolor)))
                    result = rule.relevance
                    self.add_content(rule=rule, path=path)
                    break
        return result


import json
import re
import sys
import os
import argparse
import requests
import pygit2
import sh

from datetime import datetime
from pprint import PrettyPrinter

class Issues:
    all_types = ['New Feature', 'Improvement', 'Bug', 'Task', 'Sub-task', 'Story', 'Epic', 'No Assigned Issue']
    feature_types = ['New Feature', 'Improvement']
    maint_types = ['Bug', 'Task', 'Sub-task', 'Story', 'Epic', 'No Assigned Issue']
    issues = {}

    @staticmethod
    def get_issue(issue):
        if not issue in Issues.issues:
            url = "https://openxt.atlassian.net/rest/api/latest/issue/" + issue
            try:
                r = requests.get(url)

                if r.status_code == requests.codes.ok:
                    Issues.issues[issue] = json.loads(r.text)
                else:
                    sys.stderr.write("Failed to query JIRA, %s\n" % url)
                    Issues.issues[issue] = []
            except:
                sys.stderr.write("Exception fetching JIRA issue %s\n" % issue)
                Issues.issues[issue] = []

        return Issues.issues[issue]

class CommitEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Commit):
            return {
                "hash" : obj.hash,
                "repo" : obj.repo,
                "subject" : obj.subject,
                "body" : obj.body,
                "commit_date" : obj.commit_date,
                "author_name" : obj.author_name,
                "author_email" : obj.author_email,
                "signers" : obj.signers
            }

class Commit:
    def __init__(self, repo_name, git_commit):
        issues = list(set(re.findall('OXT-[0-9]+', git_commit.message)))
        signed = re.findall('Signed-off-by: (.*)\n', git_commit.message)
        signed.extend(re.findall('Signed off by: (.*)\n', git_commit.message))

        msgsplt = git_commit.message.split('\n\n', 1)
        if len(msgsplt) == 2:
            subject = msgsplt[0].strip()
            body = msgsplt[1].strip()
        else:
            subject = msgsplt[0].strip()
            body = ""

        self.hash = git_commit.hex
        self.repo = repo_name
        self.subject = subject.replace("{", "\{")
        self.body = body
        self.raw = git_commit.message
        self.commit_date = datetime.utcfromtimestamp(
            git_commit.commit_time).strftime('%Y-%m-%dT%H:%M:%SZ')
        self.author_name = git_commit.author.name
        self.author_email = git_commit.author.email
        self.parents = [c.hex for c in git_commit.parents]
        self.issues = issues
        self.signers = signed

    @staticmethod
    def is_merge(commit):
        return True if len(commit.parents) > 1 else False

    @staticmethod
    def dedup_commits(commits):
        deduped = {v.hash:v for v in commits}.values()

        return deduped

    @staticmethod
    def categorize(commits):
        report = {}

        for t in Issues.all_types:
            report[t] = []

        for commit in commits:
            if Commit.is_merge(commit):
                continue

            commit.security = False

            if not commit.issues or 'No Assigned Issue' in commit.issues:
                report['No Assigned Issue'].append(commit)
            else:
                for issue_id in commit.issues:
                    issue = Issues.get_issue(issue_id)
                    if issue:
                        for c in issue['fields']['components']:
                            if "Security" in c['name']:
                                commit.security = True
                        report[issue['fields']['issuetype']['name']].append(commit)
                    else:
                        report['No Assigned Issue'].append(commit)

        return report

    @staticmethod
    def merge_categorized(commits1, commits2):
        report = {}

        for t in Issues.all_types:
            report[t] = commits1[t] + commits2[t]

        return report

class Repository:
    GITHUB_URL = "https://github.com/OpenXT/"

    def __init__(self, repo_name, previous, new, workdir):
        self.name = repo_name
        self.repodir = workdir + "/" + repo_name + ".git"
        self.previous = previous
        self.new = new
        self.url = self.GITHUB_URL + repo_name + ".git"

        # Check the remote for refs before attempting anything
        try:
            pc = sh.wc(sh.git("ls-remote", "--heads", "--tags", self.GITHUB_URL + repo_name, previous), "-l").stdout.decode('utf-8').strip()
            nc = sh.wc(sh.git("ls-remote", "--heads", "--tags", self.GITHUB_URL + repo_name, new), "-l").stdout.decode('utf-8').strip()

            if pc == "0":
                sys.stderr.write("No tag or head %s for repository %s\n", previous, self.GITHUB_URL + repo_name)
                raise Error
            if nc == "0":
                sys.stderr.write("No tag or head %s for repository %s\n", new, self.GITHUB_URL + repo_name)
                raise Error

        except sh.ErrorReturnCode as e:
            print(e)
            raise Error

        if not os.path.isdir(self.repodir):
            try:
                sh.git.clone("--mirror", self.url, self.repodir)
            except sh.ErrorReturnCode:
                sys.stderr.write("Failed to mirror repo url: %s\n" % self.url)
                raise Error

        try:
            repo_path = pygit2.discover_repository(self.repodir)
            self.repo = pygit2.Repository(repo_path)
        except KeyError:
            sys.stderr.write("%s is not a git repository\n" % name)
            raise RuntimeError

    def get_commits(self):
        self.commits = []

        try:
            cherry = filter(None, sh.git.cherry(self.previous, self.new, _cwd=self.repodir).stdout.decode('utf-8').split("\n"))
            cherry = filter(lambda i: i[0] == "+", cherry)
            if cherry:
                commit_list = [x[2:].strip() for x in cherry]
            else:
                return
        except sh.ErrorReturnCode:
            sys.stderr.write ("Error occurred generating cherry list for %s\n" % self.name)
            raise RuntimeError

        for ref in commit_list:
            if ref:
                commit = self.repo.get(ref)
                self.commits.append(Commit(self.name, commit))

    def get_contributors(self):
        self.authors = {}
        self.contributors = []

        for c in self.commits:
            if Commit.is_merge(c):
                continue

            if not c.author_email in self.authors:
                self.authors[c.author_email] = c.author_name

            for s in c.signers:
                if not s in self.contributors:
                    self.contributors.append(s)

class Release:
    FETCH_ISSUES = 1
    REPO_URL = "https://api.github.com/users/openxt/repos?per_page=100"
    REPO_BLACKLIST = [ "bats-suite", "bvt",
                       "docs", "openxt.github.io",
                       "blktap", "blktap3",
                       "bootage", "cdrom-daemon", "ocaml",
                       "meta-openxt-base", "meta-openxt-qt", "meta-openxt-remote-management", "meta-selinux"
                    ]

    def __init__(self, previous, new, workdir, flags=0):
        self.workdir = workdir
        self.previous = previous
        self.new = new
        self.flags = flags

        try:
            repourl = sh.curl("-s", self.REPO_URL)
            jq_raw = sh.jq(repourl, "-M", ".[].name")
            repo_list = list(filter(None, jq_raw.replace("\"", "").split("\n")))
        except sh.ErrorReturnCode:
            sys.stderr.write("Failed retrieving OpenXT repository list from Github.")
            raise Error

        self.repos = []
        whitelisted = lambda x: not x in self.REPO_BLACKLIST
        for r in list(filter(whitelisted, repo_list)):
            try:
                self.repos.append(Repository(r,previous,new,workdir))
            except:
                continue

        if not self.repos:
            sys.stderr.write("Unable to find any repo with both references, %s and %s.\n" % (previous, new))
            raise Error

    def generate(self):
        self.categorized = {}
        self.contributors = []

        for t in Issues.all_types:
            self.categorized[t] = []

        for r in self.repos:
            r.get_commits()

            if not r.commits:
                continue

            for c in r.commits:
                if c.issues and self.flags & self.FETCH_ISSUES:
                    for i in c.issues:
                        Issues.get_issue(i)

            report = Commit.categorize(r.commits)
            self.categorized = Commit.merge_categorized(self.categorized, report)

            r.get_contributors()
            self.contributors.extend(r.contributors)

        self.contributors = list(set(self.contributors))

class ReleaseDocument:
    def __init__(self, filepath="release.adoc", relnum="X.Y.Z", author="Author Name", email="author@email.com", entity="copyright holding entity", rev="1.0", rev_string="First"):
        self.relnum = relnum
        self.author = author
        self.email = email
        self.entity = entity
        self.rev = rev
        self.rev_string = rev_string

        self.fd = open(filepath, 'w')

    def close(self):
        self.fd.close()

    def header_page(self):
        fd = self.fd

        fd.write("OpenXT %s Release\n====================\n" % self.relnum)
        fd.write("%s <%s>\n" % (self.author, self.email))
        fd.write("v%s, %s: %s revision\n" % (self.rev, datetime.today().strftime("%B %Y"),self.rev_string))
        fd.write(":toc:\n\n")
        fd.flush()

    def platform_page(self, path):
        fd = self.fd

        fd.write(":numbered:\nPlatform\n--------\n")
        fd.write("\n")

        if path:
            with open(path, 'r') as body:
                fd.write(body.read())

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def features_page(self, categorized):
        fd = self.fd
        commits = []

        for t in Issues.feature_types:
            commits.extend(categorized[t])

        commits = Commit.dedup_commits(commits)

        fd.write(":numbered:\nFeature Additions\n-----------------\n")
        fd.write("\n")

        for c in commits:
            fd.write("- https://github.com/OpenXT/%s/commit/%s[%s/%s]: %s" % (c.repo, c.hash, c.repo, c.hash[0:8], c.subject))
            if c.issues:
                fd.write(", ")
                for i in c.issues:
                    fd.write("https://openxt.atlassian.net/browse/%s[%s] " % (i,i))
            fd.write("\n")

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def security_page(self, categorized):
        fd = self.fd
        commits = []

        for t in Issues.all_types:
            for c in categorized[t]:
                if c.security:
                    commits.append(c)
        commits = Commit.dedup_commits(commits)

        fd.write(":numbered:\nSecurity Fixes\n--------------\n")
        fd.write("\n")

        for c in commits:
            fd.write("- https://github.com/OpenXT/%s/commit/%s[%s/%s]: %s" % (c.repo, c.hash, c.repo, c.hash[0:8], c.subject))
            if c.issues:
                fd.write(", ")
                for i in c.issues:
                    fd.write("https://openxt.atlassian.net/browse/%s[%s] " % (i,i))
            fd.write("\n")

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def maintenance_page(self, categorized):
        fd = self.fd
        commits = []

        for t in Issues.maint_types:
            commits += categorized[t]
        commits = Commit.dedup_commits(commits)

        fd.write(":numbered:\nMaintenance Changes\n-------------------\n")
        fd.write("\n")

        for c in commits:
            fd.write("- https://github.com/OpenXT/%s/commit/%s[%s/%s]: %s" % (c.repo, c.hash, c.repo, c.hash[0:8], c.subject))
            if c.issues:
                fd.write(", ")
                for i in c.issues:
                    fd.write("https://openxt.atlassian.net/browse/%s[%s] " % (i,i))
            fd.write("\n")

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def testing_page(self, path):
        fd = self.fd

        fd.write(":numbered:\nTesting\n-------\n")
        fd.write("\n")

        if path:
            with open(path, 'r') as body:
                fd.write(body.read())

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def known_issues_page(self, path):
        fd = self.fd

        fd.write(":numbered:\nKnown Issues\n------------\n")
        fd.write("\n")

        if path:
            with open(path, 'r') as body:
                fd.write(body.read())

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def contributors_page(self, contributors):
        fd = self.fd

        fd.write(":numbered:\nContributors\n------------\n")
        fd.write("\n")

        for c in contributors:
            fd.write("- %s" % c)
            fd.write("\n")

        fd.write("\n")
        fd.write("<<<\n")
        fd.write("\n")
        fd.flush()

    def license_page(self):
        fd = self.fd

        fd.write("[appendix]\nLicense\n-------\n")
        fd.write("Copyright %s by <%s>. " % (datetime.now().year, self.entity))
        fd.write("Created by %s <%s>. " % (self.author, self.email))
        fd.write("This work is licensed under the Creative Commons " +
                 "Attribution 4.0 International License. To view a copy of " +
                 "this license, visit http://creativecommons.org/licenses/by/4.0/.\n")
        fd.flush()

def main(base_path, out, refs, publish, bodies, gen_json=False):
    try:
        release = Release(refs[0], refs[1], base_path, Release.FETCH_ISSUES)
    except:
        sys.stderr.write("Abort...\n")
        sys.exit(1)

    release.generate()

    if gen_json:
        json_data = {}
        for r in release.repos:
            json_data[r.name] = r.commits

        with open('commits.json', 'w') as fd:
            json.dump(json_data, fd, cls=CommitEncoder)

    doc = ReleaseDocument(filepath=out)

    if publish['relnum']:
        doc.relnum = publish['relnum']
    if publish['author']:
        doc.author = publish['author']
    if publish['email']:
        doc.email = publish['email']
    if publish['entity']:
        doc.entity = publish['entity']

    doc.header_page()
    doc.platform_page(bodies['platform'])
    doc.features_page(release.categorized)
    doc.security_page(release.categorized)
    doc.maintenance_page(release.categorized)
    doc.testing_page(bodies['testing'])
    doc.known_issues_page(bodies['known'])
    doc.contributors_page(release.contributors)
    doc.license_page()
    doc.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--path", help="directory where git repositories will be stored")
    parser.add_argument("-A", "--author", help="author's name")
    parser.add_argument("-E", "--email", help="author's email")
    parser.add_argument("-G", "--entity", help="copyright holding entity")
    parser.add_argument("-R", "--relnum", help="release version number")
    parser.add_argument("-P", "--platform", help="path to file with \"Platform\" body")
    parser.add_argument("-T", "--testing", help="path to file with \"Testing\" body")
    parser.add_argument("-K", "--known", help="path to file with \"Known Issues\" body")
    parser.add_argument("-o", "--output", help="file name for asciidoc file")
    parser.add_argument("-j", "--json", action="store_true", help="generate json file of commits")
    parser.add_argument("previous", help="the git reference for previous release")
    parser.add_argument("new", help="the git reference for the new release")

    args = parser.parse_args()
    if args.new:
        refs = [args.previous, args.new]
    else:
        usage()

    publish = {}
    bodies = {}

    path = args.path if args.path else "repos"
    output = args.output if args.output else "release.adoc"
    gen_json = args.json if args.json else False

    publish['author'] = args.author if args.author else ""
    publish['email'] = args.email if args.email else ""
    publish['entity'] = args.entity if args.entity else ""
    publish['relnum'] = args.relnum if args.relnum else ""

    bodies['platform'] = args.platform if args.platform else ""
    bodies['testing'] = args.testing if args.testing else ""
    bodies['known'] = args.known if args.known else ""

    main(path, output, refs, publish, bodies, gen_json)


"""
    Author: Jon Ander Gomez Adrian (jon@dsic.upv.es, http://personales.upv.es/jon)
    Version: 1.0
    Date: September 2021
    Universitat Politecnica de Valencia
    Technical University of Valencia TU.VLC

    K-Means-based sequence-based predictor

"""

import os
import sys
import numpy
import pickle

from sklearn.base import BaseEstimator
from sklearn.metrics import confusion_matrix, classification_report, plot_confusion_matrix

from machine_learning import KMeans

from matplotlib import pyplot

class MyArgmaxForPredictedLabels(BaseEstimator):
    def __init__(self, threshold = 0.5):
        self.threshold = threshold
        self._estimator_type = 'classifier'

    def fit(self, X, y):
        raise Exception('No fit implemented in this class')
        return self

    def predict(self, y_probs):
        assert type(y_probs) == numpy.ndarray
        assert len(y_probs.shape) == 1
        return y_probs

if __name__ == "__main__":

    """
    run with this command:

        hdfs dfs -cat  data/uc13-train.csv | python python/kmeans_uc13_predict.py \
                                                                --codebook models/kmeans_model-uc13-1000.pkl \
                                                                --counter-pairs models/cluster-distribution-1000.csv 2>/dev/null
    """
    label_mapping = [i for i in range(10)]

    verbose = 0
    codebook_filename = 'models/kmeans_model-uc13-200.pkl'
    counter_pairs_filename = 'models/cluster-distribution-200.csv'
    spark_context = None
    num_partitions = 40
    batch_size = 100
    num_channels = 21
    results_dir = 'results3.train'
    models_dir = 'models'
    log_dir = 'log'
    do_reshape = True
    do_standard_scaling = True

    for i in range(len(sys.argv)):
        if sys.argv[i] == "--verbosity":
            verbose = int(sys.argv[i + 1])
        elif sys.argv[i] == "--codebook":
            codebook_filename = sys.argv[i + 1]
        elif sys.argv[i] == "--counter-pairs":
            counter_pairs_filename = sys.argv[i + 1]
        elif sys.argv[i] == "--results-dir":
            results_dir = sys.argv[i + 1]
        elif sys.argv[i] == "--reduce-labels":
            label_mapping[3] = 0
            label_mapping[4] = 0
            label_mapping[5] = 0
            label_mapping[6] = 1
            label_mapping[7] = 0
            label_mapping[8] = 0
            label_mapping[9] = 0
        elif sys.argv[i] == "--from-pca":
            models_dir = 'models.pca'
            log_dir = 'log.pca'
            do_reshape = False
            do_standard_scaling = False

    # Load the codebook
    with open(codebook_filename, 'rb') as f:
        centers = pickle.load(f)
        f.close()

    kmeans = KMeans()
    kmeans.n_clusters = len(centers)
    kmeans.cluster_centers_ = numpy.array(centers)
    print(f'loaded the codebook of {kmeans.n_clusters} clusters')

    # Load the confusion matrix
    counter_pairs = None
    with open(counter_pairs_filename, 'rt') as f:
        counter_pairs = list()
        for line in f:
            counter_pairs.append([float(x) for x in line.split(';')])
        f.close()
        counter_pairs = numpy.array(counter_pairs)

    # Compute the conditional probabilities
    conditional_probabilities = counter_pairs.copy()
    for i in range(counter_pairs.shape[0]):
        conditional_probabilities[i, :] = counter_pairs[i, :] / sum(counter_pairs[i, :])

    # Compute the a priori probabilities of target classes
    target_class_a_priori_probabilities = counter_pairs.sum(axis = 1) / counter_pairs.sum()

    def csv_line_to_patient_label_and_sample_reshape(line):
        parts = line.split(';')
        return (parts[0], label_mapping[int(parts[1])], numpy.array([float(x) for x in parts[2:]]).reshape(num_channels, 14))

    def csv_line_to_patient_label_and_sample(line):
        parts = line.split(';')
        return (parts[0], label_mapping[int(parts[1])], numpy.array([float(x) for x in parts[2:]]))

    ####################################################################
    # begin: load standard scaling
    ####################################################################
    statistics_filename = f'{models_dir}/mean_and_std.pkl'
    if do_standard_scaling and os.path.exists(statistics_filename):
        with open(statistics_filename, 'rb') as f:
            stats = pickle.load(f)
            f.close()
        x_mean, x_std = stats
    else:
        x_mean = numpy.zeros(14)
        x_std = numpy.ones(14)
    #
    ####################################################################
    # end: load standard scaling
    ####################################################################

    def classify_sample(t):
        patient, label, sample = t
        if do_standard_scaling:
            sample = (sample - x_mean) / x_std

        if do_reshape:
            cluster_assignment = kmeans.predict(sample)
            probs = numpy.zeros(conditional_probabilities.shape[0]) # one per target class
            for j in cluster_assignment:
                probs += conditional_probabilities[:, j]
        else:
            cluster_assignment = kmeans.predict([sample])
            j = cluster_assignment[0]
            probs = conditional_probabilities[:, j]
        #probs *= target_class_a_priori_probabilities
        k = probs.argmax()
        return (patient, label, k)

    y_true_and_pred = list()
    #
    sliding_window_length = 5 * 60 # 1800 seconds -> 30 minutes
    sliding_window_step = 60 * 5 # 300 seconds -> 5 minute
    #
    target_class_counters = [0] * conditional_probabilities.shape[0]
    list_of_predictions = list()
    list_of_alarms = list()
    current_time = 0
    state = 'inter-ictal'

    old_patient = "non-existent-yet"
    for line in sys.stdin:
        if do_reshape:
            patient, label, predicted_label = classify_sample(csv_line_to_patient_label_and_sample_reshape(line))
        else:
            patient, label, predicted_label = classify_sample(csv_line_to_patient_label_and_sample(line))
        #
        if patient != old_patient:
            # reset variables
            #
            old_patient = patient
            target_class_counters = [0] * conditional_probabilities.shape[0]
            list_of_predictions = list()
            list_of_alarms = list()
            current_time = 0
            state = 'inter-ictal'
        #
        list_of_predictions.append(predicted_label)
        target_class_counters[predicted_label] += 1
        if len(list_of_predictions) > sliding_window_length:
            target_class_counters[list_of_predictions[0]] -= 1
            del list_of_predictions[0]
        #
        target_class_probs = numpy.array([float(x) for x in target_class_counters]) / sum(target_class_counters)
        #
        current_time += 2 # add 2 seconds because each sample comes 2 seconds after the previous one
        #
        if current_time > 0 and current_time % sliding_window_step == 0:
            print(patient, label, " ".join("{:8.2f}".format(100 * x) for x in target_class_probs))

        if label == 1: # an ictal period is reached
            if state == 'inter-ictal':
                print('ictal period starts, press enter ...')
                for pred_label, pred_time in list_of_alarms:
                    if current_time - pred_time <= 60 * 60: # 3600 seconds -> one hour
                        y_true_and_pred.append((1, pred_label))
                    else:
                        y_true_and_pred.append((0, pred_label))
                #
                state = 'ictal'
                list_of_alarms = list()
            #
        elif label in [0, 2, 3, 4, 5]: # exclude post-ictal periods
            state = 'inter-ictal'
            #
            if current_time > 0 and current_time % sliding_window_step == 0:
                if target_class_probs[1:4].sum() > 0.33:
                    list_of_alarms.append((1, current_time))
                else:
                    list_of_alarms.append((0, current_time))
            #
    #
    #
    for pred_label, pred_time in list_of_alarms:
        y_true_and_pred.append((0, pred_label))
    #
    y_true = numpy.array([x[0] for x in y_true_and_pred])
    y_pred = numpy.array([x[1] for x in y_true_and_pred])

    #####################################################################
    ### presentation of results
    #####################################################################
    os.makedirs(results_dir, exist_ok = True)
    f_results = open(f'{results_dir}/prediction-results-%03d.txt' % kmeans.n_clusters, 'wt')
    _cm_ = confusion_matrix(y_true, y_pred)
    for i in range(_cm_.shape[0]):
        for j in range(_cm_.shape[1]):
            f_results.write(' %10d' % _cm_[i, j])
        f_results.write('\n')
    f_results.write('\n')
    print(classification_report(y_true, y_pred), file = f_results)
    f_results.close()
    #
    fig, axes = pyplot.subplots(nrows = 1, ncols = 2, figsize = (16, 7))
    #fig.suptitle(title)
    #
    plot_confusion_matrix(estimator = MyArgmaxForPredictedLabels(),
                          X = y_pred, y_true = y_true,
                          normalize = 'true', ax = axes[0], cmap = 'Blues', colorbar = False)
    #
    plot_confusion_matrix(estimator = MyArgmaxForPredictedLabels(),
                          X = y_pred, y_true = y_true,
                          normalize = 'pred', ax = axes[1], cmap = 'Oranges', colorbar = False)
    #
    pyplot.tight_layout()
    pyplot.savefig(f'{results_dir}/prediction-results-%03d.svg' % kmeans.n_clusters, format = 'svg')
    pyplot.savefig(f'{results_dir}/prediction-results-%03d.png' % kmeans.n_clusters, format = 'png')
    del fig


import json
import random
import re
from os import listdir
from os.path import exists, isdir, join as path_join, splitext as path_splitext
from typing import Literal, Union

from omegaconf import OmegaConf, listconfig, dictconfig
from pydantic import Field, validator

from invokeai.app.invocations.baseinvocation import (
    BaseInvocation,
    BaseInvocationOutput,
    InvocationContext,
    invocation,
    invocation_output,
    InputField,
    OutputField,
    UIType,
    WithMetadata,
    WithWorkflow,
)

@invocation_output("lookups_output")
class LookupTableOutput(BaseInvocationOutput):
    """Base class for invocations that output a JSON lookup table"""
    lookups: str = OutputField(default=None, description="The output lookup table")

@invocation_output("halved_prompt_output")
class HalvedPromptOutput(BaseInvocationOutput):
    """Base class for invocations that return a prompt as well as each half independently"""
    prompt: str = OutputField(default="", description="The output prompt")
    part_a: str = OutputField(default="", description="First part of the output prompt")
    part_b: str = OutputField(default="", description="Second part of the output prompt")

@invocation(
    "lookup_table_from_file",
    title="Lookup Table from File",
    tags=["prompt", "lookups", "grammar", "file"],
    category="prompt",
    version="1.1.0",
)
class LookupTableFromFileInvocation(BaseInvocation):
    """Loads a lookup table from a YAML file"""
    file_path: str = InputField(description="Path to lookup table YAML file")

    @validator("file_path")
    def file_path_exists(cls, v):
        if not exists(v):
            raise ValueError(FileNotFoundError)
        return v

    def parsedTemplateLines(self, lines: list):
        "Parse the rows of a template key, creating copies etc."
        results = []
        for line in lines:
            if isinstance(line, str):
                if (0 < len(line)) and (line[0] == "\\"):  # We use backslash to escape parsing
                    results.append(line[1:])
                else:
                    # - N * ..., where N is a number, adds N copies of prompt "..."
                    _match = re.match(r"\s*(\d+)\s*\*\s*(.*)", line)
                    if _match is not None:
                        for i in range(int(_match[1])):
                            results.append(_match[2])
                    # - ?N, where N is a number, is treated as an instruction to add N "empty" choices:
                    elif re.match(r"\?[\d]+$", line):
                        results.extend(["" for number_of_times in range(int(line[1:]))])
                    else:
                        results.append(line)
            elif isinstance(line, listconfig.ListConfig) and (len(line) == 3):
                for i in range(int(line[0])):
                    results.append(line[1:])
            elif isinstance(line, list):
                results.append(line)
            else:
                results.append(OmegaConf.to_object(line))
        return results

    def lookupTableFromFile(self, file_path: str):
        lookup_table = {}
        raw_conf = OmegaConf.load(file_path)
        prompt_section = raw_conf.get("prompt")
        if isinstance(prompt_section, dictconfig.DictConfig):
            for k, v in prompt_section.items():
                # Add singular "template" and "negative" entries
                #  to "templates" and "negatives" lists:
                if (k == "template"):
                    if "templates" not in lookup_table:
                        lookup_table['templates'] = []
                    lookup_table['templates'].extend(self.parsedTemplateLines([v]))
                elif (k == "negative"):
                    if "negatives" not in lookup_table:
                        lookup_table['negatives'] = []
                    lookup_table["negatives"].extend(self.parsedTemplateLines([v]))
                else:
                    if (k not in lookup_table):
                        lookup_table[k] = []
                    lookup_table[k].extend(self.parsedTemplateLines(v))
        return lookup_table

    def invoke(self, context: InvocationContext) -> LookupTableOutput:
        lookups = None
        if isdir(self.file_path):
            lookups = {}
            for pathname in listdir(self.file_path):
                if (not isdir(pathname)) and (path_splitext(pathname)[-1].lower() == ".yaml"):
                    pathname = path_join(self.file_path, pathname)
                    this_lookup_table = self.lookupTableFromFile(pathname)
                    for k, v in iter(this_lookup_table.items()):
                        if k not in lookups:
                            lookups[k] = []
                        lookups[k].extend(v)
        else:
            lookups = self.lookupTableFromFile(self.file_path)
        return LookupTableOutput(lookups=json.dumps(lookups))

@invocation(
    "lookup_from_prompt",
    title="Lookups Entry from Prompt",
    tags=["prompt", "lookups", "grammar"],
    category="prompt",
    version="1.1.0",
)
class LookupsEntryFromPromptInvocation(BaseInvocation):
    """Creates a lookup table of a single heading->value"""
    heading: str = InputField(default=None, description="Heading for the lookup table entry")
    lookup: str = InputField(default="", description="The entry to place under Heading in the lookup table")

    def invoke(self, context:InvocationContext) ->  LookupTableOutput:
        lookups = {}
        lookups[self.heading] = [self.lookup]
        return LookupTableOutput(lookups=json.dumps(lookups))

@invocation(
    "prompt_from_lookup_table",
    title="Prompt from Lookup Table",
    tags=["prompt", "lookups", "grammar"],
    category="prompt",
    version="1.1.0",
)
class PromptFromLookupTableInvocation(BaseInvocation):
    """Creates prompts using lookup table templates"""
    lookups: list[str] = InputField(
        description="Lookup table(s) containing templates (JSON)",
        default_factory=list,
    )
    remove_negatives: bool = InputField(default=False, description="Whether to strip out text between []")
    strip_parens_probability: float = InputField(default=0.0, ge=0.0, le=1.0,
                                            description="Probability of removing attention group weightings")

    @validator("lookups")
    def validate_lookups(cls, v):
        valid = False
        if isinstance(v, list):
            for i in v:
                loaded = json.loads(i)
                if "templates" in loaded:
                    valid = True
                    break
    #    else:
    #        if (not (v is None)) and ("templates" in json.loads(v)):
    #            valid = True
        if (not valid) and (not (len(v) == 0)):
            raise ValueError("'templates' key must be present in lookup table(s)")
        return v

    def templateExpand(self,
                       s :          str,
                       lookups :    dict,
                       reflection : str    = ""):
        "Used internally to replace words with their template lookups. Single pass."
        _split = re.split(r'({[\d:\w]+})', s)
        result = ""
        for word in _split:
            _lookup = None
            if re.fullmatch(r'({\w+})', word):
                _lookup = random.choice(lookups[word[1:-1]])
            else:
                _lookup = word
            if isinstance(_lookup, (list, listconfig.ListConfig)):
                result = result + _lookup[0]
                reflection = " " + _lookup[1] + reflection
            else:
                result = result + _lookup
        return result, reflection

    def cleanup(self, string_in : str):
        "Regex to clean formatting typos occurring during generation (TODO: improve)"
        # Condense whitespace
        _str = re.sub(r"\s+", " ", string_in, 0)
        _str = re.sub(r"\s([,;])\s", lambda match: (match.group(1) + " "), _str, 0)
        # Remove empty sets of parens
        _str = re.sub(r"\(\)[\+\-]*", "", _str)
        # Condense whitespace again
        _str = re.sub(r"\s+", " ", _str, 0)
        _str = re.sub(r"\s([,;])\s", lambda match: (match.group(1) +  " "), _str, 0)
        _str = re.sub(r",+\s", ", ", _str, 0)
        # Now we attempt to combine prefixes and suffixes.
        _collapsed = re.sub(r"(\S+?)\s*@@\s*(\S+)", lambda match: self.prefixer(match.group(1), match.group(2)), _str, 0)
        while _collapsed != _str:
            _str = _collapsed
            _collapsed = re.sub(r"(\S+?)\s*@@\s*(\S+)", lambda match: self.prefixer(match.group(1), match.group(2)), _str, 0)
        # Remove instances of "a apple" errors by subbing "an", w/ or w/o parens in the way.
        string_out = re.sub(
            r"(^[Aa]\s|\s[Aa]\s)([\(]*)([aeiouAEIOU])",
            lambda match: ((" a" if (len(match.group(1))==3) else "a") + "n " + match.group(2) + match.group(3)), _str, 0
        )
        return string_out

    def prefixer(self, a : str, b: str):
        "Attach pre/suffixes (indicated w/'@@') and rearrange parens/+/-"
        result = None
        if re.match(r"[\)\+\-]+$", b):
            result = a + b + " @@"
        else:
            # Allow a single trailing hyphen but move the rest of ), +, -:
            _aWeighting = re.match(r"\S*?(-?)([\)*\+\-]*)\s*$", a)
            _newA = a
            if _aWeighting is not None:
                _newA = a[0:_aWeighting.start(2)]
                _aWeighting = _aWeighting[2]
            # grab parens from the front of b:
            _bWeighting = re.match(r"\s*(\(*)(\S*)", b)
            _newB = b
            if _bWeighting is not None:
                _newB = _bWeighting[2]
                _bWeighting = _bWeighting[1]
            # now subtract the parens that cancel out:
            _newAWeighting = _aWeighting.replace(")", "", _bWeighting.count("("))
            _nSubs = len(_aWeighting) - len(_newAWeighting)
            _newBWeighting = _bWeighting.replace("(", "", _nSubs) if (0 < _nSubs) else _bWeighting
            _raw = _newBWeighting + _newA + _newB + _newAWeighting
            # subtract trailing +-'s that cancel:
            result = _raw.replace("+-", "", 1)
            while _raw != result:
                _raw = result
                result = _raw.replace("+-", "", 1)
        return result

    def invoke(self, context: InvocationContext) -> HalvedPromptOutput:
        lookups = {}
#        if isinstance(self.lookups, list):
        for lookup_table in reversed(self.lookups):
            lookup_table = json.loads(lookup_table)
            for k, v in iter(lookup_table.items()):
                if k not in lookups:
                    lookups[k] = []
                lookups[k].extend(v)
#        else:
#            lookups = json.loads(self.lookups)
        template_strings = lookups['templates']
        base_negatives = lookups['negatives'] if ('negatives' in lookups) else []
        result = None
        appendices = None

        for i in range(1):
            base, reflection = self.templateExpand(random.choice(template_strings), lookups=lookups, reflection="")
            next, reflection = self.templateExpand(base, lookups=lookups, reflection=reflection)
            while ((next != base) or (re.search(r'{\w+}', base))):
                base = next
                next, reflection = self.templateExpand(base, lookups=lookups, reflection=reflection)
            appendices = ""
            while reflection != "":
                appendix = reflection
                next, reflection = self.templateExpand(appendix, lookups=lookups, reflection="")
                while ((next != appendix) or (re.search(r'{\w+}', appendix))):
                    appendix = next
                    next, reflection = self.templateExpand(appendix, lookups=lookups, reflection=reflection)
                appendices = appendices + appendix

            if random.random() < self.strip_parens_probability:
                # Strip off parentheses and pluses, then trailing minuses w/o and w/ commas afterward
                base = re.sub(r"[\(\)+]", "", base, 0)
                base = re.sub(r"\-+\s", " ", base, 0)
                base = re.sub(r"\-+,", ",", base, 0)
                appendices = re.sub(r"[\(\)+]", "", appendices, 0)
                appendices = re.sub(r"\-+\s", " ", appendices, 0)
                appendices = re.sub(r"\-+,", ",", appendices, 0)
            if self.remove_negatives:
                # strip out anything between []
                base = re.sub(r"\[[^\]\[]*\]", "", base, 0)
                appendices = re.sub(r"\[[^\]\[]*\]", "", appendices, 0)

            base = self.cleanup(base)
            appendices = self.cleanup(appendices)

            if not (self.remove_negatives or (not base_negatives)):
                appendices = appendices + " " + random.choice(base_negatives)

            result = (base+appendices).strip()

        return HalvedPromptOutput(prompt=result, part_a=base.strip(), part_b=appendices.strip())


import sqlite3

#connect to database
db = sqlite3.connect('tour_merchandise.db')

#allows you to access columns by column name
db.row_factory = sqlite3.Row

#create cursor object to execute commands
cur = db.cursor()

#create a tables for the databse if they don't exist
cur.execute('create table if not exists venue(Venue text, Date date, Street text, City text, State text, ZipCode text)')
cur.execute('create table if not exists merchandise(ItemName text, Description text, Price float, Quantity integer)')
cur.execute('create table if not exists sales(Venue text, ItemName text, NumItems integer, '
            'FOREIGN KEY(Venue) REFERENCES venue(Venue), FOREIGN KEY(ItemName) REFERENCES merchandise(ItemName))')

#method used to add venues to the database
def add_venue_to_db(venue, concertDate, street, city, state, zipCode):
    try:
        with db:
            cur.execute('insert into venue values(?, ?, ?, ?, ?, ?)', (venue, concertDate, street, city, state, zipCode))
    except sqlite3.Error as error:
        print('Error adding venue.')
        print(error)

#method used to add new merchandise to database
def add_merch_to_db(itemName, description, price, quantity):
    try:
        with db:
            cur.execute('insert into merchandise values(?, ?, ?, ?)', (itemName, description, price, quantity))
    except sqlite3.Error as error:
        print('Error adding merchandise.')
        print(error)

#method used to add sales information to database
def add_sales_to_db(venue, itemName, numItems):
    try:
        with db:
            cur.execute('insert into sales values(?, ?, ?)', (venue, itemName, numItems))
    except sqlite3.Error as error:
        print('Error adding sale.')
        print(error)

#method to perform searches on the venue table
def search_venue(choice, venue, itemName):
    if choice == '1':
        results = cur.execute('select Max(NumItems) from sales group by venue')
        print(results[0])

#method used to perform searches on the merchandise table
def search_merchandise():
    return

#method used to perform searches on the sales table
def search_sales():
    return

#prints menu selections and gets/returns selection
def main_menu_print():
    print('1. Search')
    print('2. Add Venue')
    print('3. Add Merchandise')
    print('4. Add Sales Information')
    print('5. Exit')
    user_input = int(input('Enter the number of your selection. '))
    return user_input

#used to give the user a choice on which table to add/make changes
def secondary_menu_print():
    #get required information
    print('1. Venue')
    print('2. Merchandise')
    print('3. Sale')
    choice = input('Enter the number of your choice. ')
    return choice

#method used to query the database.
def search_record(choice):
    #determine which table to look in
    tableName = ''
    if choice == '1':
        tableName = 'venue'
        print('What search would you like to perform?')
        print('1. How much merchandise was sold at single venue.')
        print('2. Where was the most merchandise sold.')
        print('3. Where was the least merchandise sold.')
        search = input('Enter the number of your choice. ')
        if search == '1':
            venue = input('Enter the name of the venue. ')
            itemName = input('Enter the name of the item. ')
            search_venue(search, venue, itemName)
        elif search == '2':
            search_venue(2)
    elif choice == '2':
        tableName = 'merchandise'
        print('What search would you like to perform?')
        print('1. ')
        print('2. ')
        search = input('Enter the number of your choice. ')
        if search == '1':
            search_merchandise()
    elif choice == '3':
        tableName = 'sales'
        print('What search would you like to perform?')
        print('1. How much merchandise was sold at single venue.')
        print('2. What is the biggest seller.')
        print('3. What sells the least.')

    #print the results

#method to add records to the database
def add_venue():
    venue = input('What is the venue name? ')
    concertDate = input('What is the concert date? (yyyy-mm-dd) ')
    street = input('What is the street address? ')
    city = input('What city is the venue in? ')
    state = input('What state is the venue in? ')
    zipCode = input('Enter the Zip Code. ')
    #add record to database
    add_venue_to_db(venue, concertDate, street, city, state, zipCode)

def add_merchandise():
    #get new merchandise info
    itemName = input('What is the item name? ')
    description = input('Enter a description of the item. ')
    price = float(input('Enter a price for the item. '))
    quantity = int(input('How many of these do you have? '))
    #add record to database
    add_merch_to_db(itemName, description, price, quantity)

def add_sales():
    #get sale information
    venue = input('At what venue was this sale completed? ')
    itemName = input('What is the item name? ')
    numItems = input('How many were sold? ')
    #add record to database
    add_sales_to_db(venue, itemName, numItems)

#main method
def main():
    #display menu and get user input
    user_input = main_menu_print()
    #call appropriate method
    while user_input != 5:
        if user_input == 1:
            print('Where do you want to search?')
            choice = secondary_menu_print()
            search_record(choice)
        elif user_input == 2:
            add_venue()
        elif user_input == 3:
            add_merchandise()
        elif user_input == 4:
            add_sales()
        #re-display the menu
        user_input = main_menu_print()

    #when 5 is selected, exit the program
    exit()

main()


import csv
from datetime import datetime

def read_csv(csv_file, float_numbers: bool):
    with open(csv_file, newline='') as f:
        reader = csv.DictReader(f)
        data = []
        for row in reader:
            if '-' not in row["price"] and float(row["price"]) > 0:
                if float_numbers:
                    row["price"] = int(float(row["price"]) * 100)
                    row["profit"] = int(float(row["profit"]) * 100)
                else:
                    row["price"] = float(row["price"])
                    row["profit"] = float(row["profit"])
                data.append(row)

    return data

def timer(func):
    def wrapper(*args, **kwargs):
        start_time = datetime.now()
        func(*args, **kwargs)
        end_time = datetime.now()
        print(end_time - start_time)
        return func(*args)

    return wrapper


import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
}

res = requests.get(
    'https://www.thewhiskyexchange.com/c/35/japanese-whisky?pg=2#productlist-filter')

soup = BeautifulSoup(res.content, 'lxml')

product = soup.find('img', class_="product-main__image")['src']

print(product)


#I have made this file
from django.http import HttpResponse
from django.shortcuts import render

def index(request):
    #maps={'name':'jarvis','owner':'ironman'}
    #return render(request,'index.html',maps)  {{owner}} has {{name}} Write this in html
    #return HttpResponse("
No end point     hello
")
    return render(request,'index.html')

def about(request):
    return HttpResponse('''In about Hello Click Here''')

def intxt(request):
    f1=open("D:\python\one.txt","r")
    #return HttpResponse("
%s
"%(f1.read()))
    return HttpResponse("%s"%f1.read())

def analyze(request):
    #Get the text
    #print(request.GET.get('text','default'))   #print this in terminal window

    djtext=request.POST.get('text','default')
    rempunc=request.POST.get('removepunc','off')
    fulca=request.POST.get('fullcap','off')
    newlinerem=request.POST.get('newlineremove','off')
    spacerem=request.POST.get('spaceremove','off')
    chcount=request.POST.get('charcount','off')

    #print("remove punction:",rempunc)
    #print("full cap:",fulca)
    print(djtext)

    if rempunc=="on":
    #return HttpResponse("removepunc")
    #Now instead of Httprsponse we will use render
    #analyzed=djtext
        analyzed=""
        punctions=''',./';[]!@#$%^&*()_+\}{":?><'''
        for i in djtext:
            if i not in punctions:
                analyzed= analyzed + i

        para={'purpose':'Removed Punctuation','analyzed_text': analyzed}
        djtext=analyzed
        #return render(request,'analyze.html',para)

    if fulca =="on":
        analyzed=""
        for char in djtext:
            analyzed=analyzed + char.upper()

        para={'purpose':'Change to UPPERCASE','analyzed_text': analyzed}
        djtext=analyzed
        #return render(request,'analyze.html',para)

    if newlinerem == "on":
        analyzed=""
        for char in djtext:
            if char!="\n" and char!="\r":
                analyzed+= char
        para={'purpose':'New Line Removed','analyzed_text': analyzed}
        djtext=analyzed
        #return render(request,'analyze.html',para)

    if spacerem == "on":
        analyzed=""
        for index,char in enumerate(djtext):
            if not(djtext[index]==" " and djtext[index+1]==" "):
                analyzed+= char
        para={'purpose':'Space Removed','analyzed_text': analyzed}
        djtext=analyzed
        #return render(request,'analyze.html',para)

    if chcount == "on":
        para={'purpose':'Number Of charcters','analyzed_text': len(djtext)}
        #return render(request,'analyze.html',para)

    if rempunc!="on" and fulca !="on" and newlinerem != "on" and  spacerem != "on" and chcount != "on":
        return HttpResponse("ERROR")

    return render(request,'analyze.html',para)
#def removepunc(request):
#    #Get the text
#    #print(request.GET.get('text','default'))   #print this in terminal window
#    djtext=request.GET.get('text','default')
#    print(djtext)
#    return HttpResponse("removepunc")

#def capit(request):
#    return HttpResponse("capit")

#def newlineremove(request):
#    return HttpResponse("newlineremove")

#def spaceremove(request):
 #   return HttpResponse("spaceremove Back")
#def charcount(request):
#    return HttpResponse("charcount")


#%%
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import pandas as pd
import geopandas as gpd
import wlfun as wl
from arcgis.gis import GIS
import matplotlib.dates as mdates
from pygeostat.data import iotools
from sklearn.linear_model import LinearRegression
from shapely.geometry import Point
import matplotlib
from wlob import wlob
from scipy.spatial import cKDTree
from tqdm import tqdm
import cmcrameri.cm as cmc

sys.path.insert(0,'src')
import config

well_filter = 0
# %%
# Loading relevant shapefiles
# First, we must load some relevant shapefiles. We'll grab California, the Central Valley, the Corcorcan Clay, and the zone map for CVHM (to do: add CV2Sim). I project everything onto Albers as an arbitrary choice, but as long as all crs's for the dataframes align, any geographic projection should be fine.
CA_map = gpd.read_file(config.shapefile_dir / 'CA_State_TIGER2016.shp')
CA_map.to_crs(epsg=config.crs_latlon, inplace=True)

CV_map = gpd.read_file(config.shapefile_dir / 'Alluvial_Bnd.shp')
CV_map.to_crs(epsg=config.crs_latlon, inplace=True)

CC_map = gpd.read_file(config.shapefile_dir / 'corcoran_clay_depth_feet.shp')
CC_map.to_crs(epsg=config.crs_latlon, inplace=True)

#===============================
## Periodic Data
# %%
well_measurements = pd.read_csv(config.data_raw / 'CASGEM/measurements.csv', parse_dates=['msmt_date'])
well_stations = pd.read_csv(config.data_raw /'CASGEM/stations.csv')
well_perforations = pd.read_csv(config.data_raw / 'CASGEM/perforations.csv')

# calling the well object
wlo = wlob(well_stations,well_measurements,well_perforations, CA_map, CV_map, CC_map)

#Read CVHM zone data
zone_map = wlo.get_cvhm_lyr()
zone_map.info()

# %%
#plot study domain
wl.get_cv_map(CA = CA_map,CV = CV_map, corcoran = CC_map, zone_map = None)
# print(well_measurements["wlm_qa_desc"].unique())

# If interested to remove wells with specific comments, such as recent pumping. use following
if well_filter == 1:
    well_stations = wlo.well_filter_qa()
#well_stations.info()

# %%
# combine well perforation, stations, and measurements
well_msp = wlo.well_meas_perfor_combine()
well_msp.head()

#%%
cv_all_sites = wlo.get_well_from_use_region(filter_qa = None, well_use = None, all_wells = 1, CV_map = CV_map, CC_map = None)
cv_obs_sites = wlo.get_well_from_use_region(filter_qa = None, well_use = 'Observation', all_wells = None, CV_map = CV_map, CC_map = None)
cv_ag_sites = wlo.get_well_from_use_region(filter_qa = None, well_use = 'Irrigation', all_wells = None, CV_map = CV_map, CC_map = None)
cv_industry_sites = wlo.get_well_from_use_region(filter_qa = None, well_use = 'Industrial', all_wells = None, CV_map = CV_map, CC_map = None)
cv_resident_sites = wlo.get_well_from_use_region(filter_qa = None, well_use = 'Residential', all_wells = None, CV_map = CV_map, CC_map = None)

#%%
# Process zones and get shallow wells
shallow_wells_all = wlo.get_shallow_wells(zone_map, cv_all_sites,cv_obs_sites, well_type = 'all')
shallow_wells = wlo.get_shallow_wells(zone_map, cv_all_sites,cv_obs_sites, well_type = 'obs')

# thickness files are in meters
shallow_wells["layer1_thk"].describe()
# %%
#plt.figure(figsize=(8,6))

# %%
shallow_measurements = well_msp[well_msp["site_code"].isin(shallow_wells["site_code"])]
shallow_measurements_all = well_msp[well_msp["site_code"].isin(shallow_wells_all["site_code"])]
shallow_measurements.head()

#%%
shallow_measurements_all.to_csv(config.data_processed / 'shallow_wells_all_cvhm.csv')
#%%
st_list = ['351296N1193938W004','361600N1197800W001','362122N1192962W001','362301N1192828W001','362400N1198300W002']
wlo.get_wells_by_aquifer(cv_obs_sites,shallow_wells,st_list)

#%%

# Plot well time series if number of observation is above a threshold
site_codes = list(shallow_measurements["site_code"].unique())
fig = plt.figure(figsize=(16,12))
count = 0
for site_code in site_codes:
    single_well = shallow_measurements[shallow_measurements["site_code"].isin([site_code])]
    single_well = single_well.sort_values(["msmt_date"], ascending=[True])
    if single_well["gse_gwe"].count() > 100:
        plt.plot(single_well["msmt_date"], single_well["gse_gwe"], label=site_codes[count])
        count += 1
    if count == 5:
        break
plt.title('Select Monitoring Wells Time Series', fontsize=20)
plt.ylabel('Depth to Groundwater (ft)', fontsize=16)
plt.setp(plt.gca().get_xticklabels(), rotation=30, horizontalalignment='right')
plt.gca().xaxis.set_major_locator(mdates.YearLocator())
#plt.gca().invert_xaxis()
plt.grid(zorder=0)
plt.legend(fontsize=16)

# %%
wlo.get_well_plot_by_type(cv_sites = cv_resident_sites, well_label = 'Residential')
wlo.get_well_plot_by_type(cv_sites = cv_obs_sites, well_label = 'Observation')
wlo.get_well_plot_by_type(cv_sites = cv_ag_sites, well_label = 'Agricultural')
wlo.get_well_plot_by_type(cv_sites = cv_industry_sites, well_label = 'Industrial')
#%%
wlo.get_well_plot_by_type(cv_sites = cv_all_sites, well_label = 'All')

#----------------------------------
# well interpolation
#----------------------------------

#%%
CA_map = gpd.read_file(config.shapefile_dir / 'CA_State_TIGER2016.shp')
CA_map.to_crs(epsg='4326', inplace=True)

CV_map = gpd.read_file(config.shapefile_dir / 'Alluvial_Bnd.shp')
CV_map.to_crs(epsg='4326', inplace=True)
# %%

# user input
in_path = config.data_processed / 'shallow_wells_all_cvhm.csv'
target_year = '2021'
target_season = 'spring'
monitoring_weight = 0.8
n_realizations = 10000
crs_latlon='epsg:4326' # WGS 84
crs_projected='epsg:3488' # California Albers

x_ = np.arange(-123, -118, 0.10)
y_ = np.arange(34.5, 41, 0.10)
print(len(x_))
print(len(y_))
xx,yy = np.meshgrid(x_,y_,indexing = 'ij')
xx_flat = xx.reshape(-1)
yy_flat = yy.reshape(-1)

meshgrid_points = [Point(xy) for xy in zip(xx_flat,
                                           yy_flat)]
meshgrid_frame = gpd.GeoDataFrame(meshgrid_points, crs=crs_latlon, geometry=meshgrid_points)
# %%
new_binary = mpl.colors.ListedColormap(np.array([[1., 1., 1., 1.],
       [0., 0., 0., 0.]]))
CV_overlay = np.zeros(xx.shape)
CV_geom = CV_map['geometry'].values[0]
for i in tqdm(range(len(CV_overlay))):
    for j in range(len(CV_overlay[0,:])):
        potentialPoint = Point(xx[i,j], yy[i,j])
        if potentialPoint.within(CV_geom):
            CV_overlay[i,j] = 1
# %%
well_measurements = pd.read_csv(in_path, parse_dates=['msmt_date'])
print(len(well_measurements))
site_points = [Point(xy) for xy in zip(well_measurements["longitude"], well_measurements["latitude"])]

well_measurements = gpd.GeoDataFrame(well_measurements, crs=crs_latlon, geometry=site_points)
well_measurements = well_measurements.to_crs(crs_projected)
# %%
if target_season == 'spring':
    start_month = '01'
    mid_month = '03'
    end_month = '06'
else:
    start_month = '06'
    mid_month = '09'
    end_month='12'

mask = ((well_measurements['msmt_date'] > target_year+start_month+'01') &
        (well_measurements['msmt_date'] <= target_year+end_month+'01'))
# %%

timeframed_measurements = well_measurements.loc[mask]
timeframed_measurements = timeframed_measurements[timeframed_measurements['gse_gwe'].notnull()]
# %%

timeframed_measurements = timeframed_measurements[timeframed_measurements['gse_gwe'] > 0]
# %%
timeframed_measurements['drop'] = True
# %%
timeframed_measurements_x = timeframed_measurements['geometry'].x
timeframed_measurements_y = timeframed_measurements['geometry'].y
timeframed_measurements_lat = timeframed_measurements["latitude"]
timeframed_measurements_lon = timeframed_measurements["longitude"]
timeframed_measurements_z = timeframed_measurements['gse_gwe']
timeframed_measurements_type = timeframed_measurements["well_use"]
# %%

meshgrid_frame = meshgrid_frame.to_crs(crs_projected)
# %%

xx_flat = meshgrid_frame['geometry'].x
yy_flat = meshgrid_frame['geometry'].y
xx_yy = np.array((xx_flat,yy_flat)).T

# %%
# Stacking the x, y, and z data together in a new array
timeframed_measurements_xyz = np.vstack((timeframed_measurements_x, timeframed_measurements_y, timeframed_measurements_z)).T
# %%
bootstrap_timeframed_measurements_xyz = np.zeros((n_realizations, int(len(timeframed_measurements_xyz)*0.8), 3))
p_array = np.ones((int(len(timeframed_measurements_xyz))))*(1-monitoring_weight)/len(
    timeframed_measurements_type[timeframed_measurements_type!='Observation'])
p_array[timeframed_measurements_type=='Observation']=monitoring_weight/len(
timeframed_measurements_type[timeframed_measurements_type=='Observation'])
for i in range(len(bootstrap_timeframed_measurements_xyz)):
    bootstrap_indices = np.random.choice(np.arange(len(timeframed_measurements_xyz)),
                                         size=int(len(timeframed_measurements_xyz)*0.8),
                                        replace=False,
                                        p=p_array)
    bootstrap_timeframed_measurements_xyz[i] = timeframed_measurements_xyz[bootstrap_indices]
# %%
bootstrap_IDW = []
for i in tqdm(range(len(bootstrap_timeframed_measurements_xyz))):
    bootstrap_IDW.append(wl.tree(bootstrap_timeframed_measurements_xyz[i,:,:2],
                            bootstrap_timeframed_measurements_xyz[i,:,2]))

#%%
bootstrap_realizations = []
for i in tqdm(range(len(bootstrap_IDW))):
    z = bootstrap_IDW[i](xx_yy)
    bootstrap_realizations.append(z)

# %%
z_mean = np.array(bootstrap_realizations)
z_mean = np.reshape(z_mean, (n_realizations,len(x_),len(y_)))
z_mean = np.mean(z_mean, axis=0)

# %%
z_var = np.array(bootstrap_realizations)
z_var = np.sqrt(np.var(z_var, axis=0))
z_var = np.reshape(z_var, (len(x_),len(y_)))

# %%
fig, ax = plt.subplots(figsize=(15,15))
#CA_map.boundary.plot(ax=ax, zorder=3)
#plt.grid(zorder=2.5, color='k')
CV_map.boundary.plot(ax=ax, zorder=3)
plt.pcolormesh(xx, yy, z_mean, zorder=-2, cmap=cmc.batlow)
#plt.colorbar()
plt.colorbar().set_label(label='Groundwater Depth (ft)',size=20)
#plt.scatter(timeframed_measurements_lon, timeframed_measurements_lat, color='red', s=10)
plt.pcolormesh(xx, yy, CV_overlay, cmap=new_binary, zorder=-1)
plt.title('IDW interpolation: spring 2021 (mean of 10000 realizations)', fontsize=20)
plt.xlabel('Longitude', fontsize=20)
plt.ylabel('Latitude', fontsize=20)

#%%
# Export groundwater depth data
loc_file = f'{config.data_processed}/GWdepth_interpolated'
with open(f'{loc_file}/GWdepth_spring.npy', 'wb') as f:
    np.save(f, z_mean)

with open(f'{loc_file}/X_cv_gwdepth.npy', 'wb') as f:
    np.save(f, xx)
with open(f'{loc_file}/Y_cv_gwdepth.npy', 'wb') as f:
    np.save(f, yy)

# %%
fig, ax = plt.subplots(figsize=(15,15))
#CA_map.boundary.plot(ax=ax, zorder=3)
CV_map.boundary.plot(ax=ax, zorder=4)
plt.pcolormesh(xx, yy, z_var/z_mean, zorder=1, cmap=cmc.batlow)
plt.colorbar().set_label(label='Groundwater Depth Standard Deviation (ft)',size=20)
plt.pcolormesh(xx, yy, CV_overlay, cmap=new_binary, zorder=2)
#plt.scatter(timeframed_measurements_lon, timeframed_measurements_lat, color='red', s=10)
plt.scatter(timeframed_measurements_lon, timeframed_measurements_lat, color='red', s=10)
plt.title('IDW interpolation: spring 2021 variance', fontsize=20)
plt.xlabel('Longitude', fontsize=20)
plt.ylabel('Latitude', fontsize=20)
plt.grid(zorder=0)
#plt.savefig('OK_CV_only_vari.png', dpi=300)
    # %%

#%%
# Find ids for given coordinate
# Convert x y to meshgrid
coord = [-121.00000000000011, 37.50000000000004]

gwd_sel = wl.resistivity_value_from_xy(xx = xx, yy = yy,values = z_mean, coord = coord)
print(gwd_sel)


"""
Helper functions for working with S3 and STS
"""
import base64
import datetime
import hashlib
import hmac
import os
import posixpath
import sys
import urllib.parse
import uuid
from typing import List, Optional, Set, Tuple, TypedDict, Union

import boto3
import minio
from botocore.client import Config

from . import schemas

class Boto3ClientCache:
    """
    There may be many s3 nodes in the cluser.  Once a client has been established,
    cache it for future use.  This class works for sts and s3 type clients.
    """

    def __init__(self):
        self.cache: Dict[str, boto3.Session] = {}

    @staticmethod
    def _get_primary_key(client_type: str, node: schemas.StorageNodeOperator) -> str:
        if not client_type in ["s3", "sts"]:
            raise ValueError(f"{client_type} unsupported by cache")
        primary_key = (
            (
                f"{client_type}{node.region_name}{node.api_url}"
                f"{node.access_key_id}{node.secret_access_key}"
            )
            .lower()
            .encode("utf-8")
        )
        return hashlib.sha256(primary_key).hexdigest()

    def get_client(
        self,
        client_type: str,
        node: schemas.StorageNodeOperator,
    ) -> boto3.Session:
        primary_key_short_sha256 = Boto3ClientCache._get_primary_key(client_type, node)
        client = self.cache.get(primary_key_short_sha256, None)
        if client is None:
            config = Config()
            if client_type == "s3":
                config = Config(signature_version="s3v4")
                client = boto3.client(
                    client_type,
                    region_name=node.region_name,
                    endpoint_url=node.api_url,
                    aws_access_key_id=node.access_key_id,
                    aws_secret_access_key=node.secret_access_key,
                    config=config,
                )
            elif client_type == "sts":
                is_aws = node.assume_role_arn is not None
                api_url = node.api_url
                if node.sts_api_url:
                    api_url = node.sts_api_url
                elif is_aws:
                    api_url = f"https://sts.{node.region_name}.amazonaws.com"
                client = boto3.client(
                    client_type,
                    region_name=node.region_name,
                    endpoint_url=api_url,
                    aws_access_key_id=node.access_key_id,
                    aws_secret_access_key=node.secret_access_key,
                )
            else:
                raise NotImplementedError("Client type not implemented")
            self.cache[primary_key_short_sha256] = client
        return client

    def get_minio_sdk_client(self, node: schemas.StorageNodeOperator) -> minio.Minio:
        primary_key_short_sha256 = (
            Boto3ClientCache._get_primary_key("s3", node) + "minio"
        )
        client = self.cache.get(primary_key_short_sha256, None)
        url = urllib.parse.urlparse(node.api_url)
        if client is None:
            client = minio.Minio(
                url.netloc,
                access_key=node.access_key_id,
                secret_key=node.secret_access_key,
                secure=False,
            )
            self.cache[primary_key_short_sha256] = client
        return client

def sanitize(name: str) -> str:
    # TODO
    return name

def getWorkspaceKey(
    workspace: schemas.WorkspaceDB,
    root: Optional[schemas.WorkspaceRootDB] = None,
) -> str:
    """
    Determine full object prefix for a given workspace
    """
    root_base_path = root.base_path if root else workspace.root.base_path
    inner_path = posixpath.join(workspace.owner.username, sanitize(workspace.name))
    if workspace.base_path != None:
        inner_path = str(workspace.base_path)
    return posixpath.join(
        root_base_path,
        inner_path,
    ).strip("/")

S3V4Headers = TypedDict(
    "S3V4Headers",
    {
        "x-amz-date": str,
        "Authorization": str,
    },
)

def get_s3v4_headers(
    access_key: str,
    secret_key: str,
    region="us-east-1",
    service="s3",
    host="localhost:9000",
    endpoint="http://localhost:9000",
    uri="/",
    request_parameters="",
) -> S3V4Headers:
    method = "GET"
    # Key derivation functions. See:
    # http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
    def sign(key, msg):
        return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()

    def getSignatureKey(key, dateStamp, regionName, serviceName):
        kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp)
        kRegion = sign(kDate, regionName)
        kService = sign(kRegion, serviceName)
        kSigning = sign(kService, "aws4_request")
        return kSigning

    # Create a date for headers and the credential string
    t = datetime.datetime.utcnow()
    amzdate = t.strftime("%Y%m%dT%H%M%SZ")
    datestamp = t.strftime("%Y%m%d")  # Date w/o time, used in credential scope

    # ************* TASK 1: CREATE A CANONICAL REQUEST *************
    # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html

    # Step 1 is to define the verb (GET, POST, etc.)--already done.

    # Step 2: Create canonical URI--the part of the URI from domain to query
    # string (use '/' if no path)
    canonical_uri = uri

    # Step 3: Create the canonical query string. In this example (a GET request),
    # request parameters are in the query string. Query string values must
    # be URL-encoded (space=%20). The parameters must be sorted by name.
    # For this example, the query string is pre-formatted in the request_parameters variable.
    canonical_querystring = request_parameters

    # Step 4: Create the canonical headers and signed headers. Header names
    # must be trimmed and lowercase, and sorted in code point order from
    # low to high. Note that there is a trailing \n.
    canonical_headers = "host:" + host + "\n" + "x-amz-date:" + amzdate + "\n"

    # Step 5: Create the list of signed headers. This lists the headers
    # in the canonical_headers list, delimited with ";" and in alpha order.
    # Note: The request can include any headers; canonical_headers and
    # signed_headers lists those that you want to be included in the
    # hash of the request. "Host" and "x-amz-date" are always required.
    signed_headers = "host;x-amz-date"

    # Step 6: Create payload hash (hash of the request body content). For GET
    # requests, the payload is an empty string ("").
    payload_hash = hashlib.sha256(("").encode("utf-8")).hexdigest()

    # Step 7: Combine elements to create canonical request
    canonical_request = (
        method
        + "\n"
        + canonical_uri
        + "\n"
        + canonical_querystring
        + "\n"
        + canonical_headers
        + "\n"
        + signed_headers
        + "\n"
        + payload_hash
    )

    # ************* TASK 2: CREATE THE STRING TO SIGN*************
    # Match the algorithm to the hashing algorithm you use, either SHA-1 or
    # SHA-256 (recommended)
    algorithm = "AWS4-HMAC-SHA256"
    credential_scope = datestamp + "/" + region + "/" + service + "/" + "aws4_request"
    string_to_sign = (
        algorithm
        + "\n"
        + amzdate
        + "\n"
        + credential_scope
        + "\n"
        + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()
    )

    # ************* TASK 3: CALCULATE THE SIGNATURE *************
    # Create the signing key using the function defined above.
    signing_key = getSignatureKey(secret_key, datestamp, region, service)

    # Sign the string_to_sign using the signing_key
    signature = hmac.new(
        signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256
    ).hexdigest()

    # ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
    # The signing information can be either in a query string value or in
    # a header named Authorization. This code shows how to use a header.
    # Create authorization header and add to request headers
    authorization_header = (
        algorithm
        + " "
        + "Credential="
        + access_key
        + "/"
        + credential_scope
        + ", "
        + "SignedHeaders="
        + signed_headers
        + ", "
        + "Signature="
        + signature
    )

    # The request can include any headers, but MUST include "host", "x-amz-date",
    # and (for this scenario) "Authorization". "host" and "x-amz-date" must
    # be included in the canonical_headers and signed_headers, as noted
    # earlier. Order here is not significant.
    # Python note: The 'host' header is added automatically by the Python 'requests' library.
    headers: S3V4Headers = {
        "x-amz-date": amzdate,
        "Authorization": authorization_header,
    }
    return headers


# from loan import *
#https://xxxcomparelistxxx.azurewebsites.net/lista/a,b,c,d,e,f/listb/a,b,x
#http://127.0.0.1:5000/lista/a,b,c,d,e,f/listb/a,b,x

from flask import Flask
app = Flask(__name__)

def intersect(listx,listy):
    intersect = []
    not_in_y = []
    for elem_a in listx:
            if elem_a in listy:
                intersect.append(elem_a)
            else:
                not_in_y.append(elem_a)
    return intersect,not_in_y

@app.route("/lista/<a>/listb/<b>")
def compare(a,b):
    lista = a.split(',')
    listb = b.split(',')

    intersect_a_b, not_in_y  = intersect(lista,listb)
    dummy, not_in_x  = intersect(listb,lista)

    return {'lista':lista,
    'listb':listb,
    'intersection_a_b':intersect_a_b,
    'elements_a_not_in_listb':not_in_y,
    'elements_b_not_in_lista':not_in_x
    }

if __name__ == "__main__":
    app.run(debug=True)


#!/usr/bin/python3
import zipfile
import os
import sys
import random
import string

if '-h' in sys.argv:
    print("""Usage:
contestID""")
print(sys.argv)

probNames = sum([[j + str(i) for j in string.ascii_uppercase] for i in [''] + list(range(10))], [])

#exit()
#problemFile = "/home/ejudge/Downloads/archive_646_93349.zip"
#problemName = "propGraph"
#contestID = "000017"

problemTemplate = """[problem]
type = "standard"
scoring_checker = 0
interactive_valuer = 0
manual_checking = 0
check_presentation = 0
use_stdin
input_file = "input.txt"
combined_stdin
use_stdout
output_file = "output.txt"
combined_stdout
binary_input = 0
ignore_exit_code = 0
test_sfx = ""
test_pat = "%02d"
use_corr
corr_sfx = ""
corr_pat = "%02d.a"
use_info = 0
info_sfx = ""
info_pat = ""
use_tgz = 0
tgz_sfx = ""
tgz_pat = ""
tgzdir_sfx = ""
tgzdir_pat = ""
autoassign_variants = 0
disable_auto_testing = 0
disable_testing = 0
enable_compilation = 0
valuer_sets_marked = 0
disable_stderr = 0
normalization = "nl"
time_limit_millis = 2000
real_time_limit = 5
max_vm_size = 256M
max_stack_size = 256M
enable_text_form
"""
if (len(sys.argv) < 2):
    print('Not enought args')
    exit()

contestID = (6 * '0' + sys.argv[1])[-6:]
index = 1
if len(sys.argv) > 2:
    index = int(sys.argv[2])
print('Now enter ID and checker')
while(1):
    dat = input().split()
    archivePath = os.path.join('/home', 'ejudge', 'archives',
        'archive_%s.zip' % dat[0])
    zf = zipfile.ZipFile(archivePath)
    #problemName = string.ascii_uppercase[index - 1]
    problemFolder = os.path.join(contestID, 'problems')
    problemName = 'error'
    probIndex = -1
    if not os.path.isdir(problemFolder):
        os.makedirs(problemFolder)
    prev = os.listdir(problemFolder)
    for index, i in enumerate(probNames):
        if i not in prev:
            problemName = i
            probIndex = index + 1
            break
    print('\ncreate problem with {} name and id={}\n'.format(problemName, str(probIndex)))

    path = os.path.join(contestID, 'problems', problemName)

    if not os.path.isdir(path):
        os.makedirs(path)
    else:
        print('Contest already have problem with this ID')
        quit()
        break
    zf.extractall(path)

    probID = probIndex
    servePath = os.path.join(contestID, 'conf', 'serve.cfg')
    serveFile = open(servePath, 'r').read()
    probCfg = problemTemplate
    probCfg += "id = " + str(probID) + '\n'
    probCfg += 'short_name = "{}"\n'.format(problemName)
    if (len(dat) > 1):
        if dat[1] != '-':
            probCfg += 'standard_checker = "{}"\n'.format(dat[1])
            if 'double' in dat[1]:
                probCfg += 'checker_env = "EPS=1e-3"\n'
        else:
            curPath = os.path.realpath('.')
            os.chdir(path)
            files = os.listdir('.')
            if 'checker.dpr' in files:
                print('Found checker.dpr, compiling...')
                os.system('cp "/home/ejudge/archives/testlib.pas" ./')
                os.system('fpc checker.dpr > fpcOutput.txt')
                probCfg += 'check_cmd = "checker"\n'
                print('Ok, set non-standart checker')
            elif 'checker.cpp' in files:
                print('Found checker.cpp, compiling...')
                os.system('cp "/home/ejudge/archives/testlib.h" ./')
                os.system('g++ checker.cpp -o checker > gccOutput.txt')
                probCfg += 'check_cmd = "checker"\n'
                print('Ok, set non-standart checker')
            elif 'checker.pas' in files:
                print('Found checker.pas, compiling...')
                os.system('cp "/home/ejudge/archives/testlib.pas" ./')
                os.system('fpc checker.pas > fpcOutput.txt')
                probCfg += 'check_cmd = "checker"\n'
                print('Ok, set non-standart checker')
            else:
                checkPath = os.path.join('/home/ejudge/checkers/', 'checker_{}.cpp'.format(dat[0]))
                if os.path.isfile(checkPath):
                    os.system('cp {} ./'.format(checkPath))
                    os.system('cp "/home/ejudge/archives/testlib.h" ./')
                    os.system('g++ checker_{}.cpp -o checker > gccOutput.txt'.format(dat[0]))
                    probCfg += 'check_cmd = "checker"\n'
                    print('Ok, set non-standart checker')
                else:
                    print('no checker for this problem error path\n', checkPath)

            os.chdir(curPath)

    titlePath = '/home/ejudge/archives/names/' + dat[0]
    if os.path.isfile(titlePath):
        probCfg += 'long_name = "{}"\n'.format(open(titlePath).read().strip())
    open(servePath, 'a').write(probCfg)
    index += 1


# class that will be used to get the information from the ipinfo.io website
import ipinfo
from setup_log import setup_logger
# token = "7269b8c71f592f" # my privet token

# setting up log File
log = setup_logger.setup_logger(logger_name="ipinfo_py", log_file='log_files\ipinfo_class.log')

# creating a class to get lat and log
class LatLon():
    def __init__(self):
        try:
            log.info("Connecting to ipinfo api")
            # connection to ipinof api through the api token
            self.token = "7269b8c71f592f" # my secret token
            self.handler = ipinfo.getHandler(self.token)
            self.res = self.handler.getDetails()
            log.info("Connection to ipinfo api successful")
        except Exception as e:
            log.info("Connection to ipinfo api FAILED:- " + str(e))

    def get_latlon(self):
        """Function will return Latitude and Longitude"""
        try:
            log.info("Getting Latitude and Longitude")
            # getting the latitude and longitude from the ipinfo api
            lat = self.res.latitude
            lon = self.res.longitude
            return lat, lon
        except Exception as e:
            log.info("Failed to returning Latitude Longitude:- " + str(e))


import os
import pathlib
import unittest
from unittest.mock import patch

import yaml
from PIL import Image

from yaticker.dashboard import Dashboard
from yaticker.yaticker import YaTicker

class TestUtil(unittest.TestCase):
    TEST_CONFIG_NAME = "test_config.yaml"

    @classmethod
    def setUpClass(cls):
        cls.config_dict = {
            "watchlist": ["FOO", "BAR"],
            "cycle": False,
            "updatefrequency": 10,
            "showvolume": True,
            "period": "1y",
        }
        with open(cls.TEST_CONFIG_NAME, "w") as outfile:
            yaml.dump(cls.config_dict, outfile)

    @classmethod
    def tearDownClass(cls):
        os.remove(cls.TEST_CONFIG_NAME)

    @patch.object(Dashboard, "__abstractmethods__", set())
    def test_init(self):
        dashboard = Dashboard(width=200, height=100, dpi=100)
        self.assertEqual(200, dashboard.width)
        self.assertEqual(100, dashboard.height)
        self.assertEqual(100, dashboard.dpi)
        self.assertEqual(True, dashboard.cycle)
        self.assertEqual(300, dashboard.update_frequency)
        self.assertEqual(False, dashboard.show_volume)
        self.assertEqual("5d", dashboard.period)
        self.assertEqual(["AMZN", "FB", "APPL"], dashboard.watchlist)

    def test_load_config(self):
        expected_config = self.config_dict
        actual_config = Dashboard.load_config(self.TEST_CONFIG_NAME)
        self.assertDictEqual(expected_config, actual_config)

    def test_load_config_None(self):
        expected_config = {}
        actual_config = Dashboard.load_config(None)
        self.assertDictEqual(expected_config, actual_config)

    def test_stock_graph_resolution(self):
        data = YaTicker.get_ticker_data(ticker="amc", period="5d", interval="1h")
        stock_image = Image.open(
            Dashboard.stock_graph(data, filename="test_candle.png")
        )
        width, height = stock_image.size
        print(f"directory: {pathlib.Path().absolute()}")
        self.assertAlmostEqual(264, width, delta=1)
        self.assertAlmostEqual(116, height, delta=1)

if __name__ == "__main__":
    unittest.main()