import datetime
import logging
import os
import subprocess
import typing
import urllib.parse

from dnas.config import config as cfg

class git:
    """
    A class for commiting and pushing files to GitHub.
    """

    @staticmethod
    def add(filename: str = None):
        """
        Add files to the git index, to be commited.
        """
        if not filename:
            raise ValueError(
                f"Missing required arguments: filename={filename}."
            )

        if type(filename) != str:
            raise TypeError(
                f"filename is not a string: {type(filename)}"
            )

        ret = subprocess.run(
            ["git", "add", filename],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't add file {filename} to git index:\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(f"Added {filename} to git index")

    @staticmethod
    def clean():
        """
        Remove any untracked files
        """
        ret = subprocess.run(
            ["git", "clean", "-f"],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't git clear -f:\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(f"Removed untracked fit files in {cfg.GIT_BASE}")

    @staticmethod
    def clear():
        """
        Remove all files currently in the git index for commit.
        """
        ret = subprocess.run(
            ["git", "restore", "--staged", "*"],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't clear git index of existing staged entries:\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(f"Cleared git index in {cfg.GIT_BASE}")

    @staticmethod
    def clone():
        """
        Clone the DNS Stats repo.
        """
        os.makedirs(cfg.GIT_BASE, exist_ok = True)

        ret = subprocess.run(
            ["git", "clone", cfg.GIT_STAT_CLONE_URL],
            cwd=cfg.BASE_DIR,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't clone git repo {cfg.GIT_STAT_CLONE_URL} to directory "
                f"{cfg.BASE_DIR}:\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(
            f"Cloned git repo {cfg.GIT_STAT_CLONE_URL} to {cfg.BASE_DIR}"
        )

    @staticmethod
    def commit(msg: str = None):
        """
        Commit staged changes to git with commit message "msg".
        """
        if not msg:
            raise ValueError(
                f"Missing required arguments: msg={msg}."
            )

        if type(msg) != str:
            raise TypeError(
                f"msg is not a string: {type(msg)}"
            )

        ret = subprocess.run(
            ["git", "commit", "-m", msg],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            if "Your branch is up to date with" in ret.stdout.decode():
                raise ChildProcessError(
                    "Nothing committed, no changes with remote"
                )
            else:
                raise ChildProcessError(
                    f"Couldn't commit staged changes to git in {cfg.GIT_BASE}\n"
                    f"args: {ret.args}\n"
                    f"stdout: {ret.stdout.decode()}\n"
                    f"stderr: {ret.stderr.decode()}"
                )
        logging.debug(f"Committed to git in {cfg.GIT_BASE}: {msg}")

    @staticmethod
    def diff():
        """
        Return True if there are files in the git index, with uncommitted
        changes, else False.
        """
        ret = subprocess.run(
            ["git", "diff", "--cached"],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't check if git has staged changed:\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        if ret.stdout.decode() == "":
            logging.debug(f"No changes staged in git cache in {cfg.GIT_BASE}")
            return False
        else:
            logging.debug(
                f"Changes are staged git in cache in {cfg.GIT_BASE}")
            return True

    @staticmethod
    def git_exists():
        """
        Return True if DNAS Stats repo exists locally.
        """
        if not os.path.isdir(cfg.GIT_BASE):
            logging.debug(f"Local git directory doesn't exist: {cfg.GIT_BASE}")
            return False

        ret = subprocess.run(
            ["git", "status"],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            logging.debug(f"No git repo detected in {cfg.GIT_BASE}")
            return False

        logging.debug(f"Git repo found in {cfg.GIT_BASE}")
        return True

    @staticmethod
    def gen_git_path_ymd(ymd: str = None) -> str:
        """
        Generate and return the path to the report files for a specific date.
        """
        if not ymd:
            raise ValueError(
                f"Missing required arguments: ymd={ymd}."
            )

        if type(ymd) != str:
            raise TypeError(
                f"ymd is not a string: {type(ymd)}"
            )

        day = datetime.datetime.strptime(ymd, cfg.DAY_FORMAT)

        git_dir = os.path.join(cfg.GIT_BASE, str(day.year))
        git_dir = os.path.join(git_dir, f"{day.month:02}")
        git_dir = os.path.join(git_dir, f"{day.day:02}")

        return git_dir

    @staticmethod
    def gen_git_url_ymd(ymd: str = None) -> str:
        """
        Generate and return the URL to the report files for a specific date.
        """
        if not ymd:
            raise ValueError(
                f"Missing required arguments: ymd={ymd}."
            )

        if type(ymd) != str:
            raise TypeError(
                f"ymd is not a string: {type(ymd)}"
            )

        day = datetime.datetime.strptime(ymd, cfg.DAY_FORMAT)

        git_url = os.path.join(str(day.year), f"{day.month:02}")
        git_url = os.path.join(git_url, f"{day.day:02}")
        git_url = urllib.parse.urljoin(cfg.GIT_STAT_BASE_URL, git_url)

        return git_url

    @staticmethod
    def pull():
        """
        Perform a git pull to make sure the local repo is up to date
        """
        ret = subprocess.run(
            ["git", "pull"],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't pull from remote branch "
                f"{cfg.GIT_REPORT_BRANCH}\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(
            f"Git pull succeeded"
        )

    @staticmethod
    def push():
        """
        Push commits to GitHub.
        """
        ret = subprocess.run(
            ["git", "push", "origin", cfg.GIT_REPORT_BRANCH],
            cwd=cfg.GIT_BASE,
            capture_output=True,
        )
        if ret.returncode != 0:
            raise ChildProcessError(
                f"Couldn't push commits to remote branch "
                f"{cfg.GIT_REPORT_BRANCH}\n"
                f"args: {ret.args}\n"
                f"stdout: {ret.stdout.decode()}\n"
                f"stderr: {ret.stderr.decode()}"
            )
        logging.debug(
            f"Pushed commit(s) to remote branch {cfg.GIT_REPORT_BRANCH}"
        )


import scrapy

class JobsSpider(scrapy.Spider):
    name = "jobs"
    allowed_domains = ["craigslist.org"]
    start_urls = ['https://vancouver.craigslist.org/search/sof']

    def parse(self, response):
        titles = response.xpath('//a[@class="result-title hdrlnk"]/text()').extract()
        print(titles)


from csv import reader
from flask import Flask, request, Response
from flask_jwt_extended import JWTManager
from json import dumps
from redis import Redis
from shared.config import ShopConfiguration
from shared.errors import MissingField, ValidationError
from shared.shop_models import db
from shared.utils import check_is_warehouse
from tempfile import NamedTemporaryFile

app = Flask(__name__)
app.config.from_object(ShopConfiguration)
jwt = JWTManager(app)

@app.route('/update', methods=['POST'])
@check_is_warehouse()
def update():
    if not request.files or not 'file' in request.files:
        raise MissingField('file')
    values: list[str] = []
    with NamedTemporaryFile(suffix='.csv') as tf:
        request.files['file'].save(tf.name)
        with open(tf.name) as f:
            csv_file = reader(f)
            for index, row in enumerate(csv_file):
                if len(row) != 4:
                    raise ValidationError(f'Incorrect number of values on line {index}.')
                categories = row[0].split('|')
                name = row[1]
                quantity = 0
                price = 0.0
                try:
                    quantity = int(row[2])
                    price = float(row[3])
                except ValueError:
                    # Error reporting happens below
                    pass
                if quantity <= 0:
                    raise ValidationError(f'Incorrect quantity on line {index}.')
                if price <= 0:
                    raise ValidationError(f'Incorrect price on line {index}.')
                values.append(dumps({
                    'name': name,
                    'quantity': quantity,
                    'price': price,
                    'categories': categories
                }))
    with Redis(app.config['REDIS_HOST']) as client:
        client.rpush('products', *values)
    return Response(status=200)

if __name__ == '__main__':
    db.init_app(app)
    app.run('0.0.0.0', 5002, debug=True)


import numpy as np

from whylogs.core.metrics.model_metrics import ModelMetrics
from whylogs.proto import ModelProfileMessage, ModelType

SUPPORTED_TYPES = ("binary", "multiclass")

class ModelProfile:
    """
    Model Class for sketch metrics for model outputs

    Attributes
    ----------
    metrics : ModelMetrics
        the model metrics object
    model_type : ModelType
        Type of mode, CLASSIFICATION, REGRESSION, UNKNOWN, etc.
    output_fields : list
        list of fields that map to model outputs
    """

    def __init__(self, output_fields=None, metrics: ModelMetrics = None):
        super().__init__()

        if output_fields is None:
            output_fields = []
        self.output_fields = output_fields
        if metrics is None:
            metrics = ModelMetrics()
        self.metrics = metrics

    def add_output_field(self, field: str):
        if field not in self.output_fields:
            self.output_fields.append(field)

    def compute_metrics(
        self,
        targets,
        predictions,
        scores=None,
        model_type: ModelType = None,
        target_field=None,
        prediction_field=None,
        score_field=None,
    ):
        """
        Compute and track metrics for confusion_matrix

        Parameters
        ----------
        targets : List
            targets (or actuals) for validation, if these are floats it is assumed the model is a regression type model
        predictions : List
            predictions (or inferred values)
        scores : List, optional
            associated scores for each prediction (for binary and multiclass problems)
        target_field : str, optional
        prediction_field : str, optional
        score_field : str, optional (for binary and multiclass problems)

        Raises
        ------
        NotImplementedError

        """

        metric_type = self.metrics.init_or_get_model_type(scores)

        if metric_type == ModelType.REGRESSION:
            self.metrics.compute_regression_metrics(
                predictions=predictions,
                targets=targets,
                target_field=target_field,
                prediction_field=prediction_field,
            )

        elif metric_type == ModelType.CLASSIFICATION:

            # if score are not present set them to 1.
            if scores is None:
                scores = np.ones(len(targets))

            scores = np.array(scores)

            # compute confusion_matrix
            self.metrics.compute_confusion_matrix(
                predictions=predictions,
                targets=targets,
                scores=scores,
                target_field=target_field,
                prediction_field=prediction_field,
                score_field=score_field,
            )
        else:
            raise NotImplementedError(f"Model type {metric_type} not supported yet")

    def to_protobuf(self):
        return ModelProfileMessage(output_fields=self.output_fields, metrics=self.metrics.to_protobuf())

    @classmethod
    def from_protobuf(cls, message: ModelProfileMessage):
        # convert google.protobuf.pyext._message.RepeatedScalarContainer to a list
        output_fields = [f for f in message.output_fields]
        return ModelProfile(
            output_fields=output_fields,
            metrics=ModelMetrics.from_protobuf(message.metrics),
        )

    def merge(self, model_profile):
        if model_profile is None:
            return self

        output_fields = list(set(self.output_fields + model_profile.output_fields))
        metrics = self.metrics.merge(model_profile.metrics)

        return ModelProfile(output_fields=output_fields, metrics=metrics)


#!/usr/bin/env python3

import sys
import re
import datetime
from optparse import OptionParser

class Info(object):
    def __init__(self, name, summary=None, description=None, synopsis=None, copyright=None, options=None):
        self.name = name
        self.summary = summary
        self.description = description
        self.synopsis = synopsis
        self.copyright = copyright
        self.options = options  # list of dictionaries with keys ["long_name", "short_name", "description", "arg_description"]

    def gen_rst(self):
        rst  = ".. -*- coding: utf-8 -*-\n\n"
        rst += "%s\n" % ("=" * len(self.name),)
        rst += "%s\n" % self.name
        rst += "%s\n\n" % ("=" * len(self.name),)

        # Add summary
        if self.summary:
            rst += "%s\n" % ("-" * len(self.summary))
            rst += "%s\n" % self.summary
            rst += "%s\n" % ("-" * len(self.summary))
            rst += "\n"

        # Add copyright
        if self.copyright:
            rst += ":Copyright: %s\n" % self.copyright

        # Add manual page section
        rst += ":Manual section: 8\n"

        # Add date
        rst += ":Date: $Date: %s $\n\n" % datetime.datetime.strftime(datetime.datetime.utcnow(), format="%F %X")

        # Add synopsis
        if self.synopsis:
            rst += "SYNOPSIS\n"
            rst += "========\n\n"
            for line in self.synopsis:
                rst += "%s\n\n" % line
            rst += "\n"

        # Add description
        if self.description:
            rst += "DESCRIPTION\n"
            rst += "===========\n\n"
            for line in self.description:
                rst += "%s\n\n" % line
            rst += "\n"

        # Add options
        rst += "OPTIONS\n"
        rst += "=======\n"

        for command in self.options:
            cmd  = ""
            if command["short_name"]:
                cmd += "-%s " % command["short_name"]
            cmd += "--%s" % command["long_name"]
            if command["arg_description"]:
                cmd += " %s" % command["arg_description"]

            rst += "%s\n" % cmd
            rst += "%s\n" % ("-" * len(cmd))
            rst += "%s\n" % command["description"]

            rst += "\n"

        return rst

def parse_arguments_from_c_file(filename):
    args = []

    try:
        content = open(filename, "r").read()
    except IOError:
        print("Error: Cannot open file %s" % filename)
        return args

    re_cmd_entries = re.compile(r"\s*(static|const)[ ]+GOptionEntry[^{]*{(?P<entries>.*)\s*NULL\s*}[,]?\s*};", re.MULTILINE|re.DOTALL)
    match = re_cmd_entries.search(content)
    if not match:
        print("Warning: Cannot find GOptionEntry section in %s" % filename)
        return args

    re_single_entry = re.compile(r"""{\s*"(?P<long_name>[^"]*)"\s*,        # long name
                                     \s*'?(?P<short_name>[^',]*)'?\s*,     # short name
                                     \s*[^,]*\s*,                          # flags
                                     \s*[^,]*\s*,                          # arg type
                                     \s*[^,]*\s*,                          # arg data pointer
                                     \s*("(?P<description>.*?)")\s*,       # description
                                     \s*"?(?P<arg_description>[^"}]*)"?    # arg description
                                  \s*},""", re.MULTILINE|re.DOTALL|re.VERBOSE)
    raw_entries_str = match.group("entries")

    start = 0
    entry_match = re_single_entry.search(raw_entries_str)
    i = 1
    while entry_match:
        long_name = entry_match.group("long_name").strip()
        short_name = entry_match.group("short_name").strip()
        description = entry_match.group("description").strip()
        arg_description = entry_match.group("arg_description").strip()

        # Normalize short name
        if short_name in ("NULL", "0", "\\0"):
            short_name = None

        # Normalize description
        description = description.replace('\\"', "\\\\'")
        description = description.replace("\\\\'", '"')
        description = description.replace('"', "")
        description = description.replace('\n', '')
        description = description.replace('\t', '')

        # Remove multiple whitespaces from description
        while True:
            new_description = description.replace("  ", " ")
            if new_description == description:
                break
            description = new_description
        description = description.strip()

        # Normalize arg_description
        if arg_description and (arg_description == "NULL" or arg_description == "0"):
            arg_description = None

        # Store option into list
        args.append({'long_name': long_name,
                     'short_name': short_name,
                     'description': description,
                     'arg_description': arg_description
                     })

        # Continue to next option
        i += 1
        start += entry_match.end(0)
        entry_match = re_single_entry.search(raw_entries_str[start:])
    # End while

    print("Loaded %2d arguments" % (i,), file=sys.stderr)
    return args

if __name__ == "__main__":
    parser = OptionParser('usage: %prog [options] <filename> [--mergerepo|--modifyrepo|--sqliterepo]')
    parser.add_option('-m', '--mergerepo', action="store_true", help="Gen rst for mergerepo")
    parser.add_option('-r', '--modifyrepo', action="store_true", help="Gen rst for modifyrepo")
    parser.add_option('-s', '--sqliterepo', action="store_true", help="Gen rst for sqliterepo")
    options, args = parser.parse_args()

    if len(args) < 1:
        print("Error: Must specify a input filename. (Example: ../src/cmd_parser.c)", file=sys.stderr)
        sys.exit(1)

    args = parse_arguments_from_c_file(args[0])

    if options.mergerepo:
        NAME = "mergerepo_c"
        info = Info(NAME,
                summary="Merge multiple rpm-md format repositories together",
                synopsis=["%s [options] --repo repo1 --repo repo2" % (NAME,)],
                options=args)
    elif options.modifyrepo:
        NAME = "modifyrepo_c"
        info = Info(NAME,
                summary="Modify a repomd.xml of rpm-md format repository",
                synopsis=["%s [options] <input metadata> <output repodata>" % (NAME,),
                          "%s --remove <metadata type> <output repodata>" % (NAME,),
                          "%s [options] --batchfile <batch file> <output repodata>" % (NAME,) ],
                options=args)
    elif options.sqliterepo:
        NAME = "sqliterepo_c"
        info = Info(NAME,
                summary="Generate sqlite db files for a repository in rpm-md format",
                synopsis=["%s [options] <repo_directory>" % (NAME,) ],
                options=args)
    else:
        NAME = "createrepo_c"
        info = Info(NAME,
                summary="Create rpm-md format (xml-rpm-metadata) repository",
                synopsis=["%s [options] <directory>" % (NAME,)],
                description=["Uses rpm packages from <directory> to create repodata.",
                             "If compiled with libmodulemd support modular metadata inside <directory> identified by the patterns below are automatically collected, merged and added to the repodata.",
                             "The patterns are:",
                             " - \*.modulemd.yaml (recommended file name: N:S:V:C:A.modulemd.yaml)",
                             " - \*.modulemd-defaults.yaml (recommended file name: N.modulemd-defaults.yaml)",
                             " - modules.yaml (recommended way of importing multiple documents at once)"],
                options=args)

    ret = info.gen_rst()
    if not ret:
        print("Error: Rst has not been generated", file=sys.stderr)
        sys.exit(1)

    print(ret)


from pipeline_dsl.resources.resource import AbstractResource, ConcourseResource
from pipeline_dsl.concourse import concourse_context
from typing import Optional
import os
import re

class GithubReleaseResource:
    def __init__(self, name: str) -> None:
        self.name = name
        self.path = os.path.abspath(self.name)

    def __str__(self) -> str:
        return self.name

    def tag(self, default: str = None) -> Optional[str]:
        if concourse_context():
            with open(os.path.join(self.path, "tag")) as f:
                return f.read().strip()
        return default

class GithubRelease(AbstractResource[GithubReleaseResource]):
    def __init__(
        self,
        owner: str,
        repo: str,
        access_token: str = None,
        pre_release: bool = False,
        release: bool = True,
        github_api_url: str = None,
        github_v4_api_url: str = None,
        github_uploads_url: str = None,
        tag_filter: str = None,
        order_by: str = None,
    ):
        self.owner = owner
        self.repo = repo
        self.access_token = access_token
        self.pre_release = pre_release
        self.release = release
        self.github_api_url = github_api_url
        self.github_v4_api_url = github_v4_api_url
        if github_api_url and not github_v4_api_url:
            self.github_v4_api_url = re.sub(r"/v[0-9]/*", "/graphql", github_api_url)
        self.github_uploads_url = github_uploads_url
        self.tag_filter = tag_filter
        self.order_by = order_by

    def resource_type(self) -> Optional[dict]:
        return {
            "name": "github-release",
            "type": "docker-image",
            "source": {
                "repository": "concourse/github-release-resource",
                "tag": "1.6.3",
            },
        }

    def concourse(self, name: str) -> ConcourseResource:
        result = ConcourseResource(
            name=name,
            type="github-release",
            icon="github",
            source={
                "owner": self.owner,
                "repository": self.repo,
                "access_token": self.access_token,
                "pre_release": self.pre_release,
                "release": self.release,
                "github_api_url": self.github_api_url,
                "github_v4_api_url": self.github_v4_api_url,
                "github_uploads_url": self.github_uploads_url,
                "tag_filter": self.tag_filter,
                "order_by": self.order_by,
            },
        )
        result.source = dict(
            filter(lambda x: x[1] is not None, result.source.items()),
        )
        return result

    def get(self, name: str) -> GithubReleaseResource:
        return GithubReleaseResource(name)


import numpy as np
import sys
import json
import triton_python_backend_utils as pb_utils
import transformers

class TritonPythonModel:
    def initialize(self, args):
        self.model_config = model_config = json.loads(args['model_config'])
        output_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0")
        self.output_dtype = pb_utils.triton_string_to_numpy(output_config['data_type'])

    def execute(self, requests):
        output_dtype = self.output_dtype
        responses = []
        out_tensor = []
        for request in requests:
            xgb_class = pb_utils.get_input_tensor_by_name(request, "xgb_class")
            tf_class = pb_utils.get_input_tensor_by_name(request, "tf_class")
            sci_1_class = pb_utils.get_input_tensor_by_name(request, "sci_1_class")
            sci_2_class = pb_utils.get_input_tensor_by_name(request, "sci_2_class")
            out_tensor.append(pb_utils.Tensor("OUTPUT0",
            (xgb_class.as_numpy()
            + tf_class.as_numpy()
            + sci_1_class.as_numpy()
            + sci_2_class.as_numpy()) / 4.0))
            inference_response = pb_utils.InferenceResponse(output_tensors = out_tensor)
            responses.append(inference_response)
        return responses


"""
Clean reenrollment asssessment table from raw
"""
from optparse import OptionParser
from tulsa.config import create_engine_from_config_file
from tulsa.etl.sql_str_normer import normalize_colname, normalize_tablename
from collections import Counter
import sys
import json

def load_intorbust(engine):
    '''
    load the intorbust sql function into the connection.
    intorbust makes all the values of a column an int or null if not possible.

    :param engine: the engine to load intorbust into
    :type engine: sqlalchemy engine
    :param sqlfile: the filename/location of the intorbust code
    :type sqlfile: str
    '''

    sql_str = r'''create or replace function isdigits(text) returns boolean
       as 'select $1 ~ ''^(-)?[0-9]+$'' as result'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(text) returns int
    as 'select
       	  case isdigits($1) when TRUE then cast($1 as int)
	  else null
	  end'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(numeric) returns numeric
       as 'select $1'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(float) returns float
       as 'select $1'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;
    '''
    engine.execute(sql_str)

def get_tables_with_prefix(engine, prefix, schemaname):
    '''
    return a list of the table names that begin with prefix in a specific
    schema

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param prefix: the prefix to use in selecting tables
    :type prefix: str
    :param schemaname: schemaname to search with
    :type schemaname: str

    :returns: list[str] -- the table names beginning with prefix of schema
    '''
    get_tables_query = """select tablename
                              from pg_catalog.pg_tables
                              where schemaname = '{schemaname}'
                              and tablename ~ '{prefix}.?';
                              """.format(prefix=prefix, schemaname=schemaname)
    query_tables = engine.execute(get_tables_query).fetchall()
    query_tables_strs = [q['tablename'] for q in query_tables]
    return query_tables_strs

def column_names(engine, table, schemaname):
    '''
    return the columns names of a table and a schema

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param table: the table name in question
    :type table: str
    :param schemaname: the schema name from which the table exists
    :type schemaname: str
    :returns: list[str] -- the column names of the table
    '''
    return engine.execute('''SELECT * FROM {schemaname}."{table}"
                          '''.format(table=table,
                                     schemaname=schemaname)).keys()

def drop_table(engine, table, schemaname):
    '''
    drop table from schema, cascading if necessary

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param table: the table name to drop
    :type table: str
    :param schemaname: the schema name from which the table exists
    :type schemaname: str
    '''
    dropsql = '''DROP TABLE IF EXISTS {}.{} CASCADE;
              '''.format(schemaname, table)
    engine.execute(dropsql)

def clean_data_create_select_column_from_old(
        old_colnames_list, columns_in_all_tables, corrections):
    '''
    Creates select column subclauses with AS statments to clean names,
    returns "as" string of select colnames as new.

    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are common
                                                after normalization
    :type columns_in_all tables: set
    :returns str: a string of "as" statements
     '''
    new_colnames_list = [normalize_colname(name, corrections=corrections)
                         for name in old_colnames_list]
    old_new_colnames = list(zip(old_colnames_list, new_colnames_list))
    intersection_old_new_colnames = [(old, new) for
                                     old, new in old_new_colnames
                                     if new in columns_in_all_tables]
    as_statements = []
    for old, new in intersection_old_new_colnames:
            if new in ['studentid']:
                as_statement = 'intorbust("{old}") AS {new}'.format(old=old,
                                                                    new=new)
                as_statements.append(as_statement)
            else:
                as_statement = '"{old}" AS {new}'.format(old=old, new=new)
                as_statements.append(as_statement)

    as_string = ', '.join(as_statements)
    return as_string

def clean_data_create_select_statement(tablename, old_colnames_list,
                                       columns_in_all_tables, corrections):
    '''
    Creates select statements given AS statments

    :param tablename: the table name from which were selecting
    :type tablename: str
    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are
                                        common after normalization
    :type columns_in_all tables: set
    :returns str: select statement
    '''
    as_statements = clean_data_create_select_column_from_old(
        old_colnames_list, columns_in_all_tables, corrections)
    select_statement = """SELECT {as_statements}
                        FROM raw_data."{tablename}"
                        """.format(as_statements=as_statements,
                                   tablename=tablename)
    return select_statement

def clean_data_create_copy_statement(raw_tablename, old_colnames_list,
                                     columns_in_all_tables, corrections):
    '''
    makes a CREATE TABLE statement given a raw_tablename and does casting to
    numeric where possible

    :param raw_tablename: the table name from which were selecting
    :type str:
    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are common
                                                after normalization
    :type columns_in_all tables: set
    :returns str: sql create table statement
    '''
    select_statement = clean_data_create_select_statement(
        raw_tablename, old_colnames_list, columns_in_all_tables, corrections)
    clean_table_name = normalize_tablename(raw_tablename)
    copy_statement = """CREATE TABLE clean_data.{tablename} AS
                        {select_statement};
                    """.format(tablename=clean_table_name,
                               select_statement=select_statement)

    return copy_statement

def create_select_statement(tablename, colnames_list, time_from_title):
    """
    create select statement from column name
    additionally making dates form titles if specificed
    format should be _{YY}_{season} where YY is a year
    and season is one of 'fall', 'winter', or 'spring'

    :param tablename: the table name to select from
    :type tablename: str
    :param colnames_list: the colnames to select
    :type colnames: list[str]
    :param time_from_title: whether to try and get date information from title
    :type time_from_tile: bool
    :returns str: sql select statements
    """
    extra_time_cols = ''
    if time_from_title:
        tablename_parts = tablename.split('_')
        cal_year = tablename_parts[1]
        season = tablename_parts[-1]
        if season == 'fall':
            measured_year = cal_year + '_' + str(int(cal_year) + 1)
        elif season in ['winter', 'spring']:
            measured_year = str(int(cal_year) - 1) + '_' + cal_year
        extra_time_cols = """, '{measured_year}'::varchar as measured_year,
                            '{season}'::varchar as season,
                            '{cal_year}'::int as cal_year
                          """.format(measured_year=measured_year,
                                     season=season,
                                     cal_year=cal_year)
    all_columns = ', '.join(colnames_list)
    select_statement = """SELECT {all_columns} {extra_time_cols}
                        FROM clean_data.{tablename}
                        """.format(tablename=tablename,
                                   extra_time_cols=extra_time_cols,
                                   all_columns=all_columns)
    return select_statement

def create_view_statement(columns_dict, view_name, time_from_title):
    """
    create view statement from all the tables and columns in columns dict
    additionally making dates form titles if specificed

    :param columns_dict: a mapping between table names and their column names
    :type columns_dict: dict[str: list]
    :param view_name: what to name the view
    :type view_name: str
    :param time_from_title: whether to try and get date information from title
    :type time_from_tile: bool
    :returns str: sql create view statement
    """

    select_statements_list = [create_select_statement(tablename,
                                                      old_colnames_list,
                                                      time_from_title)
                              for tablename, old_colnames_list
                              in columns_dict.items()]
    select_statements = ''' UNION
                        '''.join(select_statements_list)
    view_statement = """CREATE OR REPLACE VIEW clean_data.{view_name} AS
                        {select_statements};
                    """.format(select_statements=select_statements,
                               view_name=view_name)
    return view_statement

def main(engine, options):
    load_intorbust(engine)

    # find the raw tables to operate over
    raw_tables = []
    if options.raw_table:
        raw_tables.append(options.raw_table)
    elif options.raw_table_prefix:
        prefix_table_names = get_tables_with_prefix(engine,
                                                    options.raw_table_prefix,
                                                    'raw_data')
        raw_tables.extend(prefix_table_names)
    else:
        print('no tables specified... use --prefix or --table')
        sys.exit(1)
    print('going to operate on these raw tables: {}'.format(raw_tables))

    # make a dictionary of raw_tables and their associated column names
    raw_dict = {raw_table: column_names(engine, raw_table, 'raw_data')
                for raw_table in raw_tables}

    if options.corrections:
        corrections = json.load(open(options.corrections, 'r'))
    else:
        corrections = None

    # find the common columns in all tables -- after column name normalization.
    column_count = Counter()
    for raw_table, raw_columns in raw_dict.items():
        normed_colnames = [normalize_colname(col, corrections=corrections)
                           for col in raw_columns]
        for normed_colname in normed_colnames:
            column_count[normed_colname] += 1

    columns_in_all_tables = [col for col, cnt in column_count.items()
                             if cnt == len(raw_tables)]

    # make the destination tables and drop them if necessary
    clean_tables = [normalize_tablename(raw_table) for raw_table in raw_tables]
    for clean_table in clean_tables:
        drop_table(engine, clean_table, 'clean_data')

    # perform the raw -> clean copy
    for raw_table, raw_columns in raw_dict.items():
        copy_statement = clean_data_create_copy_statement(
            raw_table,
            raw_columns,
            columns_in_all_tables,
            corrections=corrections)

        print(copy_statement)
        engine.execute(copy_statement)
    print("look for clean_data.{}".format(clean_tables))

    # now to make the view
    # make a dictionary of clean_tables and their associated column names
    clean_dict = {clean_table: column_names(engine, clean_table, 'clean_data')
                  for clean_table in clean_tables}

    view_statement = create_view_statement(clean_dict,
                                           options.view_name,
                                           options.time_from_title)
    engine.execute(view_statement)

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('-t', '--table', dest='raw_table',
                      help='specify 1 raw table name')
    parser.add_option('-p', '--table_prefix', dest='raw_table_prefix',
                      help='specificy a prefix of all raw table names')
    parser.add_option('-c', '--config', dest='config_file',
                      help='where to get database credential file')
    parser.add_option('-v', '--view_name', dest='view_name',
                      help='what the final view name should be')
    parser.add_option('-e', '--date_cols_from_title',
                      action="store_true",
                      dest='time_from_title',
                      help='''turn this flag on if you want date/season columns
                              extracted from the table names''')
    parser.add_option('-o', '--corrections',
                      dest='corrections',
                      help='''point to a json dict of column names you want to
                              change''')

    (options, args) = parser.parse_args()

    engine = create_engine_from_config_file(options.config_file)

    if not options.config_file:
        print('no database credentials json file specified use --config /path')
        sys.exit(1)
    if not options.view_name:
        print('no destination view name specified use --view_name name')
        sys.exit(1)
    else:
        main(engine, options)


import pandas as pd
from tqdm.auto import tqdm
import numpy as np
import argparse

import gc
import torch
import torch.nn as nn

from data import *
import networks
from utils import seed_everything, resize_module, str2bool

def predict(model, test_loader, device):
    test_value = []
    with torch.no_grad():
        for before_img, after_img in tqdm(iter(test_loader)):
            before_img, after_img = before_img.float().to(device), after_img.float().to(device)
            preds = model(before_img, after_img)
            preds = preds.squeeze(1).detach().cpu().float()

            test_value.extend(preds)
    return test_value

def inference_data(model, test_loader, tta, model_paths, device):
    test_values = []
    for model_path in model_paths:
        model.load_state_dict(torch.load(model_path))
        model.eval()
        model.to(device)
        for _ in range(tta):
            test_value = predict(model, test_loader, device)
            test_values.append(test_value)

    return test_values

def main(args):
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    df = make_test_df(args.test_path)

    if args.do_resize:
        for before_path, after_path in tqdm(zip(df['before_file_path'], df['after_file_path'])):
            resize_module(before_path, args.size)
            resize_module(after_path, args.size)

    test_loader = prepare_dataloader(df, 'test', args)
    test_values = []
    if args.swin:
        model = getattr(networks, 'swin')(pretrained=False)
        test_values.extend(inference_data(model, test_loader, args.tta_num, args.swin_paths, device))
        del model
        gc.collect()
        torch.cuda.empty_cache()
    if args.vit:
        model = getattr(networks, 'vit')(pretrained=False)
        test_values.extend(inference_data(model, test_loader, args.tta_num, args.vit_paths, device))
        del model
        gc.collect()
        torch.cuda.empty_cache()

    submission = pd.read_csv(args.submission_path)
    test_values = np.array(test_values) + 1
    test_values = np.mean(test_values, axis=0)
    submission['time_delta'] = test_values
    submission.to_csv(args.save_path, index=False)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--do_resize",
        default=True,
        type=str2bool
    )
    parser.add_argument(
        "--size",
        default=224,
        type=int
    )
    parser.add_argument(
        "--tta",
        default=True,
        type=str2bool
    )
    parser.add_argument(
        "--tta_num",
        default=3,
        type=int
    )
    parser.add_argument(
        "--swin",
        default=False,
        type=str2bool
    )
    parser.add_argument(
        "--swin_paths",
        default=[],
        type=str,
        nargs='*'
    )
    parser.add_argument(
        "--vit",
        default=False,
        type=str2bool
    )
    parser.add_argument(
        "--vit_paths",
        default=[],
        type=str,
        nargs='*'
    )
    parser.add_argument(
        "--test_path",
        default='./open/test_dataset',
        type=str
    )
    parser.add_argument(
        "--submission_path",
        default="./open/sample_submission.csv",
        type=str
    )
    parser.add_argument(
        "--save_path",
        default="./submission.csv",
        type=str
    )
    parser.add_argument(
        "--workers",
        default=4,
        type=int
    )

    args = parser.parse_args()
    main(args)


# coding=utf-8
import unittest

from lib import HTMLTestRunnerNew

from common import contants

discover = unittest.defaultTestLoader.discover(contants.testcases_path, pattern='*test.py', top_level_dir=None)

with open(contants.reports_html, 'wb+') as file:
    runner = HTMLTestRunnerNew.HTMLTestRunner(stream=file,
                                              title='API',
                                              description='API测试报告',
                                              tester='深圳-改变自己')
    runner.run(discover)


import requests
from flask import Flask, render_template, request, redirect
from backend.models import Hero
from backend.api import search_hero

# initialize the app
app = Flask(__name__)

#Will save the list of the previously searched superheroes
previous_heroes = []

@app.route('/')
def home():
    return render_template('index.html', heroes = previous_heroes)

@app.route('/search', methods = ['GET','POST'])
def search():
    if request.method == 'POST':
        hero = search_hero(request.form['search'])
        previous_heroes.insert(0,hero)
        return redirect('/')
    else:
        return render_template('index.html', heroes = previous_heroes)

if __name__ == "__main__":
    app.run(debug=True)


import cv2
import numpy as np

# # ----------->>> Conversion from the png to the hex file

# Read the input image
img = cv2.imread('input.png', cv2.IMREAD_COLOR)

# Convert the image to a 1-dimensional array of RGB pixels
data = img.reshape((-1, 3))

# Convert the RGB values to hex strings
hex_data = [format(int(x), '02x') for x in data.flatten()]

# Join the hex strings into a single string with line breaks after every 3 bytes
hex_data_str = '\n'.join([''.join(hex_data[i:i+3]) for i in range(0, len(hex_data), 3)])

# Write the hex data to a text file
with open('output.txt', 'w') as file:
    file.write(hex_data_str)

###################################################################################################################################

# ----------->>> Conversion from the hex to the png

# # Input to define the nuber of bytes per pexil
# BPP = 3

# # Load the hex data from the text file
# with open('eff_out.txt', 'r') as f:
#     hex_data = f.read().replace('\n', '')

# if (BPP == 3):
#     dvsr = 6
# else:
#     dvsr = 1

# num_pixels = len(hex_data) // 6
# width = int(num_pixels ** 0.5)
# height = num_pixels // width

# print(len(hex_data))
# print(num_pixels)
# print(width)
# print(width)

# # Convert the hex data to a NumPy array of uint8 values
# data = np.frombuffer(bytes.fromhex(hex_data), dtype=np.uint8)

# # Reshape the data into an array of RGB pixels
# data = data.reshape((height, width, 3))

# # Convert the array to an RGB image
# img = cv2.cvtColor(data, cv2.COLOR_RGB2RGBA)
# cv2.imwrite('drk_output.png', img)


# -*- coding: utf-8 -*-
import scrapy
import json
from time import sleep
import datetime
import requests
from scrapy.http import Request
from lxml import etree
from pangci.items import PangciItem

class pangciSpider(scrapy.Spider):

	name = "pa"
	allowed_domains = ["www.panc.cc"]
	start_urls = ["https://www.panc.cc",]

	def parse(self,response):

		print('-'*55)
		keyword=input("请输入搜索关键字：\n")

		for i in range(0,5):
			yield Request(url="https://www.panc.cc/panc/api/?s= %s &f=0&t=td&p=%s" % (keyword,str(i)),
			callback=self.page)

	def page(self,response):
		# 调用body_as_unicode()是为了能处理unicode编码的数据
		sites = json.loads(response.body_as_unicode())
		print("-"*60)
		data = sites.get("list")

		for dic in data:

			#验证url的有效性
			for short in [dic["shorturl"]]:
				url= "https://pan.baidu.com/s/"+ str(short)
				header={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36"}

				resp = requests.get(url, headers=header ,timeout=5)
				html = etree.HTML(resp.content)
				title =html.xpath("//title/text()")
				#百度网盘链接不存在时的网页标题
				tit ="百度网盘-链接不存在"

				item = PangciItem()
				#当前资源链接是否有效
				if title[0] != tit:

					print('可用链接：',resp.url)
					item["title"] = dic["title"]
					item["shorturl"] = str(resp.url)
					item["date"] = str(datetime.date.today())
					#print(item)
					yield  item

				else:
					#print("网盘链接失效")
					continue


#!/usr/bin/env python3
# This program is part of OParl and may be used to build the specification

import os
import shlex
import shutil
import subprocess
import sys
from argparse import ArgumentParser
from glob import glob
from os import path

from scripts.json_schema2markdown import schema_to_markdown

SPECIFICATION_BUILD_ACTIONS = [
    'all',
    'clean',
    'test',
    'live',
    'html',
    'pdf',
    'odt',
    'docx',
    'txt',
    'epub',
    'archives',
    'zip',
    'gz',
    'bz'
]

SPECIFICATION_BUILD_TOOLS = [
    'pandoc',
    'dot',
    'xelatex',
    'gs',
    'convert',
    'python3',
    'tar',
    'zip'
]

SPECIFICATION_BUILD_FLAGS = {
    'gs': '-dQUIET -dSAFER -dBATCH -dNOPAUSE -sDisplayHandle=0 -sDEVICE=png16m -r600 -dTextAlphaBits=4',
    'pandoc': '--from markdown --standalone --table-of-contents --toc-depth=2 --number-sections'
}

def configure_argument_parser():
    parser = ArgumentParser(
        prog='./build.py',
        epilog='''
            build.py is part of the OParl Specification and thus distributed
            under the terms of the Creative Commons SA 4.0 License.
        '''
    )

    parser.add_argument(
        '--language',
        '-l',
        help='Specification language',
        default='de',
        action='store',
    )

    parser.add_argument(
        '--version',
        '-V',
        help='This will be displayed as version in the specification. Defaults to `git desribe`',
        action='store',
    )

    parser.add_argument(
        '--print-basename',
        help='This will output the base name used for build output',
        action='store_true',
        dest='print_basename'
    )

    parser.add_argument(
        '--latex-template',
        help='Change the latex template used for PDF generation',
        action='store',
        default='resources/template.tex'
    )

    parser.add_argument(
        '--html-style',
        help='Change the CSS file used for styling the HTML output',
        action='store',
        default='resources/html5.css'
    )

    parser.add_argument(
        '--list-actions',
        help='List available build actions',
        dest='list_actions',
        action='store_true'
    )

    parser.add_argument(
        'action',
        help='Build action to take, available actions are: {}'.format(', '.join(SPECIFICATION_BUILD_ACTIONS)),
        action='store',
        nargs='*'
    )

    return parser

def check_build_action(action):
    if len(action) == 0:
        return 'all'

    if action[0] in SPECIFICATION_BUILD_ACTIONS:
        return action[0]

    raise Exception(
        'Unknown build action: {}, choose one of: {}'.format(action, ', '.join(SPECIFICATION_BUILD_ACTIONS)))

def get_git_describe_version():
    return subprocess.check_output('git describe', shell=True, universal_newlines=True).strip()

def check_available_tools():
    tools = {}
    for tool in SPECIFICATION_BUILD_TOOLS:
        executable = shutil.which(tool)
        if executable:
            tools[tool] = executable
        else:
            raise Exception('{} not found, aborting.'.format(tool))

    return tools

def get_filename_base(language, version):
    return 'OParl-{}-{}'.format(version, language)

def prepare_builddir(filename_base):
    os.makedirs('build/src/images')
    os.makedirs('build/{}'.format(filename_base))

def prepare_schema(language):
    language_file = 'schema/strings.yml'
    if language != 'de':
        language_file = 'locales/{}/schema/strings.yml'.format(language)

    output_file = 'build/src/3-99-schema.md'

    schema_to_markdown('schema', 'examples', output_file, language, language_file)

def prepare_markdown(language):
    glob_pattern = 'src/*.md'
    if language != 'de':
        glob_pattern = 'locales/{}/src/*.md'.format(language)

    files = glob(glob_pattern)
    for f in files:
        shutil.copy2(f, 'build/src/')

    os.mkdir('build/extra')
    with open('build/extra/detailed-version.md', 'w') as fp:
        fp.write("Version {}\n".format(get_git_describe_version()))

def prepare_images(tools):
    glob_pattern = 'src/images/*.*'

    files = glob(glob_pattern)
    for f in files:
        convert_command = ''
        filename, extension = os.path.splitext(f)
        fout = path.join('build', 'src', 'images', os.path.basename(filename) + '.png')

        shutil.copy2(f, fout)

        if extension == '.pdf':
            convert_command = '{} {} -sOutputFile={} -f {}'.format(
                tools['gs'],
                SPECIFICATION_BUILD_FLAGS['gs'],
                fout,
                f
            )

        if extension == '.dot':
            convert_command = '{} -Tpng {} -o {}'.format(
                tools['dot'],
                f,
                fout
            )

        if extension == '.svg':
            convert_command = '{} {} {}'.format(
                tools['convert'],
                f,
                fout
            )

        cmd = shlex.split(convert_command)
        try:
            subprocess.run(cmd, check=True)
        except subprocess.CalledProcessError:
            raise Exception(
                'Errored on image prep for {}, please check the command:\n{}'.format(
                    f, convert_command
                )
            )

def create_symlinks(filename_base):
    """ Allow having stable links when working on the spec """
    os.makedirs('build/latest')
    for specfile in os.listdir('build/{}'.format(filename_base)):
        existing = os.path.abspath('build/{}/{}'.format(filename_base, specfile))
        new = 'build/latest/oparl{}'.format(os.path.splitext(specfile)[-1])
        os.symlink(existing, new)

def get_pandoc_version(pandoc_bin):
    version_tuple = subprocess.getoutput('{} --version'.format(pandoc_bin)).split('\n')[0].split(' ')[1].split('.')
    return [int(v) for v in version_tuple]

def run_pandoc(pandoc_bin, filename_base, output_format, extra_args='', extra_files=''):
    output_file = 'build/{}/{}.{}'.format(filename_base, filename_base, output_format)
    if path.exists(output_file):
        return

    source_files = glob('build/src/*.md')
    source_files.sort(key=lambda file: path.basename(file))
    # This gets the version into the titlepage of the pdf
    detailed_version = '-M detailed-version={}'.format(get_git_describe_version())

    # NOTE: Once we can safely assume pandoc 2.0, we can use resource-path
    #       for neater include management in the markdown files
    # pandoc_command = '{} {} {} --resource-path=.:build/src -o {} {} {}'.format(
    pandoc_command = '{} {} {} -o {} {} {} {}'.format(
        pandoc_bin,
        SPECIFICATION_BUILD_FLAGS['pandoc'],
        extra_args,
        output_file,
        extra_files,
        detailed_version,
        ' '.join(source_files)
    )

    cmd = shlex.split(pandoc_command)
    try:
        subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stderr)
    except subprocess.CalledProcessError:
        raise Exception(
            'Errored on pandoc: {}'.format(pandoc_command)
        )

class Action:
    @staticmethod
    def clean():
        shutil.rmtree('build', ignore_errors=True)

    @staticmethod
    def test():
        # TODO: validate.py appears to be broken
        pass

    @staticmethod
    def live(tools, _, filename_base):
        args = '--to html5 --section-divs --no-highlight --template=resources/live.html'
        run_pandoc(tools['pandoc'], filename_base, 'html', extra_args=args)

    @staticmethod
    def html(tools, options, filename_base):
        args = '--to html5 --css {} --section-divs --self-contained'.format(options.html_style)
        run_pandoc(tools['pandoc'], filename_base, 'html', extra_args=args,
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-bild.md')

    @staticmethod
    def pdf(tools, options, filename_base):
        args = '--pdf-engine=xelatex'
        if get_pandoc_version(tools['pandoc'])[0] < 2:
            args = '--latex-engine=xelatex'

        args += ' --template {}'.format(options.latex_template)

        run_pandoc(tools['pandoc'], filename_base, 'pdf', extra_args=args)

    @staticmethod
    def odt(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'odt',
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-text.md')

    @staticmethod
    def docx(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'docx',
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-text.md')

    @staticmethod
    def txt(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'txt')

    @staticmethod
    def epub(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'epub', extra_files='build/extra/detailed-version.md')

    @staticmethod
    def all(tools, options, filename_base):
        Action.html(tools, options, filename_base)
        Action.pdf(tools, options, filename_base)
        Action.odt(tools, options, filename_base)
        Action.docx(tools, options, filename_base)
        Action.txt(tools, options, filename_base)
        Action.epub(tools, options, filename_base)

    @staticmethod
    def zip(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.zip'.format(filename_base)
        subprocess.run(['zip', '-qr', archive_name, filename_base], cwd='build')

    @staticmethod
    def gz(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.tar.gz'.format(filename_base)
        subprocess.run(['tar', '-czf', archive_name, filename_base], cwd='build')

    @staticmethod
    def bz(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.tar.bz2'.format(filename_base)
        subprocess.run(['tar', '-cjf', archive_name, filename_base], cwd='build')

    @staticmethod
    def archives(tools, options, filename_base):
        Action.zip(tools, options, filename_base)
        Action.gz(tools, options, filename_base)
        Action.bz(tools, options, filename_base)

def main():
    options = configure_argument_parser().parse_args()
    action = check_build_action(options.action)

    if options.version is None:
        options.version = get_git_describe_version()

    filename_base = get_filename_base(options.language, options.version)

    if options.print_basename:
        print(filename_base)
        exit(0)

    if options.list_actions:
        for action in SPECIFICATION_BUILD_ACTIONS:
            print('- ' + action)
        exit()

    tools = check_available_tools()

    # always clean
    Action.clean()

    if action == 'clean':
        exit(0)

    prepare_builddir(filename_base)
    prepare_schema(options.language)
    prepare_markdown(options.language)
    prepare_images(tools)

    # Avoid much boilerplate
    getattr(Action, action)(tools, options, filename_base)

    create_symlinks(filename_base)

if __name__ == '__main__':
    main()


from django.conf import settings
from django.core.validators import MinValueValidator
from django.db import models

class PizzaOrder(models.Model):
    # Flavor choices
    FLAVOR_HAWAII = 'Hawaii'
    FLAVOR_REGINA = 'Regina'
    FLAVOR_QUATTRO_FORMAGGI = 'Quattro-Formaggi'
    FLAVOR_CHOICES = (
        (FLAVOR_HAWAII, 'Hawaii'),
        (FLAVOR_REGINA, 'Regina'),
        (FLAVOR_QUATTRO_FORMAGGI, 'Quattro-Formaggi')
    )

    # Size choices
    SIZE_LARGE = "Large"
    SIZE_MEDIUM = "Medium"
    SIZE_CHOICES = (
        (SIZE_LARGE, 'Large'),
        (SIZE_MEDIUM, 'Medium')
    )

    # Crust choices
    CRUST_THIN = "Thin"
    CRUST_CHOICES = [
        (CRUST_THIN, 'Thin')
    ]

    Flavor = models.CharField(
        max_length=16,
        choices=FLAVOR_CHOICES,
        default=FLAVOR_HAWAII
    )

    Size = models.CharField(
        max_length=6,
        choices=SIZE_CHOICES,
        default=SIZE_MEDIUM
    )

    Crust = models.CharField(
        max_length=4,
        choices=CRUST_CHOICES,
        default=CRUST_THIN
    )

    Table_No = models.IntegerField(MinValueValidator(30000), blank=True, null=True)
    Order_ID = models.IntegerField(blank=True, null=True)
    Timestamp = models.DateTimeField(blank=True, null=True)

    # This field ensure I can return a users pizza orders
    Ordered_By = models.ForeignKey(settings.AUTH_USER_MODEL,
                                   on_delete=models.CASCADE,
                                   related_name="ordered_by")

    def timeAsString(self):
        return self.Timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")


from django.conf.urls import patterns, url
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.conf import settings

# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()
from django.views.generic.simple import direct_to_template

if settings.MAINTENANCE_MODE:
    urlpatterns = patterns(
        '',
        url(r'^$', 'gd.views.home_maintenance', name='home'),
        url(r'^detay/', 'gd.views.home_maintenance', name='detail'),
    )

else:
    urlpatterns = patterns(
        '',
        url(r'^$', 'gd.views.home', name='home'),
        url(r'^getmvdata/all/$', 'gd.views.mvnames'),
        url(r'^getmvdata/(?P<id>[\w-]+)/$', 'gd.views.getmvdata'),
        url(r'^detay/mv/$', 'gd.views.detail', name='detailmv'),
        url(r'^detay/parti/$', 'gd.views.detailParty', name='detailparty'),
        url(r'^detay/sehir/$', 'gd.views.detailCity', name='detailcity'),
        url(r'^enler/$', 'gd.views.toptemp', name='toptemp'),
        url(r'^en/$', 'gd.views.top', name='top'),
        url(r'^twitanaliz/$', 'twitanaliz.views.tweetAnalysis', name="twitanaliz"),
        url(r'^twitanaliz/dokum/$', 'twitanaliz.views.tweetResults', name="twitresults"),
        url(r'^nedir/$', direct_to_template, {'template': 'about.html'}, name="readme"),

        # url(r'^admin/', include(admin.site.urls)),
        url(r'^accounts/login/$', 'django.contrib.auth.views.login', name="login"),
    )

urlpatterns += staticfiles_urlpatterns()


import numpy as np
import torch
from torch.nn import functional as F

from mmdet.core.bbox import BaseBBoxCoder
from mmdet.core.bbox.builder import BBOX_CODERS

INF = 1e8
EPS = 1e-12
PI = np.pi

@BBOX_CODERS.register_module()
class UniYOLOCoder(BaseBBoxCoder):
    """Bbox Coder for MonoFlex.

    Args:
        depth_mode (str): The mode for depth calculation.
            Available options are "linear", "inv_sigmoid", and "exp".
        base_depth (tuple[float]): References for decoding box depth.
        depth_range (list): Depth range of predicted depth.
        combine_depth (bool): Whether to use combined depth (direct depth
            and depth from keypoints) or use direct depth only.
        uncertainty_range (list): Uncertainty range of predicted depth.
        base_dims (tuple[tuple[float]]): Dimensions mean and std of decode bbox
            dimensions [l, h, w] for each category.
        dims_mode (str): The mode for dimension calculation.
            Available options are "linear" and "exp".
        multibin (bool): Whether to use multibin representation.
        num_dir_bins (int): Number of Number of bins to encode
            direction angle.
        bin_centers (list[float]): Local yaw centers while using multibin
            representations.
        bin_margin (float): Margin of multibin representations.
        code_size (int): The dimension of boxes to be encoded.
        eps (float, optional): A value added to the denominator for numerical
            stability. Default 1e-3.
    """

    def __init__(self,
                 depth_mode='linear',
                 num_alpha_bins=12,
                 eps=1e-3):
        super(UniYOLOCoder, self).__init__()
        # output related
        self.num_alpha_bins = num_alpha_bins
        self.depth_mode = depth_mode
        self.eps = eps

    def decode_bboxes2d(self, bbox_preds, priors):
        xys = (bbox_preds[..., :2] * priors[:, 2:]) + priors[:, :2]
        whs = bbox_preds[..., 2:].exp() * priors[:, 2:]

        tl_x = (xys[..., 0] - whs[..., 0] / 2)
        tl_y = (xys[..., 1] - whs[..., 1] / 2)
        br_x = (xys[..., 0] + whs[..., 0] / 2)
        br_y = (xys[..., 1] + whs[..., 1] / 2)

        decoded_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], -1)
        return decoded_bboxes

    def decode_bboxes3d(self, bbox_3d_preds, alpha_preds, priors, cam2imgs):
        direct_depth = bbox_3d_preds[..., 2]
        direct_depth_log_var = bbox_3d_preds[..., 3]
        dims = bbox_3d_preds[..., 4:].view(-1, 3)
        depth = self.decode_depth(direct_depth)
        sigma = torch.exp(-direct_depth_log_var)

        center2d = (bbox_3d_preds[..., :2] * priors[:, 2:]) + priors[:, :2]
        # 1. decode alpha
        alpha = self.decode_alpha_multibin(alpha_preds[..., :self.num_alpha_bins],
                                            alpha_preds[..., self.num_alpha_bins:])  # (b, k, 1)

        # 2. recover rotY
        rot_y = self.recover_rotation(center2d, alpha, cam2imgs)  # (b, k, 1)

        # 2.5 recover box3d_center from center2d and depth
        center3d = torch.cat([center2d, depth.view(1, -1, 1)], dim=-1).squeeze(0)
        center3d = self.pts2Dto3D(center3d, np.array(cam2imgs)).unsqueeze(0)

        # 3. compose 3D box
        batch_bboxes_3d = torch.cat([center3d, bbox_3d_preds[..., 4:], rot_y], dim=-1)

        return batch_bboxes_3d, sigma.view(1, -1)

    def decode_depth(self, direct_depth):

        if self.depth_mode == 'exp':
            direct_depth = direct_depth.exp()
        elif self.depth_mode == 'scale':
            direct_depth = direct_depth * 64 / priors[:, 3:4]
        elif self.depth_mode == 'inv_sigmoid':
            direct_depth = 1 / torch.sigmoid(direct_depth) - 1

        return direct_depth

    def pts2Dto3D(self, points, view):
        """
        Args:
            points (torch.Tensor): points in 2D images, [N, 3], \
                3 corresponds with x, y in the image and depth.
            view (np.ndarray): camera instrinsic, [3, 3]

        Returns:
            torch.Tensor: points in 3D space. [N, 3], \
                3 corresponds with x, y, z in 3D space.
        """
        assert view.shape[0] <= 4
        assert view.shape[1] <= 4
        assert points.shape[1] == 3

        points2D = points[:, :2]
        depths = points[:, 2].view(-1, 1)
        unnorm_points2D = torch.cat([points2D * depths, depths], dim=1)

        viewpad = torch.eye(4, dtype=points2D.dtype, device=points2D.device)
        viewpad[:view.shape[0], :view.shape[1]] = points2D.new_tensor(view)
        inv_viewpad = torch.inverse(viewpad).transpose(0, 1)

        # Do operation in homogenous coordinates.
        nbr_points = unnorm_points2D.shape[0]
        homo_points2D = torch.cat(
            [unnorm_points2D,
             points2D.new_ones((nbr_points, 1))], dim=1)
        points3D = torch.mm(homo_points2D, inv_viewpad)[:, :3]

        return points3D

    def inv_affine(self, points, trans_mats):
        # number of points
        N = points.shape[0]
        # batch_size
        N_batch = trans_mats.shape[0]
        batch_id = torch.arange(N_batch).unsqueeze(1)
        obj_id = batch_id.repeat(1, N // N_batch).flatten()
        trans_mats_inv = trans_mats.inverse()[obj_id]
        centers2d_extend = torch.cat((points, points.new_ones(N, 1)),
                                     dim=1)
        # expand project points as [N, 3, 1]
        centers2d_extend = centers2d_extend.unsqueeze(-1)
        # transform project points back on original image
        centers2d_img = torch.matmul(trans_mats_inv, centers2d_extend).squeeze(2)

        return centers2d_img[:, :2]

    def angle2class(self, angle):
        ''' Convert continuous angle to discrete class and residual. '''
        angle = angle % (2 * PI)
        assert (angle >= 0 and angle <= 2 * PI)
        angle_per_class = 2 * PI / float(self.num_alpha_bins)
        shifted_angle = (angle + angle_per_class / 2) % (2 * PI)
        class_id = int(shifted_angle / angle_per_class)
        residual_angle = shifted_angle - (class_id * angle_per_class + angle_per_class / 2)
        return class_id, residual_angle

    def class2angle(self, cls, residual):
        ''' Inverse function to angle2class. '''
        angle_per_class = 2 * PI / float(self.num_alpha_bins)
        angle_center = cls * angle_per_class
        angle = angle_center + residual
        return angle

    def decode_alpha_multibin(self, alpha_cls, alpha_offset):
        alpha_score, cls = alpha_cls.max(dim=-1)
        cls = cls.unsqueeze(2)
        alpha_offset = alpha_offset.gather(2, cls)
        alpha = self.class2angle(cls, alpha_offset)

        alpha[alpha > PI] = alpha[alpha > PI] - 2 * PI
        alpha[alpha < -PI] = alpha[alpha < -PI] + 2 * PI
        return alpha

    def recover_rotation(self, kpts, alpha, calib):
        device = kpts.device
        calib = torch.tensor(calib).type(torch.FloatTensor).to(device).unsqueeze(0)

        si = torch.zeros_like(kpts[:, :, 0:1]) + calib[:, 0:1, 0:1]
        rot_y = alpha + torch.atan2(kpts[:, :, 0:1] - calib[:, 0:1, 2:3], si)

        while (rot_y > PI).any():
            rot_y[rot_y > PI] = rot_y[rot_y > PI] - 2 * PI
        while (rot_y < -PI).any():
            rot_y[rot_y < -PI] = rot_y[rot_y < -PI] + 2 * PI

        return rot_y

    def encode(self, gt_bboxes_3d):
        pass

    def decode(self, bbox, base_centers2d, labels, downsample_ratio, cam2imgs):
        pass


import numpy as np
import ast
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib import cm
from datetime import datetime
from matplotlib.colors import LinearSegmentedColormap

n = 366
data = []
cols = [[] for i in range(n)]
avgs = np.zeros((n, 3))
limit = 1000000
i = 0
with open("image_parser_results.txt", "r") as f:
	for line in f:
		if (line.count("22.33589744") == 3):
			continue
		d = ast.literal_eval(line.replace("array(", "").replace("))", ")"))
		if (1.5*d[2][1] > d[2][0]+d[2][2]) or (1.5*d[2][0] > d[2][1]+d[2][2]) or (1.5*d[2][2] > d[2][0]+d[2][1]):
			data.append(d)
		if i == limit:
			break
		i += 1
		# print(data[-1])

for timestamp, url, col in data:
	date = datetime.fromtimestamp(timestamp)
	day_of_year = date.timetuple().tm_yday
	cols[day_of_year-1].append(col)
	# cols[(timestamp % 86400)// 60].append(col)

for day in range(len(cols)):
	cols[day] = np.array(cols[day])
	# print(cols[day])
	avgs[day] = np.mean(cols[day], axis = 0)

avgs /= 255
# print(avgs)

fig = plt.figure()

display_axes = fig.add_axes([0.1,0.1,0.8,0.8], projection='polar')
display_axes._direction = 2*np.pi ## This is a nasty hack - using the hidden field to
								  ## multiply the values such that 1 become 2*pi
								  ## this field is supposed to take values 1 or -1 only!!

norm = mpl.colors.Normalize(0.0, 2*np.pi)

# Plot the colorbar onto the polar axis
# note - use orientation horizontal so that the gradient goes around
# the wheel rather than centre out

# colors = [(1, 0, 0), (0, 1, 0), (0, 0, 1)]  # R -> G -> B
# n_bins = [3, 6, 10, 100]  # Discretizes the interpolation into bins
cmap_name = 'my_list'
cm = LinearSegmentedColormap.from_list(
		cmap_name, avgs, N=n)

cb = mpl.colorbar.ColorbarBase(display_axes, cmap = cm,
								   norm=norm,
								   orientation='horizontal')

# aesthetics - get rid of border and axis labels
cb.outline.set_visible(False)
display_axes.set_axis_off()
plt.show() # Replace with plt.savefig if you want to save a file


"""LSTM with CTC for handwritten text recognition within a line."""
import torch
from torch import nn
from text_recognizer.networks.lenet import lenet
from text_recognizer.networks.misc import slide_window

def line_crnn(input_shape, output_shape):
    '''
    CRNN model that shrinks the H dim to 1, and use W dim as the time step,
    thus avoid manual generation of patches with slide_window
    '''
    class BidirectionalLSTM(nn.Module):

        def __init__(self, nIn, nHidden, nOut):
            super(BidirectionalLSTM, self).__init__()

            self.rnn = nn.LSTM(nIn, nHidden, bidirectional=True)
            self.embedding = nn.Linear(nHidden * 2, nOut)

        def forward(self, x):
            recurrent, _ = self.rnn(x)
            T, b, h = recurrent.size()
            t_rec = recurrent.view(T * b, h)

            output = self.embedding(t_rec)  # (T*B, nOut)
            output = output.view(T, b, -1)

            return output

    class CRNN(nn.Module):

        def __init__(self, imgH, nc, nclass, nh ):
            super(CRNN, self).__init__()
            # (B, 1, 28, 952)
            cnn = nn.Sequential(
                nn.Conv2d(1, 64, 3, 1, 1),
                nn.ReLU(),
                nn.MaxPool2d(2,2), # (B, 64, 14, 476)
                nn.Conv2d(64, 128, 3, 1, 1),
                nn.ReLU(),
                nn.MaxPool2d(2,2), # (B, 128, 7, 238)
                nn.Conv2d(128, 128, 3, 1), # (B, 128, 5, 236)
                nn.ReLU(),
                nn.BatchNorm2d(128),
                nn.Conv2d(128, 128, 3, 1), # (B, 128, 3, 234)
                nn.ReLU(),
                nn.Conv2d(128, 128, 3, 1), # (B, 128, 1, 232)
                nn.ReLU(),
            )

            self.cnn = cnn
            self.rnn = BidirectionalLSTM(nh, nh, nclass)

        def forward(self, x):
            # conv features
            if len(x.shape) < 4:
                x = x.unsqueeze(1) # ensure a channel dim
            conv = self.cnn(x)
            conv = conv.squeeze(2)
            conv = conv.permute(2, 0, 1)  # (w, b, c)
            w, b, c = conv.shape

            # rnn features
            output = self.rnn(conv) # (w, b, n_class)
            input_lengths = torch.full((b, ), w)

            return output, input_lengths

    H, W = input_shape
    output_lengths, nclass = output_shape
    model = CRNN(imgH=H, nc=1, nclass=nclass, nh=128)

    return model


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

#print(list(df))
#print(len(df))
#df['id'] = df.astype(str).values.sum(axis=1)
#print(len(df['id'].unique()))  # Registry ID

def read_data(fname, feature_dict=False, keep_cols=False, sep=','):
    df = pd.read_csv(fname, low_memory=False, sep=sep, error_bad_lines=False, encoding='latin-1')
    if feature_dict != False:
        df.rename(columns=feature_dict, inplace=True)
    if keep_cols != False:
        df = df[keep_cols]
    return df

def get_hosp_from_previous(df, fname='data/bd_srag_08-06-2020.csv',feature_dict=False, keep_cols=False, categorical_cols=False):
    # Read old dataset
    categorical_cols.append('Hospital ID')
    keep_cols.append('Hospital ID')
    df_old = read_data(fname, feature_dict, keep_cols, sep=';')
    df_old = filter_positive_test(df_old)
    df_old = add_public_private_var(df_old, fname='data/ICU_beds.csv')
    categorical_cols.append('Public Hospital')
    keep_cols.append('Public Hospital')
    df_old.drop(columns=['Hospital ID'], inplace=True)
    categorical_cols.remove('Hospital ID')

    # Create hash to compare
    a = ['Gender', 'Race', 'Schooling', 'Acute Respiratory Distress Syndrome', 'Contracted At Hospital', 'Fever',
         'Cough', 'Throat', 'Dyspnea', 'Respiratory Discomfort', 'SpO2 less 95%', 'Diarrhea', 'Vomiting',
         'Other Symptoms', 'Postpartum', 'Cardiovascular Disease', 'Hematologic Disease', 'Down Syndrome',
         'Liver Chronic Disease', 'Asthma', 'Diabetes', 'Neurological Disease', 'Another Chronic Pneumopathy',
         'Immunosuppression', 'Renal Chronic Disease', 'Obesity', 'Other Risks', 'Antiviral Use', 'Hospitalization',
         'ICU', 'Ventilator', 'Xray Thorax Result', 'Result Final']
    df['id'] = df[a].apply(lambda row: '_'.join(row.values.astype(str)), axis=1)
    df_old['id'] = df_old[a].apply(lambda row: '_'.join(row.values.astype(str)), axis=1)

    # Sort
    df = df.sort_values(list(df))
    df_old = df_old.sort_values(list(df_old))

    # Merge /vlookup
    df = df.drop_duplicates(subset=['id'])
    df_old = df_old.drop_duplicates(subset=['id'])
    df_old = df_old[['id', 'Public Hospital']]
    df2 = df.merge(df_old, on='id', how='left')

    df2['Public Hospital'] = df2['Public Hospital'].fillna(9)
    df2.rename(columns={'Public Hospital':'Hospital'}, inplace=True)
    return df2

def get_var_stats(df, col):
    serie = pd.Series(df[col].values)
    v = pd.DataFrame(serie.value_counts(dropna=False, normalize=True), columns=[col])
    return v

def get_categorical_stats(df, plot=True, fname=''):
    df2 = pd.DataFrame()
    for col in list(df):
        v = get_var_stats(df, col)
        df2 = pd.concat([df2, v], axis=1, sort=False)
    df2.round(decimals=2)
    if plot:
        table2sns(df2.T, fname, show=False)
    return df2

def set_mode_on_NA(df, numeric_cols):
    for i in numeric_cols:
        df[i] = df[i].fillna(df[i].mode()[0])
    return df

def var_to_categorical(df, var_name, bins):
    for i in range(len(bins)-1):
        df[var_name + ' ' + str(bins[i]) + '-' + str(bins[i + 1])] = 0
        df[var_name + ' ' + str(bins[i]) + '-' + str(bins[i + 1])][(df[var_name]>bins[i])&(df[var_name]<=bins[i+1]) ] = 1
        if sum(df[var_name + ' ' + str(bins[i]) + '-' + str(bins[i + 1])])==0:
            df.drop(var_name + ' ' + str(bins[i]) + '-' + str(bins[i + 1]), axis=1, inplace=True)
    df.drop(var_name, axis=1, inplace=True)
    return df

def table2sns(df, fname, show=False, round=2):
    df.round(decimals=round)
    n = len(df)
    m = len(df.T)
    plt.figure(figsize = (m, n/3))
    g = sns.heatmap(df,
                annot=True,
                cmap=sns.cm.rocket_r,
                linewidth=1
                )
    plt.tight_layout()
    if show:
        plt.show()
    plt.savefig(fname+'.png')
    plt.savefig(fname+'.pdf')

def one_hot_encoding(df, categorical_features):
    for feature in categorical_features:
        dummies = pd.get_dummies(df[feature], drop_first=False, prefix=feature)
        df = pd.concat([df, dummies], axis=1)
        df.drop(feature, axis=1, inplace=True)
        #col_d = {i: i.split('_1')[0] for i in list(df)}
        #df = df.rename(columns=col_d)
    return df

def remove_corr_features(df, print_=False, plot_=False):
    correlated_features = set()
    correlated_matrix = df.corr()
    if plot_:
        plt.matshow(correlated_matrix)
        plt.show()
    for i in range(len(correlated_matrix.columns)):
        for j in range(i):
            if abs(correlated_matrix.iloc[i, j]) > 0.8:
                colname = correlated_matrix.columns[i]
                correlated_features.add(colname)
    for i in correlated_features:
        if 'Death' not in i:
            df.drop(i, axis=1, inplace=True)
    if print_:
        print('correlated vars: ' + str(correlated_features))
    return df

def remove_features_containing(df, text):
    features = list(df)
    features2 = []
    for feature in features:
        if  text not in feature:
            features2.append(feature)
    return df[features2]

def remove_NA_features(df):
    features = list(df)
    features2 = []
    for feature in features:
        if 'Na' not in feature:
            features2.append(feature)
    return df[features2]

def filter_positive_test(df):
    return df[df['Result Final']==5]

def create_training_and_test_sets(df, y, remove_vars, percentage_train):
    df0 = df.copy()
    for i in remove_vars:
        for  l in [s for s in list(df0) if i in str(s)]:
            if l != y :
                df0 = df0.drop(columns=[l])
    x_train = df0.sample(frac=percentage_train)
    x_test = pd.concat([df0, x_train]).drop_duplicates(keep=False)
    y_train = x_train.pop(y)
    y_test = x_test.pop(y)
    return x_train, x_test, y_train, y_test

def rename_features(df, var_dictionary, ignore_vars):
    sp_vars = list(var_dictionary.keys())
    for feature in list(df):
        f = feature.split('_')
        if len(f) > 1:
            if f[0] not in ignore_vars:
                if f[0] in sp_vars:
                    df = df.rename(columns={feature:f[0]+' '+var_dictionary[f[0]][f[1]]})
                else:
                    df = df.rename(columns={feature:f[0]+' '+var_dictionary['typical'][f[1]]})
    return df

def add_public_private_var(df, fname='data/ICU_beds.csv'):
    pp_df = pd.read_csv(fname)
    pp_df.drop(columns=['public nonICU', 'private nonICU', 'public ICU', 'private ICU'], inplace=True)
    df = df.merge(pp_df, how='left', on='Hospital ID')
    del pp_df
    df.rename(columns={'public hospital':'Public Hospital'}, inplace=True)
    df['Public Hospital'].loc[df['Public Hospital'] == 0] = 2.0
    df['Public Hospital'].loc[df['Public Hospital'] == 1] = 1.0
    df["Public Hospital"] = df["Public Hospital"].fillna(9.0)
    return df

def polyfit(x, y, degree):
    results = {}

    coeffs = np.polyfit(x, y, degree)

    # Polynomial Coefficients
    results['polynomial'] = coeffs.tolist()

    # r-squared
    p = np.poly1d(coeffs)
    # fit values, and mean
    yhat = p(x)
    ybar = np.sum(y) / len(y)
    ssreg = np.sum((yhat - ybar) ** 2)
    sstot = np.sum((y - ybar) ** 2)
    results['determination'] = ssreg / sstot

    return results

def create_summary(df, categorical_vars, out_dir):
    summ = {}
    n = len(df.Age)
    n_hosp = len(df[df.Hospitalization==1])
    n_death = len(df[df.Evolution==2])
    summ['N'] = [n]
    summ['N_Hosp'] = [n_hosp]
    summ['N_Death'] = n_death
    for feature in categorical_vars:
        if feature == 'Gender':
            Ni = len(df[df[feature] == 'F'])
            summ['Female'] = [Ni]
            Ni = len(df[df[feature] == 'M'])
            summ['Male'] = [Ni]
        #else:
        #    for cat in df.feature.unique():
         #       Ni = len(df[df[feature] == cat])
         #       summ[feature +'-' + cat] = [Ni]
        else:
            Ni = len(df[df[feature] == 1])
            summ[feature] = [Ni]
    print(pd.DataFrame.from_dict(summ).T)
    pd.DataFrame.from_dict(summ).T.to_latex(out_dir+'/summary.tex')
    pd.DataFrame.from_dict(summ).T.to_csv(out_dir+'/summary.csv' )

def create_age_hosp_plot(df, out_dir):
    N = len(df)
    fig, ax = plt.subplots(3, figsize=(7,8))
    age_list = []
    percentage_death_list = []
    ages = list(np.linspace(0,100,51))
    for j in range(len(ages)-1):# sorted(list(df.EDAD.unique())):
        num = len(df[(df.Age >= ages[j]) & (df.Age < ages[j+1]) &(df.Hospitalization==1)])
        den = len(df[(df.Age >= ages[j]) & (df.Age < ages[j+1])])
        ax[0].bar(x=ages[j+1] , height=num/den, color='blue')
        plt.grid(True)
        ax[2].scatter(x=ages[j+1], y=den, marker='*', color='red')
        plt.grid(True)
        age_list.append(ages[j+1])

        # Death
        num = len(df[(df.Age >= ages[j]) & (df.Age < ages[j + 1]) & (df.Evolution == 2)])
        ax[1].bar(x=ages[j + 1], height=num / den, color='blue')
        percentage_death_list.append(num / den)
        plt.grid(True)

    ax[0].set(ylabel="Percentage being Hospitalized")

    ## plot wih linear regression :
    x = age_list[-35:]
    y = percentage_death_list[-35:]
    results = polyfit(x, y, 1)
    [m,b]= results['polynomial']
    r2 = results['determination']
    yp = np.polyval([m, b], x)
    ax[1].plot(x, yp, color='orangered', label=str(round(b,2))+'+' + str(round(m,4))+'*Age; R2='+str(round(r2,2)))
    ax[1].legend()
    ax[1].set(ylabel="Deaths")
    ax[2].set(ylabel="Counts")
    plt.xlabel('Age')
    plt.grid(True)
    plt.tight_layout()
    plt.savefig(out_dir + '/hosp_age.png')
    plt.savefig(out_dir + '/hosp_age.pdf')
    plt.close('all')

def create_cat_table(df, categorical_vars, out_dir):
    #print(categorical_vars)
    features = categorical_vars
    #features.remove('Evolution')
    #features.remove('Hospitalized')
    db1 = []
    db2 = []
    features_l = []
    special_features = ['Gender', 'Color', 'Race', 'Schooling', 'XrayToraxResult']
    for feature in features:
        if feature in special_features:
            for f in df[feature].unique():
                if f!=9.0:
                    Ni = len(df[df[feature] == f])
                    if Ni>10:
                        db1.append(len(df[(df[feature] == f) & (df['Hospitalization'] == 1)]) / Ni)
                        db2.append(len(df[(df[feature] == f) & (df['Evolution'] == 2)]) / Ni)
                        features_l.append(feature+'-'+str(f)+' (' + str(Ni) + ')')
                    else:
                        db1.append(np.NaN)
                        db2.append(np.NaN)
                        features_l.append(feature + ' (' + str(Ni) + ')')
        else:
            Ni = len(df[df[feature] == 1])
            if Ni>3:
                db1.append(len(df[(df[feature] == 1) & (df['Hospitalization'] == 1)]) / Ni)
                db2.append(len(df[(df[feature] == 1) & (df['Evolution'] == 2)]) / Ni)
                features_l.append(feature + ' (' + str(Ni) + ')')
            else:
                db1.append(np.NaN)
                db2.append(np.NaN)
                features_l.append(feature + ' (' + str(Ni) + ')')
    df_f = pd.DataFrame({'Death': db2}, index=features_l)
    df_f = df_f.sort_values('Death', ascending = False)
    df_f_transpose = df_f
    plt.figure(figsize = (8, 13))
    sns.heatmap(df_f_transpose, annot=True, cmap=sns.cm.rocket_r, linewidth=1)
    plt.tight_layout()
    plt.savefig(out_dir + '/hosp_features.png')
    plt.savefig(out_dir + '/hosp_features.pdf')

def create_cat_age_table(df, categorical_vars, out_dir):
    dic = {}
    features = categorical_vars
    preconditions = ['Diabetes', 'Renal_Comorbidity', 'Cardio_Comorbidity', 'Lung_Comorbidity', 'Tobbaco-Use',
                     'Obesity', 'Respiratory distress']
    special_features = ['Gender', 'Color', 'Race', 'Schooling', 'XrayToraxResult']

    ages = np.arange(0, 105, 10)
    for i in range(len(ages)-1):
        df2 = df[(df.Age>ages[i]) & (df.Age<=ages[i+1])]
        d0 = df[(df.Age>ages[i]) & (df.Age<=ages[i+1])]
        db1 = []
        features_v = []
        for feature in features:
            if feature in special_features:
                for f in df[feature].unique():
                    if f !=9.0:
                        df2 = df[(df.Age>ages[i]) & (df.Age<=ages[i+1]) & (df[feature] == f)]
                        Ni = len(df2)
                        if Ni>=10:
                            db1.append(len(df2[(df2['Evolution'] == 2)]) / Ni)
                        else:
                            db1.append(np.nan)

                        features_v.append(feature + ' - ' + str(f))
            else:
                df2 = df[(df.Age>ages[i]) & (df.Age<=ages[i+1]) & (df[feature] == 1)]
                Ni = len(df2)
                if Ni >= 10:
                    db1.append(len(df2[df2['Evolution'] == 2]) / Ni)
                else:
                    db1.append(np.nan)
                if feature in preconditions:
                    d0 = d0[d0[feature] != 1]
                features_v.append(feature)

        N0 = len(d0)
        if N0 > 10:
            db1.append(len(d0[d0['Evolution'] == 2]) / N0)
        else:
            db1.append(np.nan)
        dic[str(ages[i])+'-'+str(ages[i+1])] = db1

    features_v.append('No preconditions')
    df_f = pd.DataFrame(dic, index=features_v)
    sns.set(font_scale=0.7)
    plt.figure(figsize=(15 * .7, 15 * .7))
    g = sns.heatmap(df_f,
                    annot=True,
                    cmap=sns.cm.rocket_r,
                    linewidth=1
                    )
    g.invert_yaxis()
    plt.xlabel('Age Group')

    plt.savefig(out_dir + '/hosp_age_features.png')
    plt.savefig(out_dir + '/hosp_age_features.pdf')
    '''
    #print(categorical_vars)
    features = categorical_vars
    #features.remove('Evolution')
    #features.remove('Hospitalized')
    db1 = []
    db2 = []
    features_l = []
    special_features = ['Gender', 'Color', 'Race', 'Schooling', 'XrayToraxResult']
    for feature in features:
        if feature in special_features:
            for f in df[feature].unique():
                Ni = len(df[df[feature] == f])
                if Ni>10:
                    db1.append(len(df[(df[feature] == f) & (df['Hospitalization'] == 1)]) / Ni)
                    db2.append(len(df[(df[feature] == f) & (df['Evolution'] == 2)]) / Ni)
                    features_l.append(feature+'-'+str(f)+' (' + str(Ni) + ')')
                else:
                    db1.append(np.NaN)
                    db2.append(np.NaN)
                    features_l.append(feature + ' (' + str(Ni) + ')')
        else:
            Ni = len(df[df[feature] == 1])
            if Ni>3:
                db1.append(len(df[(df[feature] == 1) & (df['Hospitalization'] == 1)]) / Ni)
                db2.append(len(df[(df[feature] == 1) & (df['Evolution'] == 2)]) / Ni)
                features_l.append(feature + ' (' + str(Ni) + ')')
            else:
                db1.append(np.NaN)
                db2.append(np.NaN)
                features_l.append(feature + ' (' + str(Ni) + ')')
    df_f = pd.DataFrame({'Death': db2}, index=features_l)
    df_f = df_f.sort_values('Death', ascending = False)
    df_f_transpose = df_f

    plt.figure(figsize = (8, 13))
    sns.heatmap(df_f_transpose, annot=True, cmap=sns.cm.rocket_r, linewidth=1)
    plt.tight_layout()
    plt.savefig(out_dir + '/hosp_features.png')
    plt.savefig(out_dir + '/hosp_features.pdf')
'''

def create_basic_analytics(df, categorical_vars, out_dir, type=1):
    # Build basic statistics table
    #features = list(df)
    #features.remove('Age')
    #features.remove('Evolution')
    #features.remove('Hospitalized')

    create_summary(df, categorical_vars, out_dir)
    create_age_hosp_plot(df, out_dir)
    create_cat_table(df, categorical_vars, out_dir)
    create_cat_age_table(df, categorical_vars, out_dir)

'''

    # Table Edad Test positive
    dic = {}
    preconditions = ['Diabetes', 'Renal_Comorbidity', 'Cardio_Comorbidity', 'Lung_Comorbidity', 'Tobbaco-Use', 'Obesity', 'Respiratory distress']
    for age_group in sorted(df.Age.unique()):
        d0 = df[(df.Age == age_group)]
        db1 = []
        features_v = []
        for feature in features:
            if feature == 'Gender':
                df2 = df[(df.Age == age_group) & (df[feature] == 1)]
                Ni = len(df2)
                if Ni > 5:
                    db1.append(len(df2[(df2['Hospitalized'] == 1)]) / Ni)
                else:
                    db1.append(np.nan)
                features_v.append(feature + '- Female')

                df2 = df[(df.Age == age_group) & (df[feature] == 0)]
                Ni = len(df2)
                if Ni > 5:
                    db1.append(len(df2[(df2['Hospitalized'] == 1)]) / Ni)
                else:
                    db1.append(np.nan)
                features_v.append(feature + '- Male')

            elif feature == 'Color':
                for color in df[feature].unique():
                    df2 = df[(df.Age == age_group) & (df[feature] == color)]
                    Ni = len(df2)
                    if Ni>5:
                        db1.append(len(df2[(df2['Hospitalized'] == 1)]) / Ni)
                    else:
                        db1.append(np.nan)
                    features_v.append(feature + ' - ' + color)

            else:
                df2 = df[(df.Age == age_group) & (df[feature] == 1)]
                Ni = len(df2)
                if Ni > 5:
                    db1.append(len(df2[df2['Hospitalized'] == 1]) / Ni)
                else:
                    db1.append(np.nan)
                if feature in preconditions:
                    d0 = d0[d0[feature] != 1]
                features_v.append(feature)

        N0 = len(d0)
        if N0 > 5:
            db1.append(len(d0[d0['Hospitalized'] == 1]) / N0)
        else:
            db1.append(np.nan)
        dic[age_group] = db1

    features_v.append('No preconditions')
    df_f = pd.DataFrame(dic, index=features_v)
    sns.set(font_scale=0.7)
    plt.figure(figsize = (15*.7,9*.7))
    g = sns.heatmap(df_f,
                annot=True,
                cmap=sns.cm.rocket_r,
                linewidth=1
                )
    g.invert_yaxis()
    plt.xlabel('Age Group')

    plt.savefig(out_dir + '/hosp_age_features.png')
    plt.savefig(out_dir + '/hosp_age_features.pdf')

'''


"""Functional tests for catalog import"""

import os
import shutil

import hipscat.pixel_math as hist
import pytest
from hipscat.catalog.catalog import Catalog

import hipscat_import.catalog.run_import as runner
from hipscat_import.catalog.arguments import ImportArguments
from hipscat_import.catalog.resume_plan import ResumePlan

def test_empty_args():
    """Runner should fail with empty arguments"""
    with pytest.raises(ValueError, match="args is required"):
        runner.run(None, None)

def test_bad_args():
    """Runner should fail with mis-typed arguments"""
    args = {"output_artifact_name": "bad_arg_type"}
    with pytest.raises(ValueError, match="ImportArguments"):
        runner.run(args, None)

@pytest.mark.dask
def test_resume_dask_runner(
    dask_client,
    small_sky_parts_dir,
    resume_dir,
    tmp_path,
    assert_parquet_file_ids,
):
    """Test execution in the presence of some resume files."""
    ## First, copy over our intermediate files.
    ## This prevents overwriting source-controlled resume files.
    temp_path = os.path.join(tmp_path, "resume", "intermediate")
    shutil.copytree(
        os.path.join(resume_dir, "intermediate"),
        temp_path,
    )
    plan = ResumePlan(tmp_path=temp_path, progress_bar=False)
    histogram = hist.empty_histogram(0)
    histogram[11] = 131
    empty = hist.empty_histogram(0)
    for file_index in range(0, 5):
        ResumePlan.touch_key_done_file(temp_path, ResumePlan.SPLITTING_STAGE, f"split_{file_index}")
        ResumePlan.write_partial_histogram(
            tmp_path=temp_path,
            mapping_key=f"map_{file_index}",
            histogram=histogram if file_index == 0 else empty,
        )

    ResumePlan.touch_key_done_file(temp_path, ResumePlan.REDUCING_STAGE, "0_11")

    shutil.copytree(
        os.path.join(resume_dir, "Norder=0"),
        os.path.join(tmp_path, "resume", "Norder=0"),
    )

    args = ImportArguments(
        output_artifact_name="resume",
        input_path=small_sky_parts_dir,
        input_format="csv",
        output_path=tmp_path,
        dask_tmp=tmp_path,
        tmp_dir=tmp_path,
        overwrite=True,
        highest_healpix_order=0,
        pixel_threshold=1000,
        progress_bar=False,
    )

    runner.run(args, dask_client)

    # Check that the catalog metadata file exists
    catalog = Catalog.read_from_hipscat(args.catalog_path)
    assert catalog.on_disk
    assert catalog.catalog_path == args.catalog_path
    assert catalog.catalog_info.ra_column == "ra"
    assert catalog.catalog_info.dec_column == "dec"
    assert catalog.catalog_info.total_rows == 131
    assert len(catalog.get_healpix_pixels()) == 1

    # Check that the catalog parquet file exists and contains correct object IDs
    output_file = os.path.join(args.catalog_path, "Norder=0", "Dir=0", "Npix=11.parquet")

    expected_ids = [*range(700, 831)]
    assert_parquet_file_ids(output_file, "id", expected_ids)

    ## Re-running the pipeline with fully done intermediate files
    ## should result in no changes to output files.
    shutil.copytree(
        os.path.join(resume_dir, "intermediate"),
        temp_path,
    )
    plan = args.resume_plan
    plan.touch_stage_done_file(ResumePlan.MAPPING_STAGE)
    plan.touch_stage_done_file(ResumePlan.SPLITTING_STAGE)
    plan.touch_stage_done_file(ResumePlan.REDUCING_STAGE)

    args = ImportArguments(
        output_artifact_name="resume",
        input_path=small_sky_parts_dir,
        input_format="csv",
        output_path=tmp_path,
        dask_tmp=tmp_path,
        tmp_dir=tmp_path,
        overwrite=True,
        highest_healpix_order=0,
        pixel_threshold=1000,
        progress_bar=False,
    )

    runner.run(args, dask_client)

    catalog = Catalog.read_from_hipscat(args.catalog_path)
    assert catalog.on_disk
    assert catalog.catalog_path == args.catalog_path
    assert catalog.catalog_info.ra_column == "ra"
    assert catalog.catalog_info.dec_column == "dec"
    assert catalog.catalog_info.total_rows == 131
    assert len(catalog.get_healpix_pixels()) == 1
    assert_parquet_file_ids(output_file, "id", expected_ids)

@pytest.mark.dask
def test_dask_runner(
    dask_client,
    small_sky_parts_dir,
    assert_parquet_file_ids,
    tmp_path,
):
    """Test basic execution."""
    args = ImportArguments(
        output_artifact_name="small_sky_object_catalog",
        input_path=small_sky_parts_dir,
        input_format="csv",
        output_path=tmp_path,
        dask_tmp=tmp_path,
        highest_healpix_order=1,
        progress_bar=False,
    )

    runner.run(args, dask_client)

    # Check that the catalog metadata file exists
    catalog = Catalog.read_from_hipscat(args.catalog_path)
    assert catalog.on_disk
    assert catalog.catalog_path == args.catalog_path
    assert catalog.catalog_info.ra_column == "ra"
    assert catalog.catalog_info.dec_column == "dec"
    assert catalog.catalog_info.total_rows == 131
    assert len(catalog.get_healpix_pixels()) == 1

    # Check that the catalog parquet file exists and contains correct object IDs
    output_file = os.path.join(args.catalog_path, "Norder=0", "Dir=0", "Npix=11.parquet")

    expected_ids = [*range(700, 831)]
    assert_parquet_file_ids(output_file, "id", expected_ids)

@pytest.mark.dask
def test_dask_runner_stats_only(dask_client, small_sky_parts_dir, tmp_path):
    """Test basic execution, without generating catalog parquet outputs."""
    args = ImportArguments(
        output_artifact_name="small_sky",
        input_path=small_sky_parts_dir,
        input_format="csv",
        output_path=tmp_path,
        dask_tmp=tmp_path,
        highest_healpix_order=1,
        progress_bar=False,
        debug_stats_only=True,
    )

    runner.run(args, dask_client)

    metadata_filename = os.path.join(args.catalog_path, "catalog_info.json")
    assert os.path.exists(metadata_filename)

    # Check that the catalog parquet file DOES NOT exist
    output_file = os.path.join(args.catalog_path, "Norder=0", "Dir=0", "Npix=11.parquet")

    assert not os.path.exists(output_file)