import asyncio
import re
from typing import Any, Dict, Iterable, List, Set, Tuple
import click
from errata_tool import Erratum

from elliottlib import bzutil, constants, logutil
from elliottlib.cli.common import cli, click_coroutine, pass_runtime
from elliottlib.errata_async import AsyncErrataAPI, AsyncErrataUtils
from elliottlib.runtime import Runtime
from elliottlib.util import (minor_version_tuple, red_print)
from elliottlib.bzutil import Bug
from elliottlib.cli.attach_cve_flaws_cli import get_flaws
from elliottlib.cli.find_bugs_sweep_cli import FindBugsSweep, categorize_bugs_by_type

logger = logutil.getLogger(__name__)

@cli.command("verify-attached-bugs",
             short_help="Run validations on bugs attached to release advisories and report results")
@click.option("--verify-bug-status", is_flag=True,
              help="Check that bugs of advisories are all VERIFIED or more",
              type=bool, default=False)
@click.option("--verify-flaws", is_flag=True,
              help="Check that flaw bugs are attached and associated with respective builds",
              type=bool, default=False)
@click.option("--no-verify-blocking-bugs", is_flag=True,
              help="Don't check if there are open bugs for the next minor version blocking bugs for this minor version",
              type=bool, default=False)
@click.option("--skip-multiple-advisories-check", is_flag=True,
              help="Do not check if bugs are attached to multiple advisories",
              type=bool, default=False)
@click.argument("advisories", nargs=-1, type=click.IntRange(1), required=False)
@pass_runtime
@click_coroutine
async def verify_attached_bugs_cli(runtime: Runtime, verify_bug_status: bool, advisories: Tuple[int, ...], verify_flaws: bool,
                                   no_verify_blocking_bugs: bool, skip_multiple_advisories_check: bool):
    """
    Validate bugs in release advisories (specified in arguments or as part of an assembly).
    Requires group param (-g openshift-X.Y)

    Default validations:
    - Target Release: All bugs belong to the group's target release.
    - Bug Sorting: Bugs are attached to the correct advisory type (image/rpm/extras) using
    find-bugs:sweep logic. This only runs with --assembly. To skip manually specify advisories.
    - Improper Tracker Bugs: Bugs that have CVE in their title but do not have Tracker Bug labels set.
    - Regression Check: If there are any backport bugs, make sure they don't get ahead of their original bug.
    This is to make sure we don't regress when upgrading from OCP X.Y to X.Y+1. To skip use --no-verify-blocking-bugs.
    - Bugs aren't attached to multiple advisories. To skip use --skip-multiple-advisories-check.

    Additional validations:
    - Bug Status (--verify-bug-status): All bugs are in at least in VERIFIED state
    - Flaw Bugs (--verify-flaws): All flaw bugs are attached to advisories (using attach-cve-flaws logic) and
    associated with respective builds. Verify advisory type is RHSA when applicable, and CVE names field is correct.

    If any verification fails, a text explanation is given and the return code is 1.
    Otherwise, prints the number of bugs in the advisories and exits with success.
    """
    runtime.initialize()
    if advisories:
        click.echo("WARNING: Cannot verify advisory bug sorting. To verify that bugs are attached to the "
                   "correct release advisories, run with --assembly=<release>")
        advisory_id_map = {'?': a for a in advisories}
    else:
        advisory_id_map = runtime.get_default_advisories()
    if not advisory_id_map:
        red_print("No advisories specified on command line or in [group.yml|releases.yml]")
        exit(1)
    await verify_attached_bugs(runtime, verify_bug_status, advisory_id_map, verify_flaws, no_verify_blocking_bugs, skip_multiple_advisories_check)

async def verify_attached_bugs(runtime: Runtime, verify_bug_status: bool, advisory_id_map: Dict[str, int], verify_flaws:
                               bool, no_verify_blocking_bugs: bool, skip_multiple_advisories_check: bool):
    validator = BugValidator(runtime, output="text")
    advisory_bug_map = validator.get_attached_bugs(list(advisory_id_map.values()))
    bugs = {b for bugs in advisory_bug_map.values() for b in bugs}

    # bug.is_ocp_bug() filters by product/project, so we don't get flaw bugs or bugs of other products or
    # placeholder
    non_flaw_bugs = [b for b in bugs if b.is_ocp_bug()]

    validator.validate(non_flaw_bugs, verify_bug_status, no_verify_blocking_bugs)

    # skip advisory type check if advisories are
    # manually passed in, and we don't know their type
    if '?' not in advisory_id_map.keys():
        validator.verify_bugs_advisory_type(non_flaw_bugs, advisory_id_map, advisory_bug_map)

    if not skip_multiple_advisories_check:
        await validator.verify_bugs_multiple_advisories(non_flaw_bugs)

    if verify_flaws:
        await validator.verify_attached_flaws(advisory_bug_map)

    # Close client session
    await validator.close()

    if validator.problems:
        if validator.output != 'slack':
            red_print(f"Found the following problems: {validator.problems}")
            red_print("Some bug problems were listed above. Please investigate.")
        exit(1)

@cli.command("verify-bugs", short_help="Verify bugs included in an assembly (default --assembly=stream)")
@click.option("--verify-bug-status", is_flag=True, help="Check that bugs of advisories are all VERIFIED or more",
              type=bool, default=False)
@click.option("--no-verify-blocking-bugs", is_flag=True,
              help="Don't check if there are open bugs for the next minor version blocking bugs for this minor version",
              type=bool, default=False)
@click.option('--output', '-o',
              required=False,
              type=click.Choice(['text', 'json', 'slack']),
              default='text',
              help='Applies chosen format to command output')
@pass_runtime
@click_coroutine
async def verify_bugs_cli(runtime, verify_bug_status, output, no_verify_blocking_bugs: bool):
    """
    Validate bugs that qualify as being part of an assembly (specified as --assembly)
    Requires group param (-g openshift-X.Y). By default runs for --assembly=stream

    Default validations:
    - Target Release: All bugs belong to the group's target release.
    - Improper Tracker Bugs: Bugs that have CVE in their title but do not have Tracker Bug labels set.
    - Regression Check: If there are any backport bugs, make sure they don't get ahead of their original bug.
    This is to make sure we don't regress when upgrading from OCP X.Y to X.Y+1. To skip use --no-verify-blocking-bugs.

    Additional validations:
    - Bug Status (--verify-bug-status): All bugs are at least in VERIFIED state

    """
    runtime.initialize()
    await verify_bugs(runtime, verify_bug_status, output, no_verify_blocking_bugs)

async def verify_bugs(runtime, verify_bug_status, output, no_verify_blocking_bugs):
    validator = BugValidator(runtime, output)
    find_bugs_obj = FindBugsSweep(cve_only=False)
    ocp_bugs = []
    logger.info(f'Using {runtime.assembly} assembly to search bugs')
    for b in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
        bugs = find_bugs_obj.search(bug_tracker_obj=b, verbose=runtime.debug)
        logger.info(f"Found {len(bugs)} {b.type} bugs: {[b.id for b in bugs]}")
        ocp_bugs.extend(bugs)

    validator.validate(ocp_bugs, verify_bug_status, no_verify_blocking_bugs)

    # Close client session
    await validator.close()

    if validator.problems:
        if validator.output != 'slack':
            red_print("Some bug problems were listed above. Please investigate.")
        exit(1)

class BugValidator:

    def __init__(self, runtime: Runtime, output: str = 'text'):
        self.runtime = runtime
        self.target_releases: List[str] = runtime.get_bug_tracker('jira').config['target_release']
        self.et_data: Dict[str, Any] = runtime.get_errata_config()
        self.errata_api = AsyncErrataAPI(self.et_data.get("server", constants.errata_url))
        self.problems: List[str] = []
        self.output = output

    async def close(self):
        await self.errata_api.close()

    def validate(self, non_flaw_bugs: List[Bug], verify_bug_status: bool, no_verify_blocking_bugs: bool):
        non_flaw_bugs = self.filter_bugs_by_release(non_flaw_bugs, complain=True)
        self._find_invalid_trackers(non_flaw_bugs)
        if not no_verify_blocking_bugs:
            blocking_bugs_for = self._get_blocking_bugs_for(non_flaw_bugs)
            self._verify_blocking_bugs(blocking_bugs_for)

        if verify_bug_status:
            self._verify_bug_status(non_flaw_bugs)

    def verify_bugs_advisory_type(self, non_flaw_bugs, advisory_id_map, advisory_bug_map):
        bugs_by_type = categorize_bugs_by_type(non_flaw_bugs, advisory_id_map)
        for kind, advisory_id in advisory_id_map.items():
            actual = {b for b in advisory_bug_map[advisory_id] if b.is_ocp_bug()}
            expected = bugs_by_type[kind]
            if actual != expected:
                bugs_not_found = expected - actual
                extra_bugs = actual - expected
                if bugs_not_found:
                    self._complain(f'Expected Bugs not found in {kind} advisory ({advisory_id}):'
                                   f' {[b.id for b in bugs_not_found]}')
                if extra_bugs:
                    self._complain(f'Unexpected Bugs found in {kind} advisory ({advisory_id}):'
                                   f' {[b.id for b in extra_bugs]}')

    async def verify_bugs_multiple_advisories(self, non_flaw_bugs: List[Bug]):
        logger.info(f'Checking {len(non_flaw_bugs)} bugs, if any bug is attached to multiple advisories')

        async def get_all_advisory_ids(bug):
            all_advisories_id = bug.all_advisory_ids()
            if len(all_advisories_id) > 1:
                return f'Bug <{bug.weburl}|{bug.id}> is attached in multiple advisories: {all_advisories_id}'
            return None

        results = await asyncio.gather(*[asyncio.create_task(get_all_advisory_ids(bug)) for bug in non_flaw_bugs])
        for message in results:
            if message:
                self._complain(message)

    async def verify_attached_flaws(self, advisory_bugs: Dict[int, List[Bug]]):
        futures = []
        for advisory_id, attached_bugs in advisory_bugs.items():
            attached_trackers = [b for b in attached_bugs if b.is_tracker_bug()]
            attached_flaws = [b for b in attached_bugs if b.is_flaw_bug()]
            logger.info(f"Verifying advisory {advisory_id}: attached-trackers: "
                        f"{[b.id for b in attached_trackers]} "
                        f"attached-flaws: {[b.id for b in attached_flaws]}")
            futures.append(self._verify_attached_flaws_for(advisory_id, attached_trackers, attached_flaws))
        await asyncio.gather(*futures)

    async def _verify_attached_flaws_for(self, advisory_id: int, attached_trackers: Iterable[Bug], attached_flaws: Iterable[Bug]):
        flaw_bug_tracker = self.runtime.get_bug_tracker('bugzilla')
        brew_api = self.runtime.build_retrying_koji_client()
        tracker_flaws, first_fix_flaw_bugs = get_flaws(flaw_bug_tracker, attached_trackers, brew_api,
                                                       self.runtime.logger)

        # Check if attached flaws match expected flaws
        first_fix_flaw_ids = {b.id for b in first_fix_flaw_bugs}
        attached_flaw_ids = {b.id for b in attached_flaws}
        missing_flaw_ids = first_fix_flaw_ids - attached_flaw_ids
        if missing_flaw_ids:
            self._complain(f"On advisory {advisory_id}, these flaw bugs are not attached: "
                           f"{', '.join(sorted(map(str, missing_flaw_ids)))} but "
                           "they are referenced by attached tracker bugs. "
                           "You need to attach those flaw bugs or drop corresponding tracker bugs.")
        extra_flaw_ids = attached_flaw_ids - first_fix_flaw_ids
        if extra_flaw_ids:
            self._complain(f"On advisory {advisory_id}, these flaw bugs are attached: "
                           f"{', '.join(sorted(map(str, extra_flaw_ids)))} but "
                           f"there are no tracker bugs referencing them. "
                           "You need to drop those flaw bugs or attach corresponding tracker bugs.")

        # Check if advisory is of the expected type
        advisory_info = await self.errata_api.get_advisory(advisory_id)
        advisory_type = next(iter(advisory_info["errata"].keys())).upper()  # should be one of [RHBA, RHSA, RHEA]
        if not first_fix_flaw_ids:
            if advisory_type == "RHSA":
                self._complain(f"Advisory {advisory_id} is of type {advisory_type} "
                               f"but has no first-fix flaw bugs. It should be converted to RHBA or RHEA.")
            return  # The remaining checks are not needed for a non-RHSA.
        if advisory_type != "RHSA":
            self._complain(f"Advisory {advisory_id} is of type {advisory_type} but has first-fix flaw bugs "
                           f"{first_fix_flaw_ids}. It should be converted to RHSA.")

        # Validate CVE mappings (of CVEs and builds)
        flaw_id_bugs = {f.id: f for f in first_fix_flaw_bugs}
        cve_components_mapping: Dict[str, Set[str]] = {}
        for tracker in attached_trackers:
            component_name = tracker.whiteboard_component
            flaw_ids = tracker_flaws[tracker.id]
            for flaw_id in flaw_ids:
                if flaw_id not in flaw_id_bugs:  # This means associated flaw wasn't considered a first fix
                    continue
                alias = [k for k in flaw_id_bugs[flaw_id].alias if k.startswith('CVE-')]
                if len(alias) != 1:
                    raise ValueError(f"Bug {flaw_id} should have exactly 1 CVE alias.")
                cve = alias[0]
                cve_components_mapping.setdefault(cve, set()).add(component_name)

        attached_builds = await self.errata_api.get_builds_flattened(advisory_id)
        try:
            extra_exclusions, missing_exclusions = await AsyncErrataUtils.validate_cves_and_get_exclusions_diff(
                self.errata_api,
                advisory_id, attached_builds,
                cve_components_mapping)
        except ValueError as e:
            self._complain(e)

        for cve, cve_package_exclusions in extra_exclusions.items():
            if cve_package_exclusions:
                self._complain(f"On advisory {advisory_id}, {cve} is not associated with Brew components "
                               f"{', '.join(sorted(cve_package_exclusions))}."
                               " You may need to associate the CVE with the components "
                               "in the CVE mapping or drop the tracker bugs.")
        for cve, cve_package_exclusions in missing_exclusions.items():
            if cve_package_exclusions:
                self._complain(f"On advisory {advisory_id}, {cve} is associated with Brew components "
                               f"{', '.join(sorted(cve_package_exclusions))} without a tracker bug."
                               " You may need to explicitly exclude those Brew components from the CVE "
                               "mapping or attach the corresponding tracker bugs.")

        # Validate `CVE Names` field of the advisory
        advisory_cves = advisory_info["content"]["content"]["cve"].split()
        extra_cves = cve_components_mapping.keys() - advisory_cves
        if extra_cves:
            self._complain(f"On advisory {advisory_id}, bugs for the following CVEs are already attached "
                           f"but they are not listed in advisory's `CVE Names` field: {', '.join(sorted(extra_cves))}")
        missing_cves = advisory_cves - cve_components_mapping.keys()
        if missing_cves:
            self._complain(f"On advisory {advisory_id}, bugs for the following CVEs are not attached but listed in "
                           f"advisory's `CVE Names` field: {', '.join(sorted(missing_cves))}")

    def get_attached_bugs(self, advisory_ids: List[str]) -> Dict[int, Set[Bug]]:
        """ Get bugs attached to specified advisories
        :return: a dict with advisory id as key and set of bug objects as value
        """
        logger.info(f"Retrieving bugs for advisories: {advisory_ids}")
        advisories = [Erratum(errata_id=advisory_id) for advisory_id in advisory_ids]

        attached_bug_map = {advisory_id: set() for advisory_id in advisory_ids}
        for bug_tracker_type in ['jira', 'bugzilla']:
            bug_tracker = self.runtime.get_bug_tracker(bug_tracker_type)
            advisory_bug_id_map = {advisory.errata_id: bug_tracker.advisory_bug_ids(advisory)
                                   for advisory in advisories}
            bug_map = bug_tracker.get_bugs_map([bug_id for bug_list in advisory_bug_id_map.values()
                                                for bug_id in bug_list])
            for advisory_id in advisory_ids:
                set_of_bugs = {bug_map[bid] for bid in advisory_bug_id_map[advisory_id] if bid in bug_map}
                attached_bug_map[advisory_id] = attached_bug_map[advisory_id] | set_of_bugs
        return attached_bug_map

    def filter_bugs_by_release(self, bugs: List[Bug], complain: bool = False) -> List[Bug]:
        # filter out bugs with an invalid target release
        filtered_bugs = []
        for b in bugs:
            # b.target release is a list of size 0 or 1
            if any(target in self.target_releases for target in b.target_release):
                filtered_bugs.append(b)
            elif complain:
                self._complain(
                    f"bug {b.id} target release {b.target_release} is not in "
                    f"{self.target_releases}. Does it belong in this release?"
                )
        return filtered_bugs

    def _get_blocking_bugs_for(self, bugs):
        # get blocker bugs in the next version for all bugs we are examining
        candidate_blockers = []
        for b in bugs:
            if b.depends_on:
                candidate_blockers.extend(b.depends_on)
        jira_ids, bz_ids = bzutil.get_jira_bz_bug_ids(set(candidate_blockers))

        v = minor_version_tuple(self.target_releases[0])
        next_version = (v[0], v[1] + 1)

        def is_next_target(target_v):
            pattern = re.compile(r'^\d+\.\d+\.(0|z)$')
            return pattern.match(target_v) and minor_version_tuple(target_v) == next_version

        def managed_by_art(b: Bug):
            components_not_managed_by_art = self.runtime.get_bug_tracker('jira').component_filter()
            return b.component not in components_not_managed_by_art

        # retrieve blockers and filter to those with correct product and target version
        blockers = []
        if jira_ids:
            blockers.extend(self.runtime.get_bug_tracker('jira')
                            .get_bugs(jira_ids))
        if bz_ids:
            blockers.extend(self.runtime.get_bug_tracker('bugzilla')
                            .get_bugs(bz_ids))
        logger.debug(f"Candidate Blocker bugs found: {[b.id for b in blockers]}")
        blocking_bugs = {}
        for bug in blockers:
            if not bug.is_ocp_bug() or not managed_by_art(bug):
                continue
            target_release = []
            # A bug without `Target Version` shouldn't be considered as a blocking bug
            try:
                target_release = bug.target_release
            except ValueError as err:  # bug.target_release will raise ValueError if Target Version is not set
                if "does not have `Target Version` field set" not in str(err):
                    raise
            next_target = any(is_next_target(target) for target in target_release)
            if next_target:
                blocking_bugs[bug.id] = bug
        logger.info(f"Blocking bugs for next target release ({next_version[0]}.{next_version[1]}): "
                    f"{list(blocking_bugs.keys())}")

        k = {bug: [blocking_bugs[b] for b in bug.depends_on if b in blocking_bugs] for bug in bugs}
        return k

    def _verify_blocking_bugs(self, blocking_bugs_for):
        # complain about blocking bugs that aren't verified or shipped
        for bug, blockers in blocking_bugs_for.items():
            for blocker in blockers:
                message = str()
                if blocker.status not in ['VERIFIED', 'RELEASE_PENDING', 'CLOSED', 'Release Pending', 'Verified',
                                          'Closed'] and not (blocker.status == bug.status == "ON_QA"):
                    if self.output == 'text':
                        message = f"Regression possible: {bug.status} bug {bug.id} is a backport of bug " \
                            f"{blocker.id} which has status {blocker.status}"
                    elif self.output == 'slack':
                        message = f"`{bug.status}` bug <{bug.weburl}|{bug.id}> is a backport of " \
                                  f"`{blocker.status}` bug <{blocker.weburl}|{blocker.id}>"
                    self._complain(message)
                if blocker.status in ['CLOSED', 'Closed'] and \
                    blocker.resolution not in ['CURRENTRELEASE', 'NEXTRELEASE', 'ERRATA', 'DUPLICATE', 'NOTABUG', 'WONTFIX',
                                               'Done', 'Fixed', 'Done-Errata',
                                               'Current Release', 'Errata', 'Next Release',
                                               "Won't Do", "Won't Fix",
                                               'Duplicate', 'Duplicate Issue',
                                               'Not a Bug']:
                    if self.output == 'text':
                        message = f"Regression possible: {bug.status} bug {bug.id} is a backport of bug " \
                            f"{blocker.id} which was CLOSED {blocker.resolution}"
                    elif self.output == 'slack':
                        message = f"`{bug.status}` bug <{bug.weburl}|{bug.id}> is a backport of bug " \
                            f"<{blocker.weburl}|{blocker.id}> which was CLOSED `{blocker.resolution}`"
                    self._complain(message)

    def _verify_bug_status(self, bugs):
        # complain about bugs that are not yet VERIFIED or more.
        for bug in bugs:
            if bug.is_flaw_bug():
                continue
            if bug.status in ["VERIFIED", "RELEASE_PENDING", "Verified", "Release Pending"]:
                continue
            if bug.status in ["CLOSED", "Closed"] and bug.resolution in ["ERRATA", 'Errata']:
                continue
            status = f"{bug.status}"
            if bug.status in ['CLOSED', 'Closed']:
                status = f"{bug.status}: {bug.resolution}"
            self._complain(f"Bug <{bug.weburl}|{bug.id}> has status {status}")

    def _find_invalid_trackers(self, bugs):
        logger.info("Checking for invalid tracker bugs")
        # complain about invalid tracker bugs
        for bug in bugs:
            if bug.is_invalid_tracker_bug():
                self._complain(f"Bug <{bug.weburl}|{bug.id}> is an invalid tracker bug. Please fix")

    def _complain(self, problem: str):
        red_print(problem)
        self.problems.append(problem)


#!/usr/bin/env python3

'''

'''
from matplotlib import pyplot as plt

import matplotlib.patches as patches
import numpy as np
import csv
import math

# Map storing a set of landmarks
# landmarks are stored in a dictionary mapping  cube-IDs on Frame2D (indicating their true position)
class Track:
    def __init__(self, landmarks):
        self.landmarks = landmarks
        self.minX = None
        self.minY = None
        self.maxX = None
        self.maxY = None

class Cone:
    def __init__(self):
        self.pos = np.identity(3)
        self.colour = 'red'
        self.size = None #change at a later time
        self.visible = False

    def __init__(self, pos: np.array, colour):
        self.pos = pos
        self.colour = colour
        self.size = None
        self.visible = False

#This function loads a "blergh.csv" file in. If you want to make your own then look at the format on Oval.csv and just
#change the filename from runSimTrack.py (line 15 or nearby)
def loadTrack(trackName):
    scale = 1

    coneArray = []
    with open(trackName, 'r') as file:
        reader = csv.reader(file, delimiter=',')
        for row in reader:
            conePos = np.array([[math.cos(0), -math.sin(0), float(row[0])*scale], [math.sin(0), math.cos(0), float(row[1])*scale], [0., 0.,1.]])
            coneColour = row[2]
            coneArray.append(Cone(conePos, coneColour))

    minX = coneArray[0].pos[0,2]
    minY = coneArray[0].pos[1,2]
    maxX = coneArray[0].pos[0,2]
    maxY = coneArray[0].pos[1,2]

    for cone in coneArray:
        x = cone.pos[0,2]
        y = cone.pos[1,2]
        if(x < minX):
            minX = x
        if (x > maxX):
            maxX = x
        if (y < minY):
            minY = y
        if (y > maxY):
            maxY = y

    T = Track(coneArray)
    border = 2
    T.minX = minX - border
    T.minY = minY - border
    T.maxX = maxX + border
    T.maxY = maxY + border
    return T

#Can be used as a standalone program if you don't want a sim but want to see the track
def plotTrack(ax, t: Track, gridOn, color="blue"):
    if(gridOn == True):
        minX = t.minX
        maxX = t.maxX
        minY = t.minY
        maxY = t.maxY
        tick = 0.5
        numX = abs(maxX-minX)
        numY = abs(maxY-minY)

        for xIndex in range(0, math.ceil(numX/tick) + 1):
            x = minX + xIndex * tick
            bold = 0.8 if (xIndex - 1) % 5 == 0 else 0.4
            plt.plot([x, x], [minY, maxY], color, alpha=bold, linewidth=bold)
        for yIndex in range(0, math.ceil(numY/tick) + 1):
            y = minY + yIndex * tick
            bold = 0.8 if (yIndex - 1) % 5 == 0 else 0.4
            plt.plot([minX, maxX], [y, y], color, alpha=bold, linewidth=bold)

    for landmark in t.landmarks:
        x = landmark.pos[0,2]
        y = landmark.pos[1,2]
        size = 0.25
        if(landmark.colour.lower()=="blue"):
            colour = "blue"
        elif(landmark.colour.lower()=="yellow"):
            colour = "yellow"
        elif(landmark.colour.lower()=="orange"):
            colour = "orange"
        else:
            colour = "red"
        rect = patches.Rectangle((float(x) - size, float(y) - size), 2 * size, 2 * size, linewidth=2, edgecolor=str(colour), facecolor="none", zorder=0)
        ax.add_patch(rect)


import sys
import logging

INDENT_CHAR = "\t"

class CloudMinerlogger(logging.Logger):
    """
    The main program logging
    """
    def __init__(self, name: str, level: int = 0) -> None:
        super().__init__(name, level)
        self.indent = 0

    def _log(self, level, msg, *args, **kwargs) -> None:
        if level == logging.INFO:
            bullet = '[+]'
        elif level == logging.DEBUG:
            bullet = '[*]'
        elif level == logging.ERROR or level == logging.WARNING:
            bullet = '[!]'
        elif level == 100: #header
            bullet = '[#]'
        else:
            bullet = '[~]'

        msg = f"{INDENT_CHAR*self.indent}{bullet} {msg}"
        return super()._log(level, msg, *args, **kwargs)

    def add_indent(self):
        self.indent += 1

    def remove_indent(self):
        self.indent -= 1

def init_logging() -> CloudMinerlogger:
    logger = CloudMinerlogger("root")
    handler = logging.StreamHandler(sys.stdout)
    logger.addHandler(handler)
    return logger

logger = init_logging()


"""
Parts of the codes from https://git.iyunxiao.com/DeepVision/pytorch-ocr-framework
by hzc wt
"""

import os
import copy
import cv2
import torch
import torch.utils.data as data
import random
import numpy as np
# from functools import partial
# import math
# from losses.box_utils import jaccard
# from .utils import load_annoataion_quard, random_crop
from itertools import islice
from itertools import cycle
from torchvision import transforms
from PIL import Image
from . import register_dataset
label_prefix = 'labels'

def get_max_shape(arrays):
    """
    Args:
        images: list of arrays

    """
    shapes = list(map(lambda x: list(x.shape), arrays))
    ndim = len(arrays[0].shape)
    max_shape = []
    for d in range(ndim):
        max_shape += [max(shapes, key=lambda x: x[d])[d]]

    return max_shape

def pad_batch_images(images, max_shape=None):
    """
    Args:
        images: list of arrays
        target_shape: shape at which we want to pad

    """

    # 1. max shape
    if max_shape is None:
        max_shape = get_max_shape(images)

    # 2. apply formating
    batch_images = 255 * np.ones([len(images)] + list(max_shape))
    # batch_images = np.squeeze(batch_images, 1)
    # print(batch_images.shape, max_shape)
    for idx, img in enumerate(images):
        batch_images[idx, :, :img.shape[1], :img.shape[2]] = img

    return batch_images.astype(np.float32)

def pad_batch_formulas(formulas, id_pad, id_end, max_len=None):
    """Pad formulas to the max length with id_pad and adds and id_end token
    at the end of each formula

    Args:
        formulas: (list) of list of ints
        max_length: length maximal of formulas

    Returns:
        array: of shape = (batch_size, max_len) of type np.int32
        array: of shape = (batch_size) of type np.int32

    """
    if max_len is None:
        max_len = max(list(map(lambda x: len(x), formulas)))

    batch_formulas = id_pad * np.ones([len(formulas), max_len+1],
            dtype=np.int32)
    formula_length = np.zeros(len(formulas), dtype=np.int32)
    for idx, formula in enumerate(formulas):
        batch_formulas[idx, :len(formula)] = np.asarray(formula,
                dtype=np.int32)
        batch_formulas[idx, len(formula)] = id_end
        formula_length[idx] = len(formula) + 1

    return batch_formulas, formula_length

def img_sort_key(ex):
    """Sort using the size of the image: (width, height)."""
    return ex[0].shape[2], ex[0].shape[1]

def filter_example(ex, use_src_len=True, use_tgt_len=True,
                   min_src_len=1, max_src_len=float('inf'),
                   min_tgt_len=1, max_tgt_len=float('inf')):
    """Return whether an example is an acceptable length.
    If used with a dataset as ``filter_pred``, use :func:`partial()`
    for all keyword arguments.
    Args:
        ex (torchtext.data.Example): An object with a ``src`` and ``tgt``
            property.
        use_src_len (bool): Filter based on the length of ``ex.src``.
        use_tgt_len (bool): Similar to above.
        min_src_len (int): A non-negative minimally acceptable length
            (examples of exactly this length will be included).
        min_tgt_len (int): Similar to above.
        max_src_len (int or float): A non-negative (possibly infinite)
            maximally acceptable length (examples of exactly this length
            will be included).
        max_tgt_len (int or float): Similar to above.
    """
    src_len = len(ex.src[0])
    tgt_len = len(ex.tgt[0])
    return (not use_src_len or min_src_len <= src_len <= max_src_len) and \
        (not use_tgt_len or min_tgt_len <= tgt_len <= max_tgt_len)

def split_corpus(data, chunk_size):
    """
    Args:
        data: cycle iterator, 返回图片路径和对应的formula
        chunk_size: 一次性读取到内存中的图片张数
    Return:
        data_sort: yield 排序后的img和对应的formula
    """
    if chunk_size <= 0:
        yield data
    else:
        while True:
            shard = list(islice(data, chunk_size))  # 每次取shard size的图片进行排序
            if not shard:
                break
            data_pair = []
            # max_h, max_w = 0, 0
            for img_path, formula in shard:
                if not os.path.exists(img_path):
                    img_path = img_path.replace('media', 'home')
                img = transforms.ToTensor()(
                    Image.fromarray(cv2.imread(img_path, 0)))  # 1 * h * w
                data_pair.append((img, formula, img_path))
            data_sort = sorted(data_pair, key=img_sort_key)
            yield data_sort

@register_dataset
class ImageIm2LatexDataset(data.Dataset):
    @staticmethod
    def modify_commandline_options(parser, is_train):
        return parser
    def __init__(self, dataroot):
        super(ImageIm2LatexDataset, self).__init__()
        self.dataroot = dataroot
        data_list = []
        # if opt.isTrain:
        file_name = "all_train_random.txt"
        # else:
        #     file_name = "all_test.txt"
        with open(os.path.join(self.dataroot, file_name), encoding='utf-8') as f:
            for line in f.readlines():
                img_file, formula = line.split(" ", 1)
                data_list.append((img_file, formula))
        random.shuffle(data_list)
        print("load {} images".format(len(data_list)))
        self.data_len = len(data_list)
        self.data = cycle(data_list)
        self.chunk_size = 500  # 一次加载chunk_size数量的图片进入内存，并按照w,h的顺序排序
        self.chunk_data = iter(next(split_corpus(self.data, self.chunk_size)))
        pass

    def __len__(self):

        return self.data_len

    def __getitem__(self, index):
        err_count = 100
        while True and err_count:
            try:
                img, formula, img_name = next(self.chunk_data)
            except:
                self.chunk_data = iter(next(split_corpus(self.data, self.chunk_size)))
                img, formula, img_name = next(self.chunk_data)
            # print("img", img.shape, torch.max(img), torch.min(img), img.dtype)
            # print(img_name)
            if len(formula) == 0 or img.shape[1] > 1900 or img.shape[2] > 1900 \
                    or img.shape[1] < 20 or img.shape[2] < 20 \
                    or img.shape[1] * img.shape[2] > 600000:
                err_count -= 1
                continue
            # if self.opt.isTrain:
            return img, formula
            # else:
            #     return img, formula, img_name

    @staticmethod
    def name():
        return 'ImageIm2LatexDataset'

    @staticmethod
    def collate_fn(batch):
        images = []
        formulas = []
        filenames = []
        c = batch[-1][0].shape[0]
        max_h = max([t[0].shape[1] for t in batch])
        max_w = max([t[0].shape[2] for t in batch])
        batch_len = len(batch[0])
        if batch_len == 2:
            for item, formula in batch:
                img = np.ones((c, max_h, max_w))
                img[:, 0:item.shape[1], 0:item.shape[2]] = item
                images.append(img[np.newaxis, :, :, :])
                formulas.append(formula.strip())
        elif batch_len == 3:
            for item, formula, filename in batch:
                img = np.ones((c, max_h, max_w))
                img[:, 0:item.shape[1], 0:item.shape[2]] = item
                images.append(img[np.newaxis, :, :, :])
                formulas.append(formula.strip())
                filenames.append(filename)
        # random.shuffle(images)
        images = np.concatenate(images, axis=0)
        # formulas = np.concatenate((formulas), axis=0)
        images = images.astype(np.float32)

        # print(images.shape, np.max(images), np.min(images), images.dtype)
        if batch_len == 2:
            return images, formulas
        else:
            return images, formulas, filenames

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--dataroot', type=str, default='/media/Data/hzc/datasets/formulas/')
    opt = parser.parse_known_args()
    opt = parser.parse_args()
    dataset = ImageIm2LatexDataset(opt)
    data_loader = torch.utils.data.DataLoader(
        dataset, shuffle=True, batch_size=5,
        num_workers=1, collate_fn=dataset.collate_fn)
    for i, _ in enumerate(data_loader):
        pass
    print("sa sa sa!")
    print("su su su!")
    os._exit(0)


from flask import Flask, render_template, request
from scraper import Scraper

app = Flask(__name__)

def check_validity(url):
    if 'github.com' not in url:
        return False
    return True

@app.route("/")
def home():
    return render_template("input.html")

@app.route('/issues', methods=['POST', 'GET'])
def result():
    """
    renders the html output based on the input url
    """
    if request.method == 'POST':
        url = request.form['repo_url']
        if not url:
            return render_template('input.html', invalid=True)
        if not check_validity(url):
            return render_template('input.html', invalid=True)
        scraper = Scraper(url)
        scraper.get_data()
        ctx = scraper.send_final_data()
        return render_template("output.html", ctx=ctx)

@app.route('/test')
def test():
    return render_template('test.html')

if __name__ == "__main__":
    app.run(debug=True)


import os
import pygame
import time
import random
import math

count = 0

def setup(screen, etc):
    pass

def draw(screen, etc):
    global count
    speed = etc.knob1*20
    count = int(count + speed)
    thick = int(etc.knob3*100)+1
    color = etc.color_picker()

    peak = 0
    for i in range(0,100) :
        if etc.audio_in[i] > peak:
            peak = etc.audio_in[i]

    R = 4 * etc.knob2 * peak / 128
    x = R * math.cos((count /  1000.) * 6.28) + 640
    y = R * math.sin((count /  1000.) * 6.28) + 360
    pygame.draw.line(screen, color, [640, 360], [x, y], thick)


from django.conf.urls import url

# from apps.user import views
# from django.contrib.auth.decorators import login_required

from apps.user.views import (RegisterView, ActiveView, LoginView, UserInfoView, UserOrderView,
                             AddressView, LogoutView, TestView)

urlpatterns = [
    url(r"^register/$", RegisterView.as_view(), name='register'),    # 将注册改为类视图的使用模式
    # url(r"^register/$", views.register, name='register'),
    # url(r"^register_handle/$", views.register_handle, name='register_handle'),

    # url(r"^active/(?P<token>.*)$", ActiveView.as_view(), name='active'),    # 用户激活 拼接 url 的模式
    url(r"^active/$", ActiveView.as_view(), name='active'),    # 用户激活 将 token 作为参数进行传递

    url(r"^login/$", LoginView.as_view(), name='login'),    # 用户登录
    url(r"^logout/$", LogoutView.as_view(), name='logout'),    # 用户登出

    # login_required 要在视图函数上使用 as_view() 的返回值是函数 所以我们要在这里手动调用一次
    # url(r"^$", login_required(UserInfoView.as_view()), name='info'),    # info
    # url(r"^order/$", login_required(UserOrderView.as_view()), name='order'),    # order
    # url(r"^site/$", login_required(AddressView.as_view()), name='site'),    # site

    url(r"^$", UserInfoView.as_view(), name='info'),  # info
    url(r"^order/$", UserOrderView.as_view(), name='order'),  # order
    url(r"^site/$", AddressView.as_view(), name='site'),  # site

    url(r"^test/$", TestView.as_view(), name='test'),  # test

]


# importing libraries
import requests
import string
import util
from tagger.tagger_rule_based import speller

# consts
NEW_SENTENCE = 'sent_id'
REAL_WORD_LOC = 1
BASE_WORD_LOC = 2
DEP_LOC = 7

class TagParser:
    def __init__(self):
        # api-endpoint
        self.URL = "http://lindat.mff.cuni.cz/services/udpipe/api/process"
        self.speller = speller.Speller()

    def parse_text(self, data):
        # defining a params dict for the parameters to be sent to the API
        params = {'data': data, 'model': 'hebrew-ud-2.0-170801', 'tokenizer': True, 'tagger_rule_based': True, 'parser': True}

        # sending get request and saving the response as response object
        r = requests.get(url=self.URL, params=params)

        # extracting data in json format
        data = r.json()

        # parse data
        result = data['result']
        result_in_list = result.split('\n')
        ans = []
        for i in range(0, len(result_in_list)):
            if result_in_list[i].find(NEW_SENTENCE) != -1:
                i = i + 2  # to get into the parsed sentence
                sentence = {}
                while i < len(result_in_list):
                    words = result_in_list[i].split('\t')
                    if words[0] == '\n' or words[0] == '':
                        ans.append(sentence)
                        break
                    if words[REAL_WORD_LOC] not in string.punctuation:
                        word_to_clean = util.remove_punctuation_from_str(str(words[REAL_WORD_LOC]))
                        cleaned_word_to_add = self.speller.get_base_word(word_to_clean)

                        util.add_word_and_context_to_dict(sentence, str(words[DEP_LOC]), cleaned_word_to_add)
                    i = i + 1

        return ans


import os
from pathlib import Path
from typing import Dict, Tuple

import pytest
from cosmian_secure_computation_client import (ComputationOwnerAPI, CodeProviderAPI,
                                               DataProviderAPI, ResultConsumerAPI, CryptoContext,
                                               Side)

from keys import *

def pytest_addoption(parser):
    parser.addoption("--cosmian_api_token",
                     action="store",
                     default=os.getenv("COSMIAN_TOKEN", default=""))
    parser.addoption("--cosmian_email_account",
                     action="store",
                     default=os.getenv("COSMIAN_ACCOUNT",
                                       default="alice@cosmian.com"))
    parser.addoption("--url",
                     action="store",
                     default=os.getenv("COSMIAN_BASE_URL",
                                       default="https://backend.cosmian.com"))

@pytest.fixture(scope="session")
def cosmian_api_token(pytestconfig):
    return pytestconfig.getoption("cosmian_api_token")

@pytest.fixture(scope="session")
def cosmian_email_account(pytestconfig):
    return pytestconfig.getoption("cosmian_email_account")

@pytest.fixture(scope="module")
def cp_root_path():
    return Path(__file__).parent / "data" / "cp"

@pytest.fixture(scope="module")
def code_path():
    return Path(__file__).parent / "data" / "cp" / "enclave-join"

@pytest.fixture(scope="module")
def dp1_root_path():
    return Path(__file__).parent / "data" / "dp1"

@pytest.fixture(scope="module")
def dp2_root_path():
    return Path(__file__).parent / "data" / "dp2"

@pytest.fixture(scope="module")
def rc_root_path():
    return Path(__file__).parent / "data" / "rc"

@pytest.fixture(scope="module")
def words():
    return ("cargo", "error", "thank")

@pytest.fixture(scope="module")
def co(cosmian_api_token):
    yield ComputationOwnerAPI(token=cosmian_api_token)

@pytest.fixture(scope="module")
def computation_uuid(co, cosmian_email_account):
    computation = co.create_computation(
        name="test-e2e-secure-computation",
        code_provider_email=f"{cosmian_email_account}",
        data_providers_emails=[f"{cosmian_email_account}"],
        result_consumers_emails=[f"{cosmian_email_account}"])

    yield computation.uuid

@pytest.fixture(scope="module")
def cp(cosmian_api_token, words, computation_uuid):
    cp_ctx = CryptoContext.from_dict({
        "computation_uuid": computation_uuid,
        "side": "CodeProvider",
        "words": words,
        "ed25519_seed": CP_ED25519_SEED,
        "symkey": CP_SYMKEY
    })
    cp = CodeProviderAPI(token=cosmian_api_token, ctx=cp_ctx)

    yield cp

@pytest.fixture(scope="module")
def dp1(cosmian_api_token, words, computation_uuid):
    dp1_ctx = CryptoContext.from_dict({
        "computation_uuid": computation_uuid,
        "side": "DataProvider",
        "words": words,
        "ed25519_seed": DP1_ED25519_SEED,
        "symkey": DP1_SYMKEY
    })
    dp1 = DataProviderAPI(token=cosmian_api_token, ctx=dp1_ctx)

    yield dp1

@pytest.fixture(scope="module")
def dp2(cosmian_api_token, words, computation_uuid):
    dp2_ctx = CryptoContext.from_dict({
        "computation_uuid": computation_uuid,
        "side": "DataProvider",
        "words": words,
        "ed25519_seed": DP2_ED25519_SEED,
        "symkey": DP2_SYMKEY
    })
    dp2 = DataProviderAPI(token=cosmian_api_token, ctx=dp2_ctx)

    yield dp2

@pytest.fixture(scope="module")
def rc(cosmian_api_token, words, computation_uuid):
    rc_ctx = CryptoContext.from_dict({
        "computation_uuid": computation_uuid,
        "side": "ResultConsumer",
        "words": words,
        "ed25519_seed": RC_ED25519_SEED,
        "symkey": RC_SYMKEY
    })
    rc = ResultConsumerAPI(token=cosmian_api_token, ctx=rc_ctx)

    yield rc

_test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {}

def pytest_runtest_makereport(item, call):
    if "incremental" in item.keywords:
        if call.excinfo is not None:
            cls_name = str(item.cls)
            parametrize_index = (
                tuple(item.callspec.indices.values())
                if hasattr(item, "callspec")
                else ()
            )
            test_name = item.originalname or item.name
            _test_failed_incremental.setdefault(cls_name, {}).setdefault(
                parametrize_index, test_name
            )

def pytest_runtest_setup(item):
    if "incremental" in item.keywords:
        cls_name = str(item.cls)
        if cls_name in _test_failed_incremental:
            parametrize_index = (
                tuple(item.callspec.indices.values())
                if hasattr(item, "callspec")
                else ()
            )
            test_name = _test_failed_incremental[cls_name].get(
                parametrize_index, None
            )
            if test_name is not None:
                pytest.xfail("previous test failed ({})".format(test_name))


from django.shortcuts import render
from random import randint

from .forms import FeedbackForm
from .models import Dapp, Feedback

st = {
    "prototype": "mvp",
    "mvp": "alpha",
    "alpha": "beta",
    "beta": "live",
    "live": None
}
def home(request):
    if request.method == "POST":
        add = int(request.POST.get("eth"))
        name = request.POST.get("name")
        print(name, add)
        x = Dapp.objects.all().filter(name__exact=name)[0]
        x.current_fund += add
        #x.current_fund = 0
        if x.current_fund >= x.fund_next_stage:
            x.status = st[x.status.lower()]
            x.fund_next_stage += randint(10, 30)
        x.save()

    dapps = Dapp.objects.all()
    prototypes = dapps.filter(status__iexact="prototype")
    mvps = dapps.filter(status__iexact="mvp")
    alphas = dapps.filter(status__iexact="alpha")
    betas = dapps.filter(status__iexact="beta")
    lives = dapps.filter(status__iexact="live")
    context = {
        "prototypes": prototypes,
        "mvps": mvps,
        "alphas": alphas,
        "betas": betas,
        "lives": lives,
        "title": "Welcome to The dAppMap!",
    }
    return render(request, "home.html", context)

def feedback(request, name):
    dapp = Dapp.objects.all().filter(name__iexact=name)[0]
    form = FeedbackForm()
    if request.method == "POST":
        form = FeedbackForm(request.POST)
        if form.is_valid():
            name = form.cleaned_data["name"]
            text = form.cleaned_data["text"]
            rating = form.cleaned_data["rating"]
            f = Feedback(dapp=dapp, text=text, rating=rating)
            f.save()

    feedbacks = Feedback.objects.all().filter(dapp__exact=dapp)

    context = {
        "form": form,
        "name": name,
        "feedbacks": feedbacks,
    }
    return render(request, "feedback.html", context)


from constructs import Construct
from aws_cdk import Stack
import aws_cdk.aws_route53 as route53
from requests import get

class Route53CdkDyndnsStack(Stack):
    def __init__(
        self,
        scope: Construct,
        construct_id: str,
        hosted_zone_id: str,
        zone_name: str,
        record_name: str,
        **kwargs
    ) -> None:
        super().__init__(scope, construct_id, **kwargs)

        ip = get("https://api.ipify.org").text

        zone = route53.HostedZone.from_hosted_zone_attributes(
            self,
            "MyZone",
            zone_name=zone_name,
            hosted_zone_id=hosted_zone_id,
        )

        route53.ARecord(
            self,
            "ExternalIP",
            record_name=record_name,
            zone=zone,
            target=route53.RecordTarget.from_ip_addresses(ip),
        )


# -*- coding: utf-8 -*-
"""
Created on Thu Oct  4 10:16:53 2018

Catbot 2.0 uses discord.py version 1.x

@author: Mei Eisenbach
"""

import sys
import os
import re
import datetime as dt
import dotenv
import pickle
from discord import utils
from discord import Embed
from discord.ext import commands
from dateutil.parser import parse
from gyms import Gyms
from pokemon import Pokemon
from catbot_log import CatbotLog
from discord.ext.commands import CommandNotFound
import traceback

BOT_PREFIX = '!'
GYMFILE = 'gyms.csv'
POKEMONFILE = 'pokedex.csv'
PARAMSFILE = 'params.pickle'
HOST_LOGFILE = 'host.log'
ERROR_LOGFILE = 'error.log'
EGG_URL_BASE = 'https://ironcreek.net/catbot/eggs/'
EGG1 = 'egg1.png'
EGG3 = 'egg3.png'
EGG_LEGENDARY = 'legendary_egg.png'
EGG_MEGA = 'mega_egg.png'

gyms = None
pokemon = None
params = {}
params['legendaries'] = []

# =============================================================================
# String constants
# =============================================================================
MSG_HELP = "!whereis [gym] will return a location pin for the gym.\n\
Examples:\n\
    !whereis Irvington Community Park\n\
    !whereis icp\n\
    !whereis irvington\n\
will all return\n\
Irvington Community Park (ICP) is here http://maps.google.com/maps?q=37.522771,-121.963727"

REPORT_CHANNEL_NAME = 'raid_alerts_only'
LEGENDARY_ROLE_NAME = 'LegendaryRaid'

ERR_GYM_NOT_FOUND = '"{}" not found. Please check your spelling or use fewer words.'
ERR_TOO_MANY_RESULTS = 'Too many matches. Please be more specific.'
ERR_REPORT_MULTIPLE_MATCHES = 'More than one gym matches "{}"'

ERR_LEGENDARY_ROLE_MISSING = 'Legendary Raid role not found'

ERR_RAID_NOT_FOUND = 'Raid not found'
ERR_INVALID_DATETIME = "Invalid date or time"

# =============================================================================
# Helper and test functions
# =============================================================================

def generate_post(is_egg, identifier, mins, gym, reporter):
    time = dt.datetime.now() + dt.timedelta(minutes=int(mins))
    link = gyms.get_link(gym)
    gym_name = gyms.get_name(gym)
    if gyms.is_ex(gym):
        gym_name = gym_name + ' 🎫'
    if is_egg:
        content = "Level {} 🥚 at {}\n{}\nhatches at {} (in {} mins)\nreported by {}"\
            .format(identifier,
                    gym_name,
                    link,
                    time.strftime("%I:%M %p"),
                    mins,
                    reporter)
    else:
        content = "{} at {}\n{}\nuntil {} ({} mins remaining)\nreported by {}"\
            .format(identifier.title(),
                    gym_name,
                    link,
                    time.strftime("%I:%M %p"),
                    mins,
                    reporter)
    return content

def parse_args(arg_str):
    p = re.compile(r'^(.*) (at|in) (\S*) ?(\S*)? ?(\S*)?')
    m = p.match(arg_str)
    if m:
        return m.groups()
    return []

def format_do(gym, start, despawn, boss):
    link = gyms.get_link(gym)
    gym_name = gyms.get_name(gym)
    content = '-new "{}" {} {} {} {}'\
        .format(boss.title(),
                link,
                start.strftime("%I:%M%p"),
                despawn.strftime("%I:%M%p"),
                gym_name)
    return content

def process_do(arg_str):
    arguments = parse_args(arg_str)

    if len(arguments) == 0:
        return 'Usage: !do gym_name at/in start_time/min [despawn_time] [boss_name]'

    gym_name = arguments[0]

    # minutes given, calculate start time
    if arguments[1] == 'in':
        start = dt.datetime.now() + dt.timedelta(minutes=int(arguments[2]))
    else:
        try:
            start = parse(arguments[2])
        except:
            return 'Invalid start time. Valid formats: 00:00 or 0:00pm'

    # assign rest of arguments
    if arguments[3] == '':
        despawn = start + dt.timedelta(minutes=1)
        boss = "egg"
    elif arguments[4] == '':
        despawn = start + dt.timedelta(minutes=1)
        boss = arguments[3]
    else:
        try:
            despawn = parse(arguments[3])
        except:
            return 'Invalid despawn time. Valid formats: 00:00 or 0:00pm'
        boss = arguments[4]

    found = gyms.find(gym_name)
    if len(found) == 0:
        content = ERR_GYM_NOT_FOUND.format(gym_name)
    elif len(found) == 1:
        content = format_do(found[0], start, despawn, boss)
    else:
        content = ERR_REPORT_MULTIPLE_MATCHES.format(gym_name)

    return content

def parse_host_args(args: list):
    args = ' '.join(args)
    args = re.sub(r'[\[\](){}]', '', args)

    p = r'(.+) (hatch|end|start)\D*(\d+) ?(?:min)?(?:s|utes?)? ?(.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'boss': m.groups()[0], 'verb': m.groups()[1],
                'mins': m.groups()[2], 'notes': m.groups()[3]}
    return {}

def parse_host_now(args: list):
    args = ' '.join(args)

    p = r'(.+) (start|hatch)\w* ?(now) ?(.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'boss': m.groups()[0], 'verb': m.groups()[1],
                'mins': '0', 'notes': m.groups()[3]}
    return {}

def parse_host_mins_left(args: list):
    args = ' '.join(args)

    p = r'(.+) (\d+) ?(?:min)?(?:s|utes?)? ?(left) ?(.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'boss': m.groups()[0], 'mins': m.groups()[1],
                'verb': 'end', 'notes': m.groups()[3]}

    return {}

def censor_notes(notes):
    friendcode_pat = re.compile(r'\d{4}[-\s]*\d{4}[-\s]*\d{4}')
    notes = friendcode_pat.sub('<Friend code removed>', notes)

    dm_me_pat = re.compile(r'[dp]\W*m(\s+|$)(me)?', re.IGNORECASE)
    notes = dm_me_pat.sub('...', notes)

    dm_me_pat2 = re.compile(r'(direct\s*)?me?ss?a?ge?(\s*|$)(me)?',
                            re.IGNORECASE)
    notes = dm_me_pat2.sub('...', notes)

    dm_me_pat3 = re.compile(r'te?xt me', re.IGNORECASE)
    notes = dm_me_pat3.sub('...', notes)

    return notes

def get_egg_url(level):
    if level <= 2:
        thumbnail = 'egg1.png'
    elif level <= 4:
        thumbnail = 'egg3.png'
    elif level == 5:
        thumbnail = 'legendary_egg.png'
    else:
        thumbnail = 'mega_egg.png'
    return EGG_URL_BASE+thumbnail

def save_params(paramsfile, params):
    with open(paramsfile, 'wb') as f:
        pickle.dump(params, f)

def load_params(paramsfile):
    with open(paramsfile, 'rb') as f:
        params = pickle.load(f)
    return params

# =============================================================================
# Test functions
# =============================================================================
def test_whereis(name):
    found = gyms.find(name)
    num_found = len(found)

    if num_found == 0:
        content = ERR_GYM_NOT_FOUND.format(name)
    elif num_found <= 3:
        locations = []
        for gym in found:
            link = gyms.get_link(gym)
            locations.append(gyms.get_name(gym) + " is here " + link)
        content = '\n'.join(locations)
    print(content)

def test_do():
    # absolute start time
    assert process_do('Rings Arch at 8:30pm 9:15pm Kyurem') ==\
        '-new "Kyurem" http://maps.google.com/maps?q=37.584377,-122.068447 08:30PM 09:15PM Rings Arch'
    assert process_do('Rings Arch at 20:30 Kyurem') ==\
        '-new "Kyurem" http://maps.google.com/maps?q=37.584377,-122.068447 08:30PM 08:31PM Rings Arch'
    assert process_do('Rings Arch at 8:30pm') ==\
        '-new "Egg" http://maps.google.com/maps?q=37.584377,-122.068447 08:30PM 08:31PM Rings Arch'

    # errors
    assert process_do('Rings Arch 8:30pm 9:15pm Kyurem') ==\
        'Usage: !do gym_name at/in start_time/min [despawn_time] [boss_name]'
    assert process_do('Ring Arch at 8:30pm 9:15pm Kyurem') ==\
        '"Ring Arch" not found. Please check your spelling or use fewer words.'
    assert process_do('Rings Arch at 20:30pm 9:15pm Kyurem') ==\
        'Invalid start time. Valid formats: 00:00 or 0:00pm'
    assert process_do('Rings Arch at 8:30pm 21:15pm Kyurem') ==\
        'Invalid despawn time. Valid formats: 00:00 or 0:00pm'
    print('All asserts passed.')

    # relative start time
    print('Check output from relative times...')
    print(process_do('Rings Arch in 30 9:15pm Kyurem'))
    print(process_do('Rings Arch in 30 Kyurem'))
    print(process_do('Rings Arch in 30'))

# =============================================================================
# Bot code
# =============================================================================
bot = commands.Bot(command_prefix=BOT_PREFIX)

@bot.event
async def on_ready():
    print('We have logged in as {0.user}'.format(bot))
    print('Bot Prefix: ', BOT_PREFIX)

@bot.event
async def on_message(message):
    content = ''
    if bot.user.mentioned_in(message) and message.mention_everyone is False:
        message_lowered = message.content.lower()
        if 'hello' in message_lowered or 'hi' in message_lowered:
            content = 'Hello, {}'.format(message.author.mention)
        elif 'thanks' in message_lowered:
            content = "{} You're welcome".format(message.author.mention)
        elif 'coming' in message_lowered or 'going' in message_lowered:
            content = "{} Sorry, I'm a bot and stuck in this server room."\
                .format(message.author.mention)
        await message.channel.send(content)

    await bot.process_commands(message)

@bot.event
async def on_command_error(ctx, exception):
    if isinstance(exception, CommandNotFound):
        log = open(ERROR_LOGFILE, 'a')
        log.write('{}, {}\n'.format(exception, ctx.guild))
        log.close()
        return

    print('Ignoring exception in command {}:'.format(ctx.command),
          file=sys.stderr)
    traceback.print_exception(type(exception), exception,
                              exception.__traceback__, file=sys.stderr)

@bot.command()
async def whereis(ctx, *args):
    name = ' '.join(args)

    found = gyms.find(name)
    num_found = len(found)

    if num_found == 0:
        content = ERR_GYM_NOT_FOUND.format(name)
    elif num_found <= 3:
        locations = []
        for gym in found:
            link = gyms.get_link(gym)
            locations.append(gyms.get_name(gym) + " is here " + link)
        content = '\n'.join(locations)
    else:
        content = ERR_TOO_MANY_RESULTS

    await ctx.send(content)

def parse_egg(args: list):
    args = ' '.join(args)
    args = re.sub(r'[\[\](){}]', '', args)

    p = r't?([1-6]) hatches in (\d+) at (.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'egg': m.groups()[0], 'mins': m.groups()[1],
                'gym': m.groups()[2]}
    return {}

def parse_egg_old(args: list):
    args = ' '.join(args)
    p = r'([1-6]) (\d+) (.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'egg': m.groups()[0], 'mins': m.groups()[1],
                'gym': m.groups()[2]}
    return {}

@bot.command()
async def egg(ctx, *args):
    if ctx.guild is None:
        await ctx.send('This command can only be used on a server.')
        return

    report_channel = utils.get(ctx.guild.channels, name=REPORT_CHANNEL_NAME)
    if report_channel is None:
        await ctx.send(REPORT_CHANNEL_NAME + ' channel not found')
        return

    parsed = parse_egg(args)
    if not parsed:
        parsed = parse_egg_old(args)

    if not parsed:
        CatbotLog.write('Parse Error', 'egg_new ' + ' '.join(args), ctx.guild)
        content = 'Usage: !egg [1-6] hatches in [mins] at [gym] \nExample: !egg 5 hatches in 10 at ICP'
        await ctx.send(content)
        return

    thumbnail = get_thumbnail(parsed['egg'])

    found = gyms.find(parsed['gym'])
    if len(found) == 0:
        CatbotLog.write('Gym Error', 'egg_new ' + ' '.join(args), ctx.guild)
        await ctx.send(ERR_GYM_NOT_FOUND.format(parsed['gym']))
        return
    elif len(found) == 1:
        time = dt.datetime.now() + dt.timedelta(minutes=int(parsed['mins']))
        when = 'hatches at {} (in {} mins)'.format(time.strftime("%I:%M %p"),
                                                   parsed['mins'])
        if parsed['egg'] == '6':
            title = 'Tier 6 - Mega Raid'
        else:
            title = 'Tier '+parsed['egg']
        embed = create_egg_raid_embed(title, ctx.author.mention,
                                      when, found[0], thumbnail)

        msg = await report_channel.send(embed=embed)
        await msg.add_reaction('👍')
        await ctx.send('Egg reported to ' + report_channel.mention)
    else:
        await ctx.send(ERR_REPORT_MULTIPLE_MATCHES.format(parsed['gym']))

@bot.command()
async def egg_old(ctx, egg_level, until_hatch, *args):
    report_channel = utils.get(ctx.guild.channels, name=REPORT_CHANNEL_NAME)
    if report_channel is None:
        print(REPORT_CHANNEL_NAME + ' channel not found')
        return

    legendary_role = utils.get(ctx.guild.roles, name=LEGENDARY_ROLE_NAME)
    if legendary_role is None:
        await ctx.send(ERR_LEGENDARY_ROLE_MISSING)
        return

    mega_role = utils.get(ctx.guild.roles, name='MegaRaid')

    if not egg_level.isnumeric():
        await ctx.send('"{}" is not a number. Egg levels should be 1-6'.format(egg_level))
        return
    if int(egg_level) < 1 or int(egg_level) > 6:
        await ctx.send('"{}" is not valid. Egg levels should be 1-6'.format(egg_level))
        return

    if not until_hatch.isnumeric():
        await ctx.send('"{}" is not a number. Minutes until hatch should be a number'.format(until_hatch))
        return

    gym = ' '.join(args)
    found = gyms.find(gym)
    if len(found) == 0:
        await ctx.send(ERR_GYM_NOT_FOUND.format(gym))
    elif len(found) == 1:
        reporter = ctx.message.author.mention
        content = generate_post(
            True, egg_level, until_hatch, found[0], reporter)

        if egg_level == '5':
            content = '{} {}'.format(legendary_role.mention, content)

        if egg_level == '6' and mega_role is not None:
            content = '{} {}'.format(mega_role.mention, content)

        await report_channel.send(content)
        await ctx.send('Egg reported to ' + report_channel.mention)
    else:
        await ctx.send(ERR_REPORT_MULTIPLE_MATCHES.format(gym))

def parse_raid(args: list):
    args = ' '.join(args)
    args = re.sub(r'[\[\](){}]', '', args)

    p = r'([\w\s]+) (ends|despawns) in (\d+) at (.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'boss': m.groups()[0], 'mins': m.groups()[2],
                'gym': m.groups()[3]}
    return {}

def parse_raid_old(args: list):
    args = ' '.join(args)
    p = r'(.*) (\d+) (.*)'
    m = re.match(p, args, re.IGNORECASE)
    if m:
        return {'boss': m.groups()[0], 'mins': m.groups()[1],
                'gym': m.groups()[2]}
    return {}

def create_egg_raid_embed(title, reporter, when, gym, thumbnail=''):
    embed = Embed(title=title.title(),
                  description='React with thumbs up if interested')
    embed.add_field(name="Where", value=gyms.get_name(gym))
    embed.add_field(name="When",
                    value=when)
    embed.add_field(name="Reported by", value=reporter)
    embed.add_field(name="Location", value=gyms.get_link(gym))

    if thumbnail != '':
        embed.set_thumbnail(url=thumbnail)

    return embed

@bot.command()
async def raid(ctx, *args):
    if ctx.guild is None:
        await ctx.send('This command can only be used on a server.')
        return

    report_channel = utils.get(ctx.guild.channels, name=REPORT_CHANNEL_NAME)
    if report_channel is None:
        await ctx.send(REPORT_CHANNEL_NAME + ' channel not found')
        return

    parsed = parse_raid(args)
    if not parsed:
        parsed = parse_raid_old(args)

    if not parsed:
        CatbotLog.write('Parse Error', 'raid_new ' + ' '.join(args), ctx.guild)
        content = 'Usage: !raid [boss] ends|despawns in [mins] at [gym] \nExample: !raid Regirock ends in 30 at ICP'
        await ctx.send(content)
        return

    found = pokemon.find(parsed["boss"])
    if found != '' and found != parsed["boss"].lower():
        parsed["boss"] = found

    thumbnail = get_thumbnail(parsed['boss'])

    found = gyms.find(parsed['gym'])
    if len(found) == 0:
        CatbotLog.write('Gym Error', 'raid_new ' + ' '.join(args), ctx.guild)
        await ctx.send(ERR_GYM_NOT_FOUND.format(parsed['gym']))
        return
    elif len(found) == 1:
        time = dt.datetime.now() + dt.timedelta(minutes=int(parsed['mins']))
        when = 'despawns at {} (in {} mins)'.format(time.strftime("%I:%M %p"),
                                           parsed['mins'])
        embed = create_egg_raid_embed(parsed['boss'], ctx.author.mention,
                                      when, found[0], thumbnail)

        msg = await report_channel.send(embed=embed)
        await msg.add_reaction('👍')
        await ctx.send('Raid reported to ' + report_channel.mention)
    else:
        await ctx.send(ERR_REPORT_MULTIPLE_MATCHES.format(parsed['gym']))

@bot.command()
async def raid_old(ctx, boss, time_left, *args):
    mention = ['shinx']
    report_channel = utils.get(ctx.guild.channels, name=REPORT_CHANNEL_NAME)
    if report_channel is None:
        await ctx.send(REPORT_CHANNEL_NAME + ' channel not found')
        return

    legendary_role = utils.get(ctx.guild.roles, name=LEGENDARY_ROLE_NAME)
    if legendary_role is None:
        await ctx.send(ERR_LEGENDARY_ROLE_MISSING)
        return

    if boss.isnumeric():
        await ctx.send('"{}" is not a raid boss.'.format(boss))
        return

    if not time_left.isnumeric():
        await ctx.send('"{}" is not a number. \
                       Minutes remaining should be a number'.format(time_left))
        return

    if boss.lower() in mention:
        boss_role = utils.get(ctx.guild.roles, name=boss)
        if boss_role is not None:
            boss = boss_role

    gym = ' '.join(args)
    found = gyms.find(gym)
    if len(found) == 0:
        await ctx.send(ERR_GYM_NOT_FOUND.format(gym))
    elif len(found) == 1:
        reporter = ctx.message.author.mention
        content = generate_post(False, boss, time_left, found[0], reporter)

        if is_legendary(boss):
            content = '{} {}'.format(legendary_role.mention, content)

        await report_channel.send(content)
        await ctx.send('Raid reported to ' + report_channel.mention)
    else:
        await ctx.send(ERR_REPORT_MULTIPLE_MATCHES.format(gym))

@bot.command()
@commands.has_any_role('Developer', 'Admin', 'mod')
async def set_legendaries(ctx, *args):
    if len(args) == 0:
        await ctx.send('Usage: !set_legendaries leg1 leg2 ...')
        return

    params['legendaries'] = list(map(str.lower, args))
    await ctx.message.add_reaction('👍')
    save_params(PARAMSFILE, params)

@bot.command()
@commands.has_any_role('Developer', 'Admin', 'mod')
async def get_legendaries(ctx):
    legendaries = params.get('legendaries', [])
    msg = ', '.join([el.title() for el in legendaries])
    if msg == '':
        msg = 'No legendaries set.'
    await ctx.send(msg)

@bot.command()
@commands.has_any_role('Developer')
async def reload_gyms(ctx):
    gyms.read_csv(GYMFILE)
    await ctx.message.add_reaction('👍')

@bot.command()
@commands.has_any_role('Developer')
async def reload_pokemon(ctx):
    pokemon.read_csv(POKEMONFILE)
    await ctx.message.add_reaction('👍')

@bot.command(pass_context=True)
async def get_invite(ctx):
    if ctx.message.guild is None:
        msg = 'Please use this command from a server channel.'
        await ctx.message.author.send(msg)
        return

    if ctx.message.guild.name == "(Official) Pokémon GO: Fremont":
        link = await ctx.channel.create_invite(
            max_age=3600*24, max_use=1, unique=True)
        if link is not None:
            await ctx.message.author.send(
                'Here is your invite {}'.format(link))
            await ctx.send('Invite sent via dm')
        else:
            await ctx.send('Unable to create invite')
    else:
        await ctx.send('Sorry, this command is not allowed on this server.')

# Eventually add: 'Pin', 'UC Agent', 'FMT Agent', 'SF Agent', 'HWD Agent'
@commands.has_any_role('Developer', 'TR Scientist')
@bot.command()
async def do(ctx, *args):
    message = process_do(' '.join(args))
    await ctx.send(message)

@commands.has_any_role('TR Scientist', 'Admin', 'mod')
@bot.command()
async def purge_fc(ctx, limit=None):
    def check_msg(msg):
        if msg.pinned:
            return False
        if msg.id == ctx.message.id:
            return True
        if re.search(r'(^|\D)\d{4}[-\s]*\d{4}[-\s]*\d{4}($|\D)', msg.content)\
                is None:
            return False
        return True

    # limit is the number of messages to search through
    if limit is not None:
        limit = int(limit)
    deleted = await ctx.channel.purge(limit=limit, check=check_msg)

    # debug
    for msg in deleted:
        print(msg.content)

    await ctx.send('Deleted {} message(s)'.format(len(deleted)-1),
                   delete_after=5)

def get_reporting_channels(ctx):
    channel_names = {'mega': 'active-mega-raids',
                     'legendary': 'active-t5-raids',
                     'other': 'active-t1-t3-raids'}

    reporting_channels = {}

    if ctx.guild is not None:
        for key, name in channel_names.items():
            reporting_channels[key] = utils.get(ctx.guild.channels, name=name)

    return reporting_channels

def is_mega(boss):
    if re.search(r'mega(\s|$)', boss, re.IGNORECASE) is None:
        return False
    return True

def is_legendary(boss):
    if 'legendaries' in params and boss.lower() in params['legendaries']:
        return True
    return False

def get_thumbnail(boss):
    thumbnail = ''

    m = re.match('t?([123456])', boss, re.IGNORECASE)
    if m is not None:
        thumbnail = get_egg_url(int(m.groups()[0]))
        return thumbnail

    thumbnail = pokemon.get_boss_url(boss)
    if thumbnail != '':
        return thumbnail

    if is_mega(boss):
        thumbnail = EGG_URL_BASE + EGG_MEGA

    return thumbnail

def create_embed(title, host, when, notes='', thumbnail=''):
    embed = Embed(title=title.title(),
                  description='React with team emoji for invite')
    embed.add_field(name="Host", value=host)
    embed.add_field(name="When", value=when)

    if notes != '':
        embed.add_field(name="Notes", value=notes)

    if thumbnail != '':
        embed.set_thumbnail(url=thumbnail)

    return embed

def get_raid_tier(boss):
    if is_mega(boss):
        return 'mega'

    if is_legendary(boss) or\
            re.match('t5', boss, re.IGNORECASE) is not None:
        return 'legendary'

    return 'other'

@bot.command(aliases=["Host"])
async def host(ctx, *args):
    if ctx.guild is None:
        CatbotLog.write('Error', 'Host command sent via DM')
        await ctx.send('This command can only be used on a server.')
        return

    if ctx.channel.name != '💥-hosting-commands':
        await ctx.send('This command can only be used from #💥-hosting-commands.')
        return

    reporting_channels = get_reporting_channels(ctx)

    for key, channel in reporting_channels.items():
        if channel is None:
            await ctx.send('Reporting channels not found')
            return

    parsed = parse_host_now(args)
    if not parsed:
        parsed = parse_host_args(args)
    if not parsed:
        parsed = parse_host_mins_left(args)

    if not parsed:
        content = \
        'Usage: !host [T1-5 or boss] [hatches|starts|ends] in mins (optional notes)\nExample: !host heatran ends in 30 mins'
        await ctx.send(content)

        CatbotLog.write('Parse Error', 'host ' + ' '.join(args), ctx.guild)
        return

    when = '{}ing in {} mins'.format(parsed["verb"], parsed["mins"])

    if parsed["notes"] != '':
        parsed["notes"] = censor_notes(parsed["notes"])

    found = pokemon.find(parsed["boss"])
    if found != '' and found != parsed["boss"].lower():
        CatbotLog.write('Correction', '{} to {}'.format(parsed["boss"], found),
                        ctx.guild)
        parsed["boss"] = found

    thumbnail = get_thumbnail(parsed['boss'])

    embed = create_embed(parsed['boss'], ctx.author, when, parsed['notes'],
                         thumbnail)

    tier = get_raid_tier(parsed['boss'])

    msg = await reporting_channels[tier].send(embed=embed,
                                              delete_after=2*60*60)

    CatbotLog.write('Success', 'host ' + ' '.join(args), ctx.guild)

    for team_logo in ['instinctlogo', 'mysticlogo', 'valorlogo']:
        emoji = utils.get(ctx.guild.emojis, name=team_logo)
        if emoji is not None:
            await msg.add_reaction(emoji)

    await ctx.send('Raid reported to ' + reporting_channels[tier].mention,
                   delete_after=5)

    await ctx.message.delete()

def is_n_minutes_old(timestamp, minutes):
    return (dt.datetime.utcnow() - timestamp) > dt.timedelta(minutes=minutes)

@commands.has_any_role('TR Scientist', 'Admin', 'mod', 'Mod')
@bot.command(aliases=["purge_old"])
async def purge_old_messages(ctx, age_in_minutes=2*60):
    deleted = await ctx.channel.purge(limit=100,
                                      check=lambda m: is_n_minutes_old(m.created_at,
                                                                       age_in_minutes)
                                      and not m.pinned)
    await ctx.send('Deleted {} message(s)'.format(len(deleted)),
                   delete_after=10)
    await ctx.message.delete()

@commands.has_any_role('TR Scientist', 'Developer')
@bot.command()
async def shutdown(ctx):
    await bot.logout()

# =============================================================================
# Main
# =============================================================================

try:
    gyms = Gyms(GYMFILE)
    print('Gyms loaded')
except IOError:
    print('ERROR: Unable to load gyms from', GYMFILE)
    sys.exit()

try:
    pokemon = Pokemon(POKEMONFILE)
    print('Pokemon loaded')
except IOError:
    print('ERROR: Unable to load pokemon from', GYMFILE)
    sys.exit()

try:
    params = load_params(PARAMSFILE)
    print('Parameters loaded')
    if 'legendaries' not in params:
        print('WARNING: Legendaries not set')
except IOError:
    pass

CatbotLog.init(HOST_LOGFILE)

if __name__ == "__main__":
    dotenv.load_dotenv()
    key = os.getenv('DISCORD_TOKEN')
    if key is None:
        print('ERROR: Discord Key not found in the environment variables')
        sys.exit()
    bot.run(key)


import tushare as ts
from Entity.Price import Price
from LongtermStrategy import  LongtermStrategy
from ShortTermStrategy import ShortTermStrategy
import logging
import datetime

def main():
    logging.basicConfig(level=logging.DEBUG)

    start = '2016-01-01'
    now = datetime.datetime(2018, 4, 1)
    totalBenefit = 0
    for i in range(1):
        now = now + datetime.timedelta(days=1)
        result = simulate(start, now)
        benefit = sum(result)
        totalBenefit += benefit
        if benefit < 0:
            logging.error(result)
    logging.error('average benfit: %s', totalBenefit/365)

def simulate(start, now):
    p = generateHistoricalData(start, now.strftime('%Y-%m-%d'))
    # commandCenter = LongtermStrategy(p)
    commandCenter = ShortTermStrategy(p)

    while now < datetime.datetime(2019, 4, 1):
        next = now + datetime.timedelta(days=1)
        data = ts.get_hist_data('cyb', start=now.strftime('%Y-%m-%d'), end=next.strftime('%Y-%m-%d'), ktype="5")[::-1]
        for row in data.iterrows():
            open = float(row[1]['open'])
            close = float(row[1]['close'])
            high = float(row[1]['high'])
            low = float(row[1]['low'])
            date = row[0].split(' ')[0]

            currentPrice = Price(high, low, open, close, date)

            decision = commandCenter.do(currentPrice)
            if decision.Type == 'Quit with benefit' or decision.Type == 'First Position' \
                    or decision.Type == "Stop with loss" or decision.Type == 'Follow Buying':
                logging.info(decision)
                logging.info(row)
        now = next
    return commandCenter.account

# Exclusive end Date
def generateHistoricalData(startDate, endDate):
    data = ts.get_hist_data('cyb', start=startDate, end=endDate, ktype="D")
    dailyPrices = []

    for row in data.iterrows():
        open = float(row[1]['open'])
        close = float(row[1]['close'])
        high = float(row[1]['high'])
        low = float(row[1]['low'])
        date = row[0]
        dailyPrices.insert(0, Price(high, low, open, close, date))

    return dailyPrices

if __name__ == '__main__':
    main()


import logging
# for splitting text with regex
import re
import time
from bs4 import BeautifulSoup as bs
# use selenium to have js enabled while webcrawling
from selenium import webdriver
# used for checking dropdown menus so we have a standard view
from selenium.webdriver.support.select import Select

# options we need to have selenium run firefox in the background
options = webdriver.firefox.options.Options()
options.headless = True
options.add_argument('start-maximized')
options.set_preference('browser.download.folderList',2)
options.set_preference('browser.download.manager.showWhenStarting',False)
options.set_preference('browser.helperApps.neverAsk.saveToDisk','text/*')
driver = webdriver.Firefox(executable_path='./geckodriver',options=options)

class Parser:
    _basics = {'Plains','Island','Swamp','Mountain','Forest'}

    def __init__(self,url=''):
        self.url = url
        self.decklist = set()

    def _parse_decklist(self):
        driver.get(self.url)
        soup = bs(driver.page_source,'html5lib')
        return soup

    def _add_card(self,card):
        # ignore basic lands
        if card not in self._basics and 'Snow-Covered' not in card:
            self.decklist.add(card)

    def run(self):
        try:
            return self._parse_decklist()
        except Exception:
            logging.exception(f'Failed to crawl: {self.url}')

class ArchidektParser(Parser):
    def _parse_decklist(self):
        driver.get(self.url)
        # we need to wait for archidekt to load metadata to easily get the cmdr
        time.sleep(7.5)
        soup = bs(driver.page_source,'html5lib')
        cmdr = soup.find('meta',{'data-react-helmet':True})['content']
        if 'Commander' not in cmdr:
            return
        # TODO: there's an edgecase of partners but we're ignoring b/c lazy
        cmdr = cmdr[cmdr.index('Commander:'):cmdr.index('.')]
        cmdr = re.sub('Commander:','',cmdr).strip()
        super()._add_card(cmdr)
        # load goldfishing probability as an easy way to scrape text card names
        xpath = "//i[@class='pie chart icon']"
        element = driver.find_element_by_xpath(xpath)
        element.click()
        xpath = "//a[@class='item'][contains(text(),'Probability')]"
        element = driver.find_element_by_xpath(xpath)
        element.click()
        soup = bs(driver.page_source,'html.parser')
        for table in soup.find_all('tbody',class_=''):
            cards = re.split(r'\d+\%',table.text.strip())[:-1]
            for card in cards:
                super()._add_card(card)

class GoldfishParser(Parser):
    def _parse_decklist(self):
        soup = super()._parse_decklist()
        tab = soup.find(id='tab-paper')
        for table in tab.find_all(class_='deck-view-deck-table'):
            # remove non-breaking spaces
            text = table.text.replace(u'\xa0','')
            # ignore the sideboard section
            if 'Sideboard' in text:
                text = text[:text.index('Sideboard')]
            # regex filtering
            # remove prices
            text = re.sub(r'\$\d+\,*\d*\.\d+','',text)
            # remove counts (x) for sections
            text = re.sub(r'\(\d+\)','',text)
            # remove white space
            text = re.sub(r'\n',' ',text)
            # remove section titles
            text = text.replace('  Commander  ','')
            text = text.replace('  Creatures  ','')
            text = text.replace('  Planeswalkers  ','')
            text = text.replace('  Enchantments  ','')
            text = text.replace('  Artifacts  ','')
            text = text.replace('  Spells  ','')
            text = text.replace('  Lands  ','')
            # remove white space
            text = text.strip()
            while '  ' in text:
                text = text.replace('  ',' ')
            # split text into card names
            cards = re.split(r' *\d+ ',text)[1:]
            for card in cards:
                super()._add_card(card)

class MoxfieldParser(Parser):
    def _parse_decklist(self):
        soup = super()._parse_decklist()
        # use dropdowns to guarantee we always default to table view
        try:
            select = Select(driver.find_element_by_id('viewMode'))
        # in case moxfield doesn't properly load the page
        # we return None here, relying on main.py to retry this until it works
        except:
            return
        # option 0 = view mode as a table, if visual view is on instead
        if select.all_selected_options[0] != select.options[0]:
            # when opening moxfield links w/ visual view in headless,
            # 'deck-footer ' will obscure the dropdown that lets you change
            # use javascript to hide the div 'deck-footer '
            e = driver.find_element_by_xpath("//div[@class='deck-footer ']")
            driver.execute_script("arguments[0].style.visibility='hidden'",e)
            try:
                select.select_by_value('table')
            except:
                return
            # refresh the soup for the updated html
            html = driver.page_source
            soup = bs(html,'html.parser')
        for table in soup.find_all(class_='table-deck'):
            # split up card name from type
            text = table.text.strip().split(maxsplit=1)
            # use regex to filter card names
            cards = text[1].split(')',1)[1]
            if 'Lands' in text[0]:
                if 'mdfc' in text[1]:
                    cards = text[1].split('mdfc',1)[1]
            cards = re.split(r'[\d+]',cards)[1:]
            # check for companions in sideboard
            if 'Sideboard' in text[0]:
                for i,row in enumerate(table.tbody.find_all('tr')):
                    if row.select('#companion-icon'):
                        self.decklist.add(cards[i])
                        break
                continue
            for card in cards:
                super()._add_card(card)


import sys

from bbot.modules.base import BaseModule

"""
wrapper for https://github.com/defparam/smuggler.git
"""

class smuggler(BaseModule):
    watched_events = ["URL"]
    produced_events = ["FINDING"]
    flags = ["active", "aggressive", "slow", "web-thorough"]
    meta = {"description": "Check for HTTP smuggling"}

    in_scope_only = True
    per_host_only = True

    deps_ansible = [
        {
            "name": "Get smuggler repo",
            "git": {"repo": "https://github.com/defparam/smuggler.git", "dest": "#{BBOT_TOOLS}/smuggler"},
        }
    ]

    async def handle_event(self, event):
        command = [
            sys.executable,
            f"{self.scan.helpers.tools_dir}/smuggler/smuggler.py",
            "--no-color",
            "-q",
            "-u",
            event.data,
        ]
        async for line in self.helpers.run_live(command):
            for f in line.split("\r"):
                if "Issue Found" in f:
                    technique = f.split(":")[0].rstrip()
                    text = f.split(":")[1].split("-")[0].strip()
                    description = f"[HTTP SMUGGLER] [{text}] Technique: {technique}"
                    self.emit_event(
                        {"host": str(event.host), "url": event.data, "description": description},
                        "FINDING",
                        source=event,
                    )


#!/usr/bin/env python
# -*- coding: utf-8  -*-

import hebrew_numbers
import os
import re
from common import Span, SpanKind

MISHNAH_FOLDER = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'db', 'mishnah')
BARTENURA_FOLDER = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'db', 'bartenura')

TOC = [
	['Zeraim', 'זְרָעִים', [
		['Berakhot', 'בְּרָכוֹת'],
		['Peah', 'פֵּאָה'],
		['Demai', 'דְּמַאי'],
		['Kilayim', 'כִּלְאַיִם'],
		['Sheviit', 'שְֹבִיעִית'],
		['Terumot', 'תְּרוּמוֹת'],
		['Maaserot', 'מַעֲשְׂרוֹת'], #Maasrot
		['Maaser Sheni', 'מַעֲשֵׂר שֵׁנִי'],
		['Challah', 'חַלָּה'],
		['Orlah', 'עָרְלָה'],
		['Bikkurim', 'בִּכּוּרִים']
		]],
	['Moed', 'מוֹעֵד', [
		['Shabbat', 'שַׁבָּת'],
		['Eruvin', 'עֵרוּבִין'],
		['Pesachim', 'פְּסָחִים'],
		['Shekalim', 'שְׁקָלִים'],
		['Yoma', 'יוֹמָא'],
		['Sukkah', 'סֻכָּה'],
		['Beitzah', 'בֵּיצָה'], ###
		['Rosh Hashanah', 'רֹאשׁ הַשָּׁנָה'],
		['Taanit', 'תַּעֲנִית'],
		['Megillah', 'מְגִלָּה'],
		['Moed Katan', 'מוֹעֵד קָטָן'],
		['Chagigah', 'חֲגִיגָה']
		]],
	['Nashim', 'נָשִׁים', [
		['Yevamot', 'יְבָמוֹת'],
		['Ketubot', 'כְּתוּבּוֹת'],
		['Nedarim', 'נְדָרִים'],
		['Nazir', 'נָזִיר'],
		['Sotah', 'סוֹטָה'],
		['Gittin', 'גִּטִּין'],
		['Kiddushin', 'קִדּוּשִׁין']
		]],
	['Nezikin', 'נְזִיקִין', [
		['Bava Kamma', 'בָּבָא קַמָּא'],
		['Bava Metzia', 'בָּבָא מְצִיעָא'],
		['Bava Batra', 'בָּבָא בָּתְרָא'],
		['Sanhedrin', 'סַנְהֶדְרִין'],
		['Makkot', 'מַכּוֹת'],
		['Shevuot', 'שְׁבוּעוֹת'],
		['Eduyot', 'עֵדֻיּוֹת'],
		['Avodah Zarah', 'עֲבוֹדָה זָרָה'],
		['Avot', 'אָבוֹת'],
		['Horayot', 'הוֹרָיוֹת'],
		]],
	['Kodashim', 'קָדָשִׁים', [
		['Zevachim', 'זבחים'],
		['Menachot', 'מנחות'],
		['Chullin', 'חולין'],
		['Bekhorot', 'בְּכוֹרוֹת'],
		['Arakhin', 'ערכין'],
		['Temurah', 'תְּמוּרָה'],
		['Keritot', 'כָּרֵתוֹת'],
		['Meilah', 'מעילה'],
		['Tamid', 'תָּמִיד'],
		['Middot', 'מִדּוֹת'],
		['Kinnim', 'קִנִּים'],
		]],
	['Tohorot', 'טְהָרוֹת', [
		['Kelim', 'כלים'], ###
		['Oholot', 'אהלות'],
		['Negaim', 'נגעים'],
		['Parah', 'פרה'],
		['Tohorot', 'טְהָרות'], #Tahorot
		['Mikvaot', 'מִקֶוָאוֹת'],
		['Niddah', 'נידה'],
		['Makhshirin', 'מכשירים'],
		['Zavim', 'זבים'],
		['Tevul Yom', 'טבול יום'],
		['Yadayim', 'יָדַיִם'],
		['Uktzim', 'עוקצים']#Oktzin
		]]
	]

class Verse:
	def __init__(self, number, chapter, mishnah_text, bartenura_text):
		self.number = number
		self.chapter = chapter
		self._mishnah_text = mishnah_text
		self._bartenura_text = bartenura_text
		self.hebrew_clean_number = hebrew_numbers.int_to_gematria(number, gershayim=False)
		self.hebrew_number = hebrew_numbers.int_to_gematria(number)

	@property
	def mishnah_text(self):
		return self._mishnah_text

	@mishnah_text.setter
	def mishnah_text(self, value):
		self._mishnah_text = value

	@property
	def mishnah(self):
		return parse(self._mishnah_text)

	@property
	def bartenura_text(self):
		return self._bartenura_text

	@bartenura_text.setter
	def bartenura_text(self, value):
		self._bartenura_text = value

	@property
	def bartenura(self):
		return parse_bartenura(self._bartenura_text)

class Chapter:
	def __init__(self, number, tractate, text):
		self.number = number
		self.tractate = tractate
		self.hebrew_number = hebrew_numbers.int_to_gematria(number)
		self.hebrew_clean_clean = hebrew_numbers.int_to_gematria(number, gershayim=False)
		self.verses = []
		self.text = text

class Tractate:
	def __init__(self, name, latin_name, number, order):
		self.name = name
		self.latin_name = latin_name
		self.number = number
		self.order = order
		self._chapters = None

	@property
	def mishnah_text(self):
		filename = '%d.%02d.txt'%(self.order.number, self.number)
		filename = os.path.join(MISHNAH_FOLDER, filename)
		return open(filename).read()

	@mishnah_text.setter
	def mishnah_text(self, value):
		filename = '%d.%02d.txt'%(self.order.number, self.number)
		filename = os.path.join(MISHNAH_FOLDER, filename)
		open(filename, 'w').write(value)

	@property
	def bartenura_text(self):
		filename = '%d.%02d.txt'%(self.order.number, self.number)
		filename = os.path.join(BARTENURA_FOLDER, filename)
		return open(filename).read().replace(':\u2028', '.\u2028')

	@bartenura_text.setter
	def bartenura_text(self, value):
		filename = '%d.%02d.txt'%(self.order.number, self.number)
		filename = os.path.join(BARTENURA_FOLDER, filename)
		open(filename, 'w').write(value)

	#def save(self):
	#	#self.mishnah_text = '\n\n'.join(['\n'.join([verse.mishnah_text for verse in chapter.verses]) for chapter in tractate.chapters])
	#	data = [[verse.mishnah_text for verse in chapter.verses] for chapter in tractate.chapters]
	#	print (data)

	@property
	def chapters(self):
		if self._chapters is None:
			self._chapters = []
			mishnah = [c.split('\n') for c in self.mishnah_text.split('\n\n')]
			bartenura = [c.split('\n') for c in self.bartenura_text.split('\n\n')]
			#print ("LEN", len(mishnah), len(bartenura))
			for chapter_nr in range(1, len(mishnah) + 1):
				#print ("ch nr", chapter_nr)
				#print ("LENCH", len(mishnah[chapter_nr - 1]), len(bartenura[chapter_nr - 1]))
				chapter = Chapter(chapter_nr, self, mishnah[chapter_nr - 1])
				for verse_nr in range(1, len(mishnah[chapter_nr - 1]) + 1):
					#print ("v nr", verse_number)
					#mishnah_text = verses_mishnah_parts[verse_number - 1]
					verse = Verse(verse_nr, chapter, mishnah[chapter_nr - 1][verse_nr - 1],
							#mishnah[chapter_nr - 1][verse_nr - 1])
							bartenura[chapter_nr - 1][verse_nr - 1])
					chapter.verses.append(verse)
				self._chapters.append(chapter)
		return self._chapters

class Order:
	def __init__(self, name, latin_name, number):
		self.name = name
		self.latin_name = latin_name
		self.number = number
		self.tractates = []

orders = []
for x in range(len(TOC)):
	latin_name, name, toc = TOC[x]
	order = Order(name, latin_name, x + 1)
	for t in range(len(toc)):
		tractate_latin_name, tractate_name = toc[t]
		tractate = Tractate(tractate_name, tractate_latin_name, t + 1, order)
		order.tractates.append(tractate)
	orders.append(order)

def parse(text=''):
	if not text:
		return []
	text = re.sub('"([^"]+)"', r'”\1”', text)
	text = re.sub("'([^']+)'", r"’\1’", text)
	text = re.sub('^- ', '\u2015 ', text)#―
	text = re.sub(' -', ' \u2013', text)#–
	text = re.sub('-', '\u2011', text)
	#text = re.sub('{[^}]*}', '', text) #remove zohar page no
	#text = re.sub(' +', ' ', text)
#	h1_items = list(re.finditer('^=([^\n]+)', text, re.M))
#	for item in h1_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	alternative_items = list(re.finditer('\[([^|\]]*)\|([^]]+)\]', text))
#	for item in alternative_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	nonliteral_items = list(re.finditer('_([^_]+)_', text))
#	for item in nonliteral_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	addition_items = list(re.finditer('\+([^+]+)\+', text))
#	for item in addition_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	synonym_items = list(re.finditer('\(\=([^)]+)\)', text))
#	for item in synonym_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	explanation_items = list(re.finditer('\(\~([^)]+)\)', text))
#	for item in explanation_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
#	correction_items = list(re.finditer('\[([^]]+)\]', text))
#	for item in correction_items:
#		start, end = item.span()
#		text = text[0:start] + (end - start) * 'X' + text[end:]
	citation_items = list(re.finditer('\{([^}]+)\}', text))
	for item in citation_items:
		start, end = item.span()
		text = text[0:start] + (end - start) * 'X' + text[end:]
	link_items = list(re.finditer('\(([\u05d0-\u05ea־]+ [\u05d0-\u05ea]{1,3} [\u05d0-\u05ea]{1,3})\)', text)) + \
			list(re.finditer('\^([^^]+)\^', text))
	for item in link_items:
		start, end = item.span()
		text = text[0:start] + (end - start) * 'X' + text[end:]
	plain_items = list(re.finditer('([^X]+)', text))
	for item in plain_items:
		start, end = item.span()
		text = text[0:start] + (end - start) * '.' + text[end:]
	spans = []
	for idx in range(len(text)):
		for item in citation_items + link_items + plain_items:
			#nonliteral_items + addition_items + alternative_items + \
			#	synonym_items + explanation_items + correction_items + \
			# + h1_items:
			if idx == item.start():
				groups = item.groups()
				value = groups[0]
				#if len(groups) > 1:
				#	alt = groups[1]
				span = None
				#if item in h1_items:
				#	span = Span(SpanKind.H1, value)
				#elif item in nonliteral_items:
				#	span = Span(SpanKind.NONLITERAL, value)
				#elif item in addition_items:
				#	span = Span(SpanKind.ADDITION, value)
				#elif item in alternative_items:
				#	span = Span(SpanKind.ALTERNATIVE, value, alt)
				#elif item in synonym_items:
				#	span = Span(SpanKind.SYNONYM, value)
				#elif item in explanation_items:
				#	span = Span(SpanKind.EXPLANATION, value)
				#elif item in correction_items:
				#	span = Span(SpanKind.CORRECTION, value)
				if item in citation_items:
					span = Span(SpanKind.SCRIPTURE, value)
				elif item in link_items:
					span = Span(SpanKind.LINK, value)
				elif item in plain_items:
					span = Span(SpanKind.PLAIN, value)
				spans.append(span)
	return spans

#@property
def parse_bartenura(text):
	if 1:
		#if not text:
		#	return []
		text = text.replace('\u2028', '\n')
		text = re.sub('"([^"]+)"', r'”\1”', text)
		text = re.sub("'([^']+)'", r"’\1’", text)
		text = re.sub('^- ', '\u2015 ', text)#―
		text = re.sub(' -', ' \u2013', text)#–
		text = re.sub('-', '\u2011', text)
		legend_items = list(re.finditer('^([^\.]+)\.', text, re.M))
		for item in legend_items:
			start, end = item.span()
			text = text[0:start] + (end - start) * 'X' + text[end:]
		citation_items = list(re.finditer('\{([^}]+)\}', text))
		for item in citation_items:
			start, end = item.span()
			text = text[0:start] + (end - start) * 'X' + text[end:]
		link_items = list(re.finditer('\(([^\)]+)\)', text))
		for item in link_items:
			start, end = item.span()
			text = text[0:start] + (end - start) * 'X' + text[end:]
		reference_items = list(re.finditer('\{([^\}]+)\}', text))
		for item in reference_items:
			start, end = item.span()
			text = text[0:start] + (end - start) * 'X' + text[end:]
		plain_items = list(re.finditer('([^X]+)', text))
		for item in plain_items:
			start, end = item.span()
			text = text[0:start] + (end - start) * '.' + text[end:]
		spans = []
		for idx in range(len(text)):
			for item in legend_items + citation_items + link_items + reference_items + plain_items:
				if idx == item.start():
					groups = item.groups()
					value = groups[0]
					value = value.replace('\n', '')
					span = None
					if item in legend_items:
						#value = value[:-1]
						span = Span(SpanKind.LEGEND, value)
					elif item in citation_items:
						value = re.sub('"([^"]+)"', r'“\1”', value)
						span = Span(SpanKind.SCRIPTURE, value)
					elif item in link_items:
						#value = re.sub(' ', '\u00a0', value)
						span = Span(SpanKind.LINK, value)
					elif item in reference_items:
						#value = re.sub(' ', '\u00a0', value)
						span = Span(SpanKind.REFERENCE, value)
					#elif item in plain_items:
					else:
						#value = re.sub("'([^']+)'", r"‘\1’", value)
						span = Span(SpanKind.PLAIN, value)
					spans.append(span)
		return spans
#	print (spans)