from itertools import chain
from typing import cast, Any, Callable, Union

import numpy as np
import pandas as pd
import numpy as np
from pandas.api.types import is_list_like, CategoricalDtype

from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
from pyspark.pandas.base import column_op, IndexOpsMixin
from pyspark.pandas.data_type_ops.base import _sanitize_list_like, DataTypeOps
from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import pandas_on_spark_type
from pyspark.sql import functions as F
from pyspark.sql.column import Column

class CategoricalOps(DataTypeOps):
    """
    The class for binary operations of pandas-on-Spark objects with categorical types.
    """

    @property
    def pretty_name(self) -> str:
        return "categoricals"

    def restore(self, col: pd.Series) -> pd.Series:
        """Restore column when to_pandas."""
        return pd.Series(
            pd.Categorical.from_codes(
                col.replace(np.nan, -1).astype(int),
                categories=cast(CategoricalDtype, self.dtype).categories,
                ordered=cast(CategoricalDtype, self.dtype).ordered,
            )
        )

    def prepare(self, col: pd.Series) -> pd.Series:
        """Prepare column when from_pandas."""
        return col.cat.codes

    def astype(self, index_ops: IndexOpsLike, dtype: Union[str, type, Dtype]) -> IndexOpsLike:
        dtype, _ = pandas_on_spark_type(dtype)

        if isinstance(dtype, CategoricalDtype) and (
            (dtype.categories is None) or (index_ops.dtype == dtype)
        ):
            return index_ops.copy()

        return _to_cat(index_ops).astype(dtype)

    def eq(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__eq__, is_equality_comparison=True)

    def ne(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__ne__, is_equality_comparison=True)

    def lt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__lt__)

    def le(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__le__)

    def gt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__gt__)

    def ge(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
        _sanitize_list_like(right)
        return _compare(left, right, Column.__ge__)

def _compare(
    left: IndexOpsLike,
    right: Any,
    f: Callable[..., Column],
    *,
    is_equality_comparison: bool = False
) -> SeriesOrIndex:
    """
    Compare a Categorical operand `left` to `right` with the given Spark Column function.

    Parameters
    ----------
    left: A Categorical operand
    right: The other operand to compare with
    f : The Spark Column function to apply
    is_equality_comparison: True if it is equality comparison, ie. == or !=. False by default.

    Returns
    -------
    SeriesOrIndex
    """
    if isinstance(right, IndexOpsMixin) and isinstance(right.dtype, CategoricalDtype):
        if not is_equality_comparison:
            if not cast(CategoricalDtype, left.dtype).ordered:
                raise TypeError("Unordered Categoricals can only compare equality or not.")
        # Check if categoricals have the same dtype, same categories, and same ordered
        if hash(left.dtype) != hash(right.dtype):
            raise TypeError("Categoricals can only be compared if 'categories' are the same.")
        if cast(CategoricalDtype, left.dtype).ordered:
            return column_op(f)(left, right)
        else:
            return column_op(f)(_to_cat(left), _to_cat(right))
    elif not is_list_like(right):
        categories = cast(CategoricalDtype, left.dtype).categories
        if right not in categories:
            raise TypeError("Cannot compare a Categorical with a scalar, which is not a category.")
        right_code = categories.get_loc(right)
        return column_op(f)(left, right_code)
    else:
        raise TypeError("Cannot compare a Categorical with the given type.")

def _to_cat(index_ops: IndexOpsLike) -> IndexOpsLike:
    categories = cast(CategoricalDtype, index_ops.dtype).categories
    if len(categories) == 0:
        scol = SF.lit(None)
    else:
        kvs = chain(*[(SF.lit(code), SF.lit(category)) for code, category in enumerate(categories)])
        map_scol = F.create_map(*kvs)
        scol = map_scol[index_ops.spark.column]
    return index_ops._with_new_scol(scol)


import datetime
import os
import uuid
import random
import boto3
from botocore.exceptions import ClientError

from lambda_python_powertools.logging import (
    logger_inject_process_booking_sfn,
    logger_setup,
    MetricUnit,
    log_metric,
)

from lambda_python_powertools.tracing import Tracer

logger = logger_setup()
tracer = Tracer()

session = boto3.Session()
dynamodb = session.resource("dynamodb")
table_name = os.getenv("BOOKING_TABLE_NAME", "undefined")
table = dynamodb.Table(table_name)
#

def get_config(config_id, values):
    extract_values = lambda items: [value for value in [values for values in items[0].values()][0].values()][0]
    client = boto3.client('dynamodb')
    items = client.query(TableName='configuration_table', KeyConditionExpression='configID = :config',
                         ExpressionAttributeValues={
                             ':config': {'S': config_id}
                         }, ProjectionExpression=values)['Items']
    return extract_values(items)

_cold_start = True

class BookingReservationException(Exception):
    def __init__(self, message=None, status_code=None, details=None):

        super(BookingReservationException, self).__init__()

        self.message = message or "Booking reservation failed"
        self.status_code = status_code or 500
        self.details = details or {}

def is_booking_request_valid(booking):
    return all(x in booking for x in ["outboundFlightId", "customerId", "chargeId"])

@tracer.capture_method
def reserve_booking(booking,dow_executeAnomaly,update_item_executeAnomaly,rid):
    """Creates a new booking as UNCONFIRMED
    Parameters
    ----------
    booking: dict
        chargeId: string
            pre-authorization charge ID
        stateExecutionId: string
            Step Functions Process Booking Execution ID
        chargeId: string
            Pre-authorization payment token
        customer: string
            Customer unique identifier
        bookingOutboundFlightId: string
            Outbound flight unique identifier
    Returns
    -------
    dict
        bookingId: string
    """
    try:
        booking_id = str(uuid.uuid4())
        state_machine_execution_id = booking["name"]
        outbound_flight_id = booking["outboundFlightId"]
        customer_id = booking["customerId"]
        payment_token = booking["chargeId"]

        booking_item = {
            "id": booking_id,
            "stateExecutionId": state_machine_execution_id,
            "__typename": "Booking",
            "bookingOutboundFlightId": outbound_flight_id,
            "checkedIn": False,
            "customer": customer_id,
            "paymentToken": payment_token,
            "status": "UNCONFIRMED",
            "createdAt": str(datetime.datetime.now()),
        }

        logger.debug(
            {"operation": "reserve_booking", "details": {"outbound_flight_id": outbound_flight_id}}
        )
        num_of_oper = 1

        if update_item_executeAnomaly:
            num_of_oper = 0
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'updateItem','UpdateItemInsteadOfPutItem'))
            ret = table.update_item(Key={'id':'bf313090-82f4-4698-8eb8-29489f242c7d'},
                                UpdateExpression="set checkedIn=:r",
                                ExpressionAttributeValues={
                                    ':r': True
                                },
                                ReturnValues="UPDATED_NEW")
        if dow_executeAnomaly:
            num_of_oper = 20
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'putObject','DenialOfWalletMany'))

        for i in range(num_of_oper):
            ret = table.put_item(Item=booking_item)

        if dow_executeAnomaly:
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'putObject','DenialOfWalletMany'))

        anomaly =  update_item_executeAnomaly

        logger.info({"operation": "reserve_booking", "details": ret})
        logger.debug("Adding put item operation result as tracing metadata")
        tracer.put_metadata(booking_id, booking_item, "booking")

        return {"bookingId": booking_id, "anomaly": anomaly}
    except ClientError as err:
        logger.debug({"operation": "reserve_booking", "details": err})
        raise BookingReservationException(details=err)

@tracer.capture_lambda_handler(process_booking_sfn=True)
@logger_inject_process_booking_sfn
def lambda_handler(event, context):
    """AWS Lambda Function entrypoint to reserve a booking
    Parameters
    ----------
    event: dict, required
        Step Functions State Machine event
        chargeId: string
            pre-authorization charge ID
        stateExecutionId: string
            Step Functions Process Booking Execution ID
        chargeId: string
            Pre-authorization payment token
        customerId: string
            Customer unique identifier
        bookingOutboundFlightId: string
            Outbound flight unique identifier
    context: object, required
        Lambda Context runtime methods and attributes
        Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
    Returns
    -------
    bookingId: string
        booking ID generated
    Raises
    ------
    BookingReservationException
        Booking Reservation Exception including error message upon failure
    """
    global _cold_start
    print("Reserve")
    anomaly_mode = get_config('anomaly_mode', 'Activate')
    dow_anomaly_prob = float(get_config('Airline-ReserveBooking-master-DOW', 'anomaly_prob'))
    dow_anomaluseExecution = random.random() < dow_anomaly_prob
        # if both ANOMALY_MODE and anomaluseExecution are true - execute anomaly
    dow_executeAnomaly = anomaly_mode == True & dow_anomaluseExecution == True

    update_item_anomaly_prob = float(get_config('Airline-ReserveBooking-master-UpdateItem', 'anomaly_prob'))
    update_item_anomaluseExecution = random.random() < update_item_anomaly_prob
        # if both ANOMALY_MODE and anomaluseExecution are true - execute anomaly
    update_item_executeAnomaly = anomaly_mode == True & update_item_anomaluseExecution == True

    # If two attacks were chosen  cancel one of them randomly:
    if update_item_executeAnomaly and dow_executeAnomaly:
        if random.random() < 0.5:
            update_item_executeAnomaly = False
        else:
            dow_executeAnomaly = False

    if not (update_item_executeAnomaly or dow_executeAnomaly):
        cancel_path = get_config('cancel_mode', 'Activate')
        cancel_prob = float(get_config('cancel_mode', 'Prob'))
        cancelExecution = random.random() < cancel_prob
        executeCancel = cancel_path == True & cancelExecution == True

        if executeCancel:
            raise ValueError("Cancel booking request")

    if _cold_start:
        log_metric(
            name="ColdStart", unit=MetricUnit.Count, value=1, function_name=context.function_name
        )
        _cold_start = False

    if not is_booking_request_valid(event):
        log_metric(
            name="InvalidBookingRequest",
            unit=MetricUnit.Count,
            value=1,
            operation="reserve_booking",
        )
        logger.error({"operation": "invalid_event", "details": event})
        raise ValueError("Invalid booking request")

    try:
        logger.debug(f"Reserving booking for customer {event['customerId']}")
        rid = context.aws_request_id
        ret = reserve_booking(event,dow_executeAnomaly,update_item_executeAnomaly,rid)

        log_metric(name="SuccessfulReservation", unit=MetricUnit.Count, value=1)
        logger.debug("Adding Booking Reservation annotation")
        tracer.put_annotation("Booking", ret["bookingId"])
        tracer.put_annotation("BookingStatus", "RESERVED")

        if update_item_executeAnomaly or dow_executeAnomaly:
            raise ValueError("Cancel booking request")

        # Step Functions use the return to append `bookingId` key into the overall output
        return ret["bookingId"]
    except BookingReservationException as err:
        log_metric(name="FailedReservation", unit=MetricUnit.Count, value=1)
        logger.debug("Adding Booking Reservation annotation before raising error")
        tracer.put_annotation("BookingStatus", "ERROR")
        logger.error({"operation": "reserve_booking", "details": err})
        raise BookingReservationException(details=err)


from __future__ import print_function

import os
import sys
import subprocess

from .srcinfo import SrcInfoPool, iter_packages
from .pacman import PacmanPackage
from .utils import version_is_newer_than, version_cmp

def sorted_with_cmp(sequence, cmp_func, **kwargs):
    # sort with a cmp func which works under both Python 2 and 3

    if sys.version_info[0] == 3:
        from functools import cmp_to_key

        key = cmp_to_key(cmp_func)

        def do_sort(x):
            return sorted(x, key=key, **kwargs)
    else:
        def do_sort(x):
            return sorted(x, cmp=cmp_func, **kwargs)

    return do_sort(sequence)

def cmp_(a, b):
    return (a > b) - (a < b)

def get_pkgbuilds_to_build_in_order(packages_todo):
    """Returns a list of PKGBUILD files in the order they need to be build"""

    pool = SrcInfoPool()
    pkgbuilds = {}
    for package in packages_todo:
        pool.add_package(package)
        pkgbuilds.setdefault(package.pkgbuild_path, set()).add(package)

    # now sort the pkgbuilds by the dependencies of the contained packages

    def cmp_func(aa, bb):
        """Takes two sets of packages ands sorts according to their
        dependencies.
        """

        at = set()
        for a in aa:
            a_name = a.pkgname
            at.update(pool.get_transitive_dependencies(a))

        bt = set()
        for b in bb:
            b_name = b.pkgname
            bt.update(pool.get_transitive_dependencies(b))

        # make the result deterministic,
        # packages with fewer dependencies first
        a_key = (len(at), a_name)
        b_key = (len(bt), b_name)

        if a_name in bt and b_name in at:
            # cyclic dependency!
            return cmp_(a_key, b_key)
        elif a_name in bt:
            return -1
        elif b_name in at:
            return 1
        else:
            return cmp_(a_key, b_key)

    return pool, sorted_with_cmp(
        pkgbuilds.items(), lambda a, b: cmp_func(a[1], b[1]))

class BuildError(Exception):
    pass

def build_source(pkgbuild, packages, targetdir):
    """Build source packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)

    try:
        output = subprocess.check_output(
            ["bash", "/usr/bin/makepkg", "--noconfirm", "--noprogressbar",
             "--skippgpcheck", "--allsource", "--config",
             "/etc/makepkg_mingw64.conf", "-f",
             "-p", os.path.basename(pkgbuild),
             "SRCPKGDEST=%s" % targetdir],
            cwd=os.path.dirname(pkgbuild),
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        output = e.output
        raise BuildError(e)
    else:
        tarballs = set()
        for entry in os.listdir(targetdir):
            for p in packages:
                name = "%s-%s" % (p.pkgbase, p.build_version)
                if name in entry and ".src." in entry:
                    tarballs.add(os.path.join(targetdir, entry))
        return tarballs
    finally:
        some_pkg = list(packages)[0]
        target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)
        logpath = os.path.join(targetdir, target_key + ".src.log")
        with open(logpath, "wb") as h:
            h.write(output)

def build_and_install_binary(pkgbuild, packages, targetdir):
    """Build binary packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)

    try:
        output = subprocess.check_output(
            ["bash", "/usr/bin/makepkg-mingw", "--noconfirm",
             "--noprogressbar", "--skippgpcheck", "--nocheck", "--syncdeps",
             "--rmdeps", "--cleanbuild", "--install", "-f", "--noconfirm",
             "-p", os.path.basename(pkgbuild),
             "PKGDEST=%s" % targetdir],
            cwd=os.path.dirname(pkgbuild),
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        output = e.output
        raise BuildError(e)
    else:
        tarballs = set()
        for entry in os.listdir(targetdir):
            for p in packages:
                name = "%s-%s" % (p.pkgname, p.build_version)
                if name in entry and ".pkg." in entry:
                    tarballs.add(os.path.join(targetdir, entry))
        return tarballs
    finally:
        some_pkg = list(packages)[0]
        target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)
        logpath = os.path.join(targetdir, target_key + ".pkg.log")
        with open(logpath, "wb") as h:
            h.write(output)

def build(pkgbuild, packages, targetdir):
    """Build packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    results = set()
    some_pkg = list(packages)[0]
    target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)
    fail_path = os.path.join(targetdir, "%s.failed" % target_key)

    if os.path.exists(fail_path):
        print("%s found, build aborted. Delete the file to not skip "
              "the build." % fail_path)
        raise BuildError

    try:
        results.update(build_source(pkgbuild, packages, targetdir))
        results.update(
            build_and_install_binary(pkgbuild, packages, targetdir))
    except BuildError:
        open(fail_path, "wb").close()

        # something failed, try to clean up
        for path in results:
            try:
                os.unlink(path)
            except EnvironmentError:
                pass
        raise
    else:
        return results

def add_parser(subparsers):
    parser = subparsers.add_parser("build",
        help="Auto builds PKGBUILD files where the packages in the database "
             "are out of date. Builds them in the right order according to "
             "their dependency relation.")
    parser.add_argument(
        "path", help="path to the directory containg PKGBUILD files or a "
                     "PKGBUILD file itself")
    parser.add_argument(
        "target", help="path to the directory where the build packages will "
                       "be saved to")
    parser.add_argument('--dry-run', action='store_true',
                        help="Only show which packages will be build")
    parser.set_defaults(func=main)

def main(args):
    repo_path = os.path.abspath(args.path)
    target_path = os.path.abspath(args.target)

    repo_packages = PacmanPackage.get_all_packages()
    repo_packages = dict((p.pkgname, p) for p in repo_packages)

    # Find packages which not are not VCS and which are out of date
    packages_todo = set()
    for package in iter_packages(repo_path):
        if package.is_vcs:
            continue
        if package.pkgname in repo_packages:
            repo_pkg = repo_packages[package.pkgname]
            if version_is_newer_than(package.build_version,
                                     repo_pkg.build_version):
                packages_todo.add(package)

    # Throw away PKGBUILDS which build a package which is available from
    # another PKGBUILD but newer there.
    per_pkgname = {}
    for p in packages_todo:
        per_pkgname.setdefault(p.pkgname, set()).add(p)
    pkgbuilds_to_skip = set()
    for pkgname, packages in per_pkgname.items():
        # last is the newest
        packages_by_version = sorted_with_cmp(
            packages, lambda a, b: version_cmp(a.build_version,
                                               b.build_version))
        for to_blacklist in packages_by_version[:-1]:
            if to_blacklist.pkgbuild_path != \
                    packages_by_version[-1].pkgbuild_path:
                pkgbuilds_to_skip.add(to_blacklist.pkgbuild_path)
    packages_todo = set([p for p in packages_todo
                         if p.pkgbuild_path not in pkgbuilds_to_skip])

    # TODO: Support MSYS2 packages
    for package in packages_todo:
        print(package.repo)
        if package.repo == "msys":
            raise Exception(
                "Only mingw builds supported atm, please add "
                "support! (%s isn't)" % (package.pkgbuild_path,))

    # Sort them according to their dependencies so no package is build
    # before any of its dependencies
    pool, pkgbuilds = get_pkgbuilds_to_build_in_order(packages_todo)

    print("%d PKGBUILDs to build" % len(pkgbuilds))

    if args.dry_run:
        for path, packages in pkgbuilds:
            print(path)
            for package in packages:
                print("    -> ", package.pkgname)
        return

    if pkgbuilds:
        try:
            os.makedirs(target_path)
        except EnvironmentError:
            pass

    failed_packages = set()
    skipped = set()
    for path, packages in pkgbuilds:
        # In case some build failed, check if this one needs to be
        # skipped because it depends on the failed one.
        can_build = True
        reason = set()
        for failed in failed_packages:
            for p in packages:
                if failed in pool.get_transitive_dependencies(p):
                    can_build = False
                    reason.add(failed)
        if not can_build:
            skipped.update(packages)
            print("SKIPPING %s because %s failed" % (path, ", ".join(reason)))
            continue

        # start the build
        print("STARTING %s" % path)
        try:
            build(path, packages, target_path)
        except BuildError:
            print("FAILED")
            failed_packages.update(packages)
        else:
            print("DONE")

    # Final report
    print("All done.")
    if failed_packages:
        print("The following packages failed to build:")
        for p in sorted(failed_packages):
            print(p.pkgname)
        print("As a result, the following packages got skipped:")
        for p in sorted(skipped):
            print(p.pkgname)


'''
Decision Tree
Predict if it is possible to default on the loan
'''
import numpy as np
from sklearn import tree

data = np.genfromtxt("exercise.csv", delimiter=",")
# get train data set
x_data = data[1:, 1:-1]
# get test data set
y_data = data[1:, -1]

print(x_data)
print(y_data)

# Create decision tree
dtree = tree.DecisionTreeClassifier(min_samples_leaf=5)
dtree.fit(x_data, y_data)
print(dtree.score(x_data, y_data))


import logging.handlers
import os

import Ops
from Miscellaneous import exe_vers, find_exe

LONG_LINE = 120
TAF_VER = ".".join([str(i) for i in exe_vers(find_exe())])

if not os.path.exists('Logs'):
    os.mkdir('Logs')

log_file = "TafnitClientLog.log"
logging.basicConfig(filename=f'Logs\\{log_file}',
                    level=logging.DEBUG,
                    filemode='w',
                    format='%(asctime)s.%(msecs)03d - %(process)d:[%(levelname)7s]:[%(levelno)s]  %(message)s',
                    datefmt="%d-%m-%Y %H:%M:%S")
# root_looger = logging.getLogger()
# root_looger.setLevel(logging.DEBUG)
# handler = logging.FileHandler(f'Logs\\{log_file}', 'w', 'utf-8')
# handler.setFormatter(
#     logging.Formatter('%(asctime)s.%(msecs)03d - %(process)d:[%(levelname)7s]:[%(levelno)s]  %(message)s'))
# root_looger.addHandler(handler)

def init():
    logging.info("Initializing end user client configuration tool for eTafnit usage")
    logging.info("Updated as of May 2019")
    # logging.info("Tested on InterSystems Cache version 2018.1.2.309.0.18861")
    logging.info(f"Current eTafnit client version installed: {TAF_VER}")

def log_section(msg):
    char = '>'
    line = char * 10
    print_line = f'{line} {msg} {line}'
    print_line_len = len(print_line)
    logging.debug("")
    logging.debug(print_line_len * char)
    logging.debug(print_line)
    logging.debug(print_line_len * char)

def start_title(msg):
    char = '>'
    line = char * 10
    print_line = f'{line} {msg} {line}'
    logging.debug('')
    logging.debug(print_line)
    return len(print_line)

def end_title(num):
    char = '>'
    logging.debug(char * num)
    logging.debug('')

def end_log():
    from math import ceil
    import traceback
    HALF_LINE = ceil(LONG_LINE / 2)
    logging.info(f'\n{LONG_LINE * "_"}\n{HALF_LINE * ".-"}')
    if not Ops.operations:
        logging.info(f"The client is ready.")
        logging.info("Done.")
        return False

    # Errors found
    err_len = 0
    logging.info(">" * LONG_LINE)
    logging.info(f"THE CLIENT IS NOT READY FOR ETAFNIT")
    errors = []
    for i, e in enumerate(Ops.operations):
        line = f">> {i + 1}) {str(Ops.operations[i])} "
        errors.append(line)
        if err_len < len(line):
            err_len = len(line)

    err_len += 2
    logging.info(">" * err_len)
    for err in errors:
        times = err_len - len(err) - 2
        logging.info(f"{err}{times * ' '}>>")
    logging.info(">" * err_len)
    return True

def error_file():
    try:
        os.remove("Logs\\TafnitErrors.log")
    except FileNotFoundError:
        pass
    if Ops.operations:
        f = open("Logs\\TafnitErrors.log", "w", encoding='utf8')
        f.write('Error File\n')
        err_len = 0  # longest error line
        errors = []  # errors list
        for i, e in enumerate(Ops.operations):
            line = f'>> {i + 1}) {str(Ops.operations[i])} '
            errors.append(line)
            if err_len < len(line):
                err_len = len(line)
        if err_len == 0:
            f.write('No errors were found.\nA bug or a feature?')
            return
        err_len += 2
        f.write(f"{'>' * err_len}\n")
        for err in errors:
            times = err_len - len(err) - 2
            f.write(f'{err}{times * " "}>>\n')
        f.write(f"{'>' * err_len}\n")
        f.close()
    return

if __name__ == '__main__':
    pass


import logging

import IdeaAndIdmefConverter.InnerConverterStructure.Helper.Exceptions as OwnExceptions
from IdeaAndIdmefConverter.InnerConverterStructure.IdeaToIdmef.IdeaParser import IdeaParser
from IdeaAndIdmefConverter.InnerConverterStructure.IdeaToIdmef.IdeaToIdmefConverter import IdeaToIdmefConverter
from IdeaAndIdmefConverter.InnerConverterStructure.IdeaToIdmef.IdmefExporter import IdmefExporter
from IdeaAndIdmefConverter.InnerConverterStructure.IdeaToIdmef.ToIdmefMessageConverter import IdeaToIdmefMessageConverter
from IdeaAndIdmefConverter.InnerConverterStructure.IdeaToIdmef.ToIdmefValueConverter import IdeaToIdmefValueConverter
from IdeaAndIdmefConverter.InnerConverterStructure.IdmefToIdea.IdeaExporter import IdeaExporter
from IdeaAndIdmefConverter.InnerConverterStructure.IdmefToIdea.IdmefParser import IdmefParser
from IdeaAndIdmefConverter.InnerConverterStructure.IdmefToIdea.IdmefToIdeaConverter import IdmefToIdeaConverter
from IdeaAndIdmefConverter.InnerConverterStructure.IdmefToIdea.ToIdeaValueConverter import IdmefToIdeaValueConverter

from IdeaAndIdmefConverter.InnerConverterStructure.IdmefToIdea.ToIdeaMessageConverter import IdmefToIdeaMessageConverter

def convert_file_idea_into_idmef(input_filename, output_filename, time_output_filename=None, unconverted_filename=None,
                                 single_message=False, prettify=False, idmef_namespace=False):
    """
    Function which takes file with IDEA message (messages) and converts it into file with IDMEF message (messages).

    IMPORTANT: if you want to convert more than one message, each IDEA message must be in single line. Otherwise IDEA
               will not be parsed correctly.

    Each message will be converted to valid IDMEF with root IDMEF-Message, but root of all
    these messages in output file will be IDMEF-Messages (only if there are more input messages, if it is just single,
    absolute root will be IDMEF-Message). It means single valid IDMEF message is enclosed to tag
    (<IDMEF-Message></IDMEF-Message>)

    Be careful before you decide to convert, many fields from IDEA cannot be converted to IDMEF. If you do not provide
    unconverted_filename, where this unconverted data are stored converted naturally from JSON to XML, you may lose some
    data from IDEA.

    :param input_filename: Path to input file with IDEA message (messages).
    :param output_filename: Path to output file, which will contain converted IDMEF messages. Whole file will be
                            rewritten with IDMEF messages, so it is recommended to use path to file which does not exist
                            or is empty.
    :param time_output_filename: file for saving time of conversion
    :param unconverted_filename: Path to output file, which will contain XML data, which could not be converted properly
                                 from IDEA to IDMEF and therefore are converted naturally. Each messsage is
                                 encloded to tag UnconvertedFromIDEA, all messages (if more than one input IDEA is
                                 provided) are enclosed to root tag AllUnconvertedFromIDEA. Whole file will be
                                 rewritten, so it is recommended to use path to file which does not exist or is empty.
    :param single_message: boolean, information whether input file contains one or more messages
    :param prettify: boolean, information whether output XML has to be prettified (indentation) or not.
                    If you use your library at Windows os, this function always writes output prettified, you do not
                    need to use this parameter (use it from Linux systems)
    :param idmef_namespace: boolean, information, whether output IDMEF message (messages) has to contain "idmef"
                            namespace or not
    """
    input_file, output_file, time_file, unconverted_file = None, None, None, None
    converter = None
    try:
        input_file = open(input_filename, "r")
        output_file = open(output_filename, "w")
        if time_output_filename:
            time_file = open(time_output_filename, "w")
        if unconverted_filename:
            unconverted_file = open(unconverted_filename, "w")

        idea_parser = IdeaParser(input_file)
        idea_to_idmef = IdeaToIdmefMessageConverter()
        values_to_idmef = IdeaToIdmefValueConverter()
        idmef_exporter = IdmefExporter(output_file, unconverted_file)
        converter = IdeaToIdmefConverter(idea_parser, idea_to_idmef, values_to_idmef, idmef_exporter, time_file)
        converter.convert_file(single_message, prettify, idmef_namespace)
    except FileNotFoundError as error:
        logging.exception(repr(error))
    except PermissionError as error:
        logging.exception(repr(error))
    except OwnExceptions.JsonParserException as error:
        logging.exception(repr(error))
        if converter:
            converter.correct_file_after_interruption()
    except OwnExceptions.IdeaToIdmefMessageConverterException as error:
        logging.exception(repr(error))
        if converter:
            converter.correct_file_after_interruption()
    except OwnExceptions.IdeaToIdmefValueConverterException as error:
        logging.exception(repr(error))
        if converter:
            converter.correct_file_after_interruption()
    finally:
        if input_file:
            input_file.close()
        if output_file:
            output_file.close()
        if time_file:
            time_file.close()
        if unconverted_file:
            unconverted_file.close()

def convert_file_idmef_into_idea(input_filename, output_filename, time_output_filename=None, prettify=False):
    """
    Function which takes file with IDEA message (messages) and converts it into file with IDMEF message (messages).

    IMPORTANT: If you want to convert more than one message, each message must be enclosed in IDMEF-Message tag and
               because of valid XML it must have some root element, which will be ancestor of each IDMEF-Message (and
               must have different tag name than IDMEF-Message). Indentation of input is not important, in this case
               we do not require one message in one line if more input messages are provided.

    Fields, which could not be converted from IDMEF properly are in result provided in last "Attach" in each IDEA.

    :param input_filename: Path to input file with IDMEF message (messages).
    :param output_filename: Path to output file, which will contain converted IDEA messages. Whole file will be
                            rewritten with IDMEF messages, so it is recommended to use path to file which does not exist
                            or is empty.
    :param time_output_filename: file for saving time of conversion
    :param prettify: boolean, information whether output JSON has to be prettified (indentation) or not
    """
    input_file, output_file, time_file = None, None, None
    try:
        input_file = open(input_filename, "r")
        output_file = open(output_filename, "w")
        if time_output_filename:
            time_file = open(time_output_filename, "w")

        idmef_parser = IdmefParser(input_file)
        idmef_to_idea = IdmefToIdeaMessageConverter()
        idea_exporter = IdeaExporter(output_file)
        values_to_idea = IdmefToIdeaValueConverter()
        converter = IdmefToIdeaConverter(idmef_parser, idmef_to_idea, values_to_idea, idea_exporter, time_file)
        converter.convert_file(prettify=prettify)
    except FileNotFoundError as error:
        logging.exception(repr(error))
    except PermissionError as error:
        logging.exception(repr(error))
    except OwnExceptions.XmlParserException as error:
        logging.exception(repr(error))
    except OwnExceptions.IdmefToIdeaMessageConverterException as error:
        logging.exception(repr(error))
    except OwnExceptions.IdmefToIdeaValueConverterException as error:
        logging.exception(repr(error))
    finally:
        if input_file:
            input_file.close()
        if output_file:
            output_file.close()
        if time_file:
            time_file.close()

def convert_string_idea_into_idmef(input_string, idmef_namespace=False):
    """
    Function which takes string with single IDEA message and converts it into string with single IDMEF message.

    :param input_string: string containing JSON object with IDEA message
    :param idmef_namespace: boolean, information, whether output IDMEF message (messages) has to contain "idmef"
                            namespace or not
    :return: output string with converted IDMEF message
    """
    try:
        idea_parser = IdeaParser(input_string)
        idea_to_idmef = IdeaToIdmefMessageConverter()
        values_to_idmef = IdeaToIdmefValueConverter()
        idmef_exporter = IdmefExporter()
        converter = IdeaToIdmefConverter(idea_parser, idea_to_idmef, values_to_idmef, idmef_exporter)
        return converter.convert_string(idmef_namespace)
    except OwnExceptions.IdeaToIdmefMessageConverterException as error:
        logging.exception(repr(error))
    except OwnExceptions.IdeaToIdmefValueConverterException as error:
        logging.exception(repr(error))

def convert_string_idmef_into_idea(input_string):
    """
    Function which takes string with single IDMEF message and converts it into string with IDEA message (or messages
    when IDMEF message contains more alerts).

    :param input_string: string containing XML with IDEA message
    :return: output string with converted IDEA message (messages)
    """
    try:
        idmef_parser = IdmefParser(input_string)
        idmef_to_idea = IdmefToIdeaMessageConverter()
        idea_exporter = IdeaExporter()
        values_to_idea = IdmefToIdeaValueConverter()
        converter = IdmefToIdeaConverter(idmef_parser, idmef_to_idea, values_to_idea, idea_exporter)
        return converter.convert_string()
    except OwnExceptions.IdmefToIdeaMessageConverterException as error:
        logging.exception(repr(error))
    except OwnExceptions.IdmefToIdeaValueConverterException as error:
        logging.exception(repr(error))

"""
def main():
    convert_file_idea_into_idmef("../ExternalData/InputData/idea-examples.json",
                                 "../ExternalData/OutputData/idea-examples.xml", "idea-examples-time.txt",
                                 unconverted_filename="../ExternalData/OutputData/idea-examples-unconvertable.xml")
    convert_file_idmef_into_idea("../ExternalData/InputData/mid-level-phase-4.xml",
                                 "../ExternalData/OutputData/mid-level-phase-4.json", "mid-level-phase-time.txt")
    convert_file_idmef_into_idea("../ExternalData/InputData/rfc4765-examples.xml",
                                 "../ExternalData/OutputData/rfc4765-examples.json", "rfc4765-examples-time.txt")

if __name__ == "__main__":
    # execute only if run as a script
    main()
"""


# importar bibliotecas
import time
import requests
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from bs4 import BeautifulSoup
from colorama import init, Fore, Back

# 2 inserir init() para que o colocarama funcione no windows
init(autoreset = True)

# 3 definir número de jogos
def num_jogos():
    global número_de_jogos
    número_de_jogos = int(input('Quantos jogos você gostaria de cadastrar? '))

# 4 definir os jogos
def jogos():
    jogo = list()
    global jogos_num
    jogos_num = list()
    posição = 1
    número = 1
    for c in range(1, número_de_jogos+1):
        for números in range(1, 16):
            jogo.append(int(input(f'Qual o {posição}º número do {número}º jogo: ')))
            posição += 1
        print(f'{"----------":>70}')
        jogos_num.append(jogo[:])
        jogo.clear()
        número += 1
        posição = 1
    return jogos_num

# 5 criar função para mostrar os jogos
def mostra_jogos():
    for pos, números in enumerate(jogos_num):
        time.sleep(0.3)
        print(f'jogo {pos+1}: {números}')

# 6 criar funções para as opções do Menu
def opção1():
    num_jogos()
    jogos()
    print(f'Total de {len(jogos_num)} jogos cadastrados')
    print('--'*35)
    print('Jogos cadastrados com sucesso.')
    print('No menu, vá para a opção desejada.')
    print('--'*35)

def opção1_menu():
    números_novos = int(input('Quantos jogos a mais você gostaria de cadastrar? '))
    jogo = list()
    número = 1
    posição = 1
    for c in range(1, números_novos+1):
        for números in range(1, 16):
            jogo.append(int(input(f'Qual o {posição}º número do {número}º novo jogo: ')))
            posição += 1
        print(f'{"----------":>70}')
        jogos_num.append(jogo[:])
        jogo.clear()
        número += 1
        posição = 1
    return jogos_num

def opção2():
    print('--'*35)
    print(f'{"JOGOS CADASTRADOS":^70}')
    print('--'*35)
    if len(jogos_num) > 0:
        mostra_jogos()
        print('--'*35)
    else:
        print(f'Você não possui jogos cadastrados.\nPor favor, volte ao menu anterior e selecione a opção 1.')

def opção3():
    pergunta_jogos = str(input('Gostaria que seus jogos fossem mostrados novamente? S/N  ')).upper()
    print(f'{"----------":>70}')
    if pergunta_jogos == 'S':
        mostra_jogos()
        print(f'{"----------":>70}')
    pergunta_num = int(input('Qual jogo você gostaria de alterar?  '))
    print('--'*35)
    if pergunta_num == 0 or pergunta_num > len(jogos_num):
        time.sleep(0.3)
        print('Opção inválida, por favor refaça.')
        print(f'{"----------":>70}')
        print(f'   O jogo {pergunta_num} não existe.')
        print('   Volte ao menu anterior.')
    else:
        jogos_num.pop(pergunta_num-1)
        substituir_jogo = list()
        for c in range(1, 16):
            substituir_jogo.append(int(input(f'Qual o {c}º número do novo {pergunta_num}º jogo: ')))
        jogos_num.insert(pergunta_num-1, substituir_jogo[:])
        substituir_jogo.clear()
        print(f'jogo {pergunta_num} alterado com sucesso.')

def opção4():
    time.sleep(0.3)
    pergunta_jogos = str(input('Gostaria que seus jogos fossem mostrados novamente? S/N  ')).upper()
    print(f'{"----------":>70}')
    time.sleep(0.3)
    if pergunta_jogos == 'S':
        mostra_jogos()
        print(f'{"----------":>70}')
    pergunta = int(input('Qual jogo você gostaria de excluir?  '))
    print(f'{"----------":>70}')
    if pergunta == 0 or pergunta > len(jogos_num):
        time.sleep(0.3)
        print('Opção inválida, por favor refaça.')
        print(f'{"----------":>70}')
        print(f'  * O jogo {pergunta} não existe.')
        print('  * Volte ao menu anteior')
    else:
        jogos_num.pop(pergunta-1)
        time.sleep(0.3)
        print(f'jogo {pergunta} excluído com sucesso.')
        print(f'{"----------":>70}')
        time.sleep(0.3)
        print('   Seus jogos receberam uma nova numeração.')
        time.sleep(0.3)
        print('   Selecione a OPÇÃO 2 para nova visualização.')

def opção5():
    url = 'http://loterias.caixa.gov.br/wps/portal/loterias/landing/lotofacil/'

    option = Options()
    option.headless = True
    driver = webdriver.Firefox()

    driver.get(url)
    time.sleep(15)

    def concurso():
        try:
            element = driver.find_element_by_xpath("//div[@class='title-bar clearfix']//h2//span")
            html_content = element.get_attribute('outerHTML')
            soup = BeautifulSoup(html_content, 'html.parser')
            return soup.get_text().split()
        except:
            return 'Tente mais tarde'

    print(f' {concurso()[0]}  {concurso()[1]}  {concurso()[2]}')

    def números_sorteio():
        try:
            element2 = driver.find_element_by_xpath("//table[@class='simple-table lotofacil']//tbody")
            html_content2 = element2.get_attribute('outerHTML')
            soup2 = BeautifulSoup(html_content2, 'html.parser')
            números = soup2.get_text().split()
            num = list()
            índice = 0
            # for c in range(0,3): # Forma maior
            #     for pos, núm in enumerate(números[índice]):
            #         num.append(núm)
            #     índice += 1
            for c in range(0, 3):
                [num.append(núm) for pos, núm in enumerate(números[índice])] # Forma contraída
                índice += 1
            soma = 0
            posi = 0
            posiç = 1
            resultado = list()
            for c in range(0, 15):
                resultado.append(num[posi] + num[posiç])
                soma += 1
                posi += 2
                posiç += 2
            resultado_inteiros = [int(num) for num in resultado]
            return resultado_inteiros
        except:
            return 'Tente mais tarde'

    print(f' Num sorteados: {números_sorteio()}')

    def acertos():
        global acertos_jogos
        acertos_jogos = list()
        for números in jogos_num:
            acertos_jogos.append(list(set(números)&set(números_sorteio())))
        return acertos_jogos
    acertos()
    print(f'{"-----------":>70}')

    def mostra_jogos():
        for pos, números in enumerate(acertos_jogos):
            print(Fore.GREEN + f' jogo {pos+1}:  ACERTOS  {números}  TOTAL = {len(números)}')

    mostra_jogos()

    time.sleep(0.3)

    print('--'*35)
    print(f'{"Confira o seu bilhete no site oficial da Loteria !":^70}')
    print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
    print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
    print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
    print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
    print('--'*35)

    driver.quit()

# 7 criar Menu de opções
def menu():
    print('--'*35)
    print(f'{"BEM VINDO AO VERIFICADOR DE JOGOS DE 15 NÚMEROS":^70}')
    print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
    print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
    print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
    print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
    print('--'*35)
    print('--'*35)
    print((Back.WHITE + Fore.BLACK + f'{"Menu de Opções":^70}'))
    print('--'*35)
    time.sleep(0.3)
    print(Fore.BLUE + '''    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
    print('--'*35)
    print('As opções não podem ser acessadas.')
    print('ANTES CADASTRE PELO MENOS UM JOGO.')
    print('--'*35)
    opção1()
    resp = 'S'
    while resp == 'S':
        print('--'*35)
        print((Back.WHITE + Fore.BLACK + f'{"Menu de Opções":^70}'))
        print('--'*35)
        time.sleep(0.3)
        print('''    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
        print('--'*35)
        time.sleep(0.3)
        print('--'*35)
        time.sleep(0.3)
        opção = int(input('    Qual a opção desejada:  '))
        print('--'*35)
        while opção < 1 or opção > 8:
            print('Desculpe a opção digitada não é válida, por favor, refaça.')
            resp = str(input('Deseja continuar? S/N  ')).upper()
            if resp != 'S':
                print(f'Programa finalizado, volte sempre.')
                break
            if resp == 'S':
                time.sleep(0.3)
                print('--'*35)
                print((f'{"Menu de Opções":^70}'))
                print('--'*35)
                print('''
    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
            print('--'*35)
            opção = int(input('    Qual a opção desejada:  '))
            print('--'*35)

        if opção == 1:
            opção1_menu()
            print(f' Total de {len(jogos_num)} jogos cadastrados')

        if opção == 2:
            opção2()

        if opção == 3:
            opção3()

        if opção == 4:
            opção4()

        if opção == 5:
            if len(jogos_num) == 0:
                time.sleep(0.3)
                print(Back.WHITE + 'Você não pode executar o programa.')
                print(Fore.RED + '  * Ainda não foram cadastrados jogos.')
                print(Fore.RED + '  * Volte ao menu anterior')
            else:
                opção5()

        if opção == 6:
            print('--'*35)
            print(f'{"Obrigado, volte sempre !":^70} ')
            print(f'{"Programa encerrado com sucesso !":^70}')
            print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
            print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
            print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
            print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
            print('--'*35)
            break

menu()


# -*- coding: utf-8 -*-
"""
Created on Thu Feb 13 12:50:44 2020

@author: tl
"""
from numpy.linalg import matrix_power
from numpy.linalg import inv
from numpy.linalg import cond
from itertools import combinations
from scipy.linalg import circulant

import scipy as sy
import numpy as np
import random as rd

# Generate a random matrix with dimension dim
def genRandMat(low, high, dim):
    mat = np.random.randint(low, high, dim).astype(np.float64)
    return mat

# Random pick k worker nodes from worker set. Rest of them are stragglers
def pickWorkers(threshold, numWorkers):
    workersSet = rd.sample(set(range(numWorkers)), threshold)
    return workersSet

# add noise to matrix
def addGaussianNoise(mat, SNR):
    noisePower = 10**(-SNR/10)
    noise = np.random.randn(mat.shape[0], mat.shape[1]) * np.sqrt(noisePower)
    return mat + noise

# encode matrix in the master node
def encodeMat(mat, generator, blockSize, threshold, numWorkers, embedMatSize):
    matSplit = np.split(mat, threshold*embedMatSize, axis=1)
    row = mat.shape[0]
    matEncoded = [None] * numWorkers
    for j in range(numWorkers):
        encodeTemp = np.zeros([row, embedMatSize*blockSize], dtype=generator.dtype)
        for k in range(embedMatSize):
            encodeTemp[:, k*blockSize:k*blockSize+blockSize] = sum([matSplit[i]*generator[i][embedMatSize*j+k] for i in range(embedMatSize*threshold)])
        matEncoded[j] = encodeTemp
    return matEncoded

# transmit the corresponding encoded matrix to workers
def transmitSubMat2Workers(matEncoded, numWorkers, blockSize, embedMatSize):
    matsTransmit = [None] * numWorkers
    for i in range(numWorkers):
        matsTransmit[i] = matEncoded[i]
    return matsTransmit

# decode the matrix multiplication result based on the result from worker nodes
def decodeResult(matReceived, numWorkers, generator, workersSet, blockSize, sample, embedSize):
    threshold = len(workersSet)
    matReceived = np.reshape(matReceived.T,(blockSize*sample,threshold*embedSize), order='F')
    G2Workers = np.zeros((threshold*embedSize, threshold*embedSize), dtype=generator.dtype)
    for i in range(threshold):
        worker = workersSet[i]
        G2Workers[:, i*embedSize:i*embedSize+embedSize] = generator[:, worker*embedSize:worker*embedSize+embedSize]
    # directly inverse G in time domain to get result
    Cdecode = np.matmul(matReceived, inv(G2Workers))
    #Cdecode = np.linalg.solve(G2Workers, matReceived.T)
    return Cdecode

# computation in worker nodes
def computeWorker(A, B):
    C = np.matmul(A.T, B)
    return C

# Get the correct reuslt based on A and x we randomly generate
def getMultResult(A, B, blockSize, sample, threshold, embedSize):
    ASplit = np.split(A, threshold*embedSize, axis=1)
    C = []
    for i in range(threshold*embedSize):
        C.append(np.matmul(ASplit[i].T, B).ravel())
    C = np.asarray(C).T
    return C

# Find the set of worker nodes that are not stagglers
# This set of worker nodes correspond to the worst condition number of corresponding encoding matrix
def findWorkerSet2WorstCond(generatorMat, threshold, numWorkers, embedSize):
    comb = list(combinations(range(numWorkers), threshold))
    condCollect = []
    for i in range(len(comb)):
        G2Workers = np.zeros((threshold*embedSize, threshold*embedSize), dtype=generatorMat.dtype)
        for j in range(threshold):
            worker = comb[i][j]
            G2Workers[:, j*embedSize:j*embedSize+embedSize] = generatorMat[:, worker*embedSize:worker*embedSize+embedSize]
        condCollect.append(cond(G2Workers))
    print("Average condition number is %f"%np.mean(condCollect))
    worstCond = np.max(condCollect)
    print("Worst condition number is %f"%worstCond)
    workerSet2WorstCond = comb[condCollect.index(worstCond)]
    return workerSet2WorstCond

# check the worst and average condition number for circulant embed method
# Note: since the encoding matrix in circulant matrix method is not full rank
# checking the conditio number is different from other methods()
# We check the worst and average condition number of the full rank block matrix of B
# 1) B is the block diagnoal matrix obtained by eigenvalue decomposition of encoding matrix G
# 2) only the first block matrix in B is singular
def checkCondNumberForCirEmbed(B, embedSize, threshold, numWorkers):
    comb = list(combinations(range(numWorkers), threshold))
    for i in range(len(comb)):
        condCollect = []
        for j in range(1, embedSize):
            Bsub = B[j*threshold:j*threshold+threshold, j*numWorkers:j*numWorkers+numWorkers]
            Bsub = Bsub[:, comb[i]]
            condCollect.append(cond(Bsub))
    print("Average condition number is %f"%np.mean(condCollect))
    print("Worst condition number is %f"%np.max(condCollect))

# Compare the correct result to the decoded result
def getMSE(orginalRst, decodeRst):
    mse = np.sum(abs((orginalRst - decodeRst)**2))/np.sum(abs(decodeRst**2))
    print("MSE is %s"%mse)
    return mse

#############################################################################################################
# The following function is to create encoding matrix for different methods

# Encoding matrix in Vandermonde embedded by rotation matrix scheme
def getRotEmbedEncodingMat(threshold, numWorkers):
    alpha = np.exp(2*1j*np.pi/numWorkers) #n-th root of unity
    rotMat = [[np.real(alpha), -np.imag(alpha)], [np.imag(alpha), np.real(alpha)]]
    G = np.zeros((2*threshold, 2*numWorkers)) #generator matrix in time domain: a vandmond like matrix
    for i in range(threshold):
        for j in range(numWorkers):
            #generator matrix in time domain, G(i,j)= rotMat^(i*j)
            G[2*i:2*i+2, 2*j:2*j+2] = matrix_power(rotMat, i*j)
    return G

# Encoding matrix in real Vendermonde method
def getRealVandEncodingMat(activeWorkers, totalWorkers):
    G = np.zeros((activeWorkers, totalWorkers)) #generator matrix in time domain: a vandmond like matrix
    evl = np.linspace(-1, 1, totalWorkers)
    for i in range(activeWorkers):
        for j in range(totalWorkers):
            #generator matrix in time domain, G(i,j)= rotMat^(i*j)
            G[i, j] = evl[j]**(i)
    return G

# Encoding matrix in complex Vendermonde method
def getCompVandEncodingMat(activeWorkers, totalWorkers):
    alpha = np.exp(2*1j*np.pi/totalWorkers) #n-th root of unity
    G = np.zeros((activeWorkers, totalWorkers), dtype=complex) #generator matrix in time domain: a vandmond like matrix
    for i in range(activeWorkers):
        for j in range(totalWorkers):
            #generator matrix in time domain, G(i,j)= rotMat^(i*j)
            G[i, j] = alpha**(i*j)
    return G

# Encoding matrix in Vandermonde matrix embedding by circulant matrix method
def getCirEncodingMat(threshold, numWorkers, embedSize):
    # embedding matrix is a circulant matrix cirMat
    genCol = np.zeros(embedSize)
    genCol[embedSize-1] = 1
    cirMat = circulant(genCol).T

    G = np.zeros((threshold*embedSize, numWorkers*embedSize)) #generator matrix in time domain: a vandmond like matrix
    B = np.zeros((threshold*embedSize, numWorkers*embedSize), dtype=complex) #generator matrix in freq domain

    for i in range(threshold):
        for j in range(numWorkers):
            #generator matrix in time domain, G(i,j)= cirMat^(i*j)
            cir = matrix_power(cirMat, i*j)
            G[i*embedSize:i*embedSize+embedSize, j*embedSize:j*embedSize+embedSize] = cir

            #generator matrix in freq domain, B(i,j) = fft(G(i,j))
            head = cir[0,:]
            headFeq = sy.fft(head)
            B[i*embedSize:i*embedSize+embedSize, j*embedSize:j*embedSize+embedSize] = np.diag(headFeq)

    ##permute matrix B into D(block diagnoal matrix)
    permCol = embedSize*np.mod(range(embedSize*numWorkers), numWorkers) + [np.floor(x/numWorkers) for x in range(embedSize*numWorkers)]
    permRow = embedSize*np.mod(range(embedSize*threshold), threshold) + [np.floor(x/threshold) for x in range(embedSize*threshold)]
    permCol = permCol.astype(int)
    permRow = permRow.astype(int)
    B = B[permRow, :]
    B = B[:, permCol]
    return G, B

##########################################################################################
# The following three functions are only for circulant embedding method
# preprocessing the matrix, only used in circulant embedding methrod
def createFakeSymbol(mat, threshold, blockSize, embedSize):
    row = mat.shape[0]
    for i in range(threshold):
        temp = np.zeros((row, blockSize))
        for j in range(embedSize-1):
            temp = temp + mat[:, i*blockSize*embedSize+j*blockSize:i*blockSize*embedSize+j*blockSize+blockSize]
        mat[:,(i+1)*blockSize*embedSize-blockSize:(i+1)*blockSize*embedSize] = -temp
    return mat

def encodeMatForCirEmbed(mat, generator, blockSize, threshold, numWorkers, embedMatSize):
    matSplit = np.split(mat, threshold*embedMatSize, axis=1)
    row = mat.shape[0]
    matEncoded = [None] * numWorkers
    for j in range(numWorkers):
        encodeTemp = np.zeros([row, embedMatSize*blockSize], dtype=generator.dtype)
        for k in range(embedMatSize):
            nonZeroEle = np.nonzero(generator[:, embedMatSize*j+k])
            encodeTemp[:, k*blockSize:k*blockSize+blockSize] = sum([matSplit[nonZeroEle[0][i]] for i in range(threshold)])
        matEncoded[j] = encodeTemp
    return matEncoded

# decoding algorithm for circulant embeding method. The decoding algorithm over frequency domain
def decodeResultFreqCirEmbed(matReceived, numWorkers, generator, workersSet, blockSize, sample, embedSize):
    threshold = len(workersSet)
    #matReceived = np.reshape(matReceived.T,(blockSize*sample,threshold*embedSize), order='F').T
    matReceived = np.reshape(matReceived,(threshold*embedSize,blockSize*sample))
    matRecFreq = np.zeros((threshold*embedSize, blockSize*sample), dtype=complex) #iDFT each block of received codeword(c_rec)

    for i in range(threshold):
        matRecFreq[i*embedSize:i*embedSize+embedSize, :] = np.fft.fft(matReceived[i*embedSize:i*embedSize+embedSize,:], axis=0)

    permMatEncodeFreqPerm = embedSize*np.mod(range(embedSize*threshold), threshold) + [np.floor(x/threshold) for x in range(embedSize*threshold)]
    permMatEncodeFreqPerm = permMatEncodeFreqPerm.astype(int)
    matRecFreqPerm = matRecFreq[permMatEncodeFreqPerm]

    matDecodeFreqPerm = np.zeros((threshold*embedSize,blockSize*sample), dtype=complex)

    for i in range(1,embedSize):
        blockdiagMat = generator[i*threshold:i*threshold+threshold, i*numWorkers:i*numWorkers+numWorkers]
        blockdiagMat = blockdiagMat[:,workersSet]
        blockdiagMat = blockdiagMat.astype(complex)
        #block codeword times inverse of one of block diagonal matrices
        matDecodeFreqPerm[i*threshold:i*threshold+threshold, :] =  np.matmul(inv(blockdiagMat).T, matRecFreqPerm[i*threshold:i*threshold+threshold, :])

    ##permute C_fp_d to C_f_d, recover C by C_f_d(by FFT)
    permMatDecodeFreq = threshold*np.mod(range(embedSize*threshold), embedSize) + [np.floor(x/embedSize) for x in range(embedSize*threshold)]
    permMatDecodeFreq = permMatDecodeFreq.astype(int)
    matDecodeFreq = matDecodeFreqPerm[permMatDecodeFreq]; #permute A_fp_d to A_f_d
    matDecode = np.zeros((threshold*embedSize, blockSize*sample))

    for i in range(threshold):
        matDecode[i*embedSize:i*embedSize+embedSize,:] = np.fft.ifft(matDecodeFreq[i*embedSize:i*embedSize+embedSize,:], axis=0).real

    matDecode = matDecode.T
    return matDecode

matrix vector multiplication local (python)/matVecMul.py


import asyncio
import discord
import threading
import random
from functools import partial
from ..DiscordSwitchboard.DiscordSwitchboard import PriorityLevel
from ..Deck.Deck import Card, Deck, PlayingArea
from ..Player.Player import Player

# We don't worry about getting asynchronocity issues on registering/unregistering functors here, as the switchboard will take card of that.
# However, the objects own critical areas need protection in certain cases. Fortunately, most individual operations with standard Python objects (inserting, etc.) are threadsafe.

DEBUG = True  # Currently, allows the card czar to play on his own turn and allows for one-player games.
CARDS_PER_PLAYER = 10

class CardsAgainstGovernance:   # cards here should be passed in as a unique copy for this class.
    def __init__(self, switchboard, channel, client, cards):
        self.switchboard = switchboard
        self.channel = channel
        self.client = client
        self.players = []

        self.playing_area = PlayingArea()
        self.question_area = PlayingArea()
        self.questions_deck = Deck(cards[0])
        self.answers_deck = Deck(cards[1])
        self.turn_generator = None
        self.cur_czar = None
        self.new_question = None
        self.score = {}

        # Setup.
        asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel, "Cards against Governance is now in setup. Register with the game by mentioning the bot. Start the game with !startcardsgame."),
            asyncio.get_event_loop())
        setup_id = self.switchboard.RegisterOutput(self.SetupPlayers, PriorityLevel.PRIORITY, channel_id=channel.id, mentions=self.client.user.id)
        self.switchboard.RegisterOutput(self.StartGame, PriorityLevel.PRIORITY, channel_id=channel.id, starts_with="!startcardsgame",
                                        remove_self_when_done=True, remove_when_done=[(setup_id, PriorityLevel.PRIORITY)])

        self.player_lock = threading.Lock()
        self.has_played = {}

    # TODO: Get current round, etc.

    def GetCzar(self):  # A generator that returns the current czar. Changing self.players will change its operation.
        cur_index = -1
        while True:
            cur_index = (cur_index+ 1) % len(self.players)
            yield self.players[cur_index]

    # Handles setup and getting players.
    async def SetupPlayers(self, message):
        for player in self.players:
            if player.id == message.author.id:
                return False
        self.players.append(Player(message.author, self.switchboard, self.client))
        self.score[message.author.id] = [message.author.name, 0]
        return True

    # Handles starting the game itself.
    async def StartGame(self, message):
        if len(self.players) < 2 and not DEBUG:
            asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel, "Not enough players!"),
            asyncio.get_event_loop())
            return False
        # Deal a hand to each player. Note that it doesn't matter that we haven't determined turn order yet, as this is all randomized anyway.
        for player in self.players:
            player.hand = []  # players will get their full hands at start of turn
        asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel, "Game is started, your hand has been PM'd to you."),
            asyncio.get_event_loop())
        # Determine turn order:
        random.shuffle(list(self.players))
        asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel, "Turn order is: " + ", ".join(player.user.name for player in self.players)),
            asyncio.get_event_loop())
        self.turn_generator = self.GetCzar()
        self.SetupTurn()
        return True

    def SetupTurn(self):
        # Each player draws until they have ten cards.
        for player in self.players:
            player.hand.extend(self.answers_deck.Deal(CARDS_PER_PLAYER - len(player.hand)))
            player.DisplayHand()
        self.cur_czar = next(self.turn_generator)
        new_question = self.questions_deck.Deal(1)
        self.new_question = new_question[0]
        self.question_area.Play(self.new_question, self.cur_czar.user.id)
        asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel,
                                     ''.join(["Current Card Czar: ", self.cur_czar.user.name, "\n\nQuestion card:```", new_question[0].description, "```\n\nSubmit your reply by PM using !submit (number) (number)"])),
            asyncio.get_event_loop())
        for player in self.players:
            if not DEBUG and player.user == self.cur_czar.user:
                continue
            player.AddResponse("WaitForCardPlay", partial(self.WaitForCardPlay, player))

    async def WaitForCardPlay(self, player, message):
        words = message.content.lower().split()
        if words[0] != "!submit":
            return
        if len(words) < 2:
            return
        card_choices = set()
        for word in words[1:]:
            try:
                card_choice = int(word)
            except (ValueError, TypeError):
                player.SendMessage("Command improperly formatted! Try again.")
                return
            if card_choice < 0 or card_choice > len(player.hand) - 1:
                player.SendMessage("Card # out of range! Try Again.")
                return
            card_choices.add(card_choice)
        if len(card_choices) != self.new_question.data["num_answers"]:
            player.SendMessage("Wrong number of answers!")
            return
        actual_cards = player.GetCards(card_choices)
        for card in actual_cards:
            self.playing_area.Play(card, player.user.id)
        player.RemoveResponse("WaitForCardPlay")
        with self.player_lock:
            self.has_played[player.user.id] = True
            for id, has_played in self.has_played.items():
                if (id != self.cur_czar.user.id or DEBUG) and not has_played:
                    return
            self.has_played.clear()
            for player in self.players:
                if not DEBUG and player.user == self.cur_czar.user:
                    continue
            # TODO: fancify card list display
            asyncio.run_coroutine_threadsafe(
            self.client.send_message(self.channel,
                                     ''.join(["All responses received. ", self.cur_czar.user.name, " should choose their favorite response with !choose #\n\n", '\n'.join(str(x) + ": " + ','.join(x.description for x in self.playing_area.current_cards[source_id]) for x, source_id in enumerate(self.playing_area.current_cards.keys()))])),
            asyncio.get_event_loop())
            self.switchboard.RegisterOutput(self.ResolveTurn, PriorityLevel.PRIORITY, channel_id=self.channel.id,
                                            author_id=self.cur_czar.user.id, starts_with="!choose",
                                            remove_self_when_done=True)
            print("hmm")

    async def ResolveTurn(self, message):
        words = message.content.lower().split()
        if words[0] != "!choose":
            return
        if len(words) < 2:
            return False
        try:
            choice = int(words[1])
        except (ValueError, TypeError):
            asyncio.run_coroutine_threadsafe(
                self.client.send_message(self.channel, "Command improperly formatted! Try again."),
                asyncio.get_event_loop())
            return False
        ref_list = list(self.playing_area.current_cards.keys())
        if choice < 0 or choice > len(ref_list) - 1:
            asyncio.run_coroutine_threadsafe(
                self.client.send_message(self.channel, "Card # out of range! Try Again."),
                asyncio.get_event_loop())
            return False
        winner = ref_list[choice]
        self.score[winner][1] += 1
        asyncio.run_coroutine_threadsafe(
                self.client.send_message(self.channel, ''.join(["That belonged to ", self.score[winner][0], ", who now has ", str(self.score[winner][1]), " points!"])),
                asyncio.get_event_loop())
        self.playing_area.EndTurn()
        self.question_area.EndTurn()
        self.SetupTurn()
        return True

    def EndGame(self):
        pass


import torch
import torch.nn.functional as F

def gradient_train(epoch, network, data_loader, optimiser, log_interval=10):
    network.train()
    train_losses = []
    train_counter = []
    for batch_idx, (data, target) in enumerate(data_loader):
        if torch.cuda.is_available():
            data, target = data.cuda(), target.cuda()

        optimiser.zero_grad()
        output = network(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimiser.step()
        if batch_idx % log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(data_loader.dataset), 100.0 * batch_idx / len(data_loader),
                loss.item()))
            train_losses.append(loss.item())
            torch.save(network.state_dict(), 'results/model.pth')
            torch.save(optimiser.state_dict(), 'results/optimiser.pth')
            train_counter.append(
                (batch_idx * 64) + ((epoch - 1) * len(data_loader.dataset)))
    return train_losses, train_counter


__all__ = ["get_dataloader"]

from typing import List, Literal, cast

import numpy as np
import torch
from torch.utils.data import DataLoader, Subset, TensorDataset

def make_data(
    num_samples: int,
    num_features: int,
    num_associations: int,
    seed: int,
) -> torch.Tensor:
    if num_associations > num_features:
        raise ValueError("Number of associations cannot exceed number of features.")

    rng = np.random.RandomState(seed)

    loc = rng.randint(5, 10, size=num_features)
    con_data = rng.normal(loc=loc, scale=1, size=(num_samples, num_features))

    ids = [*range(0, num_features, num_features // num_associations)]

    mean = con_data[:, ids].mean(axis=0)
    std = con_data[:, ids].std(axis=0)

    effect = mean - rng.normal(loc=mean, scale=2 * std)

    con_data[num_samples // 2 :, ids] += effect

    return torch.FloatTensor(con_data)

def get_dataloader(
    num_samples: int = 300,
    num_features: int = 15,
    num_associations: int = 5,
    seed: int = 123,
    split: Literal["train", "test"] = "train",
    test_frac: float = 0.2,
    **dataloader_kwargs
) -> DataLoader:
    con_data = make_data(num_samples, num_features, num_associations, seed)

    train_ids = [
        *range(int(num_samples // 2 * (1 - test_frac))),
        *range(num_samples // 2, int(num_samples // 2 * (2 - test_frac))),
    ]
    test_ids = np.setdiff1d(range(num_samples), train_ids)

    mean = con_data[train_ids, :].mean(dim=0)
    scale = con_data[train_ids, :].std(dim=0)

    con_data = (con_data - mean) / scale

    cat_data = torch.zeros((num_samples, 2))
    cat_data[: (num_samples // 2), 0] = 1.0
    cat_data[(num_samples // 2) :, 1] = 1.0
    labels = cat_data.argmax(dim=-1)

    data = torch.cat((cat_data, con_data), dim=1)

    dataset = TensorDataset(data, labels)
    if split == "train":
        return DataLoader(Subset(dataset, train_ids), **dataloader_kwargs)
    elif split == "test":
        return DataLoader(
            Subset(dataset, cast(List[int], test_ids)), **dataloader_kwargs
        )

    raise ValueError("Unsupported split")


from torch.nn.utils.clip_grad import clip_grad_norm_
from torch.optim import Adam, SGD

class TripletLossModel:
    def __init__(self, img_encoder, text_encoder, criterion, fineturn=True, use_cuda=False, grad_clip=2, lr=1e-2):
        self.image_encoder = img_encoder
        self.text_encoder = text_encoder
        self.criterion = criterion
        self.use_cuda = use_cuda
        self.grad_clip = grad_clip

        params = list(self.text_encoder.parameters())
        if fineturn:
            params += list(self.image_encoder._classifier[0].parameters())
            params += list(self.image_encoder._classifier[3].parameters())
            params += list(self.image_encoder._classifier[6].parameters())

        self.params = params
        self.optimizer = Adam(self.params, lr=lr, weight_decay=5e-4)
        self.optim_dict = {'Adam': Adam, 'SGD': SGD}

    def cuda(self):
        """switch cuda
        """
        self.image_encoder.cuda()
        self.text_encoder.cuda()

    def cpu(self):
        """switch cpu
        """
        self.image_encoder.cpu()
        self.text_encoder.cpu()

    def state_dict(self):
        state_dict = [self.image_encoder.state_dict(), self.text_encoder.state_dict()]
        return state_dict

    def load_state_dict(self, state_dict):
        self.image_encoder.load_state_dict(state_dict[0])
        self.text_encoder.load_state_dict(state_dict[1])

    def train_start(self):
        """switch to train mode
        """
        self.image_encoder.train()
        self.text_encoder.train()

    def val_start(self):
        """switch to evaluate mode
        """
        self.image_encoder.eval()
        self.text_encoder.eval()

    def forward(self, image_triple, caption_triple):
        # execute image_triple
        image, pos_cap, neg_cap = image_triple.get_batch()
        image_encoded = self.image_encoder(image)
        pos_text_encoded = self.text_encoder(pos_cap[0], pos_cap[1])
        neg_text_encoded = self.text_encoder(neg_cap[0], neg_cap[1])
        image_triple_loss = self.criterion(image_encoded, pos_text_encoded, neg_text_encoded)

        # execute caption_triple
        caption, pos_img, neg_img = caption_triple.get_batch()
        text_encoded = self.text_encoder(caption[0], caption[1])
        pos_image_encoded = self.image_encoder(pos_img)
        neg_image_encoded = self.image_encoder(neg_img)
        caption_triple_loss = self.criterion(text_encoded, pos_image_encoded, neg_image_encoded)

        loss = image_triple_loss + caption_triple_loss

        # measure accuracy and record loss
        self.optimizer.zero_grad()

        # compute gradient and do SGD step
        loss.backward()
        if self.grad_clip > 0:
            clip_grad_norm_(self.params, self.grad_clip)
        self.optimizer.step()

        return loss.item()


import random
import re
from tsv_io import TSVDataset, tsv_reader, tsv_writer
from qd_common import write_to_file, basename_no_ext
from qd_common import load_list_file
from taxonomy import synset_to_noffset, LabelTree, LabelToSynset
import json
import os
import os.path as op
import logging

def ensure_dataset_sample(source_dataset, sample_label, sample_image, out_data):
    if op.exists(out_data):
        logging.info('skip to generate the samle data since it exists')
        return
    logging.info('start to generate the sample data')
    assert source_dataset.name == 'imagenet2012', 'only cls dataset is supp'
    assert sample_label > 0 and sample_label <= 1
    assert sample_image > 0 and sample_image <= 1
    random.seed(777)
    labels = source_dataset.load_labelmap()
    num_labels = len(labels)
    num_sampled_labels = int(num_labels * sample_label)
    assert num_sampled_labels > 0
    label_idx = range(num_labels)
    random.shuffle(label_idx)
    sampled_labels_idx = label_idx[:num_sampled_labels]
    train_rows = tsv_reader(source_dataset.get_train_tsv())
    train_images = [row for row in train_rows if int(row[1]) in sampled_labels_idx]
    for row in train_images:
        row[1] = str(sampled_labels_idx.index(int(row[1])))
    random.shuffle(train_images)
    tsv_writer(train_images[: int(len(train_images) * sample_image)],
            op.join(out_data, 'train.tsv'))
    # process the test set
    test_rows = tsv_reader(source_dataset.get_test_tsv_file())
    test_images = [row for row in test_rows if int(row[1]) in sampled_labels_idx]
    for row in test_images:
        row[1] = str(sampled_labels_idx.index(int(row[1])))
    tsv_writer(test_images, op.join(out_data, 'test.tsv'))
    # save teh label map
    sampled_labels = [labels[i] for i in sampled_labels_idx]
    write_to_file('\n'.join(sampled_labels), op.join(out_data, 'labelmap.txt'))

def dynamic_process_tsv(source_dataset, output_data_path,
        tree,
        **kwargs):
    if 'dataset_ops' not in kwargs:
        kwargs['dataset_ops'] = []
    return process_dataset(source_dataset, output_data_path,
            kwargs['dataset_ops'], tree)

    # deprecate the following (use process_dataset())
    source = source_dataset.get_train_tsv()
    source_label = None
    desc = ''
    if 'remove_bb_label' in kwargs and kwargs['remove_bb_label']:
        if 'remove_bb_label_prob' in kwargs and kwargs['remove_bb_label_prob']:
            desc = desc + '_rb_{}{}'.format(kwargs['remove_bb_label'],
                    kwargs['remove_bb_label_prob'])
            source_label = os.path.join(output_data_path,
                    '{}_label_remove_{}_bb_{}.tsv'.format(os.path.splitext(os.path.basename(source))[0],
                        kwargs['remove_bb_label'].replace(',','_'),
                        kwargs['remove_bb_label_prob']))
        else:
            desc = desc + '_rb_{}1'.format(kwargs['remove_bb_label'])
            source_label = os.path.join(output_data_path,
                    '{}_label_remove_{}_bb.tsv'.format(os.path.splitext(os.path.basename(source))[0],
                        kwargs['remove_bb_label'].replace(',','_')))
        if not os.path.isfile(source_label):
            remove_bb(source, source_label, **kwargs)

    source_shuffle = None
    if 'remove_image' in kwargs and kwargs['remove_image']:
        desc = desc + '_ri_{}'.format(kwargs['remove_image'].replace(',', '_'))
        if 'remove_image_prob' in kwargs and kwargs['remove_image_prob']:
            desc = desc + '_{}'.format(kwargs['remove_image_prob'])
        if kwargs['remove_image'] == 'all' and \
                ('remove_image_prob' not in kwargs or \
                kwargs['remove_image_prob'] == 1):
            source = None
            source_label = None
        else:
            if 'remove_image_prob' in kwargs and kwargs['remove_image_prob']:
                source_shuffle = os.path.join(output_data_path,
                        '{}_image_remove_{}_{}.shuffle'.format(os.path.splitext(os.path.basename(source))[0],
                            kwargs['remove_image'].replace(',', '_'),
                            kwargs['remove_image_prob']))
            if not os.path.isfile(source_shuffle):
                remove_image(source, source_shuffle, **kwargs)

    if 'add_image_rule' in kwargs and kwargs['add_image_rule']:
        add_image_rule = kwargs['add_image_rule']
        desc = desc + '_ai_{}'.format(kwargs['add_image_rule'].replace(':',
            '_'))
        out_source = os.path.join(output_data_path,
                '{}{}.tsv'.format(
                    source_dataset._name,
                    desc))
        if not os.path.exists(out_source):
            source_labelmap = source_dataset.get_labelmap_file()
            merge_image(source, source_label, source_labelmap, add_image_rule, out_source)
        source = out_source
        source_label = None
        source_shuffle = randomize_tsv_file(out_source)

    return source, source_label, source_shuffle

def construct_low_shot(source_dataset, labels, num_train_images, shuffle_file):
    assert len(labels) == 1
    label_imageIdx = {}
    target_images = []
    rows = tsv_reader(source_dataset.get_train_tsv())
    shuffle = []
    for i, row in enumerate(rows):
        curr_labels = json.loads(row[1])
        if any([l['class'] == labels[0] for l in curr_labels]):
            target_images.append((i, row[0]))
        else:
            shuffle.append(i)
    assert len(target_images) >= num_train_images
    random.seed(777)
    random.shuffle(target_images)
    selected = target_images[:num_train_images]
    num_duplicate = (len(target_images) + num_train_images - 1) / num_train_images
    assert num_duplicate >= 1
    for d in range(num_duplicate):
        shuffle.extend([s[0] for s in selected])

    random.shuffle(shuffle)
    write_to_file('\n'.join(map(str, selected)), shuffle_file + '.selected')
    write_to_file('\n'.join(map(str, shuffle)), shuffle_file)

def process_dataset(source_dataset, output_root_data_path, dataset_ops,
        synset_tree):
    sources, source_labels = [], []
    source_shuffles, data_batch_weights = [], []
    labelmap = None
    if synset_tree:
        noffsets = synset_tree.noffsets
        labelmap = op.join(output_root_data_path,
            op.basename(synset_tree.basefilename))
        write_to_file('\n'.join(noffsets), labelmap)
    else:
        synset_tree = None
        labelmap = source_dataset.get_labelmap_file()

    source = source_dataset.get_train_tsvs()
    sources.append(source)
    train_label_files = source_dataset.get_train_tsvs('label')
    if all(op.isfile(f) for f in train_label_files):
        source_labels.append(train_label_files)
    shuffle_file = source_dataset.get_train_shuffle_file()
    if op.isfile(shuffle_file):
        source_shuffles.append(shuffle_file)

    for i, operator in enumerate(dataset_ops):
        if operator['op'] == 'remove':
            # remove the current dataset. That is, we don't want to use the
            # current dataset at all
            sources = []
            source_labels = []
            data_batch_weights = []
            source_shuffles = []
        elif operator['op'] == 'select_top':
            # dataset_ops = [{'op':'select_top', 'num_top': 1}]
            # only select the first few training images. This is used to verify
            # if the training loss can be reduced if we only have a few
            # training samples.
            num_top = operator['num_top']
            shuffle_file = op.join(output_root_data_path, source_dataset.name,
                    'train_shuffle{}.txt'.format(num_top))
            write_to_file('\n'.join(map(str, range(num_top))), shuffle_file)
            assert len(sources) == 1
            assert len(source_shuffles) == 0
            source_shuffles.append(shuffle_file)
        elif operator['op'] == 'add':
            # add a new dataset.
            if 'path' not in operator:
                extra_dataset = TSVDataset(operator['name'])
                if operator['source'] == 'train':
                    curr_source = extra_dataset.get_train_tsvs()
                    curr_labels = extra_dataset.get_train_tsvs('label')
                elif operator['source'] == 'trainval':
                    raise NotImplemented
                    curr_source = extra_dataset.get_trainval_tsv()
                else:
                    assert False
                #curr_labelmap = extra_dataset.load_labelmap()
                if op.isfile(extra_dataset.get_train_shuffle_file()):
                    source_shuffles.append(extra_dataset.get_train_shuffle_file())
                sources.append(curr_source)
                if len(curr_labels) == len(curr_source):
                    source_labels.append(curr_labels)
                data_batch_weights.append(operator['weight'])
            else:
                assert False
                curr_source = op.join(operator['path'], operator['source'] + '.tsv')
                #curr_labelmap = load_list_file(op.join(operator['path'],
                    #'labelmap.txt'))
            #source_label = op.join(output_root_data_path, operator['name'],
                    #'{}_{}_{}'.format(basename_no_ext(synset_tree.basefilename),
                        #basename_no_ext(curr_source), 'label.tsv'))
            #if not op.isfile(source_label):
                #map_label(curr_source,
                        #curr_labelmap,
                        #synset_tree,
                        #source_label)
        elif operator['op'] == 'sample':
            # randomly sample the training data by a probability for all
            assert len(dataset_ops) == 1
            sample_label = operator['sample_label']
            sample_image = operator['sample_image']
            out_folder = op.join(output_root_data_path,
                    '{}_{}_{}'.format(source_dataset.name,
                        operator['sample_label'],
                        operator['sample_image']))
            ensure_dataset_sample(source_dataset, sample_label, sample_image, out_folder)
            sources = [op.join(out_folder, 'train.tsv')]
            labelmap = op.join(out_folder, 'labelmap.txt')
        elif operator['op'] == 'low_shot':
            assert len(dataset_ops) == 1
            labels = operator['labels'].split(',')
            num_train = operator['num_train']
            shuffle_file = op.join(output_root_data_path, source_dataset.name,
                    'low_shot_shuffle_{}_{}.txt'.format('_'.join(labels),
                        num_train))
            source_shuffles.append(shuffle_file)
            if not op.isfile(shuffle_file):
                construct_low_shot(source_dataset, labels,
                        num_train, shuffle_file)
        elif operator['op'] == 'mask_background':
            assert i == 0
            sources = []
            new_label_idx = operator['new_label_idx']
            target_folder = op.join(output_root_data_path,
                    '{}_{}_{}'.format(source_dataset.name,
                        '.'.join(map(str, operator['old_label_idx'])),
                        new_label_idx))
            if not op.exists(target_folder):
                old_labels = source_dataset.load_labelmap()
                num_label = len(old_labels)
                old_new = create_mask_label_map(operator['old_label_idx'],
                        new_label_idx, num_label)
                mask_background(source_dataset.get_train_tsv(),
                        op.join(target_folder, 'train.tsv'),
                        old_new)
                mask_background(source_dataset.get_test_tsv_file(),
                        op.join(target_folder, 'test.tsv'),
                        old_new)
                new_labels = [None] * (num_label - len(operator['old_label_idx']) + 1)
                new_labels[new_label_idx] = 'background'
                for curr_old_label_idx, curr_new_label_idx in enumerate(old_new):
                    if new_labels[curr_new_label_idx] is None:
                        if curr_new_label_idx != new_label_idx:
                            new_labels[curr_new_label_idx] = old_labels[curr_old_label_idx]
                write_to_file('\n'.join(new_labels), op.join(target_folder,
                    'labelmap.txt'))
            sources.append(op.join(target_folder, 'train.tsv'))
        else:
            assert False
    return sources, source_labels, source_shuffles, data_batch_weights, labelmap

def create_mask_label_map(old_label_idx,
        new_label_idx, num_label):
    old_new = [None] * num_label
    idx = 0
    for i in range(num_label):
        if i in old_label_idx:
            old_new[i] = new_label_idx
        else:
            if idx == new_label_idx:
                idx = idx + 1
            old_new[i] = idx
            idx = idx + 1

    return old_new

def mask_background(tsv_file,
                    out_file,
                    old_new):

    def gen_rows():
        rows = tsv_reader(tsv_file)
        for row in rows:
            row[1] = str(old_new[int(float(row[1]))])
            yield row

    tsv_writer(gen_rows(), out_file)

def is_noffset_list(labels):
    has = False
    for label in labels:
        if re.match('^n[0-9]{8}$', label) == None:
            return False
        else:
            return True

def map_label(source, labels, synset_tree, source_label):
    assert synset_tree
    all_idx = []

    if is_noffset_list(labels):
        noffsets = labels
    else:
        mapper = LabelToSynset()
        noffsets = [synset_to_noffset(mapper.convert(l)) for l in labels]
        for noffset in noffsets:
            assert len(synset_tree.root.search_nodes(name=noffset)) == 1

    label_noffset = { label:noffset for label, noffset in zip(labels, noffsets)}
    rows = tsv_reader(source)
    def convert_label(rows, labels, label_noffset):
        is_cls_set = False
        for i, row in enumerate(rows):
            if row[1].isdigit():
                infos = [{'class': label_noffset[labels[int(row[1])]],
                    'rect': [0,0,0,0]}]
                is_cls_set = True
            else:
                assert not is_cls_set
                infos = json.loads(row[1])
                for info in infos:
                    info['class'] = label_noffset[info['class']]
            if (i % 1000) == 0:
                logging.info(i)
            yield row[0], json.dumps(infos)
    tsv_writer(convert_label(rows, labels, label_noffset), source_label)


from mpi4py import MPI
import numpy as np
import random

comm = MPI.COMM_WORLD
rank = comm.Get_rank()
numProcess = comm.Get_size()

clientsList = []
clientsPerRank = 3

if rank == 0:
    print('Entering the 0th processes')
    data = {'key1' : [1,2, 3], 'key2' : ( 'abc', 'xyz')}
    comm.send(data,dest=1)
elif rank==1:
    data = comm.recv(source=0)
    data1 = {'key1' : [1,2,3]}
    comm.send(data1,dest=2)
elif rank==2:
    data = comm.recv(source=1)
else:
    data = {'key3' : [1],'key4' : ('acd', 'aba')}

print(rank,data)

MPI.Finalize()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/3/10 23:45
# @Author  : lcyanxi
# @Email    : lcyanxi@163.com
# @File    : stock.py
# @Software: PyCharm
# 东方财富网股票的信息和百度股票个体信息
import requests
from bs4 import BeautifulSoup
import re
import traceback

def getHtmlText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        print("获取页面出错了")

def getStcokList(lst, stcokUrl):
    html = getHtmlText(stcokUrl)
    soup = BeautifulSoup(html, "html.parser")
    a = soup.find_all('a')
    for i in a:
        try:
            herf = i.attrs['href']
            lst.append(re.findall(r'[s][hz]\d{6}', herf)[0])
            print("获取股票列表完成")
        except:
            print("获取股票列表出错")
            continue

def getStcokInfo(lst, stockInfoUrl, fpath):
    for stock in lst:
        url = stockInfoUrl + stock + ".html"
        print(url)
        html = getHtmlText(url)
        try:
            if html == "":
                continue
                infoDict = {}
                soup = BeautifulSoup(html, "html.parser")
                stockinfo = soup.find('div', attrs={"class": "stock-bet"})
                name = stockinfo.find_all(attrs={"class": "bets-name"})[0]
                infoDict.update({"股票名称": name.text.split()[0]})
                keyLst = stockinfo.find_all('dt')
                valueLst = stockinfo.find_all('dd')
                for i in range(len(keyLst)):
                    key = keyLst[i].text
                    value = valueLst[i].text
                    infoDict[key] = value

                with open(fpath, 'a', encoding='utf-8') as f:
                    f.write(str(infoDict) + '\n')
        except:
            traceback.print_exc()  # 获取错误信息
            print("获取股票信息出错")

if __name__ == '__main__':
    stock_list_url = "http://quote.eastmoney.com/stocklist.html"
    stock_info_url = "https://gupiao.baidu.com/stock/"
    fpath = "D:\myStock.txt"
    infoList = []
    getStcokList(infoList, stock_list_url)
    getStcokInfo(infoList, stock_info_url, fpath)


from itertools import combinations
from operator import mul
import functools

def main():
    with open('advent2015_day24.txt') as file:
        weights = [int(x) for x in file.read().splitlines()]
        group_size = sum(weights) / 4
        for i in range(len(weights)):
            list = [functools.reduce(mul, c) for c in combinations(weights, i)
                    if sum(c) == group_size]
            if list:
                return min(list)

if __name__ == "__main__":
    print(main())

2015/Day_24 - sleigh balancing/advent2015_day24.py


# pylint: disable=E1101
"""ReadVotingView module.
"""
from django.shortcuts import render

from vote.models import Voting
from vote.views.generic_vote_view import GenericVoteView

class ReadVotingView(GenericVoteView):
    """ReadVotingView class.
    """

    def __init__(self):
        super().__init__()
        self.view_template = 'vote/read_voting.html'
        self.context = {
            'voting': None,
            'voting_operation': None,
            'voting_result': None,
            'voting_status': None
        }
        self.msg_unauthenticated = "Authentification requise"

    def get(self, request, id_voting):
        """Detailed voting view method on client get request.
        """
        voting = Voting.objects.get(pk=id_voting)
        self.context = self.manager.set_context(
            self.context, voting, 'read'
        )
        return render(request, self.view_template, self.context)