"""
Created on Wed Sep 20 16:56:37 2017

TODO: Add npy, npz types for saving plain types
TODO: Use npy by default on numpy arrays

@author: alex
"""

import sys
import os
import os.path
from warnings import warn
import builtins
from pathlib import Path
import io
from collections import namedtuple, OrderedDict
import logging
import numpy as np
import dill
from inspect import isclass
from parameters import ParameterSet
logger = logging.getLogger('mackelab.iotools')

# TODO: Make dill an optional dependency, use pickle otherwise (with '.dill', '.pkl' extensions).
#       Do this by pulling out all the format-specific export code from `save`
#       into separate functions – then we can just define the fallback type as
#       either 'dill' or 'pkl' depending on whether `dill` could be loaded.
#       (c.f. comment in `save` starting with "Define the save functions below at top level of module […]")
# TODO: (Partly supersedes the one below) Remove any serialization code that
#       duplicates Pydantic functionality.

if sys.version_info.major >= 3 and sys.version_info.minor >= 6:
    PathTypes = (str, os.PathLike)
else:
    PathTypes = (str,)

Format = namedtuple("Format", ['ext', 'save', 'load', 'bytes'])
    # save: save function, `save(filename, data)`
    # load: load function, `load(filename)`
    # bytes: whether to use the 'b' flag when opening a file for loading/saving
    # TODO: Allow multiple extensions per format
    # TODO: Use namedtuple defaults statt None once Python3.7 is standard
defined_formats = OrderedDict([
    # List of formats known to the load/save functions
    # The order of these formats also defines a preference, for when two types might be used
    ('npr',  Format('npr', None, None, True)),
    ('repr', Format('repr', None, None, False)),
    ('brepr', Format('brepr', None, None, True)), # Binary version of 'repr'
    ('dill', Format('dill', None, None, True))
    ])

_load_types = {}
_format_types = {}

def register_datatype(type, typename=None, format=None, alias=None):
    """
    :param alias: alternative name, usually a shortened form of the full typename.
    """
    global _load_types, _format_types
    assert(isclass(type))
    if typename is None:
        typename = type.__module__ + '.' + type.__qualname__
    if alias is None:
        alias = type.__qualname__
        if alias in _load_types:  # Don't let default override existing alias
            alias = None
    assert(isinstance(typename, str))
    _load_types[typename] = type
    if alias is not None:
        _load_types[alias] = type
    if format is not None:
        assert isinstance(format, str)
        _format_types[typename] = format

def find_registered_typename(type):
    """
    If `type` is a registered datatype, return it's associated name.
    Otherwise, find the nearest parent of `type` which is registered and return its name.
    If no parents are registered, return `type.__name__`.
    """
    def get_name(type):
        # _load_types is always a superset of _format_types
        for registered_name, registered_type in _load_types.items():
            if registered_type is type:
                return registered_name
        return None

    typename = get_name(type)
    if typename is None:
        if not isinstance(type, builtins.type):
            type = builtins.type(type)
        for base in type.mro():
            typename = get_name(base)
            if typename is not None:
                break
    if typename is None:
        typename = type.__name__
            # No registered type; return something sensible (i.e. the type's name)
    return typename

def get_format_from_ext(ext):
    format = None
    ext = ext.strip('.')
    for name, info in defined_formats.items():
        exts = [info.ext] if isinstance(info.ext, str) else info.ext
        for e in exts:
            if e.strip('.') == ext:
                format = name
                break
        if format is not None:
            break
    if format is None:
        raise ValueError("No format matching extension {} was found."
                         .format(ext))
    return format

# Register basic data types
# Python structures can be arbitrarily nested, so we can't assume that they
# would compatible with any strict format
register_datatype(list, format='dill')
register_datatype(tuple, format='dill')
register_datatype(set, format='dill')
register_datatype(dict, format='dill')

def get_free_file(path, bytes=True, max_files=100, force_suffix=False, start_suffix=None):
    """
    Return a file handle to an unused filename. If 'path' is free, return a handle
    to that. Otherwise, append a number to it until a free filename is found or the
    number exceeds 'max_files'. In the latter case, raise 'IOError'.

    Returning a file handle, rather than just a file name, avoids the possibility of a
    race condition (a new file of the same name could be created between the time
    where one finds a free filename and then opens the file).

    Parameters
    ----------
    path: str
        Path name. Can be absolute or relative to the current directory.
    bytes: bool (Default: True)
        (Optional) Specify whether to open the file for byte (True) or plain
        text (False) output. Default is to open for byte output, which
        is suitable for passing to `numpy.save`.
    max_files: int
        (Optional) Maximum allowed number of files with the same name. If this
        number is exceeded, IOError is raised.
    force_suffix: bool (default False)
        (Optional) If True, a suffix '_#', where # is a number, is always added
        to the file name. Forcing suffixes also changes the default value
        of 'start_suffix' to 1.
    start_suffix: int (default 2)
        If creating a file with 'path' is unsuccessful (or 'force_suffix is
        set to True), this is the first number to try appending to the file name.

    Returns
    -------
    filehandle
        Write-only filehandle, as obtained from a call to
        `open(pathname, 'mode='xb')`.
    pathname: str
        Pathname (including the possibly appended number) of the opened file.
    """

    # Get a full path
    # TODO: is cwd always what we want here ?
    if isinstance(path, Path):
        pathname = str(path.absolute())
    elif path[0] == '/':
        #path is already a full path name
        pathname = path
    else:
        #Make a full path from path
        pathname = os.path.abspath(path)

    # Set the default value for start_suffix
    if start_suffix is None:
        start_suffix = 1 if force_suffix else 2

    # Set the mode
    if bytes:
        mode = 'xb'
    else:
        mode = 'x'

    # Make sure the directory exists
    os.makedirs(os.path.dirname(pathname), exist_ok=True)

    try:
        if force_suffix:
            raise IOError
        else:
            f = open(pathname, mode=mode)
            return f, pathname
    except IOError:
        name, ext = os.path.splitext(pathname)
        for i in range(start_suffix, max_files+start_suffix):
            appendedname = name + "_" + str(i) + ext
            try:
                f = open(appendedname, mode=mode)
                return f, appendedname
            except IOError:
                continue

        raise IOError("Number of files with the name '{}' has exceeded limit."
                      .format(path))

def save(file, data, format=None, overwrite=False):
    """Save `data`.
    First the function checks if :param:data defines a `save()` method; if so,
    the method is called as `save(output_path)`. If this is successful, the
    function terminates.
    If the call is not successful, or :param:data does not define a `save()`
    method, then the function attempts to save to the formats defined by
    `format`. By default, only the 'numpy_repr' representation is saved,
    if `data` defines a numpy representation.
    Not only is the numpy representation format more future-proof, it can be an
    order of magnitude more compact.
    If the numpy_repr save is unsuccessful (possibly because `data` does not provide a
    `numpy_repr` method), then `save()` falls back to saving a plain (dill) pickle of 'data'.

    Parameters
    ----------
    file: str
        Path name or file object. Note that the file extension is mostly
        ignored and will be replaced by the one associated with the format.
        This is to allow saving to multiple formats.
    data: Python object
        Data to save
    format: str
        The format in which to save the data. Possible values are:
          - 'npr' (default) Save with the numpy_repr format. This is obtained by calling the
            method 'nprepr' on the `data`. If this call fails, a warning is issued
            and the 'dill' format is used.
            Output file have the extension 'npr'.
            Objects using this format should implement the `from_nprepr` method.
          - 'repr' Call `repr` on the data and save the resulting string to file. The save will
            fail (and fall back to 'dill' format) if the `repr` is simply inherited from object,
            as simply saving the object address is not useful for reconstructing it. Still, there
            is no way of ensuring that the `repr` is sufficiently informative to reconstruct the
            object, so make sure it is before using this format.
            Output file have the extension 'repr'.
            Objects using this format should implement the `from_repr` method.
          - 'dill' A dill pickle.
            Output file has the extension 'dill'
        Formats can also be combined as e.g. 'npr+dill'.
    overwrite: bool
        If True, allow overwriting previously saved files. Default is false, in
        which case a number is appended to the filename to make it unique.

    Returns
    -------
    List of output paths.
        List because many formats may be specified, leading to multiple outputs.
    """
    if isinstance(format, str):
        selected_formats = format
    else:
        if format is None:
            typename = find_registered_typename(type(data))
        else:
            if not isinstance(format, type):
                logger.error("The `format` argument should be either a string "
                             "or type. Provided value: {}"
                             "Attempting to infer type from data".format(format))
                typename = find_registered_typename(type(data))
            typename = find_registered_typename(format)
        if typename in _format_types:
            format = _format_types[typename]
        else:
            logger.error("Type '{}' has no associated format".format(typename))
            format = 'npr'

    selected_formats = set(format.split('+'))

    # Check argument - format
    bad_formats = [f for f in selected_formats if f not in defined_formats]
    selected_formats = selected_formats.difference(bad_formats)
    if len(bad_formats) > 0:
        format_names = ["'" + f + "'" for f in defined_formats]
        bad_format_names = ["'" + f + "'" for f in bad_formats]
        formatstr = "format"
        if len(format_names) > 1:
            format_names = ", ".join(format_names[:-1]) + " and " + format_names[-1]
        if len(bad_format_names) > 1:
            formatstr = "formats"
            bad_format_names = ", ".join(bad_format_names[:-1]) + " and " + bad_format_names[-1]
        logger.warning("Unrecognized save {} {}.".format(formatstr, bad_format_names)
                       + "Recognized formats are " + format_names)
        if len(selected_formats) == 0:
            logger.warning("Setting the format to {}.".format_names)
            # We don't want to throw away the result of a long calculation because of a
            # flag error, so instead we will try to save into every format and let the user
            # sort out the files later.
            format = '+'.join(format_names)

    get_output = None
    def set_str_file(filename):
        nonlocal get_output
        def _get_output(filename, ext, bytes, overwrite):
            return output(filename, ext, bytes, overwrite)
        get_output = _get_output

    # Check argument - file
    if isinstance(file, io.IOBase):
        thisfilename = os.path.realpath(file.name)
        if 'luigi' in os.path.basename(thisfilename):
            # 'file' is actually a Luigi temporary file
            luigi = True
        else:
            luigi = False
        filename = thisfilename  # thisfilename used to avoid name clashes
        if not any(c in file.mode for c in ['w', 'x', 'a', '+']):
            logger.warning("File {} not open for writing; closing and reopening.")
            file.close()
            set_str_file(thisfilename)
        else:
            def _get_output(filename, ext, bytes, overwrite):
                # Check that the file object is compatible with the arguments,
                # and if succesful, just return the file object unmodified.
                # If it is not successful, revert to opening a file as though
                # a filename was passed to `save`.
                # TODO: Put checks in `dummy_file_context`
                fail = False
                if (os.path.splitext(os.path.realpath(filename))[0]
                    != os.path.splitext(os.path.realpath(thisfilename))[0]):
                    logger.warning("[iotools.save] Given filename and file object differ.")
                    fail = True
                thisext = os.path.splitext(thisfilename)[1].strip('.')
                if not luigi and thisext != ext.strip('.'):
                    # Luigi adds 'luigi' to extensions of temporary files; we
                    # don't want that to trigger closing the file
                    logger.warning("[iotools.save] File object has wrong extension.")
                    fail = True
                if (bytes and 'b' not in file.mode
                    or not bytes and 'b' in file.mode):
                    if luigi:
                        # Luigi's LocalTarget always saves to bytes, and it's
                        # the Format class that takes care of converting data
                        # (possibly text) to and back from bytes.
                        logger.warning("\n"
                            "WARNING [iotools]: Attempted to save a 'luigi' target with the wrong "
                            "mode (binary or text). Note that Luigi targets "
                            "always use the same mode internally; use the "
                            "`format` argument to convert to/from in your code. "
                            "In particular, LocalTarget writes in binary. "
                            "Consequently, the file will not be saved as {}, "
                            "but as {}; specify the correct value to `bytes` "
                            "to avoid this message.\n"
                            .format("bytes" if bytes else "text",
                                    "text" if bytes else "bytes"))
                    else:
                        logger.warning("[iotools.save] File object has incorrect byte mode.")
                        fail = True
                if (overwrite and 'a' in file.mode):
                    # Don't check for `not overwrite`: in that case the damage is already done
                    logger.warning("[iotools.save] File object unable to overwrite.")
                    fail = True
                if fail:
                    logger.warning("[iotools.save] Closing and reopening file object.")
                    file.close()
                    set_str_file(thisfilename)
                    return output(filename, ext, bytes, overwrite)
                else:
                    return dummy_file_context(file)
            get_output = _get_output
    else:
        assert isinstance(file, PathTypes)
        filename = file
        set_str_file(file)

    # Ensure target directory exists
    dirname = os.path.dirname(filename)
    if dirname != "":
        os.makedirs(dirname, exist_ok=True)

    output_paths = []

    # If data provides a "save" method, use that
    # This overrides the "format" argument – only exception is if save fails,
    # then we reset it to what it was and try the other formats
    if isinstance(data, ParameterSet):
        # Special case of data with `save` attribute
        _selected_formats_back = selected_formats
        selected_formats = []  # Don't save to another format if successful
        with get_output(filename, ext="", bytes=False, overwrite=overwrite) as (f, output_path):
            # Close the file since Parameters only accepts urls as filenames
            # FIXME: This introduces a race condition; should use `f` to save
            #        This would require fixing the parameters package to
            #        accept file objects in `save()`
            pass
        try:
            logger.info("Saving ParameterSet using its own `save` method...")
            data.save(output_path, expand_urls=True)
        except (AttributeError, PermissionError) as e:
            logger.warning("Calling the data's `save` method failed with '{}'."
                           .format(str(e)))
            selected_formats = _selected_formats_back
        else:
            output_paths.append(output_path)
    elif hasattr(data, 'save'):
        _selected_formats_back = selected_formats
        selected_formats = []  # Don't save to another format if successful
        # See if this type is in the registered formats, so we can get the
        # expected extension
        typename = find_registered_typename(data)
            # Always returns a type name: if none is found, returns that of data
        format = _format_types.get(typename, None)
        if format is None or format not in defined_formats:
            ext = ""
        else:
            ext = defined_formats[format].ext
        with get_output(filename, ext=ext, bytes=False, overwrite=overwrite) as (f, output_path):
            # TODO: Use `f` if possible, and only `output_path` if it fails.
            pass
        try:
            logger.info("Saving data using its own `save` method...")
            data.save(output_path)
        except (AttributeError, PermissionError) as e:
            logger.warning("Calling the data's `save` method failed with '{}'."
                           .format(str(e)))
            selected_formats = _selected_formats_back
        else:
            output_paths.append(output_path)

    # Save to all specified formats
    for name, formatinfo in defined_formats.items():
        if name in ('npr', 'repr', 'brepr', 'dill'):
            # TODO: Define the save functions below at top level of module
            # and treat these formats as any other
            #       Make sure 'dill' is still used as backup
            continue
        if name in selected_formats:
            if formatinfo.save is None:
                logger.error("Format '{}' does not define a save function"
                             .format(name))
                fail = True
            else:
                fail = False
                ext = formatinfo.ext
                try:
                    with get_output(filename, ext, formatinfo.bytes, overwrite) as (f, output_path):
                        formatinfo.save(f, data)
                except IOError:
                    fail = True
                except Exception as e:
                    logger.error("Silenced uncaught exception during saving process to attempt another format.")
                    logger.error("Silenced exception was: " + str(e))
                    fail = True
                else:
                    output_paths.append(output_path)
            if fail:
                try: os.remove(output_path)  # Ensure there are no leftover files
                except: pass
                logger.warning("Unable to save to {} format."
                               .format(name))
                if 'dill' not in selected_formats:
                    # Warn the user that we will use another format
                    logger.warning("Will try a plain (dill) pickle dump.")
                    selected_formats.add('dill')
    # Save data as numpy representation
    if 'npr' in selected_formats:
        fail = False
        ext = defined_formats['npr'].ext
        try:
            with get_output(filename, ext, True, overwrite) as (f, output_path):
                try:
                    logger.info("Saving data to 'npr' format...")
                    np.savez(f, **data.repr_np)
                except AttributeError:
                    fail = True
                else:
                    output_paths.append(output_path)
        except IOError:
            fail = True
        if fail:
            # TODO: Use custom error type
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save to numpy representation ('npr') format.")
            if 'dill' not in selected_formats:
                # Warn the user that we will use another format
                logger.warning("Will try a plain (dill) pickle dump.")
                selected_formats.add('dill')

    # Save data as representation string ('repr' or 'brepr')
    for format in [format
                   for format in selected_formats
                   if format in ('repr', 'brepr')]:
        bytes = (format == 'brepr')
        fail = False
        if data.__repr__ is object.__repr__:
            # Non-informative repr -- abort
            fail = True
        else:
            ext = defined_formats['repr'].ext
            try:
                with get_output(filename, ext=ext, bytes=bytes, overwrite=overwrite) as (f, output_path):
                    try:
                        logger.info("Saving data to plain-text 'repr' format'")
                        f.write(repr(data))
                    except:
                        fail = True
                    else:
                        output_paths.append(output_path)
            except IOError:
                fail = True
        if fail:
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save to numpy representation ('npr') format.")
            if 'dill' not in selected_formats:
                # Warn the user that we will use another format
                logger.warning("Will try a plain (dill) pickle dump.")
                selected_formats.add('dill')

    # Save data in dill format
    if 'dill' in selected_formats:
        ext = defined_formats['dill'].ext
        try:
            with get_output(filename, ext, True, overwrite) as (f, output_path):
                logger.info("Saving data as a dill pickle.")
                dill.dump(data, f)
                output_paths.append(output_path)
        except IOError:
            # There might be other things to save, so don't terminate
            # execution because this save failed
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save picke at location {}."
                           .format(output_path))

    # Return the list of output paths
    return [Path(path) for path in output_paths]

def find_file(file, format=None):
    """
    If `file` has no extension and `format` is None:
    Returns the list of paths matching the file. If the list has length one,
    a simple path is returned
    Otherwise, returns the file path with the specified format.
    In both cases, if no file is found, raises FileNotFoundError.

    Parameters
    ----------
    file: str | path-like
    format: str | type
        Should correspond to an entry in `defined_formats`.
    """
    if isinstance(file, PathTypes):
        basepath, ext = os.path.splitext(file)
        dirname, basename = os.path.split(basepath)
        if dirname == '':
            dirname = '.'

        # Normalize the format and see if it is recognized
        if isinstance(format, str):
            formatext = defined_formats.get(format, None).ext
        elif isinstance(format, type):
            def_format = defined_formats.get(find_registered_typename(format), None)
            if def_format is not None:
                formatext = def_format.ext
            else:
                warn("The type {} does not correspond to one of the types "
                     "registered with `mackelab_toolbox.iotools.register_datatype`"
                     .format(format))
                formatext = None
        else:
            formatext = None

        # Find the file
        if len(ext) > 0 and formatext is None:
            format = ext.strip('.')
        if len(ext) == 0 and formatext is None:
            # Try every file whose name without extension matches `file`
            match = lambda fname: os.path.splitext(fname)[0] == basename
            fnames = [name for name in os.listdir(dirname) if match(name)]
            # Order the file names so we try most likely formats first (i.e. npr, repr, dill)
            # We do not attempt to load other extensions, since we don't know the format
            ordered_fnames = []
            for formatinfo in defined_formats.values():
                name = basename + '.' + formatinfo.ext
                if name in fnames:
                    ordered_fnames.append(os.path.join(dirname, name))
            if len(ordered_fnames) == 0:
                # No file was found
                raise FileNotFoundError(
                    "No file with base name '{}' and a recognized extension was "
                    "found.\nDirectory searched: {}"
                    .format(basename, dirname))
            elif len(ordered_fnames) == 1:
                ordered_fnames = ordered_fnames[0]
            return ordered_fnames

        elif os.path.exists(file):
            return file
        else:
            return basepath + "." + formatext.strip('.')
    else:
        raise ValueError("File finding is only implemented for path-like "
                         "arguments.")

def load(file, types=None, load_function=None, format=None, input_format=None):
    """
    Load file at `file`. How the data is loaded is determined by the input format,
    which is inferred from the filename extension. It can also be given by `format`
    explicitly.
    If `load_function` is provided, it is be applied to the loaded data. Otherwise,
    we try to infer type from the loaded data.
      - For 'npr' data, we look for a 'type' entry.
      - For 'repr' data, we look at the first part of the returned string, before
        any whitespace or special characters.
    Type inference can be completely disabled by passing `load_function=False`.
    If a type is found, we then try to match it to one of the entries in `types`. Thus
    `types` should be a dictionary of str:Type pairs, where the keys match the type name
    stored in files, and the Type is a loaded Python class which is associated to that
    name; typically, the type name will be the class name, although that is not required.
    It is also possible to add to the list of default types by calling this module's
    `add_load_type` method.

    Parameters
    ----------
    file: str | file object (TODO)

    types: dict
        (Optional)
    load_function: function
        (Optional) Post-processing function, called on the result of the loaded
        data. I.e. does not override `type`, but provides a handle to process
        the result.
    format: str
        Specify the format of the data; overrides any deductions from the
        filename. Effectively this specifies the loading function, and thus
        should correspond to a key in `types`.
    input_format: str (DEPRECATED)
        Deprecated synonym for `format`.
    """
    global _load_types
    if types is not None:
        types = {**_load_types, **dict(types)}
    else:
        types = _load_types

    if format is None:
        format = input_format

    if isinstance(file, PathTypes):
        # _, ext = os.path.splitext(file)
        # if len(ext) > 0 and format is None:
        #     format = ext.strip('.')
        ordered_fnames = find_file(file, format)
        if isinstance(ordered_fnames, list):
            # Try to load every file name in sequence. Terminate after the first success.
            for fname in ordered_fnames:  # Empty list -> FileNotFound below
                try:
                    # Recursively call `load`
                    data = load(fname, types, load_function)
                except (FileNotFoundError):
                    # Basically only possible to reach here with a race condition, where
                    # file is deleted after having been listed
                    # TODO: Also catch loading errors ?
                    continue
                else:
                    return data
            # No file was found
            raise FileNotFoundError(
                "No file with base name '{}' and a recognized extension was "
                "found.\nDirectory searched: {}"
                .format(basename, dirname))

        elif os.path.exists(ordered_fnames):
            # Normal exit from `load` recursion
            openfilename = ordered_fnames
            basepath, ext = os.path.splitext(openfilename)
        else:
            # No file was found
            raise FileNotFoundError("No file '{} was found. (Specified "
                                    " format: '{}')".format(file, format))

    elif isinstance(file, io.IOBase):
        ext = os.path.splitext(file.name)[1]
    elif hasattr(file, 'fn'):
        # Treats Luigi LocalTarget
        ext = os.path.splitext(file.fn)[1]
    else:
        raise TypeError("[iotools.load] File '{}' is of unrecognized type '{}'."
                        .format(file, type(file)))

    if format is None:
        format = get_format_from_ext(ext[1:])
    if format not in defined_formats:
        warn("mackelab_toolbox.iotools.load: Unrecognized format `{}`. "
             "Ignoring `format` argument and using file extension instead."
             .format(format))
        format = get_format_from_ext(ext[1:])

    if format not in ('npr', 'repr', 'dill'):
        # TODO: Define following load functions separately, add them to
        # to defined_formats, and then treat these as any other format
        formatinfo = defined_formats[format]
        data = defined_formats[format].load(basepath+ext)
    elif format == 'npr':
        with wrapped_open(file, 'rb') as f:
            data = np.load(f)
                # np.load provides dict access to the file object;
                # but doesn't load it to memory. We must make sure it's in
                # memory before we exit `with` block.
            in_memory = False
            if load_function is False:
                pass
            elif load_function is not None:
                data = load_function(data)
                # Is it safe to assume that `load_function` always puts data in memory ?
                in_memory = True
            elif 'type' in data:
                # 'type' stored as a 0D array
                if (data['type'].ndim == 0
                    and data['type'].dtype.kind in {'S', 'U'}
                    and str(data['type']) in types) :
                    # make sure it's really 0D
                    cls = types[str(data['type'])]
                    if hasattr(cls, 'from_repr_np'):
                        data = cls.from_repr_np(data)
                        in_memory = True
                else:
                    # TODO: Error message
                    pass
            if not in_memory:
                # Since it's still not in memory, just load it as a dictionary.
                data = dict(data)

    elif format == 'repr':
        with wrapped_open(openfilename, 'r') as f:
            data = f.read()
        if load_function is False:
            pass
        elif load_function is not None:
            data = load_function(data)
        else:
            # Search for the type name in the initial data
            if data[0] == '<':
                i, j = data.index('>'), data.index('<')
                if i == -1 or (j > -1 and j < i):
                    # Misformed type specification
                    pass
                else:
                    test_clsname = data[1:i]
                    if test_clsname in types:
                        cls = types[test_clsname]
                        if hasattr(cls, 'from_repr'):
                            data = cls.from_repr(data)
    elif format == 'dill':
        with wrapped_open(openfilename, 'rb') as f:
            try:
                data = dill.load(f)
            except EOFError:
                logger.warning("File {} is corrupted or empty. You should "
                               "delete it to prevent confusion.".format(file))
                raise FileNotFoundError

    return data

class output():
    def __init__(self, path, ext, bytes, overwrite=False):

        # Add extension
        basepath, _ = os.path.splitext(path)
            # Remove possible extension from path
        if len(ext) > 0 and ext[0] != ".":
            ext = "." + ext
        self.output_path = basepath + ext
        self.orig_output_path = self.output_path
        self.overwrite = overwrite
        self.bytes = bytes

    def __enter__(self):
        # Open file
        try:
            if not self.overwrite:
                self.f, self.actual_path = get_free_file(self.output_path, bytes=self.bytes)
            else:
                mode = 'wb' if self.bytes else 'w'
                self.f = open(self.orig_output_path, mode)
                self.actual_path = self.orig_output_path
        except IOError:
            logger.error("Could not create a file at '{}'."
                         .format(self.orig_output_path))
            raise

        return self.f, self.actual_path

    def __exit__(self, type, value, traceback):
        self.f.close()

class dummy_file_context:
    def __init__(self, file):
        self.f = file

    def __enter__(self):
        return self.f, os.path.realpath(self.f.name)

    def __exit__(self, type, value, traceback):
        # Since the file was not created in this context, don't close it
        pass

class wrapped_open:
    # Acts like `open`, but if given a file object, just returns the object.
    def __init__(self, file, mode):
        self.file = file
        self.mode = mode
        self.exit = None

    def __enter__(self):
        if isinstance(self.file, io.IOBase):
            if self.file.mode != mode:
                logger.warning("Already open file object has mode '{}', but "
                               " '{}' is expected."
                               .format(self.file.mode, self.mode))
            self.f = self.file
            self.exit = False  # Don't close a file we didn't open
        elif hasattr(self.file, 'open'):
            # Treats Luigi LocalTarget
            self.f = self.file.open(self.mode)
            self.exit = True
        else:
            self.f = open(self.file, self.mode)
            self.exit = True
        return self .f

    def __exit__(self, type, value, traceback):
        if self.exit is None:
            logger.warning("ERROR [iotools.wrapped_open]: `self.exit()` was "
                           "never set.")
        if self.exit:
            self.f.__exit__()


#!/usr/bin/env python3

"""
Helper script for printing out the similarity breakdown between two patients
"""

__author__ = 'Orion Buske'

import os
import sys
import logging

from math import log, exp
from collections import defaultdict

from disease import Diseases
from orphanet import Orphanet
from patient_similarity import PatientComparator, Patient

def script(patient1, patient2, patient_hpo_filename, hpo_filename,
           disease_phenotype_filename,
           orphanet_lookup_filename, orphanet_prevalence_filename,
           **kwargs):
    hpo = HPO(hpo_filename, new_root='HP:0000118')
    patients = dict([(patient.id, patient)
                     for patient in Patient.iter_from_file(patient_hpo_filename, hpo)
                     if patient.hp_terms])

    p1 = patients[patient1]
    p2 = patients[patient2]

    diseases = Diseases(disease_phenotype_filename)
    orphanet = Orphanet(orphanet_prevalence_filename, lookup_filename=orphanet_lookup_filename)
    comparator = PatientComparator(hpo, diseases, orphanet)

    clusters = comparator.similarity_breakdown(p1, p2)
    cluster_strs = []
    for cluster in clusters:
        score, root, p1_terms, p2_terms = cluster
        if p1_terms:
            p1_terms = '\n'.join(['\t{}: {}'.format(t.id, t.name) for t in p1_terms])
        else:
            p1_terms = '\tNo matching terms'

        if p2_terms:
            p2_terms = '\n'.join(['\t{}: {}'.format(t.id, t.name) for t in p2_terms])
        else:
            p2_terms = '\tNo matching terms'

        if root:
            root = '{}: {} ({:.2f})'.format(root.id, root.name, score)
        else:
            root = 'Unmatched terms'

        cluster_strs.append('{}\n\t-- {} --\n{}\n\t-- {} --\n{}'.format(root, patient1, p1_terms, patient2, p2_terms))

    print('\n\n'.join(cluster_strs))

def parse_args(args):
    from argparse import ArgumentParser
    description = __doc__.strip()

    parser = ArgumentParser(description=description)
    parser.add_argument('patient1')
    parser.add_argument('patient2')
    parser.add_argument('patient_hpo_filename', metavar='patients.hpo')
    parser.add_argument('hpo_filename', metavar='hp.obo')
    parser.add_argument('disease_phenotype_filename', metavar='phenotype_annotations.tab')
    parser.add_argument('orphanet_lookup_filename', metavar='orphanet_lookup')
    parser.add_argument('orphanet_prevalence_filename', metavar='orphanet_prevalence')
    parser.add_argument('--log', dest='loglevel', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='WARNING')

    return parser.parse_args(args)

def main(args=sys.argv[1:]):
    args = parse_args(args)
    logging.basicConfig(level=args.loglevel)

    script(**vars(args))

if __name__ == '__main__':
    sys.exit(main())


import os
import smtplib
import secrets
import string
import pickle

#this is the function which send the OTP code to given email
def send_email(otp,email):
	msg = "\r\n".join([
			"From: SENDERGMAIL@gmail.com",
			"To: "+email,
			"Subject: OTP code : ",
			"",
			"OTP CODE : "+otp])

	server = smtplib.SMTP('smtp.gmail.com:587')
	server.ehlo()
	server.starttls()
	server.ehlo()
	server.login("SENDERGMAIL@gmail.com", "YOURPASSWORD")
	server.sendmail("SENDERGMAIL@gmail.com","RECEIVER@gmail.com", msg)

#this is the function which generate OTP code (default given length of OTP code is 7)
def generateOTP(len=7):
	alphabet = string.ascii_letters + string.digits
	return  ''.join(secrets.choice(alphabet) for i in range(len))

#cashing OTP until validate or expire(TOTP)
def secretsStore(otp,email):

	listOTP = dict()
	secretOTP = {
				"OTP": otp,
				"validity": False
				}

	with open('secret.pickle', 'rb') as secretRead:
		listOTP = pickle.load(secretRead)

	listOTP[email] = secretOTP

	with open('secret.pickle', 'wb') as secretWrite:
		pickle.dump(listOTP, secretWrite, protocol=pickle.HIGHEST_PROTOCOL)

#validate
def secretsCheck(otp):

	listOTP = dict()

	with open('secret.pickle', 'rb') as secretRead:
		listOTP = pickle.load(secretRead)

	for emailKey in listOTP:
		if listOTP[emailKey]["OTP"] == otp and not listOTP[emailKey]["validity"]:
			print("Access Granted")
			listOTP[emailKey]["validity"] = True
			break
		if 	listOTP[emailKey]["OTP"] == otp and listOTP[emailKey]["validity"]:
			print("OTP has already used")
			break
		else:
			print("Access Denied")

	with open('secret.pickle', 'wb') as secretWrite:
		pickle.dump(listOTP, secretWrite, protocol=pickle.HIGHEST_PROTOCOL)

#main function
def main():
	print("welcome to OTP generator")
	email = input("Enter Your Email : ").strip()
	OTP = generateOTP()
	try:
		print("Sending Email ...... ")
		send_email(OTP,email)
		print("OTP has sent to the {} email right now".format(email))
		secretsStore(OTP,email)
		print("~~~~~~~~~~~~~~~~~~~~~~")
		check = input("Enter Your OTP code : ").strip()
		secretsCheck(check)
	except:
		print("Error")

#excute programe
if __name__ == '__main__':
	if not os.path.isfile('secret.pickle'):
		with open('secret.pickle', 'wb') as secretWrite:
			pickle.dump(dict(), secretWrite, protocol=pickle.HIGHEST_PROTOCOL)
	main()


from solution import print_depth, Person
import unittest
from unittest import mock
import io
import sys

class TestSolution(unittest.TestCase):
    """
    Test the solution for problem 1
    """

    @unittest.mock.patch('sys.stdout', new_callable=io.StringIO)
    def assert_stdout(self, input_val, expected_output, mock_stdout):
        print_depth(input_val)
        self.assertEqual(mock_stdout.getvalue(), expected_output)

    def test_dictionary_with_object(self):
        person_a = Person("User", "1", None)
        person_b = Person("User", "2", person_a)

        a = {
            "key1": 1,
            "key2": {
                "key3": 1,
                "key4": {
                    "key5": 4,
                    "user": person_b
                }
            }
        }
        output = "key1 : 1\nkey2 : 1\nkey3 : 2\nkey4 : 2\nkey5 : 3\nuser : 3\nfather : 4\nfirst_name : 4\nlast_name : 4\nfather : 5\nfirst_name : 5\nlast_name : 5\n"
        self.assert_stdout(a, output)

if __name__ == '__main__':
    unittest.main()


"""create Post model

Revision ID: 5796a5eb7b81
Revises: fef0b6e1d1a0
Create Date: 2019-01-23 16:00:53.299784

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '5796a5eb7b81'
down_revision = 'fef0b6e1d1a0'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('post',
                    sa.Column('id', sa.Integer(), nullable=False),
                    sa.Column('writer_id', sa.Integer(), nullable=False),
                    sa.Column('board_id', sa.Integer(), nullable=False),
                    sa.Column('title', sa.String(length=240), nullable=False),
                    sa.Column('body', sa.Text(), nullable=True),
                    sa.Column('is_published', sa.Boolean(), nullable=False, default=True),
                    sa.Column('like_count', sa.Integer(), nullable=False),
                    sa.Column('view_count', sa.Integer(), nullable=False),
                    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
                    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
                    sa.ForeignKeyConstraint(['board_id'], ['board.id'], ),
                    sa.ForeignKeyConstraint(['writer_id'], ['user.id'], ),
                    sa.PrimaryKeyConstraint('id')
                    )
    op.alter_column('post', 'is_published',
                    existing_type=mysql.TINYINT(display_width=1),
                    type_=sa.Boolean(),
                    nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('post')
    # ### end Alembic commands ###


#  Created by Bogdan Trif on 09-10-2017 , 9:56 PM.

from pylab import *
import itertools
import random

Colors = ['mediumturquoise',  'darkslategrey', 'springgreen', 'saddlebrown', 'olivedrab', 'wheat', 'brown',
          'chocolate', 'lightslategrey', 'palegreen', 'darkorange', 'darkorchid', 'hotpink', 'yellowgreen', 'silver', 'palevioletred',
          'oldlace', 'navy', 'peachpuff', 'orangered', 'mediumorchid', 'burlywood', 'tan', 'forestgreen', 'yellow', 'darksage',
          'darkslategray', 'bisque', 'indigo', 'lightsteelblue', 'aquamarine', 'blue', 'lightgrey', 'khaki', 'slategrey', 'darkred', 'lime', 'peru',
          'darkmagenta', 'midnightblue', 'lightslategray', 'darkgrey', 'lightcoral', 'teal', 'lightsage', 'deepskyblue', 'limegreen', 'skyblue',
          'aqua', 'mediumpurple', 'dodgerblue', 'plum', 'indianred', 'mediumslateblue', 'darkturquoise', 'violet',
          'darkgoldenrod', 'thistle', 'lawngreen', 'palegoldenrod', 'lightsalmon', 'lightpink', 'coral', 'rosybrown', 'goldenrod', 'lemonchiffon',
          'mediumseagreen',  'mediumblue', 'crimson', 'darksalmon', 'cornflowerblue', 'red', 'magenta', 'mistyrose',
          'mediumspringgreen', 'green', 'lightblue', 'lightgreen', 'darkgreen', 'purple', 'royalblue', 'sandybrown',
          'gray', 'blanchedalmond', 'turquoise', 'paleturquoise', 'blueviolet', 'darkslateblue', 'lightseagreen', 'orange', 'pink',
          'darkcyan',  'steelblue', 'gold', 'darkkhaki', 'black', 'papayawhip', 'lightskyblue', 'sienna', 'sage', 'cornsilk', 'dimgrey',
          'darkgray', 'deeppink', 'moccasin', 'chartreuse', 'fuchsia', 'darkviolet', 'honeydew', 'olive', 'mediumvioletred', 'powderblue',
          'lightgoldenrodyellow', 'grey', 'gainsboro', 'firebrick', 'slateblue', 'greenyellow', 'cadetblue', 'seagreen',
          'lavender', 'slategray', 'darkblue', 'dimgray', 'beige', 'mediumaquamarine', 'salmon',
          'cyan', 'orchid', 'darkseagreen', 'darkolivegreen', 'maroon', 'tomato', 'lightgray']

def grid(major,minor, x1, x2, y1, y2):
    # major ticks every 5, minor ticks every 1
    major_ticks = np.arange(x1, x2, major)
    minor_ticks = np.arange(y1, y2, minor)

    ax.set_xticks(major_ticks)
    ax.set_xticks(minor_ticks, minor=True)
    ax.set_yticks(major_ticks)
    ax.set_yticks(minor_ticks, minor=True)

    # and a corresponding grid
    # ax.grid(which='both')

    # or if you want differnet settings for the grids:
    # ax.grid(which='minor', alpha=0.2)
    ax.grid(which='minor', alpha=0.8)
    ax.grid(which='major', alpha=1)

fig = plt.figure(figsize=(11, 10))
ax = fig.add_subplot(1,1,1 ) #, aspect='equal')

ax.grid(which='both')
ax.axis('equal')

x1 , x2, y1, y2 = -5, 5, -5, 5
# ax.set_ylim([-x1, x2])
# ax.set_xlim([-y1, y2])
# ax.autoscale_view(True, True ,True)
plt.xlim([x1 , x2])
plt.ylim([y1, y2])

ax.axhline(y=0, color='k')
ax.axvline(x=0, color='k')

grid(1 , 5, x1, x2, y1, y2)

TRIANGL =   [((0, 3), (4, 0), (-3, -2)), ((0, 3), (4, 2), (-1, -4)), ((0, 4), (3, 2), (-4, -3)), ((0, 4), (4, 1), (-3, -1)), ((0, 4), (4, 3), (-1, -3)), ((1, 3), (4, 1), (-3, -4)), ((1, 4), (4, 2), (-3, -3))]

for triangle in TRIANGL :
    culoare = random.choice( Colors )
    plt.plot(*zip(*itertools.chain.from_iterable(itertools.combinations(triangle, 2)))  , color = culoare, marker = 'o',  linewidth=2.5 )

plt.ylabel('Triangle points')
plt.show()


import numpy as np
import tensorflow as tf
import argparse
import time, datetime
import os
import pickle
import sys

from utils import TextLoader
from model import Model

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type = str, default = 'data/reddit-data/reddit', help = 'data directory containing input.txt')
    parser.add_argument('--save_dir', type = str, default = 'model/new_save', help = 'directory for checkpointed models (load from here if one is already present)')
    parser.add_argument('--block_size', type = int, default = 2048, help = 'number of cells per block')
    parser.add_argument('--num_blocks', type = int, default = 3, help = 'number of blocks per layer')
    parser.add_argument('--num_layers', type = int, default = 3, help = 'number of layers')
    parser.add_argument('--model', type = str, default = 'gru', help = 'rnn, gru, lstm or nas')
    parser.add_argument('--batch_size', type = int, default = 40, help = 'minibatch size')
    parser.add_argument('--seq_length', type = int, default = 40, help = 'RNN sequence length')
    parser.add_argument('--num_epochs', type = int, default = 50, help = 'number of epochs')
    parser.add_argument('--save_early', type = int, default = 5000, help = 'save frequency')
    parser.add_argument('--grad_clip', type = float, default = 5., help = 'clip gradients at this value')
    parser.add_argument('--learning_rate', type = float, default = 1e-5, help = 'learning rate')
    parser.add_argument('--decay_rate', type = float, default = 0.975, help = 'how much to decay the learning rate')
    parser.add_argument('--decay_steps', type = int, default = 100000, help = 'how often to decay the learning rate')
    parser.add_argument('--set_learning_rate', type = float, default = -1, help = 'reset learning rate to this value (if greater than zero)')
    args = parser.parse_args()
    train(args)

def train(args):
    data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
    args.vocab_size = data_loader.vocab_size

    load_model = False
    if not os.path.exists(args.save_dir):
        print("Creating directory %s" % args.save_dir)
        os.mkdir(args.save_dir)
    elif (os.path.exists(os.path.join(args.save_dir, 'config.pkl'))):
        ckpt = tf.train.get_checkpoint_state(args.save_dir)
        if ckpt and ckpt.model_checkpoint_path:
            with open(os.path.join(args.save_dir, 'config.pkl'), 'rb') as f:
                saved_args = pickle.load(f)
                args.block_size = saved_args.block_size
                args.num_blocks = saved_args.num_blocks
                args.num_layers = saved_args.num_layers
                args.model = saved_args.model
                print("Found a previous checkpoint. Overwritin model description arguments to:")
                print(" model: {}, block_size: {}, num_blocks: {}, num_layers: {}".format(saved_args.model, saved_args.block_size, saved_args.num_blocks, saved_args.num_layers))
                load_model = True

    with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
        pickle.dump(args, f)

    with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'wb') as f:
        pickle.dump((data_loader.chars, data_loader.vocab), f)

    print("Building the model")
    model = Model(args)
    print("Total trainable parameters: {:,d}".format(model.trainable_parameter_count()))

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    config = tf.ConfigProto(log_device_placement = False)
    with tf.Session(config = config) as sess:
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(model.save_variables_list(), max_to_keep = 3)
        if (load_model):
            print("Loading saved parameters")
            saver.restore(sess, ckpt.model_checkpoint_path)
        global_epoch_fraction = sess.run(model.global_epoch_fraction)
        global_seconds_elapsed = sess.run(model.global_seconds_elapsed)
        if load_model:
            print("Resuming from global epoch fraction {:.3f}, total trained time: {}, learning rate: {}".format(global_epoch_fraction, datatime.timedelta(seconds = float(gloabal_seconds_elapsed)). sess.run(model.lr)))
        if args.set_learning_rate > 0:
            sess.run(tf.assign(model.lr, args.set_learning_rate))
            print("Reset learning rate to {}".format(args.set_learning_rate))

        data_loader.cue_batch_pointer_to_epoch_fraction(global_epoch_fraction)
        initial_batch_step = int((global_epoch_fraction - int(global_epoch_fraction)) * data_loader.total_batch_count)
        epoch_range = (int(global_epoch_fraction), args.num_epochs + int(global_epoch_fraction))
        writer = tf.summary.FileWriter(args.save_dir, graph = tf.get_default_graph())
        outputs = [model.cost, model.final_state, model.train_op, model.summary_op]
        global_step = epoch_range[0] * data_loader.total_batch_count + initial_batch_step
        avg_loss = 0
        avg_steps = 0

        try:
            for e in range(*epoch_range):
                state = sess.run(model.zero_state)
                batch_range = (initial_batch_step, data_loader.total_batch_count)
                initial_batch_step = 0
                for b in range(*batch_range):
                    global_step += 1
                    if global_step % args.decay_steps == 0:
                        current_learning_rate = sess.run(model.lr)
                        current_learning_rate *= args.decay_rate
                        sess.run(tf.assign(model.lr, current_learning_rate))
                        print("Decayed learning rate to {}".format(current_learning_rate))
                    start = time.time()

                    x, y = data_loader.next_batch()

                    feed = {model.input_data: x, model.targets: y}
                    model.add_state_to_feed_dict(feed, state)

                    train_loss, state, _, summary = sess.run(outputs, feed)
                    elapsed = time.time() - start
                    global_seconds_elapsed += elapsed
                    writer.add_summary(summary, e * batch_range[1] + b + 1)

                    if avg_steps < 100:
                        avg_steps += 1
                    avg_loss = 1 / avg_steps * train_loss + (1 - 1 / avg_steps) * avg_loss
                    print("{:, d} / {:, d} (epoch: {:.3f} / {}), loss {:.3f} (avg {:.3f}), {:.3f}s".format(b, batch_range[1], e + b / batch_range[1], epoch_range[1], train_loss, avg_loss, elapsed))

                    if (e * batch_range[1] + b + 1) % args.save_every == 0 or (e == epoch_range[1] - 1 and b == batch_range[1] - 1):
                        save_model(sess, saver, model, args.save_dir, global_step, data_loader.total_batch_count, global_seconds_elapsed)
        except KeyboardInterrupt:
            print()
        finally:
            writer.flush()
            global_step = e * data_loader.total_batch_count + b
            save_model(sess, saver, model, args.save_dir, global_step, data_loader.total_batch_count, global_seconds_elapsed)

def save_model(sess, saver, model, save_dir, global_step, steps_per_epoch, global_seconds_elapsed):
    global_epoch_fraction = float(global_step) / float(steps_per_epoch)
    checkpoint_path = os.path.join(save_dir, 'model.ckpt')
    print("Saving model to {} (epoch fraction {:.3f})...".format(checkpoint_path, global_epoch_fraction), end = '', flush = True)
    sess.run(tf.assign(model.global_epoch_fraction, global_epoch_fraction))
    saver.save(sess, checkpoint_path, global_step = global_step)
    print("\rSaved model to {} (epoch fraction {:.3f}).    ".format(checkpoint_path, global_epoch_fraction))

if __name__ == '__main__':
    main()


from nbconvert.preprocessors import Preprocessor

class WrapHtmlPreprocessor(Preprocessor):
    """
    Wraps html ouput from cells in the shortcode `rawhtml`.

    The `rawhtml` shortcode is a file which should be placed in `layouts/shortcodes/rawhtml.html` with this contents:

        <!-- raw html -->
        {{.Inner}}

    Hugo starting from version 0.60 (see discussion: https://discourse.gohugo.io/t/raw-html-getting-omitted-in-0-60-0/22032)
    doesn't render html tags by default, it needs the `unsafe = true` flag instead.

    However, it doesn't handle blocks of HTML very well even if `unsafe = true` in the config.toml and the approach of
    having a shortcode works better.
    """

    def preprocess_cell(self, cell, resources, index):
        outputs = cell.get("outputs", [])
        for output in outputs:
            data = output.get("data", {})
            if "text/html" in data:
                data["text/html"] = (
                    "{{< rawhtml >}}\n" + data["text/html"] + "\n{{< /rawhtml >}}"
                )
        return cell, resources


import sys
sys.path.append('..')
from common.core import *
from common.gfxutil import *
from common.audio import *
from common.mixer import *
from common.note import *
from common.wavegen import *
from common.wavesrc import *

from common.kivyparticle import ParticleSystem

from kivy.core.window import Window
from kivy.clock import Clock as kivyClock
from kivy.uix.label import Label
from kivy.graphics.instructions import InstructionGroup
from kivy.graphics import Color, Ellipse, Rectangle
from kivy.graphics import PushMatrix, PopMatrix, Translate, Scale, Rotate

from leap.LeapHelper import *

from random import random, randint, choice
import numpy as np

# part 1
class MainWidget(BaseWidget) :
    c_major_pitches = [72, 74, 76, 77, 79, 81, 83, 84]
    num_octaves = 3

    hsv_luminosity_factor = .3
    # hsv values in roy g biv in reverse order. last value refers to red for next octave
    roygbiv_hsv = [(0, 1, .41), (18./360, 1, .567), (50./360, 1, .565), (106./360, 1, .433), \
                    (229./360, .846, .433), (260./360, 1, .2), (285./360, 1, .133), (285./360, 1, .1)]
    # reverse standard roygbiv
    roygbiv_hsv = roygbiv_hsv[::-1]

    def __init__(self):
        super(MainWidget, self).__init__()
        self.info = topleft_label()
        self.add_widget(self.info)

        self.objects = AnimGroup()
        self.canvas.add(self.objects)

        # Audio framework init
        self.audio = Audio(2)
        self.note_duration = 1.0
        self.duration_step = 0.2

        # global mixer for all objects within the scope of this widget
        self.mixer = Mixer()
        self.audio.set_generator(self.mixer)

        self.leftHandBubble = HandBubble((20, 20), 20, Color(1, 0, 0))
        self.rightHandBubble = HandBubble((20, 20), 20, Color(1, 1, 0))
        self.centerMarker = HandBubble((Window.width / 2, Window.height / 2), 30, Color(0, 0, 1))
        self.objects.add(self.leftHandBubble); self.objects.add(self.rightHandBubble); self.objects.add(self.centerMarker)

        # Mock flames
        h = Window.height / 8
        w = Window.width / 12
        self.flame1 = Flame((2*w, h), 25, Color(1, .65, 0))
        self.flame2 = Flame((6*w, h), 25, Color(1, .65, 0))
        self.flame3 = Flame((10*w, h), 25, Color(1, .65, 0))
        self.objects.add(self.flame1); self.objects.add(self.flame2); self.objects.add(self.flame3)

        self.flames = [self.flame1, self.flame2, self.flame3]

        self.timbre = "sine"
        self.timbre_map = {"sine": NoteGenerator.sine, "square": NoteGenerator.square, "sawtooth": NoteGenerator.saw, \
                            "triangle": NoteGenerator.tri, "steelpan": NoteGenerator.square}

        self.segments = {"sine": 40, "square": 4, "sawtooth": 5, "triangle": 3, "steelpan": 10}

        self.leapController = Leap.Controller()
        self.leapController.set_policy(Leap.Controller.POLICY_IMAGES) # allow access to images

    def on_touch_down(self, touch) :
        # init Bubble settings
        pos = touch.pos

        # generate note and color
        (noteGen, color) = self.make_note(*pos)

        radius = 20 * noteGen.duration

        self.mixer.add(noteGen)

        self.objects.add(Bubble(pos, radius, color, self.mixer, self.segments[self.timbre], self.note_duration))

    def on_update(self):
        self.objects.on_update()
        self.audio.on_update()

        self.info.text = str(Window.mouse_pos)
        self.info.text += '\nfps:%d' % kivyClock.get_fps()
        self.info.text += '\nobjects:%d' % len(self.objects.objects)

        self.info.text += "\nNote duration: %.1fs [Toggle with up/down]" % self.note_duration
        self.info.text += "\nTimbre: %s [Toggle with s]" % self.timbre

        if self.leapController.is_connected:
            frame = self.leapController.frame()
            self.handleFrame(frame)

    def handleFrame(self, frame):
        # Remove bubbles; re-add later if appropriate hand is present
        self.objects.remove(self.leftHandBubble)
        self.objects.remove(self.rightHandBubble)
        self.leftHandBubble.isDrawn = False
        self.rightHandBubble.isDrawn = False

        hands = frame.hands

        for hand in hands:
            handBubble = self.leftHandBubble if hand.is_left else self.rightHandBubble
            if not handBubble.isDrawn:
                self.objects.add(handBubble)
                handBubble.isDrawn = True
            handPos = self.move_hand(hand, handBubble)

            if handBubble.has_flame:
                if hand.grab_strength < .1:
                    handBubble.release_flame()
            else:
                for flame in self.flames:
                    if LeapHelper.point_is_hovered(hand, flame.get_pos()) and handBubble.handOpen and hand.grab_strength == 1.0:
                        # TODO: in actual code, make sure flames know original pos, track grab
                        handBubble.grab_flame(flame)

    def move_hand(self, hand, handBubble):
        pos = LeapHelper.position_as_pixels(hand)

        def checkBounds(x, y):
            return 0 <= x and x <= Window.width and 0 <= y and y <= Window.height

        if not checkBounds(*pos):
            handBubble.release_flame()
            self.objects.remove(handBubble)
            handBubble.isDrawn = False
        else:
            if hand.is_left:
                self.leftHandBubble.set_pos(pos)
            else:
                self.rightHandBubble.set_pos(pos)
        return pos

    def on_key_down(self, keycode, modifiers):

        if keycode[1] == 's':
            self.toggle_timbre()
        if keycode[1] == "up":
            self.note_duration += self.duration_step
        if keycode[1] == "down" and self.note_duration >= 2 * self.duration_step: # Mult. by two to account for floating point (im)precision
            self.note_duration -= self.duration_step

    def make_note(self, x, y):

        # Chosoe pitch based on horizontal position
        pitchIndex = int( (1.0 * x) / Window.width * 8 )

        # Choose octave based on vertical position. num_octaves octaves are supported
        center_y = Window.height / 2.0
        band_size = 1.0 * Window.height / self.num_octaves

        octave_delta = 0
        if self.num_octaves % 2 != 0 and y < center_y:
            octave_delta = int((y - (center_y + band_size / 2)) / band_size)
        elif self.num_octaves % 2 != 0 and y >= center_y:
            octave_delta = int((y - (center_y - band_size / 2)) / band_size)
        else:
            octave_delta = int((y - center_y) / band_size)
        pitch_delta = octave_delta * 12

        pitch = self.c_major_pitches[pitchIndex] + pitch_delta
        gain = 1.0
        duration = self.note_duration

        hsv_val = (self.roygbiv_hsv[pitchIndex][0], self.roygbiv_hsv[pitchIndex][1], \
                    self.roygbiv_hsv[pitchIndex][2] * self.hsv_luminosity_factor ** -octave_delta)

        color = Color(*hsv_val, mode="hsv")

        timbre = self.timbre_map[self.timbre]

        return (NoteGenerator(pitch, gain, duration, harmonics = timbre), color)

    def toggle_timbre(self):
        keys_list = sorted(self.timbre_map.keys())
        i = keys_list.index(self.timbre)

        i += 1
        if i >= len(keys_list):
            i = 0
        self.timbre = keys_list[i]

class Bubble(InstructionGroup):
    def __init__(self, pos, r, color, mixer, segments = 40, duration=1.5):
        super(Bubble, self).__init__()

        center_x = Window.width/2
        center_y = Window.height/2

        # Duration of the bubble animation and its corresponding note
        self.duration = duration

        # pointer to a global mixer object
        self.mixer = mixer

        self.color = color
        self.add(self.color)

        self.shape = CEllipse(cpos = pos, size = (2*r, 2*r), segments = segments)
        self.add(self.shape)

        self.radius_anim = KFAnim((0, r), (.1, 2*r), (self.duration, 0))
        self.pos_anim = KFAnim((0, pos[0], pos[1]), (self.duration, pos[0], Window.height + r))

        self.time = 0
        self.on_update(0)

    def on_update(self, dt):
        # animate radius
        rad = self.radius_anim.eval(self.time)
        self.shape.csize = (2*rad, 2*rad)

        # animate position
        pos = self.pos_anim.eval(self.time)
        self.shape.cpos = pos

        # advance time
        self.time += dt
        # continue flag
        return self.radius_anim.is_active(self.time)

class HandBubble(InstructionGroup):
    def __init__(self, pos, r, color, segments = 40, duration=1.5):
        super(HandBubble, self).__init__()

        center_x = Window.width/2
        center_y = Window.height/2

        self.isDrawn = True # Is it currently visible on the canvas?

        self.pos = pos

        self.has_flame = False
        self.grabbed_flame = None
        self.handOpen = True

        # Duration of the bubble animation and its corresponding note
        self.duration = duration

        self.color = color
        self.add(self.color)

        self.shape = CEllipse(cpos = pos, size = (2*r, 2*r), segments = segments)
        self.add(self.shape)

        self.time = 0
        self.on_update(0)

    def on_update(self, dt):
        pass

    def set_pos(self, pos):
        self.pos = pos
        self.shape.set_cpos(pos)

        if self.has_flame:
            self.move_flame()

    def get_pos(self):
        return self.pos

    def move_flame(self):
        if self.has_flame:
            self.grabbed_flame.set_pos(self.pos)

    def grab_flame(self, flame):
        if not self.has_flame:
            self.grabbed_flame = flame
            self.has_flame = True
            self.grabbed_flame.set_pos(self.pos)
            self.handOpen = False

    def release_flame(self):
        if self.has_flame:
            self.grabbed_flame.reset_pos()
            self.has_flame = False
            self.grabbed_flame = None
            self.handOpen = True

class Flame(InstructionGroup):
    def __init__(self, pos, r, color, segments = 40, duration=1.5):
        super(Flame, self).__init__()

        center_x = Window.width/2
        center_y = Window.height/2

        self.pos = pos
        self.original_pos = pos

        # Duration of the bubble animation and its corresponding note
        self.duration = duration

        self.color = color
        self.add(self.color)

        self.shape = CEllipse(cpos = pos, size = (2*r, 2*r), segments = segments)
        self.add(self.shape)

        self.time = 0
        self.on_update(0)

    def on_update(self, dt):
        pass

    def set_pos(self, pos):
        self.pos = pos
        self.shape.set_cpos(pos)

    def reset_pos(self):
        self.pos = self.original_pos
        self.shape.set_cpos(self.pos)

    def get_pos(self):
        return self.pos

run(eval('MainWidget'))


from flask import Flask, jsonify
from .api.v0_1 import v0_1

application = Flask(__name__)
versions = dict()
versions["versions"] = dict()

def add_version(blueprint, prefix):
    application.register_blueprint(blueprint, url_prefix=prefix)
    versions["versions"][blueprint.name] = prefix

add_version(v0_1, "/v0-1")

@application.route("/")
def summary():
    return jsonify(versions)

if __name__ == "__main__":
    application.run()


"""Contains code for the smart_virtual platform."""
# smart_virtual.py
# Mission Pinball Framework
# Written by Brian Madden & Gabe Knuth
# Released under the MIT License. (See license info at the end of this file.)

# Documentation and more info at http://missionpinball.com/mpf

import logging

import time

from mpf.system.tasks import DelayManager
from mpf.system.utility_functions import Util
from mpf.platform.virtual import (HardwarePlatform as VirtualPlatform,
                                  VirtualDMD, VirtualDriver, VirtualGI,
                                  VirtualLED, VirtualMatrixLight,
                                  VirtualSwitch)

class HardwarePlatform(VirtualPlatform):
    """Base class for the smart_virtual hardware platform."""

    def __init__(self, machine):
        super(HardwarePlatform, self).__init__(machine)
        self.log = logging.getLogger("Smart Virtual Platform")
        self.log.debug("Configuring smart_virtual hardware interface.")

        self.delay = DelayManager()

    def __repr__(self):
        return '<Platform.SmartVirtual>'

    def initialize(self):
        self.machine.events.add_handler('machine_reset_phase_1',
                                        self._initialize2)

    def _initialize2(self):
        for device in self.machine.ball_devices:
            if not device.is_playfield() and device.config['eject_coil']:
                device.config['eject_coil'].hw_driver.register_ball_switches(
                    device.config['ball_switches'])

                if device.config['eject_targets'][0] is not self.machine.playfield:
                    device.config['eject_coil'].hw_driver.set_target_device(device.config['eject_targets'][0])

    def configure_driver(self, config, device_type='coil'):
        # todo should probably throw out the number that we get since it could
        # be a weird string and just return an incremental int?

        driver = SmartVirtualDriver(config['number'], self.machine, self)

        driver.driver_settings = config
        driver.driver_settings['pulse_ms'] = 30

        return driver, config['number']

    def write_hw_rule(self, *args, **kwargs):
        pass

    def clear_hw_rule(self, sw_name):
        sw_num = self.machine.switches[sw_name].number

        for entry in self.hw_switch_rules.keys():  # slice for copy
            if entry.startswith(
                    self.machine.switches.number(sw_num).name):
                del self.hw_switch_rules[entry]

    def tick(self):
        # ticks every hw loop (typically hundreds of times per sec)

        self.delay._process_delays(self.machine)

    def add_ball_to_device(self, device):
        if device.config['entrance_switch']:
            pass # todo

        found_switch = False

        if device.config['ball_switches']:
            for switch in device.config['ball_switches']:
                if self.machine.switch_controller.is_inactive(switch.name):
                    self.machine.switch_controller.process_switch(switch.name,
                                                                  1,
                                                                  True)
                    found_switch = True
                    break

            if not found_switch:
                raise AssertionError("KABOOM! We just added a ball to {} which"
                                     "was already full.".format(device.name))

class SmartVirtualDriver(VirtualDriver):

    def __init__(self, number, machine, platform):
        self.log = logging.getLogger('SmartVirtualDriver')
        self.number = number
        self.machine = machine
        self.platform = platform
        self.ball_switches = list()
        self.target_device = None

    def __repr__(self):
        return "SmartVirtualDriver.{}".format(self.number)

    def disable(self):
        pass

    def enable(self):
        pass

    def pulse(self, milliseconds=None):
        for switch in self.ball_switches:
            if self.machine.switch_controller.is_active(switch.name):
                self.machine.switch_controller.process_switch(switch.name, 0,
                                                              logical=True)
                break

        if self.target_device:
            self.platform.delay.add(ms=100,
                                    callback=self.platform.add_ball_to_device,
                                    device=self.target_device)

        if milliseconds:
            return milliseconds
        else:
            return self.driver_settings['pulse_ms']

    def register_ball_switches(self, switches):
        self.ball_switches.extend(switches)

    def set_target_device(self, target):
        self.target_device = target

# The MIT License (MIT)

# Oringal code on which this module was based:
# Copyright (c) 2009-2011 Adam Preble and Gerry Stellenberg

# Copyright (c) 2013-2015 Brian Madden and Gabe Knuth

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.


from __future__ import print_function

import sys

import cplex
from cplex.exceptions import CplexError
import numpy as np
import random
import itertools
import pandas as pd
# np.random.seed = 10

import heapq
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt

def getopts(argv):
    opts = {}  # Empty dictionary to store key-value pairs.
    while argv:  # While there are arguments left to parse...
        if argv[0][0] == '-':  # Found a "-name value" pair.
            opts[argv[0]] = argv[1]  # Add key and value to the dictionary.
        argv = argv[1:]  # Reduce the argument list by copying it starting from index 1.
    return opts

from sys import argv
myargs = getopts(argv)
## -t is time limit, -p is p, -Q is number of people, -s is store values and make graphs

TIMELIMIT = int(myargs["-t"])
p = int(myargs["-p"])
Q = int(myargs["-Q"])# number of people per desired team
s = int(myargs["-s"])
W_slack = int(myargs["-slack"])
# k = int(myargs["-k"])
filename = str(myargs["-input"])
print(TIMELIMIT, p, Q)

data = pd.read_csv(filename, header = 0)
V = data.values
M = V.shape[0]
N = V.shape[1] - 1
print("Total Number of Participants:", N)

W_s = N
K_s = int(np.floor(N/Q))
W_reg = W_s - W_slack
Q_reg = int(np.floor(W_reg/K_s))
Q_slack = int(np.floor(W_slack/K_s))
## minimum number of teams assigned to a skill
b = int(np.floor(K_s/M))

print("Number of People: ", W_s)
print("Number of Teams: ", K_s)
print("Number of Skills: ", M)

### TODO randomly order the data, and mark participant's ID's, so that the output knows which person is which

# first column is the participant's ID
V = V[:, 1:]

# normalize the data if we are using real data, but in simulation, the standard deviation is already determined
# for j in range(M):
# 	V[j] = (V[j]- np.mean(V[j]))/np.std(V[j])
# 	print(np.mean(V[j]))
# 	print(np.std(V[j]))

G_js = [sum(heapq.nlargest(Q + 1, V[i])) for i in range(M)]
G = max(G_js)

print("G", G)
my_obj = np.array([1.0]*(K_s * M) + [0.0] * (K_s * M + W_s * K_s))
my_obj = my_obj.flatten()
num_vars = 2* K_s * M  + W_s * K_s
my_ub = [G] * (K_s * M) + [1.0] * (K_s * M + W_s * K_s)
my_lb = [-cplex.infinity] * num_vars
# print("Number of Variables: ", num_vars)
assert(len(my_ub) == len(my_obj))

# G = Q

my_ctype = "C" * (K_s * M) + "B" * (K_s * M + W_s * K_s)
# NOTE: in the nonzero populate function we don't need the column or row name
## TODO RHS
my_rhs = [0] * (2 * K_s * M ) + [p]* K_s + [1] * W_s + [-Q_reg] * K_s \
        + [Q_reg + 1] * K_s + [-b] * (M) + [-Q_slack] * K_s + [Q_slack + 1] * K_s
my_sense = "L" * (2 * K_s * M) + "L" *  (K_s) + "E" * (W_s) + "L" * (K_s * 4 + M)
flatten = lambda l: [item for sublist in l for item in sublist]
assert(len(my_rhs) == len(my_sense))
# print(len(my_sense))

def printshape(l):
    print(np.array(l).shape)

def populatebynonzero(prob):
    prob.objective.set_sense(prob.objective.sense.maximize)

    prob.linear_constraints.add(rhs=my_rhs, senses=my_sense)
    prob.variables.add(obj=my_obj, lb=my_lb, ub=my_ub, types=my_ctype)
    # prob.variables.add(obj=my_obj,  ub=my_ub, types=my_ctype)

    rows1 = [[i] * (2) for i in range(K_s * M)]
    rows2 = [[[K_s * M + k*M + j] * (1 + W_s) for j in range(M)] for k in range(K_s)]
    rows2 = flatten(rows2)
    rows3 = [[K_s * M * 2 + rownum]  * (M) for rownum in range(K_s)]
    rows4 = [[K_s * M * 2 + K_s + rownum] * (K_s) for rownum in range(W_s)]
    rows5 = [[K_s * M * 2 + K_s + W_s + rownum] * (W_reg) for rownum in range(K_s)]
    ## no team too big
    rows6 = [[K_s * M * 2 + K_s + W_s + K_s + rownum] * (W_reg) for rownum in range(K_s)]
    # b constraint
    rows7 = [[K_s * M * 2 + K_s + W_s + 2 * K_s + rownum] * (K_s) for rownum in range(M)]

    rows8 = [[K_s * M * 2 + K_s + W_s + 2 * K_s  + M + rownum] * (W_slack) for rownum in range(K_s)]
    rows9 = [[K_s * M * 2 + K_s + W_s + 2 * K_s  + M + K_s + rownum] * (W_slack) for rownum in range(K_s)]

    rows = [rows1, rows2, rows3, rows4, rows5, rows6, rows7, rows8, rows9]
    # print("ROWSSHAPE")
    # for r in rows:
    #     printshape(r)

    rows = flatten(flatten(rows))

    cols1 = [[[k * M + j, k * M + j + K_s * M] for j in range(M)] for k in range(K_s)]
    cols1 = flatten(cols1)
    cols2 = [[[k * M + j] + [K_s * M * 2 + k * W_s + i for i in range(W_s)] for j in range(M)] for k in range(K_s)]
    cols2 = flatten(cols2)
    cols3 = [[K_s * M + k * M + j for j in range(M)] for k in range(K_s)]
    cols4 = [[K_s * M  * 2 + k * W_s + i for k in range(K_s)] for i in range(W_s)]
    cols5 = [[K_s * M  * 2 + k * W_s + i for i in range(W_reg)] for k in range(K_s)]
    cols6 = [[K_s * M  * 2 + k * W_s + i for i in range(W_reg)] for k in range(K_s)]
    cols7 = [[K_s * M + k * M + j for k in range(K_s)] for j in range(M)]

    cols8 = [[K_s * M  * 2 + k * W_s + i for i in range(W_reg, W_reg + W_slack)] for k in range(K_s)]
    cols9 = [[K_s * M  * 2 + k * W_s + i for i in range(W_reg, W_reg + W_slack)] for k in range(K_s)]

    cols = [cols1, cols2, cols3, cols4, cols5, cols6, cols7, cols8, cols9]
    # print("COLSHAPES")
    # for c in cols:
    #     printshape(c)
    cols = flatten(flatten(cols))

    assert(len(cols) == len(rows))

    rowcol = list(zip(rows,cols))
    # print("VALSHAPES")
    vals1 = [1.0, -G] * (M * K_s)
    # printshape(vals1)
    temp = [[[1.0] + [-V[j][i] for i in range(W_s)] for j in range(M)] for k in range(K_s)]
    vals2 = flatten(flatten(temp))
    # printshape(vals2)
    vals3 = [1.0] * (K_s * M + K_s * W_s) + [-1.0] * (K_s * W_reg) + [1.0] * (K_s * W_reg) + [-1.0] * (K_s * M + K_s * W_slack) \
        + [1.0] * (K_s * W_slack)
    vals = flatten([vals1,vals2, vals3])
    assert(len(vals) == len(rows))
    # print(list(zip(rows, cols, vals)))
    prob.linear_constraints.set_coefficients(zip(rows, cols, vals))

my_prob = cplex.Cplex()
handle = populatebynonzero(my_prob)

my_prob.parameters.timelimit.set(TIMELIMIT)
my_prob.solve()

print("Solution status = ", my_prob.solution.get_status(), ":", end=' ')
# the following line prints the corresponding string
print(my_prob.solution.status[my_prob.solution.get_status()])
print("Objective value  = ", my_prob.solution.get_objective_value())

numcols = my_prob.variables.get_num()
# print("Number of Variables", numcols)
numrows = my_prob.linear_constraints.get_num()

slack = my_prob.solution.get_linear_slacks()
x = my_prob.solution.get_values()

## s is whether to store values or not
# if s == 1:
#     ### -slack + G is the team value, nonzero value is the skill that is assigned to that team
#     team_values = np.array(slack[:K_s * M])
#     team_values = team_values.reshape(K_s, M)
#     # print([(idx,x) for idx, x in enumerate(team_values)])
#     team_values_out = [[idx, -sum(team) + G, np.nonzero(team)[0]] for idx, team in enumerate(team_values)]

#     df_team_values = pd.DataFrame(team_values_out, columns = ["Team Number", "Skill Total", "Skill Number"])
#     df_team_values.to_csv(str(filename) + "NewSpecTeamValues" + str(N) + ".csv")

if s == 1:
    team_assigns = x[K_s * M * 2:]
    team_assigns = np.array(team_assigns)
    team_assigns = team_assigns.reshape(K_s, W_s)

if s ==1:
    nonzeros = []
    for i in range(K_s):
        for j in range(W_s):
            if team_assigns[i,j] >= 0.5:
                nonzeros.append([i,j])
    assert(len(nonzeros) == W_s)
    a = [b[1] for b in nonzeros]
    assert(len(set(a)) == len(nonzeros))

if s ==1:
    df_team_assigns = pd.DataFrame(nonzeros, columns = ["Team Number", "User ID"])
    ## NOTE THAT all indices must add # of diverse participants
    df_team_assigns.to_csv(str(filename) + "NewSpecAssignments" + str(N) + ".csv")

    ## dictionary of team assignments
    team_assigns_dict = {}
    for k,i in nonzeros:
        if k in team_assigns_dict:
            team_assigns_dict[k].append(i)
        else:
            team_assigns_dict[k] = [i]

    XKJ = x[K_s * M: 2 * K_s * M]
    XKJ = np.array(XKJ).reshape(K_s, M)

    team_skills = []
    for k,team in enumerate(XKJ):
        for j,skill in enumerate(team):
            if skill == 1:
                team_skills.append([k,j])
    df_team_skills = pd.DataFrame(team_skills, columns = ["Team Number", "Skill Number"])
    df_team_skills.to_csv(str(filename) + "NewSpecTeamSkills" + str(N) + ".csv")

if s ==1:
    # credited skills
    for team_id in range(K_s):
        matrices = []
        for k,team in enumerate(XKJ):
            team_members = team_assigns_dict[k]
            matrix = []
            for j, skill in enumerate(team):
                matrix.append([(V[j,i] if skill == 1 else 0) for i in team_members])
            matrices.append(matrix)
        ax = sns.heatmap(matrices[team_id], annot = True)
        plt.ylabel("Skills")
        title = "A" + str(filename) + str(W_s) +"Specialized_Selected" + str(team_id) + "p=" + str(p) + ".png"
        plt.title(title)
        plt.savefig(title)
        plt.clf()

if s == 1:
    for team_id in range(K_s):
        # all skills
        matrices_all = []
        for k,team in enumerate(XKJ):
            team_members = team_assigns_dict[k]
            matrix = []
            for j, skill in enumerate(team):
                matrix.append([(V[j,i]) for i in team_members])
            matrices_all.append(matrix)
        ax = sns.heatmap(matrices_all[team_id], annot = True)
        plt.ylabel("Skills")
        title = "A" + str(filename) + str(W_s) + "Specialized_All" + str(team_id) + "p=" + str(p) + ".png"
        plt.title(title)
        plt.savefig(title)
        plt.clf()

    ##### Save as pkl then read in the other file!! :)


#Vanilla TD Actor-Critic
import gym
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Categorical

#Hyperparameters
learning_rate = 0.005
gamma         = 0.99

class ActorCritic(nn.Module):
    def __init__(self):
        super(ActorCritic, self).__init__()
        self.loss_lst = []

        self.fc1 = nn.Linear(4, 128)
        self.fc_pi = nn.Linear(128, 2)
        self.fc_v = nn.Linear(128, 1)
        self.optimizer = optim.Adam(self.parameters(), lr=learning_rate)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        pi = self.fc_pi(x)
        pi = F.softmax(pi, dim=0)
        v = self.fc_v(x)
        return pi, v

    def gather_loss(self, loss):
        self.loss_lst.append(loss.unsqueeze(0))

    def train(self):
        loss = torch.cat(self.loss_lst).sum()
        loss = loss/len(self.loss_lst)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()
        self.loss_lst = []

def main():
    env = gym.make('CartPole-v1')
    model = ActorCritic()

    score = 0.0
    print_interval = 20

    for n_epi in range(10000):
        obs = env.reset()
        loss_lst = []
        for t in range(600):
            obs = torch.from_numpy(obs).float()
            pi, v = model(obs)
            m = Categorical(pi)
            action = m.sample()

            obs, r, done, info = env.step(action.item())
            _, next_v = model(torch.from_numpy(obs).float())
            delta = r + gamma * next_v - v
            loss = -torch.log(pi[action]) * delta.item() + delta * delta.item()
            model.gather_loss(loss)

            score += r
            if done:
                break

        model.train()

        if n_epi%print_interval==0 and n_epi!=0:
            print("# of episode :{}, avg score : {:.1f}".format(n_epi, score/print_interval))
            score = 0.0

    env.close()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

#PyQT5中主窗口例子

import sys
from PyQt5.QtWidgets import QMainWindow, QApplication,QDesktopWidget,QPushButton,QHBoxLayout,QWidget
from PyQt5.QtGui import QIcon

class MainWidget(QMainWindow):
    def __init__(self, parent=None):
        super(MainWidget, self).__init__(parent)
        # 设置主窗体标签
        self.setWindowTitle("QMainWindow 例子")
        # 设置主窗体的大小
        self.resize(400, 200)
        # 设置状态栏
        self.status = self.statusBar()
        self.status.showMessage("这是状态栏提示", 5000)
        # 创建一个按钮
        self.button1 = QPushButton("关闭窗口")
        self.button1.clicked.connect(self.onButtonClick)

        layout = QHBoxLayout()
        layout.addWidget(self.button1)

        main_frame = QWidget()
        main_frame.setLayout(layout)
        self.setCentralWidget(main_frame)

    def onButtonClick(self):
        # 发送信号
        sender = self.sender()
        print(sender.text() +"被按下了")
        qApp= QApplication.instance()
        qApp.quit()

    def center(self):
        #   获取屏幕大小
        screen = QDesktopWidget.geometry()
        # 获取QWidget的大小
        size = self.geometry()
        # 将窗口移动到屏幕中间
        self.move((screen.width() - size.width()) /2,(screen.height() - size.height())/2)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    app.setWindowIcon(QIcon("./images/cartoon1.ico"))
    main = MainWidget()
    main.show()
    sys.exit(app.exec_())


import time
import random
import mmap
import os
from typing import Tuple

def test_speed(qd: int, ttime: float) -> Tuple[float, float]:
    etime = time.time() + ttime
    sz = 190 * (2 ** 20)
    dt = 0
    writes = 0
    with open("/media/data/test.db", "r+b") as fd:
        mm = mmap.mmap(fd.fileno(), 0)
        while time.time() < etime:
            for i in range(qd):
                mm[random.randint(0, sz)] = 123
            t1 = time.perf_counter()
            os.fsync(fd.fileno())
            dt += time.perf_counter() - t1
            writes += qd
    return writes / ttime, dt / (writes / qd)

db_size = 1000000
# create_db()
# fill_data(db_size)

rps1, clat1 = test_speed(50, 5)
# rps10, clat10 = test_speed(10, db_size, 1)
# rps100, clat100 = test_speed(100, db_size, 1)

print(f"   QD      RPS    LAT(ms) ")
print(f"   ~6      {int(rps1):>5d}     {int(clat1 * 1000)}")
# print(f"     10     {int(rps10):>5d}     {int(clat10 * 1000)}")
# print(f"     100    {int(rps100):>5d}     {int(clat100 * 1000)}")


#!/usr/bin/env python
import website.bigfoot.models as models
from website.bigfoot.utils import config, siteaction, ajaxian, mailman, dataplus
import datetime

def handle(request):
    try:
        logged_in_type = siteaction.getLoggedInAccountType(request)
        if logged_in_type == 'U':
            myself = siteaction.getLoggedInUser(request)
        elif logged_in_type == 'R':
            myself = siteaction.getLoggedInRecruiter(request)
        else:
            return ajaxian.getFailureResp('not_logged_in')

        quest_to_user = dataplus.dictGetVal(request.REQUEST, 'questToUser')
        question_text = dataplus.dictGetSafeVal(request.REQUEST, 'question')

        question = models.ResumeQuestion()
        question.sender = myself.account
        question.receiver = models.User.objects.get(username=quest_to_user)
        question.question = question_text
        question.answer = ''
        question.is_answered = False
        question.is_public = False
        question.sent_at = datetime.datetime.utcnow()
        question.save()

        html_message = myself.name + ' wants to know more about your resume and has asked this question, "' + \
            question_text + '"

You may want to Answer this question'

        text_message = myself.name + ' wants to know more about your resume and has asked this question, "' + \
            question_text + '"\r\n\r\nYou can answer this question by visiting this url: ' + \
            config.server_base_url + '/me/answerresumequestion.htm?qid=' + str(question.id)

        def internalSender(rcvr_accounts):
            mailman.sendToInbox(myself.username, rcvr_accounts[0].username, 'A question for you!', html_message, 'TM', 'H')

        def externalSender(rcvr_accounts):
            sender = '"' + myself.name + '" <' + myself.username + '-profile@socialray.org>'
            receivers = ['"' + rcvr.name + '" <' + rcvr.email + '>'  for rcvr in rcvr_accounts]
            mailman.sendMail(sender, receivers, 'A question for you!', html_message,
                None, None, text_message, reply_to=myself.email)

        mailman.sendBySettings([question.receiver.account], internalSender, externalSender, 'ResumeQuestion')

        return ajaxian.getSuccessResp('')
    except:
        return ajaxian.getFailureResp('unknown')


import torch
import time
from torch.autograd import Variable
import numpy as np

def mAP(bicode, label):
    if type(bicode)==torch.autograd.variable.Variable:
        bicode=bicode.data
    if type(label)==torch.autograd.variable.Variable:
        label=label.data
    bicode = torch.sign(bicode.cpu()).type(torch.FloatTensor)
    label = label.cpu()
    sampleNum = label.size(0)

    hashingDistance = -torch.mm(bicode, bicode.transpose(0,1))
    sorted, indices = torch.sort(hashingDistance)
    sorted_distance_labels = torch.gather(torch.stack([label]*sampleNum),1,indices)
    print('1')
    sorted_sim_matrix = (torch.stack([label]*sampleNum).transpose(0,1)==sorted_distance_labels).type(torch.FloatTensor)
    sorted_sim_matrix[:,0] = 0
    rank_matrix = torch.mm(sorted_sim_matrix.type(torch.FloatTensor), torch.FloatTensor(np.triu(np.ones([sampleNum,sampleNum])))) * sorted_sim_matrix
    print(1)
    tmp = torch.FloatTensor([list(range(sampleNum))]*sampleNum)
    tmp[:,0] = 1
    mapvalue = ((rank_matrix/tmp).sum(1)/(sorted_sim_matrix.sum(1)+1e-7)).sum()/float(sampleNum)
    return mapvalue

# cuda verison is smaller than cpu version
# have no idea why this happend now
def mAP_cuda(bicode, label):
    bicode = torch.sign(bicode).type(torch.cuda.FloatTensor)
    laebl = label.type(torch.cuda.FloatTensor)

    sampleNum = label.size(0)

    hashingDistance = -torch.mm(bicode, bicode.transpose(0,1))
    sorted, indices = torch.sort(hashingDistance)

    sorted_distance_labels = torch.gather(torch.stack([label]*sampleNum),1,indices)

    sorted_sim_matrix = (torch.stack([label]*sampleNum).transpose(0,1)==sorted_distance_labels).type(torch.cuda.FloatTensor)
    sorted_sim_matrix[:,0] = 0

    rank_matrix = torch.mm(sorted_sim_matrix, Variable(torch.FloatTensor(np.triu(np.ones([sampleNum,sampleNum])))).cuda()) * sorted_sim_matrix

    tmp = Variable(torch.FloatTensor([list(range(sampleNum))]*sampleNum)).cuda()
    tmp[:,0] = 1
    mapvalue = ((rank_matrix/tmp).sum(1)/(sorted_sim_matrix.sum(1)+1e-7)).sum()/float(sampleNum)
    mapvalue = mapvalue.cpu().data[0]

    return mapvalue

# mapvalue = mAP(bicode, label)


__author__ = 'jsmorrison'
from dnd.management.config import config

from django.core.management.base import BaseCommand
from ...models import OscarCategories, Winners
import os
import pandas as pd
import datetime

# python manage.py oscar_csv_to_db

class Command(BaseCommand):
    def add_arguments(self, parser):
        pass

    def handle(self, *args, **options):
        self.add_oscar()
        self.add_place_holder_winners()

    def add_oscar(self):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        file_name = dir_path + "/../../static/oscar/csv/Oscars_2021.csv"

        df = pd.read_csv(file_name)
        df.columns = ["Year", "Cat", "Selection"]

        for index, row in df.iterrows():
            year = row["Year"]
            cat = row["Cat"]
            sel = row["Selection"].replace('“','').replace('”','').replace('"','')

            (oscar_instance, oscar) = OscarCategories.objects.update_or_create(
                Year =year,
                Cat = cat,
                Name = sel,
            )

    def add_place_holder_winners(self):
        now = datetime.datetime.now()
        y = str(now.year)

        catarr = {"Best Picture": 3, "Best Actor": 3, "Best Actress": 3, "Best Supporting Actor": 3, "Best Supporting Actress": 3,
                  "Best Director": 3, "Best Animated Feature": 3, "Best Animated Short": 1 , "Best Adapted Screenplay": 3, "Best Original Screenplay": 3,
                  "Best Cinematography": 2, "Best Documentary Feature": 2, "Best Live-Action Short Film": 1, "Best International Feature": 2,
                  "Best Film Editing": 2, "Best Sound": 2, "Best Production Design": 2,
                  "Best Original Score": 2, "Best Original Song": 2, "Best Hair and Makeup": 2, "Best Costume Design": 2, "Best Visual Effects": 2,
                  "Best Documentary Short Subject": 1}

        for key,val in catarr.items():
            (oscar_instance, oscar) = Winners.objects.update_or_create(
                Year=y,
                Cat=key,
                Name="NA",
                Weight = val,
            )


import string
from twitter_specials import *
import math
import csv
import json
labels = {"positive":0,"negative":1,"neutral":2,"irrelevant":3}
reversed_labels = {0:"positive",1:"negative",2:"neutral",3:"irrelevant"}
WCOUNT = 4
total = [0,0,0,0,0]
word_counts_dict={}
word_prob_dict = {}
def parsing():
    with open("data/labeled_corpus.tsv",encoding="utf-8") as csvfile:
        readCSV = csv.reader(csvfile, delimiter='\t')
        for row in readCSV:
            line_arr = list(row)
            tweet = line_arr[0]
            tweet = clean_tweet(tweet, emo_repl_order, emo_repl, re_repl)
            words = tweet.split()
            word_set = set()
            punc = set(string.punctuation)
            label = str(line_arr[1])
            if label not in labels:
                continue
            for w in words:
                if '#' not in w and '@' not in w:
                    s = ""
                    for ch in w:
                        if (ch not in punc):
                            s += ch
                    w = s
                    word_set.add(w)

            for w in word_set:
                if w not in word_counts_dict:
                    word_counts_dict[w] = [0,0,0,0,0]
                word_counts_dict[w][WCOUNT]+= 1
                word_counts_dict[w][labels[label]]+=1
            total[WCOUNT]+=1
            total[labels[label]] += 1
        for w,counts in word_counts_dict.items():
            for i in range(4):
                if w not in word_prob_dict:
                    word_prob_dict[w] = [0,0,0,0]
                word_prob_dict[w][i]=word_counts_dict[w][i]/total[i]
    csvfile.close()

def classifier():
    with open("data/geo_twits_squares.tsv", encoding="utf-8") as csvfile:
        readCSV = csv.reader((line.replace('\0', '') for line in csvfile), delimiter='\t')
        output = []
        for row in readCSV:
            line_arr = list(row)
            latitude = line_arr[0]
            longitude = line_arr[1]
            prob = [0,0,0,0]
            tweet = clean_tweet(line_arr[2], emo_repl_order, emo_repl, re_repl)
            words = tweet.split()
            punc = set(string.punctuation)
            tested = 0
            for w in words:
                if '#' not in w and '@' not in w:
                    s = ""
                    for ch in w:
                        if (ch not in punc):
                            s += ch
                    w = s
                    if w in word_prob_dict:
                        tested = 1
                        try:
                            for i in range(4):
                                prob[i] = math.log(word_prob_dict[w][i])+math.log(total[i]/total[WCOUNT])
                        except:pass
            if tested == 1:
                row_output = [latitude,longitude, reversed_labels[prob.index(max(prob))]]
            else : row_output = [latitude,longitude, "irrelevant"]
            output.append(row_output)
        with open("locations_classified.tsv","w") as tsvfile:
            writer = csv.writer(tsvfile, delimiter='\t')
            writer.writerows(output)
    tsvfile.close()
    csvfile.close()
pos_dict = {}
def pscore():
    pos_dict = {}
    locations = None
    with open("locations_classified.tsv", encoding="utf-8") as csvfile:
        readCSV = csv.reader(csvfile, delimiter='\t')
        for row in readCSV:
            line_arr = list(row)
            label = line_arr[2]
            if (line_arr[0], line_arr[1]) not in pos_dict:
                if locations != None:
                    pos_dict[locations] = (count[0]/count[-1] - count[1]/count[-1] + 1)/2
                locations = (line_arr[0], line_arr[1])
                count = [0.0, 0.0, 0.0, 0.0, 1.0]
                pos_dict[locations] = 0
                count[labels[label]] += 1
            else:
                count[labels[label]] += 1
                count[-1] += 1
        pos_dict[locations] = (count[0]/count[-1] - count[1]/count[-1] + 1)/2
        #I will do the json file as well here
        input = []
        for loc,score in pos_dict.items():
            dict = {}
            dict["score"]=score
            dict["g"]=float(loc[1])+0.025
            dict["t"]=float(loc[0])+0.025
            input.append(dict)
        with open("data.js","w") as jsonfile:
            jsonfile.write("var data =")
            json.dump(input,jsonfile)
if __name__ == '__main__':
    parsing()
    classifier()
    pscore()


import lightgbm as lgb
import pickle
import sys
import numpy as np
import yaml

in_path = sys.argv[1]
out_path = sys.argv[2]

with open(in_path, 'rb') as f:
    df = pickle.load(f)

params = yaml.safe_load(open('params.yaml'))

# time related features
timestamps = df.index.get_level_values('valid_datetime')
seconds_in_day = 24*60*60
df.loc[:, 'sin_time_hd'] = np.sin(2*np.pi*(timestamps-timestamps.round("D")).total_seconds()/seconds_in_day)
df.loc[:, 'cos_time_hd'] = np.cos(2*np.pi*(timestamps-timestamps.round("D")).total_seconds()/seconds_in_day)
df.loc[:, 'time_hod'] = timestamps.hour

if 'dow' in params['featurize']['features']:
    df.loc[:, 'dow'] = timestamps.dayofweek

df.loc[:, 'cal_weekday'] = timestamps.dayofweek.isin([0, 1, 2, 3, 4]).astype('int')
df.loc[:, 'cal_weekend'] = timestamps.dayofweek.isin([5, 6]).astype('int')

print(df.columns)

df_X = df.drop(columns=['Load(MWh)'])
df_y = df['Load(MWh)']

dataset = lgb.Dataset(df_X, label=df_y, params={'verbose': -1}, free_raw_data=False)

with open(out_path, 'wb') as f:
    pickle.dump(dataset, f)