import json
import logging
import logging.handlers
import splunk.admin as admin
import splunk.entity as entity
import splunk.rest as rest
import splunk.util as util
import sys
import time

if sys.platform == "win32":
    import os, msvcrt
    msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)

from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log     import setup_logger, SHORT_FORMAT
from SolnCommon.kvstore import KvStoreHandler
from SolnCommon.pooling import should_execute

logger = setup_logger('reviewstatuses_rest_handler', level=logging.INFO, format=SHORT_FORMAT)

def time_function_call(fx):
    """
    This decorator will provide a log message measuring how long a function call took.

    Arguments:
    fx -- The function to measure
    """

    def wrapper(*args, **kwargs):
        t = time.time()

        r = fx(*args, **kwargs)

        logger.debug('%s, duration=%.3f', fx.__name__, time.time() - t)

        return r
    return wrapper

"""
This class provides a mechanism for determining how long operations take. Results are submitted as debug
calls to the logger provided in the constructor or the instance in the global variable logger if no logger
is provided in the constructor.

Example:
with TimeLogger("doing_something", Logger()):
    time.sleep(2)
"""
class TimeLogger():

    def __init__(self, title, logger=None):
        self.title = title
        self.logger = logger

    def __enter__(self):

        # Define the start time
        self.start_time = int(time.time())

    def __exit__(self, type, value, traceback):

        # Determine how long the operation took
        time_spent = int(time.time()) - self.start_time

        # See if we can find a logger as a global variable
        if self.logger is None:
            try:
                self.logger = logger
            except NameError:
                raise Exception('Could not get a logger instance for the purposes of recording performance')

        # Log the time spent
        self.logger.debug('%s, duration=%d', self.title, time_spent)

class UnauthorizedUserException(Exception):
    pass

class InvalidConfigException(Exception):
    pass

class InvalidParameterValueException(InvalidConfigException):
    """
    Describes a config parameter that has an invalid value.
    """

    def __init__(self, field, value, value_must_be):
        message = "The value for the parameter '%s' is invalid: %s (was %s)" % (field, value_must_be, value)
        super(InvalidConfigException, self).__init__( message)

class UnsupportedParameterException(InvalidConfigException):
    """
    Describes a config parameter that is unsupported.
    """
    pass

class MissingTransitionException(InvalidConfigException):
    """
    Describes a capability that is missing.
    """
    def __init__(self, transitions):
        self.transitions = transitions
        super(InvalidConfigException, self).__init__( 'Missing transition detected' )

def _getFieldValue(args, name, default_value=None, max_length=None):
    """
    Get the field value from the argument list.
    """

    ## Get the value if defined or the default value if not defined
    value = args[name][0] or default_value if name in args else default_value

    ## Check the length
    if value and max_length and len(value) > max_length:
        raise admin.ArgValidationException(
            'App %s cannot be longer than %s character%s.' % (name, max_length, 's' if max_length > 1 else ''))
    ## return the value
    return value

def _addToDictIfNonNull(dict, name, value):
      """
      Add the given name and value to the dictionary if the value is not none.

      Arguments:
      dict -- the dictionary to add to
      name -- the name of the object to add
      value -- the value of the object to add (if not none)
      """
      if value is not None:
          dict[name] = value

class ReviewStatuses(admin.MConfigHandler):
    '''
    Set up supported arguments
    '''
    ## admin.py constants
    REQUESTED_ACTIONS       = { '1': 'ACTION_CREATE', '2': 'ACTION_LIST', '4': 'ACTION_EDIT', '8': 'ACTION_REMOVE', '16': 'ACTION_MEMBERS', '32': 'ACTION_RELOAD' }
     ## Permissions
    WRITE_CAPABILITY        = 'edit_reviewstatuses'
     ## Default Params
    PARAM_DISABLED          = 'disabled'
    PARAM_LABEL             = 'label'
    PARAM_DESCRIPTION       = 'description'
    PARAM_DEFAULT           = 'default'
    PARAM_SELECTED          = 'selected'
    PARAM_HIDDEN            = 'hidden'
    PARAM_END               = 'end'

    BOOLEAN_PARAMS          = [PARAM_DISABLED, PARAM_DEFAULT, PARAM_SELECTED, PARAM_HIDDEN, PARAM_END]

    VALID_PARAMS            = [PARAM_DISABLED,
                               PARAM_LABEL,
                               PARAM_DESCRIPTION,
                               PARAM_DEFAULT,
                               PARAM_SELECTED,
                               PARAM_HIDDEN,
                               PARAM_END]

    ## collections do not support '.' or '$' in params
    COLLECTION_PARAMS       = [PARAM_DISABLED,
                               PARAM_LABEL,
                               PARAM_DESCRIPTION,
                               PARAM_DEFAULT,
                               PARAM_SELECTED,
                               PARAM_HIDDEN,
                               PARAM_END]

    REQUIRED_PARAMS         = [PARAM_DISABLED, PARAM_LABEL, PARAM_DEFAULT, PARAM_HIDDEN]
     ## Default Vals
    DEFAULT_NAMESPACE       = 'SA-ThreatIntelligence'
    DEFAULT_OWNER           = 'nobody'
    DEFAULT_DISABLED        = 0
    DEFAULT_DEFAULT         = 0
    DEFAULT_SELECTED        = 0
    DEFAULT_HIDDEN          = 0
    DEFAULT_END             = 0

    PARAM_UNASSIGNED_STANZA = '0'
    PARAM_UNASSIGNED_DICT   = {PARAM_DISABLED:    DEFAULT_DISABLED,
                               PARAM_LABEL:       'Unassigned',
                               PARAM_DESCRIPTION: 'An error is preventing the issue from having a valid status assignment',
                               PARAM_DEFAULT:     DEFAULT_DEFAULT,
                               PARAM_SELECTED:    DEFAULT_SELECTED,
                               PARAM_HIDDEN:      DEFAULT_HIDDEN,
                               PARAM_END:         DEFAULT_END}

    def setup(self):
        logger.info('Setting up reviewstatuses_rest_handler')

        ## set write capability
        self.setWriteCapability(ReviewStatuses.WRITE_CAPABILITY)

        if self.requestedAction == admin.ACTION_EDIT or self.requestedAction == admin.ACTION_CREATE:
            ## Fill required params
            for arg in ReviewStatuses.REQUIRED_PARAMS:
                self.supportedArgs.addReqArg(arg)

            ## Fill valid params
            for arg in ReviewStatuses.VALID_PARAMS:
                if arg not in ReviewStatuses.REQUIRED_PARAMS:
                    self.supportedArgs.addOptArg(arg)

    @time_function_call
    def handleCreate(self, confInfo):
        """
        Handles creation of a review status
        """
        sessionKey = self.getSessionKey()
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        authorizeDict         = self.readConf('authorize')
        reviewstatusesDict    = self.readConfCtx('reviewstatuses')

        name = self.callerArgs.id
        args = self.callerArgs.data

        ## Check name
        if not name:
            raise admin.ArgValidationException('The name (stanza) of the status must not be empty')
        ## Make sure the item is not '0'
        ## '0' is reserved for the Unassigned status
        elif name==ReviewStatuses.PARAM_UNASSIGNED_STANZA:
            raise admin.ArgValidationException('The name (stanza) of the status must not be 0 (this is reserved for the Unassigned status)')
        ## Make sure the item does not already exist
        ## This should not be possible based on getUID, but no hurt in double checking
        elif name in reviewstatusesDict:
            raise admin.AlreadyExistsException('A reviewstatuses.conf entry already exists for status ID %s' % name)

        ## Get the field values
        disabled    = _getFieldValue(args, ReviewStatuses.PARAM_DISABLED)
        label       = _getFieldValue(args, ReviewStatuses.PARAM_LABEL)
        description = _getFieldValue(args, ReviewStatuses.PARAM_DESCRIPTION)
        default     = _getFieldValue(args, ReviewStatuses.PARAM_DEFAULT)
        selected    = _getFieldValue(args, ReviewStatuses.PARAM_SELECTED)
        hidden      = _getFieldValue(args, ReviewStatuses.PARAM_HIDDEN)
        end         = _getFieldValue(args, ReviewStatuses.PARAM_END)

        ## Add the field values to a configuration dictionary (that will be verified)
        conf           = entity.getEntity('configs/conf-reviewstatuses', '_new', sessionKey=sessionKey)
        conf.namespace = self.appName # always save things to SOME app context.
        conf.owner     = ((self.context == admin.CONTEXT_APP_AND_USER) and self.userName) or "-"
        conf['name']   = name

        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DISABLED, disabled)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_LABEL, label)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DESCRIPTION, description)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DEFAULT, default)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_SELECTED, selected)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_HIDDEN, hidden)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_END, end)

        ## Check the configuration
        ## on create we don't care about other default statuses because we are going to unset them
        ## we do however need to ensure that we are not creating a disabled default
        try:
            if util.normalizeBoolean(default):
                ReviewStatuses.checkConf(conf, name, defaultStatuses=[name], checkDefault=True)
            else:
                ReviewStatuses.checkConf(conf, name, checkDefault=False)

        except InvalidConfigException as e:
            e = "The configuration for the new review status is invalid and could not be created: %s" % e
            logger.error(e)
            raise admin.ArgValidationException(e)

        ## switch default
        if util.normalizeBoolean(default):
            self.switchDefault(reviewstatusesDict)

        ## Write out an update to the reviewstatuses config file
        entity.setEntity(conf, sessionKey=sessionKey)
        logger.info('Successfully added review status: %s', label)

        ## Create new transitions
        transitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', name, toOnly=True))

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                transitions.extend(ReviewStatuses.makeTransitions(name, stanza))

        transitions.sort()

        ## Write out an update to the authorize config file
        conf             = entity.getEntity('configs/conf-authorize', '_new', sessionKey=sessionKey)
        conf.namespace   = self.appName # always save things to SOME app context.
        conf.owner       = ((self.context == admin.CONTEXT_APP_AND_USER) and self.userName) or "-"
        conf[ReviewStatuses.PARAM_DISABLED] = '0'
        with TimeLogger('setting_transitions', logger):
            for transition in transitions:
                if transition in authorizeDict:
                    logger.warn('An authorize.conf entry already exists for %s', transition)

                else:
                    conf['name'] = transition
                    entity.setEntity(conf, sessionKey=sessionKey)
                    logger.info('Successfully added transition %s', transition)

        ## Reload reviewstatuses (makeKVS)
        self.handleReload()

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def switchDefault(self, reviewstatusesDict):
        """
        Handles the unsetting of the default flag if this is the new default

        raises Exceptions for any issues with unsetting perceived defaults
        """
        for stanza in reviewstatusesDict:
            reviewstatus = reviewstatusesDict[stanza]
            if stanza != 'default' and util.normalizeBoolean(reviewstatus[ReviewStatuses.PARAM_DEFAULT]):
                logger.info('Unsetting default param for reviewstatus %s', stanza)
                response, content = rest.simpleRequest(
                    '/servicesNS/nobody/%s/configs/conf-reviewstatuses/%s' % (reviewstatus[admin.EAI_ENTRY_ACL]['app'], stanza),
                    sessionKey=self.getSessionKey(), postargs={'default': 0}, raiseAllErrors=True
                )

    @time_function_call
    def handleList(self, confInfo):
        """
        Handles listing of a review statuses
        """
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        ## Get the configurations from reviewstatuses.conf
        reviewstatusesDict = self.readConfCtx('reviewstatuses')
        authorizeDict      = self.readConf('authorize')

        ## Get all correlations searches and provide the relevant options
        if reviewstatusesDict:
            ## Check default count
            defaultCount = len(ReviewStatuses.getSpecialStatuses(reviewstatusesDict))
            if defaultCount!=1:
                logger.error('The reviewstatuses.conf configurations are invalid because %s default statuses are set; should be one', defaultCount)

            ## Check end count
            endCount = len(ReviewStatuses.getSpecialStatuses(reviewstatusesDict, ReviewStatuses.PARAM_END))
            if endCount!=1:
                logger.error('The reviewstatuses.conf configurations are invalid because %s end statuses are set; should be one', endCount)

            ## Check each conf
            for stanza, settings in reviewstatusesDict.items():
                ## Make sure the item is not '0'
                ## '0' is reserved for the Unassigned status
                if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                    try:
                        ## Check config
                        ReviewStatuses.checkConf(settings, stanza, confInfo)

                        ## Check transitions
                        ReviewStatuses.checkTransitions(stanza, reviewstatusesDict, authorizeDict)

                    except MissingTransitionException as e:
                        for exc in e.transitions:
                            logger.error("The configuration for status ID '%s' is invalid: %s", stanza, exc)

                    except InvalidConfigException as e:
                        logger.error("The configuration for status ID '%s' is invalid: %s", stanza, e)

            ## Add static "Unassigned"
            for key, val in ReviewStatuses.PARAM_UNASSIGNED_DICT.items():
                confInfo[ReviewStatuses.PARAM_UNASSIGNED_STANZA].append(key, val)

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def handleReload(self, confInfo=None, makeKVS=True):
        """
        Handles refresh/reload of the configuration options
        """
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)
        logger.info('Refreshing reviewstatuses configurations via properties endpoint')
        try:
            refreshInfo = entity.refreshEntities('properties/reviewstatuses', sessionKey=self.getSessionKey())
        except Exception as e:
            logger.warn('Could not refresh reviewstatuses configurations via properties endpoint: %s', e)

        logger.info('Refreshing authorize configurations via properties endpoint')
        try:
            refreshInfo = entity.refreshEntities('properties/authorize', sessionKey=self.getSessionKey())
        except Exception as e:
            logger.warn('Could not refresh authorize configurations via properties endpoint: %s', e)

        if makeKVS:
            # Only execute on standalone or on cluster master.
            exec_status, exec_status_msg = should_execute(session_key=self.getSessionKey())
            logger.info(exec_status_msg)
            if exec_status:
                self.makeKVS()

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def makeKVS(self, collection='reviewstatuses', reviewstatusesDict=None):
        """
        Handles maintenance of reviewstatuses kvstore collection
        """
        logger.info('Creating %s collection', collection)
        ## options
        options = {'app': ReviewStatuses.DEFAULT_NAMESPACE, 'owner': ReviewStatuses.DEFAULT_OWNER, 'collection': collection}
        ## get reviewstatuses conf
        reviewstatusesDict = reviewstatusesDict or self.readConf('reviewstatuses')

        if not reviewstatusesDict:
            logger.error('Reviewstatuses dictionary is None; cannot makeKVS')
        else:
            ## get reviewstatuses collection
            gr, gc = KvStoreHandler.get(None, self.getSessionKey(), options)

            if gr.status==200:
                existing_keys  = [x['_key'] for x in json.loads(gc)]
                reviewstatuses = []

                ## Iterate reviewstatuses
                for stanza, settings in sorted(reviewstatusesDict.items()):
                    if stanza!='default':
                        reviewstatus = { '_key': stanza, 'status': stanza }
                        is_default   = False
                        for key, val in settings.items():
                            if not is_default and key == ReviewStatuses.PARAM_DEFAULT:
                                if util.normalizeBoolean(val):
                                    is_default = True
                            if key in ReviewStatuses.COLLECTION_PARAMS:
                                ## kvstore prefers None over empty string
                                reviewstatus[key] = val or None
                        reviewstatuses.append(reviewstatus)

                        if is_default:
                            default_reviewstatus = reviewstatus.copy()
                            default_reviewstatus['_key'] = '-1'
                            reviewstatuses.append(default_reviewstatus)

                pr, pc = KvStoreHandler.batch_create(reviewstatuses, self.getSessionKey(), options)
                if pr.status!=200:
                    logger.error('Error in upserting records to %s collection: %s', collection, pc)

                removed_keys = list(set(existing_keys).difference([x['_key'] for x in reviewstatuses]))
                if len(removed_keys)>0:
                    duri   = '/servicesNS/{owner}/{app}/storage/collections/data/{collection}'.format(**options)
                    dquery = {'$or': []}

                    for removed_key in removed_keys:
                        dquery['$or'].append({'_key': removed_key})

                    dr, dc = rest.simpleRequest(duri, sessionKey=self.getSessionKey(), method='DELETE', getargs={'query': json.dumps(dquery)})

                    if dr.status!=200:
                        logger.error('Error in deleting records from %s collection: %s', collection, dc)
            else:
                logger.error('Error retrieving records from %s collection: %s', collection, gc)

    @time_function_call
    def correctMissingTransition(self, transition, save=True):
        """
        Create the messing transition and return the created transition.
        """
        logger.info('Transition %s does not exist but should, it will be created', transition)
        # Get the _new entity in order to start editing
        conf = entity.getEntity('configs/conf-authorize', '_new', sessionKey=self.getSessionKey())
        # Set the namespace and owner
        conf.namespace   = self.appName
        conf.owner       = ReviewStatuses.DEFAULT_OWNER
        # Set the transition
        conf['name']     = transition
        conf[ReviewStatuses.PARAM_DISABLED] = '1'

        # Save the transition
        if save:
            entity.setEntity(conf, sessionKey=self.getSessionKey())
            logger.info('Successfully added transition %s', transition)
        else:
            logger.info('Preparing the creation of the missing transition %s; save is deferred until other edits are complete', transition)

        return conf

    @time_function_call
    def handleEdit(self, confInfo):
        """
        Handles edits to the configuration options
        """
        sessionKey = self.getSessionKey()
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        reviewstatusesDict = self.readConfCtx('reviewstatuses')

        name = self.callerArgs.id
        args = self.callerArgs

        # Stop if no name was provided
        if not name:
            raise admin.ArgValidationException('No name provided')
        elif name==ReviewStatuses.PARAM_UNASSIGNED_STANZA:
            raise admin.ArgValidationException('The name (stanza) of the status must not be 0 (this is reserved for the Unassigned status)')
        try:
            conf = entity.getEntity('configs/conf-reviewstatuses', name, sessionKey=sessionKey)
        except ResourceNotFound:
            raise admin.NotFoundException("A status with the given name '%s' could not be found" % name)

        # Create the resulting configuration that would be persisted if the settings provided are applied
        for key, val in conf.items():
            if key in args.data:
                # Get the new value
                new_value = args[key][0]
                # Set the value to a single space if empty or none, otherwise, Splunk won't save it (SOLNPCI-532)
                if new_value in [None, '']:
                    new_value = ' '
                # Assign the value
                conf[key] = new_value

        ## namespace/owner
        if admin.EAI_ENTRY_ACL in conf:
            conf.namespace = conf[admin.EAI_ENTRY_ACL].get('app', None) or ReviewStatuses.DEFAULT_NAMESPACE
            conf.owner     = conf[admin.EAI_ENTRY_ACL].get('owner', None) or ReviewStatuses.DEFAULT_OWNER
        else:
            conf.namespace = ReviewStatuses.DEFAULT_NAMESPACE
            conf.owner     = ReviewStatuses.DEFAULT_OWNER

        ## Check the resulting configuration
        ## here we need to pass the existing defaults
        defaultStatuses = ReviewStatuses.getSpecialStatuses(reviewstatusesDict)
        try:
            ReviewStatuses.checkConf(conf, name, defaultStatuses=defaultStatuses, checkDefault=True)
        except InvalidConfigException as e:
            e = "The edit attempt for review status '%s' produced an invalid configuration: %s" % (name, e)
            logger.error(e)
            raise admin.ArgValidationException(e)

        ## switch default
        if util.normalizeBoolean(conf['default']):
            self.switchDefault(reviewstatusesDict)

         ## Retain is_disabled
        status_disabled = util.normalizeBoolean(conf[ReviewStatuses.PARAM_DISABLED])

        ## Enable/Disable transitions
        transitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', name, toOnly=True))
        logger.info('transitions: %s', transitions)

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                transitions.extend(ReviewStatuses.makeTransitions(name, stanza))

        transitions.sort()
        logger.info('transitions: %s', transitions)

        # Get the existing set of transition capabilities; we are going to store these so that we don't have to call
        # getEntity on each capability since this takes a long time (about 20 seconds each)
        existing_transitions = entity.getEntities('configs/conf-authorize', sessionKey=sessionKey, count=-1)

        ## Write out an update to the authorize config file
        for transition in transitions:
            changed = False

            # Get the entry that we are editing
            if transition in existing_transitions:
                transition_conf = existing_transitions[transition]
            else:
                # Uh oh, the transition wasn't found. Go ahead and create it.
                transition_conf = self.correctMissingTransition(transition, save=False)

                # If the transition was not returned, then skip this entry.
                if transition_conf is None:
                    continue

                changed = True

            transition_conf.namespace = self.appName
            transition_conf.owner = ReviewStatuses.DEFAULT_OWNER

            ## Disable only transitions to the disabled status
            if status_disabled and transition.find('to_%s' % name) != -1:
                action = 'disabled'

                # Note that we are changing the value
                if str(transition_conf['disabled']) != '1':
                    changed = True
                    transition_conf['disabled'] = '1'

            else:
                action = 'enabled'

                # Note that we are changing the value
                if str(transition_conf['disabled']) != '0':
                    changed = True
                    transition_conf['disabled'] = '0'

            # Set the settings if they have changed; try to avoid performing the changes unless we need to since this is a slow operation
            if changed:
                with TimeLogger( ('Set %s transition in handleEdit: %s' % (action, transition)), logger):
                    entity.setEntity(transition_conf, sessionKey=sessionKey)
                logger.info('Successfully %s transition: %s', action, transition)

        ## Write out an update to the reviewstatuses config file
        ## Note that we are doing this after setting the transitions
        with TimeLogger('setting entity in handleEdit', logger):
            entity.setEntity(conf, sessionKey=sessionKey)

        logger.info('Successfully updated review status id %s', name)

        ## Reload reviewstatuses (makeKVS)
        self.handleReload()

        logger.info('%s completed successfully', actionStr)

    def handleRemove(self, confInfo):
        pass

    @staticmethod
    def getUID(reviewstatuses=[]):
        """
        Returns a unique identifier to be used as a stanza name
        """
        statusInts = []
        for reviewstatus in reviewstatuses:
            try:
                statusInt = int(reviewstatus)
                if statusInt > 0:
                    statusInts.append(statusInt)
            except:
                pass
        if len(statusInts)==0:
            uid = 1
        else:
            uid = sorted(statusInts)[-1] + 1

        return uid

    @staticmethod
    def getSpecialStatuses(confDict, typ=PARAM_DEFAULT):
        logger.info('Retrieving statuses with %s set', typ)
        statuses = [x for x in confDict if util.normalizeBoolean(confDict[x][typ])]
        logger.info('Successfully retrieved status(es) with %s set: %s', typ, statuses)
        return statuses

    @staticmethod
    def checkConf(settings, stanza=None, confInfo=None, defaultStatuses=[], checkDefault=False, throwExceptionOnError=False):
        """
        Checks the settings and raises an exception if the configuration is invalid.
        """
        ## Below is a list of the required fields. The entries in this list will be removed as they
        ## are observed. An empty list at the end of the config check indicates that all necessary
        ## fields where provided.
        required_fields = ReviewStatuses.REQUIRED_PARAMS[:]

        if stanza is not None and confInfo is not None:
            # Add each of the settings
            for key, val in settings.items():
                ## Set val to empty if None
                if val is None:
                    val = ''

                if key in ReviewStatuses.VALID_PARAMS:
                    confInfo[stanza].append(key, val)

                ## Key is eai;acl Set meta
                elif key.startswith(admin.EAI_ENTRY_ACL):
                    confInfo[stanza].setMetadata(key, val)

                ## Key is eai; userName/appName
                elif key.startswith(admin.EAI_META_PREFIX):
                    confInfo[stanza].append(key, val)

                ## Key is not proper
                else:
                    pass

        ## Check each of the settings individually
        logger.info("Checking general settings for the '%s' review status", stanza)
        for key, val in settings.items():
            ## Set val to empty if None
            if val is None:
                val = ''

            ## Check the disabled/selected value
            if key in ReviewStatuses.BOOLEAN_PARAMS:
                try:
                    util.normalizeBoolean(val, enableStrictMode=True)

                    ## Remove the field from the list of required fields
                    try:
                        required_fields.remove(key)

                    except ValueError:
                        pass # Field not available, probably because it is not required

                except ValueError:
                    raise InvalidParameterValueException(key, val, 'must be a valid boolean')

            elif key in ReviewStatuses.REQUIRED_PARAMS:
                ## Remove the field from the list of required fields
                try:
                    required_fields.remove(key)

                except ValueError:
                    pass # Field not available, probably because it is not required

            elif key in ReviewStatuses.VALID_PARAMS:
                pass

            ## Key is eai
            elif key.startswith(admin.EAI_META_PREFIX):
                pass

            ## Key is not proper
            else:
                if throwExceptionOnError:
                    raise UnsupportedParameterException()

                else:
                    logger.warn("The configuration for the '%s' review status contains an unsupported parameter: %s", stanza, key)
         ## Error if some of the required fields were not provided
        if len(required_fields) > 0:
            raise InvalidConfigException('The following fields must be defined in the configuration but were not: ' + ', '.join(required_fields).strip())

        ## Error if checkDefault and...
        if checkDefault:
            ## disabled
            disabled = util.normalizeBoolean(ReviewStatuses.DEFAULT_DISABLED)
            if ReviewStatuses.PARAM_DISABLED in settings:
                disabled = util.normalizeBoolean(settings[ReviewStatuses.PARAM_DISABLED])
            ## default
            default = util.normalizeBoolean(ReviewStatuses.DEFAULT_DEFAULT)
            if ReviewStatuses.PARAM_DEFAULT in settings:
                default = util.normalizeBoolean(settings[ReviewStatuses.PARAM_DEFAULT])
            if default:
                defaultStatuses.append(stanza)
            else:
                try:
                    defaultStatuses.remove(stanza)
                except:
                    pass

            ## end
            end = util.normalizeBoolean(ReviewStatuses.DEFAULT_END)
            if ReviewStatuses.PARAM_END in settings:
                end = util.normalizeBoolean(settings[ReviewStatuses.PARAM_END])

            ## 1. No disabled defaults
            if default and disabled:
                raise InvalidConfigException('Default review statuses cannot be disabled.  If you want to disable this status, then unset default.')

            ## 2. One default always
            defaultStatuses = set(defaultStatuses)
            if len(defaultStatuses)==0:
                raise InvalidConfigException('There must be one default review status at all times')
            elif len(defaultStatuses)>1:
                logger.warn('More than one default status detected: %s', defaultStatuses)

            ## 3. No end defaults
            if default and end:
                raise InvalidConfigException('Default review status cannot also be end status.  If you want this status to be default, then unset end.')

    @staticmethod
    def makeTransitions(statusA, statusB, toOnly=False):
        transitions = []

        if statusA != statusB:
            transitions.append('capability::transition_reviewstatus-%s_to_%s' % (statusA, statusB))
            if not toOnly:
                transitions.append('capability::transition_reviewstatus-%s_to_%s' % (statusB, statusA))

        return transitions

    @staticmethod
    def checkTransitions(status, reviewstatusesDict, authorizeDict):
        transitions = []
        missingTransitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', status, toOnly=True))

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != '0':
                transitions.extend(ReviewStatuses.makeTransitions(status, stanza))

        for transition in transitions:
            if transition not in authorizeDict:
                missingTransitions.append('Missing capability ' + transition)

        if len(missingTransitions) > 0:
            raise MissingTransitionException(missingTransitions)

# initialize the handler
admin.init(ReviewStatuses, admin.CONTEXT_APP_AND_USER)


'''
分析字幕文件中出现的所有英文单词，并累计出现的次数
统计单词中所有字母的频次

steps:
    1. 识别文件类型，txt、rar、zip
        zip: 50 4B 03
        rar: 52 61 72 21 1A 07 00
        else:
    2. 打开/解压文件
        7z x 14.zip -y -oE:\tmp
    3. 提取单词
    4. 统计

https://blog.csdn.net/weixin_42902669/article/details/102806154
'''
import os
import shutil
import re
import subprocess
import json
import stat

class data():
    def __init__(self):
        self.words = dict()
        self.letters = dict()

        for i in range(65, 91):
            self.letters[chr(i)] = 0
        for i in range(97, 123):
            self.letters[chr(i)] = 0

    def add_word(self, word):
        # print('add_word: ', word)
        if word in self.words:
            self.words[word] += 1
        else:
            self.words[word] = 1

        for a in word:
            self.letters[a] += 1

    def save(self, filename):
        with open(f'{results}\{filename}_letters.json', mode='wt') as f:
            f.write(json.dumps(sorted(self.letters.items(),
                                      key=lambda i: i[1], reverse=True)))

        with open(f'{results}\{filename}_words.json', mode='wt') as f:
            f.write(json.dumps(sorted(self.words.items(),
                                      key=lambda i: i[1], reverse=True)))

    def save_total(dl: list):
        words = dict()
        letters = dict()

        for i in range(65, 91):
            letters[chr(i)] = 0
        for i in range(97, 123):
            letters[chr(i)] = 0

        for d in dl:
            for k, v in d.letters.items():
                letters[k] += v

            for k, v in d.words.items():
                if k in words:
                    words[k] += v
                else:
                    words[k] = v

        with open(f'{results}\\total_letters.json', mode='wt') as f:
            f.write(json.dumps(sorted(letters.items(),
                                      key=lambda i: i[1], reverse=True)))

        with open(f'{results}\\total_words.json', mode='wt') as f:
            f.write(json.dumps(sorted(words.items(),
                                      key=lambda i: i[1], reverse=True)))

ds = 'requests\\download'
# ds = 'test'
tmp = os.path.join(os.getcwd(), '.tmp')
results = 'results'

def get_file_type(file):
    '''
    判断文件类型：txt, zip,  rar
    '''
    if os.path.getsize(file) < 8:
        return 'txt'

    with open(file, 'rb') as f:
        bs = f.read(8)
        if bs[0] == 0x50 and bs[1] == 0x4B and bs[2] == 0x03:
            return 'zip'
        elif bs[0] == 0x52 and bs[1] == 0x61 and bs[2] == 0x72:
            return 'rar'
        else:
            return 'txt'

def readonly_handler(func, path, execinfo):
    '''
    PermissionError: [WinError 5] 拒绝访问
    '''
    os.chmod(path, stat.S_IWRITE)
    func(path)

def mk_tmp():
    if os.path.exists(tmp):
        shutil.rmtree(tmp, onerror=readonly_handler)
        os.mkdir(tmp)
    else:
        os.mkdir(tmp)

def unfile(file):
    mk_tmp()
    # os.popen(f"7z x {file} -y -o{tmp}")
    ex=subprocess.Popen(f"7z x {file} -y -o{tmp}",
                          stdout = subprocess.PIPE, shell = True)
    out, err=ex.communicate()
    status=ex.wait()
    # print("cmd out: ", out.decode('gbk'))

def yield_tmp():
    for path, dir_list, file_list in os.walk('.tmp'):
        for file in file_list:
            f=os.path.join(path, file)
            yield f

def get_files(file):
    t=get_file_type(file)

    if t == 'txt':
        yield file
        return

    if t == 'rar' or t == 'zip':
        unfile(file)

    for f in yield_tmp():
        yield f

def da(file):
    d=data()
    r=re.compile(r'\b([A-Za-z]+)\b')
    rl = re.compile(r"}([a-zA-Z\d\s,\.?!']+)\n") # 取对白部分

    with open(file, mode = 'rt', encoding = 'utf-8', errors = 'ignore') as f:
        fl = f.readline().replace('\x00', '')
        if '[Script Info]' in fl:
            suf = 'ass'
        else:
            suf = 'srt'

        for line in f:
            line=line.replace('\x00', '')

            if suf=='ass':
                line = line.replace('{\\r}', '')
                m = rl.search(line)
                if m:
                    line = m.group(1)
                else:
                    continue

            # print(line)
            for m in r.finditer(line):
                w=m.group(1)
                d.add_word(w)

    return d

if __name__ == '__main__':
    if(os.path.exists(results)):
        shutil.rmtree(results)
    os.mkdir(results)

    dl=list()
    for path, dir_list, file_list in os.walk(ds):
        for file in file_list:
            f=os.path.join(path, file)
            for fp in get_files(f):
                print(fp)
                d=da(fp)
                d.save(os.path.basename(fp))
                dl.append(d)
    data.save_total(dl)


from __future__ import print_function
import os
import re
import sys
from setuptools import setup, find_packages
from distutils.command.build_py import build_py as _build_py
from glob import glob
from os.path import join

python_requires = (3, 6)

if sys.version_info < python_requires:
    raise SystemExit('Python 3.6 or later is required!')

install_requires = [
    'numpy>=1.11.3',
    'scipy>=0.18.1',
    'matplotlib>=2.0.0',
]

extras_require = {
    'docs': [
        'sphinx',
        'sphinx_rtd_theme',
        'pillow',
    ],
    'test': [
        'pytest>=3.6.1',
        'pytest-xdist>=1.22.1',
    ]
}

with open('README.rst') as fd:
    long_description = fd.read()

# Get the current version number:
with open('ase/__init__.py') as fd:
    version = re.search("__version__ = '(.*)'", fd.read()).group(1)

package_data = {'ase': ['spacegroup/spacegroup.dat',
                        'collections/*.json',
                        'db/templates/*',
                        'db/static/*'],
                'ase.test': ['pytest.ini',
                             'data/*']}

class build_py(_build_py):
    """Custom distutils command to build translations."""
    def __init__(self, *args, **kwargs):
        _build_py.__init__(self, *args, **kwargs)
        # Keep list of files to appease bdist_rpm.  We have to keep track of
        # all the installed files for no particular reason.
        self.mofiles = []

    def run(self):
        """Compile translation files (requires gettext)."""
        _build_py.run(self)
        msgfmt = 'msgfmt'
        status = os.system(msgfmt + ' -V')
        if status == 0:
            for pofile in sorted(glob('ase/gui/po/*/LC_MESSAGES/ag.po')):
                dirname = join(self.build_lib, os.path.dirname(pofile))
                if not os.path.isdir(dirname):
                    os.makedirs(dirname)
                mofile = join(dirname, 'ag.mo')
                print()
                print('Compile {}'.format(pofile))
                status = os.system('%s -cv %s --output-file=%s 2>&1' %
                                   (msgfmt, pofile, mofile))
                assert status == 0, 'msgfmt failed!'
                self.mofiles.append(mofile)

    def get_outputs(self, *args, **kwargs):
        return _build_py.get_outputs(self, *args, **kwargs) + self.mofiles

setup(name='ase-koopmans',
      version=version,
      description='Atomic Simulation Environment for Koopmans',
      url='https://github.com/elinscott/ase_koopmans',
      maintainer='Edward Linscott',
      maintainer_email='edwardlinscott@gmail.com',
      license='LGPLv2.1+',
      platforms=['unix'],
      packages=find_packages(),
      install_requires=install_requires,
      extras_require=extras_require,
      package_data=package_data,
      entry_points={'console_scripts': ['ase=ase.cli.main:main',
                                        'ase-db=ase.cli.main:old',
                                        'ase-gui=ase.cli.main:old',
                                        'ase-run=ase.cli.main:old',
                                        'ase-info=ase.cli.main:old',
                                        'ase-build=ase.cli.main:old']},
      long_description=long_description,
      cmdclass={'build_py': build_py},
      classifiers=[
          'Development Status :: 2 - Pre-Alpha',
          'License :: OSI Approved :: '
          'GNU Lesser General Public License v2 or later (LGPLv2+)',
          'Operating System :: OS Independent',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.6',
          'Programming Language :: Python :: 3.7',
          'Topic :: Scientific/Engineering :: Physics'])


import os
from data.raw_data_loader_settings import *
import h5py
import logging
import numpy as np
import pandas as pd
import time

logger = logging.getLogger(__name__)

# TODO: process all files to standard format

class DataProcessor:

    def __init__(self, name, ReferenceDataLoader, RawDataLoader, org_structure=FileOrgStructure.DATECOLUMN):
        t = time.perf_counter()
        self.name = name
        self.outputpath = OutputDataPath
        self.ReferenceDataLoader = ReferenceDataLoader
        self.PVDataLoader = RawDataLoader
        self.org_structure = org_structure
        self.processed_halt_date = None
        self.processed_st_date = None
        self.processed_columns = set()

    def process_with_args(self, args):
        temp_names = args[0]
        temp_data_processors = args[1]
        temp_date = args[2]

        try:
            self.process(temp_data_processors, temp_date, temp_names)
            print("calculating for " + str(temp_date) + " " + str(temp_names))
        except:
            print("skipping for " + str(temp_date) + " " + str(temp_names))

    def process(self, pv_data_processors, date, names, pv_data="1min_PV"):

        if self.processed_halt_date is None:
            self.processed_halt_date = self.process_halt_date()
            self.processed_st_date = self.process_st_date()

        t = time.perf_counter()
        pv_data = self.PVDataLoader.load_pv_data(pv_data, date, date)
        logger.debug(f"load_pv_data finished in {time.perf_counter() - t} seconds")
        for pv_data_processor, name in zip(pv_data_processors, names):
            processed_daily_pv = self.process_1min_pv_to_daily(pv_data_processor, pv_data)
            logger.debug(f"process_1min_pv_to_daily finished in {time.perf_counter() - t} seconds")
            processed_daily_pv = processed_daily_pv.reset_index().rename(columns={"index": "code"})
            processed_daily_pv["code"] = processed_daily_pv["code"].astype(np.string_)
            self.write_data(date, processed_daily_pv, name, self.org_structure)
        logger.debug(f"wrote data in {time.perf_counter() - t} seconds")

    """ Helper Functions """

    def write_data(self, date, daily_df, name, org_structure):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        with h5py.File(self.outputpath + "\\" + name + ".hdf5", 'a') as f:
            dategroup = f.create_group(str(date))
            for column in daily_df.columns:
                try:
                    dategroup.create_dataset(column, data=daily_df[column].to_numpy())
                except:
                    dategroup.create_dataset(column, data=daily_df[column].to_numpy().astype(np.float64))
                    logger.info(f"forced conversion on {date} for {column}")
                self.processed_columns.add(column)
            f.close()

    @classmethod
    def write_performance_data(cls, name, performance_df, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        with h5py.File(PerformanceOutputPath + "\\" + name + ".hdf5", 'w') as f:
            for column in performance_df.columns:
                try:
                    f.create_dataset(column, data=performance_df[column].to_numpy())
                except:
                    f.create_dataset(column, data=performance_df[column].to_numpy().astype(np.float64))
            f.close()

    @classmethod
    def write_alpha_data_all(cls, daily_df, name, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        all_dates = list(daily_df["date"].drop_duplicates())
        directory = AlphaOutputPath + "\\" + name
        if not os.path.exists(directory):
            os.mkdir(directory)
        for date in all_dates:
            data = daily_df[daily_df["date"] == date]
            with h5py.File(directory + "\\" + str(date) + ".hdf5", 'w') as f:
                for column in daily_df.columns:
                    try:
                        f.create_dataset(column, data=data[column].to_numpy())
                    except:
                        f.create_dataset(column, data=data[column].to_numpy().astype(np.float64))
                f.close()

    @classmethod
    def write_alpha_data(cls, date, daily_df, name, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        directory = AlphaOutputPath + "\\" + name
        if not os.path.exists(directory):
            os.mkdir(directory)
        with h5py.File(directory + "\\" + date + ".hdf5", 'w') as f:
            for column in daily_df.columns:
                try:
                    f.create_dataset(column, data=daily_df[column].to_numpy())
                except:
                    f.create_dataset(column, data=daily_df[column].to_numpy().astype(np.float64))
            f.close()

    def process_halt_date(self):
        halt_date = self.ReferenceDataLoader.load_reference_data_by_name("HaltDate").copy()
        halt_date["restart_inclusive"] = halt_date.apply(lambda x: x["restart_time"] > 93000, axis=1)
        trade_date = self.ReferenceDataLoader.load_reference_data_by_name("Calendar").copy()
        trade_date = trade_date[trade_date["is_open"] == 1]

        output = []
        for row in halt_date.iterrows():
            code = row[1]["code"]
            halt_start = row[1]["halt_date"]
            restart_date = row[1]["restart_date"]
            if row[1]["restart_inclusive"]:
                tmp = trade_date[(trade_date["date"] >= halt_start) & (trade_date["date"] <= restart_date)].assign(
                    code=code).assign(is_halt=1)
            else:
                tmp = trade_date[(trade_date["date"] >= halt_start) & (trade_date["date"] < restart_date)].assign(
                    code=code).assign(is_halt=1)
            output.append(tmp[["code", "date", "is_halt"]])
        return pd.concat(output)

    def process_st_date(self):

        trade_date = self.ReferenceDataLoader.load_reference_data_by_name("Calendar").copy()
        trade_date_open = trade_date[trade_date["is_open"] == 1][["date", "is_open"]].set_index("date")
        trade_date_all = trade_date[["date", "is_open"]].set_index("date")

        st_status = self.ReferenceDataLoader.load_reference_data_by_name("STDate").copy()[
            ["code", "status_id", "eff_date"]]
        st_status = st_status.pivot_table(index="eff_date", columns="code", values="status_id")
        st_status = trade_date_open.join(st_status).ffill().drop(columns="is_open")
        st_status = trade_date_all.join(st_status).drop(columns="is_open")
        st_status = st_status.stack().reset_index()
        st_status.columns = ["date", "code", "status_id"]
        st_status["date"] = st_status["date"].astype("int32")
        st_status["status_id"] = st_status["status_id"].astype("int32")
        return st_status

    def process_1min_pv_to_daily(self, pv_data_processor, pv_data):

        processed_1min_pv = pv_data_processor(pv_data)
        current_base = self.ReferenceDataLoader.load_reference_data_by_name("ListDelistDate").copy().set_index("code")
        joined_df = current_base.join(processed_1min_pv)
        return joined_df[processed_1min_pv.columns]

    def add_reference_data(self, processed_daily_pv, date):

        cumulative_adjustment_factor = \
            self.ReferenceDataLoader.load_reference_data_by_name("CumulativeAdjustmentFactor")[
                self.ReferenceDataLoader.load_reference_data_by_name("CumulativeAdjustmentFactor")["date_int"] == int(
                    date)].set_index("code")[
                ["cum_adjf"]]
        cumulative_adjustment_factor["cum_adjf"] = cumulative_adjustment_factor["cum_adjf"].astype(np.float64)

        halt_date = self.processed_halt_date.copy()
        halt_date = halt_date[halt_date["date"] == int(date)].set_index("code")[["is_halt"]]
        halt_date["is_halt"] = halt_date["is_halt"].astype(np.float64)

        current_status = self.ReferenceDataLoader.load_reference_data_by_name("ListDelistDate").copy()[
            ["code", "lst_status", "lst_date", "delst_date"]].set_index("code")
        current_status["lst_status"] = current_status["lst_status"].astype(np.string_)
        current_status["lst_date"] = current_status["lst_date"].astype(np.float64)
        current_status["delst_date"] = current_status["delst_date"].astype(np.float64)

        st_date = self.processed_st_date.copy()
        st_date = st_date[st_date["date"] == int(date)].set_index("code")[["status_id"]]
        st_date["status_id"] = st_date["status_id"].astype(np.float64)

        sector_classification = self.ReferenceDataLoader.load_reference_data_by_name("Sector")[
            self.ReferenceDataLoader.load_reference_data_by_name("Sector")["date_int"] == int(date)].set_index("code")[
            ["sw1",
             "sw2",
             "sw3"]]

        limit_price = self.ReferenceDataLoader.load_reference_data_by_name("LimitPrices")[
            self.ReferenceDataLoader.load_reference_data_by_name("LimitPrices")["date_int"] == int(date)].set_index(
            "code")[["up_limit",
                     "down_limit"]]
        limit_price["up_limit"] = limit_price["up_limit"].astype(np.float64)
        limit_price["down_limit"] = limit_price["down_limit"].astype(np.float64)

        market_value = self.ReferenceDataLoader.load_reference_data_by_name("MarketValue")[
            self.ReferenceDataLoader.load_reference_data_by_name("MarketValue")["date_int"] == int(date)].set_index(
            "code")[["neg_mkt_val",
                     "mkt_val",
                     "neg_shares",
                     "shares"]]
        market_value["neg_mkt_val"] = market_value["neg_mkt_val"].astype(np.float64)
        market_value["mkt_val"] = market_value["mkt_val"].astype(np.float64)
        market_value["neg_shares"] = market_value["neg_shares"].astype(np.float64)
        market_value["shares"] = market_value["shares"].astype(np.float64)

        processed_daily_pv = processed_daily_pv.join(cumulative_adjustment_factor)
        processed_daily_pv = processed_daily_pv.join(halt_date)
        processed_daily_pv = processed_daily_pv.join(current_status)
        processed_daily_pv = processed_daily_pv.join(st_date)

        processed_daily_pv = processed_daily_pv.join(sector_classification)
        processed_daily_pv[["sw1", "sw2", "sw3"]] = processed_daily_pv[["sw1", "sw2", "sw3"]].fillna(value="")
        processed_daily_pv["sw1"] = processed_daily_pv["sw1"].str.encode('utf-8').astype(np.string_)
        processed_daily_pv["sw2"] = processed_daily_pv["sw2"].str.encode('utf-8').astype(np.string_)
        processed_daily_pv["sw3"] = processed_daily_pv["sw3"].str.encode('utf-8').astype(np.string_)

        processed_daily_pv = processed_daily_pv.join(limit_price)
        processed_daily_pv = processed_daily_pv.join(market_value)
        processed_daily_pv = processed_daily_pv.reset_index()
        processed_daily_pv["code"] = processed_daily_pv["code"].astype(np.string_)

        return processed_daily_pv


# patricia.roxanne.kwarwp_v2.py
# SPDX-License-Identifier: GPL-3.0-or-later
# patricia.roxanne.kwarwp.py
# SPDX-License-Identifier: GPL-3.0-or-later

"""
Meu Kwarwp Isaac
.. codeauthor:: Isaac D'Césares <isaac.dcesares@gmail.com>
Changelog
---------
.. versionadded::    20.08
        Classe Índio

.. versionadded::    20.07
        Criando novo Kwarwp!
"""
from collections import namedtuple as nt

IMGUR = "https://imgur.com/"

MAPA_INICIO = """
@....&
......
......
.#.^..
"""

MAPA_CERCA = """
%%%%%%%
%..%..&
%.....%
%^.%..%
%%%%%%%
"""

"""Mapa com o posicionamento inicial dos elementos."""
Ponto = nt("Ponto", "x y")
"""Par de coordenadas na direção horizontal (x) e vertiacal (y)."""
Rosa = nt("Rosa", "n l s o")
"""Rosa dos ventos com as direções norte, leste, sul e oeste."""

class Indio():
    """ Cria o personagem principal na arena do Kwarwp na posição definida.
        :param imagem: A figura representando o índio na posição indicada.
        :param x: Coluna em que o elemento será posicionado.
        :param y: Cinha em que o elemento será posicionado.
        :param cena: Cena em que o elemento será posicionado.
    """

    def __init__(self, imagem, x, y, cena):
        self.lado = lado = Kwarwp.LADO
        self.indio = Kwarwp.VITOLLINO.a(imagem, w=lado, h=lado, x=x, y=y, cena=cena)

class Kwarwp():
    """ Arena onde os desafios ocorrem.

        :param vitollino: Empacota o engenho de jogo Vitollino.
        :param mapa: Um texto representando o mapa do desafio.
    """
    VITOLLINO = None
    """Referência estática para obter o engenho de jogo."""
    LADO = None
    """Referência estática para definir o lado do piso da casa."""

    def __init__(self, vitollino=None, mapa=MAPA_INICIO, medidas={}):
        Kwarwp.VITOLLINO = self.v = vitollino()
        """Cria um matriz com os elementos descritos em cada linha de texto"""
        self.mapa = mapa.split()
        """Largura da casa da arena dos desafios, número de colunas no mapa"""
        self.lado, self.col, self.lin = 100, len(self.mapa[0]), len(self.mapa)+1
        Kwarwp.LADO = self.lado
        w, h = self.col *self.lado, self.lin *self.lado
        self.taba = {}
        """Dicionário que a partir de coordenada (i,J) localiza um piso da taba"""
        medidas.update(width=w, height=f"{h}px")
        self.cena = self.cria(mapa=self.mapa) if vitollino else None

    def cria(self, mapa=""):
        """ Fábrica de componentes.

            :param mapa: Um texto representando o mapa do desafio.
        """
        Fab = nt("Fab", "objeto imagem")
        """Esta tupla nomeada serve para definir o objeto construido e sua imagem."""

        fabrica = {
        "&": Fab(self.coisa, f"{IMGUR}dZQ8liT.jpg"), # OCA
        "^": Fab(self.indio, f"{IMGUR}8jMuupz.png"), # INDIO
        ".": Fab(self.vazio, f"{IMGUR}npb9Oej.png"), # VAZIO
        "_": Fab(self.coisa, f"{IMGUR}sGoKfvs.jpg"), # SOLO
        "#": Fab(self.coisa, f"{IMGUR}ldI7IbK.png"), # TORA
        "@": Fab(self.coisa, f"{IMGUR}tLLVjfN.png"), # PICHE
        "~": Fab(self.coisa, f"{IMGUR}UAETaiP.gif"), # CEU
        "*": Fab(self.coisa, f"{IMGUR}PfodQmT.gif"), # SOL
        "|": Fab(self.coisa, f"{IMGUR}uwYPNlz.png")  # CERCA
        }
        """Dicionário que define o tipo e a imagem do objeto para cada elemento."""
        mapa = mapa if mapa != "" else self.mapa
        """Cria um cenário com imagem de terra de chão batido, céu e sol"""
        mapa = self.mapa
        lado = self.lado
        print(f"cria(self, mapa={mapa}, col={len(self.mapa[0])}")
        cena = self.v.c(fabrica["_"].imagem)
        ceu = self.v.a(fabrica["~"].imagem, w=lado*self.col, h=lado, x=0, y=0, cena=cena)
        sol = self.v.a(fabrica["*"].imagem, w=60, h=60, x=0, y=40, cena=cena)
        self.taba = {(i, j): fabrica[imagem].objeto(fabrica[imagem].imagem, x=i*lado, y=j*lado+lado, cena=cena)
            for j, linha in enumerate(mapa) for i, imagem in enumerate(linha)}
        """Posiciona os elementos segundo suas posições i, j na matriz mapa"""
        cena.vai()
        return cena

    def coisa(self, imagem, x, y, cena):
        """ Cria um elemento na arena do Kwarwp na posição definida.
            :param x: coluna em que o elemento será posicionado.
            :param y: linha em que o elemento será posicionado.
            :param cena: cena em que o elemento será posicionado.
        """
        lado = self.lado
        return self.v.a(imagem, w=lado, h=lado, x=x, y=y, cena=cena)

    def vazio(self, imagem, x, y, cena):
        """ Cria um espaço vazio na arena do Kwarwp na posição definida.
            :param x: coluna em que o elemento será posicionado.
            :param y: linha em que o elemento será posicionado.
            :param cena: cena em que o elemento será posicionado.
        """
        lado = self.lado
        return self.v.a(imagem, w=lado, h=lado, x=x, y=y, cena=cena)

    def indio(self, imagem, x, y, cena):
        """ Cria o personagem principal na arena do Kwarwp na posição definida.
            :param x: coluna em que o elemento será posicionado.
            :param y: linha em que o elemento será posicionado.
            :param cena: cena em que o elemento será posicionado.
        """
        lado = self.lado
        return Indio(imagem, x=x, y=y, cena=cena)

def main(vitollino, medidas={}):
    """ Rotina principal que invoca a classe Kwarwp.

        :param vitollino: Empacota o engenho de jogo Vitollino.
        :param medidas: Um dicionário usado para redimensionar a tela.
    """
    # print(f"main(vitollino={vitollino} medidas={medidas}")
    Kwarwp(vitollino, medidas=medidas)

if __name__ == "__main__":
    from _spy.vitollino.main import Jogo, STYLE
    STYLE.update(width=600, height="500px")
    main(Jogo, STYLE)


import os

import psycopg2

from flask import current_app
from flask import g

def init_db(app):
    try:
        app.config['db_dsn'] = ('host={DB_HOST} '
                                'dbname={DB_NAME} '
                                'user={DB_USER} '
                                'password={DB_PASSWORD}').format(**os.environ)
    except KeyError as e:
        app.logger.error('Missing database configuration: %s', e)

    @app.teardown_appcontext
    def close_db(error):
        """Close the database at the end of the request"""
        if hasattr(g, 'db'):
            g.db.close()

def connect_db():
    dsn = current_app.config['db_dsn']
    return psycopg2.connect(dsn)

def get_db():
    """Open a new database connection if there is none yet for the
    current application context.
    """
    if not hasattr(g, 'db'):
        g.db = connect_db()
    return g.db


"""Event image handlers."""

from flask import request

from hinews.messages.image import NoSuchImage, ImageDeleted, ImagePatched
from his import authenticated, authorized
from wsgilib import Binary, JSON

from hievents.orm import Image

__all__ = ["ROUTES"]

def get_image(ident):
    """Returns the respective image."""

    try:
        return Image.get(Image.id == ident)
    except Image.DoesNotExist:
        raise NoSuchImage()

@authenticated
@authorized("hievents")
def list_all():
    """Lists all available images."""

    return JSON([image.to_json() for image in Image])

@authenticated
@authorized("hievents")
def get(ident):
    """Returns a specific image."""

    return Binary(get_image(ident).data)

@authenticated
@authorized("hievents")
def delete(ident):
    """Deletes the respective image."""

    get_image(ident).delete_instance()
    return ImageDeleted()

@authenticated
@authorized("hievents")
def patch(ident):
    """Modifies image meta data."""

    image = get_image(ident)
    image.patch_json(request.json)
    image.save()
    return ImagePatched()

ROUTES = (
    ("GET", "/image", list_all, "list_images"),
    ("GET", "/image/<int:ident>", get, "get_image"),
    ("DELETE", "/image/<int:ident>", delete, "delete_image"),
    ("PATCH", "/image/<int:ident>", patch, "patch_image"),
)


import requests
import os
from ..utils import get_headers

DEFAULT_EXECUTE_URL = "https://cloud-api.abstra.cloud/cli/connectors/execute"
EXECUTE_URL = os.getenv("ABSTRA_CONNECTORS_EXECUTE_URL", DEFAULT_EXECUTE_URL)

class ConnectorExecutionError(Exception):  # public api
    def __init__(self, name: str, method_name: str, errors: list) -> None:
        self.name = name
        self.method_name = method_name
        self.errors = errors

class Method:
    def __init__(self, name: str, connector: "Connector") -> None:
        self.name = name
        self.connector = connector

    def __call__(self, **kwargs):  # public api
        j_body = {"name": self.connector.name, "method": self.name, "params": kwargs}

        headers = {
            "Content-Type": "application/json",
            "api-authorization": f"Bearer {os.getenv('ABSTRA_API_TOKEN')}",
        }

        if EXECUTE_URL == DEFAULT_EXECUTE_URL and not os.getenv("ABSTRA_API_TOKEN"):
            raise Exception("You must be logged in to execute a connector method")

        r = requests.post(EXECUTE_URL, headers=headers, json=j_body)
        if not r.ok:
            raise Exception(f"Error executing method {self.name}: {r.text}")

        result = r.json()
        if result["errors"]:
            raise ConnectorExecutionError(
                self.connector.name, self.name, result["errors"]
            )

        return result["returns"]

class Connector:
    def __init__(self, name: str) -> None:
        self.name = name

    def get_method(self, method_name: str) -> "Method":  # public api
        return Method(method_name, self)

    def execute(self, method_name: str, **kwargs):  # public api
        return self.get_method(method_name)(**kwargs)

    def __getattr__(self, method_name: str) -> "Method":
        return self.get_method(method_name)

def get_connector(name: str) -> "Connector":  # public api
    return Connector(name)


# This script was created in 2021 to attempt to book an appointment on the RiteAid site.
# This doesn't do web scraping, instead it calls the json services that are used on the website.
# This was discussed on an episode of Data Science Imposters.
import json
import os
import requests
import time

# The script failed on these zip codes so let's ignore them all together
IGNORE_ZIPS = {'08651', '08351', '08726', '08276', '08851', '08126', '07651', '08451', '07551', '07226', '07151', '07301', '08951', '08426', '08576', '07376', '08151', '07951', '07526', '08301', '07251', '07351'}

STORE_ZIP = dict()

CHECKED_STORES = set()

def get_store_ids(zip_code):
    url = 'https://www.riteaid.com/services/ext/v2/stores/getStores?address={}&attrFilter=PREF-112&radius=25'.format(zip_code)
    #print('trying url: {}'.format(url))
    response = requests.get(url)
    data = json.loads(response.content)
    stores = []
    try:
        for store in data['Data']['stores']:
            if store['state'] == 'NJ':
                stores.append(store['storeNumber'])
                STORE_ZIP[store['storeNumber']] = zip_code
    except:
        IGNORE_ZIPS.add(zip_code)
        return None

    return stores

def get_store_availability(store_number):
    url = 'https://www.riteaid.com/services/ext/v2/vaccine/checkSlots?storeNumber={}'.format(store_number)
    response = requests.get(url)
    data = json.loads(response.content)
    for idx, slot_available in data['Data']['slots'].items():
        if slot_available:
            print('\n\n========Available for ZIP store_number {}, {}==========\n\n'.format(STORE_ZIP[store_number], store_number, idx))
            os.system("beep -f 555 -l 460")

def main():
    for zip in range(7001,8989,50):
        if zip in IGNORE_ZIPS:
            continue

        stores = get_store_ids(str(zip).zfill(5))
        if not stores:
            continue
        #stores = [1977, ]
        for store in stores:
            if store in CHECKED_STORES:
                continue
            else:
                get_store_availability(store)
                CHECKED_STORES.add(store)
        #print('sleeping 5 seconds')
        time.sleep(5)

    print('Ignore these zips:')
    print(IGNORE_ZIPS)

while True:
    main()
    print('sleeping 2  minutes')
    time.sleep(2*60)


import datetime
from kiteconnect import KiteConnect

kite = KiteConnect(api_key="")

def make_zerodha_login(api_key, access_token):
    global kite
    try:
        kite.api_key = api_key
        kite.set_access_token(access_token)
        return 1
    except Exception as e:
        return 0


import numpy as np
from PIL import Image, ImageFilter
from os import path, walk
import pandas as pd
from scipy.ndimage.filters import gaussian_filter
import shutil

def Gaussian33(img):
    kernel = ImageFilter.Kernel((3, 3), (1/16, 1/8, 1/16, 1/8, 1/4, 1/8, 1/16, 1/8, 1/16), 1, 0)
    im2 = img.filter(kernel)
    return im2

def Laplacian(img):
    kernel = ImageFilter.Kernel((3, 3), (-1, -1, -1, -1, 8, -1, -1, -1, -1), 1, 0)
    im2 = img.filter(kernel)
    return im2

def sharpe(dir, filename, label):
    img = Image.open(path.join(dir, filename)).convert('L')
    print(path.join(dir, filename))
    # crop
    (height, width) = img.size
    (top, idx) = (0, 1)
    while top + 32 <= height:
        left = 0
        while left + 32 <= width:
            imgCrop = img.crop((top, left, top + 32, left + 32))
            left += 32
            imgLP = Gaussian33(imgCrop)
            imgLP = Laplacian(imgLP)
            imgLP = np.array(imgLP)
            shar = np.var(imgLP)
            if shar < 2000:
                continue
            fn = str(idx) + '_' + filename
            dd = dir.replace('image', 'imageSharp')
            imgCrop.save(path.join(dd, fn))
            label['file'].append(fn)
            label['sharp'].append(shar)
            idx += 1
        top += 32

def sharpAll(rootdir):
    label = { 'file': [], 'sharp': [] }
    labelPath = rootdir.replace('image', 'sharp.csv')
    for subdir, dirs, files in walk(rootdir):
        for file in files:
            pd.DataFrame(label).to_csv(labelPath, index=False)
            sharpe(subdir, file, label)

def select_sharp(sharp):
    rootdir = path.join('data', 'train', 'image')
    labelPath = rootdir.replace('image', 'sharp2.csv')
    df = pd.read_csv(labelPath)
    print(df.shape)
    data = df.loc[df['sharp'] > sharp].reset_index(drop=True)
    print(data.shape)
    # labelPath = rootdir.replace('image', 'sharp4.csv')
    # data.to_csv(labelPath, index=False)

    rootdir = path.join('data', 'test', 'image')
    labelPath = rootdir.replace('image', 'sharp2.csv')
    df = pd.read_csv(labelPath)
    print(df.shape)
    data = df.loc[df['sharp'] > sharp].reset_index(drop=True)
    print(data.shape)
    # labelPath = rootdir.replace('image', 'sharp4.csv')
    # data.to_csv(labelPath, index=False)

    rootdir = path.join('data', 'valid', 'image')
    labelPath = rootdir.replace('image', 'sharp2.csv')
    df = pd.read_csv(labelPath)
    print(df.shape)
    data = df.loc[df['sharp'] > sharp].reset_index(drop=True)
    print(data.shape)
    # labelPath = rootdir.replace('image', 'sharp4.csv')
    # data.to_csv(labelPath, index=False)

def move_to_valid():
    rootdir = path.join('data', 'train', 'image')
    labelPath = rootdir.replace('image', 'sharp2.csv')
    df = pd.read_csv(labelPath)
    N = int(df.shape[0] / 5 * 4)
    train_df = df.iloc[:N,:].reset_index(drop=True)
    valid_df = df.iloc[N:,:].reset_index(drop=True)
    train_df.to_csv(labelPath, index=False)
    labelPath = labelPath.replace('train', 'valid')
    valid_df.to_csv(labelPath, index=False)

    from_dir = rootdir.replace('image', 'imageSharp')
    to_dir = from_dir.replace('train', 'valid')
    for file in valid_df['file']:
        print(file)
        shutil.move(path.join(from_dir, file), path.join(to_dir, file))

def check(rootdir):
    for subdir, dirs, files in walk(rootdir):
        print(len(files))
    sharpfile = rootdir.replace('imageBlur5000', 'groundTruth5000.csv')
    df = pd.read_csv(sharpfile)
    print(df.shape)

def createLabel(rootdir):
    sharpfile = rootdir.replace('imageSharp', 'sharp2.csv')
    df = pd.read_csv(sharpfile)
    labelDf = { 'file': [], 'label': [] }
    for file in df['file']:
        blur(rootdir, file, labelDf)
    trainGT = pd.DataFrame(labelDf)
    trainGT.to_csv(rootdir.replace('imageSharp', 'groundTruth5000.csv'), index=False)

def blur(dir, filename, gt):
    img = Image.open(path.join(dir, filename)).convert('L')
    # print(path.join(dir, filename)
    imgArr = np.array(img)
    # for i in range(20):
    z = int(np.random.rand(1)[0] * 20)
    newArr = gaussian_filter(imgArr, sigma=z)
    newArr.astype(int)
    data = Image.fromarray(newArr, 'L')
    # fn = str(i) + '_' + filename
    fn = filename
    dd = dir.replace('imageSharp', 'imageBlur5000')
    gt['file'].append(fn)
    gt['label'].append(z)
    data.save(path.join(dd, fn))

if __name__ == '__main__':
    # rootdir = path.join('data', 'train', 'image')
    # sharpAll(rootdir)

    # rootdir = path.join('data', 'test', 'image')
    # sharpAll(rootdir)

    select_sharp(5000)
    # move_to_valid()

    createLabel(path.join('data', 'train', 'imageSharp'))
    createLabel(path.join('data', 'valid', 'imageSharp'))
    createLabel(path.join('data', 'test', 'imageSharp'))

    check(path.join('data', 'train', 'imageBlur5000'))
    check(path.join('data', 'valid', 'imageBlur5000'))
    check(path.join('data', 'test', 'imageBlur5000'))


#coding=utf-8
##异步的操作都需要用await 也就是阻塞声明
#拿到章节的url=http://dushu.baidu.com/api/pc/getCatalog?data={%22book_id%22:%224306063500%22}
###协程爬取的注意事项
#1.必须创建一个任务集合 里面保存的是异步任务函数的调用 tasks['xxx()',xxx()]
#2.异步函数 必须用async声明 并且使用的使用需要用await进行阻塞
import  asyncio
import re
import requests
import aiohttp
import aiofiles
async def download_chapter(cid,cname):
    chapter_url="http://dushu.baidu.com/api/pc/getChapterContent?data={%22book_id%22:%224306063500%22,%22cid%22:%224306063500|"+cid+"%22,%22need_bookinfo%22:1}"

    chapter_url=re.sub("%22","\"",chapter_url)
    async with aiohttp.ClientSession() as session:
        async with session.get(url=chapter_url) as result:
            dic= await result.json()
            #print(dic)
            async with aiofiles.open(f"./novel/{cname}.txt","w",encoding="utf-8") as f:
                await  f.write(dic["data"]["novel"]["content"])

async def test():
    form_url = "http://dushu.baidu.com/api/pc/getCatalog?data={%22book_id%22:%224306063500%22}"
    form_url = re.sub("%22", "\"", form_url)
    result = requests.get(form_url)
    tasks=[]#这个为异步任务
    for it in result.json()["data"]["novel"]["items"]:
        tasks.append(download_chapter(it["cid"],it["title"]))
    await asyncio.wait(tasks)

if __name__ == '__main__':
    asyncio.run(test())

基础练习/协程爬取多章小说.py


import geopandas as gpd
from shapely.geometry import Point
import json

def calculate_distance(city1, city2, cities_df):
    point1 = cities_df.loc[cities_df['city'] == city1]['geometry'].values[0]
    point2 = cities_df.loc[cities_df['city'] == city2]['geometry'].values[0]
    return point1.distance(point2)

def create_distance_matrix(city_pairs, cities_datafile):
    cities = list(set([city for pair in city_pairs for city in pair]))
    n = len(cities)
    with open(cities_datafile) as f:
        cities_data = json.load(f)
    print(cities_data.columns())
    cities_df = gpd.GeoDataFrame(cities_data, crs='EPSG:4326', geometry=[Point(xy) for xy in cities_data['coordinates']])
    distances = np.zeros((n, n))
    for i in range(n):
        for j in range(i+1, n):
            city1, city2 = cities[i], cities[j]
            if (city1, city2) in city_pairs:
                distance = calculate_distance(city1, city2, cities_df)
                distances[i][j] = distance
                distances[j][i] = distance
    return distances, cities

def clarke_and_wright(num_cars, capacity, distances):

    n = len(distances)
    savings = np.zeros((n, n))
    for i in range(n):
        for j in range(i+1, n):
            savings[i][j] = distances[i][0] + distances[0][j] - distances[i][j]

    indices = [(i, j) for i in range(1, n) for j in range(i+1, n)]
    sorted_indices = sorted(indices, key=lambda x: savings[x[0]][x[1]], reverse=True)

    routes = [[0] * (capacity + 1) for _ in range(num_cars)]
    remaining_capacity = [capacity for _ in range(num_cars)]
    penalties = [0 for _ in range(num_cars)]
    distances_traveled = [0 for _ in range(num_cars)]
    empty_distances_traveled = [0 for _ in range(num_cars)]

    for (i, j) in sorted_indices:
        for k in range(num_cars):
            if (i in routes[k]) and (j not in routes[k]) and (remaining_capacity[k] >= 1):
                index = routes[k].index(i)
                routes[k].insert(index+1, j)
                distance = distances[i][j]
                distances_traveled[k] += distance
                remaining_capacity[k] -= 1
                if remaining_capacity[k] > 0:
                    penalties[k] = remaining_capacity[k]
                    empty_distances_traveled[k] += distance
                break

            elif (j in routes[k]) and (i not in routes[k]) and (remaining_capacity[k] >= 1):

                index = routes[k].index(j)
                routes[k].insert(index, i)
                distance = distances[i][j]
                distances_traveled[k] += distance
                remaining_capacity[k] -= 1
                if remaining_capacity[k] > 0:
                    penalties[k] = remaining_capacity[k]
                    empty_distances_traveled[k] += distance
                break

    for k in range(num_cars):
        for i in range(1, n):
            if i not in routes[k]:
                min_penalty = min(penalties)
                index = penalties.index(min_penalty)
                routes[index].append(i)
                distance = distances[routes[index][-2]][i]
                distances_traveled[index] += distance
                remaining_capacity[index] -= 1
                if remaining_capacity[index] > 0:
                    penalties[index] = remaining_capacity[index]
                    empty_distances_traveled[index] += distance
                else:
                    penalties[index] = 0

    return routes, distances_traveled, empty_distances_traveled


import torch
import torch.nn as nn
from torch.nn.parallel import DataParallel, DistributedDataParallel

import os
import copy
import pickle
import numpy as np
from termcolor import colored
from collections import defaultdict
from typing import Any, Dict, List, Optional

from .missing import get_missing_parameters_message, get_unexpected_parameters_message
from .heuristic import align_and_update_state_dicts

from vistem.utils.logger import setup_logger

__all__ = ['Checkpointer']

class Checkpointer:
    def __init__(
        self,
        model: nn.Module,
        save_dir: str = "",
        *,
        save_to_disk: bool = True,
        **checkpointables: object,
    ) -> None:
        if isinstance(model, (DistributedDataParallel, DataParallel)):
            model = model.module

        self.model = model
        self.save_dir = save_dir
        self.save_to_disk = save_to_disk
        self.checkpointables = copy.copy(checkpointables)

        self._logger = setup_logger(__name__)
        if not self.save_to_disk:
            self._logger.warning('No saving checkpoint mode')
        if not self.save_dir:
            self._logger.error('Not clarify saving directory')
            self.save_to_disk = False

    def _load_file(self, filename) -> object:
        if filename.endswith(".pkl"):
            with open(filename, "rb") as f:
                data = pickle.load(f, encoding="latin1")

            if "model" in data and "__author__" in data:
                # file is in Detectron2 model zoo format
                self._logger.info(f"Reading a file from '{data['__author__']}'")
                return data
            else:
                # assume file is from Caffe2 / Detectron1 model zoo
                data = {k: v for k, v in data.items() if not k.endswith("_momentum")}
                return {"model": data, "__author__": "Caffe2", "matching_heuristics": True}

        loaded = torch.load(filename, map_location=torch.device("cpu"))
        if '__author__' not in loaded:
            if filename.split('/')[-1].startswith('efficientnet'):
                loaded = {'model' : loaded, '__author__' : 'lukemelas', 'matching_heuristics' : True} # https://github.com/lukemelas/EfficientNet-PyTorch

        else :
            assert loaded['__author__'] == 'vistem'

        return loaded

    def _load_model(self, checkpoint: Any):
        if checkpoint.get("matching_heuristics", False):
            self._convert_ndarray_to_tensor(checkpoint["model"])
            model_state_dict = self.model.state_dict()
            align_and_update_state_dicts(
                model_state_dict,
                checkpoint["model"],
                c2_conversion=checkpoint.get("__author__", None),
            )
            checkpoint["model"] = model_state_dict

        checkpoint_state_dict = checkpoint.pop("model")
        self._convert_ndarray_to_tensor(checkpoint_state_dict)
        self._strip_prefix_if_present(checkpoint_state_dict, "module.")

        model_state_dict = self.model.state_dict()
        incorrect_shapes = []
        for k in list(checkpoint_state_dict.keys()):
            if k in model_state_dict:
                shape_model = tuple(model_state_dict[k].shape)
                shape_checkpoint = tuple(checkpoint_state_dict[k].shape)
                if shape_model != shape_checkpoint:
                    incorrect_shapes.append((k, shape_checkpoint, shape_model))
                    checkpoint_state_dict.pop(k)

        incompatible = self.model.load_state_dict(checkpoint_state_dict, strict=False)

        return incompatible, incorrect_shapes

    def load(self, path: str, checkpointables: Optional[List[str]] = None) -> object:
        if not path:
            self._logger.warning("No checkpoint found. Initializing model from scratch")
            return {}

        self._logger.info("Loading checkpoint from {}".format(path))
        assert os.path.isfile(path), "Checkpoint {} not found!".format(path)

        checkpoint = self._load_file(path)
        incompatible, incorrect_shapes = self._load_model(checkpoint)
        self._log_incompatible_keys(incompatible, incorrect_shapes)
        for key in self.checkpointables if checkpointables is None else checkpointables:
            if key in checkpoint:
                self._logger.info("Loading {} from {}".format(key, path))
                obj = self.checkpointables[key]
                obj.load_state_dict(checkpoint.pop(key))

        return checkpoint

    def save(self, name: str, **kwargs: Dict[str, str]) -> None:
        data = {}
        data['__author__'] = 'vistem'
        data["model"] = self.model.state_dict()
        for key, obj in self.checkpointables.items():
            data[key] = obj.state_dict()
        data.update(kwargs)

        basename = "{}.pth".format(name)
        save_file = os.path.join(self.save_dir, basename)
        assert os.path.basename(save_file) == basename, basename

        self._logger.info("Saving checkpoint to {}".format(save_file))
        with open(save_file, "wb") as f:
            torch.save(data, f)

        self.tag_last_checkpoint(basename)

    def resume_or_load(self, path: str, *, resume: bool = True) -> object:
        if resume and self.has_checkpoint():
            path = self.get_checkpoint_file()
            return self.load(path)
        else:
            return self.load(path, checkpointables=[])

    def has_checkpoint(self) -> bool:
        save_file = os.path.join(self.save_dir, "last_checkpoint")
        return os.path.isfile(save_file)

    def get_checkpoint_file(self) -> str:
        save_file = os.path.join(self.save_dir, "last_checkpoint")
        try:
            with open(save_file, "r") as f:
                last_saved = f.read().strip()
        except IOError:
            return ""
        return os.path.join(self.save_dir, last_saved)

    def tag_last_checkpoint(self, last_filename_basename: str) -> None:
        save_file = os.path.join(self.save_dir, "last_checkpoint")
        with open(save_file, "w") as f:
            f.write(last_filename_basename)

    def _log_incompatible_keys(self, incompatible, incorrect_shapes) -> None:
        for k, shape_checkpoint, shape_model in incorrect_shapes:
            self._logger.error(
                f"Unable to load '{k}' to the model due to incompatible shapes: "
                f"{shape_checkpoint} in the checkpoint "
                f"but {shape_model} in the model!"
            )

        if incompatible.missing_keys:
            msg = get_missing_parameters_message(incompatible.missing_keys)
            self._logger.info(msg)

        if incompatible.unexpected_keys:
            msg = get_unexpected_parameters_message(incompatible.unexpected_keys)
            self._logger.info(msg)

    def _convert_ndarray_to_tensor(self, state_dict: Dict[str, Any]) -> None:
        for k in list(state_dict.keys()):
            v = state_dict[k]
            if not isinstance(v, np.ndarray) and not isinstance(v, torch.Tensor):
                raise ValueError(f"Unsupported type found in checkpoint! {k}: {type(v)}")
            if not isinstance(v, torch.Tensor):
                state_dict[k] = torch.from_numpy(v)

    def _strip_prefix_if_present(self, state_dict: Dict[str, Any], prefix: str) -> None:
        keys = sorted(state_dict.keys())
        if not all(len(key) == 0 or key.startswith(prefix) for key in keys):
            return

        for key in keys:
            newkey = key[len(prefix) :]
            state_dict[newkey] = state_dict.pop(key)

        try:
            metadata = state_dict._metadata
        except AttributeError:
            pass
        else:
            for key in list(metadata.keys()):
                # for the metadata dict, the key can be:
                # '': for the DDP module, which we want to remove.
                # 'module': for the actual model.
                # 'module.xx.xx': for the rest.

                if len(key) == 0:
                    continue
                newkey = key[len(prefix) :]
                metadata[newkey] = metadata.pop(key)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Note: To use the 'upload' functionality of this file, you must:
#   $ pipenv install twine --dev

import io
import os
import sys
from shutil import rmtree

from setuptools import Command, find_packages, setup

NAME = "papers"
DESCRIPTION = "Simple reference manager in Python"
URL = "https://github.com/jan-matthis/papers"
EMAIL = "mail@jan-matthis.de"
AUTHOR = "Jan-Matthis Lueckmann"
REQUIRES_PYTHON = ">=3.6.0"

REQUIRED = ["unidecode", "jinja2", "markdown2", "pybtex", "requests"]

EXTRAS = {"dev": ["black", "flake8", "isort", "pytest",]}

here = os.path.abspath(os.path.dirname(__file__))

try:
    with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
        long_description = "\n" + f.read()
except FileNotFoundError:
    long_description = DESCRIPTION

about = {}
project_slug = NAME.lower().replace("-", "_").replace(" ", "_")
with open(os.path.join(here, project_slug, "__version__.py")) as f:
    exec(f.read(), about)

class UploadCommand(Command):
    """Support setup.py upload."""

    description = "Build and publish the package."
    user_options = []

    @staticmethod
    def status(s):
        """Prints things in bold."""
        print("\033[1m{0}\033[0m".format(s))

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        try:
            self.status("Removing previous builds…")
            rmtree(os.path.join(here, "dist"))
        except OSError:
            pass

        self.status("Building Source and Wheel (universal) distribution…")
        os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable))

        self.status("Uploading the package to PyPI via Twine…")
        os.system("twine upload dist/*")

        self.status("Pushing git tags…")
        os.system("git tag v{0}".format(about["__version__"]))
        os.system("git push --tags")

        sys.exit()

setup(
    name=NAME,
    version=about["__version__"],
    description=DESCRIPTION,
    long_description=long_description,
    long_description_content_type="text/markdown",
    author=AUTHOR,
    author_email=EMAIL,
    python_requires=REQUIRES_PYTHON,
    url=URL,
    packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]),
    entry_points={
        "console_scripts": [
            "papers=papers.cli:cli_papers",
            "papers-import=papers.cli:cli_import",
            "papers-export=papers.cli:cli_export",
        ],
    },
    install_requires=REQUIRED,
    extras_require=EXTRAS,
    include_package_data=True,
    license="MIT",
    classifiers=[
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: Implementation :: PyPy",
    ],
    cmdclass={"upload": UploadCommand,},
)


from itertools import permutations

if __name__ == '__main__':
    S, k = input().split()
    S = str(S).upper()
    k = int(k)

# Create the list of permutations
perms = list(permutations(S, k))

# Extract the characters from the permutations into a string
perm_strings = []
for perm in perms:
    perm_string = ''
    for char in perm:
        perm_string = perm_string + char
    perm_strings.append(perm_string)

# Sort the list of strings and print each string
perm_strings.sort()
[print(string) for string in perm_strings]