from StringIO import StringIO
from ast import literal_eval
from collections import defaultdict
from contextlib import contextmanager
import logging
import os
import re
from threading import Thread
import uuid

# noinspection PyUnresolvedReferences
from boto.sdb.domain import Domain
# noinspection PyUnresolvedReferences
from boto.s3.bucket import Bucket
# noinspection PyUnresolvedReferences
from boto.s3.connection import S3Connection
# noinspection PyUnresolvedReferences
from boto.sdb.connection import SDBConnection
from boto.sdb.item import Item
import boto.s3
from boto.exception import SDBResponseError, S3ResponseError
import itertools
import time

from toil.jobStores.abstractJobStore import AbstractJobStore, NoSuchJobException, \
    ConcurrentFileModificationException, NoSuchFileException
from toil.batchJob import BatchJob

log = logging.getLogger( __name__ )

# FIXME: Command length is currently limited to 1024 characters

# NB: Number of messages per batchjob is limited to 256-x, 1024 bytes each, with x being the number of
# other attributes in the item

# FIXME: enforce SimpleDB limits early

class AWSJobStore( AbstractJobStore ):
    """
    A batchjob store that uses Amazon's S3 for file storage and SimpleDB for storing batchjob info and
    enforcing strong consistency on the S3 file storage. The schema in SimpleDB is as follows:

    Jobs are stored in the "xyz.jobs" domain where xyz is the name prefix this batchjob store was
    constructed with. Each item in that domain uses the batchjob store batchjob ID (jobStoreID) as the item
    name. The command, memory and cpu fields of a batchjob will be stored as attributes. The messages
    field of a batchjob will be stored as a multivalued attribute.
    """

    # FIXME: Eliminate after consolidating behaviour with FileJobStore

    resetJobInLoadState = True
    """Whether to reset the messages, remainingRetryCount and children attributes of a batchjob when
    it is loaded by loadToilState."""

    def fileExists(self, jobStoreFileID ):
        return bool(self.versions.get_item(item_name=jobStoreFileID, consistent_read=True))

    def jobs( self ):
        for attempt in retry_sdb( ):
            with attempt:
                result = list( self.jobDomain.select(
                    query="select * from `{domain}` ".format( domain=self.jobDomain.name ),
                    consistent_read=True ) )
        jobList = []
        for jobItem in result:
            yield AWSJob.fromItem(jobItem)

    def create( self, command, memory, cpu, disk,updateID=None,
                predecessorNumber=0 ):
        jobStoreID = self._newJobID( )
        log.debug( "Creating batchjob %s for '%s'",
                   jobStoreID, '<no command>' if command is None else command )
        batchjob = AWSJob( jobStoreID=jobStoreID,
                             command=command, memory=memory, cpu=cpu, disk=disk,
                             remainingRetryCount=self._defaultTryCount( ), logJobStoreFileID=None,
                             updateID=updateID, predecessorNumber=predecessorNumber)
        for attempt in retry_sdb( ):
            with attempt:
                assert self.jobDomain.put_attributes( item_name=jobStoreID,
                                                 attributes=batchjob.toItem( ) )
        return batchjob

    def __init__( self, region, namePrefix, config=None ):
        log.debug( "Instantiating %s for region %s and name prefix '%s'",
                   self.__class__, region, namePrefix )
        self.region = region
        self.namePrefix = namePrefix
        self.jobDomain = None
        self.versions = None
        self.files = None
        self.stats = None
        self.db = self._connectSimpleDB( )
        self.s3 = self._connectS3( )
        create = config is not None
        self.jobDomain = self._getOrCreateDomain( 'jobs', create )
        self.versions = self._getOrCreateDomain( 'versions', create )
        self.files = self._getOrCreateBucket( 'files', create, versioning=True )
        self.stats = self._getOrCreateBucket( 'stats', create, versioning=True )
        super( AWSJobStore, self ).__init__( config=config )

    def exists( self, jobStoreID ):
        for attempt in retry_sdb( ):
            with attempt:
                return bool( self.jobDomain.get_attributes( item_name=jobStoreID,
                                                       attribute_name=[ ],
                                                       consistent_read=True ) )
    def getPublicUrl( self,  jobStoreFileID):
        """
        For Amazon SimpleDB requests, use HTTP GET requests that are URLs with query strings.
        http://awsdocs.s3.amazonaws.com/SDB/latest/sdb-dg.pdf
        Create url, check if valid, return.
        """
        key = self.files.get_key( key_name=jobStoreFileID)
        return key.generate_url(expires_in=3600) # one hour

    def getSharedPublicUrl(self, FileName):
        jobStoreFileID = self._newFileID( FileName )
        return self.getPublicUrl(jobStoreFileID)

    def load( self, jobStoreID ):
        # TODO: check if mentioning individual attributes is faster than using *
        for attempt in retry_sdb( ):
            with attempt:
                result = list( self.jobDomain.select(
                    query="select * from `{domain}` "
                          "where itemName() = '{jobStoreID}'".format( domain=self.jobDomain.name,
                                                                   jobStoreID=jobStoreID ),
                    consistent_read=True ) )
        if len(result)!=1:
            raise NoSuchJobException(jobStoreID)
        batchjob = AWSJob.fromItem(result[0])
        if batchjob is None:
            raise NoSuchJobException( jobStoreID )
        log.debug( "Loaded batchjob %s", jobStoreID )
        return batchjob

    def update( self, batchjob ):
        log.debug( "Updating batchjob %s", batchjob.jobStoreID )
        for attempt in retry_sdb( ):
            with attempt:
                assert self.jobDomain.put_attributes( item_name=batchjob.jobStoreID,
                                                 attributes=batchjob.toItem( ) )

    def delete( self, jobStoreID ):
        # remove batchjob and replace with jobStoreId.
        log.debug( "Deleting batchjob %s", jobStoreID )
        for attempt in retry_sdb( ):
            with attempt:
                self.jobDomain.delete_attributes( item_name=jobStoreID )
        for attempt in retry_sdb( ):
            with attempt:
                items = list( self.versions.select(
                    query="select * from `%s` "
                          "where jobStoreID='%s'" % (self.versions.name, jobStoreID),
                    consistent_read=True ) )
        if items:
            log.debug( "Deleting %d file(s) associated with batchjob %s", len( items ), jobStoreID )
            for attempt in retry_sdb( ):
                with attempt:
                    self.versions.batch_delete_attributes( { item.name: None for item in items } )
            for item in items:
                if 'version' in item:
                    self.files.delete_key( key_name=item.name,
                                           version_id=item[ 'version' ] )
                else:
                    self.files.delete_key( key_name=item.name)

    def writeFile( self, jobStoreID, localFilePath ):
        jobStoreFileID = self._newFileID( )
        firstVersion = self._upload( jobStoreFileID, localFilePath )
        self._registerFile( jobStoreFileID, jobStoreID=jobStoreID, newVersion=firstVersion )
        log.debug( "Wrote initial version %s of file %s for batchjob %s from path '%s'",
                   firstVersion, jobStoreFileID, jobStoreID, localFilePath )
        return jobStoreFileID

    @contextmanager
    def writeFileStream( self, jobStoreID ):
        jobStoreFileID = self._newFileID( )
        with self._uploadStream( jobStoreFileID, self.files ) as (writable, key):
            yield writable, jobStoreFileID
        firstVersion = key.version_id
        assert firstVersion is not None
        self._registerFile( jobStoreFileID, jobStoreID=jobStoreID, newVersion=firstVersion )
        log.debug( "Wrote initial version %s of file %s for batchjob %s",
                   firstVersion, jobStoreFileID, jobStoreID )

    @contextmanager
    def writeSharedFileStream( self, sharedFileName ):
        assert self._validateSharedFileName( sharedFileName )
        jobStoreFileID = self._newFileID( sharedFileName )
        oldVersion = self._getFileVersion( jobStoreFileID )
        with self._uploadStream( jobStoreFileID, self.files ) as (writable, key):
            yield writable
        newVersion = key.version_id
        jobStoreId = str( self.sharedFileJobID ) if oldVersion is None else None
        self._registerFile( jobStoreFileID,
                            jobStoreID=jobStoreId, oldVersion=oldVersion, newVersion=newVersion )
        if oldVersion is None:
            log.debug( "Wrote initial version %s of shared file %s (%s)",
                       newVersion, sharedFileName, jobStoreFileID )
        else:
            log.debug( "Wrote version %s of file %s (%s), replacing version %s",
                       newVersion, sharedFileName, jobStoreFileID, oldVersion )

    def updateFile( self, jobStoreFileID, localFilePath ):
        oldVersion = self._getFileVersion( jobStoreFileID )
        newVersion = self._upload( jobStoreFileID, localFilePath )
        self._registerFile( jobStoreFileID, oldVersion=oldVersion, newVersion=newVersion )
        log.debug( "Wrote version %s of file %s from path '%s', replacing version %s",
                   newVersion, jobStoreFileID, localFilePath, oldVersion )

    @contextmanager
    def updateFileStream( self, jobStoreFileID ):
        oldVersion = self._getFileVersion( jobStoreFileID )
        with self._uploadStream( jobStoreFileID, self.files ) as (writable, key):
            yield writable
        newVersion = key.version_id
        self._registerFile( jobStoreFileID, oldVersion=oldVersion, newVersion=newVersion )
        log.debug( "Wrote version %s of file %s, replacing version %s",
                   newVersion, jobStoreFileID, oldVersion )

    def readFile( self, jobStoreFileID, localFilePath ):
        version = self._getFileVersion( jobStoreFileID )
        if version is None: raise NoSuchFileException( jobStoreFileID )
        log.debug( "Reading version %s of file %s to path '%s'",
                   version, jobStoreFileID, localFilePath )
        self._download( jobStoreFileID, localFilePath, version )

    @contextmanager
    def readFileStream( self, jobStoreFileID ):
        version = self._getFileVersion( jobStoreFileID )
        if version is None: raise NoSuchFileException( jobStoreFileID )
        log.debug( "Reading version %s of file %s", version, jobStoreFileID )
        with self._downloadStream( jobStoreFileID, version, self.files ) as readable:
            yield readable

    @contextmanager
    def readSharedFileStream( self, sharedFileName ):
        assert self._validateSharedFileName( sharedFileName )
        jobStoreFileID = self._newFileID( sharedFileName )
        version = self._getFileVersion( jobStoreFileID )
        if version is None: raise NoSuchFileException( jobStoreFileID )
        log.debug( "Read version %s from shared file %s (%s)",
                   version, sharedFileName, jobStoreFileID )
        with self._downloadStream( jobStoreFileID, version, self.files ) as readable:
            yield readable

    def deleteFile( self, jobStoreFileID ):
        version, bucket = self._getFileVersionAndBucket( jobStoreFileID )
        if bucket:
            for attempt in retry_sdb( ):
                with attempt:
                    if version:
                        self.versions.delete_attributes( jobStoreFileID,
                                                         expected_values=[ 'version', version ] )
                    else:
                        self.versions.delete_attributes( jobStoreFileID)

            bucket.delete_key( key_name=jobStoreFileID, version_id=version )
            if version:
                log.debug( "Deleted version %s of file %s", version, jobStoreFileID )
            else:
                log.debug( "Deleted unversioned file %s", version, jobStoreFileID )
        else:
            log.debug( "File %s does not exist", jobStoreFileID)

    def getEmptyFileStoreID( self, jobStoreID ):
        jobStoreFileID = self._newFileID( )
        self._registerFile( jobStoreFileID, jobStoreID=jobStoreID )
        log.debug( "Registered empty file %s for batchjob %s", jobStoreFileID, jobStoreID )
        return jobStoreFileID

    def writeStatsAndLogging( self, statsAndLoggingString ):
        jobStoreFileId = self._newFileID( )
        with self._uploadStream( jobStoreFileId, self.stats, multipart=False ) as (writeable, key):
            writeable.write( statsAndLoggingString )
        firstVersion = key.version_id
        self._registerFile( jobStoreFileId, bucketName='stats', newVersion=firstVersion )

    def readStatsAndLogging( self, statsCallBackFn ):
        itemsProcessed = 0
        for attempt in retry_sdb( ):
            with attempt:
                items = list( self.versions.select(
                    query="select * from `%s` "
                          "where bucketName='stats'" % (self.versions.name,),
                    consistent_read=True ) )
        for item in items:
            with self._downloadStream( item.name, item[ 'version' ], self.stats ) as readable:
                statsCallBackFn( readable )
            self.deleteFile( item.name )
            itemsProcessed += 1
        return itemsProcessed

    # Dots in bucket names should be avoided because bucket names are used in HTTPS bucket
    # URLs where the may interfere with the certificate common name. We use a double
    # underscore as a separator instead.
    bucketNameRe = re.compile( r'^[a-z0-9][a-z0-9-]+[a-z0-9]$' )

    nameSeparator = '--'

    @classmethod
    def _parseArgs( cls, jobStoreString ):
        region, namePrefix = jobStoreString.split( ':' )
        # See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html,
        # reserve 10 characters for separator and suffixes
        if not cls.bucketNameRe.match( namePrefix ):
            raise ValueError( "Invalid name prefix '%s'. Name prefixes must contain only digits, "
                              "hyphens or lower-case letters and must not start or end in a "
                              "hyphen." % namePrefix )
        # reserve 13 for separator and suffix
        if len( namePrefix ) > 50:
            raise ValueError( "Invalid name prefix '%s'. Name prefixes may not be longer than 50 "
                              "characters." % namePrefix )
        if '--' in namePrefix:
            raise ValueError( "Invalid name prefix '%s'. Name prefixes may not contain "
                              "%s." % (namePrefix, cls.nameSeparator) )

        return region, namePrefix

    def _connectSimpleDB( self ):
        """
        rtype: SDBConnection
        """
        db = boto.sdb.connect_to_region( self.region )
        if db is None:
            raise ValueError( "Could not connect to SimpleDB. Make sure '%s' is a valid SimpleDB "
                              "region." % self.region )
        assert db is not None
        return db

    def _connectS3( self ):
        """
        :rtype: S3Connection
        """
        s3 = boto.s3.connect_to_region( self.region )
        if s3 is None:
            raise ValueError( "Could not connect to S3. Make sure '%s' is a valid S3 region." %
                              self.region )
        return s3

    def _getOrCreateBucket( self, bucket_name, create=False, versioning=False ):
        """
        :rtype Bucket
        """
        bucket_name = self.namePrefix + self.nameSeparator + bucket_name
        assert self.bucketNameRe.match( bucket_name )
        assert 3 <= len( bucket_name ) <= 63
        try:
            bucket = self.s3.get_bucket( bucket_name, validate=True )
            assert versioning is self.__getBucketVersioning( bucket )
            return bucket
        except S3ResponseError as e:
            if e.error_code == 'NoSuchBucket' and create:
                bucket = self.s3.create_bucket( bucket_name, location=self.region )
                if versioning:
                    bucket.configure_versioning( versioning )
                return bucket
            else:
                raise

    def _getOrCreateDomain( self, domain_name, create=False ):
        """
        :rtype : Domain
        """
        domain_name = self.namePrefix + self.nameSeparator + domain_name
        for i in itertools.count( ):
            try:
                return self.db.get_domain( domain_name )
            except SDBResponseError as e:
                if e.error_code == 'NoSuchDomain':
                    if i == 0 and create:
                        self.db.create_domain( domain_name )
                    else:
                        log.warn( "Creation of '%s' still pending, retrying in 5s" % domain_name )
                        time.sleep( 5 )

    def _newJobID( self ):
        return str( uuid.uuid4( ) )

    # A dummy batchjob ID under which all shared files are stored.
    sharedFileJobID = uuid.UUID( '891f7db6-e4d9-4221-a58e-ab6cc4395f94' )

    def _newFileID( self, sharedFileName=None ):
        if sharedFileName is None:
            return str( uuid.uuid4( ) )
        else:
            return str( uuid.uuid5( self.sharedFileJobID, str(sharedFileName) ) )

    def _getFileVersionAndBucket( self, jobStoreFileID ):
        """
        :rtype: tuple(str version, AWS bucket)
        """
        for attempt in retry_sdb( ):
            with attempt:
                item = self.versions.get_attributes( item_name=jobStoreFileID,
                                                     attribute_name=[ 'version', 'bucketName' ],
                                                     consistent_read=True )
        bucketName = item.get( 'bucketName', None )
        if bucketName is None:
            return None, None
        else:
            return item.get( 'version', None ), getattr( self, bucketName )

    def _getFileVersion( self, jobStoreFileID, expectedBucket=None ):
        version, bucket = self._getFileVersionAndBucket( jobStoreFileID )
        if bucket is None:
            assert version is None
        else:
            if expectedBucket is None:
                expectedBucket = self.files
            assert bucket is expectedBucket
        return version

    _s3_part_size = 50 * 1024 * 1024

    def _upload( self, jobStoreFileID, localFilePath ):
        file_size, file_time = self._fileSizeAndTime( localFilePath )
        if file_size <= self._s3_part_size:
            key = self.files.new_key( key_name=jobStoreFileID )
            key.name = jobStoreFileID
            key.set_contents_from_filename( localFilePath )
            version = key.version_id
        else:
            with open( localFilePath, 'rb' ) as f:
                upload = self.files.initiate_multipart_upload( key_name=jobStoreFileID )
                try:
                    start = 0
                    part_num = itertools.count( )
                    while start < file_size:
                        end = min( start + self._s3_part_size, file_size )
                        assert f.tell( ) == start
                        upload.upload_part_from_file( fp=f,
                                                      part_num=next( part_num ) + 1,
                                                      size=end - start )
                        start = end
                    assert f.tell( ) == file_size == start
                except:
                    upload.cancel_upload( )
                    raise
                else:
                    version = upload.complete_upload( ).version_id
        key = self.files.get_key( jobStoreFileID )
        assert key.size == file_size
        assert self._fileSizeAndTime( localFilePath ) == (file_size, file_time) #why do this? No one can touch the file while it is uploaded?
        return version

    @contextmanager
    def _uploadStream( self, jobStoreFileID, bucket, multipart=True ):
        key = bucket.new_key( key_name=jobStoreFileID )
        assert key.version_id is None
        readable_fh, writable_fh = os.pipe( )
        with os.fdopen( readable_fh, 'r' ) as readable:
            with os.fdopen( writable_fh, 'w' ) as writable:
                def reader( ):
                    try:
                        upload = bucket.initiate_multipart_upload( key_name=jobStoreFileID )
                        try:
                            for part_num in itertools.count( ):
                                # FIXME: Consider using a key.set_contents_from_stream and rip ...
                                # FIXME: ... the query_args logic from upload_part_from_file in ...
                                # FIXME: ... in MultipartUpload. Possible downside is that ...
                                # FIXME: ... implicit retries won't work.
                                buf = readable.read( self._s3_part_size )
                                # There must be at least one part, even if the file is empty.
                                if len( buf ) == 0 and part_num > 0: break
                                upload.upload_part_from_file( fp=StringIO( buf ),
                                                              # S3 part numbers are 1-based
                                                              part_num=part_num + 1 )
                                if len( buf ) == 0: break
                        except:
                            upload.cancel_upload( )
                            raise
                        else:
                            key.version_id = upload.complete_upload( ).version_id
                    except:
                        log.exception( 'Exception in reader thread' )

                def simpleReader( ):
                    log.debug( "Using single part upload" )
                    try:
                        buf = StringIO( readable.read( ) )
                        assert key.set_contents_from_file( fp=buf ) == buf.len
                    except:
                        log.exception( "Exception in simple reader thread" )

                thread = Thread( target=reader if multipart else simpleReader )
                thread.start( )
                # Yield the key now with version_id unset. When reader() returns
                # key.version_id will be set.
                yield writable, key
            # The writable is now closed. This will send EOF to the readable and cause that
            # thread to finish.
            thread.join( )
            assert key.version_id is not None

    def _download( self, jobStoreFileID, localFilePath, version ):
        key = self.files.get_key( jobStoreFileID, validate=False )
        key.get_contents_to_filename( localFilePath, version_id=version )

    @contextmanager
    def _downloadStream( self, jobStoreFileID, version, bucket ):
        key = bucket.get_key( jobStoreFileID, validate=False )
        readable_fh, writable_fh = os.pipe( )
        with os.fdopen( readable_fh, 'r' ) as readable:
            with os.fdopen( writable_fh, 'w' ) as writable:
                def writer( ):
                    key.get_contents_to_file( writable, version_id=version )
                    # This close() will send EOF to the reading end and ultimately cause the
                    # yield to return. It also makes the implict .close() done by the enclosing
                    # "with" context redundant but that should be ok since .close() on file
                    # objects are idempotent.
                    writable.close( )

                thread = Thread( target=writer )
                thread.start( )
                yield readable
                thread.join( )

    def _registerFile( self, jobStoreFileID,
                       bucketName='files', jobStoreID=None, newVersion=None, oldVersion=None ):
        """
        Register a a file in the store

        :param jobStoreFileID: the file's ID, mandatory

        :param bucketName: the name of the S3 bucket the file was placed in

        :param jobStoreID: the ID of the batchjob owning the file, only allowed for first version of
                           file or when file is registered without content

        :param newVersion: the file's new version or None if the file is to be registered without
                           content, in which case jobStoreId must be passed

        :param oldVersion: the expected previous version of the file or None if newVersion is the
                           first version or file is registered without content
        """
        # Must pass either jobStoreID or newVersion, or both
        assert jobStoreID is not None or newVersion is not None
        # Must pass newVersion if passing oldVersion
        assert oldVersion is None or newVersion is not None
        attributes = dict( bucketName=bucketName )
        if newVersion is not None:
            attributes[ 'version' ] = newVersion
        if jobStoreID is not None:
            attributes[ 'jobStoreID' ] = jobStoreID
        # False stands for absence
        expected = [ 'version', False if oldVersion is None else oldVersion ]
        try:
            for attempt in retry_sdb( ):
                with attempt:
                    assert self.versions.put_attributes( item_name=jobStoreFileID,
                                                         attributes=attributes,
                                                         expected_value=expected )
            if oldVersion is not None:
                bucket = getattr( self, bucketName )
                bucket.delete_key( jobStoreFileID, version_id=oldVersion )
        except SDBResponseError as e:
            if e.error_code == 'ConditionalCheckFailed':
                raise ConcurrentFileModificationException( jobStoreFileID )
            else:
                raise

    def _fileSizeAndTime( self, localFilePath ):
        file_stat = os.stat( localFilePath )
        file_size, file_time = file_stat.st_size, file_stat.st_mtime
        return file_size, file_time

    versionings = dict( Enabled=True, Disabled=False, Suspended=None )

    def __getBucketVersioning( self, bucket ):
        """
        A valueable lesson in how to feck up a simple tri-state boolean.

        For newly created buckets get_versioning_status returns None. We map that to False.

        TBD: This may actually be a result of eventual consistency

        Otherwise, the 'Versioning' entry in the dictionary returned by get_versioning_status can
        be 'Enabled', 'Suspended' or 'Disabled' which we map to True, None and False
        respectively. Calling configure_versioning with False on a bucket will cause
        get_versioning_status to then return 'Suspended' for some reason.
        """
        status = bucket.get_versioning_status( )
        return bool( status ) and self.versionings[ status[ 'Versioning' ] ]

    def deleteJobStore( self ):
        for bucket in (self.files, self.stats):
            if bucket is not None:
                for upload in bucket.list_multipart_uploads( ):
                    upload.cancel_upload( )
                if self.__getBucketVersioning( bucket ) in (True, None):
                    for key in list( bucket.list_versions( ) ):
                        bucket.delete_key( key.name, version_id=key.version_id )
                else:
                    for key in list( bucket.list( ) ):
                        key.delete( )
                bucket.delete( )
        for domain in (self.versions, self.jobDomain):
            if domain is not None:
                domain.delete( )

# Boto converts all attribute values to strings by default, so an attribute value of None would
# becomes 'None' in SimpleDB. To truly represent attribute values of None, we'd have to always
# call delete_attributes in addition to put_attributes but there is no way to do that atomically.
# Instead we map None to the empty string and vice versa. The same applies to empty iterables.
# The empty iterable is a no-op for put_attributes, so we map that to '' instead. This means that
# we can't serialize [''] or '' because the former would be deserialized as [] and the latter as
# None.

def toNoneable( v ):
    return v if v else None

def fromNoneable( v ):
    assert v != ""
    return '' if v is None else v

sort_prefix_length = 3

def toSet( vs ):
    """
    :param vs: list[str] | str
    :return: set(str) | set()

    Lists returned by simpleDB is not guaranteed to be in their original order, but because we are converting them
    to sets, the loss of order is not a problem.

    >>> toSet(["x", "y", "z"])
    set(['y', 'x', 'z'])

    Instead of a set, a single String can also be returned by SimpleDB.

    >>> toSet("x")
    set(['x'])

    An empty set is serialized as ""

    >>> toSet("")
    set([])

    """
    return set(vs) if vs else set()

def fromSet( vs ):
    """
    :type vs: set(str)
    :rtype str|list[str]

    Empty set becomes empty string

    >>> fromSet(set())
    ''

    Singleton set becomes its sole element

    >>> fromSet({'x'})
    'x'

    Set elements are unordered, so sort_prefixes used in fromList are not needed here.

    >>> fromSet({'x','y'})
    ['y', 'x']

    Only sets with non-empty strings are allowed

    >>> fromSet(set(['']))
    Traceback (most recent call last):
    ...
    AssertionError
    >>> fromSet({'x',''})
    Traceback (most recent call last):
    ...
    AssertionError
    >>> fromSet({'x',1})
    Traceback (most recent call last):
    ...
    AssertionError
    """
    if len( vs ) == 0:
        return ""
    elif len( vs ) == 1:
        v = vs.pop()
        assert isinstance( v, basestring ) and v
        return v
    else:
        assert len( vs ) <= 256
        assert all( isinstance( v, basestring ) and v for v in vs )
        return list(vs)

def toList( vs ):
    """
    :param vs: list[str] | str
    :return: list[str] | []

    Lists are not guaranteed to be in their original order, so they are sorted based on a prefixed string.

    >>> toList(["000x", "001y", "002z"])
    ['x', 'y', 'z']

    Instead of a List of length 1, a single String will be returned by SimpleDB.
    A single element is can only have 1 order, no need to sort.

    >>> toList("x")
    ['x']

    An empty list is serialized as ""

    >>> toList("")
    []

    """
    if isinstance( vs, basestring ):
        return [ vs ] if vs else [ ]
    else:
        return [ v[ sort_prefix_length: ] for v in sorted( vs ) ]

def fromList( vs ):
    """
    :type vs: list[str]
    :rtype str|list[str]

    Empty lists becomes empty string

    >>> fromList([])
    ''

    Singleton list becomes its sole element

    >>> fromList(['x'])
    'x'

    Lists elements are prefixed with their position because lists don't retain their order in SDB

    >>> fromList(['x','y'])
    ['000x', '001y']

    Only lists with non-empty strings are allowed

    >>> fromList([''])
    Traceback (most recent call last):
    ...
    AssertionError
    >>> fromList(['x',''])
    Traceback (most recent call last):
    ...
    AssertionError
    >>> fromList(['x',1])
    Traceback (most recent call last):
    ...
    AssertionError
    """
    if len( vs ) == 0:
        return ''
    elif len( vs ) == 1:
        v = vs[ 0 ]
        assert isinstance( v, basestring ) and v
        return v
    else:
        assert len( vs ) <= 256
        assert all( isinstance( v, basestring ) and v for v in vs )
        return [ str( i ).zfill( sort_prefix_length ) + v for i, v in enumerate( vs ) ]

def passThrough( v ): return v

def skip( _ ): return None

class AWSJob( BatchJob ):
    """
    A Batchjob that can be converted to and from a SimpleDB Item
    """
    fromItemTransform = defaultdict( lambda: passThrough,
                                     predecessorNumber=int,
                                     memory=float,
                                     disk=float,
                                     cpu=float,
                                     updateID=str,
                                     command=toNoneable,
                                     stack=lambda v:map( literal_eval, toList( v )),
                                     jobsToDelete=toList,
                                     predecessorsFinished=toSet,
                                     remainingRetryCount=int,
                                     logJobStoreFileID=toNoneable )

    @classmethod
    def fromItem( cls, item, jobStoreID=None ):
        """
        :type item: Item
        :rtype: AWSJob
        """
        if jobStoreID is None: jobStoreID = item.name
        try:
            del item[ 'parentJobStoreID' ]
        except KeyError:
            pass
        item = { k: cls.fromItemTransform[ k ]( v ) for k, v in item.iteritems( ) }
        return cls( jobStoreID=jobStoreID, **item )

    toItemTransform = defaultdict( lambda: passThrough,
                                   command=fromNoneable,
                                   jobStoreID=skip,
                                   updateID=str,
                                   children=skip,
                                   stack=lambda v: fromList( map( repr, v ) ),
                                   logJobStoreFileID=fromNoneable,
                                   predecessorsFinished=fromSet,
                                   jobsToDelete=fromList ,
                                   predecessorNumber=str,
                                   remainingRetryCount=str)

    def toItem( self, parentJobStoreID=None ):
        """
        :rtype: Item
        """
        item = self.toDict( )
        if parentJobStoreID is not None:
            item[ 'parentJobStoreID' ] = parentJobStoreID
        item = ((k, self.toItemTransform[ k ]( v )) for k, v in item.iteritems( ))
        return { k: v for k, v in item if v is not None }

# FIXME: This was lifted from cgcloud-lib where we use it for EC2 retries. The only difference
# FIXME: ... between that code and this is the name of the exception.

a_short_time = 5

a_long_time = 60 * 60

def no_such_domain( e ):
    return e.error_code.endswith( 'NoSuchDomain' )

def true( _ ):
    return True

def false( _ ):
    return False

def retry_sdb( retry_after=a_short_time,
               retry_for=10 * a_short_time,
               retry_while=no_such_domain ):
    """
    Retry an SDB operation while the failure matches a given predicate and until a given timeout
    expires, waiting a given amount of time in between attempts. This function is a generator
    that yields contextmanagers. See doctests below for example usage.

    :param retry_after: the delay in seconds between attempts

    :param retry_for: the timeout in seconds.

    :param retry_while: a callable with one argument, an instance of SDBResponseError, returning
    True if another attempt should be made or False otherwise

    :return: a generator yielding contextmanagers

    Retry for a limited amount of time:
    >>> i = 0
    >>> for attempt in retry_sdb( retry_after=0, retry_for=.1, retry_while=true ):
    ...     with attempt:
    ...         i += 1
    ...         raise SDBResponseError( 'foo', 'bar' )
    Traceback (most recent call last):
    ...
    SDBResponseError: SDBResponseError: foo bar
    <BLANKLINE>
    >>> i > 1
    True

    Do exactly one attempt:
    >>> i = 0
    >>> for attempt in retry_sdb( retry_for=0 ):
    ...     with attempt:
    ...         i += 1
    ...         raise SDBResponseError( 'foo', 'bar' )
    Traceback (most recent call last):
    ...
    SDBResponseError: SDBResponseError: foo bar
    <BLANKLINE>
    >>> i
    1

    Don't retry on success
    >>> i = 0
    >>> for attempt in retry_sdb( retry_after=0, retry_for=.1, retry_while=true ):
    ...     with attempt:
    ...         i += 1
    >>> i
    1

    Don't retry on unless condition returns
    >>> i = 0
    >>> for attempt in retry_sdb( retry_after=0, retry_for=.1, retry_while=false ):
    ...     with attempt:
    ...         i += 1
    ...         raise SDBResponseError( 'foo', 'bar' )
    Traceback (most recent call last):
    ...
    SDBResponseError: SDBResponseError: foo bar
    <BLANKLINE>
    >>> i
    1
    """
    if retry_for > 0:
        go = [ None ]

        @contextmanager
        def repeated_attempt( ):
            try:
                yield
            except SDBResponseError as e:
                if time.time( ) + retry_after < expiration:
                    if retry_while( e ):
                        log.info( '... got %s, trying again in %is ...', e.error_code, retry_after )
                        time.sleep( retry_after )
                    else:
                        log.info( 'Exception failed predicate, giving up.' )
                        raise
                else:
                    log.info( 'Retry timeout expired, giving up.' )
                    raise
            else:
                go.pop( )

        expiration = time.time( ) + retry_for
        while go:
            yield repeated_attempt( )
    else:
        @contextmanager
        def single_attempt( ):
            yield

        yield single_attempt( )


import os
import sys
import cv2
import time
sys.path.append(r"/home/shushi/projects/3face/face_api/src")
from common.api_eyekey import Eyekey
from utils.base import draw_face_rectangle

def MyDetect(fn,result_file):
    face_id = ""
    result = eyekey.detect(fn)
    try:
        face = result['face'][0]
        face_id = face["face_id"]
    except:
        print("error")
        try:
            buf = str(j) + " " + fn + " : " + result["message"] + "\n"
            result_file.writelines(buf)
        except:
            buf = str(j) + " " + fn + " : 404 " + "\n"
            result_file.writelines(buf)
    return face_id

path = '/home/shushi/Pic/idealtest/idealtest/'
output_path = '../result/campare_same/eyekey/'
eyekey = Eyekey()
if not os.path.exists(output_path):
    os.makedirs(output_path)
file_name = output_path+"compare.txt"
result_file = open(file_name, 'a+')
confidence = 0.0
total = 0
file_paths = os.listdir(path)[0:20]
for j in range(len(file_paths)):
    file_path = path + file_paths[j]+"/"
    files = os.listdir(file_path)
    file1 = file_path+files[0]
    face_id1 = MyDetect(file1,result_file)
    if face_id1 != "":
        k = 1
        while k < len(files):
            file2 = file_path + files[k]
            face_id2 = MyDetect(file2,result_file)
            if face_id2 != "":
                print(file1,file2)
                print(face_id1,face_id2)
                result = eyekey.compare(face_id1, face_id2)
                try:
                    buf = file1 + " vs " + file2 + "\n" + "similarity: " + str(result["similarity"]) + "\n"
                    result_file.writelines(buf)
                    confidence += result["similarity"]
                    total += 1
                except:
                    buf = file1 + " vs " + file2 + "\n" + ":405 " + "\n"
                    result_file.writelines(buf)
            k += 1
print(str(confidence/total))
result_file.flush()
result_file.close()


import logging
import pandas as pd
from load_model.generics import task as t
from load_model.generics.file_type_enum import SupportedFileReadType

logger = logging.getLogger('LCTK_APPLICATION_LOGGER')

class HeatcoolIndexer(t.Task):
    def __init__(self, name, pipeline_artifact_dir):
        super().__init__(self)
        self.name = name
        self.pipeline_artifact_dir = pipeline_artifact_dir
        self.data_files = [
            { 'name': f'{pipeline_artifact_dir}/area_loads.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/594.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/596.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/597.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/598.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/833.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/835.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/836.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/837.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/838.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/970.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/971.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/972.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/973.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/974.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/980.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/981.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/982.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/983.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/984.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/985.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/988.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/989.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/990.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/991.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/992.csv', 'read_type': SupportedFileReadType.DATA },
            { 'name': f'{pipeline_artifact_dir}/noaa/993.csv', 'read_type': SupportedFileReadType.DATA },
        ]
        self.task_function = self._task
        self.output_artifact_enduse_loads = f'{pipeline_artifact_dir}/enduse_loads.csv'

        self.data_map = None
        self.df = None

        # these should be read from config, they are different for RBSA and CEUS
        self.theat = 15
        self.tcool = 25

    def _save_data(self):
        self.save_data(self.df)

    def _get_data(self):
        self.data_map = self.load_data(self.data_files)
        self.df = self.data_map[f'{self.pipeline_artifact_dir}/area_loads.csv']

    def _task(self):
        self._get_data()

        zipcodes = self.df.zipcode.unique()
        enduse_loads = pd.DataFrame()

        for zipcode in zipcodes:
            zipcode_df = self.df.loc[self.df.zipcode == zipcode]
            zipcode_df = zipcode_df.reset_index()

            filename = f'{self.pipeline_artifact_dir}/noaa/{str(zipcode)}.csv'
            zipcode_weather = self.data_map[filename]

            # validation for date ranges of zip codes load data date range to noaa data for that zipcode
            if (zipcode_df.time.max() > zipcode_weather.DATE.max()) | (zipcode_df.time.min() < zipcode_weather.DATE.min()):
                logger.exception(f'Task {self.name} did not pass validation. Error found in matching noaa weather file date range to {zipcode} zip code.')
                self.did_task_pass_validation = False
                self.on_failure()

            # make start and end dates of weather data match load
            start = zipcode_df.time.min()
            end = zipcode_df.time.max()

            zipcode_weather = zipcode_weather.loc[(zipcode_weather.DATE >= start) & (zipcode_weather.DATE <= end)]

            load_df = pd.DataFrame(columns=['HeatCool', 'Temperature', 'Indexer','Heating', 'Cooling', 'Ventilation', 'HeatCoolVent'])

            # apply indexing
            load_df['HeatCool'] = zipcode_df['HeatCool']
            load_df['Temperature'] = zipcode_weather['Temperature']
            load_df['Indexer'] = zipcode_weather.apply(self.temp_dir, axis=1)
            load_df['Heating'] = load_df.apply(self.heat_method, axis=1)
            load_df['Cooling'] = load_df.apply(self.cool_method, axis=1)
            load_df['HeatCoolVent'] = load_df.apply(self.vent_method, axis=1)

            load_df['Heating'] = load_df['Heating'] + (load_df['HeatCoolVent'] / 3)
            load_df['Cooling'] = load_df['Cooling'] + (load_df['HeatCoolVent'] / 3)
            load_df['Ventilation'] = load_df['HeatCoolVent'] / 3

            load_df = load_df.fillna(0)

            enduses_updated = ['Heating', 'Cooling', 'Ventilation']

            # zipcode_df['Ventilation'] = 0 # no ventilation coming in

            # apply changes
            for enduse in enduses_updated:
                zipcode_df[enduse] = zipcode_df[enduse] + load_df[enduse]

            enduse_loads = enduse_loads.append(zipcode_df)

        enduse_loads = enduse_loads.drop('HeatCool', axis=1)
        enduse_loads = enduse_loads.set_index('time')
        enduse_loads = enduse_loads.drop('index', axis=1)

        self.validate(enduse_loads)
        self.on_complete({self.output_artifact_enduse_loads: enduse_loads})

    def temp_dir(self, row):
        """
        Function used for seperating heatcool
        """
        if row['Temperature'] < self.theat:
            return "Heating"
        if row['Temperature'] > self.tcool:
            return "Cooling"
        return "Ventilation"

    def heat_method(self, row):
        """
        Function used for seperating heat from heatcool
        """
        if row['Indexer'] == "Heating":
            return row['HeatCool']
        return 0

    def cool_method(self, row):
        """
        Function used for seperating cool from heatcool
        """
        if row['Indexer'] == "Cooling":
            return row['HeatCool']
        return 0

    def vent_method(self, row):
        """
        Function used for seperating vent from heatcool
        """
        if row['Indexer'] == "Ventilation":
            return row['HeatCool']
        return 0

    def validate(self, df):
        """
        Validation
        """
        logger.info(f'Validating task {self.name}')
        if df.isnull().values.any():
            logger.exception(f'Task {self.name} did not pass validation. DataFrame contains null values when it should not.')
            self.did_task_pass_validation = False
            self.on_failure()

    def on_failure(self):
        logger.info('Perform task cleanup because we failed')
        super().on_failure()


from sympy import *
import matplotlib
import tkinter
from tkinter import *
import numpy as np
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
matplotlib.use('TkAgg')
from numpy import random
from scipy.stats import binom
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.animation as animation

root=Tk()
frame=Frame(root)
frame.pack()

#-----------------------------Labels and Entries-----------------------------------

label1=Label(frame,text="n =")
label1.grid(row=1,column=1)

text1=Entry(frame,width=10)
text1.grid(row=1,column=2)

label2=Label(frame,text="x =")
label2.grid(row=2,column=1)

text2=Entry(frame,width=10)
text2.grid(row=2,column=2)

label3=Label(frame,text="p =")
label3.grid(row=1,column=4)

text3=Entry(frame,width=10)
text3.grid(row=1,column=5)

#-----------------------Dropdown Menu----------------------------------------------

options=StringVar(root)
choices={'P(X>=x)= ','P(X<=x)= ','P(X=x)= '}
options.set('P(X=x)= ')

menu=OptionMenu(frame,options,*choices)
menu.grid(row=2,column=4)

show=StringVar()
text4=Entry(frame,width=10,textvariable=show)
text4.grid(row=2,column=5)

#--------------------------Canavas--------------------------------------------------

button1=Button(frame,text="Go",command=lambda:tempCanvas())
button1.grid(row=3,column=3)

#--------------------------Functions-----------------------------------------------

def tempCanvas():
    canvas_width=300
    canvas_height=80
    temp=Canvas(frame,width=canvas_width,height=canvas_height)
    temp.config(background="white")
    temp.grid(row=3,column=1,columnspan=5)

#--------------------------Assure that n is a positive integer---------------------

    if text1.get().isdigit():
        global n
        n=int(text1.get())
    else:
        temp.create_text(150,20,text="The number of experiments is not valid!")
        temp.create_text(150,30,text="Please enter a positive integer!")
#----------------------------------------------------------------------------------
    if text2.get().isdigit():
        global x
        x=int(text2.get())
    else:
        temp.create_text(150,40,text="X is not valid!")
        temp.create_text(150,50,text="Please enter a positive integer!")
#------------------------------------------------------------------------------------
    try:
        global p
        p=float(text3.get())
        if p<0 or p>1:
            temp.create_text(150,50,text="Please enter a number between 0 and 1")
            exit()
        else:
            p=float(text3.get())
    except ValueError:
        temp.create_text(150,60,text="The probability is not valid!")
        temp.create_text(150,70,text="Please enter a number between 0 and 1")

#----------------------------------Value of Probability-------------------------------

def Result(*args):
    h=tempCanvas()
    l=Paint()

    probability=options.get()
    if probability=='P(X>=x)= ':
        value=0
        for i in range(x,n+1):
            value=value+binomial(n,i)*(p**i)*((1-p)**(n-i))
    elif probability=='P(X<=x)= ':
        value=0
        for i in range(0,x+1):
            value=value+binomial(n,i)*(p**i)*((1-p)**(n-i))
    else:
        value=binomial(n,x)*(p**x)*((1-p)**(n-x))

    show.set(round(value,5))

def Paint():
    t=[]
    w=Figure(figsize=(5,5),dpi=100)
    a=w.add_subplot(111)
    a.set_ylabel('P(X)')
    a.set_xlabel('Number of trials')
    y=np.arange(n+1)
    for i in y:
        t.append(binomial(n,i)*(p**i)*((1-p)**(n-i)))

    barlist=a.bar(y[:],t[:])
    if options.get()=='P(X>=x)= ':
        for i in range(x,n+1):
            barlist[i].set_color('r')
    elif options.get()=='P(X<=x)= ':
        for i in range(0,x+1):
            barlist[i].set_color('r')
    else:
        barlist[i].set_color('r')

    canvas=FigureCanvasTkAgg(w,frame)
    canvas.draw()
    canvas.get_tk_widget().grid(row=3,column=1,columnspan=5)

options.trace('w',Result)
root.mainloop()


from PIL import Image, ImageDraw
import os,cv2

#datahome=os.getcwd()
# path of parent dir of current dir 上一级目录所在路径
datahome=os.path.abspath(os.path.join(os.getcwd(), ".."))

txt=os.path.join(datahome,"train/000000000077.txt")
jpg=os.path.join(datahome,"train/000000000077.jpg")
#print(txt,jpg)

def get_labels(label_file):
    """ cls, box(cx,cy,w,h), cx,cy,w,h>0,<1 """
    ftxt=open(txt,'r')
    lines = ftxt.readlines()
    ftxt.close()
    lb=[]
    for line in lines:
        a = line.strip('\n').split(' ')
        cls,box = int(a[0]), [float(x) for x in a[1:]]
        lb.append((cls,box))
    return lb

def bound(x,boundary=0):
    if(boundary==0):
        return max(x,0)
    else:
        return min(x,boundary-1)
def cxcywh2xyxy(labels,wh):
    """ labels (cls, box(4 01float numbers)); wh, imagesize """
    labels_new = []
    w, h = wh
    for cls, box in labels:
        x1,y1 = int((box[0]-box[2]/2) * w), int( (box[1]-box[3]/2) * h)
        x1,y1 = bound(x1),bound(y1)
        x2,y2 = int((box[0]+box[2]/2) * w), int( (box[1]+box[3]/2) * h)
        x2,y2 = bound(x2,w), bound(y2,h)
        labels_new.append((cls,[x1,y1,x2,y2]))
    return labels_new

def drawbox(labels_new,im):
    dr = ImageDraw.Draw(im)
    for cls, (x1,y1,x2,y2) in labels_new:
        dr.line([(x1,y1),(x1,y2),(x2,y2),(x2,y1),(x1,y1)],width=2,fill='red')
    return im

lbs=get_labels(txt)
im=Image.open(jpg)
wh=im.size
lbs_new = cxcywh2xyxy(lbs,wh)
print(len(lbs_new), lbs_new)
#print(len(lbs), lbs)

im1 = drawbox(lbs_new,im)
im1.show()


#from gensim.models import Word2Vec
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Flatten
from keras.layers.embeddings import Embedding
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

dataset = pd.read_csv('spam_or_not_spam.csv')

dataset.email = dataset.email.astype(str)

emails = dataset["email"].values
labels = dataset["label"].values

print(len(emails))

#δημιουργία tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(emails)
vocab_size = len(tokenizer.word_index) + 1

#κάνουμε integer encode στα email που έχουμε
encoded_emails = tokenizer.texts_to_sequences(emails)
print(vocab_size)

#κανουμε padding στα emails
max_length = 20
padded_emails = pad_sequences(encoded_emails, maxlen=max_length, padding='post')
print(len(padded_emails))

#κάνουμε load τo dictionary με τα embeddings
embeddings_index = dict()
f = open("glove.6B.100d.txt", encoding="utf8")
for line in f:
    values = line.split()
    word = values[0]
    coefficients = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefficients
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))

# create a weight matrix for words in training docs
embedding_matrix = np.zeros((vocab_size, 100))
for word, i in tokenizer.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector

print(embedding_matrix.shape)

#δημιουργία του νευρωνικού model
model = Sequential()
model.add(Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=20, trainable=False))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))

#κάνομυε compile το model μας
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])

# summarize the model
print(model.summary())

#κάνουμε split το dataset σε train και test data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(padded_emails, labels, test_size=0.25, random_state=42)

#κάνουμε fit το model μας
model.fit(X_train, y_train, epochs=10, verbose=0)

# evaluation του model μας
loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print('Accuracy: %f' % (accuracy*100))

from sklearn.metrics import classification_report, confusion_matrix

y_pred = model.predict_classes(X_test)
cf_matrix=confusion_matrix(y_test,y_pred)
print("\nConfusion Matrix\n\n {} \n\n {}".format(cf_matrix,classification_report(y_test,y_pred)))


#!/usr/bin/python

# Copyright 2018 Adobe. All rights reserved.
# This file is licensed to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
# OF ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

"""
This module manages consolidating DNS records from various sources.
"""

from datetime import datetime
from libs2 import MongoConnector
from bson.objectid import ObjectId

class DNSManager(object):
    """
    Marinus collects DNS information from multiple sources.
    In the original version of Marinus, there was a separate table for each source.
    Maintaining separate tables makes it difficult to search.
    Therefore, the tables have been merged into "all_dns".
    This class acts as the interface for translating a DNS record into the all_dns format.
    When submitting a DNS record, a "source" must be provided.
    The approved sources are "virustotal", "common_crawl", "sonar_dns", and "ssl".
    These will eventually be saved as constants.
    """

    all_dns_collection = None

    def __init__(self, mongo_connector):
        self.all_dns_collection = mongo_connector.get_all_dns_connection()

    @staticmethod
    def monthdelta(date, delta):
        """
        Return the date from the given delta

        :param date: The original date
        :param delta: The change from the original date that is to be calculated.
        """
        m, y = (date.month+delta) % 12, date.year + ((date.month)+delta-1) // 12
        if not m:
            m = 12
        d = min(date.day, [31, 29 if y%4==0 and not y%400==0 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][m-1])
        return date.replace(day=d, month=m, year=y)

    def insert_record(self, result, source_name):
        """
        Insert the provided source as a record from the provided source name.
        :param result: The result of a DNS lookup as a JSON object including
                      fqdn, type, value, zone, and created values.
        :param source_name: The DNS record source ("ssl","virustotal","sonar_dns","common_crawl")
        """
        check = self.all_dns_collection.find_one({'fqdn': result['fqdn'],
                                                  'type': result['type'],
                                                  'value': result['value']})

        if check is None:
            result['sources'] = []
            result['sources'].append({})
            result['sources'][0]['source'] = source_name
            result['sources'][0]['updated'] = datetime.now()
            result['updated'] = datetime.now()
            self.all_dns_collection.insert(result)
        else:
            sonar_i = -1
            for i in range(0, len(check['sources'])):
                if check['sources'][i]['source'] == source_name:
                    sonar_i = i
            if sonar_i != -1:
                name = 'sources.' + str(sonar_i) + '.updated'
                entry = {}
                entry[name] = datetime.now()
                self.all_dns_collection.update({'_id': ObjectId(check['_id'])},
                                               {"$set": entry})
                self.all_dns_collection.update({'_id': ObjectId(check['_id'])},
                                               {"$set": {'updated': datetime.now()}})
            else:
                entry = {}
                entry['source'] = source_name
                entry['updated'] = datetime.now()
                self.all_dns_collection.update({'_id': ObjectId(check['_id'])},
                                               {'$push': {'sources': entry}})
                self.all_dns_collection.update({'_id': ObjectId(check['_id'])},
                                               {"$set": {'updated': datetime.now()}})

    def find_multiple(self, criteria, source):
        """
        Find multiple records for the specified criteria.

        :param criteria: A JSON object representing the find query. No limit support.
        :param source: (Optional) The DNS record source ("ssl","virustotal","sonar_dns",etc.)
        :return: The cursor from the find operation
        """
        if source != None:
            criteria['sources.source'] = source

        check = self.all_dns_collection.find(criteria)
        return check

    def find_one(self, criteria, source):
        """
        Find a single record for the specified criteria

        :param criteria: A JSON object representing the find query. No limit support.
        :param source: (Optional) The DNS record source ("ssl","virustotal","sonar_dns",etc.)
        :return: The cursor from the find operation
        """
        if source != None:
            criteria['sources.source'] = source

        check = self.all_dns_collection.find_one(criteria)
        return check

    def find_count(self, criteria, source):
        """
        Return the count of records for the specified criteria.

        :param criteria: A JSON object representing the find query. No limit support.
        :param source: (Optional) The DNS record source ("ssl","virustotal","sonar_dns",etc.)
        :return: The cursor from the find operation
        """
        if source != None:
            criteria['sources.source'] = source

        check = self.all_dns_collection.find(criteria).count()
        return check

    def remove_by_domain_and_source(self, domain, dns_type, dns_value, source):
        """
        Remove a specific all_dns entry by providing the domain, type, value and
        source of the record to be removed.

        :param domain: The domain that is to be altered
        :param dns_type: The type of record that is to be removed
        :param dns_value: The corresponding value that is to be removed
        :param source: The source of the record that is to be removed.
        :return: A boolean indicating success or failure
        """
        result = self.all_dns_collection.find_one({'fqdn':domain,
                                                   'type': dns_type,
                                                   'value': dns_value})

        if result is None:
            return False

        if len(result['sources']) == 1:
            self.all_dns_collection.remove({'fqdn': domain})
            return True

        self.all_dns_collection.update({'_id': ObjectId(result['_id'])},
                                       {"$pull": {"sources": {"source": source}}})
        return True

    def remove_by_object_id_and_source(self, object_id, source):
        """
        Remove a specific all_dns entry by providing the object_id and source to be removed.
        If an entry is associated with multiple sources,
        then only the association with the specified source will be removed.

        :param objectid: The object ID of the record.
        :param source: The source reference that is to be removed from the object_id.
        :return: A boolean indicating success or failure
        """
        result = self.all_dns_collection.find_one({'_id': ObjectId(object_id)})

        if result is None:
            return False

        if len(result['sources']) == 1:
            self.all_dns_collection.remove({'_id': ObjectId(object_id)})
            return True

        self.all_dns_collection.update({'_id': ObjectId(result['_id'])},
                                       {"$pull": {"sources": {"source": source}}})
        return True

    def remove_all_by_source_and_date(self, source, month_delta=-2):
        """
        Remove a specific all_dns entry by providing the object_id and source to be removed.
        If an entry is associated with multiple sources,
        then only the association with the specified source will be removed.

        :param source: The source of the records that are to be aged out.
        :param month_delta: How many months to keep (e.g. Keep the last two months)
        :return: A boolean indicating success or failure
        """
        d_minus_2m = self.monthdelta(datetime.now(), month_delta)
        results = self.all_dns_collection.find({'sources.source': source,
                                                'sources.updated': {"$lt": d_minus_2m}}
                                              ).batch_size(30)

        for result in results:
            if len(result['sources']) > 1:
                self.all_dns_collection.update({'_id': ObjectId(result['_id'])},
                                               {"$pull": {"sources": {"source": source}}})
            else:
                self.all_dns_collection.remove({'_id': ObjectId(result['_id'])})

        return True

    def remove_by_source(self, source):
        """
        Remove all entries associated with a specific source.
        If an entry is associated with multiple sources,
        then only the association with the specified source will be removed.

        :param source: The source references that are to be removed.
        :return: A boolean indicating success or failure
        """
        results = self.all_dns_collection.find({'sources.source': source})

        if results is None:
            return False

        for result in results:
            if len(result['sources']) == 1:
                self.all_dns_collection.remove({'_id': ObjectId(result['_id'])})
            else:
                self.all_dns_collection.update({'_id': ObjectId(result['_id'])},
                                               {"$pull": {"sources": {"source": source}}})
        return True


#coding=utf-8

import requests
from Conf.Config import Config
from api import login
from api.BaseApi import BaseApi
import time

class userApi(BaseApi):

    def __init__(self,payload):
        self.payload = payload
        self.session = requests.session()
        self.sendVerifyCode_url = Config().user_host_debug+'sendVerifyCode'
        self.appLogin_url = Config().user_host_debug + 'appLogin'
        self.editUserInfo_url = Config().user_host_debug + 'editUserInfo'
        self.getUserInfo_url = Config().user_host_debug + 'getUserInfo'
        self.getUserInfoByMobile_url = Config().user_host_debug + 'getUserInfoByMobile'
        self.editCallRemind_url = Config().user_host_debug + 'editCallRemind'
        self.headers = {"authorization": login.Login.login(), "content - type": "application/json;charset=utf-8"}

    def sendVerifyCode(self):
        self.json_data = self.session.post(self.sendVerifyCode_url,json=self.payload).json()
        self.verbose(self.json_data)
        return self.json_data

    def appLogin(self):
        self.json_data = self.session.post(self.appLogin_url, json=self.payload).json()
        self.verbose(self.json_data)
        return self.json_data

    #编辑用户信息

    def editUserInfo(self):
        self.json_data = requests.post(self.editUserInfo_url, headers=self.headers, json=self.payload, verify=False).json()
        self.verbose(self.json_data)
        return self.json_data

    #获取用户信息

    def getUserInfo(self):
        self.json_data = requests.get(self.getUserInfo_url, headers=self.headers, json=self.payload, verify=False).json()
        self.verbose(self.json_data)
        return self.json_data

    #根据手机号查询用户信息

    def getUserInfoByMobile(self):
        self.json_data = requests.post(self.getUserInfoByMobile_url, headers=self.headers, json=self.payload, verify=False).json()
        self.verbose(self.json_data)
        return self.json_data

    #修改语音视频通话提示

    def editCallRemind(self):
        self.json_data = requests.post(self.editCallRemind_url, headers=self.headers, json=self.payload, verify=False).json()
        self.verbose(self.json_data)
        return self.json_data


# -*- encoding: utf-8 -*-
#
# Copyright © 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# NOTE(sileht): usefull for gunicon, not really for uwsgi
# import gevent
# import gevent.monkey
# gevent.monkey.patch_all()

import hmac
import json
import logging

import flask
import github
import raven.contrib.flask
import rq
import rq_dashboard
import uhashring

from mergify_engine import config
from mergify_engine import utils
from mergify_engine import worker

LOG = logging.getLogger(__name__)

app = flask.Flask(__name__)

app.config.from_object(rq_dashboard.default_settings)
app.register_blueprint(rq_dashboard.blueprint, url_prefix="/rq")
app.config["REDIS_URL"] = utils.get_redis_url()
app.config["RQ_POLL_INTERVAL"] = 10000  # ms
sentry = raven.contrib.flask.Sentry(app, dsn=config.SENTRY_URL)

# TODO(sileht): Make the ring dynamic
global RING
nodes = []
for fqdn, w in sorted(config.TOPOLOGY.items()):
    nodes.extend(map(lambda x: "%s-%003d" % (fqdn, x), range(w)))

RING = uhashring.HashRing(nodes=nodes)

def get_queue(slug, subscription):
    global RING
    name = "%s-%s" % (RING.get_node(slug),
                      "high" if subscription["subscribed"] else "low")
    return rq.Queue(name, connection=utils.get_redis_for_rq())

def authentification():  # pragma: no cover
    # Only SHA1 is supported
    header_signature = flask.request.headers.get('X-Hub-Signature')
    if header_signature is None:
        LOG.warning("Webhook without signature")
        flask.abort(403)

    try:
        sha_name, signature = header_signature.split('=')
    except ValueError:
        sha_name = None

    if sha_name != 'sha1':
        LOG.warning("Webhook signature malformed")
        flask.abort(403)

    mac = utils.compute_hmac(flask.request.data)
    if not hmac.compare_digest(mac, str(signature)):
        LOG.warning("Webhook signature invalid")
        flask.abort(403)

@app.route("/check_status_msg/<path:key>")
def check_status_msg(key):
    msg = utils.get_redis_for_cache().hget("status", key)
    if msg:
        return flask.render_template("msg.html", msg=msg)
    else:
        flask.abort(404)

@app.route("/refresh/<owner>/<repo>/<path:refresh_ref>",
           methods=["POST"])
def refresh(owner, repo, refresh_ref):
    authentification()

    integration = github.GithubIntegration(config.INTEGRATION_ID,
                                           config.PRIVATE_KEY)
    installation_id = utils.get_installation_id(integration, owner)
    if not installation_id:  # pragma: no cover
        flask.abort(400, "%s have not installed mergify_engine" % owner)

    token = integration.get_access_token(installation_id).token
    g = github.Github(token)
    r = g.get_repo("%s/%s" % (owner, repo))
    try:
        r.get_contents(".mergify.yml")
    except github.GithubException as e:  # pragma: no cover
        if e.status == 404:
            return "No .mergify.yml", 202
        else:
            raise

    if refresh_ref == "full" or refresh_ref.startswith("branch/"):
        if refresh_ref.startswith("branch/"):
            branch = refresh_ref[7:]
            pulls = r.get_pulls(base=branch)
        else:
            branch = '*'
            pulls = r.get_pulls()
        key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(),
                                         repo.lower(), r.private, branch)
        utils.get_redis_for_cache().delete(key)
    else:
        try:
            pull_number = int(refresh_ref[5:])
        except ValueError:  # pragma: no cover
            return "Invalid PR ref", 400
        pulls = [r.get_pull(pull_number)]

    subscription = utils.get_subscription(utils.get_redis_for_cache(),
                                          installation_id)

    if not subscription["token"]:  # pragma: no cover
        return "", 202

    if r.private and not subscription["subscribed"]:  # pragma: no cover
        return "", 202

    for p in pulls:
        # Mimic the github event format
        data = {
            'repository': r.raw_data,
            'installation': {'id': installation_id},
            'pull_request': p.raw_data,
        }
        get_queue(r.full_name, subscription).enqueue(
            worker.event_handler, "refresh", subscription, data)

    return "", 202

@app.route("/refresh", methods=["POST"])
def refresh_all():
    authentification()

    integration = github.GithubIntegration(config.INTEGRATION_ID,
                                           config.PRIVATE_KEY)

    counts = [0, 0, 0]
    for install in utils.get_installations(integration):
        counts[0] += 1
        token = integration.get_access_token(install["id"]).token
        g = github.Github(token)
        i = g.get_installation(install["id"])

        subscription = utils.get_subscription(utils.get_redis_for_cache(),
                                              install["id"])
        if not subscription["token"]:  # pragma: no cover
            continue

        for r in i.get_repos():
            if r.private and not subscription["subscribed"]:
                continue
            try:
                r.get_contents(".mergify.yml")
            except github.GithubException as e:  # pragma: no cover
                if e.status == 404:
                    continue
                else:
                    raise

            counts[1] += 1
            for p in list(r.get_pulls()):
                # Mimic the github event format
                data = {
                    'repository': r.raw_data,
                    'installation': {'id': install["id"]},
                    'pull_request': p.raw_data,
                }
                get_queue(r.full_name, subscription).enqueue(
                    worker.event_handler, "refresh", subscription, data)

    return ("Updated %s installations, %s repositories, "
            "%s branches" % tuple(counts)), 202

# FIXME(sileht): rename this to new subscription something
@app.route("/subscription-cache/<installation_id>", methods=["DELETE"])
def subscription_cache(installation_id):  # pragma: no cover
    authentification()
    r = utils.get_redis_for_cache()
    r.delete("subscription-cache-%s" % installation_id)

    subscription = utils.get_subscription(
        utils.get_redis_for_cache(), installation_id)

    # New subscription, create initial configuration for private repo
    # public repository have already been done during the installation
    # event.
    if subscription["token"] and subscription["subscribed"]:
        # FIXME(sileht): We should pass the slugs
        get_queue(installation_id, subscription).enqueue(
            worker.installation_handler, installation_id, "private")
    return "Cache cleaned", 200

@app.route("/event", methods=["POST"])
def event_handler():
    authentification()

    event_type = flask.request.headers.get("X-GitHub-Event")
    event_id = flask.request.headers.get("X-GitHub-Delivery")
    data = flask.request.get_json()

    subscription = utils.get_subscription(
        utils.get_redis_for_cache(), data["installation"]["id"])

    if not subscription["token"]:
        msg_action = "ignored (no token)"

    elif event_type == "installation" and data["action"] == "created":
        for repository in data["repositories"]:
            if repository["private"] and not subscription["subscribed"]:  # noqa pragma: no cover
                continue

            get_queue(repository["full_name"], subscription).enqueue(
                worker.installation_handler, data["installation"]["id"],
                [repository])
        msg_action = "pushed to backend"

    elif event_type == "installation" and data["action"] == "deleted":
        key = "queues~%s~*~*~*~*" % data["installation"]["id"]
        utils.get_redis_for_cache().delete(key)
        msg_action = "handled, cache cleaned"

    elif (event_type == "installation_repositories" and
          data["action"] == "added"):
        for repository in data["repositories_added"]:
            if repository["private"] and not subscription["subscribed"]:  # noqa pragma: no cover
                continue

            get_queue(repository["full_name"], subscription).enqueue(
                worker.installation_handler, data["installation"]["id"],
                [repository])

        msg_action = "pushed to backend"

    elif (event_type == "installation_repositories" and
          data["action"] == "removed"):
        for repository in data["repositories_removed"]:
            if repository["private"] and not subscription["subscribed"]:  # noqa pragma: no cover
                continue
            key = "queues~%s~%s~%s~*~*" % (
                data["installation"]["id"],
                data["installation"]["account"]["login"].lower(),
                repository["name"].lower()
            )
            utils.get_redis_for_cache().delete(key)
        msg_action = "handled, cache cleaned"

    elif event_type in ["installation", "installation_repositories"]:
        msg_action = "ignored (action %s)" % data["action"]

    elif event_type in ["pull_request", "pull_request_review", "status",
                        "refresh"]:

        if data["repository"]["private"] and not subscription["subscribed"]:
            msg_action = "ignored (not public or subscribe)"

        elif event_type == "status" and data["state"] == "pending":
            msg_action = "ignored (state pending)"

        elif (event_type == "pull_request" and data["action"] not in [
                "opened", "reopened", "closed", "synchronize",
                "labeled", "unlabeled"]):
            msg_action = "ignored (action %s)" % data["action"]

        else:
            get_queue(data["repository"]["full_name"], subscription).enqueue(
                worker.event_handler, event_type, subscription, data)
            msg_action = "pushed to backend"

    else:
        msg_action = "ignored (unexpected event_type)"

    if "repository" in data:
        repo_name = data["repository"]["full_name"]
    else:
        repo_name = data["installation"]["account"]["login"]

    LOG.info('[%s/%s] received "%s" event "%s", %s',
             data["installation"]["id"], repo_name,
             event_type, event_id, msg_action)

    return "", 202

# NOTE(sileht): These endpoints are used for recording cassetes, we receive
# Github event on POST, we store them is redis, GET to retreive and delete
@app.route("/events-testing", methods=["POST", "GET", "DELETE"])
def event_testing_handler():  # pragma: no cover
    authentification()
    r = utils.get_redis_for_cache()
    if flask.request.method == "DELETE":
        r.delete("events-testing")
        return "", 202
    elif flask.request.method == "POST":
        event_type = flask.request.headers.get("X-GitHub-Event")
        event_id = flask.request.headers.get("X-GitHub-Delivery")
        data = flask.request.get_json()
        r.rpush("events-testing", json.dumps(
            {"id": event_id, "type": event_type, "payload": data}
        ))
        return "", 202
    else:
        p = r.pipeline()
        p.lrange("events-testing", 0, -1)
        p.delete("events-testing")
        values = p.execute()[0]
        data = [json.loads(i) for i in values]
        return flask.jsonify(data)

@app.route("/")
def index():  # pragma: no cover
    return flask.redirect("https://mergify.io/")


from numpy import *
import matplotlib.pyplot as plt
import matplotlib
from Surface_Reaction_Tools.adsorption_fitting_tools import load_average_data
from Surface_Reaction_Tools.theoretical import Langmuir_transition_kinetics

# Setup
matplotlib.rc('font', size=10.0, family='serif')    # Set default font
width = 5
colors = ['k', 'b']
dashes = [(None,None), (8,4), (2,2)]

# Langmuir with post-adsorption transition
ka = 5.957e-8
ks = 1.143e-2
kd = 6.744e-5
area1 = 209.0
area2 = 291.0
ratio = area2 / area1
theta_max = 1.0

expFileName = '/home/cfinch/cfinch/Microfluidics/Whispering Gallery Mode Sensor/Experimental Data Analysis/GO/Data/GO_13F.h5'

NA = 6.022e23
molar_mass = 160e3      # g/mol
f = NA / molar_mass * 1e-9    # g/molecule

exp_C, exp_t, exp_mean, exp_std, labels = load_average_data(expFileName)
plt.figure(figsize=(width, width/1.618), facecolor='w')
max_time = 0
for i in range(len(exp_C)):
    if exp_t[i].max() > max_time:
        max_time = exp_t[i].max()

    theta1, theta2 = Langmuir_transition_kinetics(ka, ks, kd, ratio, exp_t[i],
                        const_C=exp_C[i] * 1000, blocking_fn_args=theta_max)

    rho1 = theta1 / (f * area1 * 1e-14)
    rho2 = theta2 / (f * area2 * 1e-14)
    surface_density = rho1 + rho2

    plt.plot(exp_t[i], surface_density, color=colors[i], dashes=dashes[i],
            label=r"$" + str(exp_C[i]) + "\, \mu g/ml$")
    plt.plot(exp_t[i], exp_mean[i], color=colors[i], ls='-')
    plt.plot(exp_t[i], exp_mean[i] + exp_std[i], color=colors[i], ls=':')
    plt.plot(exp_t[i], exp_mean[i] - exp_std[i], color=colors[i], ls=':')

plt.xlabel(r"$t(s)$")
plt.ylabel(r"$ng/cm^2$")
plt.axis([0.0, max_time, 0.0, 160.0])
plt.legend(loc='upper right', handlelength=3)
plt.subplots_adjust(bottom=0.15)
plt.show()


#!/usr/bin/env python

import sys
import argparse

import context
from helpers import utils
from helpers.subprocess_wrappers import Popen, check_output, PIPE

def main():
    config = utils.parse_vantage_points()

    parser = argparse.ArgumentParser()
    parser.add_argument('action', choices=['start', 'stop', 'status'])
    parser.add_argument('--hosts', metavar='"HOST1 HOST2..."',
                        help='space-separated list of hosts (default: all)')
    args = parser.parse_args()

    applied_hosts = None
    if args.hosts is not None:
        applied_hosts = args.hosts.split()

    procs = []
    for server, server_cfg in config['aws_servers'].iteritems():
        if applied_hosts is not None:
            if server not in applied_hosts:
                continue

        instance_id = server_cfg['id']
        region = server_cfg['region']

        if args.action == 'start':
            procs.append(Popen(
                ['aws', 'ec2', 'start-instances',
                 '--instance-ids', instance_id, '--region', region]))
        elif args.action == 'stop':
            procs.append(Popen(
                ['aws', 'ec2', 'stop-instances',
                 '--instance-ids', instance_id, '--region', region]))
        elif args.action == 'status':
            procs.append(Popen(
                ['aws', 'ec2', 'describe-instances',
                 '--instance-ids', instance_id, '--region', region,
                 '--query', 'Reservations[0].Instances[0].State.Name'],
                stdout=PIPE))

    for proc in procs:
        if args.action == 'status':
            status, _ = proc.communicate()
            sys.stdout.write(status.replace('"', ''))
        else:
            proc.wait()

if __name__ == '__main__':
    main()


# * coding:utf-8 *
# Author    : Administrator
# Createtime: 7/12/2018
import sys,os
import unittest
import xmlrunner
sys.path.insert(0, os.getcwd())
from BDP.config import constants
from BDP.common import qw_model_action_util, base_case, check_result

class testcase_Basic_Check_Setup_WiredWireless_0001(base_case.BaseCase):
    """
    Excel:              Basic_Check_Setup_WiredWireless.xlsx
    Category:           Wired & Wireless
    Test case:          1) Setup BDP Wired connection.
                        2) Retrive Internet Service List.
                        3) Play any network service (E.g: Youtube)

    Expected Result:    Confirm Local connection & Network access both OK.
                        Retrive Network Service List successful.
                        Playback OK.
    """
    def setUp(self):
        print("Initialization Test Environment")
        super(testcase_Basic_Check_Setup_WiredWireless_0001, self).setUp()

    def tearDown(self):
        if(check_result.check_pictures('Connection_Status.png')):
            qw_model_action_util.send_key("RIGHT")
            qw_model_action_util.send_key("ENTER")
            qw_model_action_util.send_key("UP")
        super(testcase_Basic_Check_Setup_WiredWireless_0001, self).tearDown()

    def testcase_Basic_Check_Setup_WiredWireless_0001(self):
        print ("Step1: Enter Into Setup screen")
        qw_model_action_util.go_home()
        qw_model_action_util.go_setup()

        print ("Step2: Enter Into Network Setting")
        qw_model_action_util.go_network_setting()

        print ("Step3: Enter Into Internet Settings")
        qw_model_action_util.go_internet_settings()

        print ("Step4: Retrive Internet Service List")
        qw_model_action_util.set_wired_setup()

        # print ("Step5: Play any network service (E.g: Digital Concert Hall)")
        # qw_model_action_util.go_digital_concert_hall()
        # self.assertTrue(check_result.check_pictures('Digital_Concert.png'),
        #                 'Play network service Digital Concert Hall Failed')

if __name__ == '__main__':
    xmlpath = constants.log_dir
    runner = xmlrunner.XMLTestRunner(output=xmlpath, stream=sys.stdout)
    suite = unittest.TestLoader().loadTestsFromTestCase(testcase_Basic_Check_Setup_WiredWireless_0001)
    runner.run(suite)


import json
import pickle

file = 'DataP.pickle'
data = pickle.load(open(file , 'rb'))

complete = []
for timeStep in data["AllData"]:
    timeData = timeStep['data']
    process_copy = {'id':timeStep['id']}
    for key , val in timeData.items():
        tempList = []
        for element in val:
            #print(element)
            try:
                temp = {str(k):v for k , v in element.items()}
            except:
                process_copy[key] = element
                continue
            tempList.append(temp)
        process_copy[key] = tempList
    complete.append(process_copy)
with open("Complete.json" , 'w') as f:
    json.dump({"AllData":complete} , f)


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required

from .models import Topic, Entry
from .forms import TopicForm, EntryForm

def index(request):
	'''The Homepage for Learning Log'''
	return render(request, 'learning_logs/index.html')

@login_required
def topics(request):
	'''Displays all topics'''
	topics = Topic.objects.filter(owner=request.user).order_by('date_added')
	context ={'topics':topics}

	return render(request, 'learning_logs/topics.html', context)

@login_required
def topic(request, topic_id):
	'''Displays a Specific Topic Page with all entries'''
	topic = get_object_or_404(Topic, id=topic_id)
	#make sure topic belongs to user
	check_topic_owner(request, topic.owner, topic.public)

	entries = topic.entry_set.order_by('-date_added')

	context = {'topic':topic, 'entries':entries}
	return render(request, 'learning_logs/topic.html', context)

@login_required
def new_topic(request):
	'''Add a new topic'''
	if request.method != 'POST':
		#no data submitted; create a blank form
		form = TopicForm()
	else:
		#POST data submitted; process data
		form = TopicForm(request.POST)
		if form.is_valid():
			new_topic = form.save(commit = False)
			new_topic.owner = request.user
			new_topic.save()
			return HttpResponseRedirect(reverse('learning_logs:topics'))

	context = {'form':form}
	return render(request, 'learning_logs/new_topic.html', context)

@login_required
def new_entry(request, topic_id):
	"""Add a new Entry for a particular topic"""
	topic = get_object_or_404(Topic, id = topic_id)
	check_topic_owner(request, topic.owner, topic.public)

	if request.method != 'POST':
		#No Data Submitted; create a blank form
		form = EntryForm()
	else:
		#POST data submitted; process data.
		form = EntryForm(data = request.POST)
		if form.is_valid():
			new_entry = form.save(commit=False)
			new_entry.topic = topic
			new_entry.owner = request.user
			new_entry.save()
			return HttpResponseRedirect(reverse('learning_logs:topic', args=[topic_id]))

	context = {'topic': topic, 'form': form}
	return render(request, 'learning_logs/new_entry.html', context)

@login_required
def edit_entry(request, entry_id):
	'''Edit an Existing Entry'''
	entry = get_object_or_404(Entry, id = entry_id)
	topic = entry.topic
	#Makes sure only the Owner can access
	check_topic_owner(request, topic.owner, topic.public)

	if request.method != 'POST':
		#Initial Request; Pre fill form with current Entry
		form = EntryForm(instance=entry)
	else:
		#POST data submitted; process data
		form = EntryForm(instance=entry, data = request.POST)
		if form.is_valid():
			form.save()
			return HttpResponseRedirect(reverse('learning_logs:topic', args = [topic.id]))

	context = {'entry':entry, 'topic':topic, 'form': form}
	return render(request, 'learning_logs/edit_entry.html', context)

def check_topic_owner(request, owner, public):
	#Make Sure the User as Access
	if owner != request.user and public == False:
		raise Http404

def public(request):
	'''Displays all Public Topics'''
	topics = Topic.objects.filter(public=True).order_by("date_added")

	context = {'topics':topics}

	return render(request, 'learning_logs/public.html', context)

@login_required
def switch(request, topic_id):
	'''Switches Public Boolean on a specific topic'''
	topic = get_object_or_404(Topic, id=topic_id)
	if topic.owner != request.user:
		raise Http404

	if topic.public == True:
		#Switch Public to False
		topic.public =False
	else:
		topic.public = True

	topic.save()
	return HttpResponseRedirect(reverse('learning_logs:topics'))


# coding: utf-8
# @Time    : 2020/7/8 16:53
# @Author  : 蟹蟹 ！！
# @FileName: a24_changeENV.py
# @Software: PyCharm

from conf import conf
from resource.regOperator import regOperator
import logging
import os

logging.basicConfig(format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s',
                    level=logging.DEBUG)

# 一键切换流程
def run(envtype):
    ro = regOperator(envtype=envtype)
    ver = ro.currentVersion() # 先判断当前版本
    exe_path = os.path.join(str(conf.envReg[f'platform_{envtype}']['InstallPath']),'NetViosVR.exe')# 创建launcher启动路径
    if ver: # 如果当前版本能够识别
        # 如果已经是新版本，打印信息
        if ver == envtype:
            logging.info(f'Current version is {envtype}!')
            print(exe_path)
            os.system('\"'+exe_path+'\"')
        # 如果是旧版本，执行备份——》更新注册表路径
        else:
            logging.info(f'To {envtype} begin~')
            logging.info('备份当前注册表')
            # 备份当前注册表
            logging.info("================== Execute saveREG ==================")
            ro.saveReg(oldtype=ver)
            logging.info("================== Finish saveREG ==================")
            # 初始化当前游戏注册表
            logging.info("================== Execute init_gameREG ==================")
            ro.del_games()
            logging.info("================== Finish init_gameREG ==================")
            # 注入环境注册表
            logging.info("================== Execute setREG ==================")
            ro.setReg()
            logging.info("================== Finish setREG ==================")
            print(exe_path)
            os.system(exe_path)
    else:
        pass

def toOnline():
    run('Online')

def toTest():
    run('Test')

def toDefault():
    run('Default')

def currentVersion():
    ro = regOperator()
    ver = ro.currentVersion()
    if ver:
        logging.info('============= Excute current version =============')
        logging.info(f"当前环境是{ver}")
        logging.info('============= Finish current version =============')
    return None

def saveReg():
    ro = regOperator()
    logging.info("================== Execute saveREG ==================")
    ver = ro.currentVersion()
    if ver:
        ro.saveReg(ver)
    else:
        logging.error('注册表保存失败！')
    logging.info("================== Finish saveREG ==================")

def setReg():
    ro = regOperator()
    logging.info("================== Execute setREG ==================")
    ver = ro.currentVersion()
    ro_2 = regOperator(envtype=ver)
    logging.info(f'当前版本是{ver}')
    if ver:
        ro_2.setReg()
    else:
        logging.error('注册表保存失败！')
    logging.info("================== Finish setREG ==================")

    pass