# -*- coding: utf-8 -*-

from django.test import TestCase

from django.core.exceptions import ValidationError
from django.db.utils import IntegrityError

from build.models import Build, BuildItem
from stock.models import StockItem
from part.models import Part, BomItem
from InvenTree import status_codes as status

class BuildTest(TestCase):
    """
    Run some tests to ensure that the Build model is working properly.
    """

    def setUp(self):
        """
        Initialize data to use for these tests.
        """

        # Create a base "Part"
        self.assembly = Part.objects.create(
            name="An assembled part",
            description="Why does it matter what my description is?",
            assembly=True,
            trackable=True,
        )

        self.sub_part_1 = Part.objects.create(
            name="Widget A",
            description="A widget",
            component=True
        )

        self.sub_part_2 = Part.objects.create(
            name="Widget B",
            description="A widget",
            component=True
        )

        # Create BOM item links for the parts
        BomItem.objects.create(
            part=self.assembly,
            sub_part=self.sub_part_1,
            quantity=10
        )

        BomItem.objects.create(
            part=self.assembly,
            sub_part=self.sub_part_2,
            quantity=25
        )

        # Create a "Build" object to make 10x objects
        self.build = Build.objects.create(
            title="This is a build",
            part=self.assembly,
            quantity=10
        )

        # Create some build output (StockItem) objects
        self.output_1 = StockItem.objects.create(
            part=self.assembly,
            quantity=5,
            is_building=True,
            build=self.build
        )

        self.output_2 = StockItem.objects.create(
            part=self.assembly,
            quantity=5,
            is_building=True,
            build=self.build,
        )

        # Create some stock items to assign to the build
        self.stock_1_1 = StockItem.objects.create(part=self.sub_part_1, quantity=1000)
        self.stock_1_2 = StockItem.objects.create(part=self.sub_part_1, quantity=100)

        self.stock_2_1 = StockItem.objects.create(part=self.sub_part_2, quantity=5000)

    def test_init(self):
        # Perform some basic tests before we start the ball rolling

        self.assertEqual(StockItem.objects.count(), 5)

        # Build is PENDING
        self.assertEqual(self.build.status, status.BuildStatus.PENDING)

        # Build has two build outputs
        self.assertEqual(self.build.output_count, 2)

        # None of the build outputs have been completed
        for output in self.build.get_build_outputs().all():
            self.assertFalse(self.build.isFullyAllocated(output))

        self.assertFalse(self.build.isPartFullyAllocated(self.sub_part_1, self.output_1))
        self.assertFalse(self.build.isPartFullyAllocated(self.sub_part_2, self.output_2))

        self.assertEqual(self.build.unallocatedQuantity(self.sub_part_1, self.output_1), 50)
        self.assertEqual(self.build.unallocatedQuantity(self.sub_part_1, self.output_2), 50)
        self.assertEqual(self.build.unallocatedQuantity(self.sub_part_2, self.output_1), 125)
        self.assertEqual(self.build.unallocatedQuantity(self.sub_part_2, self.output_2), 125)

        self.assertFalse(self.build.is_complete)

    def test_build_item_clean(self):
        # Ensure that dodgy BuildItem objects cannot be created

        stock = StockItem.objects.create(part=self.assembly, quantity=99)

        # Create a BuiltItem which points to an invalid StockItem
        b = BuildItem(stock_item=stock, build=self.build, quantity=10)

        with self.assertRaises(ValidationError):
            b.save()

        # Create a BuildItem which has too much stock assigned
        b = BuildItem(stock_item=self.stock_1_1, build=self.build, quantity=9999999)

        with self.assertRaises(ValidationError):
            b.clean()

        # Negative stock? Not on my watch!
        b = BuildItem(stock_item=self.stock_1_1, build=self.build, quantity=-99)

        with self.assertRaises(ValidationError):
            b.clean()

        # Ok, what about we make one that does *not* fail?
        b = BuildItem(stock_item=self.stock_1_1, build=self.build, install_into=self.output_1, quantity=10)
        b.save()

    def test_duplicate_bom_line(self):
        # Try to add a duplicate BOM item - it should fail!

        with self.assertRaises(IntegrityError):
            BomItem.objects.create(
                part=self.assembly,
                sub_part=self.sub_part_1,
                quantity=99
            )

    def allocate_stock(self, q11, q12, q21, output):
        # Assign stock to this build

        if q11 > 0:
            BuildItem.objects.create(
                build=self.build,
                stock_item=self.stock_1_1,
                quantity=q11,
                install_into=output
            )

        if q12 > 0:
            BuildItem.objects.create(
                build=self.build,
                stock_item=self.stock_1_2,
                quantity=q12,
                install_into=output
            )

        if q21 > 0:
            BuildItem.objects.create(
                build=self.build,
                stock_item=self.stock_2_1,
                quantity=q21,
                install_into=output,
            )

            # Attempt to create another identical BuildItem
            b = BuildItem(
                build=self.build,
                stock_item=self.stock_2_1,
                quantity=q21
            )

            with self.assertRaises(ValidationError):
                b.clean()

    def test_partial_allocation(self):
        """
        Partially allocate against output 1
        """

        self.allocate_stock(50, 50, 200, self.output_1)

        self.assertTrue(self.build.isFullyAllocated(self.output_1))
        self.assertFalse(self.build.isFullyAllocated(self.output_2))
        self.assertTrue(self.build.isPartFullyAllocated(self.sub_part_1, self.output_1))
        self.assertTrue(self.build.isPartFullyAllocated(self.sub_part_2, self.output_1))

        self.assertFalse(self.build.isPartFullyAllocated(self.sub_part_1, self.output_2))
        self.assertFalse(self.build.isPartFullyAllocated(self.sub_part_2, self.output_2))

        # Check that the part has been allocated
        self.assertEqual(self.build.allocatedQuantity(self.sub_part_1, self.output_1), 100)

        self.build.unallocateStock(output=self.output_1)
        self.assertEqual(BuildItem.objects.count(), 0)

        # Check that the part has been unallocated
        self.assertEqual(self.build.allocatedQuantity(self.sub_part_1, self.output_1), 0)

    def test_auto_allocate(self):
        """
        Test auto-allocation functionality against the build outputs
        """

        allocations = self.build.getAutoAllocations(self.output_1)

        self.assertEqual(len(allocations), 1)

        self.build.autoAllocate(self.output_1)
        self.assertEqual(BuildItem.objects.count(), 1)

        # Check that one part has been fully allocated to the build output
        self.assertTrue(self.build.isPartFullyAllocated(self.sub_part_2, self.output_1))

        # But, the *other* build output has not been allocated against
        self.assertFalse(self.build.isPartFullyAllocated(self.sub_part_2, self.output_2))

    def test_cancel(self):
        """
        Test cancellation of the build
        """

        # TODO

        """
        self.allocate_stock(50, 50, 200, self.output_1)
        self.build.cancelBuild(None)

        self.assertEqual(BuildItem.objects.count(), 0)
        """
        pass

    def test_complete(self):
        """
        Test completion of a build output
        """

        self.allocate_stock(50, 50, 250, self.output_1)
        self.allocate_stock(50, 50, 250, self.output_2)

        self.assertTrue(self.build.isFullyAllocated(self.output_1))
        self.assertTrue(self.build.isFullyAllocated(self.output_2))

        self.build.completeBuildOutput(self.output_1, None)

        self.assertFalse(self.build.can_complete)

        self.build.completeBuildOutput(self.output_2, None)

        self.assertTrue(self.build.can_complete)

        self.build.complete_build(None)

        self.assertEqual(self.build.status, status.BuildStatus.COMPLETE)

        # the original BuildItem objects should have been deleted!
        self.assertEqual(BuildItem.objects.count(), 0)

        # New stock items should have been created!
        self.assertEqual(StockItem.objects.count(), 4)

        a = StockItem.objects.get(pk=self.stock_1_1.pk)

        # This stock item has been depleted!
        with self.assertRaises(StockItem.DoesNotExist):
            StockItem.objects.get(pk=self.stock_1_2.pk)

        c = StockItem.objects.get(pk=self.stock_2_1.pk)

        # Stock should have been subtracted from the original items
        self.assertEqual(a.quantity, 900)
        self.assertEqual(c.quantity, 4500)

        # And 10 new stock items created for the build output
        outputs = StockItem.objects.filter(build=self.build)

        self.assertEqual(outputs.count(), 2)

        for output in outputs:
            self.assertFalse(output.is_building)


from match3Env import Match3Env
import numpy as np
import os
import shutil
import json
import sys
import gym
import ray
from ray.rllib.agents.a3c import A3CTrainer, DEFAULT_CONFIG
from ray.tune.registry import register_env

os.environ["CUDA_VISIBLE_DEVICES"] = '1'

checkpoint_root = 'checkpoint_4'
# Where checkpoints are written:
# shutil.rmtree(checkpoint_root, ignore_errors=True, onerror=None)
# Where some data will be written and used by Tensorboard below:
ray_results = f'{checkpoint_root}/ray_results/'
shutil.rmtree(ray_results, ignore_errors=True, onerror=None)
info = ray.init(ignore_reinit_error=True, num_gpus=1)

trainer_config = DEFAULT_CONFIG.copy()
# trainer_config['num_workers'] = 4
trainer_config["train_batch_size"] = 400

def do_training(Match3EnvClass, config = trainer_config, iterations=20):
    trainer = A3CTrainer(config, Match3EnvClass)
    if os.path.exists(checkpoint_root):
        checkpoints = os.listdir(checkpoint_root)
        for i in range(len(checkpoints)):
            checkpoints[i] = int(checkpoints[i].replace('checkpoint_', ''))
        checkpoint = max(checkpoints)
        trainer.restore(os.path.join(checkpoint_root, 'checkpoint_%06d' % checkpoint, f'checkpoint-{checkpoint}'))
    results = []
    episode_data = []
    episode_json = []
    for i in range(iterations):
        result = trainer.train()
        results.append(trainer)

        episode = {'n': i,
                'episode_reward_min': result['episode_reward_min'],
                'episode_reward_mean': result['episode_reward_mean'],
                'episode_reward_max': result['episode_reward_max'],
                'episode_len_mean': result['episode_len_mean']}

        episode_data.append(episode)
        episode_json.append(json.dumps(episode))
        file_name = trainer.save(checkpoint_root)

    return trainer

# trainer = do_training("CartPole-v1", config=trainer_config, iterations=20)

# env = gym.make('CartPole-v0')
# state = env.reset()

def env_creator(env_config):
    return Match3Env()  # return an env instance

register_env('Match3Env', env_creator)

trainer = do_training('Match3Env', config=trainer_config, iterations=1)

env = Match3Env()
state = env.reset()

done = False
max_state = -1
cumulative_reward = 0
reward = 1
# while not done:
while reward > 0 or not done:
    action = trainer.compute_action(state)
    state, reward, done, results = env.step(action)
    cumulative_reward += reward

print(f'Cumulative reward you received is: {cumulative_reward}. Congratulations!')


from itertools import count
from scipy.optimize import brentq
import logging

class FiberSolver(object):

    """Generic abstract class for callable objects used as fiber solvers."""

    logger = logging.getLogger(__name__)

    def __init__(self, fiber):
        self.fiber = fiber

    def __call__(self, *args, **kwargs):
        raise NotImplementedError()

    def _findFirstRoot(self, fct, args=(), lowbound=0, highbound=None,
                       ipoints=[], delta=0.25, maxiter=None):
        while True:
            if ipoints:
                maxiter = len(ipoints)
            elif highbound:
                maxiter = int((highbound - lowbound) / delta)
            assert maxiter is not None

            a = lowbound
            fa = fct(a, *args)
            if fa == 0:
                return a

            for i in range(1, maxiter+1):
                b = ipoints.pop(0) if ipoints else a + delta
                if highbound:
                    if ((b > highbound > lowbound) or
                            (b < highbound < lowbound)):
                        self.logger.info("_findFirstRoot: no root found within"
                                         " allowed range")
                        return float("nan")
                fb = fct(b, *args)
                if fb == 0:
                    return b

                if (fa > 0 and fb < 0) or (fa < 0 and fb > 0):
                    z = brentq(fct, a, b, args=args, xtol=1e-20)
                    fz = fct(z, *args)
                    if abs(fa) > abs(fz) < abs(fb):  # Skip discontinuities
                        return z

                a, fa = b, fb
            if highbound and maxiter < 100:
                delta /= 10
            else:
                break
        print("maxiter reached", maxiter)
        return float("nan")

    def _findBetween(self, fct, lowbound, highbound, args=()):
        v = [lowbound, highbound]
        s = [fct(lowbound, *args), fct(highbound, *args)]

        for j in count():  # probably not needed...
            for i in range(len(s)-1):
                a, b = v[i], v[i+1]
                fa, fb = s[i], s[i+1]

                if (fa > 0 and fb < 0) or (fa < 0 and fb > 0):
                    z = brentq(fct, a, b, args=args)
                    fz = fct(z, *args)
                    if abs(fa) > abs(fz) < abs(fb):  # Skip discontinuities
                        return z

            ls = len(s)
            for i in range(ls-1):
                a, b = v[2*i], v[2*i+1]
                c = (a + b) / 2
                v.insert(2*i+1, c)
                s.insert(2*i+1, fct(c, *args))


"""WSGI Middleware components

NERC Data Grid Project"""
__author__ = "P J Kershaw"
__date__ = "27/05/08"
__copyright__ = "(C) 2009 Science and Technology Facilities Council"
__license__ = "BSD - see LICENSE file in top-level directory"
__contact__ = "Philip.Kershaw@stfc.ac.uk"
__revision__ = '$Id$'
import logging
log = logging.getLogger(__name__)
import httplib
import re # for NDGSecurityPathFilter

class NDGSecurityMiddlewareError(Exception):
    '''Base exception class for NDG Security middleware'''

class NDGSecurityMiddlewareConfigError(NDGSecurityMiddlewareError):
    '''NDG Security Middleware Configuration error'''

class NDGSecurityMiddlewareBase(object):
    """Base class for NDG Security Middleware classes
    @cvar USERNAME_ENVIRON_KEYNAME: environ key name for user id as used by
    AuthKit
    @type USERNAME_ENVIRON_KEYNAME: string
    """
    USERNAME_ENVIRON_KEYNAME = 'REMOTE_USER'
    USERDATA_ENVIRON_KEYNAME = 'REMOTE_USER_DATA'
    USERNAME_SESSION_KEYNAME = 'username'

    propertyDefaults = {
        'mountPath': '/',
    }
    __slots__ = ('_app', '_environ', '_start_response', '_pathInfo', '_path',
                 '_mountPath')

    def __init__(self, app, app_conf, prefix='', **local_conf):
        '''
        @type app: callable following WSGI interface
        @param app: next middleware application in the chain
        @type app_conf: dict
        @param app_conf: PasteDeploy global configuration dictionary
        @type prefix: basestring
        @param prefix: prefix for app_conf parameters e.g. 'ndgsecurity.' -
        enables other global configuration parameters to be filtered out
        @type local_conf: dict
        @param local_conf: PasteDeploy application specific configuration
        dictionary
        '''
        self._app = app
        self._environ = {}
        self._start_response = None
        self._pathInfo = None
        self._path = None
        self._mountPath = '/'

        # Convenient utility for tracing start_response call
        self.debug_start_response = False

        opt = self.__class__.propertyDefaults.copy()

        # If no prefix is set, there is no way to distinguish options set for
        # this app and those applying to other applications
        if app_conf is not None and prefix:
            # Update from application config dictionary - filter using prefix
            self.__class__._filterOpts(opt, app_conf, prefix=prefix)

        # Similarly, filter keyword input
        self.__class__._filterOpts(opt, local_conf, prefix=prefix)

        # Set options as object attributes
        for name, val in opt.items():
            if not name.startswith('_'):
                setattr(self, name, val)

    def _initCall(self, environ, start_response):
        """Call from derived class' __call__() to set environ and path
        attributes

        @type environ: dict
        @param environ: WSGI environment variables dictionary
        @type start_response: function
        @param start_response: standard WSGI start response function
        """
        self.environ = environ

        if self.debug_start_response:
            def start_response_(status, response_headers, exc_info=None):
                log.debug('Calling start response with '
                          'environ["PATH_INFO"]=%r, '
                          'status=%r, headers=%r, exc_info=%r',
                          environ['PATH_INFO'], status, response_headers,
                          exc_info)

                return start_response(status, response_headers,
                                      exc_info=exc_info)

            self.start_response = start_response_

        else:
            self.start_response = start_response

        self.setPathInfo()
        self.setPath()

    @staticmethod
    def initCall(__call__):
        '''Decorator to __call__ to enable convenient attribute initialisation
        '''
        def __call__wrapper(self, environ, start_response):
            self._initCall(environ, start_response)
            return __call__(self, environ, start_response)

        return __call__wrapper

    def __call__(self, environ, start_response):
        """
        @type environ: dict
        @param environ: WSGI environment variables dictionary
        @type start_response: function
        @param start_response: standard WSGI start response function
        @rtype: iterable
        @return: response
        """
        self._initCall(environ, start_response)
        return self._setResponse(environ, start_response)

    def _setResponse(self,
                     environ=None,
                     start_response=None,
                     notFoundMsg=None,
                     notFoundMsgContentType=None):
        """Convenience method to wrap call to next WSGI app in stack or set an
        error if none is set

        @type environ: dict
        @param environ: WSGI environment variables dictionary defaults to
        environ object attribute.  For the latter to be available, the initCall
        decorator method must have been invoked.
        @type start_response: function
        @param start_response: standard WSGI start response function defaults
        to start_response object attribute.  For the latter to be available,
        the initCall decorator method must have been invoked.
        """
        if environ is None:
            environ = self.environ

        if start_response is None:
            start_response = self.start_response

        if self._app:
            return self._app(environ, start_response)
        else:
            return self._setErrorResponse(start_response=start_response,
                                          msg=notFoundMsg,
                                          code=404,
                                          contentType=notFoundMsgContentType)

    def _setErrorResponse(self, start_response=None, msg=None,
                          code=500, contentType=None):
        '''Convenience method to set a simple error response

        @type start_response: function
        @param start_response: standard WSGI callable to set the HTTP header
        defaults to start_response object attribute.  For the latter to be
        available, the initCall decorator method must have been invoked.
        @type msg: basestring
        @param msg: optional error message
        @type code: int
        @param code: standard HTTP error response code
        @type contentType: basestring
        @param contentType: set 'Content-type' HTTP header field - defaults to
        'text/plain'
        '''
        if start_response is None:
            start_response = self.start_response

        status = '%d %s' % (code, httplib.responses[code])
        if msg is None:
            response = status
        else:
            response = msg

        if contentType is None:
            contentType = 'text/plain'

        start_response(status,
                       [('Content-type', contentType),
                        ('Content-Length', str(len(response)))])
        return [response]

    @staticmethod
    def getStatusMessage(statusCode):
        '''Make a standard status message for use with start_response
        @type statusCode: int
        @param statusCode: HTTP status code
        @rtype: str
        @return: status code with standard message
        @raise KeyError: for invalid status code
        '''
        return '%d %s' % (statusCode, httplib.responses[statusCode])

    # Utility functions to support Paste Deploy application and filter function
    # signatures
    @classmethod
    def filter_app_factory(cls, app, app_conf, **local_conf):
        '''Function signature for Paste Deploy filter

        @type app: callable following WSGI interface
        @param app: next middleware application in the chain
        @type app_conf: dict
        @param app_conf: PasteDeploy global configuration dictionary
        @type prefix: basestring
        @param prefix: prefix for app_conf parameters e.g. 'ndgsecurity.' -
        enables other global configuration parameters to be filtered out
        @type local_conf: dict
        @param local_conf: PasteDeploy application specific configuration
        dictionary
        '''
        return cls(app, app_conf, **local_conf)

    @classmethod
    def app_factory(cls, app_conf, **local_conf):
        '''Function Signature for Paste Deploy app'''
        return cls(None, app_conf, **local_conf)

    @classmethod
    def _filterOpts(cls, opt, newOpt, prefix='', propertyDefaults=None):
        '''Convenience utility to filter input options set in __init__ via
        app_conf or keywords

        @type opt: dict
        @param opt: existing options set.  These will be updated by this
        method based on the content of newOpt
        @type newOpt: dict
        @param newOpt: new options to update opt with
        @type prefix: basestring
        @param prefix: if set, remove the given prefix from the input options
        @type propertyDefaults: iterable/None
        @param propertyDefaults: property names restricted to this dictionary
        of names.  If None, default to propertyDefaults class variable setting
        @raise KeyError: if an option is set that is not in the classes
        defOpt class variable
        '''
        if propertyDefaults is None:
            propertyDefaults = cls.propertyDefaults

        badOpt = []
        for k,v in newOpt.items():
            if prefix and k.startswith(prefix):
                subK = k.replace(prefix, '')
                filtK = '_'.join(subK.split('.'))
            else:
                # Ignore items that are not prefixed
                continue

            if propertyDefaults is not None and filtK not in propertyDefaults:
                badOpt += [k]
            else:
                opt[filtK] = v

        if len(badOpt) > 0:
            raise TypeError("Invalid input option(s) set: %s" %
                            (", ".join(badOpt)))

    def setMountPath(self, mountPath=None, environ=None):
        if mountPath:
            self._mountPath = mountPath
        else:
            if environ is None:
                environ = self._environ

            self._mountPath = environ.get('SCRIPT_URL')
            if self._mountPath is None:
                raise AttributeError("SCRIPT_URL key not set in environ: "
                                     "'mountPath' is set to None")

        # Comment out as it breaks standard for URL trailing slash
#        if self._mountPath != '/':
#            self._mountPath = self._mountPath.rstrip('/')

    def _getMountPath(self):
        return self._mountPath

    mountPath = property(fget=_getMountPath,
                        fset=setMountPath,
                        doc="URL path as assigned to SCRIPT_URL environ key")

    def setPathInfo(self, pathInfo=None, environ=None):
        if pathInfo:
            self._pathInfo = pathInfo
        else:
            if environ is None:
                environ = self._environ

            self._pathInfo = environ['PATH_INFO']

        if self._pathInfo != '/':
            self._pathInfo = self._pathInfo.rstrip('/')

    def _getPathInfo(self):
        return self._pathInfo

    pathInfo = property(fget=_getPathInfo,
                        fset=setPathInfo,
                        doc="URL path as assigned to PATH_INFO environ key")

    def setPath(self, path=None):
        if path:
            self._path = path
        else:
            self._path = self.mountPath.rstrip('/') + self._pathInfo

        if self._path != '/':
            self._path = self._path.rstrip('/')

    def _getPath(self):
        return self._path

    path = property(fget=_getPath,
                        fset=setPath,
                        doc="Full URL path minus domain name - equivalent to "
                            "self.mountPath PATH_INFO environ setting")

    def _setEnviron(self, environ):
        self._environ = environ

    def _getEnviron(self):
        return self._environ

    environ = property(fget=_getEnviron,
                       fset=_setEnviron,
                       doc="Reference to WSGI environ dict")

    def _setStart_response(self, start_response):
        self._start_response = start_response

    def _getStart_response(self):
        return self._start_response

    start_response = property(fget=_getStart_response,
                              fset=_setStart_response,
                              doc="Reference to WSGI start_response function")

    def redirect(self, url, start_response=None):
        """Do a HTTP 302 redirect

        @type start_response: callable following WSGI start_response convention
        @param start_response: WSGI start response callable
        @type url: basestring
        @param url: URL to redirect to
        @rtype: list
        @return: empty HTML body
        """
        if start_response is None:
            # self.start_response will be None if initCall decorator wasn't
            # applied to __call__
            if self.start_response is None:
                raise NDGSecurityMiddlewareConfigError("No start_response "
                                                       "function set.")
            start_response = self.start_response

        start_response(NDGSecurityMiddlewareBase.getStatusMessage(302),
                       [('Content-type', 'text/html'),
                        ('Content-length', '0'),
                        ('Location', url)])
        return []

    @staticmethod
    def parseListItem(item):
        """Utility method for parsing a space separate list of items in a
        string.  Items may be quoted.  This method is useful for parsing items
        assigned to a parameter in a config file e.g.
        fileList: "a.txt" b.csv 'My File'
        @type item: basestring
        @param item: list of space separated items in a string.  These may be
        quoted
        """
        return [i.strip("\"'") for i in item.split()]

class NDGSecurityPathFilter(NDGSecurityMiddlewareBase):
    """Specialisation of NDG Security Middleware to enable filtering based on
    PATH_INFO
    """
    propertyDefaults = {
        'errorResponseCode': 401,
        'serverName': None,
        'pathMatchList': ['/']
    }
    propertyDefaults.update(NDGSecurityMiddlewareBase.propertyDefaults)

    CSV_PAT = re.compile(',\s*')

    # TODO: refactor to:
    # * enable reading of path list from a database or some other
    # configuration source.
    # * enable some kind of pattern matching for paths
    _pathMatch = lambda self: self._pathInfo in self.pathMatchList
    pathMatch = property(fget=_pathMatch,
                         doc="Check for input path match to list of paths"
                             "to which this middleware is to be applied")

    def __init__(self, *arg, **kw):
        '''See NDGSecurityMiddlewareBase for explanation of args
        @type arg: tuple
        @param arg: single element contains next middleware application in the
        chain and app_conf dict
        @type kw: dict
        @param kw: prefix for app_conf parameters and local_conf dict
        '''
        super(NDGSecurityPathFilter, self).__init__(*arg, **kw)

    def _getPathMatchList(self):
        return self.__pathMatchList

    def _setPathMatchList(self, pathList):
        '''
        @type pathList: list or tuple
        @param pathList: list of URL paths to apply this middleware
        to. Paths are relative to the point at which this middleware is mounted
        as set in environ['PATH_INFO']
        '''

        if isinstance(pathList, basestring):
            # Try parsing a space separated list of file paths
             self.__pathMatchList=NDGSecurityPathFilter.CSV_PAT.split(pathList)

        elif not isinstance(pathList, (list, tuple)):
            raise TypeError('Expecting a list or tuple for "pathMatchList"')
        else:
            self.__pathMatchList = list(pathList)

    pathMatchList = property(fget=_getPathMatchList,
                             fset=_setPathMatchList,
                             doc='List of URL paths to which to apply SSL '
                                 'client authentication')

    def _getErrorResponseCode(self):
        """Error response code getter
        @rtype: int
        @return: HTTP error code set by this middleware on client cert.
        verification error
        """
        return self._errorResponseCode

    def _setErrorResponseCode(self, code):
        """Error response code setter
        @type code: int or basestring
        @param code: error response code set if client cert. verification
        fails"""
        if isinstance(code, int):
            self._errorResponseCode = code
        elif isinstance(code, basestring):
            self._errorResponseCode = int(code)
        else:
            raise TypeError('Expecting int or string type for '
                            '"errorResponseCode" attribute')

        if self._errorResponseCode not in httplib.responses:
            raise ValueError("Error response code [%d] is not recognised "
                             "standard HTTP response code" %
                             self._errorResponseCode)

    errorResponseCode = property(fget=_getErrorResponseCode,
                                 fset=_setErrorResponseCode,
                                 doc="Response code raised if client "
                                     "certificate verification fails")


# -*- coding: utf-8 -*-

import os
import math
import argparse
import random
import numpy
import torch
import torch.nn as nn
from bucket_iterator import BucketIterator
from sklearn import metrics
from data_utils import ABSADatesetReader
from  models.pos_gat import PoS_GAT
from  models.biff_lcf_gat import Biff_Lcf_GAT
from models.Type_aware_GAT import  Type_aware_GAT
from models.Relation_aware_Position import  Relation_aware_Pos
from models.syntax_transformer_model import Syntax_local_transformer
import torch.nn.functional as F
from torchsummary import summary
import  numpy as np
import codecs
import os
from thop import profile
# from torch.utils.tensorboard import SummaryWriter
# writer = SummaryWriter("log")

import  time
class Instructor:
    def __init__(self, opt):
        self.opt = opt

        absa_dataset = ABSADatesetReader(dataset=opt.dataset, embed_dim=opt.embed_dim,dependency_dim=opt.dependency_edge_dim,position_dim=opt.position_dim, use_bert=opt.use_bert, max_len=70,opt=opt)

        self.train_data_loader = BucketIterator(data=absa_dataset.train_data, batch_size=opt.batch_size, shuffle=True,sort=False,
                                                max_len=70,)
        self.test_data_loader = BucketIterator(data=absa_dataset.test_data, batch_size=opt.batch_size, shuffle=False,sort=False,
                                               max_len=70)

        self.model = opt.model_class(absa_dataset.embedding_matrix,absa_dataset.dependency_matrix,absa_dataset.position_matrix, opt).to(opt.device)
        # self.model = opt.model_class(absa_dataset.embedding_matrix, opt)
        # self.model = nn.DataParallel(self.model).to(opt.device)
        self._print_args()
        self.global_f1 = 0.

        if torch.cuda.is_available():
            print('cuda memory allocated:', torch.cuda.memory_allocated(device=opt.device.index))

    def _print_args(self):
        # print(self.model.gat_layer_list)
        # print(self.model.ModuleList)
        # exit()
        n_trainable_params, n_nontrainable_params = 0, 0
        for p in self.model.parameters():
            n_params = torch.prod(torch.tensor(p.shape)).item()
            if p.requires_grad:
                n_trainable_params += n_params
            else:
                n_nontrainable_params += n_params
        print('n_trainable_params: {0}, n_nontrainable_params: {1}'.format(n_trainable_params, n_nontrainable_params))
        print('> training arguments:')
        for arg in vars(self.opt):
            print('>>> {0}: {1}'.format(arg, getattr(self.opt, arg)))

    def _reset_params(self):
        for p in self.model.parameters():
            if p.requires_grad:
                if len(p.shape) > 1:
                    self.opt.initializer(p)
                else:
                    stdv = 1. / math.sqrt(p.shape[0])
                    torch.nn.init.uniform_(p, a=-stdv, b=stdv)

    def label_smoothing(self, inputs, epsilon=0.1):
        '''Applies label smoothing. See 5.4 and https://arxiv.org/abs/1512.00567.
        inputs: 3d tensor. [N, T, V], where V is the number of vocabulary.
        epsilon: Smoothing rate.

        For example,

        ```
        import tensorflow as tf
        inputs = tf.convert_to_tensor([[[0, 0, 1],
           [0, 1, 0],
           [1, 0, 0]],
          [[1, 0, 0],
           [1, 0, 0],
           [0, 1, 0]]], tf.float32)

        outputs = label_smoothing(inputs)

        with tf.Session() as sess:
            print(sess.run([outputs]))

        >>
        [array([[[ 0.03333334,  0.03333334,  0.93333334],
            [ 0.03333334,  0.93333334,  0.03333334],
            [ 0.93333334,  0.03333334,  0.03333334]],
           [[ 0.93333334,  0.03333334,  0.03333334],
            [ 0.93333334,  0.03333334,  0.03333334],
            [ 0.03333334,  0.93333334,  0.03333334]]], dtype=float32)]
        ```
        '''
        V = inputs.get_shape().as_list()[-1]  # number of channels
        return ((1 - epsilon) * inputs) + (epsilon / V)

    def _train(self, criterion, optimizer):
        max_test_acc = 0
        max_test_f1 = 0
        global_step = 0
        continue_not_increase = 0

        for epoch in range(self.opt.num_epoch):
            increase_flag = False
            print('>' * 100)
            print('epoch: ', epoch)

            n_correct, n_total = 0, 0
            attention = []
            for i_batch, sample_batched in enumerate(self.train_data_loader):

                global_step += 1

                # switch model to training mode, clear gradient accumulators'
                self.model.train()
                # print(self.model)
                optimizer.zero_grad()
                # print(sample_batched)
                # print(self.opt.inputs_cols)
                inputs = [sample_batched[col].to(self.opt.device) for col in self.opt.inputs_cols]
                targets = sample_batched['polarity'].to(self.opt.device)
                #

                # a_numpy = np.array(inputs)
                # print(a_numpy)
                # #
                # #
                # np.save('./state_dict/twitter_numpy.npy', a_numpy)
                inputs = [inputs,True]

                # print()

                # outputs = torch.nn.DataParallel(self.model(inputs),device_ids=[1,0])
                outputs,e_final  = self.model(inputs)
                attention.append(e_final)
                # flop, para = profile(self.model, input_size=(inputs), )
                # print("%.2fM" % (flop / 1e6), "%.2fM" % (para / 1e6))

                # print(outputs)?
                ## ce loss#########################################
                loss = criterion(outputs, targets)

                loss.backward()
                optimizer.step()

                # predict the best model
                # self.model.load_state_dict(torch.load('state_dict/' + self.opt.model_name + '_' + self.opt.dataset + '.pkl'))

                if global_step % self.opt.log_step == 0:
                    n_correct += (torch.argmax(outputs, -1) == targets).sum().item()
                    n_total += len(outputs)
                    train_acc = n_correct / n_total

                    test_acc, test_f1, test_loss,t_targets_all, t_outputs_all = self._evaluate_acc_f1(criterion)
                    if test_acc > max_test_acc:
                        max_test_acc = test_acc
                    if test_f1 > max_test_f1:
                        increase_flag = True
                        max_test_f1 = test_f1

                        if self.opt.save and test_f1 > self.global_f1:
                            self.global_f1 = test_f1
                            # np.save("./attention_numpy/ture_outputs_all.npy", t_targets_all)
                            # np.save("./attention_numpy/pre_outputs_all.npy", t_outputs_all)
                            torch.save(self.model,
                                       'state_dict/' + self.opt.model_name + '_' + self.opt.dataset+'_'+str(max_test_f1) + '.pkl')
                            print('>>> this best model saved.this f1 is {:.4f}'.format(max_test_f1))
                            report = metrics.classification_report(t_targets_all.cpu(),
                                                                   t_outputs_all.cpu(),
                                                                   labels=[0, 1, 2], )
                            print(report)
                    print('\r >>> this repeat f1 is {:.4f}'.format(max_test_f1 ))
                    print('\r lr:{:E} loss: {:.4f}, acc: {:.4f}, test_loss_all{:.4f}，test_acc: {:.4f}, test_f1: {:.4f}'.format(optimizer.param_groups[0]['lr'],loss.item(), train_acc,
                                                                                test_loss,test_acc, test_f1))
            print(attention)
            if increase_flag == False:
                continue_not_increase += 1
                if continue_not_increase%3==0:
                    new_lr = opt.lr_de*optimizer.param_groups[0]['lr']
                    optimizer.param_groups[0]['lr'] = new_lr
                if continue_not_increase >= 7:
                    print('early stop.')
                    return max_test_acc, max_test_f1
                    break
            else:
                continue_not_increase = 0

        return max_test_acc, max_test_f1

    def _evaluate_acc_f1(self,criterion):
        # switch model to evaluation mode
        self.model.eval()
        n_test_correct, n_test_total,test_loss_all = 0, 0,0
        t_targets_all, t_outputs_all, = None, None,
        # attention=[]
        with torch.no_grad():
            for t_batch, t_sample_batched in enumerate(self.test_data_loader):
                t_inputs = [t_sample_batched[col].to(opt.device) for col in self.opt.inputs_cols]

                t_targets = t_sample_batched['polarity'].to(opt.device)
                t_inputs = [t_inputs,False]

                t_outputs,e_final = self.model(t_inputs)
                test_loss = criterion(t_outputs,t_targets)
                test_loss_all += test_loss.item()
                # print(t_outputs)
                n_test_correct += (torch.argmax(t_outputs, -1) == t_targets).sum().item()
                n_test_total += len(t_outputs)

                if t_targets_all is None:
                    t_targets_all = t_targets
                    t_outputs_all = t_outputs
                else:
                    t_targets_all = torch.cat((t_targets_all, t_targets), dim=0)
                    t_outputs_all = torch.cat((t_outputs_all, t_outputs), dim=0)

        test_acc = n_test_correct / n_test_total
        test_loss_all = test_loss_all
        f1 = metrics.f1_score(t_targets_all.cpu(), torch.argmax(t_outputs_all, -1).cpu(), labels=[0, 1, 2],
                              average='macro')

        # print(attention)
        return test_acc, f1,test_loss_all,t_targets_all.cpu(),torch.argmax(t_outputs_all, -1).cpu()

    def run(self):
        # Loss and Optimizer
        criterion = nn.CrossEntropyLoss()
        # criterion = F.multi_margin_loss(prob, t_label)
        _params = filter(lambda p: p.requires_grad, self.model.parameters())
        optimizer = self.opt.optimizer(_params, lr=self.opt.learning_rate, weight_decay=self.opt.l2reg)
        # exp_lr_scheduler =torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)
        time_str = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))
        f_out = codecs.open('log/' + self.opt.model_name + '_' + self.opt.dataset +'_Syn_Layer_'+str(self.opt.Syn_Layer) +'_val.txt', 'a+',encoding="utf-8")
        total = sum([param.nelement() for param in self.model.parameters()])
        print("Number of parameter: %.2fM" % (total / 1e6))

        f_out.write('time:{0}\n'.format(time_str))
        arguments = " "
        for arg in vars(self.opt):
            arguments += '{0}: {1} '.format(arg, getattr(self.opt, arg))
        f_out.write(arguments)
        print('repeat: ', (i + 1))
        self._reset_params()
        # optimizer.param_groups[0]['lr'] = opt.learning_rate
        max_test_acc, max_test_f1 = self._train(criterion, optimizer)
        print("----------------------------")
        print('max_test_acc: {0}     max_test_f1: {1}'.format(max_test_acc, max_test_f1))
        print('#' * 100)
        f_out.write('max_test_acc: {0}, max_test_f1: {1}\n'.format(max_test_acc, max_test_f1))
        f_out.close()
        return  max_test_acc,max_test_f1

if __name__ == '__main__':
    # Hyper Parameters
    parser = argparse.ArgumentParser()
    # parser.add_argument('--model_name', default='asgcn_new', type=str)
    parser.add_argument('--model_name', default='Relation_aware_Pos', type=str)  # Syntax_local_transformer,pos_gat，Type_aware_gat
    parser.add_argument('--dataset', default='twitter', type=str, help='lap14,twitter, rest14, rest15, rest16')
    parser.add_argument('--optimizer', default='adam', type=str)
    parser.add_argument('--initializer', default='xavier_uniform_', type=str)
    parser.add_argument('--learning_rate', default=0.005, type=float)
    parser.add_argument('--l2reg', default=0.0001, type=float)
    parser.add_argument('--num_epoch', default=100, type=int)
    parser.add_argument('--batch_size', default=20, type=int)
    parser.add_argument('--log_step', default=5, type=int)
    parser.add_argument('--embed_dim', default=300, type=int)
    parser.add_argument('--position_dim',default=100,type=int)
    parser.add_argument('--position_drop', default=0.4, type=float)

    parser.add_argument('--use_edge_weight', default="yes", type=str)
    parser.add_argument('--dependency_edge_dim',default=100,type=int)
    parser.add_argument('--edge_hidden_dim', default=300, type=int)

    # parser.add_argument('--gat_hidden_dim',default=100,type=int )
    parser.add_argument('--hidden_dim', default=300, type=int)
    parser.add_argument('--polarities_dim', default=3, type=int)
    parser.add_argument('--save', default=True, type=bool)
    parser.add_argument('--seed', default=3, type=int)  # 776
    parser.add_argument('--device', default="cuda:0", type=str)
    parser.add_argument('--use_lstm_attention', default=True, type=bool)
    parser.add_argument('--use_bert', default=False, type=bool)
    parser.add_argument('--use_speech_weight', default=True, type=bool)
    parser.add_argument('--lcf', default="cdw",type=str)
    parser.add_argument('--Syn_Layer',default=5,type=int)
    parser.add_argument('--SRD',default=3,type=int)
    parser.add_argument('--lr_de', default=0.5, type=float)
    parser.add_argument('--context_length_type', default="context", type=str)  #context or syn
    parser.add_argument('--max_syntactic_distance', default=8, type=int)
    parser.add_argument('--GAT_alpha', default=0.2, type=float)
    parser.add_argument('--n_heads', default=4, type=int)
    parser.add_argument('--biff_layer_number', default=2, type=int)
    parser.add_argument('--use_gat_gru',default="yes",type=str)
    parser.add_argument('--use_p_embedding',default="yes" ,type =str)

    parser.add_argument('--hop', default=2, type=int)
    parser.add_argument("--train_type",default="trian",type = str)
    #transformer  encoder
    # parser.add_argument('--use_transformer',default=True,type=bool)
    parser.add_argument('--max_length', default=100, type=int)
    parser.add_argument('--num_layers', default=3, type=int)
    parser.add_argument('--model_dim', default=300, type=int)
    parser.add_argument('--num_heads', default=6, type=int)
    parser.add_argument('--ffn_dim', default=512, type=int)
    parser.add_argument('--text_embed_dropout', default=0.5, type=float)
    parser.add_argument('--edge_embed_dropout', default=0.5, type=float)
    parser.add_argument('--liner_dropout', default=0.5, type=float)

    parser.add_argument('--graph_dropout', default=0.3, type=float)

    #gat_type
    parser.add_argument('--use_scaled_dot', default=False, type=bool)
    opt = parser.parse_args()
    model_classes = {

        'pos_gat': PoS_GAT,
        'Syntax_local_transformer':Syntax_local_transformer,
        'biff_lcf_gat' :Biff_Lcf_GAT,
        'Type_aware_gat':Type_aware_GAT,
        'Relation_aware_Pos':Relation_aware_Pos,
    }
    if opt.use_speech_weight ==True:
        input_colses = {
            'Syntax_local_transformer': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_graph_redir','dependency_graph_undir','dependency_edge_matrix','dependency_edge_matrix_re','dependency_edge_matrix_undir','position_syntax_indices ','speech_list'],
            'pos_gat': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_graph_redir','dependency_graph_undir','dependency_edge_matrix','dependency_edge_matrix_re','dependency_edge_matrix_undir','position_syntax_indices','speech_list'],
            'biff_lcf_gat': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_graph_redir','dependency_graph_undir','dependency_edge_matrix','dependency_edge_matrix_re','dependency_edge_matrix_undir','position_syntax_indices', 'speech_list'],
            'Type_aware_gat': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_graph_redir','dependency_graph_undir', 'dependency_edge_matrix','dependency_edge_matrix_re','dependency_edge_matrix_undir','position_syntax_indices', 'speech_list'],
            'Relation_aware_Pos': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_graph_redir','dependency_graph_undir', 'dependency_edge_matrix','dependency_edge_matrix_re','dependency_edge_matrix_undir','position_syntax_indices', 'speech_list'],

        }
    else:
        input_colses = {
            'Syntax_local_transformer': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir','dependency_edge_matrix','dependency_graph_undir','position_syntax_indices'],
            'pos_gat': ['text_indices', 'aspect_indices', 'left_indices','dependency_graph_dir','dependency_graph_undir','position_syntax_indices'],
            'biff_lcf_gat': ['text_indices', 'aspect_indices', 'left_indices', 'dependency_graph_dir',
                        'dependency_graph_undir','position_syntax_indices'],

        }
    initializers = {
        'xavier_uniform_': torch.nn.init.xavier_uniform_,
        'xavier_normal_': torch.nn.init.xavier_normal,
        'orthogonal_': torch.nn.init.orthogonal_,
    }
    optimizers = {
        'adadelta': torch.optim.Adadelta,  # default lr=1.0
        'adagrad': torch.optim.Adagrad,  # default lr=0.01
        'adam': torch.optim.Adam,  # default lr=0.001
        'adamax': torch.optim.Adamax,  # default lr=0.002
        'asgd': torch.optim.ASGD,  # default lr=0.01
        'rmsprop': torch.optim.RMSprop,  # default lr=0.01
        'sgd': torch.optim.SGD,
    }

    opt.model_class = model_classes[opt.model_name]
    # summary(opt.model_class,input_size=(32,32,300))
    opt.inputs_cols = input_colses[opt.model_name]
    opt.initializer = initializers[opt.initializer]
    opt.optimizer = optimizers[opt.optimizer]
    # opt.device = torch.device('cpu')
    opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') \
        if opt.device is None else torch.device(opt.device)

    if opt.seed is not None:
        random.seed(opt.seed)
        numpy.random.seed(opt.seed)
        torch.manual_seed(opt.seed)
    repeats = 3
    max_test_acc_avg = 0
    max_test_f1_avg = 0
    # if opt.train_type =="test":
    #     model_state= 'state_dict/' + opt.model_name + '_' + opt.dataset + '.pkl'
    #     model = ins.model
    #     model.load_state_dict(torch.load(model_state))
    #     print(model)
    #     exit()
    for i in range(repeats):
        ins = Instructor(opt)
        max_test_acc,max_test_f1=ins.run()
        max_test_acc_avg += max_test_acc
        max_test_f1_avg += max_test_f1
    f_out = codecs.open('log/' + opt.model_name + '_' + opt.dataset +'_Syn_Layer_'+str(opt.Syn_Layer)+ '_val.txt', 'a+',encoding="utf-8")
    print("max_test_acc_avg:", max_test_acc_avg / repeats)
    print("max_test_f1_avg:", max_test_f1_avg / repeats)
    f_out.write('max_test_acc_avg: {0}, max_test_f1_avg: {1}\n'.format(max_test_acc_avg / repeats,
                                                                       max_test_f1_avg / repeats))
    f_out.write("\n")


#!/usr/bin/python
import numpy as np
import pandas as pd
import os
from sklearn.externals import joblib
import ConfigParser
import ipdb

def main(method="promp",min_max_scaler=None,emg_index=None,
         leftHand_index=None,leftJoint_index=None, datasets_path=None):

    # read csv file
    csv_path = os.path.join(datasets_path, 'info/noise/multiModal_states.csv')
    data = pd.read_csv(csv_path)
    # extract the all signals data ORIENTATION
    emg = data.values[:, 7:15].astype(float)
    left_hand = data.values[:, leftHand_index[0]:leftHand_index[-1]].astype(float)
    left_joints = data.values[:, leftJoint_index[0]:leftJoint_index[-1]].astype(float)  # robot ee actually

    if method == "promp":
        full_data = left_joints[1200:, :]
        full_data = min_max_scaler.transform(left_joints)

    elif method == "emg_ipromp":
        full_data = np.hstack([left_hand, emg, left_joints])[1200:, :]
        full_data = min_max_scaler.transform(full_data)

    elif method =="ipromp":
        full_data = np.hstack([left_hand, left_joints])[1200:, :]
        full_data = min_max_scaler.transform(full_data)

    # compute the noise observation covariance matrix
    noise_cov = np.cov(full_data.T)
    # save it in pkl
    joblib.dump(noise_cov, os.path.join(datasets_path, 'pkl/'+method+'_noise_cov.pkl'))
    print('Saved the noise covariance matrix successfully!')

if __name__ == '__main__':
    main()


import numpy as np
from sklearn.decomposition.pca import PCA
from numpy import loadtxt, genfromtxt, savetxt, var, random
from pylab import figure
from matplotlib.pyplot import plot, savefig, xlabel, ylabel, scatter, axis, xlim

#data_dir='../../data/'
data_dir='../../data_all_types/'
out_dir='./plots/'

# load the data and run PCA
derivatives = loadtxt(data_dir+'derivatives.dat')
labels = loadtxt(data_dir+'labels.dat')
spectra_data = genfromtxt(data_dir+'spectra_data.dat',dtype=None)

random.seed(1)

sne_names= set(spectra_data['f0']) # one entry for each SN
train_names=[]
for sn in sne_names:
    if random.random() > .20:
        train_names.append(sn)

train_mask_TL=[]
train_mask_no_TL=[]
test_mask=[]
for i in range(np.size(labels)):
    if labels[i]==1:
        if spectra_data['f0'][i] in train_names:
            train_mask_TL.append(i)
            train_mask_no_TL.append(i)
        else:
            test_mask.append(i)
    if labels[i]==0:
        #if spectra_data['f0'][i] in train_names:
        train_mask_TL.append(i)

#######################################################
#   To run the DL.
from rpy2.robjects import r
def DL( train, test, n ):

    input_string = '120,100,90,50,30,20,%d,20,30,50,90,100,120' % (n)
    seed = 1
    in_param = 'training_frame = train3.hex, activation = "Tanh", autoencoder=T, hidden = c(%s), epochs=100, ignore_const_cols = F, seed = %d '  % (input_string,seed)
    n_layers = 7
    size_in_data = np.shape(train)[1]
    in_param = 'x=1:%d,' % (size_in_data) + in_param
    in_file_cvs='./.temp_file.cvs'
    np.savetxt(in_file_cvs, train, delimiter=',', fmt='%.18f')
    ## define libraries
    r('library(h2o)')
    ## Read data
    r('h2oServer <- h2o.init(nthreads=-1)')
    r('TRAIN = "%s" '% (in_file_cvs))
    r('train3.hex <- h2o.importFile(h2oServer, path = TRAIN, header = F, sep = ",", destination_frame="train3.hex")')
    ## Construct deep learning model
    r('dlmodel <- h2o.deeplearning( %s ) ' % (in_param))
    ## Generate new features
    r('features_dl <- h2o.deepfeatures(dlmodel, train3.hex, layer=%d)' % (n_layers))
    r('head(features_dl)')
    ## new features
    X = r('as.matrix(features_dl)')

    in_file_cvs='./.temp_file.cvs'
    np.savetxt(in_file_cvs, test, delimiter=',', fmt='%.18f')
    ## Read data
    r('h2oServer <- h2o.init(nthreads=-1)')
    r('TEST = "%s" '% (in_file_cvs))
    r('test3.hex <- h2o.importFile(h2oServer, path = TEST, header = F, sep = ",", destination_frame="test3.hex")')
    ## Generate predictions
    r('predictions_dl <- h2o.predict(dlmodel, test3.hex)')
    r('head(predictions_dl)')
    ## new predictions
    pred = r('as.matrix(predictions_dl)')
    return var(pred -test)
################################################################

figure()
variances_table = []

for i in range(2,11,1):
    pca = PCA(n_components=i)
    der = derivatives[train_mask_TL]
    pca.fit(der)
    X = pca.transform(derivatives[test_mask])
    pred_pca_temp = (pca.inverse_transform(X))

    #
    var_fraction_pca_TL = var(pred_pca_temp-derivatives[test_mask])/var(derivatives[test_mask])
    #plot([i], [var(pred_pca_temp-derivatives[test_mask])],'D')

    var_fraction_DL_TL = DL( derivatives[train_mask_TL], derivatives[test_mask], i)/var(derivatives[test_mask])
    #plot([i], [var_DL_TL ],'Dk')

    pca = PCA(n_components=i)
    der = derivatives[train_mask_no_TL]
    pca.fit(der)
    X = pca.transform(derivatives[test_mask])
    pred_pca_temp = (pca.inverse_transform(X))

    var_fraction_pca_no_TL = var(pred_pca_temp-derivatives[test_mask])/var(derivatives[test_mask])
    #plot([i], [var(pred_pca_temp-derivatives[test_mask])],'x')

    var_fraction_DL_no_TL = DL( derivatives[train_mask_no_TL], derivatives[test_mask], i)/var(derivatives[test_mask])
    #plot([i], [var_DL_no_TL ],'xk')

    variances_table.append([i,var_fraction_pca_no_TL, var_fraction_pca_TL, var_fraction_DL_no_TL, var_fraction_DL_TL])

savetxt('./residuals.dat', variances_table, header='nPCs fraction_variance_PCA_at_max fraction_variance_PCA_with_TL fraction_variance_DL_at_max fraction_variance_DL_with_TL'   )

data_to_plot = loadtxt('./residuals.dat').T
figure()
plot(data_to_plot[0], data_to_plot[1],'xr')
plot(data_to_plot[0], data_to_plot[2],'Dr')
plot(data_to_plot[0], data_to_plot[3],'xk')
plot(data_to_plot[0], data_to_plot[4],'Dk')


from django.shortcuts import render, render_to_response
from django.http import HttpResponse

from solos.models import Solo

def index(request):

    context = {'solos': []}
    if request.GET.keys():
        solos_queryset = Solo.objects.all()
        if request.GET.get('instrument', None):
            solos_queryset = solos_queryset.filter(
                instrument=request.GET.get('instrument', None))
        if request.GET.get('artist', None):

            solos_queryset = solos_queryset.filter(artist=request.GET.get('artist', None))

        context['solos'] = solos_queryset

    return render_to_response('solos/index.html', context)


# import math
# import random
# import calendar
# import datetime
# y = datetime.timedelta(days=1)
# print(y)

# print(calendar.month(2018,8))
# num1 =int( input('input num1'))
# num2 =float( input('input num2'))1
# sum = num1 + num2
# print('numOne {0} + numTwo {1} equals: {2}'.format(num1,num2,sum))
# print('-----------')
# num3 = int(input('input numThree'))
# num4 = math.sqrt(num3)
# print('sqrt num3 {0} equals  {1}'.format(num3,num4))
# print(random.randint(1,2))
# x = input('input x\'s value')
# y = input('input y\'s value')
# x,y = (y,x)
# # print('x = {0} , y = {1}'.format(x,y))
# num5 = float(input('input num5 :'))
# if num5 >= 0:
#     if num5 == 0:
#         print('zero')
#     else:
#         print('+')
# else:
#     print('-')
# while True:
#     try:
#         num=int(input('输入一个整数：')) #判断输入是否为整数
#     except ValueError: #不是纯数字需要重新输入
#         print("输入的不是整数！")
#         continue
#     if num%2==0:
#         print('偶数')
#     else:
#         print('奇数')
#     break
# !/usr/bin/python3

# import pymysql
#
# # 打开数据库连接
# db = pymysql.connect("localhost", "root", "root111", "TESTDB1")
#
# # 使用 cursor() 方法创建一个游标对象 cursor
# cursor = db.cursor()
#
# # 使用 execute()  方法执行 SQL 查询
# cursor.execute("SELECT VERSION()")
#
# # 使用 fetchone() 方法获取单条数据.
# data = cursor.fetchone()
#
# print("Database version : %s " % data)
#
# # 关闭数据库连接
# db.close()
# !/usr/bin/python3

import pymysql

# 打开数据库连接
db = pymysql.connect("localhost", "root", "root111", "TESTDB1")

# 使用 cursor() 方法创建一个游标对象 cursor
cursor = db.cursor()

# 使用 execute() 方法执行 SQL，如果表存在则删除
#cursor.execute("DROP TABLE IF EXISTS EMPLOYEE")

# 使用预处理语句创建表
# sql = """CREATE TABLE EMPLOYEE (
#          FIRST_NAME  CHAR(20) NOT NULL,
#          LAST_NAME  CHAR(20),
#          AGE INT,
#          SEX CHAR(1),
#          INCOME FLOAT )"""
# # SQL 插入语句
# sql = """ delete from EMPLOYEE  where FIRST_NAME = 'Mac12' """
# try:
#     # 执行sql语句
#     cursor.execute(sql)
#     # x = 10/0
#     # 提交到数据库执行
#     db.commit()
# except:
#     # 如果发生错误则回滚
#     db.rollback()
# SQL 查询语句
sql = """SELECT * FROM EMPLOYEE
       WHERE INCOME > 1000 """
try:
    # 执行SQL语句
    cursor.execute(sql)
    # 获取所有记录列表
    results = cursor.fetchall()
    for row in results:
        fname = row[0]
        lname = row[1]
        age = row[2]
        sex = row[3]
        income = row[4]
        # 打印结果
        print("fname=%s,lname=%s,age=%d,sex=%s,income=%d" % \
              (fname, lname, age, sex, income))
except:
    print("Error: unable to fetch data")

# 关闭数据库连接
db.close()


import cv2
import numpy as np

img_path = 'F:\OpenCVDataSet\misc\lena_color_256.tif'
img = cv2.imread(img_path)
cv2.imshow('Lena', img)
cv2.waitKey(0)

'''
IMAGE BLURRING EX:
'''
# CREATING 3*3 KERNEL
kernel_3x3 = np.ones((3, 3), np.float32) / 9

# WE USE THE cv2.filter2S TO CONVOLVE THE KERNEL WITH AN IMAGE
blurred = cv2.filter2D(img, -1, kernel_3x3)
cv2.imshow('3x3 Kernel Blurring', blurred)

cv2.waitKey(0)

# CREATING 7X7 KERNEL
kernel_7x7 = np.ones((7, 7), np.float32) / 49
blurred2 = cv2.filter2D(img, -1, kernel_7x7)
cv2.imshow('7x7 Kernel Blurring', blurred2)

cv2.waitKey(0)
cv2.destroyAllWindows()


# -*-coding: utf-8-*-
from django.db import models
from django.core.urlresolvers import reverse
from django.dispatch import receiver
from django.db.models.signals import pre_save, pre_delete

class Group(models.Model):

    title = models.CharField(verbose_name=u'Название', max_length=300)
    head = models.ForeignKey('Student',
                             verbose_name=u'Староста',
                             related_name='groups',
                             null=True,
                             blank=True,
                             on_delete=models.SET_NULL)

    class Meta:
#       db_table = 'group'
        verbose_name = 'Группа'
        verbose_name_plural = 'Группы'

    def __unicode__(self):
        """Returns the title of the group."""

        return self.title

    def get_absolute_url(self):
        """Gives absolute url to the group."""

        return reverse('students.views.group', kwargs={'group_id': self.id})

class Student(models.Model):

    name = models.CharField(verbose_name=u'Имя', max_length=30)
    surname = models.CharField(verbose_name=u'Фамилия', max_length=40)
    patronymic = models.CharField(verbose_name=u'Отчество', max_length=40)
    birthday = models.DateField(verbose_name=u'Дата рождения')
    ticket = models.CharField(verbose_name=u'Студ-билет', max_length=20)
    group = models.ForeignKey('Group',
                              verbose_name=u'Группа',
                              related_name='students',
                              null=True,
                              blank=True,
                              on_delete=models.SET_NULL)

    class Meta:
#       db_table = 'student'
        verbose_name = 'Студент'
        verbose_name_plural = 'Студенты'

    def __unicode__(self):
        """Returns name of the student."""

        return u'{0} {1} {2}'.format(self.surname, self.name, self.patronymic)

class Log(models.Model):

    model_name = models.CharField(verbose_name=u'Имя модели', max_length=50)
    action = models.CharField(verbose_name=u'Действия', max_length=50)
    instance = models.CharField(verbose_name=u'Экземпляр', max_length=50)
    date = models.DateTimeField(verbose_name=u'Дата', auto_now_add=True)

    class Meta:
#       db_table = 'log'
        verbose_name = 'Лог'
        verbose_name_plural = 'Логи'

    def __unicode__(self):
        """Returns detailed information of the log."""

        return u'Экземпляр "{0}" модели "{1}" был {2} {3}'.format(self.instance, self.model_name, self.action, self.date.strftime("%d.%m.%Y %H:%M"))

@receiver(pre_save, sender=Group)
@receiver(pre_save, sender=Student)
def new_instance(sender, instance, **kwargs):

    new_log = Log()

    try:
        sender.objects.get(pk=instance.id)
        new_log.model_name = sender.__name__
        new_log.action = u'модифицирован'
        new_log.instance = str(instance)

    except sender.DoesNotExist:
        new_log.model_name = sender.__name__
        new_log.action = u'создан'
        new_log.instance = str(instance)

    new_log.save()

@receiver(pre_delete, sender=Group)
@receiver(pre_delete, sender=Student)
def model_delete_signal(sender, instance, **kwargs):
    new_log = Log()
    new_log.model_name = sender.__name__
    new_log.action = u'удален'
    new_log.instance = str(instance)
    new_log.save()


#!/usr/bin/env python

import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import astor

def TestMe(x, y, width=10, foo=None):
    from foo.bar.baz.murgatroyd import sally as bob

    a.b = c.d + x.y.z.a.b
    m.n = q = (w.w, x.x.y.y) = f(x.x.y.z)

func_ast = astor.codetoast(TestMe)
print(astor.dump(func_ast))
print(astor.to_source(func_ast))

class ConsolidateAttributes(astor.TreeWalk):
    def post_Attribute(self):
        node = self.cur_node
        value = node.value
        value.id += '.' + node.attr
        self.replace(value)

ConsolidateAttributes(func_ast)

class FindNames(astor.TreeWalk):
    def init_Assign(self):
        self.assignments = []
        self.current_assign = None
    def pre_Assign(self):
        self.current_assign = [], []
    def post_Assign(self):
        self.assignments.append(self.current_assign)
        self.current_assign = None

    def pre_targets_name(self):
        self.in_targets = True
    def post_targets_name(self):
        self.in_targets = False
    def post_Name(self):
        if self.current_assign:
            self.current_assign[self.in_targets].append(self.cur_node.id)

x = FindNames(func_ast)
print(x.assignments)

print('')
print('')
print(astor.dump(func_ast))
print(astor.to_source(func_ast))


import requests, re, sys, os
from urllib3.contrib.appengine import AppEngineManager

def main(ano):

    if len(ano) == 2:
        anoDef=int(ano)
    else :
        anoDef= int(ano[-2:])

    path="C:/Users/DarioMatos/Desktop/EDC/f1nalisys/f1nalisys/webapp/Corridas/20"+str(anoDef)

    os.mkdir(path)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/0", verify=False)
    f = open(path+"/20"+str(anoDef)+"_0.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response2 = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/drivers", verify=False)
    f = open(path+"/20"+str(anoDef)+"_drivers.xml", "x")
    resposta= change(response2.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/constructors", verify=False)
    f = open(path+"/20"+str(anoDef)+"_constructors.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/circuits", verify=False)
    f = open(path+"/20"+str(anoDef)+"circuits.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/2/results", verify=False)
    f = open(path+"/20"+str(anoDef)+"race2.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/1/results", verify=False)
    f = open(path+"/20"+str(anoDef)+"race1.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/3/results", verify=False)
    f = open(path+"/20"+str(anoDef)+"race3.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/driverStandings", verify=False)
    f = open(path+"/20"+str(anoDef)+"driverStandings.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

    response = requests.get("http://ergast.com/api/f1/20"+str(anoDef)+"/constructorStandings", verify=False)
    f = open(path+"/20"+str(anoDef)+"constructorStandings.xml", "x")
    resposta= change(response.text)
    f.write(resposta)

def change(stringz):
    str1=stringz.replace("utf-8", "iso-8859-1")
    str2=str1.replace('<?xml-stylesheet type="text/xsl" href="/schemas/mrd-1.4.xsl"?>', '')
    result = re.search('<MRData(.*)>', str2)
    str3=str2.replace(result.group(1), '')
    return str3

main(sys.argv[1])


"""
Copyright (c) 2017, Intel Corporation

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Intel Corporation nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import struct
import socket
import sys
import os

import pygit2

unit_tests_dir = os.path.dirname(os.path.abspath(__file__))
ezbench_dir = os.path.dirname(unit_tests_dir)
timings_db_dir = os.path.join(ezbench_dir, 'timing_DB')

sys.path.insert(0, os.path.join(ezbench_dir, 'python-modules'))
sys.path.insert(0, os.path.join(ezbench_dir, 'protocols'))
sys.path.insert(0, timings_db_dir)
sys.path.insert(0, ezbench_dir)

tmp_folder = ezbench_dir + "/unit_tests/tmp"

def send_msg(client, msg):
    buf = msg.SerializeToString()
    if len(buf) == 0:
        print("WARNING: Trying to send 0 bytes!")
        traceback.print_stack()
    pack1 = struct.pack('>I', len(buf))
    client.sendall(pack1 + buf)

def recv_msg(client, msg):
    msg_len_buf = client.recv(4, socket.MSG_WAITALL)
    if len(msg_len_buf) != 4:
        return None

    msg_len = struct.unpack('>I', msg_len_buf)[0]
    buf = bytes()
    while len(buf) < msg_len:
        buf += client.recv(msg_len - len(buf), socket.MSG_WAITALL)

    if len(buf) == msg_len:
        msg.ParseFromString(buf)
        return msg
    else:
        print("Received less than expected (got {} instead of {})".format(len(buf), msg_len), buf)
        return None

class GitRepoFactory:
    def __init__(self, path, bare=False):
        # Create a test dut-repo and a couple of commits
        self.repo_path = path
        self.is_bare = bare
        self.src_repo = pygit2.init_repository(path, self.is_bare)

    def create_commit(self, title, files=[], parents=[], branch_name="master"):
        # Create the data
        tb = self.src_repo.TreeBuilder()
        for filepath, data in files:
            data_oid = self.src_repo.create_blob(data)
            tb.insert(filepath, data_oid, pygit2.GIT_FILEMODE_BLOB)

        # Make the commit
        author = pygit2.Signature("EzBench Test Author", "test@author.com")
        reference="refs/heads/" + branch_name
        commit_id = self.src_repo.create_commit(reference, author, author, title,
                                       tb.write(), parents)
        commit = self.src_repo.revparse_single(str(commit_id))

        # Update the master branch
        try:
            branch = self.src_repo.lookup_branch(branch_name)
        except:
            branch = self.src_repo.create_branch(branch_name, commit, False)
        branch.set_target(str(commit_id))

        # Now update the working copy to the current branch
        if not self.is_bare:
            self.src_repo.checkout(reference)

        return str(commit_id)

    def create_tag(self, name, commit, msg="Nothing to say here"):
        author = pygit2.Signature("EzBench Test Author", "test@author.com")
        return self.src_repo.create_tag(name, commit, pygit2.GIT_OBJ_COMMIT,
                                        author, msg)


import logging
import os
import time

from cassandra.cluster import Cluster
from cassandra.policies import RetryPolicy, RoundRobinPolicy, TokenAwarePolicy

# Set default log.handler to avoid "No handler found" warnings.

stderrLogger = logging.StreamHandler()
f = '%(filename)s: %(levelname)s: %(funcName)s(): %(lineno)d:\t%(message)s'
stderrLogger.setFormatter(logging.Formatter(f))

log = logging.getLogger('hecuba')
log.addHandler(stderrLogger)

if 'DEBUG' in os.environ and os.environ['DEBUG'].lower() == "true":
    log.setLevel(logging.DEBUG)
elif 'HECUBA_LOG' in os.environ:
    log.setLevel(os.environ['HECUBA_LOG'].upper())
else:
    log.setLevel(logging.ERROR)

class _NRetry(RetryPolicy):
    def __init__(self, time_to_retry=5):
        self.time_to_retry = time_to_retry

    def on_unavailable(self, query, consistency, required_replicas, alive_replicas, retry_num):
        if retry_num > self.time_to_retry:
            return self.RETHROW, None
        else:
            return self.RETHROW, None

    def on_write_timeout(self, query, consistency, write_type, required_responses, received_responses, retry_num):
        if retry_num > self.time_to_retry:
            return self.RETHROW, None
        else:
            return self.RETHROW, None

    def on_read_timeout(self, query, consistency, required_responses, received_responses, data_retrieved, retry_num):
        if retry_num > self.time_to_retry:
            return self.RETHROW, None
        else:
            return self.RETHROW, None

class Config:
    class __Config:
        def __init__(self):
            self.configured = False
            ## intercepted : list of numpy intercepted calls
            self.intercepted = {}
            self.configdir = {} # map<string, string>

    instance = __Config()

    def __getattr__(self, item):
        return getattr(Config.instance, item)

    def executequery_withretries(self, query):
        """
            Executes 'query' to cassandra. If the query fails (for example due
            to a timeout) it will resend the query a maximum of NRETRIES(5)
        """
        executed=False
        nretries=0
        while not executed:
            try:
                self.instance.session.execute(query)
                executed=True
            except Exception as ir:
                log.warn("Unable to execute %s %s/5", query, nretries)
                nretries+=1
                if nretries==5: #FIXME the number of retries should be configurable
                    log.error("Too many retries. Aborting. Unable to execute %s", query)
                    raise ir
                time.sleep(1)

    def executelocked(self,query):
        if self.instance.concurrent_creation:
            r=self.instance.session.execute(self.instance._query_to_lock,[query])
            if r[0][0]:
                self.executequery_withretries(query)
            else:
                # FIXME find a better way to do this instead of an sleep... describe?
                time.sleep(.300)
        else:
            self.executequery_withretries(query)

    def __init__(self):
        singleton = Config.instance
        if singleton.configured:
            log.info('setting down')
            return

        singleton.configured = True

        if 'HECUBA_ARROW' in os.environ:
            env_var = os.environ['HECUBA_ARROW'].lower()
            singleton.arrow_enabled = False if env_var == 'no' or env_var == 'false' else True
            log.info('HECUBA_ARROW: {}'.format(singleton.arrow_enabled))
        else:
            singleton.arrow_enabled = False
            log.warn('Arrow access is DISABLED [HECUBA_ARROW=%s]', singleton.arrow_enabled)
        singleton.configdir['hecuba_arrow'] = 'true' if singleton.arrow_enabled else 'false'

        if 'CONCURRENT_CREATION' in os.environ:
            if os.environ['CONCURRENT_CREATION']=='True':
                singleton.concurrent_creation = True
            else:
                singleton.concurrent_creation = False
            log.info('CONCURRENT_CREATION: %s', str(singleton.concurrent_creation))
        else:
            singleton.concurrent_creation = False
            log.warn('Concurrent creation is DISABLED [CONCURRENT_CREATION=False]')
        singleton.configdir['concurrent_creation'] = 'true' if singleton.concurrent_creation else 'false'

        if 'LOAD_ON_DEMAND' in os.environ:
            if os.environ['LOAD_ON_DEMAND']=='False':
                singleton.load_on_demand = False
            else:
                singleton.load_on_demand = True
            log.info('LOAD_ON_DEMAND: %s', str(singleton.load_on_demand))
        else:
            singleton.load_on_demand = True
            log.warn('Load data on demand is ENABLED [LOAD_ON_DEMAND=True]')
        singleton.configdir['load_on_demand'] = 'true' if singleton.load_on_demand else 'false'

        if 'CREATE_SCHEMA' in os.environ:
            env_var = os.environ['CREATE_SCHEMA'].lower()
            singleton.id_create_schema = False if env_var == 'no' or env_var == 'false' else True
            log.info('CREATE_SCHEMA: %d', singleton.id_create_schema)
        else:
            singleton.id_create_schema = True
            log.warn('Creating keyspaces and tables by default [CREATE_SCHEMA=True]')
        singleton.configdir['id_create_schema'] = 'true' if singleton.load_on_demand else 'false'
        try:
            singleton.nodePort = int(os.environ['NODE_PORT'])
            log.info('NODE_PORT: %d', singleton.nodePort)
        except KeyError:
            log.warn('using default NODE_PORT 9042')
            singleton.nodePort = 9042
        singleton.configdir['node_port'] = str(singleton.nodePort)

        try:
            singleton.contact_names = os.environ['CONTACT_NAMES'].split(",")
            log.info('CONTACT_NAMES: %s', str.join(" ", singleton.contact_names))
            # Convert node names to ips if needed
            import socket
            contact_names_ips = []
            show_translation = False
            for h_name in singleton.contact_names:
                IP_addres = socket.gethostbyname(h_name)
                if (IP_addres != h_name):
                    show_translation=True
                contact_names_ips.append(IP_addres)
            singleton.contact_names = contact_names_ips
            if show_translation:
                log.info('CONTACT_NAMES: %s', str.join(" ", singleton.contact_names))

        except KeyError:
            log.warn('using default contact point localhost')
            singleton.contact_names = ['127.0.0.1']
        singleton.configdir['contact_names'] = str.join(",", singleton.contact_names)

        try:
            singleton.kafka_names = os.environ['KAFKA_NAMES'].split(",")
            log.info('KAFKA_NAMES: %s', str.join(" ", singleton.kafka_names))
        except KeyError:
            log.warn('kakfa names defaults to %s', str.join(" ", singleton.contact_names))
            singleton.kafka_names = singleton.contact_names
        singleton.configdir['kafka_names'] = str.join(",", singleton.kafka_names)

        if hasattr(singleton, 'session'):
            log.warn('Shutting down pre-existent sessions and cluster')
            try:
                singleton.session.shutdown()
                singleton.cluster.shutdown()
            except Exception:
                log.warn('error shutting down')
        try:
            singleton.replication_factor = int(os.environ['REPLICA_FACTOR'])
            log.info('REPLICA_FACTOR: %d', singleton.replication_factor)
        except KeyError:
            singleton.replication_factor = 1
            log.warn('using default REPLICA_FACTOR: %d', singleton.replication_factor)
        singleton.configdir['replica_factor'] = str(singleton.replication_factor)

        try:
            user_defined_execution_name = os.environ['EXECUTION_NAME']
            if user_defined_execution_name == 'hecuba':
                raise RuntimeError('Error: the application keyspace cannot be \'hecuba\'. '
                                   'This keyspace is reserved for storing metadata.')
            singleton.execution_name = user_defined_execution_name
            log.info('EXECUTION_NAME: %s', singleton.execution_name)
        except KeyError:
            singleton.execution_name = 'my_app'
            log.warn('using default EXECUTION_NAME: %s', singleton.execution_name)
        singleton.configdir['execution_name'] = singleton.execution_name

        try:
            singleton.splits_per_node = int(os.environ['SPLITS_PER_NODE'])
            log.info('SPLITS_PER_NODE: %d', singleton.splits_per_node)
        except KeyError:
            singleton.splits_per_node = 32
            log.warn('using default SPLITS_PER_NODE: %d', singleton.splits_per_node)
        singleton.configdir['splits_per_node'] = str(singleton.splits_per_node)

        try:
            singleton.token_range_size = int(os.environ['TOKEN_RANGE_SIZE'])
            log.info('TOKEN_RANGE_SIZE: %d', singleton.token_range_size)
            singleton.target_token_range_size = None
        except KeyError:
            singleton.token_range_size = None

            try:
                singleton.target_token_range_size = int(os.environ['TARGET_TOKEN_RANGE_SIZE'])
                log.info('TARGET_TOKEN_RANGE_SIZE: %d', singleton.target_token_range_size)
            except KeyError:
                singleton.target_token_range_size = 64 * 1024
                log.warn('using default TARGET_TOKEN_RANGE_SIZE: %d', singleton.target_token_range_size)
        singleton.configdir['target_token_range_size'] = str(singleton.target_token_range_size)

        try:
            singleton.max_cache_size = int(os.environ['MAX_CACHE_SIZE'])
            log.info('MAX_CACHE_SIZE: %d', singleton.max_cache_size)
        except KeyError:
            singleton.max_cache_size = 1000
            log.warn('using default MAX_CACHE_SIZE: %d', singleton.max_cache_size)
        singleton.configdir['max_cache_size'] = str(singleton.max_cache_size)

        try:
            singleton.replication_strategy = os.environ['REPLICATION_STRATEGY']
            log.info('REPLICATION_STRATEGY: %s', singleton.replication_strategy)
        except KeyError:
            singleton.replication_strategy = "SimpleStrategy"
            log.warn('using default REPLICATION_STRATEGY: %s', singleton.replication_strategy)
        singleton.configdir['replication_strategy'] = singleton.replication_strategy

        try:
            singleton.replication_strategy_options = os.environ['REPLICATION_STRATEGY_OPTIONS']
            log.info('REPLICATION_STRATEGY_OPTIONS: %s', singleton.replication_strategy_options)
        except KeyError:
            singleton.replication_strategy_options = "''"
            log.warn('using default REPLICATION_STRATEGY_OPTIONS: %s', singleton.replication_strategy_options)
        singleton.configdir['replication_strategy_options'] = singleton.replication_strategy_options

        if singleton.replication_strategy == "SimpleStrategy":
            singleton.replication = "{'class' : 'SimpleStrategy', 'replication_factor': %d}" % \
                                    singleton.replication_factor
        else:
            singleton.replication = "{'class' : '%s', %s}" % (
                singleton.replication_strategy, singleton.replication_strategy_options)
        try:
            singleton.hecuba_print_limit = int(os.environ['HECUBA_PRINT_LIMIT'])
            log.info('HECUBA_PRINT_LIMIT: %s', singleton.hecuba_print_limit)
        except KeyError:
            singleton.hecuba_print_limit = 1000
            log.warn('using default HECUBA_PRINT_LIMIT: %s', singleton.hecuba_print_limit)
        singleton.configdir['hecuba_print_limit'] = str(singleton.hecuba_print_limit)

        try:
            singleton.prefetch_size = int(os.environ['PREFETCH_SIZE'])
            log.info('PREFETCH_SIZE: %s', singleton.prefetch_size)
        except KeyError:
            singleton.prefetch_size = 10000
            log.warn('using default PREFETCH_SIZE: %s', singleton.prefetch_size)
        singleton.configdir['prefetch_size'] = str(singleton.prefetch_size)

        try:
            singleton.write_buffer_size = int(os.environ['WRITE_BUFFER_SIZE'])
            log.info('WRITE_BUFFER_SIZE: %s', singleton.write_buffer_size)
        except KeyError:
            singleton.write_buffer_size = 1000
            log.warn('using default WRITE_BUFFER_SIZE: %s', singleton.write_buffer_size)
        singleton.configdir['write_buffer_size'] = str(singleton.write_buffer_size)

        try:
            singleton.write_callbacks_number = int(os.environ['WRITE_CALLBACKS_NUMBER'])
            log.info('WRITE_CALLBACKS_NUMBER: %s', singleton.write_callbacks_number)
        except KeyError:
            singleton.write_callbacks_number = 16
            log.warn('using default WRITE_CALLBACKS_NUMBER: %s', singleton.write_callbacks_number)
        singleton.configdir['write_callbacks_number'] = str(singleton.write_callbacks_number)

        try:
            env_var = os.environ['TIMESTAMPED_WRITES'].lower()
            singleton.timestamped_writes = False if env_var == 'no' or env_var == 'false' else True
            log.info('TIMESTAMPED WRITES ENABLED? {}'.format(singleton.timestamped_writes))
        except KeyError:
            singleton.timestamped_writes = False
            log.warn('using default TIMESTAMPED_WRITES: %s', singleton.timestamped_writes)
        singleton.configdir['timestamped_writes'] = 'true' if singleton.timestamped_writes else 'false'

        if 'HECUBA_SN_SINGLE_TABLE' in os.environ:
            env_var = os.environ['HECUBA_SN_SINGLE_TABLE'].lower()
            singleton.hecuba_sn_single_table = False if env_var == 'no' or env_var == 'false' else True
            log.info('HECUBA_SN_SINGLE_TABLE: %d', singleton.hecuba_sn_single_table)
        else:
            singleton.hecuba_sn_single_table = True
            log.warn('Create storagenumpys in a single table by default [HECUBA_SN_SINGLE_TABLE=True]')
        singleton.configdir['hecuba_sn_single_table'] = 'true' if singleton.hecuba_sn_single_table else 'false'

        if singleton.max_cache_size < singleton.write_buffer_size:
            import warnings
            message = "Defining a MAX_CACHE_SIZE smaller than WRITE_BUFFER_SIZE can result " \
                      "in reading outdated results from the persistent storage"
            warnings.warn(message)

        log.info('Initializing global session')

        singleton.cluster = Cluster(contact_points=singleton.contact_names,
                                    load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
                                    port=singleton.nodePort,
                                    default_retry_policy=_NRetry(5))
        singleton.session = singleton.cluster.connect()
        singleton.session.encoder.mapping[tuple] = singleton.session.encoder.cql_encode_tuple
        if singleton.concurrent_creation:
            configure_lock=[
                """CREATE KEYSPACE IF NOT EXISTS hecuba_locks
                        WITH replication=  {'class': 'SimpleStrategy', 'replication_factor': 1};
                """,
                """CREATE TABLE IF NOT EXISTS hecuba_locks.table_lock
                        (table_name text, PRIMARY KEY (table_name));
                """,
                "TRUNCATE table hecuba_locks.table_lock;"
            ]
            for query in configure_lock:
                try:
                    self.instance.session.execute(query)
                except Exception as e:
                    log.error("Error executing query %s" % query)
                    raise e
            singleton._query_to_lock=singleton.session.prepare("INSERT into hecuba_locks.table_lock (table_name) values (?) if not exists;")

        if singleton.id_create_schema:
            queries = [
                "CREATE KEYSPACE IF NOT EXISTS hecuba  WITH replication = %s" % singleton.replication,
                """CREATE TYPE IF NOT EXISTS hecuba.q_meta(
                mem_filter text,
                from_point frozen<list<double>>,
                to_point frozen<list<double>>,
                precision float);
                """,
                """CREATE TYPE IF NOT EXISTS hecuba.np_meta (flags int, elem_size int, partition_type tinyint,
                dims list<int>, strides list<int>, typekind text, byteorder text)""",
                """CREATE TABLE IF NOT EXISTS hecuba
                .istorage (storage_id uuid,
                class_name text,name text,
                istorage_props map<text,text>,
                tokens list<frozen<tuple<bigint,bigint>>>,
                indexed_on list<text>,
                qbeast_random text,
                qbeast_meta frozen<q_meta>,
                numpy_meta frozen<np_meta>,
                block_id int,
                base_numpy uuid,
                view_serialization blob,
                primary_keys list<frozen<tuple<text,text>>>,
                columns list<frozen<tuple<text,text>>>,
                PRIMARY KEY(storage_id));
                """,
                "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (singleton.execution_name, singleton.replication)]
            for query in queries:
                try:
                    self.executelocked(query)
                except Exception as e:
                    log.error("Error executing query %s" % query)
                    raise e

        from hecuba.hfetch import connectCassandra, HArrayMetadata
        # connecting c++ bindings
        connectCassandra(singleton.contact_names, singleton.nodePort, singleton.configdir)

        if singleton.id_create_schema:
            time.sleep(10)
        singleton.cluster.register_user_type('hecuba', 'np_meta', HArrayMetadata)
        # Create a dummy arrayDataStore to generate the TokensToHost variable

global config
config = Config()

from .parser import Parser
from .storageobj import StorageObj
from .hdict import StorageDict
from .hnumpy import StorageNumpy
from .storagestream import StorageStream
from .hfilter import hfilter

if not filter == hfilter:
    import builtins

    builtins.python_filter = filter
    builtins.filter = hfilter

def _intercept_numpy_method(method_name):
    """
    Intercept Numpy.'method_name' and use StorageNumpy.'method_name' instead.
    """
    if not isinstance(method_name, str):
        raise TypeError("Intercepted method name MUST be an string")

    # INTERCEPT Numpy METHODS
    import numpy as np

    if np.__dict__[method_name] != StorageNumpy.__dict__[method_name]:
        config.instance.intercepted[method_name] = np.__dict__[method_name]
        np.__dict__[method_name] = StorageNumpy.__dict__[method_name]
    else:
        #print("WARNING: method {} already intercepted".format(method_name), flush=True)
        if not method_name in config.instance.intercepted:
            print("WARNING: method {} already intercepted but we have lost the original method!!!".format(method_name), flush=True)
            # This only happens executing tests... we think that is due to the reload of hecuba, but not numpy (not 100% sure)
            # therefore our solution is to reload numpy.
            import importlib
            importlib.reload(np)
            config.instance.intercepted[method_name] = np.__dict__[method_name]
            np.__dict__[method_name] = StorageNumpy.__dict__[method_name]

_intercept_numpy_method('dot')
_intercept_numpy_method('array_equal')
_intercept_numpy_method('concatenate')

__all__ = ['StorageObj', 'StorageDict', 'StorageNumpy', 'StorageStream', 'Parser']

{'socket': 'socket', 'warnings': 'warnings', 'connectCassandra': 'hecuba.hfetch.connectCassandra', 'HArrayMetadata': 'hecuba.hfetch.HArrayMetadata'}


# -*- coding: utf-8 -*-
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import annotations

from typing import MutableMapping, MutableSequence

import proto  # type: ignore

__protobuf__ = proto.module(
    package="google.cloud.documentai.v1beta2",
    manifest={
        "Barcode",
    },
)

class Barcode(proto.Message):
    r"""Encodes the detailed information of a barcode.

    Attributes:
        format_ (str):
            Format of a barcode. The supported formats are:

            -  ``CODE_128``: Code 128 type.
            -  ``CODE_39``: Code 39 type.
            -  ``CODE_93``: Code 93 type.
            -  ``CODABAR``: Codabar type.
            -  ``DATA_MATRIX``: 2D Data Matrix type.
            -  ``ITF``: ITF type.
            -  ``EAN_13``: EAN-13 type.
            -  ``EAN_8``: EAN-8 type.
            -  ``QR_CODE``: 2D QR code type.
            -  ``UPC_A``: UPC-A type.
            -  ``UPC_E``: UPC-E type.
            -  ``PDF417``: PDF417 type.
            -  ``AZTEC``: 2D Aztec code type.
            -  ``DATABAR``: GS1 DataBar code type.
        value_format (str):
            Value format describes the format of the value that a
            barcode encodes. The supported formats are:

            -  ``CONTACT_INFO``: Contact information.
            -  ``EMAIL``: Email address.
            -  ``ISBN``: ISBN identifier.
            -  ``PHONE``: Phone number.
            -  ``PRODUCT``: Product.
            -  ``SMS``: SMS message.
            -  ``TEXT``: Text string.
            -  ``URL``: URL address.
            -  ``WIFI``: Wifi information.
            -  ``GEO``: Geo-localization.
            -  ``CALENDAR_EVENT``: Calendar event.
            -  ``DRIVER_LICENSE``: Driver's license.
        raw_value (str):
            Raw value encoded in the barcode. For example:
            ``'MEBKM:TITLE:Google;URL:https://www.google.com;;'``.
    """

    format_: str = proto.Field(
        proto.STRING,
        number=1,
    )
    value_format: str = proto.Field(
        proto.STRING,
        number=2,
    )
    raw_value: str = proto.Field(
        proto.STRING,
        number=3,
    )

__all__ = tuple(sorted(__protobuf__.manifest))


import datetime

import factory

from core.models import Patient
from core.tests.factory.hospital import HospitalFactory

MINUTES = 60
HOURS = 60 * MINUTES
DAYS = 24 * HOURS

class PatientFactory(factory.django.DjangoModelFactory):
    class Meta:
        model = Patient

    code = factory.Sequence(lambda n: 'A001%s' % str(n).zfill(8))
    hospital = factory.LazyFunction(HospitalFactory)
    kakao_user_id = factory.Faker('sha256')
    nickname = factory.Faker('first_name')
    user = None
    treatment_started_date = datetime.datetime.today().astimezone() - datetime.timedelta(days=10)

class MedicationPatientFactory(PatientFactory):
    medication_manage_flag = True
    daily_medication_count = 3
    medication_noti_flag = True
    medication_noti_time_1 = datetime.time(hour=8)
    medication_noti_time_2 = datetime.time(hour=10)
    medication_noti_time_3 = datetime.time(hour=12)

class VisitPatientFactory(PatientFactory):
    visit_manage_flag = True
    visit_notification_flag = True
    next_visiting_date_time = datetime.datetime.today().astimezone() + datetime.timedelta(days=10)
    visit_notification_before = 1 * DAYS

class MeasurementPatientFactory(PatientFactory):
    measurement_manage_flag = True
    daily_measurement_count = 3
    measurement_noti_flag = True
    measurement_noti_time_1 = datetime.time(hour=8)
    measurement_noti_time_2 = datetime.time(hour=10)
    measurement_noti_time_3 = datetime.time(hour=12)

class VisitTodayPatientFactory(VisitPatientFactory):
    next_visiting_date_time = datetime.datetime.today().astimezone()
    visit_notification_before = 3 * HOURS

class CompletePatientFactory(MedicationPatientFactory, VisitPatientFactory, MeasurementPatientFactory):
    phone_number = factory.Sequence(lambda n: '010%s' % str(n).zfill(8))
    name = factory.LazyAttribute(lambda obj: '%s' % obj.nickname[:5])


import numpy as np
import cv2

cap0 = cv2.VideoCapture(1)
cap1 = cv2.VideoCapture(2)
cap2 = cv2.VideoCapture(3)

count = 0

while(True):
    # Capture frame-by-frame
    ret0, frame0 = cap0.read()
    ret1, frame1 = cap1.read()
    ret2, frame2 = cap2.read()

    bgr0 = frame0
    bgr1 = frame1
    bgr2 = frame2

    # Our operations on the frame come here
    gray0 = cv2.cvtColor(frame0, cv2.COLOR_BGR2GRAY)
    gray1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
    gray2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)

    # Display the resulting frame

    cv2.imshow('frame0',bgr0)
    cv2.imshow('frame1',bgr1)
    cv2.imshow('frame2',bgr2)

    # print('start')
    # if cv2.waitKey(1) & 0xFF == ord('q'):
    #     break
    k = cv2.waitKey(1)
    if k == 27:         # wait for ESC key to exit
        cv2.destroyAllWindows()
    elif k == ord('s'):
        # print('start save')
        cv2.imwrite("./test1_captured_images/gray/c_%d_v_1.jpg" % count,gray0)
        cv2.imwrite("./test1_captured_images/gray/c_%d_v_2.jpg" % count,gray1)
        cv2.imwrite("./test1_captured_images/gray/c_%d_v_3.jpg" % count,gray2)

        cv2.imwrite("./test1_captured_images/bgr/c_%d_v_1.jpg" % count,bgr0)
        cv2.imwrite("./test1_captured_images/bgr/c_%d_v_2.jpg" % count,bgr1)
        cv2.imwrite("./test1_captured_images/bgr/c_%d_v_3.jpg" % count,bgr2)
        count  = count + 1

# When everything done, release the capture
cap0.release()
cap1.release()
cap2.release()

cv2.destroyAllWindows()