text:
"""
Copyright 2015, Rob Shakir (rjs@jive.com, rjs@rob.sh)

This project has been supported by:
          * Jive Communications, Inc.
          * BT plc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
from decimal import Decimal
from bitarray import bitarray
import uuid
import re
import collections
import copy

NUMPY_INTEGER_TYPES = [np.uint8, np.uint16, np.uint32, np.uint64,
                    np.int8, np.int16, np.int32, np.int64]

# Words that could turn up in YANG definition files that are actually
# reserved names in Python, such as being builtin types. This list is
# not complete, but will probably continue to grow.
reserved_name = ["list", "str", "int", "global", "decimal", "float",
                  "as", "if", "else", "elsif", "map", "set", "class",
                  "from", "import", "pass", "return", "is", "exec",
                  "pop", "insert", "remove", "add", "delete", "local",
                  "get", "default", "yang_name"]

def is_yang_list(arg):
  if isinstance(arg, list):
    return True
  elif hasattr(arg, "_pybind_generated_by"):
    pygen = getattr(arg, "_pybind_generated_by")
    if pygen in ["TypedListType", "YANGListType"]:
      return True
  return False

def remove_path_attributes(p):
  new_path = []
  for i in p:
    if "[" in i:
      new_path.append(i.split("[")[0])
    else:
      new_path.append(i)
  return new_path

def safe_name(arg):
  """
    Make a leaf or container name safe for use in Python.
  """
  k = arg
  arg = arg.replace("-", "_")
  arg = arg.replace(".", "_")
  if arg in reserved_name:
    arg += "_"
  # store the unsafe->original version mapping
  # so that we can retrieve it when get() is called.
  return arg

def RestrictedPrecisionDecimalType(*args, **kwargs):
  """
    Function to return a new type that is based on decimal.Decimal with
    an arbitrary restricted precision.
  """
  precision = kwargs.pop("precision", False)
  class RestrictedPrecisionDecimal(Decimal):
    """
      Class extending decimal.Decimal to restrict the precision that is
      stored, supporting the fraction-digits argument of the YANG decimal64
      type.
    """
    _precision = 10.0**(-1.0*int(precision))
    def __new__(self, *args, **kwargs):
      """
        Overloads the decimal __new__ function in order to round the input
        value to the new value.
      """
      if not self._precision is None:
        if len(args):
          value = Decimal(args[0]).quantize(Decimal(str(self._precision)))
        else:
          value = Decimal(0)
      elif len(args):
        value = Decimal(args[0])
      else:
        value = Decimal(0)
      obj = Decimal.__new__(self, value, **kwargs)
      return obj
  return type(RestrictedPrecisionDecimal(*args, **kwargs))

def RestrictedClassType(*args, **kwargs):
  """
    Function to return a new type that restricts an arbitrary base_type with
    a specified restriction. The restriction_type specified determines the
    type of restriction placed on the class, and the restriction_arg gives
    any input data that this function needs.
  """
  base_type = kwargs.pop("base_type", unicode)
  restriction_type = kwargs.pop("restriction_type", None)
  restriction_arg = kwargs.pop("restriction_arg", None)
  restriction_dict = kwargs.pop("restriction_dict", None)

  # this gives deserialisers some hints as to how to encode/decode this value
  # it must be a list since a restricted class can encapsulate a restricted
  # class
  current_restricted_class_type = re.sub("<(type|class) '(?P<class>.*)'>", "\g<class>", str(base_type))
  if hasattr(base_type, "_restricted_class_base"):
    restricted_class_hint = getattr(base_type, "_restricted_class_base")
    restricted_class_hint.append(current_restricted_class_type)
  else:
    restricted_class_hint = [current_restricted_class_type]

  class RestrictedClass(base_type):
    """
      A class that restricts the base_type class with a new function that the
      input value is validated against before being applied. The function is
      a static method which is assigned to _restricted_test.
    """
    __slots__ = ('_restricted_class_base')
    _pybind_generated_by = "RestrictedClassType"

    _restricted_class_base = restricted_class_hint

    def __init__(self, *args, **kwargs):
      """
        Overloads the base_class __init__ method to check the input argument
        against the validation function - returns on instance of the base_type
        class, which can be manipulated as per a usual Python object.
      """
      try:
        self.__check(args[0])
      except IndexError:
        pass
      super(RestrictedClass, self).__init__(*args, **kwargs)

    def __new__(self, *args, **kwargs):
      self._restriction_dict = restriction_dict
      self._restriction_tests = []

      """
        Create a new class instance, and dynamically define the
        _restriction_test method so that it can be called by other functions.
      """
      def convert_regexp(pattern):
        if not pattern[0] == "^":
          pattern = "^%s" % pattern
        if not pattern[len(pattern)-1] == "$":
          pattern = "%s$" % pattern
        return pattern

      def in_range_check(low, high):
        if low is None and high is None:
          raise AttributeError("invalid means of specifying a range check")
        def int_range_check(value):
          if low is not None and value < low:
            return False
          if high is not None and value > high:
            return False
          return True
        return int_range_check

      def match_pattern_check(regexp):
        #return lambda i: True if re.compile(convert_regexp(regexp)).match(i) else False
        def mp_check(value):
          if not isinstance(value, basestring):
            return False
          if re.match(convert_regexp(regexp), value):
            return True
          return False
        return mp_check

      def in_dictionary_check(dictionary):
        return lambda i: unicode(i) in dictionary

      def length_check(maxlength, minlength=None):
        if maxlength is None and minlength is None:
          raise AttributeError("invalid means of specifying length check")
        def lgt_check(value):
          if maxlength is not None and len(value) > int(maxlength):
            return False
          if minlength is not None and len(value) < int(minlength):
            return False
          return True
        return lgt_check

      range_regex = re.compile("(?P<low>[0-9]+)([ ]+)?\.\.([ ]+)?(?P<high>([0-9]+|max))")

      val = False
      try:
        val = args[0]
      except IndexError:
        pass
      if self._restriction_dict is None:
        if restriction_type is not None and restriction_arg is not None:
          self._restriction_dict = {restriction_type: restriction_arg}
        else:
          raise ValueError, "must specify either a restriction dictionary or a type and argument"

      for rtype,rarg in self._restriction_dict.iteritems():
        if rtype == "pattern":
          tests = []
          self._restriction_tests.append(match_pattern_check(rarg))
        elif rtype == "range":
          low,high = range_regex.sub("\g<low>,\g<high>", rarg).split(",")
          high = base_type(high) if not high == "max" else None
          low = base_type(low) if not low == "min" else None
          self._restriction_tests.append(in_range_check(low, high))
          try:
            val = int(val)
          except:
            raise TypeError, "must specify a numeric type for a range argument"
        elif rtype == "length":
          if range_regex.match(rarg):
            minlength,maxlength = range_regex.sub('\g<low>,\g<high>', rarg).split(",")
            minlength = minlength if not minlength == "min" else None
            maxlength = maxlength if not maxlength == "max" else None
            self._restriction_tests.append(length_check(maxlength, minlength=minlength))
          else:
            self._restriction_tests.append(length_check(rarg))
        elif rtype == "dict_key":
          new_rarg = copy.deepcopy(rarg)
          # populate enum values
          used_values = []
          for k in new_rarg:
            if "value" in new_rarg[k]:
              used_values.append(int(new_rarg[k]["value"]))
          c = 0
          for k in new_rarg:
            while c in used_values:
              c += 1
            if not "value" in new_rarg[k]:
              new_rarg[k]["value"] = c
            c += 1
          self._restriction_tests.append(in_dictionary_check(new_rarg))
          self._enumeration_dict = new_rarg
        else:
          raise TypeError, "unsupported restriction type"

      if not val == False:
        for test in self._restriction_tests:
          passed = False
          if test(val):
            passed = True
            break
        if not passed:
          raise ValueError, "%s does not match a restricted type" % val

      obj = base_type.__new__(self, *args, **kwargs)
      return obj

    def __check(self, v):
      """
        Run the _restriction_test static method against the argument v,
        returning an error if the value does not validate.
      """
      v = base_type(v)
      for chkfn in self._restriction_tests:
        if not chkfn(v):
          raise ValueError, "did not match restricted type"
      return True

    def getValue(self, *args, **kwargs):
      """
        For types where there is a dict_key restriction (such as YANG
        enumeration), return the value of the dictionary key.
      """
      if "dict_key" in self._restriction_dict:
        value = kwargs.pop("mapped", False)
        if value:
          return self._enumeration_dict[self.__str__()]["value"]
      return self

  return type(RestrictedClass(*args, **kwargs))

def TypedListType(*args, **kwargs):
  """
    Return a type that consists of a list object where only
    certain types (specified by allowed_type kwarg to the function)
    can be added to the list.
  """
  allowed_type = kwargs.pop("allowed_type", unicode)
  if not isinstance(allowed_type, list):
    allowed_type = [allowed_type,]
  class TypedList(collections.MutableSequence):
    _pybind_generated_by = "TypedListType"
    _list = list()

    def __init__(self, *args, **kwargs):
      self._allowed_type = allowed_type
      self._list = list()
      if len(args):
        if isinstance(args[0], list):
          for i in args[0]:
            self.check(i)
          self._list.extend(args[0])
        else:
          self.check(args[0])
          self._list.append(args[0])

    def check(self,v):
      passed = False
      count = 0
      for i in self._allowed_type:
        if isinstance(v, i):
          passed = True
          break
        try:
          if hasattr(i, "_pybind_generated_by"):
            if getattr(i, "_pybind_generated_by") == "RestrictedClassType":
              tmp = i(v)
              passed = True
              break
            elif getattr(i, "_pybind_generated_by") == "ReferencePathType":
              tmp = i(v)
              passed = True
              break
          elif i == unicode and isinstance(v, str):
            tmp = unicode(v)
            passed = True
            break
          elif i in NUMPY_INTEGER_TYPES:
            # numpy has odd characteristics where
            # it supports lists, so we check against
            # int as well.
            tmp = int(v)
            tmp = i(v)
            passed = True
            break
        except (ValueError,TypeError):
          # ValueError is generated by restricted classes in general
          # TypeError is generated by int when it receives a [] which
          # is valid for numpy, but not for standard int.
          pass
        count += 1
      if not passed:
        raise ValueError("Cannot add %s to TypedList (accepts only %s)" % \
          (v, self._allowed_type))

    def __len__(self): return len(self._list)
    def __getitem__(self, i): return self._list[i]
    def __delitem__(self, i): del self._list[i]
    def __setitem__(self, i, v):
      self.check(v)
      self._list.insert(i,v)

    def insert(self, i, v):
      self.check(v)
      self._list.insert(i,v)

    def append(self, v):
      self.check(v)
      self._list.append(v)

    def __str__(self):
      return str(self._list)

    def __iter__(self):
      return iter(self._list)

    def __eq__(self, other):
      if self._list == other:
        return True
      return False

    def get(self, filter=False):
      return self._list
  return type(TypedList(*args,**kwargs))

def YANGListType(*args,**kwargs):
  """
    Return a type representing a YANG list, with a contained class.
    A dict or ordered dict is used to store the list, and the
    returned object behaves akin to a dictionary.

    Additional checks are performed to ensure that the keys of the
    list are valid before adding the value.

    .add(key) - initialises a new member of the list
    .delete(key) - removes it.

    Where a list exists that does not have a key - which can be the
    case for 'config false' lists - a uuid is generated and used
    as the key for the list.
  """
  try:
    keyname = args[0]
    listclass = args[1]
  except:
    raise TypeError, "A YANGList must be specified with a key value and a contained class"
  is_container = kwargs.pop("is_container", False)
  parent = kwargs.pop("parent", False)
  yang_name = kwargs.pop("yang_name", False)
  user_ordered = kwargs.pop("user_ordered", False)
  path_helper = kwargs.pop("path_helper", None)

  class YANGList(object):
    __slots__ = ('_pybind_generated_by', '_members', '_keyval', '_contained_class', '_path_helper')
    _pybind_generated_by = "YANGListType"

    def __init__(self, *args, **kwargs):
      if user_ordered:
        self._members = collections.OrderedDict()
      else:
        self._members = dict()
      self._keyval = keyname
      if not type(listclass) == type(int):
        raise ValueError, "contained class of a YANGList must be a class"
      self._contained_class = listclass
      self._path_helper = path_helper

    def __str__(self):
      return str(self._members)

    def __repr__(self):
      return repr(self._members)

    def __check__(self, v):
      if self._contained_class is None:
        return False
      try:
        tmp = YANGDynClass(base=self._contained_class, parent=parent, yang_name=yang_name,
                         is_container=is_container, path_helper=False)
        if not tmp.__slots__ == v.__slots__:
          return False
      except:
        return False
      return True

    def iteritems(self):
      return self._members.iteritems()

    def __iter__(self):
      return iter(self._members)

    def __contains__(self,k):
      if k in self._members:
        return True
      return False

    def __getitem__(self, k):
      return self._members[k]

    def __setitem__(self, k, v):
      self.__set(k,v)

    def __set(self, k=False, v=False):
      update=False
      if v and not self.__check__(v):
        raise ValueError, "value must be set to an instance of %s" % \
          (self._contained_class)
      if k in self._members:
        update=True
      if self._keyval:
        try:
          tmp = YANGDynClass(base=self._contained_class, parent=parent, yang_name=yang_name,
                              is_container='container', path_helper=False)
          if " " in self._keyval:
            keys = self._keyval.split(" ")
            keyparts = k.split(" ")
            if not len(keyparts) == len(keys):
              raise KeyError, "YANGList key must contain all key elements (%s)" % (self._keyval.split(" "))
            path_keystring = "["
            for kv,kp in zip(keys,keyparts):
              kv_obj = getattr(tmp, kv)
              path_keystring += "%s='%s' " % (kv_obj.yang_name(),kp)
            path_keystring = path_keystring.rstrip(" ")
            path_keystring += "]"
          else:
            if k == "":
              raise KeyError("Cannot set a null key for a list entry!")
            keys = [self._keyval,]
            keyparts = [k,]
            kv_obj = getattr(tmp, self._keyval)
            path_keystring = "[%s=%s]" % (kv_obj.yang_name(), k)

          if not update:
            tmp = YANGDynClass(base=self._contained_class, parent=parent, yang_name=yang_name, \
                  is_container='container', path_helper=path_helper, \
                  register_path=self._parent._path() + [self._yang_name+path_keystring], \
                  extmethods=self._parent._extmethods)
          else:
            # hand the value to the init, rather than simply creating an empty object.
            tmp = YANGDynClass(v, base=self._contained_class, parent=parent, yang_name=yang_name, \
                  is_container='container', path_helper=path_helper, \
                  register_path=self._parent._path() + [self._yang_name+path_keystring],
                  extmethods=self._parent._extmethods)

          for i in range(0,len(keys)):
            key = getattr(tmp, "_set_%s" % keys[i])
            key(keyparts[i])
          self._members[k] = tmp
        except ValueError, m:
          raise KeyError, "key value must be valid, %s" % m
      else:
        # this is a list that does not have a key specified, and hence
        # we generate a uuid that is used as the key, the method then
        # returns the uuid for the upstream process to use
        k = str(uuid.uuid1())
        self._members[k] = YANGDynClass(base=self._contained_class, parent=parent, yang_name=yang_name, \
                            is_container=is_container, path_helper=path_helper, extmethods=self._parent._extmethods)
        return k

    def __delitem__(self, k):
      del self._members[k]

    def __len__(self): return len(self._members)

    def keys(self): return self._members.keys()

    def add(self, k=None):
      if k in self._members:
        raise KeyError, "%s is already defined as a list entry" % k
      if self._keyval:
        if k is None:
          raise KeyError, "a list with a key value must have a key specified"
        self.__set(k)
      else:
        k = self.__set()
        return k

    def delete(self, k):
      if self._path_helper:
        current_item = self._members[k]
        if " " in self._keyval:
          keyparts = self._keyval.split(" ")
          keyargs = k.split(" ")
          key_string = "["
          for key,val in zip(keyparts,keyargs):
            kv_o = getattr(self._members[k], key)
            key_string += "%s=%s " % (kv_o.yang_name(),val)
          key_string = key_string.rstrip(" ")
          key_string += "]"
        else:
          kv_o = getattr(self._members[k], self._keyval)
          key_string = "[@%s=%s]" % (kv_o.yang_name(), k)

        obj_path = self._parent._path() + [self._yang_name + key_string]

      try:
        del self._members[k]
        if self._path_helper:
          self._path_helper.unregister(obj_path)
      except KeyError, m:
        raise KeyError, "key %s was not in list (%s)" % (k,m)

    def get(self, filter=False):
      if user_ordered:
        d = collections.OrderedDict()
      else:
        d = {}
      for i in self._members:
        if hasattr(self._members[i], "get"):
          d[i] = self._members[i].get(filter=filter)
        else:
          d[i] = self._members[i]
      return d

  return type(YANGList(*args,**kwargs))

class YANGBool(int):
  """
    A custom boolean class for using in YANG. Since bool has specific
    logic in python, it is not possible to extend the existing bool
    objects.

    This bool also accepts input matching strings to handle the
    forms that might be used in YANG modules.
  """
  def __new__(self, *args, **kwargs):
    false_args = ["false", "False", False, 0, "0"]
    true_args = ["true", "True", True, 1, "1"]
    if len(args):
      if not args[0] in false_args + true_args:
        raise ValueError, "%s is an invalid value for a YANGBool" % args[0]
      value = 0 if args[0] in false_args else 1
    else:
      value = 0
    return int.__new__(self, bool(value))

  def __repr__(self):
    return str([False, True][self])

  def __str__(self):
    return str(self.__repr__())

def YANGDynClass(*args,**kwargs):
  """
    Wrap an type - specified in the base_type arugment - with
    a set of custom attributes that YANG specifies (or are required
    for serialisation of that object). Particularly:

      - base_type:  the original type - for example, string, int.
      - default:    the YANG specified default value of the type.
      - yang_name:  the YANG name of the type (as opposed to a 'safe'
                    Python version).
      - parent:     the class which this type is a member of in the
                    YANG-specified tree.
      - choice:     The choice branch that this type is a member of.
      - is_{container,leaf}: whether this element is a container or
                             a leaf.
      - path_helper: pyangbind helper class to allow XPATH lookups.
      - supplied_register_path: an override for the path that this
                                object should register to. This is
                                used when an element is a member of
                                a list to add the key attributes to
                                the path.
      - extensions:  The list of extensions that should be stored
                     with the type.
  """
  base_type = kwargs.pop("base", False)
  default = kwargs.pop("default", False)
  yang_name = kwargs.pop("yang_name", False)
  parent_instance = kwargs.pop("parent", False)
  choice_member = kwargs.pop("choice", False)
  is_container = kwargs.pop("is_container", False)
  is_leaf = kwargs.pop("is_leaf", False)
  path_helper = kwargs.pop("path_helper", None)
  supplied_register_path = kwargs.pop("register_path", None)
  extensions = kwargs.pop("extensions", None)
  extmethods = kwargs.pop("extmethods", None)
  is_keyval = kwargs.pop("is_keyval", False)
  if not base_type:
    raise TypeError, "must have a base type"
  if base_type in NUMPY_INTEGER_TYPES and len(args):
    if isinstance(args[0], list):
      raise TypeError, "do not support creating numpy ndarrays!"
  if isinstance(base_type, list):
    # this is a union, we must infer type
    if not len(args):
      # there is no argument to infer the type from
      # so use the first type (default)
      base_type = base_type[0]
    else:
      type_test = False
      for candidate_type in base_type:
        try:
          type_test = candidate_type(args[0]) # does the slipper fit?
          break
        except:
          pass # don't worry, move on, plenty more fish (types) in the sea...
      if not type_test:
        # we're left alone at midnight -- no types fit the arguments
        raise TypeError, "did not find a valid type using the argument as a" + \
                            "hint"
      # otherwise, hop, skip and jump with the last candidate
      base_type = candidate_type

  class YANGBaseClass(base_type):
    if is_container:
      __slots__ = ('_default', '_mchanged', '_yang_name', '_choice', '_parent',
                   '_supplied_register_path', '_path_helper', '_base_type', '_is_leaf',
                   '_is_container', '_extensionsd', '_pybind_base_class', '_extmethods',
                   '_is_keyval')

    _pybind_base_class = re.sub("<(type|class) '(?P<class>.*)'>", "\g<class>", str(base_type))

    def __new__(self, *args, **kwargs):
      obj = base_type.__new__(self, *args, **kwargs)
      return obj

    def __init__(self, *args, **kwargs):
      self._default = False
      self._mchanged = False
      self._yang_name = yang_name
      self._parent = parent_instance
      self._choice = choice_member
      self._path_helper = path_helper
      self._supplied_register_path = supplied_register_path
      self._base_type = base_type
      self._is_leaf = is_leaf
      self._is_container = is_container
      self._extensionsd = extensions
      self._extmethods = extmethods
      self._is_keyval = is_keyval

      if default:
        self._default = default
      if len(args):
        if not args[0] == self._default:
          self._set()

      # lists themselves do not register, only elements within them
      # are actually created in the tree.
      if not self._is_container == "list":
        if self._path_helper:
          if self._supplied_register_path is None:
            self._path_helper.register(self._register_path(), self)
          else:
            self._path_helper.register(self._supplied_register_path, self)

      if self._is_container == 'list' or self._is_container == 'container':
        kwargs['path_helper'] = self._path_helper

      try:
        super(YANGBaseClass, self).__init__(*args, **kwargs)
      except Exception as e:
        raise TypeError, "couldn't generate dynamic type -> %s -> %s" % (type(e),e)

    def _changed(self):
      return self._mchanged

    def _extensions(self):
      return self._extensionsd

    def _path(self):
      return self._register_path()

    def _yang_path(self):
      return "/"+"/".join(self._register_path())

    def __str__(self):
      return super(YANGBaseClass, self).__str__()

    def __repr__(self):
      return super(YANGBaseClass, self).__repr__()

    def _set(self,choice=False):
      if hasattr(self, '__choices__') and choice:
        for ch in self.__choices__:
          if ch == choice[0]:
            for case in self.__choices__[ch]:
              if not case == choice[1]:
                for elem in self.__choices__[ch][case]:
                  method = "_unset_%s" % elem
                  if not hasattr(self, method):
                    raise AttributeError, "unmapped choice!"
                  x = getattr(self, method)
                  x()
      if self._choice and not choice:
        choice=self._choice
      self._mchanged = True
      if self._parent and hasattr(self._parent, "_set"):
        self._parent._set(choice=choice)

    def yang_name(self):
      return self._yang_name

    def default(self):
      return self._default

    # we need to overload the set methods
    def __setitem__(self, *args, **kwargs):
      self._set()
      super(YANGBaseClass, self).__setitem__(*args, **kwargs)

    def append(self, *args, **kwargs):
      if not hasattr(super(YANGBaseClass,self), "append"):
        raise AttributeError("%s object has no attribute append" % base_type)
      self._set()
      super(YANGBaseClass, self).append(*args,**kwargs)

    def pop(self, *args, **kwargs):
      if not hasattr(super(YANGBaseClass, self), "pop"):
        raise AttributeError("%s object has no attribute pop" % base_type)
      self._set()
      item = super(YANGBaseClass, self).pop(*args, **kwargs)
      return item

    def remove(self, *args, **kwargs):
      if not hasattr(super(YANGBaseClass, self), "remove"):
        raise AttributeError("%s object has no attribute remove" % base_type)
      self._set()
      if self._path_helper:
        elem_index = super(YANGBaseClass, self).index(*args, **kwargs)
        item = super(YANGBaseClass, self).__getitem__(elem_index)
      super(YANGBaseClass, self).remove(*args, **kwargs)

    def extend(self, *args, **kwargs):
      if not hasattr(super(YANGBaseClass, self), "extend"):
        raise AttributeError("%s object has no attribute extend" % base_type)
      self._set()
      super(YANGBaseClass, self).extend(*args, **kwargs)

    def insert(self, *args, **kwargs):
      if not hasattr(super(YANGBaseClass,self), "insert"):
        raise AttributeError("%s object has no attribute insert" % base_type)
      self._set()
      super(YANGBaseClass, self).insert(*args, **kwargs)

    def _register_path(self):
      if not self._supplied_register_path is None:
        return self._supplied_register_path
      if not self._parent is None:
        return self._parent._path() + [self._yang_name]
      else:
        return []

    def __has_extmethod(self, method):
      if not self._extmethods:
        return False
      chk_path = "/" + "/".join(remove_path_attributes(self._register_path()))
      if chk_path in self._extmethods:
        if hasattr(self._extmethods[chk_path], method):
          return getattr(self._extmethods[chk_path], method, False)
        return False
      return False

    def __return_extmethod(self, method, *args, **kwargs):
      cm = self.__has_extmethod(method)
      if cm:
        kwargs['caller'] = self._register_path()
        kwargs['path_helper'] = self._path_helper
        return cm(*args, **kwargs)
      return None

    def _presave(self, *args, **kwargs):
      return self.__return_extmethod("presave", *args, **kwargs)

    def _commit(self, *args, **kwargs):
      return self.__return_extmethod("commit", *args, **kwargs)

    def _postsave(self, *args, **kwargs):
      return self.__return_extmethod("postsave", *args, **kwargs)

    def _validate(self, *args, **kwargs):
      return self.__return_extmethod("validate", *args, **kwargs)

  return YANGBaseClass(*args, **kwargs)

def ReferenceType(*args,**kwargs):
  """
    A type which based on a path provided acts as a leafref.
    The caller argument is used to allow the path that is provided
    to be a relative (rather than absolute) path. The require_instance
    argument specifies whether errors should be thrown in the case
    that the referenced instance does not exist.
  """
  ref_path = kwargs.pop("referenced_path", False)
  path_helper = kwargs.pop("path_helper", None)
  caller = kwargs.pop("caller", False)
  require_instance = kwargs.pop("require_instance", False)
  class ReferencePathType(object):

    __slots__ = ('_referenced_path', '_path_helper', '_caller', '_referenced_object',
                 '_ptr', '_require_instance', '_type')

    _pybind_generated_by = "ReferencePathType"

    def __init__(self, *args, **kwargs):
      self._referenced_path = ref_path
      self._path_helper = path_helper
      self._referenced_object = False
      self._caller = caller
      self._ptr = False
      self._require_instance = require_instance
      self._type = "unicode"

      if len(args):
        value = args[0]
      else:
        value = None

      if self._path_helper:
        path_chk = self._path_helper.get(self._referenced_path, caller=self._caller)

        # if the lookup returns only one leaf, then this means that we have something
        # that could potentially be a pointer. However, this is not sufficient to tell
        # whether it is (it could be a single list entry) - thus perform two additional
        # checks. 1) check whether this is the key value of a list (if it is then this
        # is something that can be externally referenced) and 2) check that this is not
        # a list itself (including a leaf-list)
        if len(path_chk) == 1 and not path_chk[0]._is_keyval and not is_yang_list(path_chk[0]):
          # we are not checking whether this leaf exists, but rather
          # this is a pointer to some other value.
          path_parts = self._referenced_path.split("/")
          leaf_name = path_parts[-1]
          if ":" in leaf_name:
            # normalise the namespace
            leaf_name = leaf_name.split(":")[1]
          set_method = getattr(path_chk[0]._parent, "_set_%s" % safe_name(leaf_name))
          get_method = getattr(path_chk[0]._parent, "_get_%s" % safe_name(leaf_name))
          if value:
            set_method(value)
          self._type = re.sub("<(type|class) '(?P<class>.*)'>", "\g<class>", str(get_method()._base_type))
          self._ptr = True
        elif self._require_instance:
          if not value:
            self._referenced_object = None
          else:
            if isinstance(value, str):
              value = unicode(value)
            lookup_o = []
            path_chk = self._path_helper.get(self._referenced_path, caller=self._caller)
            found = False
            if value in path_chk:
              self._referenced_object = path_chk[path_chk.index(value)]
              found = True
            else:
              for i in path_chk:
                try:
                  self._referenced_object = i[i.index(value)]
                  found = True
                except ValueError:
                  pass
            if not found:
              raise ValueError, "no such key (%s) existed in path (%s -> %s)" % (value, self._referenced_path, path_chk)
        else:
          # require instance is not set, so act like a string
          self._referenced_object = value

    def _get_ptr(self):
      if self._ptr:
        ptr = self._path_helper.get(self._referenced_path, caller=self._caller)
        if len(ptr) == 1:
          return ptr[0]
      raise ValueError, "Invalid pointer specified"

    def __repr__(self):
      if not self._ptr:
        return repr(self._referenced_object)
      return repr(self._get_ptr())

    def _get(self):
      if not self._ptr:
        return self._referenced_object
      return self._get_ptr()

    def __str__(self):
      if not self._ptr:
        return str(self._referenced_object)
      return str(self._get_ptr())

  return type(ReferencePathType(*args,**kwargs))

max_stars_repo_name: learnflexswitch/pyangbind

max_issues_repo_name: ktbyers/pyangbind

max_forks_repo_name: ktbyers/pyangbind


text:
__author__ = "Yuri E. Corilo"
__date__ = "Jun 12, 2019"
from numpy import array, trapz
from corems.chroma_peak.calc.ChromaPeakCalc import GCPeakCalculation
from corems.molecular_id.factory.EI_SQL import LowResCompoundRef

class ChromaPeakBase():
    '''
    classdocs
    '''
    def __init__(self, chromatogram_parent, mass_spectrum_obj, start_index, index, final_index):

        self.start_index = start_index
        self.final_index = final_index
        self.index = int(index)
        self.chromatogram_parent = chromatogram_parent
        self.mass_spectrum = mass_spectrum_obj
        'updated individual calculation'
        self._area = None

        self._compounds = []

    def __len__(self):

        return len(self._compounds)

    def __getitem__(self, position) -> LowResCompoundRef:

        return self._compounds[position]

    def add_compound(self, compounds_dict, similarity):
        #implemented in child class
        pass

    def remove_compound(self, compounds_obj):

        self._compounds.remove(compounds_obj)

    def clear_compounds(self):

        self._compounds = []

    @property
    def rt(self):
        return self.mass_spectrum.rt

    @property
    def tic(self):
        return self.mass_spectrum.tic

    @property
    def area(self):
        return self._area

    @property
    def rt_list(self):
        return [self.chromatogram_parent.retention_time[i] for i in range(self.start_index, self.final_index+1) ]

    @property
    def tic_list(self):
        return [self.chromatogram_parent.tic[i] for i in range(self.start_index, self.final_index+1) ]

class GCPeak(ChromaPeakBase, GCPeakCalculation):

    def __init__(self, chromatogram_parent, mass_spectrum_obj, indexes):

        super().__init__(chromatogram_parent, mass_spectrum_obj, *indexes)

        self._ri = None

    def add_compound(self, compounds_dict, spectral_similarity_scores, ri_score=None, similarity_score=None):

        compound_obj = LowResCompoundRef(compounds_dict)

        # add all spectral similarities methods as a dict
        compound_obj.spectral_similarity_scores = spectral_similarity_scores
        # TODO need to add spectral similarity score label in the options in the parameters encapsulation class
        compound_obj.spectral_similarity_score = spectral_similarity_scores.get("cosine_correlation")

        compound_obj.ri_score = ri_score

        compound_obj.similarity_score = similarity_score

        self._compounds.append(compound_obj)

        if similarity_score:
            self._compounds.sort(key=lambda c: c.similarity_score, reverse=True)
        else:
            self._compounds.sort(key=lambda c: c.spectral_similarity_score, reverse=True)

    @property
    def ri(self): return self._ri

    @property
    def highest_ss_compound(self):
        if self:
            return max(self, key = lambda c: c.spectral_similarity_score)
        else:
            None

    @property
    def highest_score_compound(self):
        if self:
            return max(self, key = lambda c: c.similarity_score)
        else:
            return None

    @property
    def compound_names(self):
        if self:
            return [c.name for c in self]
        else:
            return []

class GCPeakDeconvolved(GCPeak):

    def __init__(self, chromatogram_parent, mass_spectra, apex_index, rt_list, tic_list ):

        self._ri = None

        self._rt_list = list(rt_list)

        self._tic_list = list(tic_list)

        self.mass_spectra = list(mass_spectra)

        super().__init__(chromatogram_parent, self.mass_spectra[apex_index], (0, apex_index, len(self.mass_spectra)-1))

    @property
    def rt_list(self):
        return self._rt_list

    @property
    def tic_list(self):
        return self._tic_list

ext: py

max_stars_repo_name: Kzra/CoreMS

max_issues_repo_name: Kzra/CoreMS

max_forks_repo_name: Kzra/CoreMS

reason: from numpy


text:
import scedar.knn as knn
import scedar.eda as eda
import numpy as np
import scipy.sparse as spsp
import pytest

class TestRareSampleDetectionSparse(object):
    """docstring for TestRareSampleDetection"""

    def test_detect_rare_samples(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7])
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_detect_rare_samples_par(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7], 3)
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_detect_rare_samples_single_run(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [101, 101],
                 [99, 99], [100, 100], [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        res = rsd._no_pdist_rare_s_detect(1, 0.1, 1)
        resl = rsd.detect_rare_samples(1, 0.1, 1)
        assert resl[0] == []
        resl2 = rsd.detect_rare_samples(1, 0.1, 5)
        assert resl2[0] == []

    def test_detect_rare_samples_empty_subset(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)

        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples(1, 10, 5)
        resl2 = rsd.detect_rare_samples([1], [10], 5)
        # scalar and list params should have the same results
        assert resl == resl2
        # result lut should be the same length
        assert len(rsd._res_lut) == 1
        assert rsd._res_lut[(1, 10, 5)][1][-1] == resl[0]

    def test_detect_rare_samples_wrong_args(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        # Invalid parameters
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0, 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 0, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0.5)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0.5, 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, -0.1, 1)
        # Parameters of different length
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples([1, 2], 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, [1, 2], 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, [1, 2])

    def test_hnsw_detect_rare_samples(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7],
                                       use_pca=True, use_hnsw=True)
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_hnsw_detect_rare_samples_par(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7], 3,
                                       use_pca=True, use_hnsw=True)
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_hnsw_detect_rare_samples_single_run(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [101, 101],
                 [99, 99], [100, 100], [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        res = rsd._no_pdist_rare_s_detect(1, 0.1, 1)
        resl = rsd.detect_rare_samples(1, 0.1, 1, use_pca=True, use_hnsw=True)
        assert resl[0] == []
        resl2 = rsd.detect_rare_samples(1, 0.1, 5, use_pca=True, use_hnsw=True)
        assert resl2[0] == []

    def test_hnsw_detect_rare_samples_empty_subset(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)

        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples(1, 10, 5, use_pca=True, use_hnsw=True)
        resl2 = rsd.detect_rare_samples([1], [10], 5, use_pca=True,
                                        use_hnsw=True)
        # scalar and list params should have the same results
        assert resl == resl2
        # result lut should be the same length
        assert len(rsd._res_lut) == 1
        assert rsd._res_lut[(1, 10, 5)][1][-1] == resl[0]

    def test_hnsw_detect_rare_samples_wrong_args(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=False)
        rsd = knn.RareSampleDetection(tsdm)
        # Invalid parameters
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0, 1, 1, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 0, 1, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0.5, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0.5, 1, 1, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, -0.1, 1, use_pca=True, use_hnsw=True)
        # Parameters of different length
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples([1, 2], 1, 1, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, [1, 2], 1, use_pca=True, use_hnsw=True)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, [1, 2], use_pca=True, use_hnsw=True)

    def test_pdist_detect_rare_samples(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=True)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7])
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_pdist_detect_rare_samples_par(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=True)
        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples([3, 4, 5], [10]*3, [5, 6, 7], 3)
        assert resl == [list(range(5, 10)), list(range(5, 10)), []]
        assert len(rsd._res_lut) == 3
        assert rsd._res_lut[(3, 10, 5)][1][-1] == resl[0]
        assert rsd._res_lut[(4, 10, 6)][1][-1] == resl[1]
        assert rsd._res_lut[(5, 10, 7)][1][-1] == resl[2]

    def test_pdist_detect_rare_samples_single_run(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [101, 101],
                 [99, 99], [100, 100], [102, 102]]),
            metric="euclidean", use_pdist=True)
        rsd = knn.RareSampleDetection(tsdm)
        res = rsd._no_pdist_rare_s_detect(1, 0.1, 1)
        resl = rsd.detect_rare_samples(1, 0.1, 1)
        assert resl[0] == []
        resl2 = rsd.detect_rare_samples(1, 0.1, 5)
        assert resl2[0] == []

    def test_pdist_detect_rare_samples_empty_subset(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=True)

        rsd = knn.RareSampleDetection(tsdm)
        resl = rsd.detect_rare_samples(1, 10, 5)
        resl2 = rsd.detect_rare_samples([1], [10], 5)
        # scalar and list params should have the same results
        assert resl == resl2
        # result lut should be the same length
        assert len(rsd._res_lut) == 1
        assert rsd._res_lut[(1, 10, 5)][1][-1] == resl[0]

    def test_pdist_detect_rare_samples_wrong_args(self):
        tsdm = eda.SampleDistanceMatrix(
            spsp.csr_matrix(
                [[0.00001, 0.00001], [1, 1], [200, 200], [200, 200],
                 [200, 200], [100, 100], [101, 101], [99, 99], [100, 100],
                 [102, 102]]),
            metric="euclidean", use_pdist=True)
        rsd = knn.RareSampleDetection(tsdm)
        # Invalid parameters
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0, 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 0, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, 0.5)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(0.5, 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, -0.1, 1)
        # Parameters of different length
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples([1, 2], 1, 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, [1, 2], 1)
        with pytest.raises(ValueError) as excinfo:
            rsd.detect_rare_samples(1, 1, [1, 2])

max_stars_repo_name: logstar/scedar

max_issues_repo_name: benstear/scedar

max_forks_repo_name: benstear/scedar

reason: import numpy,import scipy


text:
from __future__ import print_function
from __main__ import vtk, qt, ctk, slicer, os

import vtkITK
import numpy
import SimpleITK as sitk
import sitkUtils as su
import vtk.util.numpy_support
import collections
import itertools
import math
import string
import operator
import glob
import shutil

import fnmatch
import pdb
import radiomicsDatabase

def Execute(self, inputdir, outputdir, fileIdentifierSettings, applyButton):
  from datetime import datetime
  progress_filename = str(datetime.now().strftime(('%Y-%m-%d_%H-%M-%S')) + '_XYZSnapshot.txt')
  printfile = open(str(os.path.join(outputdir,progress_filename)),mode='a')
  print(fileIdentifierSettings[0], file=printfile)

  PatientDirs, PatientNames = radiomicsDatabase.getFolderList(self,inputdir,excludeDirName=False)
  for x,patientName in enumerate(PatientNames):
    print(x,patientName,PatientDirs[x])
    """
    if os.path.exists(os.path.join(outputdir,patientName)):
      radiomicsDatabase.deleteDataSlicer(self)
      slicer.mrmlScene.Clear(0)
      continue
    """
    #get button widget by reference and change text to x value
    applyButton.text = ("Processing: " + str(x+1) + " of " + str(len(PatientNames)))
    applyButton.repaint()
    #slicer.app.processEvents()

    # Set ID current patient
    IDcurrPatient = radiomicsDatabase.getIDcurrPatient(self,2,PatientDirs[x])

    #if not os.path.exists(str(os.path.join(outputdir, IDcurrPatient))):
    print("\n" + "Loading Data: " + str(self.IDcurrPatient) + " at location: " + str(PatientDirs[x]),file=printfile)

    ExtractionDirs = []
    includeSubfolders = True
    subfoldersIndependent = True
    # Normal or Treat each subfolder as independent scans (option GUI)
    if subfoldersIndependent: ExtractionDirs = radiomicsDatabase.getFolderList(self,PatientDirs[x])[0].values()
    else: ExtractionDirs.append(PatientDirs[x])

    # Load and Extract features for each selected folder
    for index in xrange(radiomicsDatabase.lenghtList(ExtractionDirs)):
      for settings in fileIdentifierSettings:
        print(patientName, ExtractionDirs[index])
        ImageFilePath, LabelFilePath = radiomicsDatabase.getDataFiles(ExtractionDirs[index],includeSubfolders,settings["Mask"],settings["selImage"],settings["selLabel"])
        if ImageFilePath:# and LabelFilePath:
          # Load Image Nodes into Slicer
          imageNode, labelNode = radiomicsDatabase.loadDataSlicer(self,ImageFilePath,LabelFilePath,printfile)
          #use binarizeLabelMapToValue
          createLabelMap(self,imageNode, labelNode, outputdir, IDcurrPatient)
        else: print("could not find image or label path: " + patientName, file=printfile)
      radiomicsDatabase.deleteDataSlicer(self)
      slicer.mrmlScene.Clear(0)
    radiomicsDatabase.deleteDataSlicer(self)
    slicer.mrmlScene.Clear(0)
  printfile.close()

def reversedim(M,k=0):
    idx = tuple((slice(None,None,-1) if ii == k else slice(None)
            for ii in xrange(M.ndim)))
    return M[idx]

def binarizeLabelMapToValue(labelNode, labelValue=1):
  labelNodeImageData = labelNode.GetImageData()
  change = slicer.vtkImageLabelChange()
  change.SetInputData(labelNodeImageData)
  change.SetOutput(labelNodeImageData)
  change.SetOutputLabel(labelValue)

  for i in xrange(1,int(labelNodeImageData.GetScalarRange()[1])+1):
    change.SetInputLabel(i)
    change.Update()

  labelNode.SetAndObserveImageData(labelNodeImageData)
  return labelNode

def createLabelMap(self, dataNode, labelNode, outputDir, IDcurrPatient):
  """
  self.sliceXarray = numpy.rollaxis(self.sliceXarray,1)[:,:,None] #512,36,1
  #self.sliceXarray = self.sliceXarray[:,:,None]

  self.sliceYarray = numpy.rollaxis(self.sliceYarray,1)[:,:,None] #512,36,1
  #self.sliceYarray = self.sliceYarray[:,:,None]

  #self.slizeZarray = numpy.rot90(self.sliceZarray)
  self.sliceZarray = self.sliceZarray[:,:,None] #512,512,1

  self.sliceXnode = slicer.vtkMRMLScalarVolumeNode()
  self.sliceYnode = slicer.vtkMRMLScalarVolumeNode()
  self.sliceZnode = slicer.vtkMRMLScalarVolumeNode()
  self.sliceXnode.SetName('SliceX')
  self.sliceYnode.SetName('SliceY')
  self.sliceZnode.SetName('SliceZ')

  self.sliceXNodeDisplayNode = slicer.vtkMRMLVectorVolumeDisplayNode()
  slicer.mrmlScene.AddNode(self.sliceXNodeDisplayNode)
  self.sliceXnode.SetAndObserveDisplayNodeID(self.sliceXNodeDisplayNode.GetID())

  self.sliceYNodeDisplayNode = slicer.vtkMRMLVectorVolumeDisplayNode()
  slicer.mrmlScene.AddNode(self.sliceYNodeDisplayNode)
  self.sliceYnode.SetAndObserveDisplayNodeID(self.sliceYNodeDisplayNode.GetID())

  self.sliceZNodeDisplayNode = slicer.vtkMRMLVectorVolumeDisplayNode()
  slicer.mrmlScene.AddNode(self.sliceZNodeDisplayNode)
  self.sliceZnode.SetAndObserveDisplayNodeID(self.sliceZNodeDisplayNode.GetID())
  """
  imageFilePath = dataNode.GetStorageNode().GetFileName()
  imageName = os.path.basename(imageFilePath).replace('.nrrd','')

  reconDir = os.path.dirname(imageFilePath)
  studyDir = os.path.dirname(reconDir)
  segDir  = [dirpath for dirpath in glob.glob(os.path.join(studyDir,'*')) if 'Segmentations' in dirpath][0]
  resourcesDir = [dirpath for dirpath in glob.glob(os.path.join(studyDir,'*')) if 'Resources' in dirpath][0]
  spacing = dataNode.GetSpacing()
  origin = dataNode.GetOrigin()
  #pdb.set_trace()

  arrays = []
  for r,d,f in os.walk(studyDir):
    [arrays.append(os.path.join(r,file)) for file in fnmatch.filter(f, '*.npy')]

  try:
    name = os.path.basename(arrays[0]).replace('.npy', '')
  except IndexError:
    print(IDcurrPatient, ': FAILED TO PROCESS NPY FILE')
    return
  arr = numpy.load(arrays[0])
  #arr = numpy.ascontiguousarray(arr)

  #nump_arr = reversedim(arr,0)
  nump_arr = numpy.swapaxes(arr, 0, 2)

  #nump_arr = nump_arr[...,::-1]
  #nump_arr = numpy.rollaxis(numpy.rollaxis(nump_arr,2),2,1)

  self.labelMapnode = slicer.vtkMRMLScalarVolumeNode()
  self.labelMapnode.SetName('labelMap')
  self.labelMapnode.LabelMapOn()

  self.labelMapnodeDisplayNode = slicer.vtkMRMLScalarVolumeDisplayNode()
  slicer.mrmlScene.AddNode(self.labelMapnodeDisplayNode)
  self.labelMapnode.SetAndObserveDisplayNodeID(self.labelMapnodeDisplayNode.GetID())
  self.labelMapnodeDisplayNode.SetAndObserveColorNodeID('vtkMRMLColorTableNodeFileGenericColors.txt')

  try:
    self.labelMapnode = renderArray(self, nump_arr, self.labelMapnode, dataNode, shape=nump_arr.shape, spacing=spacing, origin=origin)
  except ValueError:
    print(IDcurrPatient, ': FAILED TO CONVERT TO LABEL MAP')
    return
  #self.sliceXnode = self.renderSlice(self.sliceXarray, self.sliceXnode, self.dataNode, shape=self.sliceXarray.shape, spacing=(x,z,1))
  #self.sliceYnode = self.renderSlice(self.sliceYarray, self.sliceYnode, self.dataNode, shape=self.sliceYarray.shape, spacing=(y,z,1))
  #self.sliceZnode = self.renderSlice(self.sliceZarray, self.sliceZnode, self.dataNode, shape=self.sliceZarray.shape, spacing=(x,y,1))

  self.labelMapnode = binarizeLabelMapToValue(self.labelMapnode)
  self.labelMapnodeDisplayNode.SetInputImageDataConnection(self.labelMapnode.GetImageDataConnection())
  self.labelMapnodeDisplayNode.UpdateImageDataPipeline()

  volumesLogic = slicer.vtkSlicerVolumesLogic()
  volumesLogic.CenterVolume(dataNode)
  volumesLogic.CenterVolume(self.labelMapnode)

  labelMapNodeFileName = os.path.join(segDir, imageName + '_labelMap.nrrd')
  saveLabelMap = slicer.util.saveNode(self.labelMapnode, labelMapNodeFileName, properties={"filetype": ".nrrd"})
  #saveNode = slicer.util.saveNode(dataNode, imageFilePath, properties={"filetype": ".nrrd"})
  arraysName = os.path.basename(arrays[0])
  shutil.move(arrays[0],os.path.join(resourcesDir,arraysName))

  labelMapNodeSITK = sitk.ReadImage(str(labelMapNodeFileName))
  FlipZAxis = sitk.FlipImageFilter()
  FlipZAxis.SetDebug(False)
  FlipZAxis.SetFlipAxes([False, False, True])
  FlipZAxis.SetFlipAboutOrigin(True)
  FlipZAxis.SetNumberOfThreads(4)
  labelMapNodeSITK = FlipZAxis.Execute(labelMapNodeSITK)
  FlipZAxis.Execute(labelMapNodeSITK)
  sitk.WriteImage(labelMapNodeSITK, str(labelMapNodeFileName) )
  """
  sliceXfilename = os.path.join(outputDir, 'test_sliceX.tiff')
  savex = slicer.util.saveNode(self.sliceXnode, sliceXfilename, properties={"filetype": ".tiff"})

  #sliceYfilename = os.path.join(outputDir, 'test_sliceY.tiff')
  #savey = slicer.util.saveNode(self.sliceYnode, sliceYfilename, properties={"filetype": ".tiff"})

  #sliceZfilename = os.path.join(outputDir, 'test_sliceZ.tiff')
  #savez = slicer.util.saveNode(self.sliceZnode, sliceZfilename, properties={"filetype": ".tiff"})
  """

def renderArray(self, labelNodeArray, labelNode, imageNode, shape=(512,512,88), spacing=(1,1,1), origin=(0,0,0)):
  ras2ijk = vtk.vtkMatrix4x4()
  ijk2ras = vtk.vtkMatrix4x4()
  imageNode.GetRASToIJKMatrix(ras2ijk)
  imageNode.GetIJKToRASMatrix(ijk2ras)
  labelNode.SetRASToIJKMatrix(ras2ijk)
  labelNode.SetIJKToRASMatrix(ijk2ras)

  labelNodeImageData = vtk.vtkImageData()
  labelNodeImageData.DeepCopy(imageNode.GetImageData())
  #labelNodeImageData.AllocateScalars(4,1)
  #labelNodeImageData.SetDimensions(shape)
  labelNodePointData = labelNodeImageData.GetPointData()

  """
  # Numpy array is converted from signed int16 to signed vtkShortArray
  scalarArray = vtk.vtkShortArray()
  dims1D = labelNodeArray.size #labelNodePointData.GetScalars().GetSize()
  flatArray = labelNodeArray.reshape(dims1D, order='C')
  scalarArray = vtk.util.numpy_support.numpy_to_vtk(flatArray)
  """
  scalarArray = vtk.vtkShortArray()
  dims1D = labelNodePointData.GetScalars().GetSize() #labelNodePointData.GetScalars().GetSize()
  flatArray = labelNodeArray.reshape(dims1D, order='F')
  scalarArray = vtk.util.numpy_support.numpy_to_vtk(flatArray) # VTK Unsigned Char Array =3, short =4, uint = 7

  labelNodePointData.SetScalars(scalarArray)
  labelNodeImageData.Modified()

  slicer.mrmlScene.AddNode(labelNode)
  labelNode.SetAndObserveImageData(labelNodeImageData)
  labelNode.SetSpacing(spacing)
  labelNode.SetOrigin(origin)

  return labelNode

def renderSlice(self, sliceNodeArray, sliceNode, imageNode, shape=(512,512,88), spacing=(1,1,1), origin=(0,0,0)):
  # Initializes a vtkMRMLScalarVolumeNode for the SegmentCAD Output and copies ijkToRAS matrix and Image data from nodeLabel
  """
  ras2ijk = vtk.vtkMatrix4x4()
  ijk2ras = vtk.vtkMatrix4x4()
  imageNode.GetRASToIJKMatrix(ras2ijk)
  imageNode.GetIJKToRASMatrix(ijk2ras)
  sliceNode.SetRASToIJKMatrix(ras2ijk)
  sliceNode.SetIJKToRASMatrix(ijk2ras)
  """

  sliceNodeImageData = vtk.vtkImageData()
  sliceNodeImageData.AllocateScalars(4,1)
  sliceNodeImageData.SetDimensions(shape)

  sliceNodePointData = sliceNodeImageData.GetPointData()

  """
  # Numpy array is converted from signed int16 to signed vtkShortArray
  scalarArray = vtk.vtkShortArray()
  dims1D = sliceNodeArray.size #sliceNodePointData.GetScalars().GetSize()
  flatArray = sliceNodeArray.reshape(dims1D, order='C')
  scalarArray = vtk.util.numpy_support.numpy_to_vtk(flatArray)
  """

  dims1D = sliceNodeArray.size #sliceNodePointData.GetScalars().GetSize()
  flatArray = sliceNodeArray.reshape(dims1D, order='C')
  scalarArray = vtk.util.numpy_support.numpy_to_vtk(flatArray, array_type=4) # VTK Unsigned Char Array =3, short =4, uint = 7

  sliceNodePointData.SetScalars(scalarArray)
  sliceNodeImageData.Modified()

  slicer.mrmlScene.AddNode(sliceNode)
  sliceNode.SetAndObserveImageData(sliceNodeImageData)
  sliceNode.SetSpacing(spacing)
  sliceNode.SetOrigin(origin)

  return sliceNode

  """
  # Corresponding display node and color table nodes created for SegmentCAD label Output
  self.SegmentCADLabelMapDisplay = slicer.vtkMRMLLabelMapVolumeDisplayNode()
  self.SegmentCADLabelMapDisplay.SetScene(slicer.mrmlScene)
  self.SegmentCADLabelMapDisplay.SetAndObserveColorNodeID('vtkMRMLColorTableNodeFileGenericColors.txt')

  if vtk.VTK_MAJOR_VERSION <= 5:
    self.SegmentCADLabelMapDisplay.SetInputImageData(self.SegmentCADLabelMap.GetImageData())
  else:
    self.SegmentCADLabelMapDisplay.SetInputImageDataConnection(self.SegmentCADLabelMap.GetImageDataConnection())
  self.SegmentCADLabelMapDisplay.UpdateImageDataPipeline()

  slicer.mrmlScene.AddNode(self.SegmentCADLabelMapDisplay)
  self.SegmentCADLabelMap.SetAndObserveDisplayNodeID(self.SegmentCADLabelMapDisplay.GetID())
  self.SegmentCADLabelMapDisplay.UpdateScene(slicer.mrmlScene)
  """

def createNumpyArray (self, imageNode):
  # Generate Numpy Array from vtkMRMLScalarVolumeNode
  imageData = vtk.vtkImageData()
  imageData = imageNode.GetImageData()
  shapeData = list(imageData.GetDimensions())
  shapeData.reverse()
  return (vtk.util.numpy_support.vtk_to_numpy(imageData.GetPointData().GetScalars()).reshape(shapeData))

def tumorVoxelsAndCoordinates(self, arrayROI, arrayDataNode):
  coordinates = numpy.where(arrayROI != 0) # can define specific label values to target or avoid
  values = arrayDataNode[coordinates].astype('int64')
  return(values, coordinates)

def paddedTumorMatrixAndCoordinates(self, shape, targetVoxels, targetVoxelsCoordinates):
  matrix = numpy.empty(shape, dtype='float64')
  matrix[:] = numpy.NAN
  matrix[targetVoxelsCoordinates] = targetVoxels
  return(matrix, targetVoxelsCoordinates)

def padMatrix(self, a, matrixCoordinates, dims, voxelArray):
  # pads matrix 'a' with zeros and resizes 'a' to a cube with dimensions increased to the next greatest power of 2
  # numpy version 1.7 has numpy.pad function

  # center coordinates onto padded matrix  # consider padding with NaN or eps = numpy.spacing(1)
  pad = tuple(map(operator.div, tuple(map(operator.sub, dims, a.shape)), ([2,2,2])))
  matrixCoordinatesPadded = tuple(map(operator.add, matrixCoordinates, pad))
  matrix2 = numpy.zeros(dims)
  matrix2[matrixCoordinatesPadded] = voxelArray
  return (matrix2, matrixCoordinatesPadded)

reason: import numpy


text:
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

import functions
import varopt_discrete

func_name = "negative_branin_uniform"

quantile = 0.1

tf.reset_default_graph()
graph = tf.get_default_graph()
dtype = tf.float64

f_info = getattr(functions, func_name)()

func = f_info["function"]

func_tf = f_info["function_tf"]
xmin = f_info["xmin"]
xmax = f_info["xmax"]

xdim = f_info["xdim"]
zdim = f_info["zdim"]
input_dim = xdim + zdim

zvalues_np = f_info["zvalues"]
zprobs_np = f_info["zprobs"]

def ground_truth_func(x, z):
    # x: (1,nx,1,1,xdim)
    # z: (nz,zdim)
    # return (1,n_init_x,1,nz)
    nx = tf.shape(x)[1]
    nz = tf.shape(z)[0]
    x = tf.tile(x, multiples=(1, 1, 1, nz, 1))
    # (1,nx,1,nz,xdim)
    z = tf.reshape(z, shape=(1, 1, 1, nz, zdim))
    z = tf.tile(z, multiples=(1, nx, 1, 1, 1))
    # (1,nx,1,nz,zdim)

    xz = tf.concat([x, z], axis=-1)
    # (1,nx,1,nz,xdim+zdim)

    flatten_xz = tf.reshape(xz, shape=(-1, input_dim))
    # (nx*nz,xdim+zdim)

    vals = func_tf(flatten_xz)
    return tf.reshape(vals, shape=(1, nx, 1, nz))

n_rand_opt_init = 10

ground_truth_varopt = varopt_discrete.VaROpt(
    xdim,
    zdim,
    [xmin, xmax],
    ground_truth_func,
    zvalues_np.shape[0],
    zvalues_np,
    zprobs_np,
    n_rand_opt_init,
    graph,
    name="ground_truth_varopt",
    dtype=dtype,
)

x_plc = tf.placeholder(dtype=dtype, shape=(None, xdim), name="x_plc")
ground_truth_quantile_f_val = ground_truth_varopt.find_quantile(x_plc)

if xdim == 1:
    x_np = np.linspace(xmin, xmax, 1000).reshape(-1, xdim)
elif xdim == 2:
    x_np = functions.get_meshgrid(xmin, xmax, 100, xdim)
elif xdim >= 3:
    x_np = functions.get_meshgrid(xmin, xmax, 10, xdim)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    ground_truth_quantile_f_val_np = sess.run(
        ground_truth_quantile_f_val,
        feed_dict={
            x_plc: x_np,
            ground_truth_varopt.quantile_plc: quantile,
        },
    )

print(
    "VaR range: {} - {}".format(
        np.min(ground_truth_quantile_f_val_np), np.max(ground_truth_quantile_f_val_np)
    )
)

if xdim == 1:
    plt.plot(x_np, ground_truth_quantile_f_val_np)
elif xdim == 2:
    side = int(np.sqrt(ground_truth_quantile_f_val_np.shape[0]))
    X = np.tile(np.linspace(0.0, 1.0, side).reshape(side, 1), reps=(1, side))
    plt.contourf(X, X.T, ground_truth_quantile_f_val_np.reshape(side, side))
    plt.colorbar()
elif xdim == 3:

    for seed in range(5):
        fig, ax = plt.subplots()

        np.random.seed(seed)
        proj_mat = np.random.rand(xdim, 2)
        proj_mat /= np.sqrt(np.sum(np.square(proj_mat), axis=0, keepdims=True))
        proj_x = x_np[:900].dot(proj_mat)

        cb = ax.contourf(
            proj_x[:, 0].reshape(30, 30),
            proj_x[:, 1].reshape(30, 30),
            ground_truth_quantile_f_val_np[:900].reshape(30, 30),
        )
        plt.colorbar(cb)

elif xdim == 4:

    np.random.seed(2)
    proj_mat = np.random.rand(xdim, 2)
    proj_mat /= np.sqrt(np.sum(np.square(proj_mat), axis=0, keepdims=True))
    proj_x = x_np.dot(proj_mat)

    plt.contourf(
        proj_x[:, 0].reshape(100, 100),
        proj_x[:, 1].reshape(100, 100),
        ground_truth_quantile_f_val_np.reshape(100, 100),
    )
    plt.colorbar()

plt.show()

max_stars_repo_name: qphong/BayesOpt-LV

max_issues_repo_name: qphong/BayesOpt-LV

max_forks_repo_name: qphong/BayesOpt-LV