text:
"""
PySCeS - Python Simulator for Cellular Systems (http://pysces.sourceforge.net)

Copyright (C) 2004-2022 B.G. Olivier, J.M. Rohwer, J.-H.S Hofmeyr all rights reserved,

Brett G. Olivier (bgoli@users.sourceforge.net)
Triple-J Group for Molecular Cell Physiology
Stellenbosch University, South Africa.

Permission to use, modify, and distribute this software is given under the
terms of the PySceS (BSD style) license. See LICENSE.txt that came with
this distribution for specifics.

NO WARRANTY IS EXPRESSED OR IMPLIED.  USE AT YOUR OWN RISK.
Brett G. Olivier
"""

from __future__ import division, print_function
from __future__ import absolute_import
from __future__ import unicode_literals

from .version import __version__

import os
import math, operator
import numpy

from .. import output_dir, __CHGDIR_ON_START__
if __CHGDIR_ON_START__:
    CWD = output_dir
else:
    CWD = os.getcwd()

from .InfixParser import MyInfixParser

InfixParser = MyInfixParser()
InfixParser.buildlexer()
InfixParser.buildparser(
    debug=0,
    debugfile='infix.dbg',
    tabmodule='infix_tabmodule',
    outputdir=output_dir,
)
InfixParser.setNameStr('self.', '()')

os.chdir(CWD)

class MapList(list):
    def __init__(self, *args):
        list.__init__(self, *args)

    def asSet(self):
        return set(self.__getslice__(0, self.__len__()))

class NewCoreBase(object):
    __DEBUG__ = False
    name = None
    annotations = None

    def getName(self):
        return self.name

    def setName(self, name):
        self.name = name

    def get(self, attr):
        """Return an attribute whose name is str(attr)"""
        return self.__getattribute__(attr)

    def getAnnotation(self):
        """Returns an annotation dictionary"""
        if self.annotations == None:
            self.annotations = {}
        return self.annotations.copy()

    def setAnnotation(self, key, value):
        """Set an annotation as a key:value pair"""
        if self.annotations == None:
            self.annotations = {}
        self.annotations.update({key: value})

class NumberBase(NewCoreBase):
    value = None
    value_initial = None

    def __call__(self):
        return self.value

    def getValue(self):
        return self.value

    def setValue(self, v):
        self.value = v

class Compartment(NewCoreBase):
    size = None
    dimensions = None
    Compartment = None
    reactions = None
    species = None
    area = None

    def __init__(self, name, compartment=None):
        self.name = name
        self.Compartment = compartment
        self.reactions = []
        self.species = []

    def __call__(self):
        return self.size

    def setSize(self, size, dim):
        self.size = size
        assert dim in [0, 1, 2, 3], '\nOkeee! %s dimensions?' % dim
        self.dimensions = dim

    def setArea(self, area=None):
        if area == None and self.dimensions == 2:
            self.area = self.size
            if self.__DEBUG__:
                print('Setting reactive area to size for 2D compartment %s' % self.name)
        elif area == None and self.dimensions == 3:
            self.area = (113.09733552923255 * self.size ** 2.0) ** (0.33333333333333331)
            if self.__DEBUG__:
                print(
                    'Setting reactive area to surface area for 3D compartment %s (assuming a sphere geometry)'
                    % self.name
                )
        self.area = area

    def hasReactions(self):
        return MapList([r.name for r in self.reactions])

    def hasSpecies(self):
        return MapList([s.name for s in self.species])

    def addReaction(self, reaction):
        if reaction.name not in self.hasReactions():
            self.reactions.append(reaction)
            self.__setattr__(reaction.name, reaction)
            if self.__DEBUG__:
                print('Adding reaction %s' % reaction.name)

    def addSpecies(self, species):
        if species.name not in self.hasSpecies():
            self.species.append(species)
            self.__setattr__(species.name, species)
            if self.__DEBUG__:
                print('Adding species %s' % species.name)
        else:
            if self.__DEBUG__:
                print('Species %s already added' % species.name)

    def getDimensions(self):
        return self.dimensions

    def getCompartment(self):
        return self.Compartment

    def hasCompartment(self):
        if self.Compartment != None:
            return True
        else:
            return False

    def isVolume(self):
        if self.dimensions == 3:
            return True
        else:
            return False

    def isArea(self):
        if self.dimensions == 2:
            return True
        else:
            return False

    def isLength(self):
        if self.dimensions == 1:
            return True
        else:
            return False

    def isPoint(self):
        if self.dimensions == 0:
            return True
        else:
            return False

class BaseUnit(NewCoreBase):
    '''Base Unit can be of type: time, substance, volume'''

    _types = ('time', 'substance', 'volume', 'area', 'length')
    value = 1.0
    type = None

    def __init__(self, name, type):
        self.name = name
        assert type in self._types, '\nType must be one of: %s' % str(self._types)
        self.type = type

    def __call__(self):
        return self.value

    def getType(self):
        return self.type

class SimpleUnit(NewCoreBase):
    exponent = 1.0
    scale = 0.0
    multiplier = 1.0
    baseunit = None
    type = None

    def __init__(self, baseunit, name, exp=1.0, scale=0.0, mult=1.0):
        self.baseunit = baseunit
        self.exponent = exp
        self.scale = scale
        self.multiplier = mult
        self.name = name
        self.type = baseunit.type

    def __call__(self):
        return (self.multiplier * self.baseunit() * 10 ** self.scale) ** self.exponent

    def getType(self):
        return self.type

class CompoundUnit(NewCoreBase):
    units = None
    _HAS_USERNAME = False

    def __init__(self, name=None):
        self.units = []
        if name != None:
            self.name = name
            self._HAS_USERNAME = True
        else:
            self.name = ''

    def __call__(self):
        U = 1.0
        for u in self.units:
            U *= u()
        return U

    def addUnit(self, unit):
        self.units.append(unit)
        if not self._HAS_USERNAME:
            self.name = '%s%s' % (self.name, unit.getName())

    def getUnits(self):
        return self.units

    def hasUnits(self):
        return MapList([u.getName() for u in self.units])

class Species(NumberBase):
    subs = None
    prods = None
    mods = None
    fixed = False
    Compartment = None
    __amount__ = False

    def __init__(self, name, value):
        self.setName(name)
        self.value = value
        self.value_initial = value
        self.subs = []
        self.prods = []
        self.mods = []

    def getCompartment(self):
        return self.Compartment

    def setCompartment(self, c):
        self.Compartment = c

    def hasCompartment(self):
        if self.Compartment != None:
            return True
        else:
            return False

    def setSubstrate(self, reaction):
        self.__setattr__(reaction.name, reaction)
        self.subs.append(reaction)

    def setProduct(self, reaction):
        self.__setattr__(reaction.name, reaction)
        self.prods.append(reaction)

    def setModifier(self, reaction):
        self.__setattr__(reaction.name, reaction)
        self.mods.append(reaction)

    def isSubstrateOf(self):
        return MapList([r.name for r in self.subs])

    def isProductOf(self):
        return MapList([r.name for r in self.prods])

    def isModifierOf(self):
        return MapList([r.name for r in self.mods])

    def isReagentOf(self):
        return MapList(self.isSubstrateOf() + self.isProductOf())

    def setAmount(self, b):
        self.__amount__ = bool(b)

    def isAmount(self):
        return self.__amount__

class SpeciesAssignmentRule(Species):
    formula = None
    code_string = None
    _names = None
    _functions = None
    type = 'assignment'
    _TIME_ = None

    def __init__(self, name, value):
        Species.__init__(self, name, value)

    def __call__(self):
        exec(self.xcode)
        return self.value

    def addFormula(self, formula):
        formula = formula.replace('self.', '')
        self.formula = formula
        InfixParser.setNameStr('self.', '()')
        InfixParser.parse(formula)
        self.code_string = 'self.value=%s' % InfixParser.output
        self._names = InfixParser.names
        self._functions = InfixParser.functions
        self.xcode = compile(self.code_string, '<string>', 'exec')

    def addModelAttr(self, obj):
        self.__setattr__(obj.name, obj)

class Function(NewCoreBase):
    formula = None
    code_string = None
    xcode = None
    value = None
    _names = None
    args = None
    _TIME_ = None

    def __init__(self, name):
        self.setName(name)
        self.args = []

    def __call__(self, *args):
        for ar in range(len(args)):
            self.__setattr__(self.args[ar], args[ar])
        exec(self.xcode)
        return self.value

    def setArg(self, var, value=None):
        self.__setattr__(var, value)
        self.args.append(var)

    def addFormula(self, formula):
        # print('PC2 377:', formula)
        formula = formula.replace('self.', '')
        self.formula = formula
        InfixParser.setNameStr('self.', '')
        InfixParser.SymbolReplacements = {'_TIME_': '_TIME_()'}
        InfixParser.parse(formula)
        self._names = InfixParser.names
        self.code_string = 'self.value=%s' % InfixParser.output
        self.xcode = compile(self.code_string, '<string>', 'exec')

class Reaction(NewCoreBase):
    modifiers = None
    substrates = None
    products = None
    stoichiometry = None
    multistoich = None
    multistoich_enabled = False
    parameters = None
    functions = None
    reversible = True
    formula = None
    code_string = None
    rate = None
    xcode = None
    _names = None
    _functions = None
    _TIME_ = None
    Compartment = None

    def __call__(self):
        exec(self.xcode)
        return self.rate

    def __init__(self, name):
        self.setName(name)
        self.modifiers = []
        self.substrates = []
        self.products = []
        self.stoichiometry = {}
        self.parameters = []
        self.functions = []
        self.multistoich = []

    def addSubstrate(self, species):
        self.__setattr__(species.name, species)
        self.substrates.append(species)

    def addProduct(self, species):
        self.__setattr__(species.name, species)
        self.products.append(species)

    def addModifier(self, species):
        self.__setattr__(species.name, species)
        self.modifiers.append(species)

    def addFormula(self, formula):
        formula = formula.replace('self.', '')
        self.formula = formula
        InfixParser.setNameStr('self.', '()')
        InfixParser.parse(formula)
        self._names = InfixParser.names
        self._functions = InfixParser.functions
        self.code_string = 'self.rate=%s' % InfixParser.output
        self.xcode = compile(self.code_string, '<string>', 'exec')

    def addParameter(self, par):
        self.__setattr__(par.name, par)
        self.parameters.append(par)

    def addFunction(self, func):
        self.__setattr__(func.name, func)
        self.functions.append(func)

    def hasProducts(self, t=type):
        return MapList([p.name for p in self.products])

    def hasSubstrates(self):
        return MapList([s.name for s in self.substrates])

    def hasModifiers(self):
        return MapList([m.name for m in self.modifiers])

    def hasParameters(self):
        return MapList([p.name for p in self.parameters])

    def hasReagents(self):
        return MapList(self.hasSubstrates() + self.hasProducts())

    def setCompartment(self, compartment):
        self.Compartment = compartment

    def getCompartment(self):
        return self.Compartment

    def hasCompartment(self):
        if self.Compartment != None:
            return True
        else:
            return False

class Parameter(NumberBase):
    association = None

    def __init__(self, name, value):
        self.name = name
        self.value = value
        self.value_initial = value
        self.association = []

    def setAssociation(self, reac):
        self.association.append(reac)
        self.__setattr__(reac.name, reac)

    def isParameterOf(self):
        return MapList([a.name for a in self.association])

class AssignmentRule(Parameter):
    formula = None
    code_string = None
    _names = None
    _functions = None
    type = 'assignment'
    _TIME_ = None
    fixed = False  # added so that assignment rules can modify fixed species

    def __init__(self, name, value):
        Parameter.__init__(self, name, value)

    def __call__(self):
        exec(self.xcode)
        return self.value

    def addFormula(self, formula):
        formula = formula.replace('self.', '')
        self.formula = formula
        InfixParser.setNameStr('self.', '()')
        InfixParser.parse(formula)
        self.code_string = 'self.value=%s' % InfixParser.output
        self._names = InfixParser.names
        self._functions = InfixParser.functions
        # print(self.code_string)
        self.xcode = compile(self.code_string, '<string>', 'exec')

    def addModelAttr(self, obj):
        self.__setattr__(obj.name, obj)

class RateRule(NewCoreBase):
    formula = None
    rate = None
    xcode = None
    code_string = None
    _names = None
    _functions = None
    compartment = None

    def __init__(self, name, formula):
        self.name = name
        self.addFormula(formula)

    def __call__(self):
        exec(self.xcode)
        return self.rate

    def addFormula(self, formula):
        formula = formula.replace('self.', '')
        self.formula = formula.replace('()', '')
        InfixParser.setNameStr('self.', '()')
        InfixParser.parse(self.formula)
        self.code_string = 'self.rate=%s' % InfixParser.output
        self._names = InfixParser.names
        self._functions = InfixParser.functions
        self.xcode = compile(self.code_string, 'RateRule: %s' % self.name, 'exec')

    def getFormula(self):
        return self.formula

    def addModelAttr(self, obj):
        self.__setattr__(obj.name, obj)

class ODE(NewCoreBase):
    sdot = None
    value = None
    coefficients = None
    reactions = None
    independent = None
    ode_terms = None
    formula = ''
    formula_alt = ''
    code_string = 'self.value='
    code_string_alt = 'sdot='

    def __init__(self, species, independent=True):
        self.sdot = species
        self.name = 'ODE_' + species.name
        self.reactions = []
        self.coefficients = []
        self.ode_terms = []
        self.independent = independent

    def __call__(self):
        exec(self.code_string)
        return self.value

    def addReaction(self, reaction, coefficient):
        self.reactions.append(reaction)
        self.coefficients.append(coefficient)
        if coefficient > 0.0:
            if coefficient == 1.0:
                term = '+self.%s() ' % (reaction.name)
                aterm = '+(%s) ' % (reaction.code_string.replace('self.rate=', ''))
                fterm = '+%s' % (reaction.name)
                afterm = '+ (%s) ' % (reaction.formula)
            else:
                term = '+%g*self.%s() ' % (abs(coefficient), reaction.name)
                aterm = '+%g*(%s) ' % (
                    abs(coefficient),
                    reaction.code_string.replace('self.rate=', ''),
                )
                fterm = '+%g*%s' % (abs(coefficient), reaction.name)
                afterm = '+ %g*(%s) ' % (abs(coefficient), reaction.formula)
        else:
            if coefficient == -1.0:
                term = '-self.%s() ' % (reaction.name)
                aterm = '-(%s) ' % (reaction.code_string.replace('self.rate=', ''))
                fterm = '-%s' % (reaction.name)
                afterm = '- (%s) ' % (reaction.formula)
            else:
                term = '-%g*self.%s() ' % (abs(coefficient), reaction.name)
                aterm = '-%g*(%s) ' % (
                    abs(coefficient),
                    reaction.code_string.replace('self.rate=', ''),
                )
                fterm = '-%g*%s' % (abs(coefficient), reaction.name)
                afterm = '- %g*(%s) ' % (abs(coefficient), reaction.formula)
        self.ode_terms.append(term)
        self.code_string += term
        self.code_string_alt += aterm
        self.formula += fterm
        self.formula_alt += afterm
        self.__setattr__(reaction.name, reaction)

    def hasReactions(self):
        return MapList([r.name for r in self.reactions])

    def getFormula(self):
        return self.code_string

    def getGlobalFormula(self):
        return self.code_string_alt

class StructMatrix(NewCoreBase):
    """
    This class is specifically designed to store structural matrix information
    give it an array and row/col index permutations it can generate its own
    row/col labels given the label src.
    """

    array = None
    ridx = None
    cidx = None
    row = None
    col = None

    def __init__(self, array, ridx, cidx, row=None, col=None):
        """
        Instantiate with array and matching row/col index arrays, optional label arrays
        """
        self.array = array
        self.ridx = ridx
        self.cidx = cidx
        self.row = row
        self.col = col
        self.shape = array.shape

    def __call__(self):
        return self.array

    def getRowsByIdx(self, *args):
        """Return the rows referenced by index (1,3,5)"""
        return self.array.take(args, axis=0)

    def getColsByIdx(self, *args):
        """Return the columns referenced by index (1,3,5)"""
        return self.array.take(args, axis=1)

    def setRow(self, src):
        """
        Assuming that the row index array is a permutation (full/subset)
        of a source label array by supplying that source to setRow it
        maps the row labels to ridx and creates self.row (row label list)
        """
        self.row = [src[r] for r in self.ridx]

    def setCol(self, src):
        """
        Assuming that the col index array is a permutation (full/subset)
        of a source label array by supplying that src to setCol
        maps the row labels to cidx and creates self.col (col label list)
        """
        self.col = [src[c] for c in self.cidx]

    def getRowsByName(self, *args):
        """Return the rows referenced by label ('s','x','d')"""
        assert self.row != None, "\nI need row labels"
        try:
            return self.array.take([self.row.index(l) for l in args], axis=0)
        except Exception as ex:
            print(ex)
            print("\nValid row labels are: %s" % self.row)
            return None

    def getColsByName(self, *args):
        """Return the columns referenced by label ('s','x','d')"""
        assert self.col != None, "\nI need column labels"
        try:
            return self.array.take([self.col.index(l) for l in args], axis=1)
        except Exception as ex:
            print(ex)
            print("Valid column labels are: %s" % self.col)
            return None

    def getLabels(self, axis='all'):
        """Return the matrix labels ([rows],[cols]) where axis='row'/'col'/'all'"""
        if axis == 'row':
            return self.row
        elif axis == 'col':
            return self.col
        else:
            return self.row, self.col

    def getIndexes(self, axis='all'):
        """Return the matrix indexes ([rows],[cols]) where axis='row'/'col'/'all'"""
        if axis == 'row':
            return self.ridx
        elif axis == 'col':
            return self.cidx
        else:
            return self.ridx, self.cidx

    def getByIdx(self, row, col):
        assert row in self.ridx, '\n%s is an invalid index' % row
        assert col in self.cidx, '\n%s is an invalid index' % col
        return self.array[row, col]

    def getByName(self, row, col):
        assert row in self.row, '\n%s is an invalid name' % row
        assert col in self.col, '\n%s is an invalid name' % col
        return self.array[self.row.index(row), self.col.index(col)]

    def setByIdx(self, row, col, val):
        assert row in self.ridx, '\n%s is an invalid index' % row
        assert col in self.cidx, '\n%s is an invalid index' % col
        self.array[row, col] = val

    def setByName(self, row, col, val):
        assert row in self.row, '\n%s is an invalid name' % row
        assert col in self.col, '\n%s is an invalid name' % col
        self.array[self.row.index(row), self.col.index(col)] = val

    def shape(self):
        return self.array.shape

class EventAssignment(NumberBase):
    variable = None
    _names = None
    formula = None
    code_string = None
    xcode = None

    def __call__(self):
        self.variable.value = self.value
        if self.__DEBUG__:
            print('\tAssigning %s = %s' % (self.variable.name, self.value))
        return True

    def __init__(self, name='None'):
        self.setName(name)

    def setVariable(self, var):
        self.variable = var

    def setFormula(self, formula):
        self.formula = formula
        InfixParser.setNameStr('self.', '()')
        ##  InfixParser.SymbolReplacements = {'_TIME_':'_TIME_()'}
        InfixParser.parse(formula)
        self._names = InfixParser.names
        self.code_string = 'self.value=%s' % InfixParser.output
        self.xcode = compile(self.code_string, '<string>', 'exec')
        if self.__DEBUG__:
            '\t', self.name, self.code_string

    def evaluateAssignment(self):
        exec(self.xcode)

class Event(NewCoreBase):
    trigger = None
    delay = 0.0

    formula = None
    code_string = None
    xcode = None

    state0 = False
    state = False

    assignments = None
    _TIME_ = None
    _ASS_TIME_ = 0.0
    _need_action = False
    _names = None
    _time_symbol = None

    def __init__(self, name):
        self.setName(name)
        self.assignments = []

    def __call__(self, time):
        self._TIME_.set(time)
        exec(self.xcode)
        if self.state0 and not self.state:
            self.state0 = self.state
        if not self.state0 and self.state:
            for ass in self.assignments:
                ass.evaluateAssignment()
            self.state0 = self.state
            self._need_action = True
            self._ASS_TIME_ = self._TIME_() + self.delay
            if self.__DEBUG__:
                print('event %s is evaluating at %s' % (self.name, time))
        if self._need_action and self._TIME_() >= self._ASS_TIME_:
            for ass in self.assignments:
                ass()
            if self.__DEBUG__:
                print(
                    'event %s is assigning at %s (delay=%s)'
                    % (self.name, time, self.delay)
                )
            self._need_action = False

    def setTrigger(self, formula, delay=0.0):
        self.formula = formula
        self.delay = delay
        InfixParser.setNameStr('self.', '()')
        ##  print self._time_symbol
        if self._time_symbol != None:
            InfixParser.SymbolReplacements = {self._time_symbol: '_TIME_'}
            ##  self.formula = formula.replace(self._time_symbol, '_TIME_')
        InfixParser.parse(formula)
        self._names = InfixParser.names
        self.code_string = 'self.state=%s' % InfixParser.output
        if self._time_symbol != None:
            InfixParser.setNameStr('', '')
            InfixParser.SymbolReplacements = {self._time_symbol: '_TIME_'}
            InfixParser.parse(formula)
            self.formula = InfixParser.output
        self.xcode = compile(self.code_string, '<string>', 'exec')
        if self.__DEBUG__:
            self.name, self.code_string

    def setTriggerAttributes(self, core):
        # TODO: experimental
        for n in self._names:
            self.__setattr__(n, core.__getattribute__(n))

    def setAssignment(self, var, formula):
        ass = EventAssignment(var.name)
        ass.setVariable(var)
        ass.setFormula(formula)
        self.assignments.append(ass)
        self.__setattr__('_' + var.name, ass)

class PieceWise(NewCoreBase):
    """
    Generic piecewise class written by me!

    - *args* a dictionary of piecewise information generated by the InfixParser
    """

    name = None
    value = None
    formula = None
    code_string = None
    xcode = None
    _names = None
    _TIME_ = None

    def __init__(self, pwd):
        pwd = pwd.copy()
        if pwd['other'] != None:
            other = 'self.value = %s' % pwd.pop('other')
        else:
            other = 'pass'
            pwd.pop('other')
        InfixParser.setNameStr('self.', '')
        InfixParser.SymbolReplacements = {'_TIME_': '_TIME_()'}
        self._names = []
        if len(list(pwd.keys())) == 1:
            formula = pwd[0][0]
            InfixParser.parse(formula)
            for n in InfixParser.names:
                if n not in self._names and n != '_TIME_()':
                    self._names.append(n)
            formula = InfixParser.output
            self.code_string = 'if %s:\n    self.value = %s\nelse:\n    %s' % (
                formula,
                pwd[0][1],
                other,
            )
            self.formula = self.code_string.replace('self.', '')
        else:
            formula = pwd[0][0]
            InfixParser.parse(formula)
            for n in InfixParser.names:
                if n not in self._names and n != '_TIME_()':
                    self._names.append(n)

            formula = InfixParser.output
            self.code_string = 'if %s:\n    self.value = %s\n' % (formula, pwd[0][1])
            pwd.pop(0)
            for p in pwd:
                formula = pwd[p][0]
                InfixParser.SymbolReplacements = {'_TIME_': '_TIME_()'}
                InfixParser.parse(formula)
                for n in InfixParser.names:
                    if n not in self._names and n != '_TIME_()':
                        self._names.append(n)
                formula = InfixParser.output
                self.code_string += 'elif %s:\n    self.value = %s\n' % (
                    formula,
                    pwd[p][1],
                )
            self.code_string += 'else:\n    %s' % other
            self.formula = self.code_string.replace('self.', '')
        self.xcode = compile(self.code_string, 'PieceWise', 'exec')

    def __call__(self):
        exec(self.xcode)
        return self.value

class Time(object):
    value = None
    name = '__TIME__'

    def __init__(self, t=0):
        self.value = t

    def __call__(self):
        return self.value

    def set(self, t):
        self.value = t

##  def delay(*args):
##  print 'delay() ignored'
##  return 1.0

class NewCore(NewCoreBase):
    __nDict__ = None
    reactions = None
    species = None
    species_variable = None
    __model__ = None
    __InitDict__ = None
    __not_inited__ = None
    global_parameters = None
    __parameter_store__ = None
    forcing_functions = None
    __rules__ = None
    __events__ = None
    # new
    __compartments__ = None
    compartments = None
    rate_rules = None
    description = "Pysces Core2"
    __uDict__ = None
    stoichiometric_matrix = None
    struct = None
    ODEs = None
    functions = None
    _TIME_ = None
    events = None
    __sDict__ = None
    __KeyWords__ = None
    __piecewises__ = None
    piecewise_functions = None
    netStoich = None

    def __init__(self, model, iValues=True, netStoich=True):
        # setup core dictionaries
        self.__nDict__ = model.__nDict__
        self.__sDict__ = model.__sDict__
        self.__KeyWords__ = model.__KeyWords__
        if self.__KeyWords__['Modelname'] != None:
            self.setName(self.__KeyWords__['Modelname'])
        else:
            self.setName('PySCeSModel')
        if self.__KeyWords__['Description'] != None:
            self.setDescription(self.__KeyWords__['Description'])
        else:
            self.setDescription('PySCeSModel')

        self.__model__ = model
        self.__InitDict__ = model.__InitDict__
        if not iValues:
            if self.__DEBUG__:
                print(self.__InitDict__)
            for k in list(self.__InitDict__.keys()):
                self.__InitDict__[k] = getattr(self.__model__, k)
            for c in model.__compartments__:
                model.__compartments__[c]['size'] = getattr(self.__model__, c)
        self.netStoich = netStoich

        self.global_parameters = []
        self.__parameter_store__ = []
        self.__not_inited__ = []
        self.forcing_functions = []
        self.__rules__ = model.__rules__
        self.__uDict__ = model.__uDict__
        self.__piecewises__ = model.__piecewises__
        InfixParser.__pwcntr__ = 0

        # start building objects
        self.__compartments__ = model.__compartments__
        self.addCompartments()
        self._TIME_ = Time()
        self.addPieceWiseFunctions()  # this adds any piecewise functions
        self.addSpecies()

        # the order is important from here as eg functions can occur in rate equations
        try:
            self.__functions__ = model.__functions__
        except:
            self.__functions__ = {}
            if self.__DEBUG__:
                print('No functions')
        self.functions = []
        self.addFunctions()
        self.addReactions()
        self.generateMappings()
        self.setAssignmentRules()
        self.setRateRules()

        # add event support
        self.__events__ = self.__model__.__eDict__
        self.events = []
        self.addEvents()
        self.addPieceWiseFunctions(update=True)  # this updates their attributes

        ##  # get rid of _TIME_ in not intited
        ##  if '_TIME_' in self.__not_inited__:
        ##  self.__not_inited__.pop(self.__not_inited__.index('_TIME_'))
        if len(self.__not_inited__) > 0:
            print(
                "\nWARNING: Uninitialised parameters: %s (value set to zero)"
                % self.__not_inited__
            )

    def __cleanString__(self, s):
        s = s.lstrip()
        s = s.rstrip()
        return s

    ##  def parseForcingFunctions(self):
    ##  self.__rules__ = {}
    ##  try:
    ##  ff = self.__function_forced_str__.split('\n')
    ##  for f in ff:
    ##  if f != '':
    ##  f =  f.split('=')
    ##  f[0] = f[0].replace('self.','')
    ##  f[1] = f[1].replace('self.','')
    ##  self.__rules__.setdefault(self.__cleanString__(f[0]), self.__cleanString__(f[1]))
    ##  except Exception, ex:
    ##  pass
    # print 'No forcing functions (%s).' % ex

    def setDescription(self, txt):
        self.description = str(txt)

    def getDescription(self):
        return str(self.description)

    def setGlobalUnits(self, **kwargs):
        for un in list(kwargs.keys()):
            self.__uDict__[un] = (kwargs[un][0], kwargs[un][1])
            if self.__DEBUG__:
                print(
                    "Modified \"%s\" to be %i*%s*10**%i"
                    % (un, kwargs[un][0], un, kwargs[un][1])
                )

    def getGlobalUnits(self):
        return self.__uDict__

    def addPieceWiseFunctions(self, update=False):
        if not update:
            self.piecewise_functions = []
            for pw in list(self.__piecewises__.keys()):
                if self.__DEBUG__:
                    print('Info: adding piecewise function:%s' % pw)
                P = PieceWise(self.__piecewises__[pw])
                P.setName(pw)
                P.__setattr__('_TIME_', self.__getattribute__('_TIME_'))
                self.piecewise_functions.append(P)
                self.__setattr__(pw, P)
        else:
            for pw in self.piecewise_functions:
                for a in pw._names:
                    pw.__setattr__(a, self.__getattribute__(a))

    def addOneCompartment(self, name, size, dimensions, compartment=None, area=None):
        C = Compartment(name, compartment)
        C.setSize(size, dimensions)
        ##  C.setArea(area)
        self.compartments.append(C)
        self.__setattr__(name, C)

    def addCompartments(self):
        self.compartments = []
        for C in self.__compartments__:
            c2 = self.__compartments__[C]
            if self.__DEBUG__:
                print('Adding compartment %s' % c2['name'])
            self.addOneCompartment(
                c2['name'],
                c2['size'],
                c2['dimensions'],
                compartment=c2['compartment'],
                area=None,
            )

    def addOneSpecies(
        self, species, value, fix=False, comp=None, amount=False, fullName=None
    ):
        s = Species(species, value)
        ##  if comp != None:
        s.setCompartment(comp)
        s.setAmount(amount)
        s.setAnnotation('sbml_name', fullName)
        if fix:
            s.fixed = True
        self.__setattr__(species, s)
        self.species.append(s)
        if not fix:
            self.species_variable.append(s)
        if comp != None:
            comp.addSpecies(s)

    def addSpecies(self):
        self.species = []
        self.species_variable = []
        for s in self.__sDict__:
            ##  print s
            ##  print self.__sDict__[s]
            name = self.__sDict__[s]['name']
            if s in self.__InitDict__:
                val = self.__InitDict__[s]
            else:
                val = 0.0
            ##  print val
            fix = self.__sDict__[s]['fixed']
            if self.__sDict__[s]['compartment'] != None:
                comp = self.__getattribute__(self.__sDict__[s]['compartment'])
            else:
                comp = None
            amount = self.__sDict__[s]['isamount']
            fullName = None
            if 'fullName' in self.__sDict__[s]:
                fullName = self.__sDict__[s]['fullName']
            # print(self.__sDict__)
            self.addOneSpecies(
                name, val, fix=fix, comp=comp, amount=amount, fullName=fullName
            )

    def addOneFunction(self, name, args, formula):
        func = Function(name)
        # TODO: make better
        setattr(func, '_TIME_', self._TIME_)
        for a in args:
            func.setArg(a)
        func.addFormula(formula)
        self.functions.append(func)
        self.__setattr__(name, func)

    def addFunctions(self):
        for f in list(self.__functions__.keys()):
            self.addOneFunction(
                f, self.__functions__[f]['args'], self.__functions__[f]['formula']
            )

    def addOneReaction(self, rDict):
        r = Reaction(rDict['name'])
        if rDict['compartment'] != None:
            C = self.__getattribute__(rDict['compartment'])
            r.setCompartment(C)
            C.addReaction(r)
        fullName = None
        if 'fullName' in rDict:
            r.setAnnotation('sbml_name', rDict['fullName'])

        # add dummy reaction kinetic law
        # TODO: make better
        setattr(r, '_TIME_', self._TIME_)
        if rDict['RateEq'] != None:
            r.addFormula(rDict['RateEq'].replace('self.', ''))
        else:
            r.addFormula('J')
            self.addParameter('J')
        if rDict['Type'] == 'Irrev':
            r.reversible = False
        # now we can add formulas that occured in the rate equation
        if len(r._functions) > 0:
            for func in r._functions:
                try:
                    r.addFunction(self.__getattribute__(func))
                except Exception as ex:
                    print(ex)
                    print('\nHave you added the function objects yet (addFunctions())')

        # fxnames = self.hasFixedSpecies()
        processed_parameter = []

        # where parameters are defined `locally' per reaction
        for p in rDict['Params']:
            p = p.replace('self.', '')
            if p not in self.hasGlobalParameters() and not (
                p in self.hasFixedSpecies() or p in self.__compartments__
            ):
                if self.__DEBUG__:
                    print("Adding parameter %s from networkdict" % p)
                self.addParameter(p)
                par = self.__getattribute__(p)
                par.setAssociation(r)
                r.addParameter(par)
                processed_parameter.append(p)
            elif not (p in self.hasFixedSpecies() or p in self.__compartments__):
                if self.__DEBUG__:
                    print("Updating parameter %s from networkdict" % p)
                pidx = self.hasGlobalParameters().index(p)
                self.global_parameters[pidx].setAssociation(r)
                r.addParameter(self.global_parameters[pidx])
                processed_parameter.append(p)

        # print self.hasGlobalParameters()
        # where parameters are not `locally' defined and are extracted from Req (ie from SBML)
        for p in r._names:
            p = p.replace('self.', '')
            if p == '_TIME_':
                pass
            elif p in [pw.name for pw in self.piecewise_functions]:
                pass
            elif p in self.hasCompartments() and p not in processed_parameter:
                C = self.__getattribute__(p)
                C.addReaction(r)
                # TODO: this will work until isParameterOf is called on a compartment object
                r.addParameter(C)
                # dirty alternative
                # setattr(r, C.name, C)
                processed_parameter.append(p)
            elif (
                p not in processed_parameter
                and p not in self.hasGlobalParameters()
                and p not in self.hasSpecies()
            ):
                if self.__DEBUG__:
                    print("Adding parameter %s from global" % p)
                self.addParameter(p)
                par = self.__getattribute__(p)
                par.setAssociation(r)
                r.addParameter(par)
                processed_parameter.append(p)

            elif p not in processed_parameter and p not in self.hasSpecies():
                if self.__DEBUG__:
                    print("Updating parameter %s from global" % p)
                pidx = self.hasGlobalParameters().index(p)
                self.global_parameters[pidx].setAssociation(r)
                r.addParameter(self.global_parameters[pidx])
                processed_parameter.append(p)

        self.__setattr__(rDict['name'], r)
        self.reactions.append(r)

    def addParameter(self, name):
        if name not in self.__piecewises__:
            if name in self.__InitDict__:
                par = Parameter(name, self.__InitDict__[name])
            else:
                par = Parameter(name, 0.0)
                if name not in self.__not_inited__:
                    self.__not_inited__.append(name)
            self.global_parameters.append(par)
            self.__setattr__(name, par)

    def addReactions(self):
        self.reactions = []
        for r in self.__model__.reactions:
            self.addOneReaction(self.__nDict__[r])
        non_parameters = (
            self.hasGlobalParameters() + self.hasSpecies() + self.hasFixedSpecies()
        )
        for k in list(self.__InitDict__.keys()):
            if k not in non_parameters:
                if self.__DEBUG__:
                    print('Adding new parameter:', k)
                self.addParameter(k)

    def replaceParameterWithRule(self, ar):
        par = self.__getattribute__(ar.name)
        for r in par.association:
            ar.setAssociation(r)
            setattr(r, ar.name, ar)
            r.parameters[r.hasParameters().index(ar.name)] = ar
        self.global_parameters[self.hasGlobalParameters().index(ar.name)] = ar
        self.__setattr__(ar.name, ar)

    def replaceFixedSpeciesWithRule(self, ar):
        fs = self.__getattribute__(ar.name)
        ar.fixed = fs.fixed
        for r in fs.subs:
            ar.setSubstrate(r)
            setattr(r, ar.name, ar)
            r.substrates[r.hasSubstrates().index(ar.name)] = ar
        for r in fs.prods:
            ar.setProduct(r)
            setattr(r, ar.name, ar)
            r.products[r.hasProducts().index(ar.name)] = ar
        for r in fs.mods:
            ar.setModifier(r)
            setattr(r, ar.name, ar)
            r.modifiers[r.hasModifiers().index(ar.name)] = ar
        self.species[self.hasSpecies().index(ar.name)] = ar
        self.__setattr__(ar.name, ar)

    def replaceSpeciesWithRule(self, ar):
        fs = self.__getattribute__(ar.name)
        for r in fs.subs:
            ar.setSubstrate(r)
            setattr(r, ar.name, ar)
            r.substrates[r.hasSubstrates().index(ar.name)] = ar
        for r in fs.prods:
            ar.setProduct(r)
            setattr(r, ar.name, ar)
            r.products[r.hasProducts().index(ar.name)] = ar
        for r in fs.mods:
            ar.setModifier(r)
            setattr(r, ar.name, ar)
            r.modifiers[r.hasModifiers().index(ar.name)] = ar
        self.species[self.hasSpecies().index(ar.name)] = ar
        self.species_variable[self.hasVariableSpecies().index(ar.name)] = ar
        self.__setattr__(ar.name, ar)

    def setAssignmentRules(self):
        aps = [
            self.__rules__[ar]['name']
            for ar in self.__rules__
            if self.__rules__[ar]['type'] == 'assignment'
        ]
        ##  for p in self.global_parameters + [self.get(fs) for fs in self.hasFixedSpecies()]:
        for p in self.global_parameters + self.species:
            # print p.name
            if p.name in aps:
                if self.__DEBUG__:
                    print(
                        'Assigning: %s = %s'
                        % (p.name, self.__rules__[p.name]['formula'])
                    )
                p2 = None
                # TODO: make better
                if p.name in self.hasGlobalParameters():
                    p2 = AssignmentRule(p.name, self.__InitDict__[p.name])
                    setattr(p2, '_TIME_', self._TIME_)
                    self.replaceParameterWithRule(p2)
                elif p.name in self.hasFixedSpecies():
                    p2 = SpeciesAssignmentRule(p.name, self.__InitDict__[p.name])
                    p2.setCompartment(p.getCompartment())
                    setattr(p2, '_TIME_', self._TIME_)
                    self.replaceFixedSpeciesWithRule(p2)
                elif p.name in self.hasVariableSpecies():
                    p2 = SpeciesAssignmentRule(p.name, self.__InitDict__[p.name])
                    p2.setCompartment(p.getCompartment())
                    setattr(p2, '_TIME_', self._TIME_)
                    self.replaceSpeciesWithRule(p2)

                assert isinstance(p2, AssignmentRule) or isinstance(
                    p2, SpeciesAssignmentRule
                ), "\nHappy assertion error"
                # print type(p2)
                p2.addFormula(self.__rules__[p.name]['formula'])

                # add piecewise functions to assignment rules or something like that
                if 'piecewise' in p2.formula:
                    print('\n\nWe have a piecewise', p2.name, p2.formula)
                    print(p2, self.piecewise_functions[0]._names[0], self.__getattribute__(self.piecewise_functions[0]._names[0]).__call__())
                    for p in range(len(self.piecewise_functions)):
                        for n in range(len(self.piecewise_functions[p]._names)):
                            setattr(p2, self.piecewise_functions[p]._names[n], self.__getattribute__(self.piecewise_functions[p]._names[n]).__call__())
                            self.piecewise_functions[p].code_string = self.piecewise_functions[p].code_string.replace('self.{}'.format(self.piecewise_functions[p]._names[n]), 'self.{}()'.format(self.piecewise_functions[p]._names[n]))
                            self.piecewise_functions[p].xcode = compile(self.piecewise_functions[p].code_string, 'PieceWise', 'exec')
                            setattr(p2, self.piecewise_functions[p].name, self.piecewise_functions[p])
                            print(self.piecewise_functions[p].code_string)

                for n in p2._names + p2._functions:
                    p2.addModelAttr(self.__getattribute__(n))
                ##  # setup initial values
                ##  p2.value_initial = self.p2()
                if p2.name in self.__not_inited__:
                    self.__not_inited__.pop(self.__not_inited__.index(p.name))
        for p in self.global_parameters:
            if p.name in self.hasAssignmentRules():
                # TODO assignment rules need a list of properties
                for ar in p._names:
                    if ar in self.hasAssignmentRules():
                        setattr(p, ar, self.__getattribute__(ar))
        # TODO this is where things will go wrong if fs --> ar contains nested ar's

    def setRateRules(self):
        # TODO mayvbe split into two methods for now read from self.__rules__
        # TODO add functions to rules
        ars = [
            self.__rules__[ar]['name']
            for ar in self.__rules__
            if self.__rules__[ar]['type'] == 'rate'
        ]
        self.rate_rules = []
        for rr in ars:
            rrobj = RateRule(self.__rules__[rr]['name'], self.__rules__[rr]['formula'])
            ##  print 'RR:', rrobj.name, rrobj._names, rrobj._functions
            for symb in rrobj._names + rrobj._functions:
                rrobj.addModelAttr(self.__getattribute__(symb))
            self.rate_rules.append(rrobj)
            # TODO investgiate this as it is problematic, the rate rule
            # is not a model property as such more an ODE property
            ##  self.__setattr__(rrobj.name, rrobj)
            if self.__DEBUG__:
                print(
                    'Adding RateRule %s with formula: %s' % (rrobj.name, rrobj.formula)
                )

    def addOneEvent(self, e):
        """Add a single event using an event dictionary """
        # translate self.__events__[e] to e

        ev = Event(e['name'])
        ev._time_symbol = e['tsymb']
        ev.setTrigger(e['trigger'], e['delay'])
        # associate model attributes with event
        # TODO: check that this still works
        ev.setTriggerAttributes(self)
        ##  for n in ev._names:
        ##  setattr(ev, n, self.__getattribute__(n))
        # for each assignment
        for ass in e['assignments']:
            ev.setAssignment(self.__getattribute__(ass), e['assignments'][ass])
            assref = getattr(ev, '_' + ass)  # don\t like this at all :-(
            # associate model attributes with assignment
            for n in assref._names:
                setattr(assref, n, self.__getattribute__(n))
        self.events.append(ev)
        self.__setattr__(ev.name, ev)
        setattr(ev, '_TIME_', self._TIME_)

    def addEvents(self):
        # TODO: check that you can change the trigger on the fly (might need a setAttr thing in event obj)
        self.events = []
        # for each event
        for e in self.__events__:
            self.addOneEvent(self.__events__[e])

    def generateMappings(self):
        ##  self.netStoich = False
        for reac in self.reactions:
            if self.netStoich:
                for reag in self.__nDict__[reac.name]['Reagents']:
                    if self.__nDict__[reac.name]['Reagents'][reag] < 0.0:
                        reac.addSubstrate(
                            self.__getattribute__(reag.replace('self.', ''))
                        )
                        self.__getattribute__(reag.replace('self.', '')).setSubstrate(
                            self.__getattribute__(reac.name)
                        )
                    else:
                        reac.addProduct(
                            self.__getattribute__(reag.replace('self.', ''))
                        )
                        self.__getattribute__(reag.replace('self.', '')).setProduct(
                            self.__getattribute__(reac.name)
                        )
                    reac.stoichiometry.setdefault(
                        reag.replace('self.', ''),
                        self.__nDict__[reac.name]['Reagents'][reag],
                    )
            else:
                for reag in self.__nDict__[reac.name]['AllReagents']:
                    if reag[1] < 0.0:
                        reac.addSubstrate(
                            self.__getattribute__(reag[0].replace('self.', ''))
                        )
                        self.__getattribute__(
                            reag[0].replace('self.', '')
                        ).setSubstrate(self.__getattribute__(reac.name))
                    else:
                        reac.addProduct(
                            self.__getattribute__(reag[0].replace('self.', ''))
                        )
                        self.__getattribute__(reag[0].replace('self.', '')).setProduct(
                            self.__getattribute__(reac.name)
                        )
                    reac.multistoich.append((reag[0].replace('self.', ''), reag[1]))
                    if reag[0].replace('self.', '') in reac.stoichiometry:
                        reac.multistoich_enabled = True
                    reac.stoichiometry.setdefault(reag[0].replace('self.', ''), reag[1])
            for mod in self.__nDict__[reac.name]['Modifiers']:
                reac.addModifier(self.__getattribute__(mod.replace('self.', '')))
                self.__getattribute__(mod.replace('self.', '')).setModifier(
                    self.__getattribute__(reac.name)
                )
            ##  print 'I AM LEGEND'
            ##  print reac.stoichiometry
            ##  print reac.multistoich
            ##  print 'reac.multistoich_enabled', reac.multistoich_enabled
            ##  print self.__nDict__[reac.name]['Reagents']
            ##  print self.__nDict__[reac.name]['AllReagents']

    def setStoichiometricMatrix(self):
        vspec = self.hasVariableSpecies()
        react = self.hasReactions()
        nm = numpy.zeros((len(vspec), len(react)), 'd')
        for sp in vspec:
            for r in self.get(sp).isReagentOf():
                nm[vspec.index(sp)][react.index(r)] = self.get(r).stoichiometry[sp]
            # this is if absolute stoichiometry value is used
            ##  for r in self.get(sp).isSubstrateOf():
            ##  nm[vspec.index(sp)][react.index(r)] = abs(self.get(r).stoichiometry[sp])
            ##  for r in self.get(sp).isProductOf():
            ##  nm[vspec.index(sp)][react.index(r)] = -abs(self.get(r).stoichiometry[sp])
        self.stoichiometric_matrix = StructMatrix(
            nm, list(range(len(vspec))), list(range(len(react)))
        )
        self.stoichiometric_matrix.setRow(vspec)
        self.stoichiometric_matrix.setCol(react)

    def addODEs(self):
        self.ODEs = []
        for varspec in self.stoichiometric_matrix.row:
            if self.struct != None:
                if varspec not in self.struct.Nr.row:
                    if self.__DEBUG__:
                        print('Creating dependent ODE_%s' % varspec)
                    ode = ODE(self.get(varspec), independent=False)
                else:
                    if self.__DEBUG__:
                        print('Creating independent ODE_%s' % varspec)
                    ode = ODE(self.get(varspec), independent=True)
            else:
                if self.__DEBUG__:
                    print(
                        'Creating independent* ODE_%s (*assumed - no structural information available)'
                        % varspec
                    )
                ode = ODE(self.get(varspec), independent=True)
            mrow = self.stoichiometric_matrix.getRowsByName(varspec)
            for e in range(len(mrow[0])):
                if mrow[0, e] != 0.0:
                    print(
                        'Adding term: %s*%s'
                        % (mrow[0, e], self.stoichiometric_matrix.col[e])
                    )
                    ode.addReaction(
                        self.get(self.stoichiometric_matrix.col[e]), mrow[0, e]
                    )
            self.__setattr__(ode.name, ode)
            self.ODEs.append(ode)
            self.__setattr__(
                'xcode_' + ode.name, compile(ode.getGlobalFormula(), '<string>', 'exec')
            )

    def hasODEs(self):
        return MapList([o.name for o in self.ODEs])

    def evalODEs(self, odes):
        return [v() for v in odes]

    def evalXcode(self, ode):
        exec(self.__getattribute__('xcode_' + ode.name))
        return sdot

    def hasFunctions(self):
        return MapList([f.name for f in self.functions])

    def hasReactions(self):
        return MapList([r.name for r in self.reactions])

    def hasSpecies(self):
        return MapList([s.name for s in self.species])

    def hasFixedSpecies(self):
        return MapList([s.name for s in self.species if s.fixed])

    def hasVariableSpecies(self):
        return MapList([s.name for s in self.species if not s.fixed])

    def findReactionsThatIncludeAllSpecifiedReagents(self, *args):
        assert len(args) > 1, '\nNeed two or more species for this one!'
        setlist = [self.__getattribute__(s).isReagentOf().asSet() for s in args]
        isect = setlist[0]
        for s in setlist:
            isect.intersection_update(s)
        return MapList(isect)

    def hasGlobalParameters(self):
        return MapList(p.name for p in self.global_parameters)

    def hasAssignmentRules(self):
        return MapList(
            [
                ar.name
                for ar in self.global_parameters + self.species
                if hasattr(ar, 'type') == 'assignment'
            ]
        )

    #def hasAssignmentRules(self):
        #return MapList(
            #[
                #ar.name
                #for ar in self.global_parameters + self.species
                #if hasattr(ar, 'type') == 'rate'
            #]
        #)

    def hasEvents(self):
        return MapList(e.name for e in self.events)

    def hasCompartments(self):
        return MapList(c.name for c in self.compartments)

##  if __psyco_active__:
##  psyco.bind(NewCoreBase)
##  psyco.bind(NumberBase)
##  psyco.bind(Species)
##  psyco.bind(Parameter)
##  psyco.bind(AssignmentRule)
##  psyco.bind(Reaction)
##  psyco.bind(ODE)
##  psyco.bind(NewCore)

max_stars_repo_name: jmrohwer/pysces

max_issues_repo_name: jmrohwer/pysces

max_forks_repo_name: jmrohwer/pysces


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 22 12:11:55 2021

@author: giuseppeperonato, rboghetti

Original template from https://www.pyimagesearch.com/2019/01/28/keras-regression-and-cnns/
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import uuid
import json
import os
import glob
import sys
import shutil

from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, BatchNormalization, Flatten, Cropping2D, Dense, Concatenate, Dropout, Activation, UpSampling2D, ZeroPadding2D
from tensorflow.keras.models import Sequential, Model, load_model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

import prepareModel

def create_cnn(width, height, depth, filters=(16, 32, 64, 64), regress=False):
    # Original CNN from https://www.pyimagesearch.com/2019/01/28/keras-regression-and-cnns/
	# initialize the input shape and channel dimension, assuming
	# TensorFlow/channels-last ordering
	inputShape = (height, width, depth)
	chanDim = -1
	# define the model input
	inputs = Input(shape=inputShape)
	# loop over the number of filters
	for (i, f) in enumerate(filters):
		# if this is the first CONV layer then set the input
		# appropriately
		if i == 0:
			x = inputs
		# CONV => RELU => BN => POOL
		x = Conv2D(f, (3, 3), padding="same")(x)
		x = Activation("relu")(x)
		x = MaxPooling2D(pool_size=(2, 2))(x)
		x = BatchNormalization(axis=chanDim)(x)

	# flatten the volume, then FC => RELU => BN => DROPOUT
	x = Flatten()(x)

	x = Dense(64)(x)
	x = Activation("relu")(x)
	x = Dropout(0.4)(x)
	x = Dense(64)(x)
	x = Activation("relu")(x)
	x = Dropout(0.4)(x)
	# apply another FC layer, this one to match the number of nodes
	# coming out of the MLP
	x = Dense(8)(x)
	x = Activation("relu")(x)
	# check to see if the regression node should be added
	if regress:
		x = Dense(1, activation="linear")(x)
	# construct the CNN
	model = Model(inputs, x)
	# return the CNN
	return model

def create_context_aware_cnn(conv_filters = 256, hidden_neurons = 128,
                             branch_hidden_neurons = 256,
                             height = 40, width = 40, channels = 2):
    # Input layer
    input_layer = Input(shape=(height, width, channels), name= "tile_input")

    # Convolutions - full context
    xfull = Conv2D(conv_filters//4, (3, 3), activation='relu')(input_layer)
    xfull = MaxPooling2D((2, 2))(xfull)
    xfull = BatchNormalization(axis=-1)(xfull)
    xfull = Dropout(0.4)(xfull)
    xfull = Conv2D(conv_filters//2, (3, 3), activation='relu')(xfull)
    xfull = MaxPooling2D((2, 2))(xfull)
    xfull = BatchNormalization(axis=-1)(xfull)
    xfull = Dropout(0.4)(xfull)
    xfull = Conv2D(conv_filters//2, (3, 3), activation='relu')(xfull)
    xfull = MaxPooling2D((2, 2))(xfull)
    xfull = BatchNormalization(axis=-1)(xfull)
    xfull = Dropout(0.4)(xfull)
    xfull = Conv2D(conv_filters, (3, 3), activation='relu')(xfull)
    xfull = MaxPooling2D((2, 2))(xfull)
    xfull = BatchNormalization(axis=-1)(xfull)
    xfull = Dropout(0.4)(xfull)
    xfull = Conv2D(conv_filters, (3, 3), activation='relu')(xfull)
    xfull = BatchNormalization(axis=-1)(xfull)
    xfull = Dropout(0.4)(xfull)
    xfull = Flatten()(xfull)
    xfull = Dense(branch_hidden_neurons)(xfull)
    xfull = Dropout(0.4)(xfull)

    # Convolutions - image center
    s = int((height - 40)//2)
    xcenter = Cropping2D(cropping=s)(input_layer)
    xcenter = Conv2D(conv_filters//2, (3, 3), activation='relu')(xcenter)
    xcenter = MaxPooling2D((2, 2))(xcenter)
    xcenter = BatchNormalization(axis=-1)(xcenter)
    xcenter = Dropout(0.4)(xcenter)
    xcenter = Conv2D(conv_filters, (3, 3), activation='relu')(xcenter)
    xcenter = MaxPooling2D((2, 2))(xcenter)
    xcenter = BatchNormalization(axis=-1)(xcenter)
    xcenter = Dropout(0.4)(xcenter)
    xcenter = Conv2D(conv_filters, (3, 3), activation='relu')(xcenter)
    xcenter = MaxPooling2D((2, 2))(xcenter)
    xcenter = BatchNormalization(axis=-1)(xcenter)
    xcenter = Dropout(0.4)(xcenter)
    xcenter = Flatten()(xcenter)
    xcenter = Dense(branch_hidden_neurons)(xcenter)
    xcenter = Dropout(0.4)(xcenter)

    # Concatenation
    concatenate = Concatenate()([xfull, xcenter])

    # Hidden layer
    x = Dense(hidden_neurons, activation="relu", name="dense_conc")(concatenate)
    x = Dropout(0.4)(x)

    # Hidden layer
    x = Dense(4, activation="relu", name="dense_conc")(concatenate)

    # Output
    output_layer = Dense(1, activation="relu", name="output")(x)

    # Model
    model = Model(inputs=input_layer, outputs=output_layer)

    return model

if __name__ == "__main__":

    #######  INPUT
    if len(sys.argv) > 1:
        param_file = sys.argv[1]
    else:
        param_file = "parameters.json"

    with open(param_file,"r") as f:
        parameters = json.loads(f.read())

    image_size = int(parameters["tile_size"]/parameters["pixel_size"])
    #######

    # Creating results data
    # Check whether the results already exist
    Run = True
    for file in glob.glob('results/*/parameters.json'):
        with open(file,"r") as f:
            if parameters == json.loads(f.read()):
                overwrite = "n"
                overwrite = input('Overwrite? (type y to continue, something else to load the model)   ')
                if overwrite != "y":
                    Run = False
                this_uuid = os.path.basename(os.path.dirname(file))
                break
            else:
                this_uuid = str(uuid.uuid1())
    if len(glob.glob('results/*/parameters.json')) == 0:
        this_uuid = str(uuid.uuid1())

    # Create results folder and json parameters
    dest_folder = os.path.join("results","{}".format(this_uuid))
    if not os.path.exists("results"):
        os.mkdir("results")
    if not os.path.exists(dest_folder):
        os.mkdir(dest_folder)

    with open(os.path.join(dest_folder,"parameters.json"),"w") as f:
        f.write(json.dumps(parameters,indent=4))

    tiles, X_train, X_val, X_test = prepareModel.getX(parameters)

    # Select parameter
    y_train = tiles.loc[tiles.set=="train",parameters["predicting_variable"]].values
    y_val = tiles.loc[tiles.set=="validation",parameters["predicting_variable"]].values
    y_test = tiles.loc[tiles.set=="test",parameters["predicting_variable"]].values

    if Run:
        if parameters['predicting_on_tile100'] and parameters.get("context_aware"):
            model = create_context_aware_cnn(height = parameters["tile_size"] // parameters["pixel_size"],
                                          width = parameters["tile_size"] // parameters["pixel_size"])
        else:
            model = create_cnn(image_size, image_size, depth=X_train.shape[3], filters=(16, 32, 32, 64, 64), regress=True)

        opt = Adam(lr=1e-3, decay=1e-3 / 200)
        model.compile(loss="mean_absolute_percentage_error", optimizer=opt)

        checkpoint = ModelCheckpoint(
        filepath=os.path.join("results",this_uuid,"checkpoint"),
        monitor='val_loss',
        verbose=1,
        save_best_only=True,
        save_weights_only=False,
        mode='auto',
        save_freq='epoch',
        )

        early_stopping_monitor = EarlyStopping(
            monitor='val_loss',
            min_delta=0.1,
            patience=25,
            verbose=1,
            mode='auto',
            baseline=None,
            restore_best_weights=True
        )

        # train the model
        print("[INFO] training model...")
        model.fit(x=X_train, y=y_train,
            validation_data=(X_val, y_val),
            epochs=1000, batch_size=8, callbacks=[checkpoint, early_stopping_monitor])

        # Saving results
        model.save("results/{}/model".format(this_uuid))
        try:
            os.remove("results/{}/checkpoint".format(this_uuid))
        except:
            shutil.rmtree("results/{}/checkpoint".format(this_uuid))
        pd.DataFrame(model.history.history).sort_values("val_loss").to_csv(os.path.join("results",this_uuid,"losses.txt"),
                                                                           sep="\t",
                                                                           float_format='%.2f')
        tiles["set"].to_csv(os.path.join("results",this_uuid,"samples.csv"),index=True,header=True)

        # Plotting loss function
        pd.DataFrame(model.history.history).plot()
        plt.ylim([10,120])
        plt.show()

    else:
        model = load_model(os.path.join("results",this_uuid,"model"))

    print("[INFO] predicting  IDC...")
    preds = model.predict(X_test)

    print("Model")
    prepareModel.performanceStats(preds.flatten(),y_test)

ext: py

max_stars_repo_name: enermaps/heatlearn

max_issues_repo_name: enermaps/heatlearn

max_forks_repo_name: enermaps/heatlearn

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf8

"""
    Module that provides a class wrapper for source separation.

    :Example:

    >>> from spleeter.separator import Separator
    >>> separator = Separator('spleeter:2stems')
    >>> separator.separate(waveform, lambda instrument, data: ...)
    >>> separator.separate_to_file(...)
"""

import os
import logging

from time import time
from multiprocessing import Pool
from os.path import basename, join, splitext
import numpy as np
import tensorflow as tf
from librosa.core import stft, istft
from scipy.signal.windows import hann

from . import SpleeterError
from .audio.adapter import get_default_audio_adapter
from .audio.convertor import to_stereo
from .utils.configuration import load_configuration
from .utils.estimator import create_estimator, to_predictor, get_default_model_dir
from .model import EstimatorSpecBuilder, InputProviderFactory

__email__ = 'research@deezer.com'
__author__ = 'Deezer Research'
__license__ = 'MIT License'

logger = logging.getLogger("spleeter")

def get_backend(backend):
    assert backend in ["auto", "tensorflow", "librosa"]
    if backend == "auto":
        return "tensorflow" if tf.test.is_gpu_available() else "librosa"
    return backend

class Separator(object):
    """ A wrapper class for performing separation. """

    def __init__(self, params_descriptor, MWF=False, stft_backend="auto", multiprocess=True):
        """ Default constructor.

        :param params_descriptor: Descriptor for TF params to be used.
        :param MWF: (Optional) True if MWF should be used, False otherwise.
        """

        self._params = load_configuration(params_descriptor)
        self._sample_rate = self._params['sample_rate']
        self._MWF = MWF
        self._tf_graph = tf.Graph()
        self._predictor = None
        self._input_provider = None
        self._builder = None
        self._features = None
        self._session = None
        self._pool = Pool() if multiprocess else None
        self._tasks = []
        self._params["stft_backend"] = get_backend(stft_backend)

    def __del__(self):
        if self._session:
            self._session.close()

    def _get_predictor(self):
        """ Lazy loading access method for internal predictor instance.

        :returns: Predictor to use for source separation.
        """
        if self._predictor is None:
            estimator = create_estimator(self._params, self._MWF)
            self._predictor = to_predictor(estimator)
        return self._predictor

    def join(self, timeout=200):
        """ Wait for all pending tasks to be finished.

        :param timeout: (Optional) task waiting timeout.
        """
        while len(self._tasks) > 0:
            task = self._tasks.pop()
            task.get()
            task.wait(timeout=timeout)

    def _separate_tensorflow(self, waveform, audio_descriptor):
        """
        Performs source separation over the given waveform with tensorflow backend.

        :param waveform: Waveform to apply separation on.
        :returns: Separated waveforms.
        """
        if not waveform.shape[-1] == 2:
            waveform = to_stereo(waveform)
        predictor = self._get_predictor()
        prediction = predictor({
            'waveform': waveform,
            'audio_id': audio_descriptor})
        prediction.pop('audio_id')
        return prediction

    def _stft(self, data, inverse=False, length=None):
        """
        Single entrypoint for both stft and istft. This computes stft and istft with librosa on stereo data. The two
        channels are processed separately and are concatenated together in the result. The expected input formats are:
        (n_samples, 2) for stft and (T, F, 2) for istft.
        :param data: np.array with either the waveform or the complex spectrogram depending on the parameter inverse
        :param inverse: should a stft or an istft be computed.
        :return: Stereo data as numpy array for the transform. The channels are stored in the last dimension
        """
        assert not (inverse and length is None)
        data = np.asfortranarray(data)
        N = self._params["frame_length"]
        H = self._params["frame_step"]
        win = hann(N, sym=False)
        fstft = istft if inverse else stft
        win_len_arg = {"win_length": None, "length": length} if inverse else {"n_fft": N}
        n_channels = data.shape[-1]
        out = []
        for c in range(n_channels):
            d = data[:, :, c].T if inverse else data[:, c]
            s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg)
            s = np.expand_dims(s.T, 2-inverse)
            out.append(s)
        if len(out) == 1:
            return out[0]
        return np.concatenate(out, axis=2-inverse)

    def _get_input_provider(self):
        if self._input_provider is None:
            self._input_provider = InputProviderFactory.get(self._params)
        return self._input_provider

    def _get_features(self):
        if self._features is None:
            self._features = self._get_input_provider().get_input_dict_placeholders()
        return self._features

    def _get_builder(self):
        if self._builder is None:
            self._builder = EstimatorSpecBuilder(self._get_features(), self._params)
        return self._builder

    def _get_session(self):
        if self._session is None:
            saver = tf.train.Saver()
            latest_checkpoint = tf.train.latest_checkpoint(get_default_model_dir(self._params['model_dir']))
            self._session = tf.Session()
            saver.restore(self._session, latest_checkpoint)
        return self._session

    def _separate_librosa(self, waveform, audio_id):
        """
        Performs separation with librosa backend for STFT.
        """
        with self._tf_graph.as_default():
            out = {}
            features = self._get_features()

            # TODO: fix the logic, build sometimes return, sometimes set attribute
            outputs = self._get_builder().outputs
            stft = self._stft(waveform)
            if stft.shape[-1] == 1:
                stft = np.concatenate([stft, stft], axis=-1)
            elif stft.shape[-1] > 2:
                stft = stft[:, :2]

            sess = self._get_session()
            outputs = sess.run(outputs, feed_dict=self._get_input_provider().get_feed_dict(features, stft, audio_id))
            for inst in self._get_builder().instruments:
                out[inst] = self._stft(outputs[inst], inverse=True, length=waveform.shape[0])
            return out

    def separate(self, waveform, audio_descriptor=""):
        """ Performs separation on a waveform.

        :param waveform:            Waveform to be separated (as a numpy array)
        :param audio_descriptor:    (Optional) string describing the waveform (e.g. filename).
        """
        if self._params["stft_backend"] == "tensorflow":
            return self._separate_tensorflow(waveform, audio_descriptor)
        else:
            return self._separate_librosa(waveform, audio_descriptor)

    def separate_to_file(
            self, audio_descriptor, destination,
            audio_adapter=get_default_audio_adapter(),
            offset=0, duration=600., codec='wav', bitrate='128k',
            filename_format='{filename}/{instrument}.{codec}',
            synchronous=True):
        """ Performs source separation and export result to file using
        given audio adapter.

        Filename format should be a Python formattable string that could use
        following parameters : {instrument}, {filename} and {codec}.

        :param audio_descriptor:    Describe song to separate, used by audio
                                    adapter to retrieve and load audio data,
                                    in case of file based audio adapter, such
                                    descriptor would be a file path.
        :param destination:         Target directory to write output to.
        :param audio_adapter:       (Optional) Audio adapter to use for I/O.
        :param offset:              (Optional) Offset of loaded song.
        :param duration:            (Optional) Duration of loaded song (default:
                                    600s).
        :param codec:               (Optional) Export codec.
        :param bitrate:             (Optional) Export bitrate.
        :param filename_format:     (Optional) Filename format.
        :param synchronous:         (Optional) True is should by synchronous.
        """
        waveform, sample_rate = audio_adapter.load(
            audio_descriptor,
            offset=offset,
            duration=duration,
            sample_rate=self._sample_rate)
        sources = self.separate(waveform, audio_descriptor)
        self.save_to_file(  sources, audio_descriptor, destination,
                            filename_format, codec, audio_adapter,
                            bitrate, synchronous)

    def save_to_file(
            self, sources, audio_descriptor, destination,
            filename_format='{filename}/{instrument}.{codec}',
            codec='wav', audio_adapter=get_default_audio_adapter(),
            bitrate='128k', synchronous=True):
        """ export dictionary of sources to files.

        :param sources:             Dictionary of sources to be exported. The
                                    keys are the name of the instruments, and
                                    the values are Nx2 numpy arrays containing
                                    the corresponding intrument waveform, as
                                    returned by the separate method
        :param audio_descriptor:    Describe song to separate, used by audio
                                    adapter to retrieve and load audio data,
                                    in case of file based audio adapter, such
                                    descriptor would be a file path.
        :param destination:         Target directory to write output to.
        :param filename_format:     (Optional) Filename format.
        :param codec:               (Optional) Export codec.
        :param audio_adapter:       (Optional) Audio adapter to use for I/O.
        :param bitrate:             (Optional) Export bitrate.
        :param synchronous:         (Optional) True is should by synchronous.

        """

        filename = splitext(basename(audio_descriptor))[0]
        generated = []
        for instrument, data in sources.items():
            path = join(destination, filename_format.format(
                filename=filename,
                instrument=instrument,
                codec=codec))
            directory = os.path.dirname(path)
            if not os.path.exists(directory):
                os.makedirs(directory)
            if path in generated:
                raise SpleeterError((
                    f'Separated source path conflict : {path},'
                    'please check your filename format'))
            generated.append(path)
            if self._pool:
                task = self._pool.apply_async(audio_adapter.save, (
                    path,
                    data,
                    self._sample_rate,
                    codec,
                    bitrate))
                self._tasks.append(task)
            else:
                audio_adapter.save(path, data, self._sample_rate, codec, bitrate)
        if synchronous and self._pool:
            self.join()

max_stars_repo_name: gurjeet/spleeter-clone

max_issues_repo_name: gurjeet/spleeter-clone

max_forks_repo_name: gurjeet/spleeter-clone

reason: import numpy,from scipy


text:
import os
import numpy as np
import math
import cv2
from tools.convert_utils import build_voc_dirs, generate_xml

"""
gt 样例
矩形四点坐标的顺序： left-top, right-top, right-bottom, left-bottom
x1,y1,x2,y2,x3,y3,x4,y4,language,text

language 可能有：
- Arabic
- Latin
- Chinese
- Korean
- Japanese
- Bangla
- Symbols
- None

text 可能有：
- 图片上对应的文字
- 无法识别的文字用 ### 表示

只转换 Latin 和 Chinese 语言，忽略 ### 的部分
"""

DEBUG = False

img_dir = '/home/cwq/data/MLT2017/val'
gt_dir = '/home/cwq/data/MLT2017/val_gt'

out_dir = '/home/cwq/data/MLT2017/mlt_voc2'
IGNORE = '###\n'

# 允许 None，因为 None 会在后面检查 IGNORE 时被过滤掉
VALID_LANGUAGE = ['Latin', 'Chinese', 'None']

SCALE = 600
MAX_SCALE_LENGTH = 1200
step = 16

# resize 以后允许的最小高度
MIN_TEXT_HEIGHT = 5

# resize 以后允许的 anchor 数量
MIN_CONTINUE_ANCHORS = 0

# resize 以后允许text line区域最大的高宽比
MAX_HEIGHT_WIDTH_SCALE = 4

# 对文本行进行 split 后，允许最少的 anchor 数量
MIN_ANCHOR_COUNT = 10

global_k_is_none_count = 0

def parse_line(pnts, im_scale):
    """
    :param pnts:
        "x1,y1,x2,y2,x3,y3,x4,y4,language,text"
        矩形四点坐标的顺序： left-top, right-top, right-bottom, left-bottom
    :return:
        (x1,y1,x2,y2,x3,y3,x4,y4), language, text
    """
    splited_line = pnts.split(',')
    if len(splited_line) > 10:
        splited_line[-1] = ','.join(splited_line[10:])

    for i in range(8):
        splited_line[i] = int(int(splited_line[i]) * im_scale)

    pnts = (splited_line[0], splited_line[1],
            splited_line[2], splited_line[3],
            splited_line[4], splited_line[5],
            splited_line[6], splited_line[7])

    return pnts, splited_line[-2], splited_line[-1]

def get_ltrb(line):
    """
    :param line: (x1,y1,x2,y2,x3,y3,x4,y4)
    :return: (xmin, ymin, xmax, ymax)
    """
    xmin = min(line[0], line[6])
    ymin = min(line[1], line[3])
    xmax = max(line[2], line[4])
    ymax = max(line[5], line[7])

    return xmin, ymin, xmax, ymax

def get_img_scale(img, scale, max_scale):
    img_size = img.shape
    im_size_min = np.min(img_size[0:2])
    im_size_max = np.max(img_size[0:2])

    im_scale = float(scale) / float(im_size_min)
    if np.round(im_scale * im_size_max) > max_scale:
        im_scale = float(max_scale) / float(im_size_max)

    return im_scale

def clip_line(line, size):
    """
    :param line: (xmin, ymin, xmax, ymax)
    :param size: (height, width)
    :return: (xmin, ymin, xmax, ymax)
    """
    xmin, ymin, xmax, ymax = line

    if xmin < 0:
        xmin = 0
    if xmax > size[1] - 1:
        xmax = size[1] - 1
    if ymin < 0:
        ymin = 0
    if ymax > size[0] - 1:
        ymax = size[0] - 1

    return xmin, ymin, xmax, ymax

def split_text_line(line, step):
    """
    按照 Bounding box 对文字进行划分
    :param line: (xmin, ymin, xmax, ymax)
    :return: [(anchor_xmin, anchor_ymin, anchor_xmax, anchor_ymax)]
    """
    xmin, ymin, xmax, ymax = line
    width = xmax - xmin

    anchor_count = int(math.ceil(width / step))

    splited_lines = []

    for i in range(anchor_count):
        anchor_xmin = i * step + xmin
        anchor_ymin = ymin
        anchor_xmax = anchor_xmin + step - 1
        anchor_ymax = ymax

        splited_lines.append((anchor_xmin, anchor_ymin, anchor_xmax, anchor_ymax))

    return splited_lines

class Point:
    def __init__(self, x, y):
        self.x = int(x)
        self.y = int(y)

    def __str__(self):
        return "({}, {})".format(self.x, self.y)

class Line:
    def __init__(self, p0: Point, p1: Point):
        self.p0 = p0
        self.p1 = p1

        if p0.x - p1.x == 0:
            self.k = None
        else:
            self.k = (self.p0.y - self.p1.y) / (self.p0.x - self.p1.x)

        # f = ax+by+c = 0
        self.a = self.p0.y - self.p1.y
        self.b = self.p1.x - self.p0.x
        self.c = self.p0.x * self.p1.y - self.p1.x * self.p0.y

    def cross(self, line) -> Point:
        d = self.a * line.b - line.a * self.b
        if d == 0:
            return None

        x = (self.b * line.c - line.b * self.c) / d
        y = (line.a * self.c - self.a * line.c) / d

        return Point(x, y)

    def contain(self, p: Point) -> bool:
        if p is None:
            return False

        # 输入的点应该吃 cross(求出来的交点)
        # p 点是否落在 p0 和 p1 之间, 而不是延长线上
        if p.x > max(self.p1.x, self.p0.x):
            return False

        if p.x < min(self.p1.x, self.p0.x):
            return False

        if p.y > max(self.p1.y, self.p0.y):
            return False

        if p.y < min(self.p1.y, self.p0.y):
            return False

        return True

def get_clockwise(pnts):
    """
    :param pnts: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
    :return:
        [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
        left-top, right-top, right-bottom, left-bottom
    """
    out = []
    p = sorted(pnts, key=lambda x: x[1])
    if p[0][0] < p[1][0]:
        out.append(p[0])
        out.append(p[1])
    else:
        out.append(p[1])
        out.append(p[0])

    if p[2][0] > p[3][0]:
        out.append(p[2])
        out.append(p[3])
    else:
        out.append(p[3])
        out.append(p[2])

    return out

def draw_four_vectors(img, line, color=(0, 255, 0)):
    """
    :param line: (x1,y1,x2,y2,x3,y3,x4,y4)
        矩形四点坐标的顺序： left-top, right-top, right-bottom, left-bottom
    """
    img = cv2.line(img, (line[0], line[1]), (line[2], line[3]), color)
    img = cv2.line(img, (line[2], line[3]), (line[4], line[5]), color)
    img = cv2.line(img, (line[4], line[5]), (line[6], line[7]), color)
    img = cv2.line(img, (line[6], line[7]), (line[0], line[1]), color)
    return img

def draw_bounding_box(img, line, color=(255, 0, 0)):
    """
    :param line: (xmin, ymin, xmax, ymax)
    """
    img = cv2.line(img, (line[0], line[1]), (line[2], line[1]), color)
    img = cv2.line(img, (line[2], line[1]), (line[2], line[3]), color)
    img = cv2.line(img, (line[2], line[3]), (line[0], line[3]), color)
    img = cv2.line(img, (line[0], line[3]), (line[0], line[1]), color)
    return img

def split_text_line2(line, step, img=None):
    """
    按照 minAreaRect 对文本进行划分
    :param line:
        (x1,y1,x2,y2,x3,y3,x4,y4)
        矩形四点坐标的顺序： left-top, right-top, right-bottom, left-bottom
    :return: [(anchor_xmin, anchor_ymin, anchor_xmax, anchor_ymax)]
    """
    global global_k_is_none_count
    if DEBUG:
        img = draw_four_vectors(img, line)

    xmin, ymin, xmax, ymax = get_ltrb(line)
    width = xmax - xmin
    height = ymax - ymin

    if height > MAX_HEIGHT_WIDTH_SCALE * width:
        return []

    anchor_count = int(math.ceil(width / step))

    if DEBUG:
        img = draw_bounding_box(img, (xmin, ymin, xmax, ymax))

    rect = cv2.minAreaRect(np.asarray([[line[0], line[1]],
                                       [line[2], line[3]],
                                       [line[4], line[5]],
                                       [line[6], line[7]]]))
    # 获得最小 rotate rect 的四个角点
    box = cv2.boxPoints(rect)
    box = get_clockwise(box)

    if DEBUG:
        img = draw_four_vectors(img, (box[0][0], box[0][1],
                                      box[1][0], box[1][1],
                                      box[2][0], box[2][1],
                                      box[3][0], box[3][1]), color=(255, 55, 55))

    p1 = Point(box[0][0], box[0][1])
    p2 = Point(box[1][0], box[1][1])
    p3 = Point(box[2][0], box[2][1])
    p4 = Point(box[3][0], box[3][1])

    l1 = Line(p1, p2)
    l2 = Line(p2, p3)
    l3 = Line(p3, p4)
    l4 = Line(p4, p1)
    lines = [l1, l2, l3, l4]

    if l1.k is None:
        global_k_is_none_count += 1
        print("l1 K is None")
        print(p1)
        print(p2)
        print(p3)
        print(p4)
        return []

    splited_lines = []
    for i in range(anchor_count):
        anchor_xmin = i * step + xmin
        anchor_xmax = anchor_xmin + step - 1
        anchor_ymin = ymin
        anchor_ymax = ymax

        # 垂直于 X 轴的现
        left_line = Line(Point(anchor_xmin, 0), Point(anchor_xmin, height))
        right_line = Line(Point(anchor_xmax, 0), Point(anchor_xmax, height))

        left_cross_pnts = [left_line.cross(l) for l in lines]
        right_cross_pnts = [right_line.cross(l) for l in lines]

        if l1.k < 0:
            if l1.contain(right_cross_pnts[0]):
                anchor_ymin = right_cross_pnts[0].y

            if l4.contain(right_cross_pnts[3]):
                anchor_ymax = right_cross_pnts[3].y

            if l3.contain(left_cross_pnts[2]):
                anchor_ymax = left_cross_pnts[2].y

            if l2.contain(left_cross_pnts[1]):
                anchor_ymin = left_cross_pnts[1].y

        if l1.k > 0:
            if l4.contain(right_cross_pnts[3]):
                anchor_ymin = right_cross_pnts[3].y

            if l3.contain(right_cross_pnts[2]):
                anchor_ymax = right_cross_pnts[2].y

            if l1.contain(left_cross_pnts[0]):
                anchor_ymin = left_cross_pnts[0].y

            if l2.contain(left_cross_pnts[1]):
                anchor_ymax = left_cross_pnts[1].y

        if anchor_ymax - anchor_ymin <= MIN_TEXT_HEIGHT:
            continue

        if DEBUG:
            img = draw_bounding_box(img, (anchor_xmin, anchor_ymin, anchor_xmax, anchor_ymax), (0, 0, 255))
            cv2.imshow('test', img)
            cv2.waitKey()

        splited_lines.append((anchor_xmin, anchor_ymin, anchor_xmax, anchor_ymax))

    if DEBUG:
        cv2.imshow('test', img)
        cv2.waitKey()

    return splited_lines

def test():
    # p1 = Point(5, 1)
    # p2 = Point(1, 4)
    # l1 = Line(p2, p1)
    # print(l1.k)
    #
    # p1 = Point(5, 4)
    # p2 = Point(1, 1)
    # l1 = Line(p1, p2)
    # print(l1.k)

    pnts = [(40, 40), (140, 85), (140, 160), (50, 100)]
    img = np.zeros((200, 200, 3))
    a = cv2.minAreaRect(np.asarray(pnts))

    img = cv2.line(img, pnts[0], pnts[1], (0, 255, 0), thickness=1)
    img = cv2.line(img, pnts[1], pnts[2], (0, 255, 0), thickness=1)
    img = cv2.line(img, pnts[2], pnts[3], (0, 255, 0), thickness=1)
    img = cv2.line(img, pnts[3], pnts[0], (0, 255, 0), thickness=1)

    box = cv2.boxPoints(a)

    def tt(p):
        return (p[0], p[1])

    img = cv2.line(img, tt(box[0]), tt(box[1]), (255, 255, 0), thickness=1)
    img = cv2.line(img, tt(box[1]), tt(box[2]), (255, 255, 0), thickness=1)
    img = cv2.line(img, tt(box[2]), tt(box[3]), (255, 255, 0), thickness=1)
    img = cv2.line(img, tt(box[3]), tt(box[0]), (255, 255, 0), thickness=1)

    cv2.imshow('test', img.astype(np.uint8))
    cv2.waitKey(0)

def main():
    dest_label_dir, dest_img_dir, dest_set_dir = build_voc_dirs(out_dir)
    # training or val
    setname = os.path.basename(img_dir)

    img_names = os.listdir(img_dir)
    img_names.sort()

    trainval_set_file = open(os.path.join(dest_set_dir, setname + '.txt'), 'w')

    count = 0

    # img_names = img_names[0:205]
    for img_name in img_names:
        img_path = os.path.join(img_dir, img_name)

        _, basename = os.path.split(img_name)
        if basename.lower().split('.')[-1] not in ['jpg', 'png']:
            continue

        stem, ext = os.path.splitext(basename)
        gt_file = os.path.join(gt_dir, 'gt_' + stem + '.txt')

        img = cv2.imread(img_path)
        im_scale = get_img_scale(img, SCALE, MAX_SCALE_LENGTH)

        with open(gt_file, 'r') as f:
            lines = f.readlines()

        parsed_lines = [parse_line(line, im_scale) for line in lines]

        if len(parsed_lines) == 0 or \
                not all(x[1] in VALID_LANGUAGE for x in parsed_lines):  # 保证图片中只有 latin/chinese
            print("Skip image: %s" % img_name)
            continue

        parsed_lines = list(filter(lambda x: x[2] != IGNORE, parsed_lines))

        # 过滤掉高度过高或者过小的文本行
        tmp = []
        for l in parsed_lines:
            xmin, ymin, xmax, ymax = get_ltrb(l[0])
            h = ymax - ymin
            w = xmax - xmin
            if MIN_TEXT_HEIGHT < h < MAX_HEIGHT_WIDTH_SCALE * w:
                tmp.append(l)

        parsed_lines = tmp

        resize_img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_AREA)
        resize_img_size = resize_img.shape

        anchors = []
        for line in parsed_lines:
            temp_splited_lines = split_text_line2(line[0], step, resize_img)

            temp_splited_lines = [clip_line(l, resize_img_size) for l in temp_splited_lines]

            if len(temp_splited_lines) <= MIN_CONTINUE_ANCHORS:
                continue

            anchors.extend(temp_splited_lines)

        if len(anchors) <= MIN_ANCHOR_COUNT:
            print("Skip image: %s" % img_name)
            continue

        print("Process image: %s" % img_name)
        out_img_name = '{}_{}.jpg'.format(setname, stem)
        cv2.imwrite(os.path.join(dest_img_dir, out_img_name), resize_img)
        doc, objs = generate_xml(out_img_name, anchors, resize_img_size, 'icdar_mlt17_%s' % setname)

        xmlfile = os.path.join(dest_label_dir, '{}_{}.xml'.format(setname, stem))

        with open(xmlfile, 'w') as f:
            f.write(doc.toprettyxml(indent='	'))

        trainval_set_file.writelines("{}_{}\n".format(setname, stem))
        count += 1

    print("Converted image count: %d" % count)
    print("K is None: %d" % global_k_is_none_count)

if __name__ == "__main__":
    main()


text:
import numpy as np
from photonai_graph.GraphConstruction.graph_constructor import GraphConstructor

class GraphConstructorKNN(GraphConstructor):
    _estimator_type = "transformer"

    def __init__(self,
                 k_distance: int = 10,
                 one_hot_nodes: int = 0,
                 use_abs: int = 0,
                 fisher_transform: int = 0,
                 use_abs_fisher: int = 0,
                 zscore: int = 0,
                 use_abs_zscore: int = 0,
                 adjacency_axis: int = 0,
                 logs: str = None):
        """
        Transformer class for generating adjacency matrices
        from connectivity matrices. Selects the k nearest
        neighbours for each node based on pairwise distance.
        Recommended for functional connectivity.
        Adapted from Ktena et al, 2017.

        Parameters
        ----------
        k_distance: int
            the k nearest neighbours value, for the kNN algorithm.
        one_hot_nodes: int,default=0
            Whether to generate a one hot encoding of the nodes in the matrix (1) or not (0)
        use_abs: bool, default = False
            whether to convert all matrix values to absolute values before applying
            other transformations
        fisher_transform: int,default=0
            whether to perform a fisher transform of each matrix (1) or not (0)
        use_abs_fisher: int,default=0
            changes the values to absolute values. Is applied after fisher transform and before z-score transformation
        zscore: int,default=0
            performs a zscore transformation of the data. Applied after fisher transform and np_abs
        use_abs_zscore: int,default=0
            whether to use the absolute values of the z-score transformation or allow for negative values
        adjacency_axis: int,default=0
            position of the adjacency matrix, default being zero
        logs: str, default=None
            Path to the log data

        Example
        -------
        Use outside of a PHOTON pipeline

        ```python
        constructor = GraphConstructorKNN(k_distance=6,
                                          fisher_transform=1,
                                          use_abs=1)
        ```

        Or as part of a pipeline

        ```python
        my_pipe.add(PipelineElement('GraphConstructorKNN',
                                    hyperparameters={'k_distance': 6}))
        ```
       """
        super(GraphConstructorKNN, self).__init__(one_hot_nodes=one_hot_nodes,
                                                  use_abs=use_abs,
                                                  fisher_transform=fisher_transform,
                                                  use_abs_fisher=use_abs_fisher,
                                                  zscore=zscore,
                                                  use_abs_zscore=use_abs_zscore,
                                                  adjacency_axis=adjacency_axis,
                                                  logs=logs)
        self.k_distance = k_distance

    def get_knn(self, adjacency: np.ndarray) -> np.ndarray:
        """Returns kNN matrices"""
        adjacency = np.squeeze(adjacency)
        adjacency_list = []
        for i in range(adjacency.shape[0]):
            # generate adjacency matrix
            d, idx = self.distance_sklearn_metrics(adjacency[i, :, :], k=self.k_distance, metric='euclidean')
            k_adjacency = self.adjacency(d, idx).astype(np.float32)

            # turn adjacency into numpy matrix for concatenation
            k_adjacency = k_adjacency.toarray()
            adjacency_list.append(k_adjacency)

        # X = X[..., None] + adjacency[None, None, :] #use broadcasting to speed up computation
        adjacency_knn = np.asarray(adjacency_list)
        adjacency_knn = adjacency_knn[:, :, :, np.newaxis]

        return adjacency_knn

    def transform(self, X: np.ndarray) -> np.ndarray:
        """Transform matrices based on k nearest neighbours"""
        adj, feat = self.get_mtrx(X)
        # do preparatory matrix transformations
        adj = self.prep_mtrx(adj)
        # threshold matrix
        adj = self.get_knn(adj)
        # get feature matrix
        X_transformed = self.get_features(adj, feat)

        return X_transformed


text:
"""
Dishonest Casino Example
------------------------

We'll use the ubiquitous dishonest casino example to demonstrate
how to train a Hidden Markov Model (HMM) on somewhat realistic
test data (e.g. http://www.mcb111.org/w06/durbin_book.pdf
Chapter 3).

In this example, we suspect that a casino is trading out a fair
die (singular or dice) for a loaded die. We want to figure out
1) when the loaded die was used (i.e. the most likely path) 2) how
often the loaded die is used (i.e. the transition probabilities) and
3) the probabilities for each outcome of a roll for the loaded die
(i.e. the emission probabilities).
"""

# %%
# First, import necessary modules and functions.

import numpy as np
import matplotlib.pyplot as plt

from hmmlearn import hmm

# %%
# Now, let's act as the casino and exchange a fair die for a loaded one
# and generate a series of rolls that someone at the casino would
# observe.

# make our generative model with two components, a fair die and a
# loaded die
gen_model = hmm.MultinomialHMM(n_components=2, random_state=99)

# the first state is the fair die so let's start there so no one
# catches on rigth away
gen_model.startprob_ = np.array([1.0, 0.0])

# now let's say that we sneak the loaded die in:
# here, we have a 95% chance to continue using the fair die and a 5%
# chance to switch to the loaded die
# when we enter the loaded die state, we have a 90% chance of staying
# in that state and a 10% chance of leaving
gen_model.transmat_ = np.array([[0.95, 0.05],
                                [0.1, 0.9]])

# now let's set the emission means:
# the first state is a fair die with equal probabilities and the
# second is loaded by being biased toward rolling a six
gen_model.emissionprob_ = \
    np.array([[1 / 6, 1 / 6, 1 / 6, 1 / 6, 1 / 6, 1 / 6],
              [1 / 10, 1 / 10, 1 / 10, 1 / 10, 1 / 10, 1 / 2]])

# simulate the loaded dice rolls
rolls, gen_states = gen_model.sample(30000)

# plot states over time, let's just look at the first rolls for clarity
fig, ax = plt.subplots()
ax.plot(gen_states[:500])
ax.set_title('States over time')
ax.set_xlabel('Time (# of rolls)')
ax.set_ylabel('State')
fig.show()

# plot rolls for the fair and loaded states
fig, ax = plt.subplots()
ax.hist(rolls[gen_states == 0], label='fair', alpha=0.5,
        bins=np.arange(7) - 0.5, density=True)
ax.hist(rolls[gen_states == 1], label='loaded', alpha=0.5,
        bins=np.arange(7) - 0.5, density=True)
ax.set_title('Roll probabilities by state')
ax.set_xlabel('Count')
ax.set_ylabel('Roll')
ax.legend()
fig.show()

# %%
# Now, let's see if we can recover our hidden states, transmission matrix
# and emission probabilities.

# split our data into training and validation sets (50/50 split)
X_train = rolls[:rolls.shape[0] // 2]
X_validate = rolls[rolls.shape[0] // 2:]

# check optimal score
gen_score = gen_model.score(X_validate)

best_score = best_model = None
n_fits = 50
np.random.seed(13)
for idx in range(n_fits):
    model = hmm.MultinomialHMM(
        n_components=2, random_state=idx,
        init_params='se')  # don't init transition, set it below
    # we need to initialize with random transition matrix probabilities
    # because the default is an even likelihood transition
    # we know transitions are rare (otherwise the casino would get caught!)
    # so let's have an uniform random prior distribution on our transitions
    # between 0 and 0.25 to train our model faster (than uniform random 0 to 1)
    fair_to_loaded = np.random.random() / 4
    loaded_to_fair = np.random.random() / 4
    model.transmat_ = np.array([[1 - fair_to_loaded, fair_to_loaded],
                                [loaded_to_fair, 1 - loaded_to_fair]])
    model.fit(X_train)
    score = model.score(X_validate)
    print(f'Model #{idx}\tScore: {score}')
    idx += 1
    if best_score is None or score > best_score:
        best_model = model
        best_score = score

print(f'Generated score: {gen_score}\nBest score:      {best_score}')

# use the Viterbi algorithm to predict the most likely sequence of states
# given the model
states = best_model.predict(rolls)

# plot our recovered states compared to generated (aim 1)
fig, ax = plt.subplots()
ax.plot(gen_states[:500], label='generated')
ax.plot(states[:500] + 1.5, label='recovered')
ax.set_yticks([])
ax.set_title('States compared to generated')
ax.set_xlabel('Time (# rolls)')
ax.set_xlabel('State')
ax.legend()
fig.show()

# %%
# Let's check our learned transition probabilities and see if they match.

print(f'Transmission Matrix Generated:\n{gen_model.transmat_.round(3)}\n\n'
      f'Transmission Matrix Recovered:\n{best_model.transmat_.round(3)}\n\n')

# %%
# Finally, let's see if we can tell how the die is loaded.

print(f'Emission Matrix Generated:\n{gen_model.emissionprob_.round(3)}\n\n'
      f'Emission Matrix Recovered:\n{best_model.emissionprob_.round(3)}\n\n')

# %%
# In this case, we were able to get very good estimates of the transition and
# emission matrices, but decoding the states was imperfect. That's because
# the decoding algorithm is greedy and picks the most likely series of states
# which isn't necessarily what happens in real life. Even so, our model could
# tell us when to watch for the loaded die and we'd have a better chance at
# catching them red-handed.

max_stars_repo_name: mvaliadis/hmmlearn

max_issues_repo_name: mvaliadis/hmmlearn

max_forks_repo_name: mvaliadis/hmmlearn