text:
#!/usr/bin/env python
#------------------------------
"""
:py:class:`DCMethods` is a set of utilities for direct operations with calibration data
=======================================================================================

Usage::

    # Import
    import psana
    import PSCalib.DCMethods as dcm

    # Example of parameters
    dsname = 'exp=cxif5315:run=129'
    # or:
    dsname = '/reg/g/psdm/detector/data_test/xtc/cxif5315-e545-r0169-s00-c00.xtc'
    ds = psana.DataSource(dsname)
    env=ds.env()
    evt=ds.events().next()

    src      = 'Cspad.' # 'Epix100a.', etc
    ctype    = gu.PIXEL_MASK # | gu.PEDESTALS | gu.PIXEL_STATUS, etc.
    vers     = None # or e.g. 5
    calibdir = None # or e.g. './calib'
    nda      = np.zeros((32,185,388))
    pred     = 'CxiDs2.0:Cspad.0'
    succ     = 'CxiDs2.0:Cspad.0'
    range    = '1474587520-end'

    par      = evt # psana.Event | float - tsec event time
    parts    = env # psana.Env | psana.Event | float - tsec event time

    # Methods with dynamically-reconstructed calib file name
    dcm.add_constants(nda, par, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, vers=None, pred=None, succ=None, cmt=None, verb=False)
    dcm.print_content(env, src='Epix100a.', calibdir=None)
    nda = dcm.get_constants(par, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, vers=None, verb=False)
    dcm.delete_version(evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, vers=None, cmt=None, verb=False)
    dcm.delete_range  (evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, range=None, cmt=None, verb=False)
    dcm.delete_ctype  (evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, cmt=None, verb=False)

    # Methods using fname
    dcm.add_constants_to_file(data, fname, parts, env, ctype=gu.PIXEL_MASK, vers=None, pred=None, succ=None, cmt=None, verb=False)
    dcm.print_content_from_file(fname)
    nda = dcm.get_constants_from_file(fname, parts, ctype=gu.PIXEL_MASK, vers=None, verb=False)
    dcm.delete_version_from_file(fname, parts, ctype=gu.PIXEL_MASK, vers=None, cmt=None, verb=False)
    dcm.delete_range_from_file  (fname, ctype=gu.PIXEL_MASK, range=None, cmt=None, verb=False)
    dcm.delete_ctype_from_file  (fname, ctype=gu.PIXEL_MASK, cmt=None, verb=False)

Methods
    * :meth:`add_constants`,
    * :meth:`add_constants_to_file`,
    * :meth:`print_content`,
    * :meth:`print_content_from_file`,
    * :meth:`get_constants`,
    * :meth:`get_constants_from_file`,
    * :meth:`delete_version`,
    * :meth:`delete_version_from_file`,
    * :meth:`delete_range`,
    * :meth:`delete_range_from_file`,
    * :meth:`delete_ctype`
    * :meth:`delete_ctype_from_file`

Classes:
    * :class:`DCStore`
    * :class:`DCType`
    * :class:`DCRange`
    * :class:`DCVersion`
    * :class:`DCBase`
    * :class:`DCInterface`
    * :class:`DCUtils`
    * :class:`DCDetectorId`
    * :class:`DCConfigParameters`
    * :class:`DCFileName`
    * :class:`DCLogger`
    * :class:`DCMethods`
    * :class:`DCEmail`

This software was developed for the SIT project.
If you use all or part of it, please give an appropriate acknowledgment.

Created: 2016-09-23 by Mikhail Dubrovin
"""
#------------------------------

import sys
import os
#import numpy as np
#import h5py
from time import time #localtime, strftime

from PSCalib.DCLogger import log
from PSCalib.DCFileName import DCFileName
import PSCalib.DCUtils as dcu
from PSCalib.DCStore import DCStore

sp = dcu.sp
gu = dcu.gu

#------------------------------

def is_good_fname(fname, verb=False) :
    """Checks the hdf5 file name validity, returns True or False.

    Parameters

    - fname : str - full path to the file
    - verb : bool - verbosity, default=False - do not print any message

    Returns

    True/False - for existing or not file
    """
    metname = sys._getframe().f_code.co_name

    if fname is None :
        if verb : print '%s WARNING: file name is None' % metname
        return False

    if not isinstance(fname, str) :
        if verb : print '%s WARNING: parameter fname is not str' % metname
        return False

    if not os.path.exists(fname) :
        if verb : print '%s WARNING: file %s does not exist' % (metname, fname)
        return False

    return True

#------------------------------
#------------------------------
#------------------------------
#------------------------------

def add_constants_to_file(data, fname, par, env=None, ctype=gu.PIXEL_MASK,\
                          vers=None,\
                          pred=None,\
                          succ=None,\
                          cmt=None,\
                          verb=False) :
    """Adds specified numpy array to the hdf5 file.

    Parameters

    - data : numpy.array or str - array or string of calibration constants/data to save in file
    - fname: full path to the hdf5 file
    - par  : psana.Event | psana.Env | float - tsec event time
    - env  : psana.Env -> is used to get exp=env.experiment() for comments etc.
    - ctype: gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - vers : int - calibration version
    - pred : str - predecessor name
    - succ : str - successor name
    - cmt  : str - comment saved as a history record within DCRange
    - verb : bool - verbosity, default=False - do not print any message

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]

    if verb : print '  %s.add_constants_to_file  ctype: %s  vers: %s'%\
                    (metname, str_ctype, vers)

    if fname is None :
        if verb : print 'WARNING: file name is not defined - return None'
        return

    tsec_ev = dcu.par_to_tsec(par)

    cs = DCStore(fname)

    if verb : log.setPrintBits(0377) # 0377

    if os.path.exists(fname) : cs.load()

    cs.set_tscfile(tsec=tsec_ev)
    cs.set_predecessor(pred)
    cs.set_successor(succ)

    msg = 'detname:%s predecessor:%s successor:%s ts:%.0f' %\
          (cs.detname(), cs.predecessor(), cs.successor(), cs.tscfile())
    #cs.add_history_record('%s for %s' % (metname, msg))
    #cs.add_par('par-1-in-DCStore', 1)

    ct = cs.add_ctype(str_ctype, cmt='')
    if ct is None : return
    #ct.add_history_record('%s - add DCType %s' % (metname,str_ctype))
    #ct.add_par('par-1-in-DCType', 1)

    exp = env.experiment() if env is not None else ''
    exp = exp if exp != '' else 'unknown'
    runnum = par.run() if not isinstance(par,float) else 0
    msg = 'exp=%s:run=%s' % (exp, str(runnum))
    cr = ct.add_range(tsec_ev, end=None, cmt=msg)
    if cr is None : return
    cr.add_par('experiment', exp)
    cr.add_par('run', str(runnum))
    #cr.set_vnum_def(vnum=None)

    msg = '' if cmt is None else cmt
    cv = cr.add_version(vnum=vers, tsec_prod=time(), nda=data, cmt=msg)
    if cv is None : return
    #v = cr.vnum_last() if vers is None else vers
    #rec='%s vers=%d: %s' % (metname, v, cmt if cmt is not None else 'no-comments')
    #cr.add_history_record(rec)

    if verb :
        print 50*'_','\nIn %s:' % metname
        cs.print_obj()

    if verb : log.setPrintBits(02) # 0377

    cs.save()

#------------------------------

def add_constants(data, par, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None,\
                  vers=None,\
                  pred=None,\
                  succ=None,\
                  cmt=None,\
                  verb=False) :
    """Adds specified numpy array to the hdf5 file.

    Parameters

    - data : numpy.array or str - array or string of calibration constants/data to save in file
    - env  : psana.Env -> full detector name for psana.Source -> hdf5 file name
    - par  : psana.Event | float time -> event time
    - src  : str - source short/full name, alias or full -> hdf5 file name
    - ctype: gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - calibdir : str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)
    - vers : int - calibration version
    - pred : str - predecessor name
    - succ : str - successor name
    - cmt  : str - comment saved as a history record within DCRange
    - verb : bool - verbosity, default=False - do not prnt any message

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]

    if verb : print '  %s.add_constants  src: %s\n  ctype: %s\n  vers: %s\n  calibdir:%s'%\
                    (metname, src, str_ctype, vers, calibdir)

    ofn = DCFileName(env, src, calibdir)
    if verb : ofn.print_attrs()
    ofn.make_path_to_calib_file() # depth=2, mode=0775)

    fname = ofn.calib_file_path()

    add_constants_to_file(data, fname, par, env, ctype, vers, pred, succ, cmt, verb)

#------------------------------

def get_constants_from_file(fname, par, ctype=gu.PIXEL_MASK, vers=None, verb=False) :
    """Returns specified array of calibration constants.

    Parameters

    - fname : full path to the hdf5 file
    - par   : psana.Event | psana.Env | float - tsec event time
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - vers  : int - calibration version

    Returns

    - np.array - specified array of calibration constants

    See :py:class:`DCMethods`
    """
    if not is_good_fname(fname, verb) : return None

    cs = DCStore(fname)
    cs.load()
    if verb :
        print 50*'_','\nDCStore.print_obj()'
        cs.print_obj()

    str_ctype = gu.dic_calib_type_to_name[ctype]
    ct = cs.ctypeobj(str_ctype)
    if ct is None : return None
    #ct.print_obj()

    tsec = dcu.par_to_tsec(par)
    #print 'XXX: get DCRange object for time = %.3f' % tsec
    cr = ct.range_for_tsec(tsec)
    #cr = ct.range_for_evt(evt)

    if cr is None : return None
    #cr.print_obj()

    cv = cr.version(vnum=vers)
    if cv is None : return None
    #cv.print_obj()

    return cv.data()

#------------------------------

def get_constants(par, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, vers=None, verb=False) :
    """
    Returns specified array of calibration constants.

    Parameters

    - par: psana.Event | float - tsec event time
    - env: psana.Env - to get full detector name for psana.Source
    - src: str - source short/full name, alias or full
    - ctype: gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - calibdir: str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)
    - vers: int - calibration version

    Return

    numpy.array - array of calibratiopn constatnts.

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.get_constants  src: %s\n  ctype: %s\n  vers: %s\n  calibdir:%s'%\
                    (metname, src, str_ctype, vers, calibdir)

    ofn = DCFileName(env, src, calibdir)
    if verb : ofn.print_attrs()

    fname = ofn.calib_file_path()

    return get_constants_from_file(fname, par, ctype, vers, verb)

#------------------------------

def delete_version_from_file(fname, par, ctype=gu.PIXEL_MASK, vers=None, cmt=None, verb=False) :
    """Delete specified version from calibration constants.

    Parameters

    - fname : full path to the hdf5 file
    - par   : psana.Event | psana.Env | float - tsec event time
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - vers  : int - calibration version
    - cmt   : str - comment
    - verb  : bool - verbousity

    See :py:class:`DCMethods`
    """

    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_version_from_file:  ctype: %s  vers: %s'%\
                    (metname, str_ctype, vers)

    if not is_good_fname(fname, verb) : return None

    cs = DCStore(fname)
    cs.load()

    ct = cs.ctypeobj(str_ctype)
    if ct is None : return None
    #ct.print_obj()

    tsec = dcu.par_to_tsec(par)
    cr = ct.range_for_tsec(tsec)
    if cr is None : return None

    v = vers if vers is not None else cr.vnum_last()

    vdel = cr.mark_version(vnum=vers, cmt=cmt)

    if verb : log.setPrintBits(02) # 0377

    cs.save()

    if verb :
        print 50*'_','\nDCStore.print_obj() after delete version %s' % str(vdel)
        cs.print_obj()

    return vdel

#------------------------------

def delete_version(evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, vers=None, cmt=None, verb=False) :
    """Delete specified version from calibration constants.

    Parameters

    - evt : psana.Event -> event time
    - env : psana.Env -> full detector name for psana.Source
    - src : str - source short/full name, alias or full
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - calibdir : str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)
    - vers : int - calibration version
    - cmt  : str - comment
    - verb : bool - verbousity

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_version:\n  src: %s\n  ctype: %s\n  vers: %s\n  calibdir:%s'%\
                    (metname, src, str_ctype, vers, calibdir)

    ofn = DCFileName(env, src, calibdir)
    if verb : ofn.print_attrs()

    fname = ofn.calib_file_path()

    return delete_version_from_file(fname, evt, ctype, vers, cmt, verb)

#------------------------------

def delete_range_from_file(fname, ctype=gu.PIXEL_MASK, range=None, cmt=None, verb=False) :
    """Delete specified time range from calibration constants.

    Parameters

    - fname : full path to the hdf5 file
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - range : str - range, e.g. '1474587520-end'
    - cmt   : str - comment
    - verb  : bool - verbousity

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_range_from_file  ctype: %s  range: %s'%\
                    (metname, str_ctype, range)

    if not is_good_fname(fname, verb) : return None

    cs = DCStore(fname)
    cs.load()
    #cs.print_obj()

    ct = cs.ctypeobj(str_ctype)
    if ct is None : return None

    rdel = ct.mark_range_for_key(range, cmt=cmt)
    if rdel is None : return None

    if verb : log.setPrintBits(02) # 0377

    cs.save()

    if verb :
        print 50*'_','\nDCStore.print_obj() after delete range %s' % rdel
        cs.print_obj()

    return rdel

#------------------------------

def delete_range(evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, range=None, cmt=None, verb=False) :
    """Delete specified time range from calibration constants.

    Parameters

    - evt : psana.Event -> event time
    - env : psana.Env -> full detector name for psana.Source
    - src : str - source short/full name, alias or full
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - calibdir : str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)
    - range : str - range, e.g. '1474587520-end'
    - cmt   : str - comment
    - verb  : bool - verbousity

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_range  src: %s\n  ctype: %s\n  range: %s\n  calibdir:%s'%\
                    (metname, src, str_ctype, range, calibdir)

    ofn = DCFileName(env, src, calibdir)
    if verb : ofn.print_attrs()

    fname = ofn.calib_file_path()

    return delete_range_from_file(fname, ctype, range, cmt, verb)

#------------------------------

def delete_ctype_from_file(fname, ctype=gu.PIXEL_MASK, cmt=None, verb=False) :
    """Delete specified ctype from calibration constants.

    Parameters

    - fname : full path to the hdf5 file
    - ctype : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - cmt   : str - comment
    - verb  : bool - verbousity

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_ctype  ctype: %s  range: %s'%\
                    (metname, str_ctype, range)

    if not is_good_fname(fname, verb) : return None

    cs = DCStore(fname)
    cs.load()
    #cs.print_obj()

    tdel = cs.mark_ctype(str_ctype, cmt=cmt)
    if tdel is None : return None

    if verb : log.setPrintBits(02) # 0377

    cs.save()

    if verb :
        print 50*'_','\nDCStore.print_obj() after delete ctype %s' % tdel
        cs.print_obj()

    return tdel

#------------------------------

def delete_ctype(evt, env, src='Epix100a.', ctype=gu.PIXEL_MASK, calibdir=None, cmt=None, verb=False) :
    """Delete specified ctype from calibration constants.

    Parameters

    - evt      : psana.Event -> event time
    - env      : psana.Env -> full detector name for psana.Source
    - src      : str - source short/full name, alias or full
    - ctype    : gu.CTYPE - enumerated calibration type, e.g.: gu.PIXEL_MASK
    - calibdir : str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)
    - cmt      : str - comment
    - verb     : bool - verbousity

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    str_ctype = gu.dic_calib_type_to_name[ctype]
    if verb : print '  %s.delete_ctype  src: %s\n  ctype: %s\n  range: %s\n  calibdir:%s'%\
                    (metname, src, str_ctype, range, calibdir)

    ofn = DCFileName(env, src, calibdir)
    if verb : ofn.print_attrs()

    fname = ofn.calib_file_path()

    return delete_ctype_from_file(fname, ctype, cmt, verb)

#------------------------------

def print_content_from_file(fname) :
    """Prints content of the file.

    Parameters

    - fname : str - full path to the file

    See :py:class:`DCMethods`
    """
    metname = sys._getframe().f_code.co_name

    if not is_good_fname(fname, True) : return

    cs = DCStore(fname)

    t0_sec = time()
    cs.load()
    print 'File content loading time (sec) = %.6f' % (time()-t0_sec)

    print 50*'_','\nDCStore.print_obj()'
    cs.print_obj()

#------------------------------

def print_content(env, src='Epix100a.', calibdir=None) :
    """Defines the file name and prints file content.

    Parameters

    - env : psana.Env -> full detector name for psana.Source
    - src : str - source short/full name, alias or full
    - calibdir : str - fallback path to calib dir (if xtc file is copied - calib and experiment name are lost)

    See :py:class:`DCMethods`
    """
    #metname = sys._getframe().f_code.co_name

    ofn = DCFileName(env, src, calibdir)
    fname = ofn.calib_file_path()
    ofn.print_attrs()

    print_content_from_file(fname)

#------------------------------
#------------------------------
#------------------------------
#------------------------------

def get_constants_v0(*par, **opt) :
    ofn = DCFileName(par[0], opt['src'])

#------------------------------

def test_add_constants() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    import numpy as np

    vers = None
    #nda  = np.zeros((32,185,388), dtype=np.float32)
    nda  = np.zeros((1000,1000), dtype=np.float32)
    pred = None
    succ = None
    cmt  = 'my comment: %s' % metname

    add_constants(nda, gevt, genv, gsrc, gctype, gcalibdir, vers, pred, succ, cmt, gverb)
    print '%s: constants added nda.shape=%s' % (metname, nda.shape)

#------------------------------

def test_add_constants_to_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    import numpy as np

    vers = None
    #data  = np.zeros((32,185,388), dtype=np.float32)
    data  = 11*np.ones((500,500), dtype=np.float32)
    pred = None
    succ = None
    cmt  = 'my comment: %s' % metname
    par = gevt # 1474587525.3 # gevt

    add_constants_to_file(data, gfname, par, genv, gctype, vers, pred, succ, cmt, gverb)

    print '%s: constants added data.shape=%s' % (metname, data.shape)

#------------------------------

def test_add_text() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    import numpy as np

    data = 'Now we are going to save\n this little piece of text'
    vers = None
    pred = 'Predecessor_name'
    succ = 'Successor_name'
    ctype = gu.GEOMETRY
    cmt  = 'my comment: %s' % metname

    add_constants(data, gevt, genv, gsrc, ctype, gcalibdir, vers, pred, succ, cmt, gverb)
    print '%s: text is added to the file' % (metname)

#------------------------------

def test_add_text_to_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    import numpy as np

    data = 'Now we are going to save\n this little piece of text'
    vers = None
    pred = None
    succ = None
    ctype = gu.GEOMETRY
    cmt  = 'my comment: %s' % metname
    par = gevt

    add_constants_to_file(data, gfname, par, genv, gctype, vers, pred, succ, cmt, gverb)

    print '%s: text is added to the file\n  %s' % (metname, gfname)

#------------------------------

def test_add_constants_two() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    import numpy as np

    vers   = None
    pred   = None
    succ   = None
    ctype1 = gu.PIXEL_STATUS
    ctype2 = gu.PIXEL_MASK
    #nda   = np.zeros((32,185,388), dtype=np.float32)
    nda1   = np.ones((1000,1000), dtype=np.float32)
    nda2   = 2 * nda1
    cmt1   = 'my comment 1: %s' % metname
    cmt2   = 'my comment 2: %s' % metname

    add_constants(nda1, gevt, genv, gsrc, ctype1, gcalibdir, vers, pred, succ, cmt1, gverb)
    add_constants(nda2, gevt, genv, gsrc, ctype2, gcalibdir, vers, pred, succ, cmt2, gverb)
    print '%s: constants added ctype1=%s nda1.shape=%s' % (metname, str(ctype1), nda1.shape)
    print '%s: constants added ctype2=%s nda2.shape=%s' % (metname, str(ctype2), nda2.shape)

#------------------------------

def test_get_constants() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    vers = None
    data = get_constants(gevt, genv, gsrc, gctype, gcalibdir, vers, gverb)

    print '%s: retrieved constants for vers %s' % (metname, str(vers))
    print 'data:\n', data
    if isinstance(data, np.ndarray) : print 'data.shape=%s' % (str(data.shape))

#------------------------------

def test_get_constants_from_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    vers = None
    data = get_constants_from_file(gfname, gevt, gctype, vers, gverb)

    print '%s: retrieved constants for vers %s' % (metname, str(vers))
    print 'data:\n', data
    if isinstance(data, np.ndarray) : print 'data.shape=%s' % (str(data.shape))

#------------------------------

def test_delete_version() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    vers = None # for default - last version
    cmt  = 'my comment: %s' % metname
    vdel = delete_version(gevt, genv, gsrc, gctype, gcalibdir, vers, cmt, gverb)
    print '%s: deleted version %s' % (metname, str(vdel))

#------------------------------

def test_delete_version_from_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    vers = None # for default - last version
    cmt  = 'my comment: %s' % metname
    vdel = delete_version_from_file(gfname, gevt, gctype, vers, cmt, gverb)

    print '%s: deleted version %s' % (metname, str(vdel))

#------------------------------

def test_delete_range() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    range = '1474587520-end'
    cmt  = 'my comment: %s' % metname
    rdel = delete_range(gevt, genv, gsrc, gctype, gcalibdir, range, cmt, gverb)
    print '%s: deleted range %s' % (metname, str(rdel))

#------------------------------

def test_delete_range_from_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    range = '1474587520-end'
    cmt  = 'my comment: %s' % metname
    rdel = delete_range_from_file(gfname, gctype, range, cmt, gverb)
    print '%s: deleted range %s' % (metname, str(rdel))

#------------------------------

def test_delete_ctype() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    cmt  = 'my comment: %s' % metname
    tdel = delete_ctype(gevt, genv, gsrc, gctype, gcalibdir, cmt, gverb)
    print '%s: deleted ctype %s' % (metname, str(tdel))

#------------------------------

def test_delete_ctype_from_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname

    cmt  = 'my comment: %s' % metname
    tdel = delete_ctype_from_file(gfname, gctype, cmt, gverb)

    print '%s: deleted ctype %s' % (metname, str(tdel))

#------------------------------

def test_print_content() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname
    print_content(genv, gsrc, gcalibdir)
    print '%s is completed' % (metname)

#------------------------------

def test_print_content_from_file() :
    metname = sys._getframe().f_code.co_name
    print 20*'_', '\n%s' % metname
    print_content_from_file(gfname)
    print '%s is completed' % (metname)

#------------------------------

def test_misc() :
    print 20*'_', '\n%s' % sys._getframe().f_code.co_name

    import PSCalib.DCUtils as dcu

    print 20*'_', '\n%s:' % sys._getframe().f_code.co_name
    print 'get_enviroment(USER) : %s' % dcu.get_enviroment()
    print 'get_login()          : %s' % dcu.get_login()
    print 'get_hostname()       : %s' % dcu.get_hostname()
    print 'get_cwd()            : %s' % dcu.get_cwd()

#------------------------------

def set_parameters() :

    import psana
    import numpy as np; global np

    global genv, gevt, gsrc, gctype, gcalibdir, gverb, gfname

    #dsname  = 'exp=cxif5315:run=129'
    #dsname   = '/reg/g/psdm/detector/data_test/xtc/cxif5315-e545-r0169-s00-c00.xtc'
    #gsrc      = 'Cspad.'

    #dsname = 'exp=mfxn8316:run=11'
    dsname = '/reg/g/psdm/detector/data_test/types/0021-MfxEndstation.0-Epix100a.0.xtc'
    gsrc      = ':Epix100a.'

    gcalibdir = './calib'
    gctype    = gu.PIXEL_STATUS # gu.PIXEL_MASK, gu.PEDESTALS, etc.
    gverb     = True
    #gverb     = False

    gfname = './calib/epix100a/epix100a-3925999616-0996663297-3791650826-1232098304-0953206283-2655595777-0520093719.h5'

    ds = psana.DataSource(dsname)
    genv=ds.env()
    gevt=ds.events().next()

#------------------------------

def do_test() :
    from time import time

    set_parameters()

    #log.setPrintBits(0377)

    tname = sys.argv[1] if len(sys.argv) > 1 else '0'
    print 50*'_', '\nTest %s:' % tname
    t0_sec = time()
    if   tname == '0' : test_misc();

    elif tname ==  '1': test_add_constants()
    elif tname ==  '2': test_add_constants_two()
    elif tname ==  '3': test_add_text()
    elif tname ==  '4': test_print_content()
    elif tname ==  '5': test_get_constants()
    elif tname ==  '6': test_delete_version()
    elif tname ==  '7': test_delete_range()
    elif tname ==  '8': test_delete_ctype()

    elif tname == '11': test_add_constants_to_file()
    elif tname == '12': test_add_constants_to_file()
    elif tname == '13': test_add_text_to_file()
    elif tname == '14': test_print_content_from_file()
    elif tname == '15': test_get_constants_from_file()
    elif tname == '16': test_delete_version_from_file()
    elif tname == '17': test_delete_range_from_file()
    elif tname == '18': test_delete_ctype_from_file()

    else : print 'Not-recognized test name: %s' % tname
    msg = 'End of test %s, consumed time (sec) = %.6f' % (tname, time()-t0_sec)
    sys.exit(msg)

#------------------------------

if __name__ == "__main__" :
    do_test()

#------------------------------


text:
import tflib as lib

import numpy as np
import tensorflow as tf

def Batchnorm(name, input, is_training=None, momentum=0.9, epsilon=2e-5, in_place_update=True):
    if in_place_update:
        result = tf.contrib.layers.batch_norm(input,
                                              decay=momentum,
                                              center=True,
                                              scale=True,
                                              epsilon=epsilon,
                                              updates_collections=None,
                                              is_training=is_training,
                                              scope=name)
        # print(result.get_shape())
        return result
    else:
        result = tf.contrib.layers.batch_norm(input,
                                              decay=momentum,
                                              center=True,
                                              scale=True,
                                              epsilon=epsilon,
                                              is_training=is_training,
                                              scope=name)
        # print(result.get_shape())
        return result

ext: py

reason: import numpy


text:
from __future__ import division
import sys, time
sys.path.append('../../')
from datetime import timedelta  # To convert elapsed time to hh:mm:ss format
from copy import deepcopy
from tools.pyomoSolverCreator import pyomoSolverCreator
from tools.globalVariables import *
from tools.userError import userError
from tools.core.model import model
from tools.core.compound import compound
from tools.core.reaction import reaction
from copy import deepcopy
from pyomo.environ import *
from pyomo.opt import *

# The following lines change the temporary directory for pyomo
from pyutilib.services import TempfileManager
TempfileManager.tempdir = pyomo_tmp_dir

# Increse the recursion limit, otherwise deepcopy will complain
sys.setrecursionlimit(10000)

class fcf(object):
    """
    Performs flux coupling analysis
    """

    def __init__(self,model, blocked_rxns = [], build_new_optModel = True, optimization_solver = default_optim_solver, results_filename = '', simulation_conditions = '', warmstart = False, warnings = True, stdout_msgs = True):
        """
        INPUTS:
        -------
                        model: An instance of class model containing the information
                               about the metabolic model
                 blocked_rxns: A list of reactions that are blocked under the given uptake and
                               aeration conditions
           build_new_optModel: If True, a new pyomo optimization model is created
          optimization_solver: Name of the LP solver to be used to solve the LP. Current
                               allowable choices are cplex and gurobi
             results_filename: A string containing the name of the file to save the results in
                             . If an empty string is provided the results are not saved to a
                               file
        simulation_conditions: A string describing simulation conditions
                    warmstart: If True, warmstart is used
                     warnings: Can be True or False shwoing whether the warnings should be writtten to the
                               screen or not. The default is True
                  stdout_msgs: Prints a summary of the FBA and a number of other messages in the output if 'on.
                               Eligible values are True and False and the default is True

        OUTPUTS:
        --------

        Ali R. Zomorrodi - Segre Lab @ Boston University
        Last updated: 04-04-2016
        """
        # model
        self.model = model

        # Always blocked reactions
        self.blocked_rxns = blocked_rxns

        # build_new_optModel
        self.build_new_optModel = build_new_optModel

        # optimization solver
        self.optimization_solver = optimization_solver

        # results_filename
        self.results_filename = results_filename

        # simulation_conditions
        self.simulation_conditions = simulation_conditions

        # warmstart
        self.warmstart = warmstart

        # warnings and stdout_msgs
        self.warnings = warnings
        self.stdout_msgs = stdout_msgs

        # Create the optimization solver
        self.optSolver = pyomoSolverCreator(self.optimization_solver)

    def __setattr__(self,attr_name,attr_value):
        """
        Redefines funciton __setattr__
        INPUTS:
        -------
         attr_name: Attribute name
        attr_value: Attribute value
        """
        # model
        if attr_name == 'model' and not isinstance(attr_value,model):
            raise TypeError('model must be instance of class model. Type of input object: {}'.format(type(attr_value)))

        # Always blocked reactions
        if attr_value == 'blocked_rxns' and not isinstance(simulation_conditions,list):
            raise TypeError('blocked_rxns must be a list of strings')
        elif attr_value == 'blocked_rxns' and len([k for k in attr_value if not isinstance(k,str)]) > 0:
            raise TypeError('blocked_rxns must be a list of strings. Non-string objects observed in the list')

        # Solver name
        if attr_name == 'optimization_solver' and attr_value.lower() not in ['cplex','gurobi']:
            raise userError('Invalid solver name (eligible choices are cplex and gurobi)\n')

        # Simulation conditions name
        if attr_name == 'simulation_conditions' and not isinstance(attr_value,str):
            raise userError('simulation_conditions must be a striing')

        # build_new_optModel
        if attr_name == 'build_new_optModel' and not isinstance(attr_value,bool):
            raise TypeError("'build_new_optModel' must be either True or False")

        # Warnings and messages in the standard output
        if attr_name == 'stdout_msgs' and not isinstance(attr_value,bool):
            raise TypeError("'stdout_msgs' must be either True or False")
        if attr_name == 'warnings' and not isinstance(attr_value,bool):
            raise TypeError("'warnings' must be either True or False")

        self.__dict__[attr_name] = attr_value

    def split_rev_rxns(self):
        """
        Decomposes all reversible and exchange reactions in the model into irreversible forward and backward reactions
        """
        # First perform FBA for the original model
        for rxn in self.model.reactions:
            rxn.objective_coefficient = 0
        self.model.biomass_reaction.objective_coefficient = 1
        self.model.fba(stdout_msgs = False)
        max_biomass_model = self.model.fba_model.solution['objective_value']

        # map between reaction ids in model_split and those in the original model for reversible reactions
        # keys are ids of forward and backward reactions in model_split and values are their corresponding reaction id
        # in the original model
        self.model_split_model_map = {}

        # List of backward reactions
        backward_rxns = []

        self.model_split = deepcopy(self.model)

        # Assing False to both is_reversible_forward and is_reversible_backward to irreversible reactions
        for rxn in [r for r in self.model_split.reactions if not r.is_reversible]:
            rxn.is_reversible_forward = False
            rxn.is_reversible_backward = False

        for rxn in [r for r in self.model_split.reactions if r.is_reversible]:
            rxn_id = rxn.id

            # Forward reaction
            rxn.is_reversible= False
            rxn.id = rxn_id + '_f'
            rxn.is_reversible_forward = True
            rxn.is_reversible_backward = False
            self.model_split_model_map[rxn_id + '_f'] = rxn_id

            # backward reaction
            rxn_b = reaction(id = rxn_id + '_b', stoichiometry = dict([(c,-rxn.stoichiometry[c]) for c in rxn.compounds]), is_reversible = False, is_exchange = rxn.is_exchange, is_transport = rxn.is_transport, name = rxn.name, name_aliases = rxn.name_aliases, KEGG_id = rxn.KEGG_id, ModelSEED_id = rxn.ModelSEED_id, BiGG_id = rxn.BiGG_id, EC_numbers = rxn.EC_numbers, subsystem = rxn.subsystem, pathways = rxn.pathways, genes = rxn.genes, gene_reaction_rule = rxn.gene_reaction_rule, objective_coefficient = rxn.objective_coefficient, flux = rxn.flux, store_flux = rxn.store_flux, flux_bounds = rxn.flux_bounds, deltaG = rxn.deltaG, deltaG_uncertainty = rxn.deltaG_uncertainty, deltaG_range = rxn.deltaG_range, kinetics = rxn.kinetics, kinetic_compounds = rxn.kinetic_compounds, confidence_level = rxn.confidence_level, notes = rxn.notes, warnings = rxn.warnings)
            rxn_b.id = rxn_id + '_b'
            rxn_b.is_reversible_forward = False
            rxn_b.is_reversible_backward = True
            self.model_split_model_map[rxn_id + '_b'] = rxn_id

            for cpd in rxn_b.compounds:
                cpd.product_reactions = list(set(cpd.product_reactions))
                if rxn_b.stoichiometry[cpd] < 0:
                    cpd.set_reactant_reactions(list(cpd.reactant_reactions) + [rxn_b])
                elif rxn_b.stoichiometry[cpd] > 0:
                    cpd.set_product_reactions(list(cpd.product_reactions) + [rxn_b])
                cpd.set_reactions(list(cpd.reactions) + [rxn_b])

            backward_rxns.append(rxn_b)

        self.model_split.add_reactions(backward_rxns)
        self.model_split.assign_props()
        self.model_split.validate()

        # Reset flux bounds
        self.model_split.reset_flux_bounds()

        # Set upper bound on export reactions (exchange reactions in forward direction)
        for rxn in [r for r in self.model_split.reactions if r.is_exchange and r.is_reversible_forward]:
            rxn.flux_bounds[1] = 1000

        # Reset the uptake conditions
        for rxn in [r for r in self.model_split.reactions if r.is_exchange and r.is_reversible_backward]:
            rxn.flux_bounds = [0,0]
        for rxn in [r for r in self.model.reactions if r.is_exchange and r.flux_bounds[0] < 0]:
            self.model_split.reactions_by_id[rxn.id + '_b'].flux_bounds[1] = -rxn.flux_bounds[0]
        for rxn in [r for r in self.model.reactions if r.is_exchange and r.flux_bounds[0] > 0]:
            self.model_split.reactions_by_id[rxn.id + '_f'].flux_bounds[0] = rxn.flux_bounds[0]
        for rxn in [r for r in self.model.reactions if r.is_exchange and r.flux_bounds[1] not in [0,1000]]:
            self.model_split.reactions_by_id[rxn.id + '_f'].flux_bounds[1] = rxn.flux_bounds[1]

        # Non-exchange reactions
        for rxn in [r for r in self.model.reactions if (not r.is_exchange) and r.flux_bounds[0] not in [0,-1000]]:
            # Reversible reaction
            if rxn.is_reversible:
                if rxn.flux_bounds[0] > 0:
                    self.model_split.reactions_by_id[rxn.id + '_f'].flux_bounds[0] = rxn.flux_bounds[0]
                elif rxn.flux_bounds[0] < 0: # Here, LB in the reversible rxn corresponds to UB in the irreversible backward rxn
                    self.model_split.reactions_by_id[rxn.id + '_b'].flux_bounds[1] = -rxn.flux_bounds[0]
            # irreversible reaction
            else:
               self.model_split.reactions_by_id[rxn.id].flux_bounds[0] = rxn.flux_bounds[0]

        for rxn in [r for r in self.model.reactions if (not r.is_exchange) and r.flux_bounds[1] not in [0,1000]]:
            # Reversible reaction
            if rxn.is_reversible:
                if rxn.flux_bounds[1] > 0:
                    self.model_split.reactions_by_id[rxn.id + '_f'].flux_bounds[1] = rxn.flux_bounds[1]
                elif rxn.flux_bounds[1] < 0: # Here, UB in the reversible rxn corresponds to LB in the irreversible backward rxn
                    self.model_split.reactions_by_id[rxn.id + '_b'].flux_bounds[0] = -rxn.flux_bounds[1]
            # irreversible reaction
            else:
               self.model_split.reactions_by_id[rxn.id].flux_bounds[1] = rxn.flux_bounds[1]

        # Regulation reactions
        for rxn in [r for r in self.model.reactions if r.flux_bounds == [0,0]]:
            # Reversible reaction
            if rxn.is_reversible and (not rxn.is_exchange):
                    self.model_split.reactions_by_id[rxn.id + '_f'].flux_bounds = [0,0]
                    self.model_split.reactions_by_id[rxn.id + '_b'].flux_bounds = [0,0]
            else:
               self.model_split.reactions_by_id[rxn.id].flux_bounds = [0,0]

        #-- Make sure there are no reactions with a negative LB in irrev_model ---
        if len([r for r in self.model_split.reactions if r.flux_bounds[0] < 0 or r.flux_bounds[1] < 0]) > 0:
            raise userError('The following reactions have a negative LB or UB fpr the flux in the model with dcomposed reversible and exchange reactions: {}'.format([(r.id, r.flux_bounds) for r in self.model_split if r.flux_bounds[0] < 0 or r.flux_bounds[1] < 0]))

        #--- pirform FBA for irrev_model ---
        for rxn in self.model_split.reactions:
            rxn.objective_coefficient = 0
        self.model_split.biomass_reaction.objective_coefficient = 1
        self.model_split.fba(stdout_msgs = False)
        max_biomass_irrev_model = self.model_split.fba_model.solution['objective_value']

        if max_biomass_model != max_biomass_irrev_model:
            raise userError('max biomass for the original model ({}) is not equal to max biomass for the model with splitd reversible and exchange reactions ({})'.format(max_biomass_model, max_biomass_irrev_model))

    def create_ignored_list(self):
        """
        Creates the list of reactions that must be ignore in flux couplign analysis
        """
        self._ignore = self.blocked_rxns

        for rxn in self.model_split.reactions:
            if rxn.is_exchange:
                self._ignore.append(rxn.id)

    def build_optModel(self):
        """
        This optModel creates a pyomo model for FBA optModel
        """
        #--- Create a pyomo model optModel ---
        optModel = ConcreteModel()

        #--- Define sets ---
        # Set of compounds
        optModel.I = Set(initialize = self.model_split.compounds_by_id.keys())

        # Set of rxns
        optModel.J = Set(initialize = self.model_split.reactions_by_id.keys())

        #--- Define the optModel variables ---
        optModel.v = Var(optModel.J, domain = NonNegativeReals, bounds = lambda optModel, j: self.model_split.reactions_by_id[j].flux_bounds)

        optModel.t = Var(domain = NonNegativeReals)

        #--- Defiine the objective function and constraints ----
        # Objective function
        optModel.objectiveFunc = Objective(rule = lambda optModel: optModel.v[self._curr_j1], sense = maximize)

        # Mass balance
        optModel.massBalance_const = Constraint(optModel.I, rule = lambda optModel, i: sum(j.stoichiometry[self.model_split.compounds_by_id[i]]*optModel.v[j.id] for j in self.model_split.compounds_by_id[i].reactions) == 0)

        # LB(j)*t <= v(j) <= UB(j)*t
        optModel.vLB_const = Constraint([j for j in optModel.J if self.model_split.reactions_by_id[j].flux_bounds[0] != 0], rule = lambda optModel, j: optModel.v[j] >= self.model_split.reactions_by_id[j].flux_bounds[0]*optModel.t)
        optModel.vUB_const = Constraint(optModel.J, rule = lambda optModel, j: optModel.v[j] <= self.model_split.reactions_by_id[j].flux_bounds[1]*optModel.t)

        # v(j2) = 1 for the current reaction j2
        optModel.denom_const = Constraint(rule = lambda optModel: optModel.v[self._curr_j2] == 1)

        self.optModel = optModel

    def run(self):
        """
        This method runs the FBA tool.
        """
        # Total processing and wall time required to create the pyomo model, solve it and store the results
        start_total_pt = time.clock()
        start_total_wt = time.time()

        # Split reversible reactions into forward and backward
        print 'Split reversible rxns in the model ...'
        self.split_rev_rxns()
        print 'Done with spliting reversible rxns in the model ...'

        # Create the list of rections that must be ignored
        self.create_ignored_list()

        #---- Creating and instantiating the optModel ----
        start_create_optModel_pt = time.clock()
        start_create_optModel_wt = time.time()
        if self.build_new_optModel:
            # Assign two preliminary values to self._curr_j1 and self_curr_j2 so the model can be created.
            # These two will get updated inside the loops
            self._curr_j1 = self.model_split.reactions[0].id
            self._curr_j2 = self.model_split.reactions[1].id

            self.build_optModel()

            elapsed_create_optModel_pt = str(timedelta(seconds = time.clock() - start_create_optModel_pt))
            elapsed_create_optModel_wt = str(timedelta(seconds = time.time() - start_create_optModel_wt))

            if self.warmstart:
                self.optimization_solver == 'gurobi_ampl'
                self.optSolver = pyomoSolverCreator(self.optimization_solver)

                self.optSolver.options['presolve'] = 0  # Turn off presolve
                self.optSolver.options['basis'] = 3     # make sure gurobi_ampl returns and uses the sstatus suffix (solution basis info)
                self.optSolver.options['method'] = 0    # use primal simplex
                # According to the Gurobi documentation, sstatus suffix
                # values can be interpreted as:
                # 1: basic,   2: superbasic, 3: nonbasic <= (normally =) lower bound, 4: nonbasic >= (normally =) upper bound
                # 5: nonbasic at equal lower and upper bounds,  6: nonbasic between bounds
                # Ref: https://software.sandia.gov/trac/pyomo/browser/pyomo/trunk/examples/pyomo/suffixes/gurobi_ampl_basis.py?rev=11265&order=name
                self.optModel.sstatus = Suffix(direction=Suffix.IMPORT_EXPORT,datatype=Suffix.INT)
                self.optModel.dual = Suffix(direction=Suffix.IMPORT_EXPORT)

        else:
            elapsed_create_optModel_pt = 0
            elapsed_create_optModel_wt = 0

        # list of reactions ids in the model
        rxn_ids = [r.id for r in self.model_split.reactions[:100]]

        # A parameter showing if a current reactions has already been coupled with another reaction
        already_coupled = dict([(r,False) for r in rxn_ids])

        # A list of dictionaries containing the details of directionally, partiually and fully coupled rxns
        self.directionally_coupled_split = []
        self.partially_coupled_split = {}
        self.fully_coupled_split = []

        # A dictioaary with keys being a rxns id representative of a fully coupled sets and values being a list of reactions
        # ids fully coupled to it in the model_split and in the original model, e.g.,  {j1:[j5,j10], j20:[j40,j70,8100]}
        self.fully_coupled_reps_split = {}
        self.fully_coupled_reps = {}

        # By using rxn_ids[:len(rxn_ids)-1] we essential ask to not consider the last element of
        # of rxn_ids in the outer loop
        for j1 in [jj for jj in rxn_ids[:len(rxn_ids)-1] if (jj not in self._ignore and not already_coupled[jj])]:

            coupled_with_j1 = []

            for j2 in [jj for jj in rxn_ids if jj not in self._ignore and rxn_ids.index(jj) > rxn_ids.index(j1)]:

                self._curr_j1 = j1
                self._curr_j2 = j2

                #-- Maximize ---
                self.optModel.del_component('objectiveFunc')
                self.optModel.objectiveFunc = Objective(rule = lambda optModel: optModel.v[self._curr_j1], sense = maximize)
                #self.optModel.preprocess() # (no londer needed in versions 4.X or higher of pyomo)

                optSoln = self.optSolver.solve(self.optModel,tee = False, warmstart = True)

                if str(optSoln.solver.termination_condition).lower() in ['optimal','globallyoptimal','locallyoptimal']:
                    #self.optModel.load(optSoln) # (no londer needed in versions 4.X or higher of pyomo)
                    Rmax =  self.optModel.objectiveFunc()
                elif str(optSoln.solver.termination_condition).lower() == 'unbounded':
                    Rmax = float('inf')
                else:
                    Rmax = None

                #-- Minimize ---
                self.maximize = False
                self.optModel.del_component('objectiveFunc')
                self.optModel.objectiveFunc = Objective(rule = lambda optModel: optModel.v[self._curr_j1], sense = minimize)
                #self.optModel.preprocess() # (no londer needed in versions 4.X or higher of pyomo)

                optSoln = self.optSolver.solve(self.optModel,tee = False, warmstart = True)

                if str(optSoln.solver.termination_condition).lower() in ['optimal','globallyoptimal','locallyoptimal']:
                    #self.optModel.load(optSoln) (no londer needed in versions 4.X or higher of pyomo)
                    Rmin =  self.optModel.objectiveFunc()
                else:
                    Rmin = None

                #--- Check for coupling relations ---
                if Rmin != None and Rmax != None and Rmax != float('inf') and Rmin == 0 and Rmax > 0:
                    self.directionally_coupled_split.append({'j1':j1,'j2':j2,'direction':'j1 --> j2', 'Rmin':Rmin, 'Rmax':Rmax})

                elif Rmin != None and Rmax != None and Rmin > 0 and Rmax == float('inf'):
                    self.directionally_coupled_split.append({'j1':j1,'j2':j2,'direction':'j2 --> j1', 'Rmin':Rmin, 'Rmax':Rmax})

                elif Rmin != None and Rmax != None and Rmax != float('inf') and Rmin > 0 and Rmax > 0 and Rmax - Rmin >= 1e-6:
                    self.partially_coupled_split.append({'j1':j1,'j2':j2,'relation':'j1 <--> j2', 'Rmin':Rmin, 'Rmax':Rmax})
                    already_coupled[j2] = True

                elif Rmin != None and Rmax != None and Rmax != float('inf') and (Rmin > 0 and Rmax > 0) and Rmax - Rmin < 1e-6:
                    self.fully_coupled_split.append({'j1':j1,'j2':j2,'relation':'j1 <==> j2', 'Rmin':Rmin, 'Rmax':Rmax})
                    already_coupled[j2] = True
                    coupled_with_j1.append(j2)

            if len(coupled_with_j1) > 0:
                self.fully_coupled_reps_split[j1] = coupled_with_j1

        # Fully couples sets for the original model
        for j1 in self.fully_coupled_reps_split.keys():
            # Irreversible reactions fully coupled to this reaction
            irrev_rxns = [j2 for j2 in self.fully_coupled_reps_split[j1] if (not self.model_split.reactions_by_id[j1].is_reversible) and (not self.model_split.reactions_by_id[j1].is_exchange)]

            # Coupled reversible forward and backward reactions
            rev_f_rxns = [j2 for j2 in self.fully_coupled_reps_split[j1] if self.model_split.reactions_by_id[j1].is_reversible_forward]
            rev_b_rxns = [j2 for j2 in self.fully_coupled_reps_split[j1] if self.model_split.reactions_by_id[j1].is_reversible_backward]

            # If the representative reaction is irreversible
            if not self.model_split.reactions_by_id[j1].is_reversible and not self.model_split.reactions_by_id[j1].is_exchange:
                # All irreversible reactions are coupled to j1
                coupled_irrev_rxns = irrev_rxns

                # This reaction is fully coupled to a reversible reaction only only if it fully coupled to its forward and
                # backward reaction (see
                coupled_rev_rxns = []
                for rf in rev_f_rxns:
                     if self.model_split_model_map[rf]  + '_b' in rev_b_rxns: # If its backward rxn is in rev_b_rxns too
                        coupled_rev_rxns.append(self.model_split_model_map[rf])

                if len(coupled_irrev_rxns + coupled_rev_rxns) > 0:
                    self.fully_coupled_reps[j1] = coupled_irrev_rxns + coupled_rev_rxns

            # If the representative reaction is reversible forward or reversible backward
            if self.model_split.reactions_by_id[j1].is_reversible_forward or self.model_split.reactions_by_id[j1].is_reversible_backward:
                if (self.model_split.reactions_by_id[j1].is_reversible_forward and self.model_split_model_map[j1] + '_b' in rev_b_rxns) or (self.model_split.reactions_by_id[j1].is_reversible_backward and self.model_split_model_map[j1] + '_f' in rev_b_rxns):
                    coupled_irrev_rxns = irrev_rxns

                    coupled_rev_rxns = []
                    for rf in rev_f_rxns:
                        if self.model_split_model_map[rf]  + '_b' in rev_b_rxns: # If its backward rxn is in rev_b_rxns too
                            coupled_rev_rxns.append(self.model_split_model_map[rf])

                        if len(coupled_irrev_rxns + coupled_rev_rxns) > 0:
                            self.fully_coupled_reps[j1] = coupled_irrev_rxns + coupled_rev_rxns

                else: # otherwise check if the rest of reversible reactions are coupled together
                    coupled_rev_rxns = []
                    for rf in rev_f_rxns:
                        if self.model_split_model_map[rf]  + '_b' in rev_b_rxns: # If its backward rxn is in rev_b_rxns too
                            coupled_rev_rxns.append(self.model_split_model_map[rf])

                    if len(coupled_irrev_rxns + coupled_rev_rxns) > 1:
                        # Choose the first element as the representative
                        rep = coupled_irrev_rxns + coupled_rev_rxns[0]
                        self.fully_coupled_reps[rep] = (coupled_irrev_rxns + coupled_rev_rxns)[1:]

        if self.results_filename != '':
            # Save the results into a temporary file in case there are errors in save_results_toFile
            with open(self.results_filename + '.tmp','w') as f:
                f.write('directionally_coupled_split = {}\n'.format(self.directionally_coupled_split))
                f.write('\nparially_coupled_split = {}\n'.format(self.partially_coupled_split))
                f.write('\nfully_coupled_split = {}\n'.format(self.fully_coupled_split))
                f.write('\nfully_coupled_reps_split = {}\n'.format(self.fully_coupled_reps_split))
            print '\nResults were temporarily saved to {}\n'.format(self.results_filename + '.tmp')

            # Save results into file
            self.save_results_toFile()

        # Time required to perform FBA
        elapsed_total_pt = str(timedelta(seconds = time.clock() - start_total_pt))
        elapsed_total_wt = str(timedelta(seconds = time.time() - start_total_wt))

        if self.stdout_msgs:
            print 'FCF took (hh:mm:ss) {}/{} of processing/walltime\n'.format(elapsed_total_pt, elapsed_total_wt)

        return self.directionally_coupled_split, self.partially_coupled_split, self.fully_coupled_reps_split, self.fully_coupled_reps_split, self.fully_coupled_reps

    def save_results_toFile(self):
        """
        Save the results into a file
        """
        with open(self.results_filename, 'w') as f:

            # Directionally coupled
            f.write('directionally_coupled_split = [\n')
            for dcoupled in self.directionally_coupled_split:
                f.write("{{'j1':'{}', 'j2':'{}', 'direction':'{}', 'Rmin':{}, 'Rmax':{}}},\n".format(dcoupled['j1'],dcoupled['j2'], dcoupled['direction'], dcoupled['Rmin'], dcoupled['Rmax']))
            f.write(']\n\n')

            # Partially coupled
            f.write('partially_coupled_split = [\n')
            for pcoupled in self.partially_coupled_split:
                f.write("{{'j1':'{}', 'j2':'{}', 'Rmin':{}, 'Rmax':{}}},\n".format(pcoupled['j1'],pcoupled['j2'], pcoupled['Rmin'], dcoupled['Rmax']))
            f.write(']\n\n')

            # Fully coupled
            f.write('fully_coupled_split = [\n')
            for fcoupled in self.fully_coupled_split:
                f.write("{{'j1':'{}', 'j2':'{}', 'Rmin':{}, 'Rmax':{}}},\n".format(fcoupled['j1'],fcoupled['j2'], fcoupled['Rmin'], dcoupled['Rmax']))
            f.write(']\n\n')

            # Fully coupled sets
            f.write('fully_coupled_reps_split = {\n')
            for k in self.fully_coupled_reps_split.keys():
                f.write("'{}':{},".format(k,self.fully_coupled_reps_split[k]))
            f.write('}\n\n')

            # Fully coupled sets for the original model
            f.write('fully_coupled_reps = {\n')
            for k in self.fully_coupled_reps.keys():
                f.write("'{}':{},".format(k,self.fully_coupled_reps[k]))
            f.write('}')

def find_coupling_with_revrxn(self, j1_f_j2_coupling, j1_b_j2_coupling):
    """
    Assume j1 and j2 are irreversible and reversible reactions in the original model. Then given the coupling relation
    between j1 and j2_f and j1 and j2_b, it identifies the coupling relation between (i,j)

    1. Given the coupling relations for (j1_f,j2_f) & (j1_f,j2_b) find the coupling relations between (j1_f,j2)
    2. Given the coupling relations for (j1_b,j2_f) & (j1_b,j2_b) find the coupling relations between (j1_b,j2)
    2. Given the coupling relations for (j1_f,j2) & (j1_b,j2) find the coupling relations between (j1,j2)

    According to the definition of this function an irreversible reaction is fully coupled to a reversible reaciton
    only if it is fully coupled to both its forward and backard reactions. Similarly, two reversible reactions are
    fully coupled only if the reverse and forward reactions of each reversible reaction are fully coupled to those of the
    other reactions

    Reference: PMID: 21676263 (See Algorithm 2 [CouplingRelationRecompute] in Additionaly File 1)
    """
    if j1_f_j2_coupling == '<==>' and j1_b_j2_coupling == '<-->':
        coupling_relation = '<-->'
    elif j1_f_j2_coupling == '<==>' and j1_b_j2_coupling == '-->':
        coupling_relation = '-->'
    elif j1_f_j2_coupling == '<==>' and j1_b_j2_coupling == '<--':
        coupling_relation = '<--'
    elif j1_f_j2_coupling == '<==>' and j1_b_j2_coupling == 'uncoupled':
        coupling_relation = 'uncoupled'

    elif j1_f_j2_coupling == '<-->' and j1_b_j2_coupling == '-->':
        coupling_relation = '-->'
    elif j1_f_j2_coupling == '<-->' and j1_b_j2_coupling == '<--':
        coupling_relation = '<--'
    elif j1_f_j2_coupling == '<-->' and j1_b_j2_coupling == 'uncoupled':
        coupling_relation = 'uncoupled'

    elif j1_f_j2_coupling == '-->' and j1_b_j2_coupling == '<--':
        coupling_relation = 'uncoupled'
    elif j1_f_j2_coupling == '-->' and j1_b_j2_coupling == 'uncoupled':
        coupling_relation = 'uncoupled'

    elif j1_f_j2_coupling == '<--' and j1_b_j2_coupling == '-->':
        coupling_relation = 'uncoupled'
    elif j1_f_j2_coupling == '<--' and j1_b_j2_coupling == 'uncoupled':
        coupling_relation = 'uncoupled'

    else:
        coupling_relation = j1_f_j2_coupling

    return coupling_relation

reason: import pyomo,from pyomo


text:
#Indices acoording to the Aurora palette for each layer:
# Ground (water, forest, house floors, roads, etc...)
# Dwellings (walls, constructions, doors, grass, and whatever elements that shoud be on top of the floor but behind the player)
# Objects: Elements that could potentially be in front of the player (lamps, trees, etc...)
# Note that indices could be repeated across layers
import pdb
import numpy as np

#TODO: put this in a json! Layers
GROUND_LAYER = 1
OVERGROUND_LAYER = 2
DECORATION_LAYER = 3
OBJECT_LAYER = 4
META_LAYER = 5

#GROUND LAYER
WATER1 = 47
WATER2 = 48
SAND1 = 72
SAND2 = 73
SAND3 = 74
GRASS1 = 184
GRASS2 = 185
GRASS3 = 186
GRASS4 = 187
ROCK = 117
WOOD_FLOOR = 163
STONE_FLOOR = 11

STAIRS_DOWN = 100
STAIRS_UP = 101

#OVERLAY LAYER
WALL1 = 154
WALL2 = 156
RUINS = 155
DOOR1 = 243
DOOR1_OPEN = 244

WINDOW1SIDE = 39
WINDOW1 = 40
PLANT1 = 41
PLANT2 = 42
PLANT3 = 43
PLANT4 = 44
PLANT5 = 45

#OBJECT LAYER
LAMP = 76
CAMP_FIRE=75
TREE1 = 92
TREE2 = 93
TREE3 = 94
TREE4 = 95
TREE5 = 96

BENCH = 60
BARREL = 66
BED = 63
FRIDGE = 131
STOVE = 134
TABLE = 64
MANHOLE = 65
LADDER = 67

def hmap_to_ground_lid(hv):
    lids = [WATER1, WATER2, SAND1, SAND2, SAND3, GRASS1, GRASS2, GRASS3, GRASS4, ROCK]
    lid = int(hv * (len(lids)-1))
    return lids[lid]

def get_color(lid):
    return Label.palette[lid,:]

class Label:
    palette = np.array([[  0,   0,   0],
                        [ 17,  17,  17],
                        [ 34,  34,  34],
                        [ 51,  51,  51],
                        [ 68,  68,  68],
                        [ 85,  85,  85],
                        [102, 102, 102],
                        [119, 119, 119],
                        [136, 136, 136],
                        [153, 153, 153],
                        [170, 170, 170],
                        [187, 187, 187],
                        [204, 204, 204],
                        [221, 221, 221],
                        [238, 238, 238],
                        [255, 255, 255],
                        [127, 127,   0],
                        [191, 191,  63],
                        [255, 255,   0],
                        [255, 255, 191],
                        [255, 129, 129],
                        [255,   0,   0],
                        [191,  63,  63],
                        [127,   0,   0],
                        [ 80,  15,  15],
                        [127,   0, 127],
                        [191,  63, 191],
                        [245,   0, 245],
                        [255, 129, 253],
                        [203, 192, 255],
                        [129, 129, 255],
                        [  0,   0, 255],
                        [ 63,  63, 191],
                        [  0,   0, 127],
                        [ 20,  20,  85],
                        [  0,  63, 127],
                        [ 63, 127, 191],
                        [  0, 127, 255],
                        [129, 191, 255],
                        [191, 255, 255],
                        [  0, 255, 255],
                        [ 63, 191, 191],
                        [  0, 127, 127],
                        [  0, 127,   0],
                        [ 63, 191,  63],
                        [  0, 255,   0],
                        [175, 255, 175],
                        [255, 191,   0],
                        [255, 127,   0],
                        [200, 125,  75],
                        [192, 175, 188],
                        [137, 170, 203],
                        [144, 160, 166],
                        [148, 148, 126],
                        [135, 130, 110],
                        [ 96, 110, 126],
                        [ 95, 105, 160],
                        [114, 120, 192],
                        [116, 138, 208],
                        [125, 155, 225],
                        [140, 170, 235],
                        [155, 185, 245],
                        [175, 200, 246],
                        [210, 225, 245],
                        [255,   0, 127],
                        [ 59,  59,  87],
                        [ 60,  65, 115],
                        [ 85,  85, 142],
                        [115, 115, 171],
                        [143, 143, 199],
                        [171, 171, 227],
                        [218, 210, 248],
                        [171, 199, 227],
                        [115, 158, 196],
                        [ 87, 115, 143],
                        [ 59,  87, 115],
                        [ 31,  45,  59],
                        [ 35,  65,  65],
                        [ 59, 115, 115],
                        [ 87, 143, 143],
                        [ 85, 162, 162],
                        [114, 181, 181],
                        [143, 199, 199],
                        [171, 218, 218],
                        [199, 237, 237],
                        [171, 227, 199],
                        [143, 199, 171],
                        [ 85, 190, 142],
                        [ 87, 143, 115],
                        [ 62, 125,  88],
                        [ 50,  80,  70],
                        [ 15,  30,  25],
                        [ 55,  80,  35],
                        [ 59,  87,  59],
                        [ 80, 100,  80],
                        [ 73, 115,  59],
                        [ 87, 143,  87],
                        [115, 171, 115],
                        [130, 192, 100],
                        [143, 199, 143],
                        [162, 216, 162],
                        [250, 248, 225],
                        [202, 238, 180],
                        [197, 227, 171],
                        [142, 180, 135],
                        [ 95, 125,  80],
                        [ 70, 105,  15],
                        [ 35,  45,  30],
                        [ 70,  65,  35],
                        [115, 115,  59],
                        [171, 171, 100],
                        [199, 199, 143],
                        [227, 227, 171],
                        [241, 241, 199],
                        [240, 210, 190],
                        [227, 199, 171],
                        [220, 185, 168],
                        [199, 171, 143],
                        [199, 143,  87],
                        [143, 115,  87],
                        [115,  87,  59],
                        [ 45,  25,  15],
                        [ 59,  31,  31],
                        [ 87,  59,  59],
                        [115,  73,  73],
                        [143,  87,  87],
                        [170, 110, 115],
                        [202, 118, 118],
                        [199, 143, 143],
                        [227, 171, 171],
                        [248, 218, 208],
                        [255, 227, 227],
                        [199, 143, 171],
                        [199,  87, 143],
                        [143,  87, 115],
                        [115,  59,  87],
                        [ 60,  35,  60],
                        [ 70,  50,  70],
                        [114,  64, 114],
                        [143,  87, 143],
                        [171,  87, 171],
                        [171, 115, 171],
                        [225, 172, 235],
                        [245, 220, 255],
                        [227, 199, 227],
                        [210, 185, 225],
                        [190, 160, 215],
                        [185, 143, 199],
                        [160, 125, 200],
                        [145,  90, 195],
                        [ 55,  40,  75],
                        [ 35,  22,  50],
                        [ 30,  10,  40],
                        [ 17,  24,  64],
                        [  0,  24,  98],
                        [ 10,  20, 165],
                        [ 16,  32, 218],
                        [ 74,  82, 213],
                        [ 10,  60, 255],
                        [ 50,  90, 245],
                        [ 98,  98, 255],
                        [ 49, 189, 246],
                        [ 60, 165, 255],
                        [ 15, 155, 215],
                        [ 10, 110, 218],
                        [  0,  90, 180],
                        [  5,  75, 160],
                        [ 20,  50,  95],
                        [ 10,  80,  83],
                        [  0,  98,  98],
                        [ 90, 128, 140],
                        [  0, 148, 172],
                        [ 10, 177, 177],
                        [ 90, 213, 230],
                        [ 16, 213, 255],
                        [ 74, 234, 255],
                        [ 65, 255, 200],
                        [ 70, 240, 155],
                        [ 25, 220, 150],
                        [  5, 200, 115],
                        [  5, 168, 106],
                        [ 20, 110,  60],
                        [  5,  52,  40],
                        [  8,  70,  32],
                        [ 12,  92,  12],
                        [  5, 150,  20],
                        [ 10, 215,  10],
                        [ 10, 230,  20],
                        [115, 255, 125],
                        [ 90, 240,  75],
                        [ 20, 197,   0],
                        [ 80, 180,   5],
                        [ 78, 140,  28],
                        [ 50,  56,  18],
                        [128, 152,  18],
                        [145, 196,   6],
                        [106, 222,   0],
                        [168, 235,  45],
                        [165, 254,  60],
                        [205, 255, 106],
                        [255, 235, 145],
                        [255, 230,  85],
                        [240, 215, 125],
                        [213, 222,   8],
                        [222, 156,  16],
                        [ 92,  90,   5],
                        [ 82,  44,  22],
                        [125,  55,  15],
                        [156,  74,   0],
                        [150, 100,  50],
                        [246,  82,   0],
                        [189, 106,  24],
                        [220, 120,  35],
                        [195, 157, 105],
                        [255, 164,  74],
                        [255, 176, 144],
                        [255, 197,  90],
                        [250, 185, 190],
                        [240, 110, 120],
                        [255,  90,  74],
                        [246,  65,  98],
                        [245,  60,  60],
                        [218,  28,  16],
                        [189,  16,   0],
                        [148,  16,  35],
                        [ 72,  33,  12],
                        [176,  16,  80],
                        [208,  16,  96],
                        [210,  50, 135],
                        [255,  65, 156],
                        [255,  98, 189],
                        [255, 145, 185],
                        [255, 165, 215],
                        [250, 195, 215],
                        [252, 198, 248],
                        [255, 115, 230],
                        [255,  82, 255],
                        [224,  32, 218],
                        [255,  41, 189],
                        [197,  16, 189],
                        [190,  20, 140],
                        [123,  24,  90],
                        [100,  20, 100],
                        [ 98,   0,  65],
                        [ 70,  10,  50],
                        [ 55,  25,  85],
                        [130,  25, 160],
                        [120,   0, 200],
                        [191,  80, 255],
                        [197, 106, 255],
                        [185, 160, 250],
                        [140,  58, 252],
                        [120,  30, 230],
                        [ 57,  16, 189],
                        [ 77,  52, 152],
                        [ 55,  20, 145]])

    def __init__(self, lid, name, layer):
        self.lid = lid
        self.name = name
        self.layer = layer
        self.color = get_color(self.lid)

labels = [
    Label(WATER1, 'water1', GROUND_LAYER),
    Label(WATER2, 'water2', GROUND_LAYER),
    Label(SAND1, 'sand1', GROUND_LAYER),
    Label(SAND2, 'sand2', GROUND_LAYER),
    Label(SAND3, 'sand3', GROUND_LAYER),
    Label(GRASS1, 'grass1', GROUND_LAYER),
    Label(GRASS2, 'grass2', GROUND_LAYER),
    Label(GRASS3, 'grass3', GROUND_LAYER),
    Label(GRASS4, 'grass4', GROUND_LAYER),
    Label(ROCK, 'rock', GROUND_LAYER),
    Label(WOOD_FLOOR, 'wood_floor', GROUND_LAYER),
    Label(STONE_FLOOR, 'stone_floor', GROUND_LAYER),
    Label(WALL1, 'wall1', OVERGROUND_LAYER),
    Label(DOOR1, 'door1', OVERGROUND_LAYER),
    Label(WINDOW1, 'window1', OVERGROUND_LAYER)
    ]

#FLOOR LAYER

#DWELLINGS LAYER

#OBJECT LAYER

# #TILES
# GRASS_TILE = 5
# WALL_TILE = 1
# FLOOR_TILE = 2
# CLOSED_DOOR_TILE = 3
# OPEN_DOOR_TILE = 4
# WINDOW_TILE = 5

# ROAD_TILE = 7
# POND_TILE = 8
# TREE_TILE_1 = 9
# FENCE_TILE = 10

# WATER1 = 50
# WATER2 = 51
# SAND1 = 52
# SAND2 = 53
# SAND3 = 54
# GRASS1 = 55
# GRASS2 = 56
# GRASS3 = 57
# GRASS4 = 58
# ROCK = 59

# #OBJECTS
# TABLE = 102
# FRIDGE = 103
# LAMP = 104
# STOVE = 105
# BED = 106
# CLOSET = 107

# PLANT_TILE_1 = 200
# PLANT_TILE_2 = 201
# PLANT_TILE_3 = 202
# PLANT_TILE_4 = 203
# PLANT_TILE_5 = 204
# PLANT_TILE_6 = 205

max_stars_repo_name: byfron/Project-iom

max_issues_repo_name: byfron/Project-iom

max_forks_repo_name: byfron/Project-iom


text:
from config.constants import (API_TIME_FORMAT, VOICE_RECORDING, ALL_DATA_STREAMS, DEFAULT_S3_RETRIES,
    SURVEY_ANSWERS, SURVEY_TIMINGS, IMAGE_FILE)
from config.settings import (S3_BUCKET, BEIWE_SERVER_AWS_ACCESS_KEY_ID,
    BEIWE_SERVER_AWS_SECRET_ACCESS_KEY, S3_REGION_NAME)
import config.load_django
from database.models import ReceivedDataStats, UploadTracking, Participant, Study, Survey, ParticipantSurvey
from database.data_access_models import ChunkRegistry, FileProcessLock, FileToProcess
from libs.client_key_management import get_client_public_key
from config.constants import UPLOAD_FILE_TYPE_MAPPING
import argparse
import config.remote_db_env
import json
import os
import re
import requests
import random, math
import time
import datetime
import sys
import numpy.random as rnd
import pandas
import numpy as np
from io import BytesIO as StringIO
from Crypto.Cipher import AES

austin_coordinates = {'latitude': 30.2672 ,
                      'longitude': 97.7431}

gps_sd = .5
gps_altitude = 150
gps_accuracy = 65
gps_samples_per_second = 1

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Beiwe study tool, mobile upload")

    parser.add_argument('--start_date', help='date to start generating data for',
        type=str)

    parser.add_argument('--num_hours', help='number of hours of data to generate',
        type=int)

    parser.add_argument('participant_id', help='id of the participant who generated the data',
        type=str)

    args = parser.parse_args()

    if not Participant.objects.filter(patient_id=args.participant_id).exists():
        print('Could not find participant {0}'.format(args.participant_id))
        sys.exit(1)

    participant = Participant.objects.get(patient_id=args.participant_id)

    if args.start_date:
        start_datetime = datetime.datetime.strptime(args.start_date, API_TIME_FORMAT)
    else:
        start_datetime = datetime.datetime.now()

    if args.num_hours:
        num_hours = args.num_hours
    else:
        num_hours = 3

    print('Creating {0} hours worth of upload data for {1} starting at time {2}'.format(num_hours, participant,
        start_datetime))

    ## now we create the random gps data

    # create random gps data and upload it
    on_duration = participant.study.device_settings.gps_on_duration_seconds
    off_duration = participant.study.device_settings.gps_off_duration_seconds

    sampling_frequency = '{0:d}L'.format(int(np.ceil(100.0/gps_samples_per_second)))

    start_time = start_datetime
    end_time = start_datetime + datetime.timedelta(hours=num_hours)

    sample_datetimes = []
    sample_latitudes = []
    sample_longitudes = []
    sample_altitude = []
    sample_accuracy = []

    while start_time < end_time:
        if on_duration > (end_time - start_time).seconds:
            on_duration = (end_time - start_time).seconds

        sample_datetimes += pandas.date_range(start_time, periods=on_duration*gps_samples_per_second, freq='S')
        sample_latitudes += rnd.normal(loc=austin_coordinates['latitude'], scale=gps_sd,
            size=on_duration*gps_samples_per_second).tolist()
        sample_longitudes += rnd.normal(loc=austin_coordinates['longitude'], scale=gps_sd,
            size=on_duration*gps_samples_per_second).tolist()
        sample_altitude += [gps_altitude] * on_duration*gps_samples_per_second
        sample_accuracy += [gps_accuracy] * on_duration*gps_samples_per_second

        start_time += datetime.timedelta(seconds=off_duration)

    gps_dataframe = pandas.DataFrame({'times': sample_datetimes,
                                      'latitude': sample_latitudes,
                                      'longitude': sample_longitudes,
                                      'altitude': sample_altitude,
                                      'accuracy': sample_accuracy})

    def datetime_to_timestamp(date_and_time):
        return int(date_and_time.strftime('%s'))*100

    def datetime_to_timestring(date_and_time):
        return date_and_time.strftime(API_TIME_FORMAT)

    gps_dataframe['timestamp'] = gps_dataframe.apply(lambda row: datetime_to_timestamp(row['times']), axis=1)
    gps_dataframe['UTC time'] = gps_dataframe.apply(lambda row: datetime_to_timestring(row['times']), axis=1)
    gps_dataframe = gps_dataframe[['timestamp', 'UTC time', 'latitude', 'longitude', 'altitude', 'accuracy']]
    gps_dataframe = gps_dataframe.set_index('timestamp')

    string_io = StringIO()
    gps_dataframe.to_csv(string_io)
    print(string_io.getvalue())

    client_public_key = get_client_public_key(participant.patient_id, participant.study.object_id)
    print(client_public_key)

    iv = os.urandom(16)
    encrypted_table = AES.new(client_public_key, AES.MODE_CFB, segment_size=8, IV=iv).encrypt('cameron is awsome')

    #return iv + AES.new( encryption_key, AES.MODE_CFB, segment_size=8, IV=iv ).encrypt( input_string )

max_stars_repo_name: ccraddock/beiwe-backend-cc

max_issues_repo_name: ccraddock/beiwe-backend-cc

max_forks_repo_name: ccraddock/beiwe-backend-cc


text:
import os

import click
import matplotlib.pyplot as plt
import networkx as nx
import numpy as num
from pyrocko import catalog, gmtpy, model
from pyrocko.plot.automap import Map

import cargonet.visualization.colors as colors
from cargonet.preprocessing.graphs.tgraph import TransportGraph
from cargonet.utils.geo import center_between_points, dist_m_v2
from cargonet.visualization.tplot import TransportPlot

BERLIN = (52.52, 13.405)

class GMTTransportPlot(TransportPlot):
    def __init__(
        self,
        g,
        live=None,
        title=None,
        subtitle=None,
        axis=True,
        arrows=False,
        cities=True,
        legend=True,
        fontsize=15,
        thickness=1,
        node_color="black",
        node_size=2,
        node_border_color="black",
        node_border_width=0,
        filepath=None,
        filename=None,
        save=False,
        show=True,
        node_labels=False,
        edge_labels=False,
        check=False,
        delay=False,
        colorbar_range=None,
        cmap=colors.delay_cmap,
        bbox=None,
        # GMT Specific
        center=BERLIN,
        radius=150000,
        edge_limit=10_000,
        node_limit=10_000,
        width=30,
        height=30,
        fit=True,
        grid=False,
        topo=True,
        rivers=True,
    ):
        self.center = center
        self.radius = radius
        self.edge_limit = edge_limit
        self.node_limit = node_limit
        self.width = width
        self.height = height
        self.fit = fit
        self.grid = grid
        self.topo = topo
        self.rivers = rivers
        super().__init__(
            g=g,
            live=live,
            title=title,
            subtitle=subtitle,
            axis=axis,
            legend=legend,
            arrows=arrows,
            cities=cities,
            fontsize=fontsize,
            thickness=thickness,
            node_color=node_color,
            node_size=node_size,
            node_border_color=node_border_color,
            node_border_width=node_border_width,
            filepath=filepath,
            filename=filename,
            save=save,
            show=show,
            node_labels=node_labels,
            edge_labels=edge_labels,
            check=check,
            delay=delay,
            colorbar_range=colorbar_range,
            cmap=cmap,
            bbox=bbox,
        )

    def draw_cities(self, **options):
        if self.cities:
            self.m.draw_cities(font_size=self.fontsize)

    def draw_edges(self, route, pos, **options):
        c, e_c = 0, route.number_of_edges()
        for u, v in self.g.edges:
            if self.edge_limit and c > self.edge_limit:
                break
            u_pos, v_pos = self.g.nodes[u].get("pos"), self.g.nodes[v].get("pos")
            e_lats = [u_pos[0], v_pos[0]]
            e_lons = [u_pos[1], v_pos[1]]

            self.m.gmt.psxy(
                in_columns=(e_lons, e_lats),
                W="%fp" % options.get("width", 2),
                G=options.get("edge_color", "black"),
                *self.m.jxyr,
            )
            c += 1
            if c % 1_000 == 0:
                print("Plotted %d of %d edges" % (c, e_c))

    def draw_nodes(self, route, pos, **options):
        c, n_lats, n_lons = 0, [], []
        n_c = route.number_of_nodes()
        for n, data in route.nodes(data=True):
            if self.node_limit and c > self.node_limit:
                break
            n_lats.append(pos[n][0])
            n_lons.append(pos[n][1])
            c += 1
            if c % 1_000 == 0:
                print("Plotted %d of %d nodes" % (c, n_c))
        self.m.gmt.psxy(
            in_columns=(n_lons, n_lats),
            S="c%fp" % options.get("node_size", 2),
            G=options.get("node_color", "black"),
            W="%d,%s,solid" % (options.get("node_border_width", 0), options.get("node_border_color", "black")),
            *self.m.jxyr,
        )

    def draw_node_labels(self, route, pos, **options):
        c, n_c = 0, route.number_of_nodes()
        for n, data in self.g.nodes(data=True):
            if self.node_limit and c > self.node_limit:
                break
            pos = data.get("pos")
            if self.node_labels:
                self.m.add_label(pos[0], pos[1], data.get("stationId"), self.fontsize)
            c += 1
            if c % 1_000 == 0:
                print("Plotted %d of %d node labels" % (c, n_c))

    def draw_edge_labels(self, route, pos, **options):
        raise NotImplementedError

    def plot(self, fit_factor=1.5, close=True):
        gmtpy.check_have_gmt()

        if self.fit:
            # Override manual settings
            lats = sorted([pos[0] for _, pos in self.g.nodes.data("pos")])
            lons = sorted([pos[1] for _, pos in self.g.nodes.data("pos")])
            min_lat, max_lat = lats[0], lats[-1]
            min_lon, max_lon = lons[0], lons[-1]
            lb, rt = (min_lat, min_lon), (max_lat, max_lon)
            self.center = center_between_points(lb, rt)
            self.radius = dist_m_v2(lb, rt) / fit_factor

        self.m = Map(
            lat=self.center[0],
            lon=self.center[1],
            radius=self.radius,
            width=self.width,
            height=self.height,
            show_grid=self.grid,
            show_topo=self.topo,
            color_dry=(238, 236, 230),
            topo_cpt_dry="light_land",
            topo_cpt_wet="light_sea",
            illuminate=True,
            illuminate_factor_ocean=0.15,
            show_rivers=self.rivers,
            show_plates=False,
            show_boundaries=True,
            show_scale=True,
            # TODO: Use german metropoles
            # custom_cities=[]
        )

        self.draw_cities()
        self.draw_route(self.g, good_color="black", bad_color="red")
        if self.live is not None:
            self.draw_route(self.live, good_color="green", bad_color="blue")

        filepath = self.get_filepath(
            filepath=self.filepath, filename=self.filename, random=self.save
        )
        if filepath is not None:
            self.m.save(filepath, psconvert=True)

max_stars_repo_name: romnnn/rail-stgcnn

max_issues_repo_name: romnnn/rail-stgcnn

max_forks_repo_name: romnnn/rail-stgcnn

reason: import numpy,import networkx


text:
import gc
import numpy as np
import os
import queue
import time

from torch.utils.data import Dataset as TorchDataset

from opentamp.policy_hooks.data_buffer import DataBuffer

class QueuedDataset(TorchDataset):
    def __init__(self,
                 task,
                 in_queue,
                 batch_size,
                 ratios,
                 dataset=None,
                 normalize=False,
                 policy=None,
                 x_idx=None,
                 aug_f=None,
                 min_buffer=10**3,
                 feed_in_policy=None,
                 feed_prob=0.,
                 feed_inds=(None, None),
                 feed_map=None,
                 save_dir=None):
        self.in_queue = in_queue
        self.task = task
        if dataset is None:
            self.data_buf = DataBuffer(policy, x_idx=x_idx, normalize=normalize, min_buffer=min_buffer, ratios=ratios)
        else:
            self.data_buf = dataset

        self.label_ratio = None
        self.batch_size = batch_size
        self.aug_f = aug_f
        self.load_freq = 10
        self.feed_in_policy = feed_in_policy
        self.feed_prob = feed_prob
        self.feed_out_inds = feed_inds[1]
        self.feed_in_inds = feed_inds[0]
        self.feed_map = feed_map

        self.save_dir = save_dir
        self.cur_save = 0

    def __len__(self):
        return len(self.data_buf)

    def __getitem__(self, index):
        if type(index) is int:
            obs, mu, prc = self.get_batch(1)
            return obs[0], mu[0], prc[0]
        elif type(index) is slice:
            return self.get_batch(index)
        else:
            raise NotImplementedError('Cannot slice data from {} object'.format(type(index)))

    def load_from_dir(self, dname):
        self.data_buf.load_from_dir(dname, self.task)

    def write_data(self, n_data):
        self.data_buf.write_data(self.save_dir, self.task, n_data)

    def pop_queue(self):
        items = []
        n = self.in_queue.qsize()
        for _ in range(n):
            try:
                data = self.in_queue.get_nowait()
                items.append(data)
            except queue.Empty:
                break

        return items

    def load_data(self, val_ratio=0.1):
        items = self.pop_queue()
        if not len(items): return 0

        start_t = time.time()
        for data in items:
            self.data_buf.update(data, val_ratio=val_ratio)

        del items
        gc.collect()
        return 1

    def get_batch(self, size=None, label=None, val=False):
        if size is None: size = self.batch_size

        # Retrieve batch from underlying data buffer
        batch = self.data_buf.get_batch(size, label, val)
        if batch is None: return [], [], []

        obs, mu, prc, wt, x, primobs, aux = batch
        mu = np.array(mu)
        x = np.array(x)
        prc = np.array(prc)
        wt = np.array(wt)
        wt = wt.reshape((-1,1,1)) if len(prc.shape) > 2 else wt.reshape((-1,1))
        aux = np.array(aux)

        # Can replace part of a ground-truth obs with output of a higher level policy
        if self.feed_prob > 0 and self.feed_in_policy is not None:
            if type(obs) is list:
                obs = np.array(obs)
            else:
                obs = obs.copy()

            nprim = int(self.feed_prob * len(mu))
            hl_out = self.feed_in_policy.act(None, primobs[:nprim], None)
            hl_val = hl_out[:nprim, self.feed_out_inds]
            if self.feed_map is not None: hl_val = self.feed_map(hl_val, x[:nprim])
            obs[:nprim, self.feed_in_inds] = hl_val

        if self.aug_f is not None:
            mu, obs, wt, prc = self.aug_f(mu, obs, wt, prc, aux, x)

        prc = wt * prc
        return obs, mu, prc

    def gen_items(self, label=None, val=False):
        while True:
            while self.wait_for_data():
                time.sleep(0.001)

            yield self.get_batch()

    def get_size(self, label=None, val=False):
        return self.data_buf.get_size(label, val)

    def gen_load(self):
        while True:
            yield self.load_data()

    def wait_for_data(self):
        cur_n = self.data_buf.get_size()
        if cur_n < self.data_buf.min_buffer:
            self.load_data()
        return cur_n < self.data_buf.min_buffer

max_stars_repo_name: Algorithmic-Alignment-Lab/openTAMP

max_issues_repo_name: Algorithmic-Alignment-Lab/openTAMP

max_forks_repo_name: Algorithmic-Alignment-Lab/openTAMP


text:
try:
    import unittest2 as unittest
except:
    import unittest
import numpy as np
from pyrr import vector, vector3, vector4

class test_vector(unittest.TestCase):
    def test_import(self):
        import pyrr
        pyrr.vector
        from pyrr import vector

    def test_normalize_single_vector(self):
        result = vector3.normalize([1.,1.,1.])
        np.testing.assert_almost_equal(result, [0.57735, 0.57735, 0.57735], decimal=5)

    def test_normalize_batch(self):
        result = vector3.normalize([
            [1.,1.,1.],
            [-1.,-1.,-1.],
            [0.,2.,7.],
        ])
        expected = [
            [0.57735, 0.57735, 0.57735],
            [-0.57735,-0.57735,-0.57735],
            [0., 0.274721, 0.961524],
        ]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_squared_length_single_vector(self):
        result = vector3.squared_length([1.,1.,1.])
        np.testing.assert_almost_equal(result, 3., decimal=5)

    def test_squared_length_batch(self):
        result = vector3.squared_length([
            [1.,1.,1.],
            [-1.,-1.,-1.],
            [0.,2.,7.],
        ])
        expected = [
            3.,
            3.,
            53.,
        ]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_length_vector3(self):
        result = vector3.length([1.,1.,1.])
        np.testing.assert_almost_equal(result, 1.73205, decimal=5)

    def test_length_vector4(self):
        result = vector3.length([1.,1.,1.,1.])
        np.testing.assert_almost_equal(result, 2., decimal=5)

    def test_length_vector3_batch(self):
        result = vector3.length([
            [1.,1.,1.],
            [-1.,-1.,-1.],
            [0.,2.,7.],
        ])
        expected = [
            1.73205,
            1.73205,
            7.28011,
        ]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_set_length_vector3(self):
        result = vector3.set_length([1.,1.,1.],2.)
        expected = [1.15470,1.15470,1.15470]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_set_length_vector4(self):
        result = vector4.set_length([1.,1.,1.,1.],2.)
        expected = [1.,1.,1.,1.]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_set_length_batch_vector(self):
        result = vector3.set_length([
            [1.,1.,1.],
            [-1.,-1.,-1.],
            [0.,2.,7.],
            ], 2.0)
        expected = [
            [1.15470,1.15470,1.15470],
            [-1.15470,-1.15470,-1.15470],
            [0.,0.54944,1.92304],
        ]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_dot_adjacent(self):
        result = vector3.dot([1.,0.,0.], [0.,1.,0.])
        np.testing.assert_almost_equal(result, 0.0, decimal=5)

    def test_dot_parallel(self):
        result = vector3.dot([0.,1.,0.], [0.,1.,0.])
        np.testing.assert_almost_equal(result, 1.0, decimal=5)

    def test_dot_angle(self):
        result = vector3.dot([.2,.2,0.], [2.,-.2,0.])
        np.testing.assert_almost_equal(result, 0.36, decimal=5)

    def test_dot_batch(self):
        result = vector3.dot([
            [1.,0.,0.],
            [0.,1.,0.],
            [.2,.2,0.]
        ],[
            [0.,1.,0.],
            [0.,1.,0.],
            [2.,-.2,0.]
        ])
        expected = [0.,1.,0.36]
        np.testing.assert_almost_equal(result, expected, decimal=5)

    def test_interoplation(self):
        result = vector3.interpolate([0.,0.,0.], [1.,1.,1.], 0.5)
        np.testing.assert_almost_equal(result, [.5,.5,.5], decimal=5)

        result = vector3.interpolate([0.,0.,0.], [2.,2.,2.], 0.5)
        np.testing.assert_almost_equal(result, [1.,1.,1.], decimal=5)

if __name__ == '__main__':
    unittest.main()


text:
import skimage
import pandas as pd
import torch
from torch.utils.data import DataLoader
import numpy as np
from pytorch_toolbelt.inference.tiles import ImageSlicer, CudaTileMerger
from pytorch_toolbelt.utils.torch_utils import to_numpy
import sys
sys.path.append('.')

def infer_one(model, mask, tile_size=(512, 512), tile_step=(256, 256), weight='mean'):
    image = mask.cpu().numpy()
    image = np.moveaxis(image, 0, -1)

    with torch.no_grad():
        tiler = ImageSlicer((900, 900), tile_size=tile_size, tile_step=tile_step, weight=weight)
        tiles = [np.moveaxis(tile, -1, 0) for tile in tiler.split(image)]
        merger = CudaTileMerger(tiler.target_shape, 1, tiler.weight)

        for tiles_batch, coords_batch in DataLoader(list(zip(tiles, tiler.crops)), batch_size=10, pin_memory=False):
            tiles_batch = tiles_batch.float().cuda()
            pred_batch = model(tiles_batch)
            tiles_batch.cpu().detach()
            merger.integrate_batch(pred_batch, coords_batch)
    merged_mask = np.moveaxis(to_numpy(merger.merge()), 0, -1)
    merged_mask = tiler.crop_to_orignal_size(merged_mask)

    m = merged_mask[..., 0].copy()
    return m

def space_metric(dl, model, truth_df, threshold=0.5, minbuildingsize=120,
                 tile_size=(512, 512), tile_step=(256, 256), weight='mean'):
    f1metric = []
    tmc = []
    pmc = []
    icount = 0
    with torch.no_grad():
        for (x_batch, y_batch, c_batch, tile_batch, orient_batch) in dl:

            icount = icount + 1
            if icount % 10 == 0:
                print('Batch ', icount)

            for i in range(len(x_batch)):
                pred = infer_one(model, x_batch[i, ...], tile_size, tile_step, weight)

                orient = orient_batch[i]
                tile_id = tile_batch[i]

                if orient == 1:
                    pred = np.fliplr(np.flipud(pred))

                pred[np.where(pred > threshold)] = 1
                pred[np.where(pred <= threshold)] = 0

                regionlabels, regioncount = skimage.measure.label(pred, background=0, connectivity=1, return_num=True)
                regionproperties = skimage.measure.regionprops(regionlabels)
                for blab in range(regioncount):
                    if regionproperties[blab].area < minbuildingsize:
                        pred[regionlabels == blab + 1] = 0

                vectordata = solaris.vector.mask.mask_to_poly_geojson(
                    pred,
                    min_area=0,
                    bg_threshold=0.5,
                    do_transform=False,
                    simplify=True
                )

                csvaddition = pd.DataFrame({'ImageId': tile_id,
                                            'BuildingId': 0,
                                            'PolygonWKT_Pix': vectordata['geometry'],
                                            'Confidence': 1
                                            })
                csvaddition.to_csv('tmp_proposal_val_train.csv', index=False)

                #                 #
                #                 # Extract ground truth from global truth_df
                aTruth = truth_df[truth_df.ImageId == tile_id]
                aTruth.to_csv('tmp_ground_truth_val_train.csv', index=False)
                truth_masks_count = len(aTruth)
                pred_masks_count = len(csvaddition)

                #                 #
                #                 # do eval
                evaluator = solaris.eval.base.Evaluator('tmp_ground_truth_val_train.csv')
                evaluator.load_proposal('tmp_proposal_val_train.csv', proposalCSV=True,
                                        conf_field_list=[])
                report = evaluator.eval_iou_spacenet_csv(miniou=0.5, min_area=80)

                assert (len(report) == 1)
                f1metric.append(report[0]['F1Score'])
                pmc.append(pred_masks_count)
                tmc.append(truth_masks_count)

    f1metric = np.array(f1metric)
    pmc = np.array(pmc)
    tmc = np.array(tmc)
    return f1metric.mean(), pmc.mean(), tmc.mean()

from segmentation_models_pytorch.utils import functional  as F
from segmentation_models_pytorch.utils.base import Loss, Activation

class BCEDiceLoss2Class(Loss):

    def __init__(self, eps=1., beta=1., activation=None, ignore_channels=None, **kwargs):
        super().__init__(**kwargs)
        self.eps = eps
        self.beta = beta
        self.activation = Activation(activation)
        self.ignore_channels = ignore_channels

    def forward(self, y_pr, y_gt):
        # interior
        y_pr_0 = y_pr[:, 0, :, :]
        y_gt_0 = y_gt[:, 0, :, :]
        bce_0 = torch.nn.functional.binary_cross_entropy(y_pr_0, y_gt_0)
        y_pr_0 = self.activation(y_pr_0)
        dice_0 = 1 - F.f_score(y_pr_0, y_gt_0, beta=self.beta, eps=self.eps, threshold=None,
                               ignore_channels=self.ignore_channels)
        loss_0 = 0.5 * bce_0 + 0.5 * dice_0
        # outline
        y_pr_1 = y_pr[:, 1, :, :]
        y_gt_1 = y_gt[:, 1, :, :]
        bce_1 = torch.nn.functional.binary_cross_entropy(y_pr_1, y_gt_1)
        y_pr_1 = self.activation(y_pr_1)
        dice_1 = 1 - F.f_score(y_pr_1, y_gt_1, beta=self.beta, eps=self.eps, threshold=None,
                               ignore_channels=self.ignore_channels)
        loss_1 = 0.5 * bce_1 + 0.5 * dice_1

        return 0.7 * loss_0 + 0.3 * loss_1


text:
#! /usr/bin/env python

"""
File: Heun.py
Copyright (c) 2016 Michael Seaman
License: MIT
Description: Implements Heun's method for solving
a set of 2 coupled ODEs
"""

from ODE_solver import ODE_solver
import numpy as np
import matplotlib.pyplot as plt

class Heun(ODE_solver):
    """
    Implements the solve_n method using Heun's method
    """
    def solve_n(self):
        output_u = np.zeros(self.n+1) + self.u0
        output_v = np.zeros(self.n+1) + self.v0
        t_list = np.linspace(self.a, self.b, self.n+1)
        delta_t = (self.b - self.a)/ float(self.n)
        for i in xrange(len(t_list) - 1):
            geussed_u_next = output_u[i] + delta_t * self.dudt(output_u[i], output_v[i], t_list[i])
            geussed_v_next = output_v[i] + delta_t * self.dvdt(output_u[i], output_v[i], t_list[i])
            output_u[i+1] = output_u[i] + .5 * delta_t * (self.dudt(output_u[i], output_v[i], t_list[i]) + self.dudt(geussed_u_next, geussed_v_next, t_list[i+1]) )
            output_v[i+1] = output_v[i] + .5 * delta_t * (self.dvdt(output_u[i], output_v[i], t_list[i]) + self.dvdt(geussed_u_next, geussed_v_next, t_list[i+1]) )
        return output_u, output_v