text:
import astropy.io.fits as pf
from astropy.table import Table,join,unique,vstack
import desitarget
from desitarget import io, mtl
from desitarget.cuts import random_fraction_of_trues
from desitarget.mtl import get_mtl_dir, get_mtl_tile_file_name,get_mtl_ledger_format
from desitarget.mtl import get_zcat_dir, get_ztile_file_name, tiles_to_be_processed
from desitarget.mtl import make_zcat,survey_data_model,update_ledger, get_utc_date
from desitarget.targets import initial_priority_numobs, decode_targetid
from desitarget.targetmask import obsconditions, obsmask
from desitarget.targetmask import desi_mask, bgs_mask, mws_mask
from desiutil.log import get_logger
import fitsio
from LSS.bitweights import pack_bitweights
from LSS.SV3.fatools import get_fba_fromnewmtl
import LSS.SV3.fatools as fatools
import matplotlib.pyplot as plt
import numpy as np
from numpy import random as rand
import numpy.lib.recfunctions as rfn
import os
import subprocess
import sys
from time import sleep

log = get_logger()

os.environ['DESIMODEL'] = '/global/common/software/desi/cori/desiconda/current/code/desimodel/master'

zcatdatamodel = np.array([], dtype=[
    ('RA', '>f8'), ('DEC', '>f8'), ('TARGETID', '>i8'),
    ('NUMOBS', '>i4'), ('Z', '>f8'), ('ZWARN', '>i8'), ('ZTILEID', '>i4')
    ])

mtltilefiledm = np.array([], dtype=[
    ('TILEID', '>i4'), ('TIMESTAMP', 'U25'),
    ('VERSION', 'U14'), ('PROGRAM', 'U6'), ('ZDATE', 'U8')
    ])

def findTwin(altFiber, origFiberList, survey = 'sv3', obscon = 'dark'):
    if survey == 'sv3':
        if obscon == 'dark':
            altTargBits = altFiber['SV3_DESI_TARGET']
            altTargBitsSec = altFiber['SV3_BGS_TARGET']
            altTargBitsMWS = altFiber['SV3_MWS_TARGET']

            origTargBitList = origFiberList['SV3_DESI_TARGET']
            origTargBitListSec = origFiberList['SV3_BGS_TARGET']
            origTargBitListMWS = origFiberList['SV3_MWS_TARGET']

        elif obscon == 'bright':
            altTargBits = altFiber['SV3_BGS_TARGET']
            altTargBitsSec = altFiber['SV3_DESI_TARGET']
            altTargBitsMWS = altFiber['SV3_MWS_TARGET']

            origTargBitList = origFiberList['SV3_BGS_TARGET']
            origTargBitListSec = origFiberList['SV3_DESI_TARGET']
            origTargBitListMWS = origFiberList['SV3_MWS_TARGET']
        else:
            raise ValueError('Invalid value for \'obscon\': {0}'.format(obscon))
    elif survey == 'main':
        if obscon == 'dark':
            altTargBits = altFiber['DESI_TARGET']
            altTargBitsSec = altFiber['BGS_TARGET']
            altTargBitsMWS = altFiber['MWS_TARGET']

            origTargBitList = origFiberList['DESI_TARGET']
            origTargBitListSec = origFiberList['BGS_TARGET']
            origTargBitListMWS = origFiberList['MWS_TARGET']

        elif obscon == 'bright':
            altTargBits = altFiber['BGS_TARGET']
            altTargBitsSec = altFiber['DESI_TARGET']
            altTargBitsMWS = altFiber['MWS_TARGET']
            origTargBitList = origFiberList['BGS_TARGET']
            origTargBitListSec = origFiberList['DESI_TARGET']
            origTargBitListMWS = origFiberList['MWS_TARGET']

        else:
            raise ValueError('Invalid value for \'obscon\': {0}'.format(obscon))
    else:
        raise ValueError('Invalid value for \'survey\': {0}'.format(survey))

    altFS = altFiber['FIBERSTATUS']
    origFS = origFiberList['FIBERSTATUS']

    BGSBits = initialentries['SV3_BGS_TARGET']
    BGSFaintHIP = ((BGSBits & 8) == 8)
    BGSFaintAll = ((BGSBits & 1) == 1) | BGSFaintHIP

    #Set all BGS_FAINT_HIP to BGS_FAINT

    initialentries['SV3_BGS_TARGET'][BGSFaintHIP] = (BGSBits[BGSFaintHIP] & ~8)
    initialentries['PRIORITY'][BGSFaintHIP] = 102000*np.ones(np.sum(BGSFaintHIP))

    NewBGSBits = initialentries['SV3_BGS_TARGET']
    NewBGSFaintHIP = ((BGSBits & 8) == 8)
    NewBGSFaintAll = ((BGSBits & 1) == 1) | NewBGSFaintHIP
    NewBGSPriors = initialentries['PRIORITY']
    #Select 20% of BGS_FAINT to promote using function from
    BGSFaintNewHIP = random_fraction_of_trues(PromoteFracBGSFaint, BGSFaintAll)
    #Promote them

    initialentries['SV3_BGS_TARGET'][BGSFaintNewHIP] = (BGSBits[BGSFaintNewHIP] | 8)
    initialentries['PRIORITY'][BGSFaintNewHIP] = 102100*np.ones(np.sum(BGSFaintNewHIP)).astype(int)

def createFAmap(FAReal, FAAlt, TargAlt = None, changeFiberOpt = None, debug = False):
    # Options for 'changeFiberOpt':
    # None: do nothing different to version 1
    # AllTwins: Find a twin fiber with a target of the
    #   same type and similar Fiber assignment for all
    #   unsimilar target types
    # SomeTwins: Find a twin as above but only for
    #   assignments where the original fiber was unassigned

    TIDReal = FAReal['TARGETID']
    TIDAlt = FAAlt['TARGETID']
    FibReal = FAReal['FIBER']
    FibAlt = FAAlt['FIBER']
    if not (changeFiberOpt is None):
        assert(not(TargAlt is None))
        jTargs = join(FAAlt, TargAlt, keys = "TARGETID")

    Real2Alt = {}
    Alt2Real = {}
    if debug:
        inc1 = 0
        inc2 = 0
    negMisMatch = []
    for tr, fr in zip(TIDReal, FibReal):
        taMatch = TIDAlt[FibAlt == fr]
        assert(len(taMatch) == 1)
        if debug:
            try:
                assert(tr == taMatch[0])
            except:
                inc1+=1
        Real2Alt[tr] = taMatch[0]

    for ta, fa in zip(TIDAlt, FibAlt):
        trMatch = TIDReal[FibReal == fa]
        try:
            assert(len(trMatch) == 1)
        except:
            if ta < 0:
                #print('no match for negative ta {0}'.format(ta))
                negMisMatch.append(ta)
                continue
            else:
                print(ta)

                assert(0)
        if debug:
            try:
                assert(ta == trMatch[0])
            except:
                inc2+=1
        if (changeFiberOpt is None) or (changeFiberOpt == 'SomeTwins') or (ta == trMatch[0]):
            Alt2Real[ta] = trMatch[0]
        elif changeFiberOpt == 'AllTwins':
            #if jTargs['SV3_']
            pass

    print('no matches for negative tas {0}'.format(negMisMatch))
    if debug:
        print(inc1)
        print(inc2)
    return Alt2Real, Real2Alt

def makeAlternateZCat(zcat, real2AltMap, alt2RealMap, debug = False):
    from collections import Counter
    zcatids = zcat['TARGETID']
    altZCat = Table(zcat)
    if debug:
        failures = 0
        negativeIDs = 0
    for n, i in zip(zcatids, range(len(zcatids))):
        cond = (n == zcatids)
        if debug and (n < 0):
            negativeIDs +=1
        #try:
        altid = real2AltMap[n]
        altZCat['TARGETID'][i] = altid
    if debug:
        print('negIDs')
        print(negativeIDs)
        print('failures')
        print(failures)
        print('testctr')
    d =  Counter(altZCat['TARGETID'])
    res = [ k for k, v in d.items() if v > 1]
    print(res)
    if len(res):
        print('how many pre dup cuts')
        print(zcatids.shape)
        cond2 = np.ones(zcatids.shape, dtype=bool)
        for i in res:
            print('test')
            print(np.sum(zcatids == i))
            cond2 = cond2 & (altcatids != i)
        print("how many post dup cuts")
        print(np.sum(cond2))
    else:
        print("supposedly, no duplicates")
    return altZCat

def checkMTLChanged(MTLFile1, MTLFile2):
    MTL1 = io.read_mtl_ledger(MTLFile1, unique = True)
    MTL2 = io.read_mtl_ledger(MTLFile2, unique = True)
    NDiff = 0
    NDiff2 = 0
    NDiff3 = 0
    for tar1 in MTL1:
        tar2 = MTL2[MTL2['TARGETID'] == tar1['TARGETID']]

        if tar1['NUMOBS'] != tar2['NUMOBS']:
            NDiff +=1

        if tar1['TIMESTAMP'] != tar2['TIMESTAMP']:
            NDiff2 +=1

        if tar1['SUBPRIORITY'] != tar2['SUBPRIORITY']:
            NDiff3 +=1

    print('Number targets with different NUMOBS')
    print(NDiff)
    print('Number targets with different TIMESTAMP')
    print(NDiff2)
    print('Number targets with different SUBPRIORITY')
    print(NDiff3)

def trimToMTL(notMTL, MTL, debug = False):
    # JL trims a target file, which possesses all of the information in an MTL, down
    # JL to the columns allowed in the MTL data model.
    allNames = notMTL.dtype.names
    MTLNames = MTL.dtype.names
    for n in allNames:
        if n in MTLNames:
            if debug:
                print('allowed')
                print(n)
            continue
        else:
            if debug:
                print('killed')
                print(n)
            notMTL = rfn.drop_fields(notMTL, n)
    return notMTL

def initializeAlternateMTLs(initMTL, outputMTL, nAlt = 2, genSubset = None, seed = 314159, obscon = 'DARK', survey = 'sv3', saveBackup = False, overwrite = False, ztilefile = '/global/cfs/cdirs/desi/survey/ops/surveyops/trunk/ops/tiles-specstatus.ecsv', hpnum = None, shuffleBrightPriorities = False, PromoteFracBGSFaint = 0.2):

    if ('trunk' in outputMTL.lower()) or  ('ops' in outputMTL.lower()):
        raise ValueError("In order to prevent accidental overwriting of the real MTLs, please remove \'ops\' and \'trunk\' from your MTL output directory")

    ztilefn = ztilefile.split('/')[-1]
    fn = initMTL.split('/')[-1]

    allentries = Table.read(initMTL)

    meta = allentries.meta
    print('---')
    print(meta)
    print(initMTL)
    print(outputMTL)
    print('---')

    firstTS = allentries[0]["TIMESTAMP"]
    initialentries = allentries[allentries["TIMESTAMP"] == firstTS]
    subpriorsInit = initialentries["SUBPRIORITY"]
    if not genSubset is None:
        if type(genSubset) == int:
            iterloop = [genSubset]
        elif (type(genSubset) == list) or (type(genSubset) == np.ndarray):
            iterloop = genSubset
    else:
        iterloop = range(nAlt)

    for n in iterloop:
        outputMTLDir = outputMTL.format(n)
        outfile = outputMTLDir +'/' + str(survey).lower() + '/' + str(obscon).lower() + '/' + str(fn)
        if os.path.exists(outfile):
            if overwrite:
                os.remove(outfile)
            else:
                continue

        if type(hpnum) == int:
            rand.seed(seed + hpnum + n)
        else:
            rand.seed(seed + n)

        if not os.path.exists(outputMTLDir):
            os.makedirs(outputMTLDir)
        if not os.path.isfile(outputMTLDir + ztilefn):
            os.symlink(ztilefile, outputMTLDir + ztilefn)
        subpriors = initialentries['SUBPRIORITY']
        #shuffler = rand.permutation(len(subpriors))
        newSubpriors = rand.uniform(size = len(subpriors))
        #newSubpriors = subpriors[shuffler]
        try:
            assert((np.std(subpriorsInit - newSubpriors) > 0.001) | (len(subpriors) < 2))
        except:
            print('first shuffle failed')
            print('size of initial subprior array')
            print(len(subpriorsInit))

            #print('Is initial shuffler sorted')
            #print(np.all(np.diff(shuffler) >= 0))
            #shuffler = rand.permutation(len(subpriors))
            newSubpriors = rand.uniform(size = len(subpriors))
            #newSubpriors = subpriors[shuffler]
            assert((np.std(subpriorsInit - newSubpriors) > 0.001) | (len(subpriors) < 2))

        initialentries['SUBPRIORITY'] = newSubpriors

        if (obscon.lower() == 'bright') and (shuffleBrightPriorities):

            #BGSBits = initialentries['SV3_BGS_TARGET']
            #BGSFaintHIP = ((BGSBits & 8) == 8)
            #BGSFaintAll = ((BGSBits & 1) == 1) | BGSFaintHIP
            #BGSPriors = initialentries['PRIORITY']

            #BGSBits[BGSFaintHIP] = (BGSBits[BGSFaintHIP] & ~8)
            #BGSPriors[BGSFaintHIP] = 102000*np.ones(np.sum(BGSFaintHIP))

            BGSBits = initialentries['SV3_BGS_TARGET']
            BGSFaintHIP = ((BGSBits & 8) == 8)
            BGSFaintAll = ((BGSBits & 1) == 1) | BGSFaintHIP

            #Set all BGS_FAINT_HIP to BGS_FAINT

            initialentries['SV3_BGS_TARGET'][BGSFaintHIP] = (BGSBits[BGSFaintHIP] & ~8)
            initialentries['PRIORITY'][BGSFaintHIP] = 102000*np.ones(np.sum(BGSFaintHIP))

            NewBGSBits = initialentries['SV3_BGS_TARGET']
            NewBGSFaintHIP = ((BGSBits & 8) == 8)
            NewBGSFaintAll = ((BGSBits & 1) == 1) | NewBGSFaintHIP
            NewBGSPriors = initialentries['PRIORITY']
            #Select 20% of BGS_FAINT to promote using function from
            BGSFaintNewHIP = random_fraction_of_trues(PromoteFracBGSFaint, BGSFaintAll)
            #Promote them

            initialentries['SV3_BGS_TARGET'][BGSFaintNewHIP] = (BGSBits[BGSFaintNewHIP] | 8)
            initialentries['PRIORITY'][BGSFaintNewHIP] = 102100*np.ones(np.sum(BGSFaintNewHIP)).astype(int)
        print('meta passed to write_mtl')
        print(meta)
        print('--')
        io.write_mtl(outputMTLDir, initialentries, survey=survey, obscon=obscon, extra=meta, nsidefile=meta['FILENSID'], hpxlist = [meta['FILEHPX']])

        if saveBackup:
            if not os.path.exists(str(outputMTLDir) +'/' + str(survey).lower() + '/' +str(obscon).lower() + '/orig/'):
                os.makedirs(str(outputMTLDir) +'/' + str(survey).lower() + '/' +str(obscon).lower() + '/orig/')

            from shutil import copyfile

            copyfile(str(outputMTLDir) +'/' + str(survey).lower() + '/' + str(obscon).lower() + '/' + str(fn), str(outputMTLDir) +'/' + str(survey).lower() + '/' +str(obscon).lower() + '/orig/' + str(fn))

def quickRestartFxn(ndirs = 1, altmtlbasedir = None, survey = 'sv3', obscon = 'dark', multiproc =False, nproc = None):
    print('quick restart running')
    from shutil import copyfile, move
    from glob import glob as ls
    if multiproc:
        iterloop = range(nproc, nproc+1)
    else:
        iterloop = range(ndirs)
    for nRestart in iterloop:
        print(nRestart)
        altmtldirRestart = altmtlbasedir + '/Univ{0:03d}/'.format(nRestart)
        if os.path.exists(altmtldirRestart + 'mtl-done-tiles.ecsv'):
            move(altmtldirRestart + 'mtl-done-tiles.ecsv',altmtldirRestart + 'mtl-done-tiles.ecsv.old')
        restartMTLs = ls(altmtldirRestart +'/' + survey + '/' + obscon + '/' + '/orig/*')
        #print(altmtldirRestart +'/' + survey + '/' + obscon + '/' + '/orig/*')
        #print(restartMTLs)
        for fn in restartMTLs:
            #print('r')
            copyfile(fn, altmtldirRestart +'/' + survey + '/' + obscon + '/' + fn.split('/')[-1])

def loop_alt_ledger(obscon, survey='sv3', zcatdir=None, mtldir=None,
                altmtlbasedir=None, ndirs = 3, numobs_from_ledger=True,
                secondary=False, singletile = None, singleDate = None, debugOrig = False,
                    getosubp = False, quickRestart = False, redoFA = False,
                    multiproc = False, nproc = None, testDoubleDate = False, changeFiberOpt = None):
    """Execute full MTL loop, including reading files, updating ledgers.

    Parameters
    ----------
    obscon : :class:`str`
        A string matching ONE obscondition in the desitarget bitmask yaml
        file (i.e. in `desitarget.targetmask.obsconditions`), e.g. "DARK"
        Governs how priorities are set when merging targets.
    survey : :class:`str`, optional, defaults to "main"
        Used to look up the correct ledger, in combination with `obscon`.
        Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
        for the main survey and different iterations of SV, respectively.
    zcatdir : :class:`str`, optional, defaults to ``None``
        Full path to the "daily" directory that hosts redshift catalogs.
        If this is ``None``, look up the redshift catalog directory from
        the $ZCAT_DIR environment variable.
    mtldir : :class:`str`, optional, defaults to ``None``
        Full path to the directory that hosts the MTL ledgers and the MTL
        tile file. If ``None``, then look up the MTL directory from the
        $MTL_DIR environment variable.
    altmtlbasedir : :class:`str`, optional, defaults to ``None``
        Formattable path to a directory that hosts alternate MTL ledgers
        If ``None``, then look up the MTL directory from the
        $ALT_MTL_DIR environment variable. This will fail since that variable
        is not currently set in the desi code setup.
    ndirs : :class:`int`, optional, defaults to ``3``
        Number of alternate MTLs to process within altmtlbasedir
    numobs_from_ledger : :class:`bool`, optional, defaults to ``True``
        If ``True`` then inherit the number of observations so far from
        the ledger rather than expecting it to have a reasonable value
        in the `zcat.`
    secondary : :class:`bool`, optional, defaults to ``False``
        If ``True`` then process secondary targets instead of primaries
        for passed `survey` and `obscon`.
    quickRestart : :class:`bool`, optional, defaults to ``False``
        If ``True`` then copy original alternate MTLs from
        altmtlbasedir/Univ*/survey/obscon/orig and
    redoFA : :class:`bool`, optional, defaults to ``False``
        If ``True`` then automatically redo fiberassignment regardless of
        existence of fiberassign file in alternate fiberassign directory
    multiproc : :class:`bool`, optional, defaults to ``False``
        If ``True`` then run a single MTL update in a directory specified by
        nproc.
    nproc : :class:`int`, optional, defaults to None
        If multiproc is ``True`` this must be specified. Integer determines
        directory of alternate MTLs to update.

    Returns
    -------
    :class:`str`
        The directory containing the ledger that was updated.
    :class:`str`
        The name of the MTL tile file that was updated.
    :class:`str`
        The name of the ZTILE file that was used to link TILEIDs to
        observing conditions and to determine if tiles were "done".
    :class:`~numpy.array`
        Information for the tiles that were processed.

    Notes
    -----
    - Assumes all of the relevant ledgers have already been made by,
      e.g., :func:`~LSS.SV3.altmtltools.initializeAlternateMTLs()`.
    """
    if ('trunk' in altmtlbasedir.lower()) or  ('ops' in altmtlbasedir.lower()):
        raise ValueError("In order to prevent accidental overwriting of the real MTLs, please remove \'ops\' and \'trunk\' from your MTL output directory")
    assert((singleDate is None) or (type(singleDate) == bool))
    if multiproc:
        import multiprocessing as mp
        import logging

        logger=mp.log_to_stderr(logging.DEBUG)
    if quickRestart:
        quickRestartFxn(ndirs = ndirs, altmtlbasedir = altmtlbasedir, survey = survey, obscon = obscon, multiproc = multiproc, nproc = nproc)
    # ADM first grab all of the relevant files.
    # ADM grab the MTL directory (in case we're relying on $MTL_DIR).
    mtldir = get_mtl_dir(mtldir)
    # ADM construct the full path to the mtl tile file.
    mtltilefn = os.path.join(mtldir, get_mtl_tile_file_name(secondary=secondary))
    # ADM construct the relevant sub-directory for this survey and
    # ADM set of observing conditions..
    form = get_mtl_ledger_format()
    resolve = True
    msg = "running on {} ledger with obscon={} and survey={}"
    if secondary:
        log.info(msg.format("SECONDARY", obscon, survey))
        resolve = None
    else:
        log.info(msg.format("PRIMARY", obscon, survey))

    # ADM grab the zcat directory (in case we're relying on $ZCAT_DIR).
    zcatdir = get_zcat_dir(zcatdir)
    # ADM And contruct the associated ZTILE filename.
    ztilefn = os.path.join(zcatdir, get_ztile_file_name())

    if altmtlbasedir is None:
        print('This will automatically find the alt mtl dir in the future but fails now. Bye.')
        assert(0)
    if debugOrig:
        iterloop = range(1)
    elif multiproc:
        iterloop = range(nproc, nproc+1)
    else:
        iterloop = range(ndirs)
    for n in iterloop:
        if debugOrig:
            altmtldir = altmtlbasedir
        else:
            altmtldir = altmtlbasedir + '/Univ{0:03d}/'.format(n)
        altmtltilefn = os.path.join(altmtldir, get_mtl_tile_file_name(secondary=secondary))

        althpdirname = io.find_target_files(altmtldir, flavor="mtl", resolve=resolve,
                                     survey=survey, obscon=obscon, ender=form)
        # ADM grab an array of tiles that are yet to be processed.
        tiles = tiles_to_be_processed(zcatdir, altmtltilefn, obscon, survey)
        # ADM stop if there are no tiles to process.
        if len(tiles) == 0:
            if (not multiproc) and (n != ndirs - 1):
                continue
            else:
                if singleDate:
                    return 0
                else:
                    return althpdirname, mtltilefn, ztilefn, tiles
        if not (singletile is None):
            tiles = tiles[tiles['TILEID'] == singletile]

        sorttiles = np.sort(tiles, order = 'ZDATE')
        if testDoubleDate:
            print('Testing Rosette with Doubled Date only')
            cond1 = ((tiles['TILEID'] >= 298) & (tiles['TILEID'] <= 324))
            cond2 = ((tiles['TILEID'] >= 475) & (tiles['TILEID'] <= 477))
            print(tiles[tiles['TILEID' ] == 314])
            print(tiles[tiles['TILEID' ] == 315])
            tiles = tiles[cond1 | cond2 ]

        dates = np.sort(np.unique(tiles['ZDATE']))
        for date in dates:
            dateTiles = tiles[tiles['ZDATE'] == date]
            OrigFAs = []
            AltFAs = []
            AltFAs2 = []
            TSs = []
            fadates = []

            for t in dateTiles:
                ts = str(t['TILEID']).zfill(6)
                FAOrigName = '/global/cfs/cdirs/desi/target/fiberassign/tiles/trunk/'+ts[:3]+'/fiberassign-'+ts+'.fits.gz'
                fhtOrig = fitsio.read_header(FAOrigName)
                fadate = fhtOrig['RUNDATE']
                fadate = ''.join(fadate.split('T')[0].split('-'))

                fbadirbase = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/'
                if getosubp:
                    FAAltName = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/orig/fba-' + ts+ '.fits'
                    fbadir = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/orig/'
                else:

                    FAAltName = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/fba-' + ts+ '.fits'
                    fbadir = fbadirbase

                if os.path.exists(FAAltName + '.tmp'):
                    os.remove(FAAltName + '.tmp')

                if  redoFA or (not os.path.exists(FAAltName)):
                    get_fba_fromnewmtl(ts,mtldir=altmtldir + survey.lower() + '/',outdir=fbadirbase, getosubp = getosubp, overwriteFA = redoFA)
                    command_run = (['bash', fbadir + 'fa-' + ts + '.sh'])
                    result = subprocess.run(command_run, capture_output = True)
                OrigFAs.append(pf.open(FAOrigName)[1].data)
                AltFAs.append(pf.open(FAAltName)[1].data)
                AltFAs2.append(pf.open(FAAltName)[2].data)
                TSs.append(ts)
                fadates.append(fadate)
            # ADM create the catalog of updated redshifts.
            zcat = make_zcat(zcatdir, dateTiles, obscon, survey)
            # ADM insist that for an MTL loop with real observations, the zcat
            # ADM must conform to the data model. In particular, it must include
            # ADM ZTILEID, and other columns addes for the Main Survey. These
            # ADM columns may not be needed for non-ledger simulations.
            # ADM Note that the data model differs with survey type.
            zcatdm = survey_data_model(zcatdatamodel, survey=survey)
            if zcat.dtype.descr != zcatdm.dtype.descr:
                msg = "zcat data model must be {} not {}!".format(
                    zcatdm.dtype.descr, zcat.dtype.descr)
                log.critical(msg)
                raise ValueError(msg)
            # ADM useful to know how many targets were updated.
            _, _, _, _, sky, _ = decode_targetid(zcat["TARGETID"])
            ntargs, nsky = np.sum(sky == 0), np.sum(sky)
            msg = "Update state for {} targets".format(ntargs)
            msg += " (the zcats also contain {} skies with +ve TARGETIDs)".format(nsky)
            log.info(msg)

            A2RMap = {}
            R2AMap = {}
            for ofa, afa, afa2 in zip (OrigFAs, AltFAs, AltFAs2):
                if changeFiberOpt is None:
                    A2RMapTemp, R2AMapTemp = createFAmap(ofa, afa, changeFiberOpt = changeFiberOpt)
                else:

                    FAOrigName = '/global/cfs/cdirs/desi/target/fiberassign/tiles/trunk/'+ts[:3]+'/fiberassign-'+ts+'.fits.gz'

                    fbadirbase = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/'
                    if getosubp:
                        FAAltName = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/orig/fba-' + ts+ '.fits'
                        fbadir = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/orig/'
                    else:

                        FAAltName = altmtldir + '/fa/' + survey.upper() +  '/' + fadate + '/fba-' + ts+ '.fits'
                        fbadir = fbadirbase

                    A2RMapTemp, R2AMapTemp = createFAmap(ofa, afa, TargAlt = afa2, changeFiberOpt = changeFiberOpt)
                A2RMap.update(A2RMapTemp)
                R2AMap.update(R2AMapTemp)

            altZCat = makeAlternateZCat(zcat, R2AMap, A2RMap)

            # ADM update the appropriate ledger.
            update_ledger(althpdirname, altZCat, obscon=obscon.upper(),
                          numobs_from_ledger=numobs_from_ledger)
            if survey == "main":
                sleep(1)
                tiles["TIMESTAMP"] = get_utc_date(survey=survey)
            io.write_mtl_tile_file(altmtltilefn,dateTiles)

            if singleDate:
                return 1
    return althpdirname, altmtltilefn, ztilefn, tiles

def plotMTLProb(mtlBaseDir, ndirs = 10, hplist = None, obscon = 'dark', survey = 'sv3', outFileName = None, outFileType = '.png', jupyter = False):
    """Plots probability that targets were observed among {ndirs} alternate realizations
    of SV3. Uses default matplotlib colorbar to plot between 1-{ndirs} observations.

    Parameters
    ----------
    mtlBaseDir : :class:`str`
        The home directory of your alternate MTLs. Should not contain obscon
        or survey. String should be formattable (i.e. '/path/to/dirs/Univ{0:03d}')
    ndirs   : :class:`int`
        The number of alternate realizations to plot.
    survey : :class:`str`, optional, defaults to "sv3"
        Used to look up the correct ledger, in combination with `obscon`.
        Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
        for the main survey and different iterations of SV, respectively.
    obscon : :class:`str`, optional, defaults to "dark"
        Used to look up the correct ledger, in combination with `survey`.
        Options are ``'dark'`` and ``'bright``'
    hplist : :class:`arraylike`, optional, defaults to None
        List of healpixels to plot. If None, defaults to plotting all available
        healpixels
    outFileName : :class:`str`, optional, defaults to None
        If desired, save file to this location. This will
        usually be desired, but was made optional for use in
        ipython notebooks.
    outFileType : :class:`str`, optional, defaults to '.png'
        If desired, save file with name "outFileName" with
        type/suffix outFileType. This will usually be desired,
        but was made optional for use in ipython notebooks.

    Returns
    -------
    Nothing

    """
    ObsFlagList = np.array([])
    for i in range(ndirs):
        mtldir = mtlBaseDir.format(i) + '/' + survey + '/' + obscon
        MTL = np.sort(desitarget.io.read_mtl_in_hp(mtldir, 32, hplist, unique=True, isodate=None, returnfn=False, initial=False, leq=False), order = 'TARGETID')
        try:
            ObsFlagList = np.column_stack((ObsFlagList,MTL['NUMOBS'] > 0.5))
        except:
            print('e')
            ObsFlagList = MTL['NUMOBS'] > 0.5
    print(ObsFlagList.shape)
    ObsArr = np.sum(ObsFlagList, axis = 1)

    #MTLList[i] = rfn.append_fields(MTLList[i], 'OBSFLAG', MTLList[i]['NUMOBS'] > 0, dtypes=np.dtype(bool))

    hist, bins = np.histogram(ObsArr, bins = np.arange(ndirs)+ 0.1)

    plt.figure()
    plt.plot(bins[1:]- 0.01, hist)
    plt.xlabel('Number of Realizations in which a target was observed')
    plt.ylabel('Number of targets')
    #plt.yscale('log')
    if len(hplist )> 100:
        plt.ylim(0, 8000)
    elif (len(hplist) > 4) & (obscon == 'dark'):
        plt.ylim(0, 1500)
    elif (len(hplist) > 4) & (obscon == 'bright'):
        plt.ylim(0, 500)
    if not (outFileName is None):
        plt.savefig(outFileName + '_vsNtarget' + outFileType)
    if not jupyter:
        plt.close()
    plt.figure()
    plt.scatter(MTL['RA'][ObsArr > 0], MTL['DEC'][ObsArr > 0], c = ObsArr[ObsArr > 0], s = 0.1)
    plt.xlabel('RA')
    plt.ylabel('DEC')
    cbar = plt.colorbar()
    cbar.set_label('Number of Realizations in which target was observed')
    if not (outFileName is None):
        plt.savefig(outFileName + '_vsRADEC' + outFileType )
    if not jupyter:
        plt.close()

def makeBitweights(mtlBaseDir, ndirs = 64, hplist = None, obscon = 'dark', survey = 'sv3', debug = False, obsprob = False, splitByReal = False):
    """Takes a set of {ndirs} realizations of DESI/SV3 and converts their MTLs into bitweights
    and an optional PROBOBS, the probability that the target was observed over the realizations

    Parameters
    ----------
    mtlBaseDir : :class:`str`
        The home directory of your alternate MTLs. Should not contain obscon
        or survey. String should be formattable (i.e. '/path/to/dirs/Univ{0:03d}')
    ndirs   : :class:`int`
        The number of alternate realizations to process.
    survey : :class:`str`, optional, defaults to "sv3"
        Used to look up the correct ledger, in combination with `obscon`.
        Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
        for the main survey and different iterations of SV, respectively.
    obscon : :class:`str`, optional, defaults to "dark"
        Used to look up the correct ledger, in combination with `survey`.
        Options are ``'dark'`` and ``'bright``'
    hplist : :class:`arraylike`, optional, defaults to None
        List of healpixels to plot. If None, defaults to plotting all available
        healpixels
    debug : :class:`bool`, optional, defaults to False
        If True, prints extra information showing input observation information
        and output bitweight information for the first few targets as well as
        the first few targets that were observed in at least one realization
    obsprob: class:`bool`, optional, defaults to False
        If True, returns TARGETID, BITWEIGHT, and OBSPROB. Else returns TARGETID
        and BITWEIGHT only
    splitByReal: class:`bool`, optional, defaults to False
        If True, run for only a single realization but for all healpixels in hplist.

    Returns
    -------
    :class:`~numpy.array`
        Array of Target IDs
    :class:`~numpy.array`
        Array of bitweights for those target ids
    :class:`~numpy.array`, optional if obsprob is True
        Array of probabilities a target gets observed over {ndirs} realizations

    """

    TIDs = None
    if splitByReal:

        from mpi4py import MPI
        print(mtlBaseDir)
        print(mtlBaseDir.format(0))
        ntar = desitarget.io.read_mtl_in_hp(mtlBaseDir.format(0) + '/' + survey + '/' + obscon, 32, hplist, unique=True, isodate=None, returnfn=False, initial=False, leq=False).shape[0]

        comm = MPI.COMM_WORLD
        mpi_procs = comm.size
        mpi_rank = comm.rank

        print('running on {0:d} cores'.format(mpi_procs))
        n_realization = ndirs
        realizations = np.arange(ndirs, dtype=np.int32)
        my_realizations = np.array_split(realizations, mpi_procs)[mpi_rank]
        MyObsFlagList = np.empty((my_realizations.shape[0], ntar), dtype = bool)
        #MTL = np.sort(desitarget.io.read_mtl_in_hp(mtldir, 32, hplist, unique=True, isodate=None, returnfn=False, initial=False, leq=False), order = 'TARGETID')
        for i, r in enumerate(my_realizations):
            mtldir = mtlBaseDir.format(i) + '/' + survey + '/' + obscon
            MTL = np.sort(desitarget.io.read_mtl_in_hp(mtldir, 32, hplist, unique=True, isodate=None, returnfn=False, initial=False, leq=False), order = 'TARGETID')
            if TIDs is None:
                TIDs = MTL['TARGETID']
            else:
                assert(np.array_equal(TIDs, MTL['TARGETID']))

            MyObsFlagList[i][:] = MTL['NUMOBS'] > 0.5

        ObsFlagList = None
        bitweights = None
        obsprobs = None
        #gather_weights = None
        if mpi_rank == 0:
            #gather_weights = np.empty(len(bitweights), dtype=bool)
            ObsFlagList = np.empty ((ndirs, ntar), dtype = bool)
        comm.Gather(MyObsFlagList, ObsFlagList, root=0)
        if mpi_rank == 0:
            print(ObsFlagList.shape)
            ObsArr = np.sum(ObsFlagList, axis = 0)
            obsprobs = ObsArr/ndirs
            print(np.min(ObsArr))
            print(np.max(ObsArr))
            print("ObsFlagList shape here: {0}".format(ObsFlagList.shape))
            bitweights = pack_bitweights(ObsFlagList.T)
            print('bitweights shape here: {0}'.format(bitweights.shape))
            print('TIDs shape here: {0}'.format(TIDs.shape))
            assert(not (TIDs is None))
        if obsprob:
            return TIDs, bitweights, obsprobs
        else:
            return TIDs, bitweights

    else:
        ObsFlagList = np.empty(ndirs)
        for i in range(ndirs):
            mtldir = mtlBaseDir.format(i) + '/' + survey + '/' + obscon
            MTL = np.sort(desitarget.io.read_mtl_in_hp(mtldir, 32, hplist, unique=True, isodate=None, returnfn=False, initial=False, leq=False), order = 'TARGETID')
            if TIDs is None:
                TIDs = MTL['TARGETID']
            else:
                assert(np.array_equal(TIDs, MTL['TARGETID']))
            try:
                ObsFlagList = np.column_stack((ObsFlagList,MTL['NUMOBS'] > 0.5))
            except:
                print('e')
                ObsFlagList = MTL['NUMOBS'] > 0.5
        print(ObsFlagList.shape)
        ObsArr = np.sum(ObsFlagList, axis = 1)
        print(np.min(ObsArr))
        print(np.max(ObsArr))
        bitweights = pack_bitweights(ObsFlagList)

        assert(not (TIDs is None))
        if obsprob:

            obsprobs = ObsArr/ndirs

            return TIDs, bitweights, obsprobs
        else:
            return TIDs, bitweights

def writeBitweights(mtlBaseDir, ndirs = None, hplist = None, debug = False, outdir = None, obscon = "dark", survey = 'sv3', overwrite = False, allFiles = False, splitByReal = False, splitNChunks = None):
    """Takes a set of {ndirs} realizations of DESI/SV3 and converts their MTLs into bitweights
    and an optional PROBOBS, the probability that the target was observed over the realizations.
    Then writes them to (a) file(s)

    Parameters
    ----------
    mtlBaseDir : :class:`str`
        The home directory of your alternate MTLs. Should not contain obscon
        or survey. String should be formattable (i.e. '/path/to/dirs/Univ{0:03d}')
    ndirs   : :class:`int`
        The number of alternate realizations to process.
    survey : :class:`str`, optional, defaults to "sv3"
        Used to look up the correct ledger, in combination with `obscon`.
        Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
        for the main survey and different iterations of SV, respectively.
    obscon : :class:`str`, optional, defaults to "dark"
        Used to look up the correct ledger, in combination with `survey`.
        Options are ``'dark'`` and ``'bright``'
    hplist : :class:`arraylike`, optional, defaults to None
        List of healpixels to plot. If None, defaults to plotting all available
        healpixels
    debug : :class:`bool`, optional, defaults to False
        If True, prints extra information showing input observation information
        and output bitweight information for the first few targets as well as
        the first few targets that were observed in at least one realization
    obsprob: class:`bool`, optional, defaults to False
        If True, returns TARGETID, BITWEIGHT, and OBSPROB. Else returns TARGETID
        and BITWEIGHT only
    outdir : :class:`str`, optional, defaults to None
        The base directory in which to create the BitweightFiles output directory.
        If None, defaults to one level above mtlBaseDir
    overwrite: class:`bool`, optional, defaults to False
        If True, will clobber already existing bitweight files.
    ***Fix this Option to autograb all tiles***
    allfiles: class:`bool`, optional, defaults to False
        If True, do not generate a bitweight file for each healpixel, but generate
        one "allTiles" file for the combination of healpixels
    splitByReal: class:`bool`, optional, defaults to False
        If True, run for only a single realization but for all healpixels in hplist

    Returns
    -------
    :class:`~numpy.array`
        Array of Target IDs
    :class:`~numpy.array`
        Array of bitweights for those target ids
    :class:`~numpy.array`, optional if obsprob is True
        Array of probabilities a target gets observed over {ndirs} realizations

    """
    if outdir is None:
        print('No outdir provided')
        outdir = mtlBaseDir.split('/')[:-1]
        print('autogen outdir')
        print(outdir)
    if splitByReal:
        from mpi4py import MPI
        comm = MPI.COMM_WORLD
        mpi_procs = comm.size
        mpi_rank = comm.rank
        if mpi_rank == 0:
            if not os.path.exists(outdir + '/BitweightFiles/' + survey + '/' + obscon):
                os.makedirs(outdir + '/BitweightFiles/' + survey + '/' + obscon)
    elif not os.path.exists(outdir + '/BitweightFiles/' + survey + '/' + obscon):
        os.makedirs(outdir + '/BitweightFiles/' + survey + '/' + obscon)
    if type(hplist) == int:
        hplist = [hplist]
    if allFiles:
        hpstring = 'AllTiles'
    else:
        hpstring = 'hp-'

        for hp in hplist:
            hpstring += str(hp)
    fn = outdir + '/BitweightFiles/' + survey + '/' + obscon + '/{0}bw-{1}-'.format(survey.lower(), obscon.lower()) + hpstring + '.fits'

    if (not overwrite) and os.path.exists(outdir + '/BitweightFiles/' + survey + '/' + obscon + '/{0}bw-{1}-'.format(survey.lower(), obscon.lower()) + hpstring + '.fits'):
        return None

    if not (splitNChunks is None):
        print('makeBitweights1')
        print("splitting into {0} chunks".format(splitNChunks))
        splits = np.array_split(hplist, int(splitNChunks))

        for i, split in enumerate(splits):
            print('split {0}'.format(i))
            print(split)
            if i == 0:
                TIDs, bitweights, obsprobs = makeBitweights(mtlBaseDir, ndirs = ndirs, hplist = split, debug = False, obsprob = True, obscon = obscon, survey = survey, splitByReal = splitByReal)
            else:
                TIDsTemp, bitweightsTemp, obsprobsTemp = makeBitweights(mtlBaseDir, ndirs = ndirs, hplist = split, debug = False, obsprob = True, obscon = obscon, survey = survey, splitByReal = splitByReal)

                if mpi_rank == 0:
                    print('----')
                    print('mpi_rank: {0}'.format(mpi_rank))
                    print("TIDs shape: {0}".format(TIDs.shape))
                    print("bitweights shape: {0}".format(bitweights.shape))
                    print("obsprobs shape: {0}".format(obsprobs.shape))
                    print('----')
                    print('mpi_rank: {0}'.format(mpi_rank))
                    print("TIDsTemp shape: {0}".format(TIDsTemp.shape))
                    print("bitweightsTemp shape: {0}".format(bitweightsTemp.shape))
                    print("obsprobsTemp shape: {0}".format(obsprobsTemp.shape))
                    TIDs = np.hstack((TIDs, TIDsTemp))
                    bitweights = np.vstack((bitweights, bitweightsTemp))
                    obsprobs = np.hstack((obsprobs, obsprobsTemp))
    else:
        print('makeBitweights2')
        TIDs, bitweights, obsprobs = makeBitweights(mtlBaseDir, ndirs = ndirs, hplist = hplist, debug = False, obsprob = True, obscon = obscon, survey = survey, splitByReal = splitByReal)
    if splitByReal:
        print('----')
        print('mpi_rank: {0}'.format(mpi_rank))
        if mpi_rank == 0:
            print("TIDs shape: {0}".format(TIDs.shape))
            print("bitweights shape: {0}".format(bitweights.shape))
            print("obsprobs shape: {0}".format(obsprobs.shape))
            data = Table({'TARGETID': TIDs, 'BITWEIGHTS': bitweights, 'PROB_OBS': obsprobs},
                      names=['TARGETID', 'BITWEIGHTS', 'PROB_OBS'])

            data.write(fn, overwrite = overwrite)
    else:
        print("TIDs shape: {0}".format(TIDs.shape))
        print("bitweights shape: {0}".format(bitweights.shape))
        print("obsprobs shape: {0}".format(obsprobs.shape))
        data = Table({'TARGETID': TIDs, 'BITWEIGHTS': bitweights, 'PROB_OBS': obsprobs},
              names=['TARGETID', 'BITWEIGHTS', 'PROB_OBS'])

        data.write(fn, overwrite = overwrite)

max_stars_repo_name: echaussidon/LSS

max_issues_repo_name: echaussidon/LSS

max_forks_repo_name: echaussidon/LSS

reason: import numpy,from numpy,import astropy,from astropy


text:
import numpy as np
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf
from ffn.training.import_util import import_symbol
import time
from train import prepare_ffn, define_data_input, train_eval_size, get_batch, EvalTracker

import platform
if platform.system() == 'Windows':
    DIVSTR = '::'
elif platform.system() in ['Linux', 'Darwin']:
    DIVSTR = ':'
else:
    DIVSTR = ':'

FLAGS = flags.FLAGS

# Options related to training data.
flags.DEFINE_string('train_coords', None,
                    'Glob for the TFRecord of training coordinates.') # What's the use of TFRecord
flags.DEFINE_string('data_volumes', None,
                    'Comma-separated list of <volume_name>:<volume_path>:'
                    '<dataset>, where volume_name need to match the '
                    '"label_volume_name" field in the input example, '
                    'volume_path points to HDF5 volumes containing uint8 '
                    'image data, and `dataset` is the name of the dataset '
                    'from which data will be read.')
flags.DEFINE_string('label_volumes', None,
                    'Comma-separated list of <volume_name>:<volume_path>:'
                    '<dataset>, where volume_name need to match the '
                    '"label_volume_name" field in the input example, '
                    'volume_path points to HDF5 volumes containing int64 '
                    'label data, and `dataset` is the name of the dataset '
                    'from which data will be read.')
flags.DEFINE_string('model_name', None,
                    'Name of the model to train. Format: '
                    '[<packages>.]<module_name>.<model_class>, if packages is '
                    'missing "ffn.training.models" is used as default.')
flags.DEFINE_string('model_args', None,
                    'JSON string with arguments to be passed to the model '
                    'constructor.')

# Training infra options.
flags.DEFINE_string('train_dir', '/tmp',
                    'Path where checkpoints and other data will be saved.')
flags.DEFINE_string('master', '', 'Network address of the master.')
flags.DEFINE_integer('batch_size', 4, 'Number of images in a batch.')
flags.DEFINE_integer('task', 0, 'Task id of the replica running the training.')
flags.DEFINE_integer('ps_tasks', 0, 'Number of tasks in the ps job.')
flags.DEFINE_integer('max_steps', 10000, 'Number of steps to train for.')
flags.DEFINE_integer('replica_step_delay', 300,
                     'Require the model to reach step number '
                     '<replica_step_delay> * '
                     '<replica_id> before starting training on a given '
                     'replica.')
flags.DEFINE_integer('summary_rate_secs', 120,
                     'How often to save summaries (in seconds).')

# FFN training options.
flags.DEFINE_float('seed_pad', 0.05,
                   'Value to use for the unknown area of the seed.')
flags.DEFINE_float('threshold', 0.9,
                   'Value to be reached or exceeded at the new center of the '
                   'field of view in order for the network to inspect it.')
flags.DEFINE_enum('fov_policy', 'fixed', ['fixed', 'max_pred_moves'],
                  'Policy to determine where to move the field of the '
                  'network. "fixed" tries predefined offsets specified by '
                  '"model.shifts". "max_pred_moves" moves to the voxel with '
                  'maximum mask activation within a plane perpendicular to '
                  'one of the 6 Cartesian directions, offset by +/- '
                  'model.deltas from the current FOV position.')
# TODO(mjanusz): Implement fov_moves > 1 for the 'fixed' policy.
flags.DEFINE_integer('fov_moves', 1,
                     'Number of FOV moves by "model.delta" voxels to execute '
                     'in every dimension. Currently only works with the '
                     '"max_pred_moves" policy.')
flags.DEFINE_boolean('shuffle_moves', True,
                     'Whether to randomize the order of the moves used by the '
                     'network with the "fixed" policy.')

flags.DEFINE_float('image_mean', None,
                   'Mean image intensity to use for input normalization.')
flags.DEFINE_float('image_stddev', None,
                   'Image intensity standard deviation to use for input '
                   'normalization.')
flags.DEFINE_list('image_offset_scale_map', None,
                  'Optional per-volume specification of mean and stddev. '
                  'Every entry in the list is a colon-separated tuple of: '
                  'volume_label, offset, scale.')

flags.DEFINE_list('permutable_axes', ['1', '2'],
                  'List of integers equal to a subset of [0, 1, 2] specifying '
                  'which of the [z, y, x] axes, respectively, may be permuted '
                  'in order to augment the training data.')

flags.DEFINE_list('reflectable_axes', ['0', '1', '2'],
                  'List of integers equal to a subset of [0, 1, 2] specifying '
                  'which of the [z, y, x] axes, respectively, may be reflected '
                  'in order to augment the training data.')

FLAGS = flags.FLAGS

def run_training_step(sess, model, fetch_summary, feed_dict):
  """Runs one training step for a single FFN FOV."""
  ops_to_run = [model.train_op, model.global_step, model.logits]  # train_op defined in model.set_up_optimizer()

  if fetch_summary is not None:
    ops_to_run.append(fetch_summary)

  results = sess.run(ops_to_run, feed_dict)  # get prediction for the operation
  step, prediction = results[1:3]

  if fetch_summary is not None:
    summ = results[-1]
  else:
    summ = None

  return prediction, step, summ

def train_ffn(model_cls, **model_kwargs):
  with tf.Graph().as_default():
    with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks, merge_devices=True)):
      # The constructor might define TF ops/placeholders, so it is important
      # that the FFN is instantiated within the current context.
      # model instantiation!
      model = model_cls(**model_kwargs)
      eval_shape_zyx = train_eval_size(model).tolist()[::-1]

      eval_tracker = EvalTracker(eval_shape_zyx)
      load_data_ops = define_data_input(model, queue_batch=1)
      prepare_ffn(model)
      merge_summaries_op = tf.summary.merge_all()

      if FLAGS.task == 0:
        save_flags()
      # Setup the Higher Order session enviroment!
      summary_writer = None
      saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.25)
      scaffold = tf.train.Scaffold(saver=saver)
      with tf.train.MonitoredTrainingSession(
          master=FLAGS.master,
          is_chief=(FLAGS.task == 0),
          save_summaries_steps=None,
          save_checkpoint_secs=300,
          config=tf.ConfigProto(log_device_placement=True, allow_soft_placement=True),
          checkpoint_dir=FLAGS.train_dir,
          scaffold=scaffold) as sess:

        eval_tracker.sess = sess
        step = int(sess.run(model.global_step))  # evaluate the step
        # global_step is the tf inner mechanism that keeps the batches seen by
        # thus when revive a model from checkpoint the `step` can be restored
        if FLAGS.task > 0:
          # To avoid early instabilities when using multiple replicas, we use
          # a launch schedule where new replicas are brought online gradually.
          logging.info('Delaying replica start.')
          while step < FLAGS.replica_step_delay * FLAGS.task:
            time.sleep(5.0)
            step = int(sess.run(model.global_step))
        else:
          summary_writer = tf.summary.FileWriterCache.get(FLAGS.train_dir)
          summary_writer.add_session_log(
              tf.summary.SessionLog(status=tf.summary.SessionLog.START), step)
        # Prepare the batch_it object by input shifts parameters into it
        fov_shifts = list(model.shifts)  # x, y, z  vector collection formed by input deltas in 3*3*3-1 directions
        if FLAGS.shuffle_moves:
          random.shuffle(fov_shifts)

        policy_map = {
            'fixed': partial(fixed_offsets, fov_shifts=fov_shifts),
            'max_pred_moves': max_pred_offsets
        }  # delta, fov_shift come into policy map to
        batch_it = get_batch(lambda: sess.run(load_data_ops),
                             eval_tracker, model, FLAGS.batch_size,
                             policy_map[FLAGS.fov_policy])

        t_last = time.time()
        # Start major loop
        while not sess.should_stop() and step < FLAGS.max_steps:
          # Run summaries periodically.
          t_curr = time.time()
          if t_curr - t_last > FLAGS.summary_rate_secs and FLAGS.task == 0:
            summ_op = merge_summaries_op
            t_last = t_curr  # update at summary_rate_secs
          else:
            summ_op = None

          # Core lines to be modified w.r.t. Multi GPU computing
          seed, patches, labels, weights = next(batch_it)

          updated_seed, step, summ = run_training_step(
              sess, model, summ_op,
              feed_dict={
                  model.loss_weights: weights,
                  model.labels: labels,
                  model.input_patches: patches,
                  model.input_seed: seed,
              })

          # Save prediction results in the original seed array so that
          # they can be used in subsequent steps.
          mask.update_at(seed, (0, 0, 0), updated_seed)

          # Record summaries.
          if summ is not None:
            logging.info('Saving summaries.')
            summ = tf.Summary.FromString(summ)

            # Compute a loss over the whole training patch (i.e. more than a
            # single-step field of view of the network). This quantifies the
            # quality of the final object mask.
            summ.value.extend(eval_tracker.get_summaries())
            eval_tracker.reset()

            assert summary_writer is not None
            summary_writer.add_summary(summ, step)

      if summary_writer is not None:
        summary_writer.flush()

def main(argv=()):
  del argv  # Unused.
  model_class = import_symbol(FLAGS.model_name)
  # Multiply the task number by a value large enough that tasks starting at a
  # similar time cannot end up with the same seed.
  seed = int(time.time() + FLAGS.task * 3600 * 24)
  logging.info('Random seed: %r', seed)
  random.seed(seed)

  train_ffn(model_class, batch_size=FLAGS.batch_size,
            **json.loads(FLAGS.model_args))

ext: py

max_stars_repo_name: Animadversio/FloodFillNetwork-Notes

max_issues_repo_name: Animadversio/FloodFillNetwork-Notes

max_forks_repo_name: Animadversio/FloodFillNetwork-Notes

reason: import numpy


text:
'''
TO DO:
    • add open
    • add high
    • add low
    • make it a function that returns a data fram of open, close, low, high, and growth rate of each since the day before
    •
    • fix the calender so you can add more data //DONE
'''

DEBUG = True
from time import *
from sklearn.svm import SVR
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from yahoo_fin.stock_info import get_live_price, get_data
import requests, bitfinex, datetime, threading, subprocess
from pymongo import MongoClient
from os import path

''' ''' ''' '''
SYMBOL='btc'
SYMBOL+='usd'
''' ''' ''' '''

print('\n'*100)
plt.style.use('seaborn-darkgrid')

#....................................................................................
def writer(predicted):
    with open(SYMBOL+'previouslyPredicted.txt', 'w') as db:
        db.truncate(0)
        db.write(str(predicted))
#....................................................................................
def reader():
    with open(SYMBOL+'previouslyPredicted.txt') as db:
        prePredict = list(db)[0]
    return float(prePredict)
#....................................................................................
try: reader()
except: writer('0')

'''....................................................................................'''
def growthRate(pre, curr): return ((100*curr)/pre)-100
'''....................................................................................'''
def predict(SYMBOL, graph=False, graphTag='close'):
    TIMER=time()

    #....................................................................................
#    '''
    api_v2 = bitfinex.bitfinex_v2.api_v2()
    pair = SYMBOL
    bin_size = '1m'
    limit = 1000
#    t_start = datetime.datetime(2018, 4, 1, 0, 0)
#    t_start = mktime(t_start.timetuple()) * 1000
    #t_stop = datetime.datetime(2018, 4, 2, 0, 0)
    #t_stop = mktime(t_stop.timetuple()) * 1000
    result = api_v2.candles(symbol=pair, interval=bin_size,limit=limit)#, start=t_start, end=t_stop)
    tags = ['time', 'open', 'close', 'high', 'low', 'volume']
    df = pd.DataFrame(result, columns=tags)
    df.drop_duplicates(inplace=True)
    df['time'] = pd.to_datetime(df['time'], unit='ms')
    df.set_index('time', inplace=True)
    df.sort_index(inplace=True)
    #print(df)

    #....................................................................................
    '''
    df = get_data(SYMBOL)#, start_date='08-1-2020')
    if len(df)>=1260: df = df.iloc[-1260:]
#    '''
    #....................................................................................
    actual_price5 = df.tail(1)
#    print(actual_price, get_live_price(SYMBOL))
    #df = df.head(len(df)-1) #<-------- get rid of the last day's data
    days = []
    dates = []
    df_days = df.index.values
    for index, day in enumerate(df_days):
        dates.append( str(day).replace('T00:00:00.000000000','') )
        days.append([index])
    '''
    print(df.columns.tolist())
    '''
    df_adj_close5 = df.loc[:, 'close']
#    df_open = df.loc[:, 'open']
#    df_high = df.loc[:, 'high']
#    df_low = df.loc[:, 'low']
    adj_close_prices5 = [float(adj_close_price5) for adj_close_price5 in df_adj_close5]
#    open_prices = [float(open_price) for open_price in df_open]
#    high_prices = [float(high_price) for high_price in df_high]
#    low_prices = [float(low_price) for low_price in df_low]
    '''Models:'''
    #....................................................................................
    #lin_svr = SVR(kernel='linear', C=1000.0)
    #lin_svr.fit(days, adj_close_prices)
    ''''''
    #poly_svr = SVR(kernel='poly', C=1000.0, degree=2)
    #poly_svr.fit(days, adj_close_prices)
    ''''''
    rbf_svr5 = SVR(kernel='rbf', C=1000.0, gamma=0.15)
    rbf_svr5.fit(days, adj_close_prices5)

    #....................................................................................
#    '''
    api_v2 = bitfinex.bitfinex_v2.api_v2()
    pair = SYMBOL
    bin_size = '1m'
    limit = 1000
#    t_start = datetime.datetime(2018, 4, 1, 0, 0)
#    t_start = mktime(t_start.timetuple()) * 1000
    #t_stop = datetime.datetime(2018, 4, 2, 0, 0)
    #t_stop = mktime(t_stop.timetuple()) * 1000
    result = api_v2.candles(symbol=pair, interval=bin_size,limit=limit)#, start=t_start, end=t_stop)
    tags = ['time', 'open', 'close', 'high', 'low', 'volume']
    df = pd.DataFrame(result, columns=tags)
    df.drop_duplicates(inplace=True)
    df['time'] = pd.to_datetime(df['time'], unit='ms')
    df.set_index('time', inplace=True)
    df.sort_index(inplace=True)
    #print(df)

    #....................................................................................
    '''
    df = get_data(SYMBOL)#, start_date='08-1-2020')
    if len(df)>=1260: df = df.iloc[-1260:]
#    '''
    #....................................................................................
    actual_price = df.tail(1)
#    print(actual_price, get_live_price(SYMBOL))
    #df = df.head(len(df)-1) #<-------- get rid of the last day's data
    days = []
    dates = []
    df_days = df.index.values
    for index, day in enumerate(df_days):
        dates.append( str(day).replace('T00:00:00.000000000','') )
        days.append([index])
    '''
    print(df.columns.tolist())
    '''
    df_adj_close = df.loc[:, 'close']
#    df_open = df.loc[:, 'open']
#    df_high = df.loc[:, 'high']
#    df_low = df.loc[:, 'low']
    adj_close_prices = [float(adj_close_price) for adj_close_price in df_adj_close]
#    open_prices = [float(open_price) for open_price in df_open]
#    high_prices = [float(high_price) for high_price in df_high]
#    low_prices = [float(low_price) for low_price in df_low]
    '''Models:'''
    #....................................................................................
    #lin_svr = SVR(kernel='linear', C=1000.0)
    #lin_svr.fit(days, adj_close_prices)
    ''''''
    #poly_svr = SVR(kernel='poly', C=1000.0, degree=2)
    #poly_svr.fit(days, adj_close_prices)
    ''''''
    rbf_svr = SVR(kernel='rbf', C=1000.0, gamma=0.15)
    rbf_svr.fit(days, adj_close_prices)

#    with open(SYMBOL+'previouslyPredicted.txt') as db:
#        for data in db: prePredict = data
#    with open(SYMBOL+'previouslyPredicted.txt', 'a') as db: db.write(str(*rbf_svr.predict([[len(df)]]))+'\n')

#    rbf_svr_open = SVR(kernel='rbf', C=1000.0, gamma=0.15)
#    rbf_svr_open.fit(days, open_prices)
#
#    rbf_svr_high = SVR(kernel='rbf', C=1000.0, gamma=0.15)
#    rbf_svr_high.fit(days, high_prices)
#
#    rbf_svr_low = SVR(kernel='rbf', C=1000.0, gamma=0.15)
#    rbf_svr_low.fit(days, low_prices)
    #....................................................................................
    '''Outputs:'''
    '''
    print(SYMBOL.upper()+':')

    day = len(df)-1
    print(day)

    print('RBF predict today:                            $', *rbf_svr.predict([[day]]))
    print('RBF predict tomorrow:                         $', *rbf_svr.predict([[day+1]]))
    print('Projected growthRate today-tomorrow:          %', growthRate(*rbf_svr.predict([[day]]), *rbf_svr.predict([[day+1]])))
    #print('Linear predict:       ', *lin_svr.predict(day))
    #print('Poly predict:         ', *poly_svr.predict(day))
    print('Totays closing price:                         $', str(actual_price[TAG]).split('    ')[1])
    '''
#    print(SYMBOL+':\nTime:', time()-TIMER)
    #....................................................................................
    '''Graph:'''
    if graph:
        plt.figure(figsize=(8,5))
        totalDays = -59
        pricePlots=adj_close_prices[totalDays:]
        predictPlot=list(rbf_svr.predict(days))[totalDays:]
        totalDays *= -1
        totalDays += 1
        plt.plot(np.arange(1,totalDays), pricePlots, color='red', label='Original '+graphTag)

#        '''
        print(pricePlots[-1], '<-- Actual price')
        print(pricePlots[-2], '<-- Previous price')
#        '''

        plt.plot(np.arange(1,totalDays), predictPlot, color='blue', label='Predicted '+graphTag)
        totalDays -= 1

#        prePredict = reader()

#        toBeSaved = str(*rbf_svr.predict([[len(df)]]))
#        writer(toBeSaved)

#        '''
        print(*rbf_svr.predict([[len(df)]]))
        print(*rbf_svr.predict([[len(df)-1]]))
        print(dates[-1])
#        '''

        plt.plot( [1,totalDays], [*rbf_svr.predict([[len(df)]]), *rbf_svr.predict([[len(df)]])], '--', label='Predicted' )
#        plt.plot( [1,totalDays], [*rbf_svr.predict([[len(df)-1]]), *rbf_svr.predict([[len(df)-1]])], '--', label='Previously Predicted' )
        plt.plot( [1,totalDays], [pricePlots[-1], pricePlots[-1]], '--', label='Current Close' )
#        plt.plot( [1,totalDays], [pricePlots[-2], pricePlots[-2]], '--', label='Previous Close' )

        plt.plot( [1,totalDays], [*rbf_svr5.predict([[len(df)]]), *rbf_svr5.predict([[len(df)]])], '--', label='Predicted next 5 minutes' )

#        plt.plot([1,7], [*rbf_svr_open.predict([[len(df)]]), *rbf_svr_open.predict([[len(df)]])], '--', label='Predicted to OPEN at $'+'{:.5}'.format(float(*rbf_svr_open.predict([[len(df)]])))+' for the next trading day')
#        plt.plot([1,7], [*rbf_svr_low.predict([[len(df)]]), *rbf_svr_low.predict([[len(df)]])], '--', label='Predicted the LOWEST at $'+'{:.5}'.format(float(*rbf_svr_low.predict([[len(df)]])))+' for the next trading day')
#        plt.plot([1,7], [*rbf_svr_high.predict([[len(df)]]), *rbf_svr_high.predict([[len(df)]])], '--', label='Predicted the HIGHEST at $'+'{:.5}'.format(float(*rbf_svr_high.predict([[len(df)]])))+' for the next trading day')

        #plt.scatter(days, rbf_svr.predict(days), color='black', label='Tomorrows Predicted Close')
        #plt.plot(days, lin_svr.predict(days), color='red', label='Linear model')
        #plt.plot(days, poly_svr.predict(days), color='blue', label='Poly model')
#        plt.legend()
        plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05),fancybox=True, shadow=True, ncol=5, prop={'size': 6})
        plt.title(SYMBOL.upper()+'\nPredicted to '+graphTag.upper()+' at %'+str( growthRate(*rbf_svr.predict([[len(df)-1]]), *rbf_svr.predict([[len(df)]]) ) )  +'\n Predicted to '+graphTag.upper()+' at $'+str(growthRate(*rbf_svr5.predict([[len(df)-1]]), *rbf_svr5.predict([[len(df)]]) ))+' the next 5 minutes')

#        plt.show()
        plt.savefig(SYMBOL+'.pdf')
        graphAddress = '/Users/iliyadehsarvi/Desktop/summer2020Project/'+SYMBOL+'.pdf'

        subprocess.call(['osascript', '-e', 'tell application "Preview" to quit'])
        subprocess.call(['open', graphAddress])

    #....................................................................................
#    return ['Last Trading Day', *rbf_svr.predict([[len(df)-1]])],\
#           ['Next Trading Day', *rbf_svr.predict([[len(df)]])],\
#           ['Last-Next growthRate', growthRate( *rbf_svr.predict([[len(df)-1]]),*rbf_svr.predict([[len(df)]]) )]

    print(SYMBOL, growthRate( *rbf_svr.predict([[len(df)-1]]),*rbf_svr.predict([[len(df)]]) ), pricePlots[-1])
    return [SYMBOL, growthRate( *rbf_svr.predict([[len(df)-1]]),*rbf_svr.predict([[len(df)]]) ), pricePlots[-1]]

#    return ['Last Trading Day',\
#            *rbf_svr_open.predict([[len(df)-1]]), *rbf_svr_high.predict([[len(df)-1]]),\
#            *rbf_svr_low.predict([[len(df)-1]]), *rbf_svr.predict([[len(df)-1]])],\
#           ['Next Trading Day',\
#           *rbf_svr_open.predict([[len(df)]]), *rbf_svr_high.predict([[len(df)]]),\
#            *rbf_svr_low.predict([[len(df)]]), *rbf_svr.predict([[len(df)]])],\
#           ['Last-Next growthRate',\
#           growthRate( *rbf_svr_open.predict([[len(df)-1]]),*rbf_svr_open.predict([[len(df)]]) ),\
#           growthRate( *rbf_svr_high.predict([[len(df)-1]]),*rbf_svr_high.predict([[len(df)]]) ),\
#           growthRate( *rbf_svr_low.predict([[len(df)-1]]),*rbf_svr_low.predict([[len(df)]]) ),\
#           growthRate( *rbf_svr.predict([[len(df)-1]]),*rbf_svr.predict([[len(df)]]) )]
#

    #return ['open', 'high', 'low', 'close'], ['open', 'high', 'low', 'close'], [Last-Next growthRate,'%open', '%high', '%low', '%close']
'''....................................................................................'''
s = time()
symbolList = ['btcusd', 'ethusd', 'bchusd', 'ltcusd', 'etcusd', 'bsvusd', 'xrpusd']#, 'qtumusd']
#print(predict('ethusd',True, 'close'))
#'''
while True:
#    try:
    pr = predict('btcusd', True)
#        if pr[1] >= 0.1:
    print(*pr)
    sleep(30)
#    except:
#
#        break
#'''

#threads = []
#for i, data in enumerate(symbolList):
#    try:
#        threads.append(threading.Thread(target=predict, args=(data,True,'close')))
#        threads[i].start()
#    except: continue
#for thread in threads: thread.join()
#print(time()-s)

max_stars_repo_name: iliya-dehsarvi/Scripts

max_issues_repo_name: iliya-dehsarvi/Scripts

max_forks_repo_name: iliya-dehsarvi/Scripts


text:
"""
Get examples for XGBoost
"""

import numpy as np
import xgboost as xgb

PARAMETERS = {'max_depth': 2,
              'eta': 0.5,
              'lambda': 1,
              'gamma': 0,
              'min_child_weight': 0}

def example_1(objective):
    """
    Simple data
    """

    parameters = PARAMETERS.copy()
    parameters['objective'] = objective

    if objective == 'reg:squarederror':
        X_train = np.array([[0],
                            [1],
                            [2]])
        y_train = np.array([0, 1, 2])
        X_test = np.array([[-1],
                           [0],
                           [1],
                           [2],
                           [3]])
        y_test = np.array([-1, 0, 1, 2, 3])
    elif objective == 'binary:logistic':
        X_train = np.array([[0],
                            [1]])
        y_train = np.array([0, 1])
        X_test = np.array([[0],
                           [1],
                           [np.nan]])
        y_test = np.array([0, 1, 0])
    elif objective == 'multi:softprob':
        X_train = np.array([[0],
                            [1],
                            [2]])
        y_train = np.array([0, 1, 2])
        X_test = np.array([[0],
                           [1],
                           [2],
                           [np.nan]])
        y_test = np.array([0, 1, 2, 0])
        parameters['num_class'] = 3
    else:
        raise ValueError('Not a recognized objective')

    data_train = xgb.DMatrix(X_train, y_train)
    data_test = xgb.DMatrix(X_test, y_test)

    bst = xgb.train(parameters,
                    data_train,
                    3)

    dump = bst.get_dump(with_stats=True)
    for tree in dump:
        print(tree)

    print(bst.predict(data_test))

if __name__ == '__main__':
    example_1('reg:squarederror')
    example_1('binary:logistic')
    example_1('multi:softprob')


text:
import sys
import gpxpy
import gpxpy.gpx

import pyproj
import numpy as np

_projections = {}

def zone(coordinates):
    if 56 <= coordinates[1] < 64 and 3 <= coordinates[0] < 12:
        return 32
    if 72 <= coordinates[1] < 84 and 0 <= coordinates[0] < 42:
        if coordinates[0] < 9:
            return 31
        elif coordinates[0] < 21:
            return 33
        elif coordinates[0] < 33:
            return 35
        return 37
    return int((coordinates[0] + 180) / 6) + 1

def letter(coordinates):
    return 'CDEFGHJKLMNPQRSTUVWXX'[int((coordinates[1] + 80) / 8)]

def project(coordinates):
    z = zone(coordinates)
    l = letter(coordinates)
    if z not in _projections:
        _projections[z] = pyproj.Proj(proj='utm', zone=z, ellps='WGS84')
    x, y = _projections[z](coordinates[0], coordinates[1])
    if y < 0:
        y += 10000000
    return z, l, x, y

def unproject(z, l, x, y):
    if z not in _projections:
        _projections[z] = pyproj.Proj(proj='utm', zone=z, ellps='WGS84')
    if l < 'N':
        y -= 10000000
    lng, lat = _projections[z](x, y, inverse=True)
    return (lng, lat)

def calc_BB(points):
    left = top = 10000000000000000
    right = bottom = 0
    elev_max = 0
    elev_min = 1000000

    for p in points:
        if p.x < left:  left = p.x
        if p.x > right: right = p.x
        if p.y < top:   top = p.y
        if p.y > bottom: bottom = p.y
        if p.elevation > elev_max: elev_max = p.elevation
        if p.elevation < elev_min: elev_min = p.elevation

    return((left,top),(right,bottom),(elev_max,elev_min))

def map_gpx(fname):
    gpx_file = open(fname, 'r')
    gpx = gpxpy.parse(gpx_file)

    points = []

    for track in gpx.tracks:
        for segment in track.segments:
            for point in segment.points:
                z, l, x, y = project((point.longitude, point.latitude))
                point.x = x
                point.y = y
                points.append(point)

    for p in points:
        print("%f lat, %f lon, %f x, %f y, %f elev" % (p.latitude, p.longitude, p.x, p.y, p.elevation))

    top,bottom,elev = calc_BB(points)
    print("top: %s, bottom: %s" % (top, bottom))

    ## calculate the fragment

    mapsz = (10,10)

    size = np.array(bottom) - np.array(top)
    #top    = np.array(top) - np.array(top)

    for i in range(len(points)):
        points[i].x = points[i].x - top[0]
        points[i].y = points[i].y - top[1]
        points[i].elevation = points[i].elevation - elev[1]

        #points[i].i = np.interp(points[i].x,[0, size[0]],[0,mapsz[0]])
        #points[i].j = np.interp(points[i].y,[0, size[1]],[0,mapsz[1]])
        #points[i].k = np.interp(points[i].elevation,[0, size[1]],[0,mapsz[2]])
        #points[i].k = 1

        #points[i].i = int(round(points[i].i))
        #points[i].j = int(round(points[i].j))
        #points[i].k = int(round(points[i].k))

    for p in points:
        #print("%f lat, %f lon, %f x, %f y, %f elev" % (p.latitude, p.longitude, p.i, p.j, p.k))
        print("%f;%f;%f" % (p.x, p.y, p.elevation))

    print("size: %s, elev: %s" % (size,elev))
    return(points)
    raise RuntimeError("X")
if __name__ == "__main__":
    map_gpx(sys.argv[1])

max_stars_repo_name: juanmcasillas/pyGLLib

max_issues_repo_name: juanmcasillas/pyGLLib

max_forks_repo_name: juanmcasillas/pyGLLib


text:
try:
    # Python2
    import Tkinter as tk
    import ttk
except ImportError:
    # Python3
    import tkinter as tk
    import tkinter.ttk as ttk

from Stepper import stepper
from picamera import PiCamera
from time import sleep
import numpy as np
#from skimage.io import imread, imsave
import time

#stepper variables

#[stepPin, directionPin, enablePin, limitPin]
focusStepper = stepper([23, 24, 22, 4])

def num(s):
    try:
        return int(s)
    except ValueError:
        return 1
def stepUp(event=' '):
    print('hello')
    focusStepper.step(num(e1.get()), "Up",10,False); #steps, dir, speed, stayOn

def stepDown(event=' '):
    focusStepper.step(num(e1.get()), "Down",10,False); #steps, dir, speed, stayOn

def old_startCamera(event=' '):

    #camera.resolution = (3280, 2464)
    camera.resolution = (1640, 1232)
    #camera.framerate = 5
    camera.exposure_mode = 'auto'
    camera.awb_mode = 'auto'
    #camera.shutter_speed = 600000
    #camera.resolution = (640, 400)
    camera.iso = 100;
    sleep(2)
    print('Speed        = ', camera.shutter_speed )
    print('Exposure     = ', camera.exposure_speed )
    print('Analog Gain  = ', camera.analog_gain  )
    print('Digital Gain = ', camera.digital_gain )
    camera.shutter_speed = int(camera.exposure_speed*1)

    print('Speed        = ', camera.shutter_speed )
    print('Exposure     = ', camera.exposure_speed )

    camera.exposure_mode = 'off'
    g = camera.awb_gains
    camera.awb_mode = 'off'
    camera.awb_gains = g

def startPreview(event=' '):
    #camera.preview_fullscreen=False
    #camera.preview_window=(100, 100, 960, 720)
    camera.start_preview()

def stopPreview(event=' '):
    camera.stop_preview()

def captureImage2File(event=' '):
    try:
        captureImage.imgNum += 1;
    except AttributeError:
        captureImage.imgNum = 1
    camera.capture('input/image{}.jpg'.format(captureImage.imgNum))

def oldcaptureImage(event=' '):
    try:
        captureImage.imgNum += 1;
    except AttributeError:
        captureImage.imgNum = 1
    #camera.resolution = (3296, 2464)
    images = []
    shp = camera.resolution
    image = np.empty((1664*shp[1]*3), dtype=np.uint8)
    for n  in range(0,10):
        print("capture image {}".format(n))
        camera.capture(image, 'rgb')
        imager = (np.reshape(image,(shp[1],1664,3)))
        images.append(imager)

    images = np.asarray(images)
    print ("averaging")
    image = np.average(images, axis = 0)

    #image = image[:3280, :2464]
    imsave('input/image{:02d}.jpg'.format(captureImage.imgNum), image)
    #camera.resolution = (3280, 2464)

def captureImage(event=' '):
    frames = 10
    camera.resolution = (3280, 2464)
    #camera.framerate = 30
    #camera.start_preview()
    # Give the camera some warm-up time
    #time.sleep(2)
    start = time.time()

    camera.capture_sequence([
        'input/image{:02d}.jpg'.format(i)
        for i in range(frames)
        ], use_video_port=True)
    finish = time.time()

    camera.stop_preview()
    print('Captured %d frames at %.2ffps' % (frames,frames / (finish - start)))
    camera.resolution = (1024, 768)

def captureSequence(event=' '):
    stopPreview();
    powerOn()
    sleep(3)   # time to settle
    camera.resolution = (3280, 2464)

    camera.capture('input/image{:02d}.jpg'.format(1), resize=(3280, 2464) )
    for z in range(2, 10):
        focusStepper.step(num(e1.get()), "Up",3,'True'); #steps, dir, speed, stayOn
        camera.capture('input/image{:02d}.jpg'.format(z), resize=(3280, 2464) )

    for z in range(2, 10):
        focusStepper.step(num(e1.get()), "Down",10,'False'); #steps, dir, speed, stayOn

    powerOff()
    camera.resolution = (1640, 1232)

def powerOff():
    focusStepper.step(num(0), "Down",1,False)

def powerOn():
    focusStepper.step(num(0), "Down",1,True)

def exitGui(event=' '):
    powerOff()
    camera.stop_preview()
    camera.close()
    focusStepper.cleanGPIO()
    quit()

##################################################

window = tk.Tk()
window.geometry('150x300+0+100')
frame = tk.Frame(window)
frame.pack()
tk.Label(frame, text="Step Amount").grid(row=0)
e1 = tk.Entry(frame)
e1.insert(10,"100")
e1.grid(row=1, column=0)

btnUp = tk.Button(frame,
                   text="Up",
                   command=stepUp)
btnUp.grid(row=2, column=0, sticky=tk.W, pady=4)

btnDown = tk.Button(frame,
                   text="Down",
                   command=stepDown)

btnDown.grid(row=3, column=0, sticky=tk.W, pady=4)

btnStartPreview = tk.Button(frame,
                   text="Start Preview",
                   command=startPreview)
btnStartPreview.grid(row=4, column=0, sticky=tk.W, pady=4)

btnStopPreview = tk.Button(frame,
                   text="Stop Preview",
                   command=stopPreview)
btnStopPreview.grid(row=5, column=0, sticky=tk.W, pady=4)

btnCapture = tk.Button(frame,
                   text="Capture Image",
                   command=captureImage)
btnCapture.grid(row=6, column=0, sticky=tk.W, pady=4)

btnCapture = tk.Button(frame,
                   text="Capture Sequence",
                   command=captureSequence)
btnCapture.grid(row=7, column=0, sticky=tk.W, pady=4)

btnQuit = tk.Button(frame,
                   text="QUIT",
                   fg="red",
                   command=exitGui)
btnQuit.grid(row=8, column=0, sticky=tk.W, pady=4)

window.bind_all('u', stepUp)
window.bind_all('d', stepDown)
window.bind_all('c', captureImage)
window.bind_all('s', stopPreview)
window.bind_all('q', exitGui)

if True:
    camera = PiCamera()
    #camera.resolution = (3280, 2464)

    camera.resolution = (1640, 1232)
    #camera.framerate = 30
    camera.exposure_mode = 'auto'
    camera.awb_mode = 'auto'
    camera.iso = 100;
    sleep(2)
    shutterSpeed = camera.exposure_speed
    print('Speed        = ', camera.shutter_speed )
    print('Exposure     = ', camera.exposure_speed )
    print('Analog Gain  = ', camera.analog_gain  )
    print('Digital Gain = ', camera.digital_gain )
    camera.shutter_speed = int(shutterSpeed*0.5)
    camera.exposure_mode = 'off'
    g = camera.awb_gains
    camera.awb_mode = 'off'
    camera.awb_gains = g
    print('')
    print('Speed        = ', camera.shutter_speed )
    print('Exposure     = ', camera.exposure_speed )
    print('Analog Gain  = ', camera.analog_gain  )
    print('Digital Gain = ', camera.digital_gain )

window.mainloop()
camera.stop_preview()

max_stars_repo_name: johnnewto/RPI-Micro-I

max_issues_repo_name: johnnewto/RPI-Micro-I

max_forks_repo_name: johnnewto/RPI-Micro-I


text:
"""Helper methods to run the experiment
"""

import numpy as np
import matplotlib.pyplot as plt
import threading

from channel import Channel
from environment import Environment
from interaction import Interaction
from fish import Fish
from observer import Observer

def generate_distortion(type='linear', n=10, show=False):
    """Generates a distortion model represented as a vector field
    """

    X, Y = np.mgrid[0:n, 0:n]
    distortion = np.zeros((n, n, 2))

    if type == 'none':
        distortion[:, :, 0] = 0
        distortion[:, :, 1] = 0
        return distortion

    elif type == 'linear':
        X_new = 1
        Y_new = 0

    elif type == 'aggregation':
        theta = np.arctan2(Y-(n-1)/2, X-(n-1)/2)
        X_new = -np.cos(theta)
        Y_new = -np.sin(theta)

    elif type == 'dispersion':
        theta = np.arctan2(Y-(n-1)/2, X-(n-1)/2)
        X_new = np.cos(theta)
        Y_new = np.sin(theta)

    elif type == 'curl':
        X_new = -(Y-(n-1)/2)
        Y_new = X-(n-1)/2

    unit_magnitude = 1/(np.sqrt(X_new**2 + Y_new**2))
    distortion[:, :, 0] = unit_magnitude * X_new
    distortion[:, :, 1] = unit_magnitude * Y_new

    if show:
        plt.quiver(X, Y, distortion[:, :, 0], distortion[:, :, 1])
        plt.show()

    return distortion

def generate_fish(
    n,
    channel,
    interaction,
    lim_neighbors,
    neighbor_weights=None,
    fish_max_speeds=None,
    clock_freqs=None,
    verbose=False,
    names=None
):
    """Generate some fish

    Arguments:
        n {int} -- Number of fish to generate
        channel {Channel} -- Channel instance
        interaction {Interaction} -- Interaction instance
        lim_neighbors {list} -- Tuple of min and max neighbors
        neighbor_weight {float|list} -- List of neighbor weights
        fish_max_speeds {float|list} -- List of max speeds
        clock_freqs {int|list} -- List of clock speeds
        names {list} -- List of names for your fish
    """

    if neighbor_weights is None:
        neighbor_weights = [1.0] * n
    elif not isinstance(neighbor_weights, list):
        neighbor_weights = [neighbor_weights] * n

    if fish_max_speeds is None:
        fish_max_speeds = [1.0] * n
    elif not isinstance(fish_max_speeds, list):
        fish_max_speeds = [fish_max_speeds] * n

    if clock_freqs is None:
        clock_freqs = [1] * n
    elif not isinstance(clock_freqs, list):
        clock_freqs = [clock_freqs] * n

    if names is None:
        names = ['Unnamed'] * n

    fish = []
    for i in range(n):
        fish.append(Fish(
            id=i,
            channel=channel,
            interaction=interaction,
            lim_neighbors=lim_neighbors,
            neighbor_weight=neighbor_weights[i],
            fish_max_speed=fish_max_speeds[i],
            clock_freq=clock_freqs[i],
            verbose=verbose,
            name=names[i]
        ))

    return fish

def init_simulation(
    clock_freq,
    single_time,
    offset_time,
    num_trials,
    final_buffer,
    run_time,
    num_fish,
    size_dist,
    center,
    spread,
    fish_pos,
    lim_neighbors,
    neighbor_weights,
    fish_max_speeds,
    noise_magnitude,
    conn_thres,
    prob_type,
    dist_type,
    verbose,
    conn_drop=1.0,
):
    """Initialize all the instances needed for a simulation

    Arguments:
        clock_freq {int} -- Clock frequency for each fish.
        single_time {float} -- Number clock cycles per individual run.
        offset_time {float} -- Initial clock offset time
        num_trials {int} -- Number of trials per experiment.
        final_buffer {float} -- Final clock buffer (because the clocks don't
            sync perfectly).
        run_time {float} -- Total run time in seconds.
        num_fish {int} -- Number of fish.
        size_dist {int} -- Distortion field size.
        center {float} -- Distortion field center.
        spread {float} -- Initial fish position spread.
        fish_pos {np.array} -- Initial fish position.
        lim_neighbors {list} -- Min. and max. desired neighbors. If too few
            neighbors start aggregation, if too many neighbors disperse!
        neighbor_weights {float} -- Distance-depending neighbor weight.
        fish_max_speeds {float} -- Max fish speed.
        noise_magnitude {float} -- Amount of white noise added to each move.
        conn_thres {float} -- Distance at which the connection either cuts off
            or starts dropping severely.
        prob_type {str} -- Probability type. Can be `binary`, `quadratic`, or
            `sigmoid`.
        dist_type {str} -- Position distortion type
        verbose {bool} -- If `true` print a lot of stuff

    Keyword Arguments:
        conn_drop {number} -- Defined the connection drop for the sigmoid
            (default: {1.0})

    Returns:
        tuple -- Quintuple holding the `channel`, `environment`, `fish,
            `interaction`, and `observer`
    """
    distortion = generate_distortion(type=dist_type, n=size_dist)
    environment = Environment(
        node_pos=fish_pos,
        distortion=distortion,
        prob_type=prob_type,
        noise_magnitude=noise_magnitude,
        conn_thres=conn_thres,
        conn_drop=conn_drop,
        verbose=verbose
    )
    interaction = Interaction(environment, verbose=verbose)
    channel = Channel(environment)

    fish = generate_fish(
        n=num_fish,
        channel=channel,
        interaction=interaction,
        lim_neighbors=lim_neighbors,
        neighbor_weights=neighbor_weights,
        fish_max_speeds=fish_max_speeds,
        clock_freqs=clock_freq,
        verbose=verbose
    )
    channel.set_nodes(fish)

    observer = Observer(
        fish=fish,
        environment=environment,
        channel=channel,
        clock_freq=clock_freq,
        fish_pos=np.copy(fish_pos)
    )
    channel.intercept(observer)

    return channel, environment, fish, interaction, observer

def run_simulation(
    fish,
    observer,
    run_time=10,
    dark=False,
    white_axis=False,
    no_legend=False,
    no_star=False
):
    """Start the simulation.

    Arguments:
        fish {list} -- List of fish instances
        observer {Observer} -- Observer instance

    Keyword Arguments:
        run_time {number} -- Total run time in seconds (default: {10})
        dark {bool} -- If `True` plot a dark chart (default: {False})
        white_axis {bool} -- If `True` plot white axes (default: {False})
        no_legend {bool} -- If `True` do not plot a legend (default: {False})
        no_star {bool} -- If `True` do not plot a star (default: {False})
    """
    def stop():
        for f in fish:
            f.stop()

        print('It\'s time to say bye bye!')

        observer.stop()
        observer.plot(
            dark=dark,
            white_axis=white_axis,
            no_legend=no_legend,
            no_star=no_star
        )

    print('Please wait patiently {} seconds. Thanks.'.format(run_time))

    # Start the fish
    for f in fish:
        threading.Thread(target=f.start).start()

    threading.Thread(target=observer.start).start()

    # Ciao stops run time
    threading.Timer(run_time, stop).start()

max_stars_repo_name: fberlinger/FISH

max_issues_repo_name: fberlinger/FISH

max_forks_repo_name: fberlinger/FISH