text:
# Tilt Hydrometer  Tilt polling library
# Simon Bowler 22/01/2017
# simon@bowler.id.au
#
# Version: 1.5 - Bug fix for debugging log and also adding connection resilliance. Will now reset
#                bluetooth connection if dropped for any reason.
# Version: 1.4 - Added additional resiliance and debugging when parsing configuration files.
#                Also added debug parameter to settings.ini to provide additional logging.
# Version: 1.3 - Upgraded for firmware 0.4.4
# Version: 1.2 - Added ability to get smoothed value, current value (calibrated)
#                and an measure of variance (can indicate vigor of fermentation).
#                Renamed library to reflect new name of product

from __future__ import print_function
import blescan
import sys
import datetime
import time
import os

import bluetooth._bluetooth as bluez
import threading
try:
    # Python 3 (Though arguably, we could swap for functions from threading instead)
    import _thread as thread
except:
    # Python 2
    import thread

try:
    # If we can't import numpy for any reason, exit
    import numpy

    from scipy.interpolate import interp1d
    from scipy import arange, array, exp
except:
    exit(1)

import csv
import functools
try:
    # Python 3
    import configparser as ConfigParser
except:
    # Python 2
    import ConfigParser

TILTHYDROMETER_COLOURS = ['Red', 'Green', 'Black', 'Purple', 'Orange', 'Blue', 'Yellow', 'Pink']

# Default time in seconds to wait before checking config files to see if calibration data has changed.
DATA_REFRESH_WINDOW = 60

# extrap1d Sourced from sastanin @ StackOverflow:
# http://stackoverflow.com/questions/2745329/how-to-make-scipy-interpolate-give-an-extrapolated-result-beyond-the-input-range
# This function is required as the interp1d function doesn't support extrapolation in the version of scipy that is currently available on the pi.
def extrap1d(interpolator):
    xs = interpolator.x
    ys = interpolator.y

    def pointwise(x):
        if x < xs[0]:
            return ys[0] + (x - xs[0]) * (ys[1] - ys[0]) / (xs[1] - xs[0])
        elif x > xs[-1]:
            return ys[-1] + (x - xs[-1]) * (ys[-1] - ys[-2]) / (xs[-1] - xs[-2])
        else:
            return interpolator(x)

    def ufunclike(xs):
        return array(map(pointwise, array(xs)))

    return ufunclike

# Simple offset calibration if only one point available.
def offsetCalibration(offset, value):
    return value + offset

# More complex interpolation calibration if more than one calibration point available
def extrapolationCalibration(extrapolationFunction, value):
    inputValue = [value]
    returnValue = extrapolationFunction(inputValue)
    return returnValue[0]

def noCalibration(value):
    return value

# Median utility function
def median(values):
    return numpy.median(numpy.array(values))

# Class to hold a TiltHydrometer reading
class TiltHydrometerValue:
    temperature = 0
    gravity = 0
    timestamp = 0

    def __init__(self, temperature, gravity):
        self.temperature = round(temperature, 2)
        self.gravity = round(gravity, 3)
        self.timestamp = datetime.datetime.now()

    def __str__(self):
        return "T: " + str(self.temperature) + " G: " + str(self.gravity)

# TiltHydrometer class, looks after calibration, storing of values and smoothing of read values.
class TiltHydrometer:
    colour = ''
    debug = False
    values = None
    lock = None
    averagingPeriod = 0
    medianWindow = 0
    tempCalibrationFunction = None
    gravityCalibrationFunction = None
    calibrationDataTime = {}

    # Averaging period is number of secs to average across. 0 to disable.
    # Median window is the window to use for applying a median filter accross the values. 0 to disable. Median window should be <= the averaging period.
    # If Median is disabled, the returned value will be the average of all values recorded during the averaging period.
    def __init__(self, colour, averagingPeriod=0, medianWindow=0, debug=False):
        self.colour = colour
        self.lock = threading.Lock()
        self.averagingPeriod = averagingPeriod
        self.medianWindow = medianWindow
        self.debug = debug
        self.values = []
        self.calibrationDataTime = {}
        self.calibrate()

    def calibrate(self):
        """Load/reload calibration functions."""
        # Check for temperature function. If none, then not changed since last load.
        tempFunction = self.tiltHydrometerCalibrationFunction("temperature", self.colour)
        if (tempFunction is not None):
            self.tempCalibrationFunction = tempFunction

        # Check for gravity function. If none, then not changed since last load.
        gravityFunction = self.tiltHydrometerCalibrationFunction("gravity", self.colour)
        if (gravityFunction is not None):
            self.gravityCalibrationFunction = gravityFunction

    def setValues(self, temperature, gravity):
        """Set/add the latest temperature & gravity readings to the store. These values will be calibrated before storing if calibration is enabled"""
        with self.lock:
            self.cleanValues()
            self.calibrate()

            calibratedTemperature = temperature
            calibratedGravity = gravity

            try:
                calibratedTemperature = self.tempCalibrationFunction(temperature)
            except (Exception) as e:
                print("ERROR: TiltHydrometer (" + self.colour + "): Unable to calibrate temperature: " + str(
                    temperature) + " - " + e.message)

            try:
                calibratedGravity = self.gravityCalibrationFunction(gravity)
            except (Exception) as e:
                print("ERROR: TiltHydrometer (" + self.colour + "): Unable to calibrate gravity: " + str(
                    gravity) + " - " + e.message)

            self.values.append(TiltHydrometerValue(calibratedTemperature, calibratedGravity))

    def getValues(self):
        """Returns the temperature & gravity values of the Tilt Hydrometer . This will be the latest read value unless averaging / median has been enabled"""
        with self.lock:
            returnValue = None
            if (len(self.values) > 0):
                if (self.medianWindow == 0):
                    returnValue = self.averageValues()
                else:
                    returnValue = self.medianValues(self.medianWindow)

                self.cleanValues()
        return returnValue

    def averageValues(self):
        """Internal function to average all the stored values"""
        returnValue = None
        if (len(self.values) > 0):
            returnValue = TiltHydrometerValue(0, 0)
            for value in self.values:
                returnValue.temperature += value.temperature
                returnValue.gravity += value.gravity

            # average values
            returnValue.temperature /= len(self.values)
            returnValue.gravity /= len(self.values)

            # round values
            returnValue.temperature = round(returnValue.temperature, 2)
            returnValue.gravity = round(returnValue.gravity, 3)
        return returnValue

    def medianValues(self, window=3):
        """Internal function to use a median method across the stored values to reduce noise.
           window - Smoothing window to apply across the data. If the window is less than the dataset size, the window will be moved across the dataset,
                    taking a median value for each window, with the resultant set averaged"""
        returnValue = None
        # Ensure there are enough values to do a median filter, if not shrink window temporarily
        if (len(self.values) < window):
            window = len(self.values)

        # print("Median filter!")
        returnValue = TiltHydrometerValue(0, 0)

        sidebars = (window - 1) / 2
        medianValueCount = 0

        for i in range(len(self.values) - (window - 1)):
            # Work out range of values to do median. At start and end of assessment, need to pad with start and end values.
            medianValues = self.values[i:i + window]
            medianValuesTemp = []
            medianValuesGravity = []

            # Separate out Temp and Gravity values
            for medianValue in medianValues:
                medianValuesTemp.append(medianValue.temperature)
                medianValuesGravity.append(medianValue.gravity)

            # Add the median value to the running total.
            returnValue.temperature += median(medianValuesTemp)
            returnValue.gravity += median(medianValuesGravity)

            # Increase count
            medianValueCount += 1

        # average values
        returnValue.temperature /= medianValueCount
        returnValue.gravity /= medianValueCount

        # round values
        returnValue.temperature = round(returnValue.temperature, 2)
        returnValue.gravity = round(returnValue.gravity, 3)

        return returnValue

    def cleanValues(self):
        """Function to clean out stale values that are beyond the desired window"""
        nowTime = datetime.datetime.now()

        for value in self.values:
            if ((nowTime - value.timestamp).seconds >= self.averagingPeriod):
                self.values.pop(0)
            else:
                # The list is sorted in chronological order, so once we've hit this condition we can stop searching.
                break

    # Load the calibration settings from file and create the calibration functions
    def tiltHydrometerCalibrationFunction(self, type, colour):
        # This function is replaced under Fermentrack
        returnFunction = noCalibration

        originalValues = []
        actualValues = []
        csvFile = None
        filename = "tiltHydrometer/" + type.upper() + "." + colour.lower()

        # Find out last time we attempted to load.
        lastChecked = self.calibrationDataTime.get(type + "_checked", 0)
        if ((int(time.time()) - lastChecked) < DATA_REFRESH_WINDOW):
            # Only check every x seconds
            return None

        # Retrieve at the last load time.
        lastLoaded = self.calibrationDataTime.get(type, 0)
        self.calibrationDataTime[type + "_checked"] = int(time.time())

        try:
            if (os.path.isfile(filename)):
                fileModificationTime = os.path.getmtime(filename)
                if (lastLoaded >= fileModificationTime):
                    # No need to load, no change
                    return None
                csvFile = open(filename, "rb")
                csvFileReader = csv.reader(csvFile, skipinitialspace=True)
                self.calibrationDataTime[type] = fileModificationTime

                lineNumber = 1
                for row in csvFileReader:
                    if (self.debug):
                        print("TiltHydrometer (" + colour + "): File - " + filename + ", Line " + str(
                            lineNumber) + " processing [" + str(row) + "]")
                    # Skip any comment rows and rows with no configuration data
                    if ((len(row) != 2) or (row[0][:1] == "#")):
                        print("WARNING: TiltHydrometer (" + colour + "): File - " + filename + ", Line " + str(
                            lineNumber) + " was ignored as does not contain valid configuration data [" + str(row) + "]")
                    else:
                        if (self.debug):
                            print("TiltHydrometer (" + colour + "): File - " + filename + ", Line " + str(
                                lineNumber) + " processed successfully")
                        originalValues.append(float(row[0]))
                        actualValues.append(float(row[1]))

                    lineNumber += 1
                # Close file
                csvFile.close()
        except IOError:
            print("TiltHydrometer (" + colour + "):  " + type.capitalize() + ": No calibration data (" + filename + ")")
        except (Exception) as e:
            print("ERROR: TiltHydrometer (" + colour + "): Unable to initialise " + type.capitalize() + " Calibration data (" + filename + ") - " + e.message)
            # Attempt to close the file
            if (csvFile is not None):
                # Close file
                csvFile.close()

        # If more than two values, use interpolation
        if (len(actualValues) >= 2):
            interpolationFunction = interp1d(originalValues, actualValues, bounds_error=False, fill_value=1)
            returnFunction = functools.partial(extrapolationCalibration, extrap1d(interpolationFunction))
            print("TiltHydrometer (" + colour + "): Initialised " + type.capitalize() + " Calibration: Interpolation")
        # Not enough values. Likely just an offset calculation
        elif (len(actualValues) == 1):
            offset = actualValues[0] - originalValues[0]
            returnFunction = functools.partial(offsetCalibration, offset)
            print("TiltHydrometer (" + colour + "): Initialised " + type.capitalize() + " Calibration: Offset (" + str(
                offset) + ")")
        return returnFunction

# Class to manage the monitoring of all TiltHydrometers and storing the read values.
class TiltHydrometerManager:
    inFahrenheit = True
    dev_id = 0
    averagingPeriod = 0
    medianWindow = 0
    debug = False

    scanning = True
    # Dictionary to hold tiltHydrometers - index on colour
    tiltHydrometers = {}

    brewthread = None
    lastLoadTime = 0
    lastCheckedTime = 0

    def __init__(self, inFahrenheit=True, averagingPeriod=0, medianWindow=0, device_id=0):
        self.inFahrenheit = inFahrenheit
        self.dev_id = device_id
        self.averagingPeriod = averagingPeriod
        self.medianWindow = medianWindow

        self.reloadSettings()

    # Convert Tilt UUID back to colour
    def tiltHydrometerName(self, uuid):
        return {
            'a495bb10c5b14b44b5121370f02d74de': 'Red',
            'a495bb20c5b14b44b5121370f02d74de': 'Green',
            'a495bb30c5b14b44b5121370f02d74de': 'Black',
            'a495bb40c5b14b44b5121370f02d74de': 'Purple',
            'a495bb50c5b14b44b5121370f02d74de': 'Orange',
            'a495bb60c5b14b44b5121370f02d74de': 'Blue',
            'a495bb70c5b14b44b5121370f02d74de': 'Yellow',
            'a495bb80c5b14b44b5121370f02d74de': 'Pink'
        }.get(uuid)

    # Convert Temp in F to C
    def convertFtoC(self, temperatureF):
        return (temperatureF - 32) * 5.0 / 9

    # Convert Tilt SG to float
    def convertSG(self, gravity):
        return float(gravity) / 1000

    # Store function
    def storeValue(self, colour, temperature, gravity):
        # This function is replaced under Fermentrack (just so we can override the class being used)
        tiltHydrometer = self.tiltHydrometers.get(colour)
        if (tiltHydrometer is None):
            tiltHydrometer = TiltHydrometer(colour, self.averagingPeriod, self.medianWindow, self.debug)
            self.tiltHydrometers[colour] = tiltHydrometer

        tiltHydrometer.setValues(temperature, gravity)

    # Retrieve function.
    def getValue(self, colour):
        returnValue = None
        tiltHydrometer = self.tiltHydrometers.get(colour)
        if (tiltHydrometer is not None):
            returnValue = tiltHydrometer.getValues()
        return returnValue

    # Scanner function
    def scan(self):
        # Keep scanning until the manager is told to stop.
        while self.scanning:
            try:
                sock = bluez.hci_open_dev(self.dev_id)

            except (Exception) as e:
                print("ERROR: Accessing bluetooth device: " + e.message)
                sys.exit(1)

            blescan.hci_le_set_scan_parameters(sock)
            blescan.hci_enable_le_scan(sock)

            try:
                # Keep scanning until the manager is told to stop.
                while self.scanning:
                    self.processSocket(sock)

                    reloaded = self.reloadSettings()
                    if (reloaded):
                        break
            except (Exception) as e:
                print("ERROR: Accessing bluetooth device whilst scanning")
                print("Resetting Bluetooth device")

    # Processes Tilt BLE data from open socket.
    def processSocket(self, sock):
        returnedList = blescan.parse_events(sock, 10)

        for beacon in returnedList:
            beaconParts = beacon.split(",")

            # Resolve whether the received BLE event is for a Tilt Hydrometer by looking at the UUID.
            name = self.tiltHydrometerName(beaconParts[1])

            # If the event is for a Tilt Hydrometer , process the data
            if name is not None:
                if (self.debug):
                    print(name + " Tilt Device Found (UUID " + str(beaconParts[1]) + "): " + str(beaconParts))

                # Get the temperature and convert to C if needed.
                temperature = int(beaconParts[2])
                if not self.inFahrenheit:
                    temperature = self.convertFtoC(temperature)

                # Get the gravity.
                gravity = self.convertSG(beaconParts[3])

                # Store the retrieved values in the relevant Tilt Hydrometer object.
                self.storeValue(name, temperature, gravity)
            else:
                # Output what has been found.
                if (self.debug):
                    print("UNKNOWN BLE Device Found: " + str(beaconParts))

    # Stop Scanning function
    def stop(self):
        self.scanning = False

    # Start the scanning thread
    def start(self):
        self.scanning = True
        self.brewthread = thread.start_new_thread(self.scan, ())

    # Checks to see if the settings file has changed, then reloads the settings if it has. Returns True if the settings were reloaded.
    def reloadSettings(self):
        # This function is replaced under Fermentrack
        filename = "tiltHydrometer/settings.ini"
        reloadSettings = False

        # Only check every x seconds
        if ((int(time.time()) - self.lastCheckedTime) >= DATA_REFRESH_WINDOW):
            self.lastCheckedTime = int(time.time())

            # Check that file exists
            if (os.path.isfile(filename)):
                fileModificationTime = os.path.getmtime(filename)

                # Check file modification time against last loaded time.
                if (self.lastLoadTime < fileModificationTime):
                    # File has been modified since last load (or first load)
                    self.loadSettings(filename)
                    self.lastLoadTime = int(time.time())
                    reloadSettings = True

        return reloadSettings

    # Load Settings from config file, overriding values given at creation. This needs to be called before the start function is called.
    def loadSettings(self, filename):
        # This function is replaced under Fermentrack
        try:
            print("Loading settings from file: " + filename)

            config = ConfigParser.ConfigParser()
            config.read(filename)

            # Load config values
            self.inFahrenheit = config.getboolean("Manager", "FahrenheitTemperatures")
            self.dev_ID = config.getint("Manager", "DeviceID")
            self.averagingPeriod = config.getint("Manager", "AveragePeriodSeconds")
            self.medianWindow = config.getint("Manager", "MedianWindowVals")
            self.debug = config.getboolean("Manager", "Debug")

            # Reset store
            self.tiltHydrometers = {}

        except (Exception) as e:
            print("ERROR: Loading default settings file (tiltHydrometer/settings.ini): " + e.message)

max_stars_repo_name: NamelessCoder/fermentrack

max_issues_repo_name: NamelessCoder/fermentrack

max_forks_repo_name: NamelessCoder/fermentrack


text:
"""
Detection Testing Script.

This scripts reads a given config file and runs the evaluation.
It is an entry point that is made to evaluate standard models in FsDet.

In order to let one script support evaluation of many models,
this script contains logic that are specific to these built-in models and
therefore may not be suitable for your own project.
For example, your research project perhaps only needs a single "evaluator".

Therefore, we recommend you to use FsDet as an library and take
this file as an example of how to use the library.
You may want to write your own script with your datasets and other customizations.
"""
import random

import numpy as np
import torch

from laplacianshot.trainer import LaplacianTrainer

random.seed(0)
np.random.seed(0)
torch.manual_seed(0)

from fsdet.config import get_cfg, set_global_cfg
from fsdet.engine import default_argument_parser, default_setup

from detectron2.config import global_cfg
import detectron2.utils.comm as comm
import os
from detectron2.checkpoint import DetectionCheckpointer
from detectron2.engine import launch
from fsdet.evaluation import (verify_results)

def setup(args):
    """
    Create configs and perform basic setups.
    """
    cfg = get_cfg()
    cfg.merge_from_file(args.config_file)
    if args.opts:
        cfg.merge_from_list(args.opts)
    cfg.freeze()
    set_global_cfg(cfg)
    default_setup(cfg, args)
    return cfg

def main(args):
    cfg = setup(args)

    model = LaplacianTrainer.build_model(cfg)

    if args.eval_iter != -1:
        # load checkpoint at specified iteration
        ckpt_file = os.path.join(
            cfg.OUTPUT_DIR, "model_{:07d}.pth".format(args.eval_iter - 1)
        )
        resume = False
    else:
        # load checkpoint at last iteration
        ckpt_file = cfg.MODEL.WEIGHTS
        resume = True
    DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
        ckpt_file, resume=resume
    )

    res = LaplacianTrainer.test(cfg, model,
                                support_augmentation=None,
                                use_classification_layer=True,
                                use_laplacianshot=True,
                                rectify_prototypes=None,
                                leverage_classification=None,
                                embeddings_type=None,
                                do_pca=None,
                                remove_possibly_duplicates=None,
                                knn=5,
                                lambda_factor=0.5,
                                max_iters=None,
                                laplacianshot_logs=False,
                                save_checkpoints=True,
                                plots=False)

    if comm.is_main_process():
        verify_results(cfg, res)
        # save evaluation results in json
        # os.makedirs(
        #     os.path.join(cfg.OUTPUT_DIR, "inference"), exist_ok=True
        # )
        # with open(
        #         os.path.join(cfg.OUTPUT_DIR, "inference", "res_final.json"),
        #         "w",
        # ) as fp:
        #     json.dump(res, fp)
    return res

if __name__ == "__main__":
    args = default_argument_parser().parse_args()
    if args.eval_during_train or args.eval_all:
        args.dist_url = "tcp://127.0.0.1:{:05d}".format(
            np.random.choice(np.arange(0, 65534))
        )
    print("Command Line Args:", args)

    launch(
        main,
        args.num_gpus,
        num_machines=args.num_machines,
        machine_rank=args.machine_rank,
        dist_url=args.dist_url,
        args=(args,),
    )

ext: py

reason: import numpy


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from functools import partial
from itertools import product
import pymc3 as pm
from src.globs import beta_std

df = pd.read_json('data/processed/processed_data_online.json')

traces = {}
for i, (touch, d) in enumerate(df.groupby('occluded_contact')):
    with pm.Model() as logistic_model:
        a = pm.Normal('a', 0, 10)
        b = pm.Normal('b', 0, 10)
        x = d['partial_mse'] - d['partial_mse'].mean()
        x = x / x.max()
        p = 1 / (1 + np.exp(-(a + b * x)))
        s = pm.Bernoulli('s', p=p, observed=d['result'])
        trace = pm.sample(1000, tune=1000, init='adapt_diag')
    traces[touch] = trace

group = 'test_part'
c = ['blue', 'green']
legend = {
    True: 'Contact during Occlusion',
    False: 'No Contact during Occlusion'
}
data = df
fig, ax = plt.subplots()
for i, (touch, d) in enumerate(data.groupby('occluded_contact')):
    gb = d.groupby('model')
    x = np.array(gb.X.mean())
    xerr = np.array(gb.X.sem())
    y = np.array(gb.result.mean())
    yerr = np.array(gb.result.agg(beta_std))
    ax.errorbar(x,
                y,
                xerr=xerr,
                yerr=yerr,
                fmt='.',
                label=legend[touch],
                alpha=0.3,
                color=c[i])
    trace = traces[touch]
    xpred = np.linspace(-0.04, 0.11)
    for _ in range(20):
        j = np.random.choice(range(2000))
        a, b = trace['a'][j], trace['b'][j]
        ypred = 1 / (1 + np.exp(-(a + b * xpred)))
        ax.plot(xpred, ypred, alpha=0.1, color=c[i])
plt.title('Effect of Contact on Predictability')
plt.xlabel('Centered MSE')
plt.ylabel("Confusion Rate")
plt.legend()
plt.tight_layout()
plt.savefig('reports/figures/fig6.pdf')

max_stars_repo_name: Lapu-Lapu/mp-prediction

max_issues_repo_name: Lapu-Lapu/mp-prediction

max_forks_repo_name: Lapu-Lapu/mp-prediction

reason: import numpy,import pymc3


text:
''' IO utility functions for MDA filetype
source: https://github.com/flatironinstitute/mountainsort/blob/master/packages/pyms/mlpy/mdaio.py
'''

import struct
import numpy as np

class MdaHeader:
    def __init__(self, dt0, dims0):
        uses64bitdims=(max(dims0)>2e9)
        self.uses64bitdims=uses64bitdims
        self.dt_code=_dt_code_from_dt(dt0)
        self.dt=dt0
        self.num_bytes_per_entry=get_num_bytes_per_entry_from_dt(dt0)
        self.num_dims=len(dims0)
        self.dimprod=np.prod(dims0)
        self.dims=dims0
        if uses64bitdims:
            self.header_size=3*4+self.num_dims*8
        else:
            self.header_size=(3+self.num_dims)*4

class DiskReadMda:
    def __init__(self,path,header=None):
        self._path=path
        if header:
            self._header=header
            self._header.header_size=0
        else:
            self._header=_read_header(self._path)
    def dims(self):
        return self._header.dims
    def N1(self):
         return int(self._header.dims[0])
    def N2(self):
        return int(self._header.dims[1])
    def N3(self):
        return int(self._header.dims[2])
    def dt(self):
        return self._header.dt
    def readChunk(self,i1=-1,i2=-1,i3=-1,N1=1,N2=1,N3=1):
        #print("Reading chunk {} {} {} {} {} {}".format(i1,i2,i3,N1,N2,N3))
        if (i2<0):
            return self._read_chunk_1d(i1,N1)
        elif (i3<0):
            if N1 != self._header.dims[0]:
                print ("Unable to support N1 {} != {}".format(N1,self._header.dims[0]))
                return None
            X=self._read_chunk_1d(i1+N1*i2,N1*N2)
            return np.reshape(X,(N1,N2),order='F')
        else:
            if N1 != self._header.dims[0]:
                print ("Unable to support N1 {} != {}".format(N1,self._header.dims[0]))
                return None
            if N2 != self._header.dims[1]:
                print ("Unable to support N2 {} != {}".format(N2,self._header.dims[1]))
                return None
            X=self._read_chunk_1d(i1+N1*i2+N1*N2*i3,N1*N2*N3)
            return np.reshape(X,(N1,N2,N3),order='F')
    def _read_chunk_1d(self,i,N):
        f=open(self._path,"rb")
        try:
            f.seek(self._header.header_size+self._header.num_bytes_per_entry*i)
            ret=np.fromfile(f,dtype=self._header.dt,count=N)
            f.close()
            return ret
        except Exception as e: # catch *all* exceptions
            print (e)
            f.close()
            return None

class DiskWriteMda:
    def __init__(self,path,dims,dt='float64'):
        self._path=path
        self._header=MdaHeader(dt,dims)
        _write_header(path, self._header)
    def N1(self):
        return self._header.dims[0]
    def N2(self):
        return self._header.dims[1]
    def N3(self):
        return self._header.dims[2]
    def writeChunk(self,X,i1=-1,i2=-1,i3=-1):
        #print("Writing chunk {} {} {} {}".format(i1,i2,i3,X[0]))
        if (len(X.shape)>=2):
            N1=X.shape[0]
        else:
            N1=1
        if (len(X.shape)>=2):
            N2=X.shape[1]
        else:
            N2=1
        if (len(X.shape)>=3):
            N3=X.shape[2]
        else:
            N3=1
        if (i2<0):
            return self._write_chunk_1d(X,i1)
        elif (i3<0):
            if N1 != self._header.dims[0]:
                print ("Unable to support DiskWriteMda N1 {} != {}".format(N1,self._header.dims[0]))
                return None
            return self._write_chunk_1d(X.ravel(order='F'),i1+N1*i2)
        else:
            if N1 != self._header.dims[0]:
                print ("Unable to support DiskWriteMda N1 {} != {}".format(N1,self._header.dims[0]))
                return None
            if N2 != self._header.dims[1]:
                print ("Unable to support DiskWriteMda N2 {} != {}".format(N2,self._header.dims[1]))
                return None
            return self._write_chunk_1d(X.ravel(order='F'),i1+N1*i2+N1*N2*i3)
    def _write_chunk_1d(self,X,i):
        N=X.size
        f=open(self._path,"ab")
        try:
            #print('Writing data to file at position {} values: {}'.format(self._header.header_size+self._header.num_bytes_per_entry*i,X))
            f.seek(self._header.header_size+self._header.num_bytes_per_entry*i)
            X.astype(self._header.dt).tofile(f)
            f.close()
            return True
        except Exception as e: # catch *all* exceptions
            print (e)
            f.close()
            return False

def _dt_from_dt_code(dt_code):
    if dt_code == -2:
        dt='uint8'
    elif dt_code == -3:
        dt='float32'
    elif dt_code == -4:
        dt='int16'
    elif dt_code == -5:
        dt='int32'
    elif dt_code == -6:
        dt='uint16'
    elif dt_code == -7:
        dt='float64'
    elif dt_code == -8:
        dt='uint32'
    else:
        dt=None
    return dt

def _dt_code_from_dt(dt):
    if dt == 'uint8':
        return -2
    if dt == 'float32':
        return -3
    if dt == 'int16':
        return -4
    if dt == 'int32':
        return -5
    if dt == 'uint16':
        return -6
    if dt == 'float64':
        return -7
    if dt == 'uint32':
        return -8
    return None

def get_num_bytes_per_entry_from_dt(dt):
    if dt == 'uint8':
        return 1
    if dt == 'float32':
        return 4
    if dt == 'int16':
        return 2
    if dt == 'int32':
        return 4
    if dt == 'uint16':
        return 2
    if dt == 'float64':
        return 8
    if dt == 'uint32':
        return 4
    return None

def _read_header(path):
    f=open(path,"rb")
    try:
        dt_code=_read_int32(f)
        num_bytes_per_entry=_read_int32(f)
        num_dims=_read_int32(f)
        uses64bitdims=False
        if (num_dims<0):
            uses64bitdims=True
            num_dims=-num_dims
        if (num_dims<1) or (num_dims>6): # allow single dimension as of 12/6/17
            print ("Invalid number of dimensions: {}".format(num_dims))
            return None
        dims=[]
        dimprod=1
        if uses64bitdims:
            for j in range(0,num_dims):
                tmp0=_read_int64(f)
                dimprod=dimprod*tmp0
                dims.append(tmp0)
        else:
            for j in range(0,num_dims):
                tmp0=_read_int32(f)
                dimprod=dimprod*tmp0
                dims.append(tmp0)
        dt=_dt_from_dt_code(dt_code)
        if dt is None:
            print ("Invalid data type code: {}".format(dt_code))
            return None
        H=MdaHeader(dt,dims)
        if (uses64bitdims):
            H.uses64bitdims=True
            H.header_size=3*4+H.num_dims*8
        f.close()
        return H
    except Exception as e: # catch *all* exceptions
        print (e)
        f.close()
        return None

def _write_header(path,H):
    f=open(path,"wb")
    try:
        _write_int32(f,H.dt_code)
        _write_int32(f,H.num_bytes_per_entry)
        if H.uses64bitdims:
            _write_int32(f,-H.num_dims)
            for j in range(0,H.num_dims):
                _write_int64(f,H.dims[j])
        else:
            _write_int32(f,H.num_dims)
            for j in range(0,H.num_dims):
                _write_int32(f,H.dims[j])
        f.close()
        return True
    except Exception as e: # catch *all* exceptions
        print (e)
        f.close()
        return False

def readmda(path):
    H=_read_header(path)
    if (H is None):
        print ("Problem reading header of: {}".format(path))
        return None
    ret=np.array([])
    f=open(path,"rb")
    try:
        f.seek(H.header_size)
        #This is how I do the column-major order
        ret=np.fromfile(f,dtype=H.dt,count=H.dimprod)
        ret=np.reshape(ret,H.dims,order='F')
        f.close()
        return ret
    except Exception as e: # catch *all* exceptions
        print (e)
        f.close()
        return None

def writemda32(X,fname):
    return _writemda(X,fname,'float32')

def writemda64(X,fname):
    return _writemda(X,fname,'float64')

def writemda8(X,fname):
    return _writemda(X,fname,'uint8')

def writemda32i(X,fname):
    return _writemda(X,fname,'int32')

def writemda32ui(X,fname):
    return _writemda(X,fname,'uint32')

def writemda16i(X,fname):
    return _writemda(X,fname,'int16')

def writemda16ui(X,fname):
    return _writemda(X,fname,'uint16')

def _writemda(X,fname,dt):
    dt_code=0
    num_bytes_per_entry=get_num_bytes_per_entry_from_dt(dt)
    dt_code=_dt_code_from_dt(dt)
    if dt_code is None:
        print ("Unexpected data type: {}".format(dt))
        return False

    f=open(fname,'wb')
    try:
        _write_int32(f,dt_code)
        _write_int32(f,num_bytes_per_entry)
        _write_int32(f,X.ndim)
        for j in range(0,X.ndim):
            _write_int32(f,X.shape[j])
        #This is how I do column-major order
        A=np.reshape(X,X.size,order='F').astype(dt)
        A.tofile(f)
    except Exception as e: # catch *all* exceptions
        print (e)
    finally:
        f.close()
        return True

def _read_int32(f):
    return struct.unpack('<i',f.read(4))[0]

def _read_int64(f):
    return struct.unpack('<q',f.read(8))[0]

def _write_int32(f,val):
    f.write(struct.pack('<i',val))

def _write_int64(f,val):
    f.write(struct.pack('<q',val))

def mdaio_test():
    M=4
    N=12
    X=np.ndarray((M,N))
    for n in range(0,N):
        for m in range(0,M):
            X[m,n]=n*10+m
    writemda32(X,'tmp1.mda')
    Y=readmda('tmp1.mda')
    print (Y)
    print (np.absolute(X-Y).max())
    Z=DiskReadMda('tmp1.mda')
    print (Z.readChunk(i1=0,i2=4,N1=M,N2=N-4))

    A=DiskWriteMda('tmpA.mda',(M,N))
    A.writeChunk(Y,i1=0,i2=0)
    B=readmda('tmpA.mda')
    print (B.shape)
    print (B)

#mdaio_test()

max_stars_repo_name: gentnerlab/klusta-pipeline

max_issues_repo_name: gentnerlab/klusta-pipeline

max_forks_repo_name: gentnerlab/klusta-pipeline


text:
import torch
import pickle
import os.path as osp
import os
import numpy as np
import pandas as pd
from mmcv import Config

from utils.models import modelFromConfig
from utils.mics import parse_args, PoseC3DTransform, createKNN, plot
from utils.metrics import hitAtK, precisionAtK, meanAveragePrecision

def main(args):
    cfg = Config.fromfile(args.cfg_path)

    # Read file contains path
    pathFile = pd.read_csv(args.txt_path, names=['filename', 'label', 'status'], header=None)
    pathFile = pathFile.replace(r'\\','/', regex=True)
    # Read keypoint annotation
    with open(args.kp_annotation, 'rb') as f:
        kp_annotation = pickle.load(f)

    # Embed dataset
    loader = PoseC3DTransform(cfg=cfg,
                        sample_mode=args.sample_mode,
                        seed=255,
                        ram=True)

    checkpoint_paths = sorted(os.listdir(args.checkpoint))
    checkpoint_paths = [osp.join(args.checkpoint, checkpoint) for checkpoint in checkpoint_paths]

    metrics = []
    k = 1
    for checkpoint_path in checkpoint_paths:
        if not checkpoint_path.endswith(('.path', '.pth','.pth.tar')):
            continue
        # Build model
        model = modelFromConfig(cfg=cfg, checkpoint=checkpoint_path, device=args.device)
        model.eval()

        embed_path = osp.join('action-recognition/POSEC3D-LSTM/embeddings/uniform', "{}_{}".format(checkpoint_path.split('/')[-2],k))
        print(embed_path)
        k += 1
        if args.save and not os.path.exists(embed_path):
            pathList = pathFile.loc[pathFile['status'] != 'test', :]

            embeddings = np.zeros((len(pathList), model.cls_head.lstm.hidden_size))
            count = 0
            with torch.no_grad():
                for index, row in pathList.iterrows():
                    path = row['filename']

                    keypoint_dict = None
                    frameDirPath = osp.join('/content',path.split('.')[0])
                    for i in range(len(kp_annotation)):
                        if kp_annotation[i]['frame_dir'] == frameDirPath:
                            keypoint_dict = kp_annotation[i]
                            break
                    video = loader(keypoint_dict)
                    if args.device == 'cuda':
                        video = video.to(args.device)
                    pred = model(video, -1)
                    embeddings[count] = pred.to('cpu').numpy()
                    count += 1
            np.save(embed_path, embeddings)
        else:
            if not embed_path.endswith('.npy'):
                embed_path = embed_path + '.npy'
            embeddings = np.load(embed_path)

        knnModel = createKNN(embeddings, 30)
        pathList = pathFile.loc[pathFile['status'] == 'test', :].reset_index()

        with torch.no_grad():
            result = np.zeros((len(pathList), args.top))
            for index, row in pathList.iterrows():
                path = row['filename']
                keypoint_dict = None
                frameDirPath = osp.join('/content',path.split('.')[0])
                for i in range(len(kp_annotation)):
                    if kp_annotation[i]['frame_dir'] == frameDirPath:
                        keypoint_dict = kp_annotation[i]
                        break
                video = loader(keypoint_dict)
                if args.device == 'cuda':
                    video = video.to(args.device)
                pred = model(video, -1)
                dists, ids = knnModel.kneighbors(pred.to('cpu').numpy(), args.top)

                similarLabelList = pathFile.loc[pathFile['label']==row['label']].index
                isIn = np.isin(np.asarray(ids), np.asarray(similarLabelList))
                result[index,:] = isIn

            hak = hitAtK(result, args.top)
            pak = precisionAtK(result, args.top)
            mAP = meanAveragePrecision(result, args.top)

            metrics.append([hak, pak, mAP])
            print('Hit at {}: {:.2f}%'.format(args.top, hak*100))
            print('Precision at {}: {:.2f}%'.format(args.top, pak*100))
            print('Mean Average Precision at {}: {:.2f}'.format(args.top, mAP*100))
            print('\n')

    plot(metrics, ['hit at {}'.format(args.top), 'precision at {}'.format(args.top), 'mean average precision at {}'.format(args.top)], osp.join(args.checkpoint, 'metrics.csv'), osp.join(args.checkpoint, 'metrics.png'))
if __name__ == "__main__":
    args = parse_args()
    main(args)


text:
import numpy as np
import random

def pairs(l):
    for i in range(0, len(l) - 1):
        yield l[i], l[i + 1]

# ==================================================

class Memory(object):
    '''circularlist storage of samples.'''

    def __init__(self, capacity):
        self.capacity = capacity
        self.index = 0
        self.length = 0
        self.data = [None for _ in range(capacity)]

    def add(self, value, p):
        "Add sample to memory with score (that is ignored)."
        self.data[self.index] = [p, value]
        self.index = (self.index + 1) % self.capacity
        self.length = min(self.length + 1, self.capacity)

    def sample(self, n):
        if n < self.length:
            return random.sample(self.data[:self.length], n)
        else:
            return random.sample(self.data, n)

    def is_full(self):
        return self.length == self.capacity

    def remaining(self):
        return (self.capacity - self.length)

# ==================================================

class PrioritizedMemory(object):
    '''Binary tree stored in breadth-first order as array.

    For unbalanced trees, the order in memory is NOT the expected order.
    '''

    def __init__(self, capacity):
        self.capacity = capacity
        self.index = 0
        self.length = 0
        self.nodes = np.zeros(2 * capacity - 1)
        self.data = [None for _ in range(capacity)]

    def total(self):
        "Total score in memory."
        return self.nodes[0]

    def add(self, value, p):
        "Add sample to memory with score."
        self.data[self.index] = value
        self.update(self.capacity - 1 + self.index, p)
        self.index = (self.index + 1) % self.capacity
        self.length = min(self.length + 1, self.capacity)

    def update(self, idx, p):
        "Update score at given index."
        delta = p - self.nodes[idx]
        while idx > 0:
            self.nodes[idx] += delta
            idx = (idx - 1) // 2
        self.nodes[0] += delta

    def get(self, p):
        "Get last sample whose cumulative sum does not exceed value."
        idx = 0
        while idx < self.capacity - 1:
            left = 2 * idx + 1
            right = left + 1

            if p <= self.nodes[left]:
                idx = left
            else:
                idx = right
                p -= self.nodes[left]
        return idx, self.data[idx - self.capacity + 1]

    def sample(self, n):
        '''Return weighted batch of n samples.

        Binned according to Appendix B.2.1 of https://arxiv.org/pdf/1511.05952
        '''
        samples = list(range(n))
        for i, bounds in enumerate(pairs(np.linspace(0, self.total(), n + 1))):
            samples[i] = self.get(random.uniform(bounds[0], bounds[1]))
        return samples

    def is_full(self):
        return self.length == self.capacity

    def remaining(self):
        return self.capacity - self.length

# ==================================================

def make_memory(config):
    memory_type = config.get('MemoryType', fallback='Memory').lower()
    if memory_type == 'memory':
        return Memory(config.getint('MemoryCapacity', fallback=100000))
    elif memory_type == 'prioritizedmemory':
        return PrioritizedMemory(config.getint('MemoryCapacity', fallback=100000))
    else:
        raise ValueError('Unknown memory type: s' % config.get('MemoryType'))

max_stars_repo_name: katetolstaya/kernelrl

max_issues_repo_name: katetolstaya/kernelrl

max_forks_repo_name: katetolstaya/kernelrl


text:
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import tensorflow as tf
from tensorflow import keras
import json
import tempfile
import numpy as np
import base64
import argparse

from singa_auto.model import utils, ImageClfBase, IntegerKnob, CategoricalKnob, FloatKnob, FixedKnob, PolicyKnob
from singa_auto.constants import ModelDependency
from singa_auto.model.dev import test_model_class

class TfFeedForward(ImageClfBase):
    '''
    Implements a fully-connected feed-forward neural network with variable hidden layers on Tensorflow for IMAGE_CLASSIFICATION
    '''

    @staticmethod
    def get_knob_config():
        return {
            'max_epochs':
                FixedKnob(10),
            'hidden_layer_count':
                IntegerKnob(1, 2),
            'hidden_layer_units':
                IntegerKnob(2, 128),
            'learning_rate':
                FloatKnob(1e-5, 1e-1, is_exp=True),
            'batch_size':
                CategoricalKnob([16, 32, 64, 128]),
            'max_image_size':
                CategoricalKnob([16, 32, 48]),
            'quick_train':
                PolicyKnob('EARLY_STOP')  # Whether early stopping would be used
        }

    def __init__(self, **knobs):
        super().__init__()
        self._knobs = knobs
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        self._graph = tf.Graph()
        self._sess = tf.Session(graph=self._graph, config=config)
        self._model = None
        self._train_params = None

    def train(self, dataset_path, **kwargs):
        max_image_size = self._knobs['max_image_size']
        bs = self._knobs['batch_size']
        max_epochs = self._knobs['max_epochs']
        quick_train = self._knobs['quick_train']

        # Define plot for loss against epochs
        utils.logger.define_plot('Loss Over Epochs',
                                 ['loss', 'early_stop_val_loss'],
                                 x_axis='epoch')

        # Load dataset
        dataset = utils.dataset.load_dataset_of_image_files(
            dataset_path, max_image_size=max_image_size, mode='RGB')
        num_classes = dataset.classes
        (images, classes) = zip(*[(image, image_class) for (image, image_class) in dataset])
        (images, norm_mean, norm_std) = utils.dataset.normalize_images(images)

        # Setup callbacks, adding early stopping if quick train
        callbacks = [
            tf.keras.callbacks.LambdaCallback(
                on_epoch_end=self._on_train_epoch_end)
        ]
        if quick_train:
            callbacks.append(
                tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                                 patience=2))

        with self._graph.as_default():
            with self._sess.as_default():
                self._model = self._build_model(num_classes, dataset.image_size)
                self._model.fit(np.asarray(images),
                                np.asarray(classes),
                                verbose=0,
                                epochs=max_epochs,
                                validation_split=0.05,
                                batch_size=bs,
                                callbacks=callbacks)

                # Compute train accuracy
                (loss, accuracy) = self._model.evaluate(images, classes)

        utils.logger.log('Train loss: {}'.format(loss))
        utils.logger.log('Train accuracy: {}'.format(accuracy))

        self._train_params = {
            'image_size': dataset.image_size,
            'norm_mean': norm_mean,
            'norm_std': norm_std
        }

    def evaluate(self, dataset_path, **kwargs):
        max_image_size = self._knobs['max_image_size']
        norm_mean = self._train_params['norm_mean']
        norm_std = self._train_params['norm_std']

        dataset = utils.dataset.load_dataset_of_image_files(
            dataset_path, max_image_size=max_image_size, mode='RGB')

        (images, classes) = zip(*[(image, image_class)
                                  for (image, image_class) in dataset])
        (images, _, _) = utils.dataset.normalize_images(images, norm_mean,
                                                        norm_std)
        with self._graph.as_default():
            with self._sess.as_default():
                (loss, accuracy) = self._model.evaluate(np.asarray(images),
                                                        np.asarray(classes))

        utils.logger.log('Validation loss: {}'.format(loss))

        return accuracy

    def predict(self, queries):
        image_size = self._train_params['image_size']
        norm_mean = self._train_params['norm_mean']
        norm_std = self._train_params['norm_std']

        images, _ = utils.dataset.transform_images(queries,
                                                image_size=image_size,
                                                mode='RGB')
        (images, _, _) = utils.dataset.normalize_images(images, norm_mean,
                                                        norm_std)

        with self._graph.as_default():
            with self._sess.as_default():
                probs = self._model.predict(images)

        return probs.tolist()

    def destroy(self):
        self._sess.close()

    def dump_parameters(self):
        params = {}

        # Add train params
        params['train_params'] = json.dumps(self._train_params)

        # Save model parameters
        with tempfile.NamedTemporaryFile() as tmp:
            # Save whole model to temp h5 file
            with self._graph.as_default():
                with self._sess.as_default():
                    self._model.save(tmp.name)

            # Read from temp h5 file & encode it to base64 string
            with open(tmp.name, 'rb') as f:
                h5_model_bytes = f.read()

            params['h5_model_base64'] = base64.b64encode(h5_model_bytes).decode(
                'utf-8')

        return params

    def load_parameters(self, params):
        # Load model parameters
        h5_model_base64 = params['h5_model_base64']

        with tempfile.NamedTemporaryFile() as tmp:
            # Convert back to bytes & write to temp file
            h5_model_bytes = base64.b64decode(h5_model_base64.encode('utf-8'))
            with open(tmp.name, 'wb') as f:
                f.write(h5_model_bytes)

            # Load model from temp file
            with self._graph.as_default():
                with self._sess.as_default():
                    self._model = keras.models.load_model(tmp.name)

        # Add train params
        self._train_params = json.loads(params['train_params'])

    def _on_train_epoch_end(self, epoch, logs):
        loss = logs['loss']
        early_stop_val_loss = logs['val_loss']
        utils.logger.log(loss=loss,
                         early_stop_val_loss=early_stop_val_loss,
                         epoch=epoch)

    def _build_model(self, num_classes, image_size):
        units = self._knobs['hidden_layer_units']
        layers = self._knobs['hidden_layer_count']
        lr = self._knobs['learning_rate']

        model = keras.Sequential()
        model.add(keras.layers.Flatten(input_shape=(image_size, image_size, 3)))
        model.add(keras.layers.BatchNormalization())

        for _ in range(layers):
            model.add(keras.layers.Dense(units, activation=tf.nn.relu))

        model.add(keras.layers.Dense(num_classes, activation=tf.nn.softmax))

        model.compile(optimizer=keras.optimizers.Adam(lr=lr),
                      loss='sparse_categorical_crossentropy',
                      metrics=['accuracy'])
        return model

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--train_path',
                        type=str,
                        default='data/fashion_mnist_val.zip',
                        help='Path to train dataset')
    parser.add_argument('--val_path',
                        type=str,
                        default='data/fashion_mnist_val.zip',
                        help='Path to validation dataset')
    parser.add_argument('--test_path',
                        type=str,
                        default='data/fashion_mnist_val.zip',
                        help='Path to test dataset')

    parser.add_argument(
        '--query_path',
        type=str,
        default='examples/data/image_classification/0-3100.png',
        help='Path(s) to query image(s), delimited by commas')
    (args, _) = parser.parse_known_args()
    print(args.query_path.split(','))
    queries = utils.dataset.load_images(args.query_path.split(','))
    test_model_class(model_file_path=__file__,
                     model_class='TfFeedForward',
                     task='IMAGE_CLASSIFICATION',
                     dependencies={ModelDependency.TENSORFLOW: '1.12.0'},
                     train_dataset_path=args.train_path,
                     val_dataset_path=args.val_path,
                     test_dataset_path=args.test_path,
                     queries=queries)

max_stars_repo_name: zlheui/singa-auto

max_issues_repo_name: zlheui/singa-auto

max_forks_repo_name: zlheui/singa-auto


text:
import sys
import pprint
import argparse
import ruamel.yaml
import numpy as np
import pandas as pd

# constants include the paths to the config files
from python_tools.constants import *
from python_tools.util import (
    DELIMITER,
    INPUTS_FILE_DELIMITER,
    get_pos,
    reverse_complement,
    all_strings_are_substrings,
    include_yaml_resources,
    include_version_info,
)

##################################
def load_fastqs(data_dir):
    """
    Recursively find all sample files in `data_dir`

    Note:
    os.walk yields a 3-list (dirpath, dirnames, filenames)

    Todo: Consider building a scalable Sample Class to hold all data related to each sample

    :param data_dir: str - folder of Samples with fastqs and SampleSheet.csv
    """
    # Gather Sample Sub-directories (but leave out the parent dir)
    folders = list(os.walk(data_dir, followlinks=True))

    # Filter to those that contain a read 1, read 2, and sample sheet
    folders_2 = filter(
        lambda folder: any([FASTQ_1_FILE_SEARCH in x for x in folder[2]]), folders
    )
    folders_3 = filter(
        lambda folder: any([FASTQ_2_FILE_SEARCH in x for x in folder[2]]), folders_2
    )

    # Issue a warning if not all folders had necessary files (-1 to exclude topmost directory)
    if not len(folders) - 1 == len(folders_3):
        print(
            DELIMITER
            + "Warning, some samples may not have a Read 1 or Read 2 fastq"
            "Please manually check inputs.yaml"
        )
        print("All sample folders:")
        pprint.pprint(folders)
        print("Sample folders with correct result files:")
        pprint.pprint(folders_3)

    # Take just the files
    files_flattened = [
        os.path.join(dirpath, f)
        for (dirpath, dirnames, filenames) in folders_3
        for f in filenames
    ]

    # Convert to absolute paths
    files_flattened = [os.path.abspath(p) for p in files_flattened]

    # Separate into three lists
    fastq1 = filter(lambda x: FASTQ_1_FILE_SEARCH in x, files_flattened)
    fastq1 = [{"class": "File", "path": path} for path in fastq1]
    fastq2 = filter(lambda x: FASTQ_2_FILE_SEARCH in x, files_flattened)
    fastq2 = [{"class": "File", "path": path} for path in fastq2]
    sample_sheet = filter(lambda x: SAMPLE_SHEET_FILE_SEARCH in x, files_flattened)
    sample_sheet = [{"class": "File", "path": path} for path in sample_sheet]

    return fastq1, fastq2, sample_sheet

def perform_duplicate_barcodes_check(title_file):
    """
    Check that no two samples have the same barcode 1 or barcode 2

    Note that this only works when performing this check on an individual lane,
    as barcodes may be reused across lanes.
    """
    for lane in title_file[TITLE_FILE__LANE_COLUMN].unique():
        lane_subset = title_file[title_file[TITLE_FILE__LANE_COLUMN] == lane]

        if np.sum(lane_subset[TITLE_FILE__BARCODE_INDEX_1_COLUMN].duplicated()) > 0:
            raise Exception(
                DELIMITER
                + "Duplicate barcodes for barcode 1, lane {}. Exiting.".format(lane)
            )

        if np.sum(lane_subset[TITLE_FILE__BARCODE_INDEX_2_COLUMN].duplicated()) > 0:
            raise Exception(
                DELIMITER
                + "Duplicate barcodes for barcode 2, lane {}. Exiting.".format(lane)
            )

def sort_fastqs(fastq1, fastq2, sample_sheet, title_file):
    """
    Helper method to sort fastq paths based on title_file ordering.
    Lists of inputs in our yaml file need to be ordered the same order as each other.
    An alternate method might involve using Record types as a cleaner solution.
    """
    fastq1 = sorted(fastq1, key=lambda f: get_pos(title_file, f, use_investigator_sample_id=True))
    fastq2 = sorted(fastq2, key=lambda f: get_pos(title_file, f, use_investigator_sample_id=True))
    sample_sheet = sorted(sample_sheet, key=lambda s: get_pos(title_file, s, use_investigator_sample_id=True))
    return fastq1, fastq2, sample_sheet

def remove_missing_samples_from_title_file(title_file, fastq1, title_file_path):
    """
    If samples IDs from title file aren't found in data directory,
    issue a warning and remove them from the title file

    # Todo: Should we instead raise an error and not continue?
    """
    found_boolv = np.array(
        [
            any([sample in f["path"] for f in fastq1])
            for sample in title_file[TITLE_FILE__COLLAB_ID_COLUMN]
        ]
    )
    samples_not_found = title_file.loc[~found_boolv, TITLE_FILE__COLLAB_ID_COLUMN]

    if samples_not_found.shape[0] > 0:
        print(
            DELIMITER
            + "Error: The following samples were missing either a read 1 fastq, read 2 fastq, or sample sheet. "
            + "These samples will be removed from the title file so that the remaining samples can be run."
        )
        print(
            "Please perform a manual check on the inputs file before running the pipeline."
        )
        print(samples_not_found)

    # Todo: Refactor to put title_file info into inputs.yaml --> then no need to use original title_file in pipeline
    title_file = title_file.loc[found_boolv, :]
    title_file.to_csv(title_file_path, sep="\t", index=False)
    return title_file

def remove_missing_fastq_samples(fastq1, fastq2, sample_sheet, title_file):
    """
    If a sample ID from the title file is not found in any of the paths to the fastqs, remove the fastq.

    Todo: For the SampleSheet files, this relies on the parent folder containing the sample name
    """
    fastq1 = filter(
        lambda f: any(
            [sid + SAMPLE_SEP_FASTQ_DELIMETER in f["path"] for sid in title_file[TITLE_FILE__COLLAB_ID_COLUMN]]
        ),
        fastq1,
    )
    fastq2 = filter(
        lambda f: any(
            [sid + SAMPLE_SEP_FASTQ_DELIMETER in f["path"] for sid in title_file[TITLE_FILE__COLLAB_ID_COLUMN]]
        ),
        fastq2,
    )
    sample_sheet = filter(
        lambda s: any(
            [sid in s["path"] for sid in title_file[TITLE_FILE__COLLAB_ID_COLUMN]]
        ),
        sample_sheet,
    )

    return fastq1, fastq2, sample_sheet

def check_i5_index(title_file_i5, sample_sheet_i5):
    """
    The i5 index (or "Index2" in the SampleSheet.csv file) will either match as is, or as a reverse-complement,
    based on the machine the sequencing was done on.

    :param title_file_i5:
    :param sample_sheet_i5:
    :return:
    """
    rev_comp_i5_barcode = reverse_complement(sample_sheet_i5)

    i5_matches_non_reverse_complemented = sample_sheet_i5 == title_file_i5
    i5_matches_reverse_complemented = rev_comp_i5_barcode == title_file_i5

    err_string = "i5 index from title file {} does not match i5 index from SampleSheet {}. Aborting.".format(
        title_file_i5, sample_sheet_i5
    )

    assert (
        i5_matches_non_reverse_complemented or i5_matches_reverse_complemented
    ), err_string

    if i5_matches_non_reverse_complemented:
        return NON_REVERSE_COMPLEMENTED
    elif i5_matches_reverse_complemented:
        return REVERSE_COMPLEMENTED

def perform_barcode_index_checks_i5(title_file, sample_sheets):
    """
    The i5 index (or "Index2" in the SampleSheet.csv file) will either match as is, or as a reverse-complement,
    based on the machine the sequencing was done on.

    :param title_file:
    :param sample_sheets:
    :return:
    """
    i5_sequencer_types = []
    for sample_id in title_file[TITLE_FILE__SAMPLE_ID_COLUMN]:
        cur_sample = title_file[title_file[TITLE_FILE__SAMPLE_ID_COLUMN] == sample_id]

        matching_sample_sheets = [
            s
            for s in sample_sheets
            if sample_id + SAMPLE_SEP_DIR_DELIMETER in s.get("path")
        ]
        assert (
            len(matching_sample_sheets) == 1
        ), "Incorrect matching sample sheet count for sample {}".format(sample_id)
        sample_sheet = matching_sample_sheets[0]
        sample_sheet_df = pd.read_csv(sample_sheet["path"], sep=",")

        try:
            sample_sheet_i5 = sample_sheet_df["index2"].values[0]
        except KeyError:
            print(
                "Index2 not found in SampleSheet.csv. Skipping i5 barcode index validation."
            )
            return

        title_file_i5 = cur_sample[TITLE_FILE__BARCODE_INDEX_2_COLUMN].values[0]
        i5_sequencer_types.append(check_i5_index(title_file_i5, sample_sheet_i5))

    all_non_reverse_complemented = all(
        [match_type == NON_REVERSE_COMPLEMENTED for match_type in i5_sequencer_types]
    )
    all_reverse_complemented = all(
        [match_type == REVERSE_COMPLEMENTED for match_type in i5_sequencer_types]
    )

    assert (
        all_non_reverse_complemented or all_reverse_complemented
    ), "Not all barcodes followed same i5 index scheme"

    if all_non_reverse_complemented:
        print(
            DELIMITER
            + "All i5 barcodes match without reverse-complementing, sequencer was one of the following:"
        )
        print("NovaSeq\nMiSeq\nHiSeq2500")

    elif all_reverse_complemented:
        print(
            DELIMITER
            + "All i5 barcodes match after reverse-complementing, sequencer was one of the following:"
        )
        print("HiSeq4000\nMiniSeq\nNextSeq")

def perform_barcode_index_checks_i7(title_file, sample_sheets):
    """
    Confirm that the barcode indexes in the title file,
    match to what is listed in the Sample Sheet files from the Illumina Run

    :param title_file:
    :param sample_sheets:
    :return:
    """
    for sample_id in title_file[TITLE_FILE__SAMPLE_ID_COLUMN]:
        cur_sample = title_file[title_file[TITLE_FILE__SAMPLE_ID_COLUMN] == sample_id]
        title_file_i7 = cur_sample[TITLE_FILE__BARCODE_INDEX_1_COLUMN].values[0]

        matching_sample_sheets = [
            s
            for s in sample_sheets
            if sample_id + SAMPLE_SEP_DIR_DELIMETER in s.get("path")
        ]
        assert (
            len(matching_sample_sheets) == 1
        ), "Incorrect matching sample sheet count for sample {}".format(sample_id)
        sample_sheet = matching_sample_sheets[0]
        sample_sheet_df = pd.read_csv(sample_sheet["path"], sep=",")

        # i7 Sequence should always match
        sample_sheet_i7 = sample_sheet_df["index"].values[0]

        err_template = "i7 index does not match for sample {}. Aborting. {} != {}"
        err_string = err_template.format(sample_id, sample_sheet_i7, title_file_i7)
        assert sample_sheet_i7 == title_file_i7, err_string

def include_fastqs_params(fh, data_dir, title_file, title_file_path, force):
    """
    Write fastq1, fastq2, read group identifiers and sample_sheet file references to yaml inputs file.

    :param fh:
    :param data_dir:
    :param title_file:
    :param title_file_path:
    :param force:
    """
    # Load and sort our data files
    fastq1, fastq2, sample_sheets = load_fastqs(data_dir)
    # Get rid of data files that don't have an entry in the title_file
    fastq1, fastq2, sample_sheets = remove_missing_fastq_samples(
        fastq1, fastq2, sample_sheets, title_file
    )

    # If there were missing sample sheets, fill them in with a dummy sample sheet
    missing_sample_sheet_count = len(fastq1) - len(sample_sheets)
    if missing_sample_sheet_count > 0:
        from copy import deepcopy
        dummy_sample_sheet = {'class': 'File', 'path': '/home/johnsoni/vendor_tools/SampleSheet.csv'}
        # Using deepcopy to avoid anchors in resulting yaml
        dummy_sample_sheets = [deepcopy(dummy_sample_sheet) for _ in range(missing_sample_sheet_count)]
        sample_sheets = list(sample_sheets) + list(dummy_sample_sheets)

    # Get rid of entries from title file that are missing data files
    title_file = remove_missing_samples_from_title_file(
        title_file, fastq1, title_file_path
    )
    # Sort everything based on the ordering in the title_file
    fastq1, fastq2, sample_sheets = sort_fastqs(
        fastq1, fastq2, sample_sheets, title_file
    )

    if not force:
        # Check that we have the same number of everything
        perform_length_checks(fastq1, fastq2, sample_sheets, title_file)
        # Check the barcode sequences in the title_file against the sequences in the sample_sheets
        perform_barcode_index_checks_i7(title_file, sample_sheets)
        perform_barcode_index_checks_i5(title_file, sample_sheets)

    samples_info = {
        "fastq1": fastq1,
        "fastq2": fastq2,
        "sample_sheet": sample_sheets,
        "adapter": [ADAPTER_1] * len(fastq1),
        "adapter2": [ADAPTER_2] * len(fastq2),
        # Todo: what's the difference between ID & SM?
        # Todo: do we want the whole filename for ID? (see BWA IMPACT logs)
        # or abbreviate it (might be the way they do it in Roslin)
        "add_rg_ID": title_file[TITLE_FILE__SAMPLE_ID_COLUMN].tolist(),
        "add_rg_SM": title_file[TITLE_FILE__SAMPLE_ID_COLUMN].tolist(),
        "add_rg_LB": title_file[TITLE_FILE__LANE_COLUMN].tolist(),
        # Todo: should we use one or two barcodes in the PU field if they are different?
        "add_rg_PU": title_file[TITLE_FILE__BARCODE_ID_COLUMN].tolist(),
        # Patient ID needs to be a string, in case it is currently an integer
        "patient_id": [
            str(p) for p in title_file[TITLE_FILE__PATIENT_ID_COLUMN].tolist()
        ],
        "sample_class": title_file[TITLE_FILE__SAMPLE_CLASS_COLUMN].tolist(),
    }

    # Trim whitespace
    for key in samples_info:
        samples_info[key] = [
            x.strip() if type(x) == str else x for x in samples_info[key]
        ]

    fh.write(ruamel.yaml.dump(samples_info))

def perform_length_checks(fastq1, fastq2, sample_sheet, title_file):
    """
    Check whether the title file matches input fastqs

    :param fastq1: List[dict] where each dict is a ruamel file object with `class` and `path` keys,
            (`path` being the path to the read 1 fastq)
    :param fastq2: List[dict] where each dict is a ruamel file object with `class` and `path` keys,
            (`path` being the path to the read 2 fastq)
    :param sample_sheet: List[dict] where each dict is a ruamel file object with `class` and `path` keys,
            (`path` being the path to the sample sheet)
    :param title_file:
    """
    try:
        assert len(fastq1) == len(fastq2)
    except AssertionError as e:
        print(
            DELIMITER
            + "Error: Different number of read 1 and read 2 fastqs: {}".format(repr(e))
        )
        print("fastq1: {}".format(len(fastq1)))
        print("fastq2: {}".format(len(fastq2)))
    try:
        assert len(sample_sheet) == len(fastq1)
    except AssertionError as e:
        print(
            DELIMITER
            + "Error: Different number of sample sheet files & read 1 fastqs: {}".format(
                repr(e)
            )
        )
        print("fastq1: {}".format(len(fastq1)))
        print("sample_sheets: {}".format(len(sample_sheet)))
    try:
        assert title_file.shape[0] == len(fastq1)
    except AssertionError as e:
        print(
            DELIMITER
            + "Error: Different number of fastqs files and samples in title file: {}".format(
                repr(e)
            )
        )
        print("fastq1: {}".format(len(fastq1)))
        print("title file length: {}".format(title_file.shape[0]))

def write_inputs_file(args, title_file, output_file_name):
    """
    Main function to write our inputs.yaml file.
    Contains most of the logic related to which inputs to use based on the type of run

    :param args:
    :param title_file:
    :param output_file_name:
    """
    # Actually start writing the inputs file
    fh = open(output_file_name, "wb")

    include_fastqs_params(
        fh, args.data_dir, title_file, args.title_file_path, args.force
    )
    include_yaml_resources(fh, COLLAPSING_INPUTS)

    fh.write(INPUTS_FILE_DELIMITER)
    # Include title_file in inputs.yaml
    title_file_obj = {
        "title_file": {"class": "File", "path": os.path.abspath(args.title_file_path)}
    }
    fh.write(ruamel.yaml.dump(title_file_obj))
    # This file itself is an input to the pipeline,
    # to include version details in the QC PDF
    inputs_yaml_object = {
        "inputs_yaml": {"class": "File", "path": os.path.abspath(output_file_name)}
    }
    fh.write(ruamel.yaml.dump(inputs_yaml_object))
    # Write the current project name for this run
    fh.write("project_name: {}".format(args.project_name))
    # Write the current pipeline version for this pipeline
    try:
        include_yaml_resources(fh, VERSION_PARAM)
    except IOError:
        # that is if version.yaml is absent
        fh.write(INPUTS_FILE_DELIMITER)
        fh.write("# Pipeline Run Version:\n")
        include_version_info(fh)

    fh.close()

def check_final_file(output_file_name):
    """
    Check that lengths of these fields in the final file are equal:
    """
    with open(output_file_name, "r") as stream:
        final_file = ruamel.yaml.round_trip_load(stream)

    # Todo: Use CONSTANTS
    fields_per_sample = [
        "add_rg_ID",
        "add_rg_LB",
        "add_rg_PU",
        "add_rg_SM",
        "patient_id",
        "fastq1",
        "fastq2",
        "sample_sheet",
        "adapter",
        "adapter2",
    ]

    try:
        for field in fields_per_sample:
            assert len(final_file[field]) == len(final_file["fastq1"])
    except AssertionError:
        print(
            DELIMITER
            + "It looks like there aren't enough entries for one of these fields: {}".format(
                fields_per_sample
            )
        )
        print(
            "Most likely, one of the samples is missing a read 1 fastq, read 2 fastq and/or sample sheet"
        )

def parse_arguments():
    parser = argparse.ArgumentParser()

    # Required Arguments
    parser.add_argument(
        "-i",
        "--title_file_path",
        help="Title File (generated from create_title_file.py)",
        required=True,
    )
    parser.add_argument(
        "-d", "--data_dir", help="Directory with fastqs and samplesheets", required=True
    )
    parser.add_argument(
        "-p",
        "--project_name",
        help="A name that describes the current data being run",
        required=True,
    )

    # Optional Arguments
    parser.add_argument(
        "-t",
        "--test",
        help="Whether to run with test params or production params",
        required=False,
        action="store_true",
    )
    parser.add_argument(
        "-o", "--output_file_name", help="Name of yaml file for pipeline", required=True
    )
    parser.add_argument(
        "-f", "--force", help="Skip validation", required=False, action="store_true"
    )
    return parser.parse_args()

def perform_validation(title_file, title_file_path, project_name):
    """
    Make sure that we don't have any unacceptable entries in the title file:

    1. Sample IDs / Collab IDs must be unique
    2. Barcodes must be unique within each lane
    3. Sample_type is in ['Plasma', 'Buffy Coat']
    4. Sample Class is in ['Tumor', 'Normal']
    """
    if not project_name in title_file_path:
        print(
            "WARNING: project ID not found in title file path. Are you sure you are using the correct title file?"
        )

    if np.sum(title_file[TITLE_FILE__SAMPLE_ID_COLUMN].duplicated()) > 0:
        raise Exception(DELIMITER + "Duplicate sample IDs. Exiting.")

    # if set([TITLE_FILE__COLLAB_ID_COLUMN]).issubset(title_file.columns) and np.sum(title_file[TITLE_FILE__COLLAB_ID_COLUMN].duplicated()) > 0:
    #    raise Exception(DELIMITER + 'Duplicate investigator sample IDs. Exiting.')

    for lane in title_file[TITLE_FILE__LANE_COLUMN].unique():
        lane_subset = title_file[title_file[TITLE_FILE__LANE_COLUMN] == lane]

        if np.sum(lane_subset[TITLE_FILE__BARCODE_ID_COLUMN].duplicated()) > 0:
            raise Exception(
                DELIMITER + "Duplicate barcode IDs in lane {}. Exiting.".format(lane)
            )

    if np.sum(
        title_file[TITLE_FILE__SAMPLE_CLASS_COLUMN].isin(ALLOWED_SAMPLE_DESCRIPTION)
    ) < len(title_file):
        raise Exception(
            DELIMITER
            + "Not all sample classes are in {}".format(
                ",".join(ALLOWED_SAMPLE_DESCRIPTION)
            )
        )

    if np.sum(
        title_file[TITLE_FILE__SAMPLE_TYPE_COLUMN].isin(ALLOWED_SAMPLE_TYPE_DESCRIPTION)
    ) < len(title_file):
        raise Exception(
            DELIMITER
            + "Not all sample types are in {}".format(
                ",".join(ALLOWED_SAMPLE_TYPE_DESCRIPTION)
            )
        )

    if not np.issubdtype(title_file[TITLE_FILE__LANE_COLUMN], np.number):
        raise Exception(DELIMITER + "Lane column must be integers")

    if title_file[TITLE_FILE__LANE_COLUMN].isnull().any():
        raise Exception(
            DELIMITER + "Lane column of title file must not be missing or NA"
        )

def print_user_message():
    print(DELIMITER)
    print(
        "You've just created the inputs file. Please double check its entries before kicking off a run."
    )
    print("Common mistakes include:")
    print(
        "1. Trying to use test parameters on a real run (accidentally using the -t param)"
    )
    print("2. Using the wrong bedfile for the capture")
    print(
        "3. Working in the wrong virtual environment (are you sure you ran setup.py install?)"
    )
    print(
        "4. Not specifying the correct parameters for logLevel or cleanWorkDir "
        + "(if you want to see the actual commands passed to the tools, or keep the temp outputs after a successful run)"
    )
    print(
        "5. Did you source the workspace_init.sh script (to make sure the PATH variable is set correctly?)"
    )
    print(
        '6. The "Sex" column of the title file will only correctly idenfity patients with [Male, M, Female, F] entries (although other entries will still be accepted).'
    )

########
# Main #
########

def main():
    # Parse arguments
    args = parse_arguments()

    # Read title file
    title_file = pd.read_csv(args.title_file_path, sep="\t")

    # Sort based on Patient ID
    # This is done to ensure that the order of the samples is retained after indel realignment,
    # which groups the samples on a per-patient basis
    # Todo: This requirement / rule needs to be explicitly documented
    title_file = title_file.sort_values(TITLE_FILE__PATIENT_ID_COLUMN).reset_index(
        drop=True
    )

    # Perform some sanity checks on the title file
    if not args.force:
        perform_validation(title_file, args.title_file_path, args.project_name)
    # Create the inputs file for the run
    write_inputs_file(args, title_file, args.output_file_name)
    # Perform some checks on the final yaml file that will be supplied to the pipeline
    check_final_file(args.output_file_name)
    print_user_message()

if __name__ == "__main__":
    main()

max_stars_repo_name: mskcc/ACCESS-Pipeline

max_issues_repo_name: andurill/ACCESS-Pipeline

max_forks_repo_name: andurill/ACCESS-Pipeline


text:
import numpy as np
import cv2
from matplotlib import pyplot as plt

"""使用FLANN来匹配
FLANN是近似最近邻的快速库，可以在项目中自由使用"""
query_image = cv2.imread('test.jpg', cv2.IMREAD_GRAYSCALE)
training_image = cv2.imread('corner.png', cv2.IMREAD_GRAYSCALE)

# 创建SIFT
sift = cv2.xfeatures2d.SIFT_create()
# 检测并计算
kp1, des1 = sift.detectAndCompute(query_image, None)
kp2, des2 = sift.detectAndCompute(training_image, None)
# FLANN 匹配器的参数
FLANN_INDEX_KDTREE = 0
# algorithm有LinearIndex、KTreeIndex、KMeansIndex、CompositeIndex和AutotuneIbdex
# KTreeIndex非常灵活，trees是密度树的数量，理想值的数量在1~16之间
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
# search_params中的字段checks，用来指定索引树要被遍历的次数，该值越高，计算的时间越长，但也会越准确
search_params = dict(checks=50)

flann = cv2.FlannBasedMatcher(index_params, search_params)

matches = flann.knnMatch(des1, des2, k=2)

matches_mask = [[0, 0] for i in range(len(matches))]

# 丢弃距离大于0.7的匹配
for i, (m, n) in enumerate(matches):
    if m.distance < 0.7 * n.distance:
        matches_mask[i] = [1, 0]

draw_params = dict(matchColor=(0, 255, 0), singlePointColor=(255, 0, 0),
                   matchesMask=matches_mask, flags=0)

result_image = cv2.drawMatchesKnn(query_image, kp1, training_image, kp2, matches, None, **draw_params)
plt.imshow(result_image)
plt.show()

max_stars_repo_name: strawsyz/straw

max_issues_repo_name: strawsyz/straw

max_forks_repo_name: strawsyz/straw


text:
#!/usr/local/bin/python
import sys
import subprocess
import os
import numpy as np

if len(sys.argv) < 2:
    print 'usage:' + sys.argv[0] + ' <bam 1> <bam 2> <..>'
    exit(1)

order = []

for i in range(1,len(sys.argv)):
    bam_file = sys.argv[i]
    print bam_file

    p = subprocess.Popen(\
            ['samtools', 'view', '-H', bam_file], \
            stdout=subprocess.PIPE)

    for l in p.communicate()[0].split('\n'):
        if '@SQ' in l:
            A = l.rstrip().split('\t')
            name = ''
            for a in A:
                if 'SN' in a:
                    name = a.split(':')[1]
            order.append(name)

    p = subprocess.Popen(\
            'samtools view ' +  bam_file + \
            ' | cut -f3,4 | head -n 100000', \
            shell=True, \
            stdout=subprocess.PIPE)

    broke = False
    curr_chrom_index = -1
    curr_pos = -1
    for l in p.communicate()[0].split('\n'):
        if len(l) > 0 :
            a = l.split('\t')
            chrom = a[0]
            pos = int(a[1])

            if order.index(chrom) > curr_chrom_index:
                curr_chrom_index = order.index(chrom)
                curr_pos = -1
            elif order.index(chrom) < curr_chrom_index:
                print 'out of order:\t' + l + '\toccurred after\t' + \
                        order[curr_chrom_index] + '\t' + str(curr_pos)
                broke = True
                break

            if pos > curr_pos:
                curr_pos = pos
            elif pos < curr_pos:
                print 'out of order:\t' + l + '\toccurred after\t' + \
                        order[curr_chrom_index] + '\t' + str(curr_pos)
                broke = True
                break
    if not broke:
        print "in order"


text:
import tqdm
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim

from .. import attack

class Attack(attack.AttackBase):
    """Optimization-based attack."""
    name = 'optimization'

    def __init__(self, *args, **kwargs):
        super(Attack, self).__init__(*args, **kwargs)

        self._attack = tf.make_template('attack-{}'.format(self.name), self._build_attack)

    @classmethod
    def add_options(cls, parser):
        """Augment program arguments."""
        parser.add_argument('--optimization-lambda', type=float, default=0.001)
        parser.add_argument('--optimization-epochs', type=int, default=1000)
        parser.add_argument('--optimization-lr', type=float, default=0.1)

    def _build_attack(self, x, y):
        # Set of generated adversarial examples.
        x_star_var = tf.get_variable('x_star', [
            self.classifier.batch_size,
            self.model.width * self.model.height * self.model.channels
        ], tf.float32, tf.constant_initializer(0))

        # Ensure that x_star is between 0 and 1.
        x_star = (tf.nn.tanh(x_star_var) + 1.0) / 2.0

        # Loss function.
        num_classes = self.classifier.num_classes
        p_lambda = self.options.optimization_lambda

        # Distance between original and generated adversarial example.
        distance = tf.nn.l2_loss(x - x_star)

        # Classification error.
        y = tf.one_hot(y, num_classes)
        y_predict = tf.nn.softmax(self.classifier.predict_op(x_star))
        y = tf.reshape(y, [-1, num_classes, 1])
        y_predict = tf.reshape(y_predict, [-1, 1, num_classes])
        c_loss = tf.reduce_mean(tf.log(1 - tf.matmul(y, y_predict)))

        loss = p_lambda * distance - c_loss

        # Optimizer.
        optimizer = tf.train.AdamOptimizer(
            learning_rate=self.options.optimization_lr
        ).minimize(loss, var_list=[x_star_var])

        return x_star, y, loss, optimizer

    def _get_attack_inputs(self):
        """Get attack input arguments."""
        return {
            'x': self.classifier._get_input_placeholder(),
            'y': self.classifier._get_labels_placeholder(),
        }

    def _get_attack_feed(self, inputs, x, y, offset, limit):
        """Get feed dictionary for each attack batch."""
        return {
            inputs['x']: x[offset:limit, :],
            inputs['y']: y[offset:limit],
        }

    def adversarial_examples(self, x, y, intensity=0.1):
        inputs = self._get_attack_inputs()
        x_star, target, loss, optimizer = self._attack(**inputs)
        loss = tf.reduce_mean(loss)

        # Do some optimization steps to generate proper output.
        total_batch = int(np.floor(x.shape[0] / self.classifier.batch_size))

        x_star_batches = []
        target_batches = []
        for batch in tqdm.trange(total_batch, desc='Batch'):
            # Initialize optimizer.
            self.session.run(tf.variables_initializer(slim.get_variables(self._attack.variable_scope.name)))

            offset = batch * self.classifier.batch_size
            limit = (batch + 1) * self.classifier.batch_size

            progress = tqdm.tqdm(total=self.options.optimization_epochs, desc='Epoch (loss=?.???)', leave=False)
            for epoch in xrange(self.options.optimization_epochs):
                _, x_star_value, target_value, loss_value = self.session.run(
                    [optimizer, x_star, target, loss],
                    feed_dict=self.classifier._set_training(self._get_attack_feed(inputs, x, y, offset, limit), False)
                )

                progress.set_description('Epoch (loss={0:.3f})'.format(loss_value))
                progress.update()
            progress.close()

            x_star_batches.append(x_star_value)
            target_batches.append(target_value)

        return (np.concatenate(x_star_batches), np.concatenate(target_batches))

max_stars_repo_name: shenkev/Exact-Inference-VAE-Robustness

max_issues_repo_name: shenkev/Exact-Inference-VAE-Robustness

max_forks_repo_name: shenkev/Exact-Inference-VAE-Robustness


text:
from vistautils.parameters import Parameters
from vistautils.parameters_only_entrypoint import parameters_only_entry_point
from pegasus_wrapper import (
    initialize_vista_pegasus_wrapper,
    directory_for,
    run_python_on_parameters,
    Locator,
    write_workflow_description,
)
import numpy as np
import adam.experiment.log_experiment as log_experiment_script

def gaze_ablation_runner_entry_point(params: Parameters) -> None:
    """This function creates all possible gaze ablation param files within a given range"""
    initialize_vista_pegasus_wrapper(params)

    # Get the baseline experiment parameters for gaze ablation -- these are things common to all of
    # the experiments, like:
    #
    #     include_image_links: true
    #     sort_learner_descriptions_by_length: True
    #     num_pretty_descriptions: 5
    baseline_parameters = params.namespace("gaze_ablation")

    # get the minimum and maximum number of objects in a scene
    min_num_objects = params.integer("min_num_objects", default=1)
    max_num_objects = params.integer("max_num_objects", default=7)

    # this gets the number of different accuracies to try; default = increment by 0.1
    num_accuracy_increments = params.integer("num_increments", default=11)
    values_for_accuracy = np.linspace(0, 1, num_accuracy_increments)

    # the number of noise instances to be included
    min_num_noise_instances = params.integer("min_num_noise", default=0)
    max_num_noise_instances = params.integer("max_num_noise", default=0)

    # get the number of instances in the entire curriculum
    min_num_instances_in_curriculum = params.integer("min_instances", default=10)
    max_num_instances_in_curriculum = params.integer("max_instances", default=20)

    # all possible numbers of noise instances
    for num_noise_instances in range(
        min_num_noise_instances, max_num_noise_instances + 1
    ):
        # all possible numbers of instances in the curriculum
        for num_instances in range(
            min_num_instances_in_curriculum, max_num_instances_in_curriculum + 1
        ):
            # all possible numbers of instances
            for num_objects_in_instance in range(min_num_objects, max_num_objects + 1):
                # all possible accuracies
                for prob_given in values_for_accuracy:
                    for prob_not_given in values_for_accuracy:
                        # both ignoring and perceiving gaze
                        for add_gaze in [True, False]:
                            # Define the experiment name, which is used both as a job name and to
                            # choose a directory in which to store the experiment results.
                            experiment_name_string = EXPERIMENT_NAME_FORMAT.format(
                                num_instances=num_instances,
                                num_noise_instances=num_noise_instances,
                                num_objects_in_instance=num_objects_in_instance,
                                prob_given=prob_given,
                                prob_not_given=prob_not_given,
                                add_gaze=add_gaze,
                            )
                            experiment_name = Locator(experiment_name_string.split("-"))

                            # Note that the input parameters should include the root params and
                            # anything else we want.
                            experiment_params = baseline_parameters.unify(
                                FIXED_PARAMETERS
                            ).unify(
                                {
                                    "experiment": experiment_name_string,
                                    "experiment_group_dir": directory_for(
                                        experiment_name
                                    ),
                                    "hypothesis_log_dir": directory_for(experiment_name)
                                    / "hypotheses",
                                    "learner_logging_path": directory_for(
                                        experiment_name
                                    ),
                                    "log_learner_state": True,
                                    "resume_from_latest_logged_state": True,
                                    "pursuit-curriculum-params": {
                                        "num_instances": num_instances,
                                        "num_noise_instances": num_noise_instances,
                                        "num_objects_in_instance": num_objects_in_instance,
                                        "add_gaze": add_gaze,
                                        "prob_given": float(prob_given),
                                        "prob_not_given": float(prob_not_given),
                                    },
                                }
                            )

                            run_python_on_parameters(
                                experiment_name,
                                log_experiment_script,
                                experiment_params,
                                depends_on=[],
                            )

    write_workflow_description()

EXPERIMENT_NAME_FORMAT = (
    "{num_instances:d}_instances-{num_noise_instances:d}_noise_instances-{num_objects_in_instance:d}_"
    "objects_in_instance-{prob_given:.3f}_given-{prob_not_given:.3f}_not_given-{add_gaze}_gaze"
)

FIXED_PARAMETERS = {
    "curriculum": "pursuit",
    "learner": "pursuit-gaze",
    "pursuit": {
        "learning_factor": 0.05,
        "graph_match_confirmation_threshold": 0.7,
        "lexicon_entry_threshold": 0.7,
        "smoothing_parameter": 0.001,
    },
}

if __name__ == "__main__":
    parameters_only_entry_point(gaze_ablation_runner_entry_point)

max_stars_repo_name: isi-vista/adam

max_issues_repo_name: Tubbz-alt/adam

max_forks_repo_name: Tubbz-alt/adam