# initial imports
from __future__ import division
from flask import Flask
from flask import render_template
from flask import request
from flask import flash
from flask import json

# local helper function imports
from subject import subject
from define_data import define_data
from session import session
import copy

# data imports
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
from scipy.io import loadmat
import glob

# instance and config
app = Flask(__name__)
app.config['TEMPLATES_AUTO_RELOAD'] = True

#------------------------------------------------------------------------------

# database connection

# functions
def crp(recalls=None, subjects=None, listLength=None, lag_num=None, skip_first_n=0):
    """
    %CRP   Conditional response probability as a function of lag (lag-CRP).
    %
    %  lag_crps = crp(recalls_matrix, subjects, list_length, lag_num)
    %
    %  INPUTS:
    %  recalls_matrix:  a matrix whose elements are serial positions of recalled
    %                   items.  The rows of this matrix should represent recalls
    %                   made by a single subject on a single trial.
    %
    %        subjects:  a column vector which indexes the rows of recalls_matrix
    %                   with a subject number (or other identifier).  That is,
    %                   the recall trials of subject S should be located in
    %                   recalls_matrix(find(subjects==S), :)
    %
    %     list_length:  a scalar indicating the number of serial positions in the
    %                   presented lists.  serial positions are assumed to run
    %                   from 1:list_length.
    %
    %         lag_num:  a scalar indicating the max number of lag to keep track
    %
    %    skip_first_n:  an integer indicating the number of recall transitions to
    %                   to ignore from the start of the recall period, for the
    %                   purposes of calculating the CRP. this can be useful to avoid
    %                   biasing your results, as the first 2-3 transitions are
    %                   almost always temporally clustered. note that the first
    %                   n recalls will still count as already recalled words for
    %                   the purposes of determining which transitions are possible.
    %                   (DEFAULT=0)
    %
    %
    %  OUTPUTS:
    %        lag_crps:  a matrix of lag-CRP values.  Each row contains the values
    %                   for one subject.  It has as many columns as there are
    %                   possible transitions (i.e., the length of
    %                   (-list_length + 1) : (list_length - 1) ).
    %                   The center column, corresponding to the "transition of
    %                   length 0," is guaranteed to be filled with NaNs.
    %
    %                   For example, if list_length == 4, a row in lag_crps
    %                   has 7 columns, corresponding to the transitions from
    %                   -3 to +3:
    %                   lag-CRPs:     [ 0.1  0.2  0.3  NaN  0.3  0.1  0.0 ]
    %                   transitions:    -3   -2    -1   0    +1   +2   +3
    """
    if recalls is None:
        raise Exception('You must pass a recalls matrix.')
    elif subjects is None:
        raise Exception('You must pass a subjects vector.')
    elif listLength is None:
        raise Exception('You must pass a list length.')
    elif len(recalls) != len(subjects):
        raise Exception('recalls matrix must have the same number of rows as subjects.')
    if lag_num is None:
        lag_num = listLength - 1
    elif lag_num < 1 or lag_num >= listLength or not isinstance(lag_num, int):
        raise ValueError('Lag number needs to be a positive integer that is less than the list length.')
    if not isinstance(skip_first_n, int):
        raise ValueError('skip_first_n must be an integer.')

    # Convert recalls and subjects to numpy arrays
    recalls = np.array(recalls)
    subjects = np.array(subjects)
    # Get a list of unique subjects -- we will calculate a CRP for each
    usub = np.unique(subjects)
    # Number of possible lags = (listLength - 1) * 2 + 1; e.g. a length-24 list can have lags -23 through +23
    num_lags = 2 * listLength - 1
    # Initialize array to store the CRP for each subject (or other unique identifier)
    result = np.zeros((usub.size, num_lags))
    # Initialize arrays to store transition counts
    actual = np.empty(num_lags)
    poss = np.empty(num_lags)

    # For each subject/unique identifier
    for i, subj in enumerate(usub):
        # Reset counts for each participant
        actual.fill(0)
        poss.fill(0)
        # Create trials x items matrix where item j, k indicates whether the kth recall on trial j was a correct recall
        clean_recalls_mask = np.array(make_clean_recalls_mask2d(recalls[subjects == subj]))
        # For each trial that matches that identifier
        for j, trial_recs in enumerate(recalls[subjects == subj]):
            seen = set()
            for k, rec in enumerate(trial_recs[:-1]):
                seen.add(rec)
                # Only increment transition counts if the current and next recall are BOTH correct recalls
                if clean_recalls_mask[j, k] and clean_recalls_mask[j, k + 1] and k >= skip_first_n:
                    next_rec = trial_recs[k + 1]
                    pt = np.array([trans for trans in range(1 - rec, listLength + 1 - rec) if rec + trans not in seen], dtype=int)
                    poss[pt + listLength - 1] += 1
                    trans = next_rec - rec
                    # Record the actual transition that was made
                    actual[trans + listLength - 1] += 1

        result[i, :] = actual / poss
        result[i, poss == 0] = np.nan

    result[:, listLength - 1] = np.nan

    return result[:, listLength - lag_num - 1:listLength + lag_num]
def spc(recalls=None, subjects=None, listLength=None):
    """
    SPC   Serial position curve (recall probability by serial position).

    p_recall = spc(recalls, subjects, listLength)

    INPUTS:
        recalls:    a matrix whose elements are serial positions of recalled
                    items. The rows of this matrix should represent recalls
                    made by a single subject on a single trial.

        subjects:   a 1D array which indexes the rows of the recalls matrix
                    with a subject number (or other identifier). That is,
                    the recall trials of subject S should be located in
                    recalls[subjects==S, :]

        listLength: a scalar indicating the number of serial positions in the
                    presented lists. serial positions are assumed to run
                    from [1, 2, ..., list_length].

    OUTPUTS:
        p_recall:  a matrix of probablities.  Its columns are indexed by
                   serial position and its rows are indexed by subject.
    """
    if recalls is None:
        raise Exception('You must pass a recalls matrix.')
    elif subjects is None:
        raise Exception('You must pass a subjects vector.')
    elif listLength is None:
        raise Exception('You must pass a list length.')
    elif len(recalls) != len(subjects):
        raise Exception('recalls matrix must have the same number of rows as subjects.')

    # Convert inputs to numpy arrays if they are not already
    recalls = np.array(recalls)
    subjects = np.array(subjects)
    # Get a list of unique subjects
    usub = np.unique(subjects)
    # Create list of all serial positions (starting with 1)
    positions = np.arange(1, listLength+1)
    # We will return one SPC for each unique subject
    result = np.zeros((len(usub), listLength))

    for i, subj in enumerate(usub):
        # Select only the trials from the current subject
        subj_recalls = recalls[subjects == subj]
        # Create a "recalled" matrix of ones and zeroes indicating whether each presented item was correctly recalled
        subj_recalled = np.array([np.isin(positions, trial_data) for trial_data in subj_recalls])
        # Calculate the subject's SPC as the fraction of trials on which they recalled each serial position's item
        result[i, :] = subj_recalled.mean(axis=0)

    return result

def pfr(recalls, subjects, listLength):
    """"
    PFR   Probability of first recall.

    Computes probability of recall by serial position for the
    first output position.
    [p_recalls] = pfr(recalls_matrix, subjects, list_length)

    INPUTS:
        recalls:    a matrix whose elements are serial positions of recalled
                    items.  The rows of this matrix should represent recalls
                    made by a single subject on a single trial.

        subjects:   a column vector which indexes the rows of recalls_matrix
                    with a subject number (or other identifier).  That is,
                    the recall trials of subject S should be located in
                    recalls_matrix(find(subjects==S), :)

        listLength: a scalar indicating the number of serial positions in the
                    presented lists.  serial positions are assumed to run
                    from 1:list_length.

    OUTPUTS:
        p_recalls:  a matrix of probablities.  Its columns are indexed by
                    serial position and its rows are indexed by subject.
    """
    if recalls is None:
        raise Exception('You must pass a recalls matrix.')
    elif subjects is None:
        raise Exception('You must pass a subjects vector.')
    elif listLength is None:
        raise Exception('You must pass a list length.')
    elif len(recalls) != len(subjects):
        raise Exception('recalls matrix must have the same number of rows as subjects.')

    subject = np.unique(subjects)
    result = [[0] * listLength for count in range(len(subject))]
    for subject_index in range(len(subject)):
        count = 0
        for subj in range(len(subjects)):
            if subjects[subj] == subject[subject_index]:
                if recalls[subj][0] > 0 and recalls[subj][0] < 1 + listLength:
                    result[subject_index][recalls[subj][0] - 1] += 1
                count += 1
        for index in range(listLength):
            result[subject_index][index] /= count
    return result

def make_clean_recalls_mask2d(data):
    """makes a clean mask without repetition and intrusion"""
    result = copy.deepcopy(data)
    for num, item in enumerate(data):
        seen = []
        for index, recall in enumerate(item):

            if recall > 0 and recall not in seen:
                result[num][index] = 1
                seen.append(recall)
            else:
                result[num][index] = 0
    return result

def get_patients():
	files = glob.glob('./data/*.mat')
	patients = []
	for file in files:
		patients.append(file[7:file.find('_e')])

	return sorted(set(patients))

def load_session(path):
	x = loadmat(path)
	x = x['events'][0]

	return x

def get_data_info(patient):
	sessions = glob.glob('./data/' + patient + '_events_sess*.mat')
	sessions.sort()
	data_info = {
		'session_paths' : sessions,
	}

	return data_info

def getPRec(sessions):
	precs = []
	for sess in sessions:
		x = load_session(sess)
		recalled = len(x[x['recalled'] == 1])
		total = len(x[x['type'] == 'WORD'])
		precs.append(recalled/total)

	return precs

def getIntrusions(sessions):
	intrusions = []
	for sess in sessions:
		x = load_session(sess)
		intrution = len(x[x['intrusion'] == 1])
		total = len(x[x['type'] == 'WORD'])
		intrusions.append(intrution/total)

	return intrusions

def getRepeats(sessions):
	repeats = []
	for sess in sessions:
		x = load_session(sess)
		rec_words = x[x['type'] == 'REC_WORD']
		rec_words = rec_words['item'].tolist()
		total = len(x[x['type'] == 'WORD'])

		unique_recalls = set()
		for r in rec_words:
		    unique_recalls.add(r[0])
		if len(rec_words) > 0:
			repeat_rate = (len(rec_words) - len(unique_recalls)) / total
			repeats.append(repeat_rate)

	return repeats

def getAvgSPC(sessions):
	session_spc = []
	for sess in sessions:
		x = load_session(sess)
		# get recalled words
		rec_words = x[x['type'] == 'REC_WORD']
		# number of trials in this session
		num_trials = rec_words['trial'].max()[0][0]

		trial_spc = []
		for i in range(num_trials):
		    trial = rec_words[rec_words['trial'] == i+1]
		    serials = trial['serialPos']

		    # check for empty recall trial
		    if len(trial) == 0:
		    	continue

		    # flatten list of arrays to list
		    serials = np.concatenate(serials).ravel()
		    # remove nans
		    serials = np.array([ int(x) for x in serials if str(x) != 'nan'])
		    # remove negatives
		    serials = np.array([ int(x) for x in serials if x > 0])
		    # remove duplicates
		    serials = list(set(serials))

		    serial_position_counts = np.bincount(serials, minlength=13) # don't hardcode
		    serial_position_counts = serial_position_counts[1:] # don't care about 0
		    trial_spc.append(np.array(serial_position_counts))

		session_spc.append(np.mean(np.array(trial_spc), axis=0))

	return np.mean(np.array(session_spc), axis=0).tolist()

def getAvgPFR(sessions):
	session_pfr = []
	for sess in sessions:
		x = load_session(sess)
		# get recalled words
		rec_words = x[x['type'] == 'REC_WORD']
		# number of trials in this session
		num_trials = rec_words['trial'].max()[0][0]

		first_recalls = []
		for i in range(num_trials):
		    trial = rec_words[rec_words['trial'] == i+1]
		    serials = trial['serialPos']

		    # check for empty recall trial
		    if len(trial) == 0:
		        continue

		    # flatten list of arrays to list
		    serials = np.concatenate(serials).ravel()

		    # remove nans
		    serials = np.array([ int(x) for x in serials if str(x) != 'nan'])
		    # remove negatives
		    serials = np.array([ int(x) for x in serials if x > 0])

			# creates a serial position array for each trial and marks which position was first recalled
		    first_recall = np.zeros(12)

		    try:
		    	first_recall[serials[0]-1] += 1
		    except IndexError:
		    	continue

		    first_recalls.append(first_recall)

		session_pfr.append(np.mean(np.array(first_recalls), axis=0))

	return np.mean(np.array(session_pfr), axis=0).tolist()

def getLagCRP(sessions):
	session_spc = []
	lags = []
	for sess in sessions:
		x = load_session(sess)
		# get recalled words
		rec_words = x[x['type'] == 'REC_WORD']
		# number of trials in this session
		num_trials = rec_words['trial'].max()[0][0]

		for i in range(num_trials):
		    trial = rec_words[rec_words['trial'] == i+1]
		    serials = trial['serialPos']

		    # check for empty recall trial
		    if len(trial) == 0:
		    	continue

		    # flatten list of arrays to list
		    serials = np.concatenate(serials).ravel()
		    # remove nans
		    serials = np.array([ int(x) for x in serials if str(x) != 'nan'])
		    # remove negatives
		    serials = np.array([ int(x) for x in serials if x > 0])
		    # remove duplicates
		    serials = list(set(serials))
		    if len(serials) > 0:
		    	lags.append( crp([serials], ['single patient'], 12)[0] )

	lags = np.nanmean(lags, 0)
	lags[np.isnan(lags)] = 0
	indices = [-11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
	lag_crp = { str(indices[n]):round(lag, 3) for n,lag in enumerate(lags) }
	return lag_crp

# routes
@app.route("/")
def main():
	patients = get_patients()
	return render_template('index.html',
							patients=patients)

@app.route('/patient/id/<string:user_id>')
def profile(user_id):
    patient_data_paths = get_data_info(user_id)
    num_sessions = len(patient_data_paths['session_paths'])

    if num_sessions:
    	session_precs = getPRec(patient_data_paths['session_paths'])
    	session_intrusions = getIntrusions(patient_data_paths['session_paths'])
    	session_repeats = getRepeats(patient_data_paths['session_paths'])
    	average_spc = getAvgSPC(patient_data_paths['session_paths'])
    	average_pfr = getAvgPFR(patient_data_paths['session_paths'])
    	lag_crp = getLagCRP(patient_data_paths['session_paths'])

    	# round array values before sending -- might want to move this to the functions
    	session_precs = [ round(x, 3) for x in session_precs ]
    	session_intrusions = [ round(x, 3) for x in session_intrusions ]
    	session_repeats = [ round(x, 3) for x in session_repeats ]
    	average_spc = [ round(x, 3) for x in average_spc ]
    	average_pfr = [ round(x, 3) for x in average_pfr ]

    	return render_template('data.html',
    							patient_id=user_id,
    							num_sessions=num_sessions,
    							sessions=patient_data_paths['session_paths'],
    							precs=session_precs,
    							intrusions=session_intrusions,
    							repeats=session_repeats,
    							avg_spc=json.dumps(average_spc),
    							avg_pfr=json.dumps(average_pfr),
    							lag_crp=json.dumps(lag_crp),
	    					  )
    else:
    	return render_template('error.html')

@app.route("/examplePost", methods=['POST'])
def examplePost():
	user_id = request.values.get("user_id")
	patient_data_paths = get_data_info(user_id)
    # creates a subject object from ID, which has many cool methods
	try:
		su = subject(index=0, subjectList=[user_id]).concatenate(False, patient_data_paths['session_paths'])
		scs = round(np.mean(su.SCS), 3)
		tcs = round(np.mean(su.TCS), 3)
	except ValueError as err:
		print(err)
		scs = 'Error: ' + str(err) + ''
		tcs = 'Error: ' + str(err) + ''

	data = {
		"scs":str(scs),
		"tcs":str(tcs)
	}
	return json.dumps(data)

#------------------------------------------------------------------------------

if __name__ == "__main__":
	app.run()


from __future__ import print_function
import argparse, sys

def main(argv=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', nargs='*', help='filenames to check')
    parser.add_argument('--license-filepath', default='LICENSE.txt')
    parser.add_argument('--comment-style', default='#',
                        help='Can be a single prefix or a triplet: <comment-sart>|<comment-prefix>|<comment-end>'
                             'E.g.: /*| *| */')
    parser.add_argument('--detect-license-in-X-top-lines', type=int, default=5)
    parser.add_argument('--remove-header', action='store_true')
    args = parser.parse_args(argv)

    if '|' in args.comment_style:
        comment_start, comment_prefix, comment_end = args.comment_style.split('|')
    else:
        comment_start, comment_prefix, comment_end = None, args.comment_style, None

    with open(args.license_filepath) as license_file:
        prefixed_license = ['{}{}{}'.format(comment_prefix, ' ' if line.strip() else '', line)
                            for line in license_file.readlines()]
    eol = '\r\n' if prefixed_license[0][-2:] == '\r\n' else '\n'
    if not prefixed_license[-1].endswith(eol):
        prefixed_license[-1] += eol
    if comment_start:
        prefixed_license = [comment_start + eol] + prefixed_license
    if comment_end:
        prefixed_license = prefixed_license + [comment_end + eol]

    changes_made = False
    for src_filepath in args.filenames:
        with open(src_filepath) as src_file:
            src_file_content = src_file.readlines()
        license_header_index = find_license_header_index(src_file_content, prefixed_license, args.detect_license_in_X_top_lines)
        if license_header_index is not None:
            if args.remove_header:
                if src_file_content[license_header_index+len(prefixed_license)].strip():
                    src_file_content = src_file_content[:license_header_index] + src_file_content[license_header_index+len(prefixed_license):]
                else:
                    src_file_content = src_file_content[:license_header_index] + src_file_content[license_header_index+len(prefixed_license)+1:]
                with open(src_filepath, 'w') as src_file:
                    src_file.write(''.join(src_file_content))
                changes_made = True
        elif not args.remove_header:
            src_file_content = prefixed_license + [eol] + src_file_content
            with open(src_filepath, 'w') as src_file:
                src_file.write(''.join(src_file_content))
            changes_made = True

    if changes_made:
        print('')
        print('The license has been added to some source files. Now aborting the commit.')
        print('You can check the changes made. Then simply "git add --update ." and re-commit')
        return 1
    return 0

def find_license_header_index(src_file_content, prefixed_license, top_lines_count):
    '''
    Returns the line number, starting from 0 and lower than `top_lines_count`,
    where the license header comment starts in this file, or else None.
    '''
    for i in range(top_lines_count):
        license_match = True
        for j, license_line in enumerate(prefixed_license):
            if i + j >= len(src_file_content) or license_line.strip() != src_file_content[i + j].strip():
                license_match = False
                break
        if license_match:
            return i
    return None

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))


# -*- coding: utf-8 -*-
'''
#-------------------------------------------------------------------------------
# NATIONAL UNIVERSITY OF SINGAPORE - NUS
# SINGAPORE INSTITUTE FOR NEUROTECHNOLOGY - SINAPSE
# Singapore
# URL: http://www.sinapseinstitute.org
#-------------------------------------------------------------------------------
# Neuromorphic Engineering Group
# Author: Andrei Nakagawa-Silva, MSc
# Contact: nakagawa.andrei@gmail.com
#-------------------------------------------------------------------------------
'''
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#LIBRARIES
import numpy as np
from collections import deque
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#detect peaks
def detect_peaks(x, mph=None, mpd=1, threshold=0, edge='rising',
         kpsh=False, valley=False, show=False, ax=None):

    x = np.atleast_1d(x).astype('float64')
    if x.size < 3:
        return np.array([], dtype=int)
    if valley:
        x = -x
    # find indices of all peaks
    dx = x[1:] - x[:-1]
    # handle NaN's
    indnan = np.where(np.isnan(x))[0]
    if indnan.size:
        x[indnan] = np.inf
        dx[np.where(np.isnan(dx))[0]] = np.inf
    ine, ire, ife = np.array([[], [], []], dtype=int)
    if not edge:
        ine = np.where((np.hstack((dx, 0)) < 0) & (np.hstack((0, dx)) > 0))[0]
    else:
        if edge.lower() in ['rising', 'both']:
            ire = np.where((np.hstack((dx, 0)) <= 0) & (np.hstack((0, dx)) > 0))[0]
        if edge.lower() in ['falling', 'both']:
            ife = np.where((np.hstack((dx, 0)) < 0) & (np.hstack((0, dx)) >= 0))[0]
    ind = np.unique(np.hstack((ine, ire, ife)))
    # handle NaN's
    if ind.size and indnan.size:
        # NaN's and values close to NaN's cannot be peaks
        ind = ind[np.in1d(ind, np.unique(np.hstack((indnan, indnan-1, indnan+1))), invert=True)]
    # first and last values of x cannot be peaks
    if ind.size and ind[0] == 0:
        ind = ind[1:]
    if ind.size and ind[-1] == x.size-1:
        ind = ind[:-1]
    # remove peaks < minimum peak height
    if ind.size and mph is not None:
        ind = ind[x[ind] >= mph]
    # remove peaks - neighbors < threshold
    if ind.size and threshold > 0:
        dx = np.min(np.vstack([x[ind]-x[ind-1], x[ind]-x[ind+1]]), axis=0)
        ind = np.delete(ind, np.where(dx < threshold)[0])
    # detect small peaks closer than minimum peak distance
    if ind.size and mpd > 1:
        ind = ind[np.argsort(x[ind])][::-1]  # sort ind by peak height
        idel = np.zeros(ind.size, dtype=bool)
        for i in range(ind.size):
            if not idel[i]:
                # keep peaks with the same height if kpsh is True
                idel = idel | (ind >= ind[i] - mpd) & (ind <= ind[i] + mpd) \
                    & (x[ind[i]] > x[ind] if kpsh else True)
                idel[i] = 0  # Keep current peak
        # remove the small peaks and sort back the indices by their occurrence
        ind = np.sort(ind[~idel])

    if show:
        if indnan.size:
            x[indnan] = np.nan
        if valley:
            x = -x
        _plot(x, mph, mpd, threshold, edge, valley, ax, ind)

    return ind
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#this class defines a moving average algorithm with 99% overlap
class MovingAverage():
    def __init__(self,_windowSize=10,_sampfreq=100):
        self.window = deque() #processing window
        self.windowsum = 0 #keeps track of the sum of all elements in the window
        #counts the number of samples to decide how to calculate the moving
        #average
        self.samplecounter = 0
        #sampling rate
        self.sampfreq = _sampfreq
        #sampling period
        self.dt = 1.0 / self.sampfreq
        #size of the window
        self.windowSize = _windowSize
    def getSample(self,_newsample):
        #if the sample counter is greater than the size of the window, then
        #the window should move in order to output the new data
        if self.samplecounter >= self.windowSize:
            #subtracts from the first element of the window
            self.windowsum -= self.window.popleft()
            #adds the new sample to the end of the window
            self.windowsum += _newsample
            #appends the new sample to the window
            self.window.append(_newsample)
            #returns the moving average response
            return self.windowsum / self.windowSize
        else: #otherwise, just populate the window
            self.windowsum += _newsample
            self.window.append(_newsample) #appends the new sample
            self.samplecounter += 1 #increase the sample counter
            return self.windowsum / self.samplecounter
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#testing the library
if __name__ == '__main__':
    #plotting with matplotlib
    import matplotlib.pyplot as plt
    #signal parameters
    sampfreq = 100 #sampling rate
    dt = 1.0 / sampfreq # sampling period
    f = 2 #sine wave frequency
    tb = 0 #time begins
    te = 5 #time ends
    #time
    x = np.arange(tb,te,dt)
    #pure sine wave
    y = np.sin(2.0*np.pi*f*x)
    #gaussian noise
    w = np.random.uniform(0,0.8,y.shape)
    #add noise to the original signal
    y += w
    #testing the moving average
    #moving average class
    windowsize = 10 #size of the window
    mva = MovingAverage(windowsize,sampfreq)
    filts = [] #new list containing the filtered signal
    #for every sample of the sine wave, generate the moving average value
    [filts.append(mva.getSample(x)) for x in y]

    #plots
    plt.figure()
    plt.subplot(2,1,1)
    plt.plot(x,y) #sine wave with noise
    plt.ylabel('Amplitude')
    plt.subplot(2,1,2)
    plt.plot(x,filts) #filtered with moving average
    plt.ylabel('Amplitude')
    plt.xlabel('Time (s)')
    plt.show()


from flownet import *
import jacky_tool
from keras.models import load_model
import os
import random
import winsound
import numpy as np

def train_model(X, Y, stop_callback, model_func=FlowNetS, overwrite=False, summary=False, epochs=10, batch_size=16):
    if overwrite:
        model = model_func()
    else:
        if os.path.exists('model/{}.nn'.format(model_func.__name__)):
            model = load_model('model/{}.nn'.format(model_func.__name__), custom_objects={'EPE_loss': EPE_loss})
        else:
            model = model_func()

    if summary:
        print(model.summary())

    history = model.fit(X, Y,
                        epochs=epochs,
                        batch_size=batch_size,
                        validation_split=0.2,
                        callbacks=[stop_callback]).history

    history_file_path = 'model/{}.ht'.format(model_func.__name__)
    if overwrite:
        history = {
            'loss': history['loss'],
            'val_loss': history['val_loss']
        }
        jacky_tool.save(history, history_file_path)

    else:
        if os.path.exists(history_file_path):
            origin_history = jacky_tool.load(history_file_path)
            origin_history['loss'].extend(history['loss'])
            origin_history['val_loss'].extend(history['val_loss'])
            jacky_tool.save(origin_history, history_file_path)
        else:
            history = {
                'loss': history['loss'],
                'val_loss': history['val_loss']
            }
            jacky_tool.save(history, history_file_path)

    model.save('model/{}.nn'.format(model_func.__name__))

def test_train(data_file_path):
    stop_callback = EarlyStoppingByLossVal(monitor='val_loss', threshold=1)
    X, Y = jacky_tool.load(data_file_path)
    X = X[:10]
    Y = [y[:10] for y in Y]
    train_model(X / 255.0, Y, stop_callback, overwrite=True, summary=True, epochs=50, batch_size=1)

def test_train_2000():
    stop_callback = EarlyStoppingByLossVal(monitor='val_loss', threshold=1)
    X, Y = jacky_tool.load('data/flying_chairs_{:05d}_{:05d}.np'.format(1, 2000))
    train_model(X / 255.0, Y,stop_callback, overwrite=True, summary=True, epochs=15, batch_size=16)
    winsound.Beep(440, 1000)

def full_train(round_limit=20, stop_threshold=5):
    """Using flying chair to train data"""
    chair_pairs = []
    step = 2000
    for start in range(1, 22873, step):
        end = start + step if start + step < 22873 else 22873
        chair_pairs.append((start, end - 1))

    indexes = np.arange(0, len(chair_pairs))

    rounds = 0
    stop_by_threshold = False

    while not stop_by_threshold and rounds < round_limit:
        rounds += 1
        np.random.shuffle(indexes)
        stop_by_threshold = train_one_round(indexes, chair_pairs, stop_threshold)

    print('Rounds = {}'.format(rounds))
    winsound.Beep(440, 1000)

def train_one_round(indexes, chair_pairs, stop_threshold):
    for i in np.nditer(indexes):
        pair = chair_pairs[i]
        print('Use flying chairs pair:', pair)
        X, Y = jacky_tool.load(os.path.join(r'D:\Dataset\python', 'flying_chairs_{:05d}_{:05d}.np'.format(*pair)))
        X = X / 255.0
        stop_callback = EarlyStoppingByLossVal(monitor='val_loss', threshold=stop_threshold)
        train_model(X, Y, stop_callback, overwrite=False, summary=False, epochs=10, batch_size=16)

        if stop_callback.stop_by_threshold:
            return True
    return False

dataset = ['mpi-sintel-clean.np', 'flying_chairs_00001_02000.np']

# test_train(os.path.join('data', dataset[0]))
# test_train_2000()
full_train(round_limit=50, stop_threshold=2)


"""
    kekeringan.py
    Aplikasi Kekeringan

    @author: Widoyo
    @date: 05 Mei 2017
"""
import datetime
import web

urls = (
    '$', 'Index',
)

app_kekeringan = web.application(urls, locals())
session = web.session.Session(app_kekeringan, web.session.DiskStore('sessions'),
                              initializer={'username': None, 'role': None,
                                           'flash': None})

globals = {'session': session}
render = web.template.render('templates/', base='base', globals=globals)

class Index:
    def GET(self):
        return render.kekeringan.index()


from html.parser import HTMLParser
import urllib3

class RadioProgram():

    def __init__(self, start, stop, name):
        self.start = start
        self.stop = stop
        self.name = name

# create a subclass and override the handler methods
class MyHTMLParser(HTMLParser):

    def __init__(self):
        #super().__init__()
        self.reset()
        self.strict = False
        self.convert_charrefs= True
        self.fed = []
        self.breadcrumps = []
        self.programs = []
        self.current_prog_start_utc = ""
        self.current_prog_start_cat = ""
        self.current_prog_name  = ""
        self.record = False

    def handle_starttag(self, tag, attrs):
        #print("Encountered a start tag:", tag)
        self.breadcrumps.append(tag)
        marker = ['html', 'head', 'meta', 'body', 'div', 'div', 'div', 'div', 'section', 'div', 'ul', 'form', 'div', 'div', 'div', 'h5']
        if self.breadcrumps == marker:
            self.record = True

    def handle_endtag(self, tag):
        #print("Encountered an end tag :", tag)
        self.breadcrumps.pop()

    def handle_data(self, data):
        if self.record:
            if self.current_prog_start_utc == "":
                self.current_prog_start_utc = data
                # ignore UTC
                pass
            elif self.current_prog_start_cat == "":
                self.current_prog_start_cat = data
                # CAT - Centrle Africa Time
                if len(self.programs) > 0:
                    # get last recognized program and input stop-time
                    lastProg = self.programs[len(self.programs)-1]
                    lastProg.stop = self.current_prog_start_cat

            elif self.current_prog_name == "":
                self.current_prog_name = data
                # default will be overwirten, when I know that the next
                # program is starting
                # for the last program on that day it is also correct like that
                default_end_time = '24:00'

                p = RadioProgram(self.current_prog_start_cat,
                                default_end_time,
                                self.current_prog_name)
                self.programs.append(p)
                self.current_prog_start_utc = ""
                self.current_prog_start_cat = ""
                self.current_prog_name  = ""
        self.record = False
        #print("bread-crumps: ", self.breadcrumps)
        #print("Encountered some data  :", data)

    def get_data(self):
        print(self.fed)
        return ''.join(self.fed)

    def getPrograms(self):
        return self.programs

class Schedule:

    class Row:

        def __init__(self, time_formated, time_mins):
            self.time_formated = time_formated
            self.time_mins = time_mins
            self.col = []
            self.col.append("") # monday
            self.col.append("") # tuesday
            self.col.append("") # wednesday
            self.col.append("") # thursday
            self.col.append("") # friday
            self.col.append("") # saturday
            self.col.append("") # sunday

        def setProgram(self, col, program_name):
            self.col[col] = program_name

    def __init__(self, convert_comma_to_semicolon=False):
        self.convert_comma_to_semicolon = convert_comma_to_semicolon
        self.rows = []

        for minute in range(24*60):
            time_mins = minute
            time_formanted = '%02d:%02d' % (minute/60, minute%60)
            row = Schedule.Row(time_formanted, time_mins)
            self.rows.append(row)

    def getRows(self):
        return self.rows

    def addProgram(self, day, start_formated, end_formated, name):
        hour= int(start_formated.split(":")[0])
        minute = int(start_formated.split(":")[1])

        start_mins = hour*60+minute

        hour= int(end_formated.split(":")[0])
        minute = int(end_formated.split(":")[1])
        end_mins = hour*60+minute

        for i in range(start_mins, end_mins):
            if self.convert_comma_to_semicolon:
                name = name.replace(",", ";")
            self.rows[i].col[day] = name

class Formater():

    def __init__(self, seperator="-", space_for_entries=40, show_header=False):
        self.seperator = seperator
        self.space_for_entries = space_for_entries
        self.show_header = show_header

    def format(self, obj_to_format):
        output = ""
        if self.show_header:
            for header in ["Time", "Monday", "Tuesday", "Wednesday", "Thursday", "Fireday", "Saturday", "Sunday"]:
                output += header + self.seperator
            output += "\n"
        for row in range(0, len(obj_to_format.getRows())):
            output += obj_to_format.getRows()[row].time_formated + self.seperator
            for col in range(0, len(obj_to_format.getRows()[row].col)):

                output +=  " {0:^{2}} {1} ".format(obj_to_format.getRows()[row].col[col], self.seperator, 40)
            output += "\n"
        return output;

class Downloader:

    def __init__(self):
        self.mon = ["monday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=1&tx=4"]
        self.tue = ["tuesday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=2&tx=4"]
        self.wed = ["wednesday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=3&tx=4"]
        self.thu = ["thursday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=4&tx=4"]
        self.fri = ["friday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=5&tx=4"]
        self.sat = ["saturday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=6&tx=4"]
        self.sun = ["sunday", "https://www.twrafrica.org/index.php/component/schedules/?task=days.weekdays&dow=7&tx=4"]
        self.week = [self.mon, self.tue, self.wed, self.thu, self.fri, self.sat, self.sun]

    def file_name(self, day, folder):
        return folder + "/" + day[0] + ".html"

    def download(self, folder):
        progs = []

        http = urllib3.PoolManager()

        for day in self.week:
            print("download " + day[0])

            response = http.request('GET', day[1], preload_content=False)
            print("response: " + str(response.status))
            f = open(self.file_name(day,folder ), 'w')
            f.write(response.data.decode("utf-8"))
            f.close

    def get_offline(self, folder):
        progs = []

        for day in self.week:
            txt = open(self.file_name(day, folder))
            wholeFile = txt.read() # read whole file at once
            txt.close()

            parser = MyHTMLParser()
            parser.feed(wholeFile)
            progs.append(parser.getPrograms())
        return progs

if __name__ == '__main__':
    worker = Downloader()
    progsPerDay = worker.download("tmp")
    progsPerDay = worker.get_offline("tmp")
    schedule = Schedule()

    for day in range(0, len(progsPerDay)):
        #print("***********************day: ", day)
        for prog in range(0, len(progsPerDay[day])):
            progData = progsPerDay[day][prog]
            #print("progData: %s, %s, %s" % (progData.start, progData.stop, progData.name))
            schedule.addProgram(day, progData.start, progData.stop, progData.name)

    formater = Formater(seperator=",", show_header=True)

    output = formater.format(schedule)

    file = open("schedule.csv", 'w')
    file.truncate()     # delete content of file
    file.write(output)  # write text to file
    file.close()
    print("finished")


#!/usr/bin/env python3
import time
import json
import matplotlib.pyplot as plt
import pathmagic
from dataclasses import dataclass
from dacite import from_dict
import numpy as np
with pathmagic.context():
    from CBBA import CBBA
    from WorldInfo import WorldInfo
    import HelperLibrary as HelperLibrary
    from Agent import Agent
    from Task import Task
    from Obstacle import Obstacle
    from window import Window

if __name__ == "__main__":
    # a json configuration file
    config_file_name = "config_example_01.json"
    # Read the configuration from the json file
    json_file = open(config_file_name)
    config_data = json.load(json_file)

    # create a world, each list is [min, max] coordinates for x,y,z axis
    WorldInfoTest = WorldInfo([-2.0, 2.5], [-1.5, 5.5], [0.0, 20.0])

    # create a list of Agent(s) and Task(s)
    num_agents = 5
    num_tasks = 10
    num_obs = 3
    max_depth = num_tasks
    # AgentList, TaskList, obsList = HelperLibrary.create_agents_and_tasks_and_obstacles(num_agents, num_tasks, num_obs, WorldInfoTest, config_data)

    # save_agents = []
    # for agent in AgentList:
    #     al_dict = agent.__dict__
    #     save_agents.append(al_dict)

    # save_tasks = []
    # for task in TaskList:
    #     tas_dict = task.__dict__
    #     save_tasks.append(tas_dict)

    # save_obs = []
    # for obs in obsList:
    #     obs_dict = obs.__dict__
    #     save_obs.append(obs_dict)
    # case_dict = {'AgentList':save_agents, 'TaskList':save_tasks, 'obsList':save_obs}
    # with open('case3', 'w') as fout:
    #     json.dump(case_dict, fout)
    AgentList = []
    TaskList = []
    obsList = []

    with open('case2', 'r') as fin:
        data = json.load(fin)
        for agent in data['AgentList']:
            AgentList.append(from_dict(data_class=Agent, data=agent))
        for task in data['TaskList']:
            TaskList.append(from_dict(data_class=Task, data=task))

        for obs in data['obsList']:
            obsList.append(from_dict(data_class=Obstacle, data=obs))

    # create a CBBA solver
    CBBA_solver = CBBA(config_data)

    t_start = time.time()

    # path_list, times_list = CBBA_solver.solve(AgentList, TaskList, WorldInfoTest, max_depth, time_window_flag=False, obsList=obsList)
    # # solve
    if False:
        window = Window("CBBA-EGO")
        window.show(block=False)
        window.ax.set_ylim([-2.0, 2.5])
        window.ax.set_xlim([-1.5, 5.5])
        path_list, times_list = CBBA_solver.solve(AgentList, TaskList, WorldInfoTest, max_depth, time_window_flag=True, obsList=obsList)
        current_index = [0 for _ in range(num_agents)]
        ts =0
        while True:
        # path_list, times_list = CBBA_solver.solve(AgentList, TaskList, WorldInfoTest, max_depth, time_window_flag=False, obsList=obsList)
        # for i in range(0,20):
            ts += 1

            offset = (CBBA_solver.WorldInfo.limit_x[1]-CBBA_solver.WorldInfo.limit_x[0]) / 100
            window.ax.cla()
            for m in range(CBBA_solver.num_tasks):
                # track task is red
                if CBBA_solver.TaskList[m].task_type == 0:
                    color_str = 'red'
                # rescue task is blue
                else:
                    color_str = 'blue'
                window.ax.scatter(CBBA_solver.TaskList[m].x, CBBA_solver.TaskList[m].y, marker='x', color=color_str)
                window.ax.text(CBBA_solver.TaskList[m].x+offset, CBBA_solver.TaskList[m].y+offset, "T"+str(m))

            # plot agents
            for n in range(CBBA_solver.num_agents):
            # for n in range(1):
                # quad  red
                if CBBA_solver.AgentList[n].agent_type == 0:
                    color_str = 'red'
                # car agent is blue
                else:
                    color_str = 'blue'
                window.ax.scatter(CBBA_solver.AgentList[n].x, CBBA_solver.AgentList[n].y, marker='o', color=color_str)
                window.ax.text(CBBA_solver.AgentList[n].x+offset, CBBA_solver.AgentList[n].y+offset, "A"+str(n))
                if CBBA_solver.path_list[n]:
                    Task_next, m = CBBA_solver.lookup_task(CBBA_solver.path_list[n][current_index[n]])
                    dir_x = (Task_next.x - AgentList[n].x)
                    dir_y = (Task_next.y - AgentList[n].y)
                    dir_theta = np.arctan2(dir_y,dir_x )
                    print(f"Agent {[n]}: ({AgentList[n].x}, {AgentList[n].y}) Task_next: {Task_next.task_id}, ({Task_next.x}, {Task_next.y}) Theta: {dir_theta*180/np.pi}")
                    AgentList[n].x += np.cos(dir_theta)*AgentList[n].nom_velocity*0.025
                    AgentList[n].y += np.sin(dir_theta)*AgentList[n].nom_velocity*0.025

                    if np.linalg.norm([AgentList[n].x-Task_next.x, AgentList[n].y-Task_next.y]) < 0.1:
                        AgentList[n].x = Task_next.x
                        AgentList[n].y = Task_next.y
                        # break
                        if current_index[n] < len(path_list[n])-1:
                            current_index[n]+=1
                        continue
                        # TaskList[CBBA_solver.path_list[n][0]]
                        # if CBBA_solver.path_list[n][m] > -1:
                        #     Task_next = CBBA_solver.lookup_task(CBBA_solver.path_list[n][m])
                        #     window.ax.plot([Task_prev.x, Task_next.x], [Task_prev.y, Task_next.y], linewidth=2, color=color_str)
                            #Task_prev = Task(**Task_next.__dict__)

                    Task_prev, ind = CBBA_solver.lookup_task(CBBA_solver.path_list[n][current_index[n]])
                    window.ax.plot([CBBA_solver.AgentList[n].x, Task_prev.x], [CBBA_solver.AgentList[n].y, Task_prev.y],
                            linewidth=2, color=color_str)
                    for m in range(1, len(CBBA_solver.path_list[n])):
                        if CBBA_solver.path_list[n][m] > -1:
                            Task_next, ind = CBBA_solver.lookup_task(CBBA_solver.path_list[n][m])
                            window.ax.plot([Task_prev.x, Task_next.x], [Task_prev.y, Task_next.y], linewidth=2, color=color_str)
                            Task_prev = Task(**Task_next.__dict__)
            window.ax.set_xlim([-2.0, 2.5])
            window.ax.set_ylim([-1.5, 5.5])
            window.fig.canvas.draw()
            window.mypause(0.001)
    else:
        path_list, times_list = CBBA_solver.solve(AgentList, TaskList, WorldInfoTest, max_depth, time_window_flag=True, obsList=obsList)

    t_end = time.time()
    t_used = t_end - t_start
    print("Time used [sec]: ", t_used)

    # the output is CBBA_solver.path_list or path_list
    print("bundle_list")
    print(CBBA_solver.bundle_list)
    print("path_list")
    print(path_list)
    print("times_list")
    print(times_list)
    print("winners_list")
    print(CBBA_solver.winners_list)
    print("scores_list")
    print(CBBA_solver.scores_list)

    print("bid_list")
    print(CBBA_solver.bid_list)
    print("winner_bid_list")
    print(CBBA_solver.winner_bid_list)

    # plot
    # CBBA_solver.plot_assignment()
    # plt.figure()
    CBBA_solver.plot_assignment_without_timewindow()
    for t in CBBA_solver.TaskList:
        print(t)
    plt.show()


from yw.env import point_reach

try:
    from yw.env.franka_env import panda_env
except:
    panda_env = None
try:
    import gym
except:
    gym = None

class EnvManager:
    def __init__(self, env_name, env_args={}, r_scale=1, r_shift=0.05, eps_length=0):
        self.make_env = None
        # Search from our own environments
        if env_name == "Reach2D":
            env_args["dim"] = 2
            env_args["sparse"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach2DDense":
            env_args["dim"] = 2
            env_args["sparse"] = False
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach2DF":
            env_args["dim"] = 2
            env_args["order"] = 1
            env_args["sparse"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach2DFDense":
            env_args["dim"] = 2
            env_args["order"] = 1
            env_args["sparse"] = False
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach1D":
            env_args["dim"] = 1
            env_args["sparse"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach1DDense":
            env_args["dim"] = 1
            env_args["sparse"] = False
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach1DF":
            env_args["dim"] = 1
            env_args["order"] = 1
            env_args["sparse"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "Reach1DFDense":
            env_args["dim"] = 1
            env_args["order"] = 1
            env_args["sparse"] = False
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "BlockReachF":
            env_args["sparse"] = True
            env_args["order"] = 1
            env_args["block"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)
        elif env_name == "BlockReachFDense":
            env_args["sparse"] = False
            env_args["order"] = 1
            env_args["block"] = True
            self.make_env = lambda: point_reach.make("Reacher", **env_args)

        # Franka environment
        if self.make_env is None and panda_env is not None:
            # TODO add a make function
            self.make_env = panda_env.FrankaPegInHole

        # Search from openai gym
        if self.make_env is None and gym is not None:
            try:
                _ = gym.make(env_name, **env_args)
                self.make_env = lambda: gym.make(env_name, **env_args)
            except:
                pass

        if self.make_env is None:
            raise NotImplementedError

        # Add extra properties on the environment.
        self.r_scale = r_scale
        self.r_shift = r_shift
        self.eps_length = eps_length

    def get_env(self):
        return EnvManager.EnvWrapper(self.make_env, self.r_scale, self.r_shift, self.eps_length)

    class EnvWrapper:
        def __init__(self, make_env, r_scale, r_shift, eps_length):
            self.env = make_env()
            self.r_scale = r_scale
            self.r_shift = r_shift
            self.eps_length = eps_length
            # need the following properties
            self._max_episode_steps = self.eps_length if self.eps_length else self.env._max_episode_steps
            self.max_u = self.env.max_u if hasattr(self.env, "max_u") else 1  # note that 1 is just for most envs
            self.action_space = self.env.action_space

        def compute_reward(self, achieved_goal, desired_goal, info):
            reward = self.env.compute_reward(achieved_goal=achieved_goal, desired_goal=desired_goal, info=info)
            reward = (reward + self.r_shift) / self.r_scale
            return reward

        def reset(self, **kwargs):
            return self.env.reset(**kwargs)

        def render(self):
            return self.env.render()

        def seed(self, seed=0):
            return self.env.seed(seed)

        def step(self, action):
            state, r, extra, info = self.env.step(action)
            r = (r + self.r_shift) / self.r_scale
            return state, r, extra, info

        def close(self):
            return self.env.close()

if __name__ == "__main__":
    import numpy as np

    # For a robosuite env
    # env_manager = EnvManager(
    #     "SawyerLift", env_args={"has_renderer": True, "ignore_done": True, "use_camera_obs": False, "control_freq": 50}
    # )
    # env = env_manager.get_env()
    # env.reset()
    # for i in range(100):
    #     action = np.random.randn(env.action_space.shape[0])  # sample random action
    #     state, r, extra, info = env.step(action)
    #     env.render()

    # For a openai env
    env_manager = EnvManager("FetchReach-v1")
    env = env_manager.get_env()

    env.seed(0)
    for i in range(1000):
        env.reset()
        # action = np.random.randn(env.action_space.shape[0])  # sample random action
        # state, r, extra, info = env.step(action)
        input("Press Enter to continue...")
        # print(state, r, extra, info)
        env.render()


import asyncio

from .common import is_ami_action, AMICommandFailure
from .protocol import AMIProtocol

def connect(host, port=5038, username='', secret='', plaintext_login=False, loop=None):
    conn = AMIConnection(
        host=host,
        port=port,
        username=username,
        secret=secret,
        plaintext_login=plaintext_login,
        loop=loop)
    yield from conn.connect()
    return conn

class AMIConnection():
    """Asterisk AMI connection representation. Wraps protocol's actions."""
    def __init__(self, host, port=5038, username='', secret='', plaintext_login=False, loop=None):
        self.host = host
        self.port = port
        self.username = username
        self.secret = secret
        self.plaintext_login = plaintext_login

        self.closed = True

        self.loop = loop or asyncio.get_event_loop()
        self.protocol = AMIProtocol(loop=self.loop)

    # mirror ami protocol actions
    def __getattr__(self, item):
        if hasattr(self.protocol, item):
            attr = getattr(self.protocol, item)
            if is_ami_action(attr):
                return attr
        return object.__getattribute__(self, item)

    def connect(self):
        yield from self.loop.create_connection(lambda: self.protocol, host=self.host, port=self.port)
        yield from self.protocol.login(self.username, self.secret, self.plaintext_login)
        self.closed = False

    def close(self):
        self.closed = True
        self.protocol.close()

    def ping(self, timeout=3.0, reconnect=True):
        try:
            yield from asyncio.wait_for(self.protocol.ping(), timeout, loop=self.loop)
        except asyncio.TimeoutError or AMICommandFailure:
            self.closed = True
            if reconnect:
                yield from self.connect()
            else:
                raise

    def add_handler(self, event, callback):
        self.protocol.on(event, callback)


import os
from functools import partial
import numpy as np
import pandas as pd
import seaborn as sns
import torch
import torch.nn as nn

import matplotlib.pyplot as plt

import pyro
from pyro.distributions import Normal, Uniform, Delta
from pyro.infer import SVI, Trace_ELBO
from pyro.optim import Adam
from pyro.distributions.util import logsumexp
from pyro.infer import EmpiricalMarginal, SVI, Trace_ELBO, TracePredictive
from pyro.infer.mcmc import MCMC, NUTS
import pyro.optim as optim
import pyro.poutine as poutine

# for CI testing
smoke_test = ('CI' in os.environ)
assert pyro.__version__.startswith('0.4.1')
pyro.enable_validation(True)
pyro.set_rng_seed(1)
pyro.enable_validation(True)

DATA_URL = "C:/Users/PrestonPhillips/Desktop/mapinv_reference_data_carbonates_calculatedMode.csv"
data = pd.read_csv(DATA_URL, encoding="ISO-8859-1")

df = data[["POROSITY", "ROCK_INDEX","Mode","PERMEABILITY","DATA_SOURCE"]]
df = df[df["DATA_SOURCE"]=="Rosetta"]
df = data[["POROSITY", "ROCK_INDEX","Mode","PERMEABILITY"]]

df["PERMEABILITY"]= np.log(df["PERMEABILITY"])

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 6), sharey=True)
rock_1_data = data[data["ROCK_INDEX"] == 1]
rock_4_data = data[data["ROCK_INDEX"] == 4]
sns.scatterplot(rock_4_data["POROSITY"],
            rock_4_data["PERMEABILITY"],
            ax=ax[0])
ax[0].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock1")
sns.scatterplot(rock_1_data["POROSITY"],
            rock_1_data["PERMEABILITY"],
            ax=ax[1])
ax[1].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock4")

plt.show()

class RegressionModel(nn.Module):
    def __init__(self, p):
        # p = number of features
        super(RegressionModel, self).__init__()
        self.linear = nn.Linear(p, 1)
        self.factor = nn.Parameter(torch.tensor(1.))

    def forward(self, x):
        return self.linear(x) + (self.factor * x[:, 0] * x[:, 1] * x[:,2]).unsqueeze(1)

p = 3  # number of features
regression_model = RegressionModel(p)

loss_fn = torch.nn.MSELoss(reduction='sum')
optim = torch.optim.Adam(regression_model.parameters(), lr=0.05)
num_iterations = 5000 if not smoke_test else 2
data = torch.tensor(df.values, dtype=torch.float)
x_data, y_data = data[:, :-1], data[:, -1]

def model(x_data, y_data):
    # weight and bias priors
    w_prior = Normal(torch.zeros(1, 3), torch.ones(1, 3)).to_event(1)
    b_prior = Normal(torch.tensor([[8.]]), torch.tensor([[1000.]])).to_event(1)
    f_prior = Normal(0., 1.)
    priors = {'linear.weight': w_prior, 'linear.bias': b_prior, 'factor': f_prior}
    scale = pyro.sample("sigma", Uniform(0., 10.))
    # lift module parameters to random variables sampled from the priors
    lifted_module = pyro.random_module("module", regression_model, priors)
    # sample a nn (which also samples w and b)
    lifted_reg_model = lifted_module()
    with pyro.plate("map", len(x_data)):
        # run the nn forward on data
        prediction_mean = lifted_reg_model(x_data).squeeze(-1)
        # condition on the observed data
        pyro.sample("obs",
                    Normal(prediction_mean, scale),
                    obs=y_data)
        return prediction_mean

from pyro.infer.autoguide import AutoDiagonalNormal
from pyro.infer.autoguide import init_to_feasible
guide = AutoDiagonalNormal(model, init_loc_fn=init_to_feasible)

optim = Adam({"lr": 0.02})
svi = SVI(model, guide, optim, loss=Trace_ELBO(), num_samples=1000)

def train():
    pyro.clear_param_store()
    for j in range(num_iterations):
        # calculate the loss and take a gradient step
        loss = svi.step(x_data, y_data)
        if j % 100 == 0:
            print("[iteration %04d] loss: %.4f" % (j + 1, loss / len(data)))

train()

get_marginal = lambda traces, sites:EmpiricalMarginal(traces, sites)._get_samples_and_weights()[0].detach().cpu().numpy()

def summary(traces, sites):
    marginal = get_marginal(traces, sites)
    site_stats = {}
    for i in range(marginal.shape[1]):
        site_name = sites[i]
        marginal_site = pd.DataFrame(marginal[:, i]).transpose()
        describe = partial(pd.Series.describe, percentiles=[.05, 0.25, 0.5, 0.75, 0.95])
        site_stats[site_name] = marginal_site.apply(describe, axis=1) \
            [["mean", "std", "5%", "25%", "50%", "75%", "95%"]]
    return site_stats

def wrapped_model(x_data, y_data):
    pyro.sample("prediction", Delta(model(x_data, y_data)))

posterior = svi.run(x_data, y_data)

# posterior predictive distribution we can get samples from
trace_pred = TracePredictive(wrapped_model,
                             posterior,
                             num_samples=1000)
post_pred = trace_pred.run(x_data, None)
post_summary = summary(post_pred, sites= ['prediction', 'obs'])
mu = post_summary["prediction"]
y = post_summary["obs"]
predictions = pd.DataFrame({
    "POROSITY": x_data[:, 0],
    "ROCK_INDEX": x_data[:, 1],
    "MODE": x_data[:,2],
    "mu_mean": mu["mean"],
    "mu_perc_5": mu["5%"],
    "mu_perc_95": mu["95%"],
    "y_mean": y["mean"],
    "y_perc_5": y["5%"],
    "y_perc_95": y["95%"],
    "true_Perm": y_data,
})

fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(12, 6), sharey=True)
rock_1_data = predictions[predictions["ROCK_INDEX"] ==1]
rock_2_data = predictions[predictions["ROCK_INDEX"] ==2]
rock_3_data = predictions[predictions["ROCK_INDEX"] ==3]
rock_4_data = predictions[predictions["ROCK_INDEX"] == 4]
rock_1_data = rock_1_data.sort_values(by=["POROSITY"])
rock_2_data = rock_2_data.sort_values(by=["POROSITY"])
rock_3_data = rock_3_data.sort_values(by=["POROSITY"])
rock_4_data = rock_4_data.sort_values(by=["POROSITY"])
fig.suptitle("Regression line 90% CI", fontsize=16)
idx = np.argsort(rock_1_data["POROSITY"])
ax[0,0].plot(rock_1_data["POROSITY"],
           rock_1_data["mu_mean"])
ax[0,0].fill_between(rock_1_data["POROSITY"],
                   rock_1_data["mu_perc_5"],
                   rock_1_data["mu_perc_95"],
                   alpha=0.5)
ax[0,0].plot(rock_1_data["POROSITY"],
           rock_1_data["true_Perm"],
           "o")
ax[0,0].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock Type 1")
idx = np.argsort(rock_2_data["POROSITY"])
ax[0,1].plot(rock_2_data["POROSITY"],
           rock_2_data["mu_mean"])
ax[0,1].fill_between(rock_2_data["POROSITY"],
                   rock_2_data["mu_perc_5"],
                   rock_2_data["mu_perc_95"],
                   alpha=0.5)
ax[0,1].plot(rock_2_data["POROSITY"],
           rock_2_data["true_Perm"],
           "o")
ax[0,1].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock_Type 2")
idx = np.argsort(rock_3_data["POROSITY"])
ax[1,0].plot(rock_3_data["POROSITY"],
           rock_3_data["mu_mean"])
ax[1,0].fill_between(rock_3_data["POROSITY"],
                   rock_3_data["mu_perc_5"],
                   rock_3_data["mu_perc_95"],
                   alpha=0.5)
ax[1,0].plot(rock_3_data["POROSITY"],
           rock_3_data["true_Perm"],
           "o")
ax[1,0].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock_Type 3")
idx = np.argsort(rock_4_data["POROSITY"])
ax[1,1].plot(rock_4_data["POROSITY"],
           rock_4_data["mu_mean"])
ax[1,1].fill_between(rock_4_data["POROSITY"],
                   rock_4_data["mu_perc_5"],
                   rock_4_data["mu_perc_95"],
                   alpha=0.5)
ax[1,1].plot(rock_4_data["POROSITY"],
           rock_4_data["true_Perm"],
           "o")
ax[1,1].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock_Type 4")

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 6), sharey=True)
fig.suptitle("Posterior predictive distribution with 90% CI", fontsize=16)
ax[0].plot(rock_4_data["POROSITY"],
           rock_4_data["y_mean"])
ax[0].fill_between(rock_4_data["POROSITY"],
                   rock_4_data["y_perc_5"],
                   rock_4_data["y_perc_95"],
                   alpha=0.5)
ax[0].plot(rock_4_data["POROSITY"],
           rock_4_data["true_Perm"],
           "o")
ax[0].set(xlabel="Pososity",
          ylabel="log Perm",
          title="Rock1")
idx = np.argsort(rock_1_data["POROSITY"])

ax[1].plot(rock_1_data["POROSITY"],
           rock_1_data["y_mean"])
ax[1].fill_between(rock_1_data["POROSITY"],
                   rock_1_data["y_perc_5"],
                   rock_1_data["y_perc_95"],
                   alpha=0.5)
ax[1].plot(rock_1_data["POROSITY"],
           rock_1_data["true_Perm"],
           "o")
ax[1].set(xlabel="POROSITY",
          ylabel="log Perm",
          title="Rock4")

# we need to prepend `module$$$` to all parameters of nn.Modules since
# that is how they are stored in the ParamStore
weight = get_marginal(posterior, ['module$$$linear.weight']).squeeze(1).squeeze(1)
factor = get_marginal(posterior, ['module$$$factor'])
gamma_within_africa = weight[:, 1] + factor.squeeze(1)
gamma_outside_africa = weight[:, 1]
fig = plt.figure(figsize=(10, 6))
sns.distplot(gamma_within_africa, kde_kws={"label": "Rock4"},)
sns.distplot(gamma_outside_africa, kde_kws={"label": "Rock1"})
fig.suptitle("Density of Slope : log(Perm) vs. POROSITY", fontsize=16)
plt.show()

from mpl_toolkits import mplot3d

import numpy as np
import matplotlib.pyplot as plt
fig = plt.figure()
from mpl_toolkits import mplot3d
ax = plt.axes(projection='3d')
print(predictions.columns)

ax.scatter3D(rock_1_data["POROSITY"], rock_1_data["MODE"], rock_1_data["y_mean"], c=rock_1_data["y_mean"], cmap='Greens');

plt.show()


import scipy
from glob import glob
import numpy as np
from random import shuffle

class DataLoader():
    def __init__(self, img_res=(128, 128)):
        self.dataset_e2s = 'edges2shoes'
        self.dataset_e2h = 'edges2handbags'
        self.img_res = img_res

    def load_data(self, batch_size=1, is_testing=False):
        data_type = "train" if not is_testing else "val"
        path_e2s = glob('./datasets/%s/%s/*' % (self.dataset_e2s, data_type))
        path_e2h = glob('./datasets/%s/%s/*' % (self.dataset_e2h, data_type))

        batch_e2s = np.random.choice(path_e2s, size=batch_size)
        batch_e2h = np.random.choice(path_e2h, size=batch_size)

        imgs_s, imgs_h = [], []
        for img_e2s, img_e2h in zip(batch_e2s, batch_e2h):
            img_s = self.imread(img_e2s)
            img_h = self.imread(img_e2h)

            img_s = scipy.misc.imresize(img_s, self.img_res)
            img_h = scipy.misc.imresize(img_h, self.img_res)

            if not is_testing and np.random.random() > 0.5:
                    img_s = np.fliplr(img_s)
                    img_h = np.fliplr(img_h)

            imgs_s.append(img_s)
            imgs_h.append(img_h)

        imgs_s = np.array(imgs_s)/127.5 - 1.
        imgs_h = np.array(imgs_h)/127.5 - 1.

        return imgs_s, imgs_h

    def load_batch(self, batch_size=1, is_testing=False):
        data_type = "train" if not is_testing else "val"
        path_e2s = glob('./datasets/%s/%s/*' % (self.dataset_e2s, data_type))
        path_e2h = glob('./datasets/%s/%s/*' % (self.dataset_e2h, data_type))

        self.n_batches_s = int(len(path_e2s) / batch_size)
        self.n_batches_h = int(len(path_e2h) / batch_size)
        self.n_batches = min(self.n_batches_s, self.n_batches_h)

        shuffle(path_e2h)
        shuffle(path_e2s)

        for i in range(self.n_batches-1):
            batch_h = path_e2h[i*batch_size:(i+1)*batch_size]
            batch_s = path_e2s[i*batch_size:(i+1)*batch_size]
            imgs_s, imgs_h = [], []

            for img in batch_h:
                img_h = self.imread(img)
                img_h = scipy.misc.imresize(img_h, self.img_res)

                if not is_testing and np.random.random() > 0.5:
                    img_h = np.fliplr(img_h)

                imgs_h.append(img_h)

            for img in batch_s:
                img_s = self.imread(img)
                img_s = scipy.misc.imresize(img_s, self.img_res)

                if not is_testing and np.random.random() > 0.5:
                    img_s = np.fliplr(img_s)

                imgs_s.append(img_s)

            imgs_h = np.array(imgs_h)/127.5 - 1.
            imgs_s = np.array(imgs_s)/127.5 - 1.

            yield imgs_s, imgs_h

    def load_img(self, path):
        img = self.imread(path)
        img = scipy.misc.imresize(img, self.img_res)
        img = img/127.5 - 1.
        return img[np.newaxis, :, :, :]

    def imread(self, path):
        return scipy.misc.imread(path, mode='RGB').astype(np.float)


"""
This class implement a Conditional RBM, a temporal generative model
To understand this model : Taylor http://www.uoguelph.ca/~gwtaylor/publications/nips2006mhmublv/
Taylor Github : https://gist.github.com/gwtaylor/2505670 (CRBM theano implementation without modification)
@author Graham W. Taylor

@modification Francois Lasson : CERV Brest France
@mail : francois.lasson@gmail.com
"""

#Numpy and Theano library
import numpy as np
import theano
import theano.tensor as T
#theano Random
from theano.sandbox.rng_mrg import MRG_RandomStreams

#Use to compute learning time
import timeit

#This class used to generate dataset readable by RBMs from BVH files
from motion import generate_dataset #to load mocap

class CRBM(object):
    """Conditional Restricted Boltzmann Machine (CRBM)  """
    def __init__(self, input=None, input_history=None,
                n_visible=None, n_hidden=None, delay=None, freq=None,forward=None,
                A=None, B=None, W=None, hbias=None, vbias=None,
                numpy_rng=None, theano_rng=None):
        """
        CRBM constructor. Defines the parameters of the model along with
        basic operations for inferring hidden from visible (and vice-versa),
        as well as for performing CD updates.

        :param input: None for standalone RBMs or symbolic variable if RBM is
        part of a larger graph.

        :param n_visible: number of visible units

        :param n_hidden: number of hidden units

        :param A: None for standalone CRBMs or symbolic variable pointing to a
        shared weight matrix in case CRBM is part of a CDBN network; in a CDBN,
        the weights are shared between CRBMs and layers of a MLP

        :param B: None for standalone CRBMs or symbolic variable pointing to a
        shared weight matrix in case CRBM is part of a CDBN network; in a CDBN,
        the weights are shared between CRBMs and layers of a MLP

        :param W: None for standalone CRBMs or symbolic variable pointing to a
        shared weight matrix in case CRBM is part of a CDBN network; in a CDBN,
        the weights are shared between CRBMs and layers of a MLP

        :param hbias: None for standalone CRBMs or symbolic variable pointing
        to a shared hidden units bias vector in case CRBM is part of a
        different network

        :param vbias: None for standalone RBMs or a symbolic variable
        pointing to a shared visible units bias
        """

        #Initialize architecture (number of layers, of neurones)
        self.n_visible = n_visible #number of neurones in visible layer
        self.n_hidden = n_hidden #number of neurones in hidden layer
        self.delay = delay #number of past visible layers
        self.freq = freq #frequence of past visible layers actualization
        self.forward = forward # number of forward visible units

        #Initialize seed for random
        if numpy_rng is None:
            # create a number generator
            numpy_rng = np.random.RandomState(1234)
        if theano_rng is None:
            theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30))

        #Initialize weight matrices and biases

        if W is None:
            # the output of uniform if converted using asarray to dtype
            # theano.config.floatX so that the code is runable on GPU
            initial_W = np.asarray(0.01 * numpy_rng.randn(n_visible*forward,
                                                          n_hidden),
                                    dtype=theano.config.floatX)
            # theano shared variables for weights and biases
            W = theano.shared(value=initial_W, name='W')

        if A is None:
            initial_A = np.asarray(0.01 * numpy_rng.randn(n_visible * delay,
                                                           n_visible*forward),
                                    dtype=theano.config.floatX)
            # theano shared variables for weights and biases
            A = theano.shared(value=initial_A, name='A')
        if B is None:
            initial_B = np.asarray(0.01 * numpy_rng.randn(n_visible * delay,
                                                           n_hidden),
                                    dtype=theano.config.floatX)
            # theano shared variables for weights and biases
            B = theano.shared(value=initial_B, name='B')
        if hbias is None:
            # create shared variable for hidden units bias
            hbias = theano.shared(value=np.zeros(n_hidden,
                                dtype=theano.config.floatX), name='hbias')
        if vbias is None:
            # create shared variable for visible units bias
            vbias = theano.shared(value=np.zeros(n_visible*forward,
                                dtype=theano.config.floatX), name='vbias')

        # initialize input layer for standalone CRBM or layer0 of CDBN
        self.input = input
        if not input:
            self.input = T.matrix('input')
        self.input_history = input_history
        if not input_history:
            self.input_history = T.matrix('input_history')

        self.W = W
        self.A = A
        self.B = B
        self.hbias = hbias
        self.vbias = vbias
        self.theano_rng = theano_rng
        self.numpy_rng = numpy_rng
        self.params = [self.W, self.A, self.B, self.hbias, self.vbias]

    def free_energy(self, v_sample, v_history):
    #def free_energy(self, v_sample):
        ''' Function to compute the free energy of a sample conditional
        on the history '''
        wx_b = T.dot(v_sample, self.W) + T.dot(v_history, self.B) + self.hbias
        ax_b = T.dot(v_history, self.A) + self.vbias
        #wx_b = T.dot(v_sample, self.W)  + self.hbias
        #ax_b = self.vbias
        visible_term = T.sum(0.5 * T.sqr(v_sample - ax_b), axis=1)
        hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1)
        return visible_term - hidden_term #TODO Useless, this Energy Function allow net to work with continuous value, however we use binary value

    def propup(self, vis, v_history):
        ''' This function propagates the visible units activation upwards to
        the hidden units

        Note that we return also the pre-sigmoid activation of the layer. As
        it will turn out later, due to how Theano deals with optimizations,
        this symbolic variable will be needed to write down a more
        stable computational graph (see details in the reconstruction cost
        function)
        '''
        pre_sigmoid_activation = T.dot(vis, self.W) + T.dot(v_history, self.B) + self.hbias
        #pre_sigmoid_activation = T.dot(vis, self.W) + self.hbias
        return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]

    def sample_h_given_v(self, v0_sample, v_history):
        ''' This function infers state of hidden units given visible units '''
        # compute the activation of the hidden units given a sample of the
        # visibles
        #pre_sigmoid_h1, h1_mean = self.propup(v0_sample)
        pre_sigmoid_h1, h1_mean = self.propup(v0_sample, v_history)
        # get a sample of the hiddens given their activation
        # Note that theano_rng.binomial returns a symbolic sample of dtype
        # int64 by default. If we want to keep our computations in floatX
        # for the GPU we need to specify to return the dtype floatX
        h1_sample = self.theano_rng.binomial(size=h1_mean.shape, n=1,
                                             p=h1_mean,
                                             dtype=theano.config.floatX)
        #h1_sample = self.theano_rng.normal(size=h1_mean.shape,avg=h1_mean,std=0.01,dtype=theano.config.floatX)
        return [pre_sigmoid_h1, h1_mean, h1_sample]

    def propdown(self, hid, v_history):
        '''This function propagates the hidden units activation downwards to
        the visible units

        Note that we return also the pre_sigmoid_activation of the layer. As
        it will turn out later, due to how Theano deals with optimizations,
        this symbolic variable will be needed to write down a more
        stable computational graph (see details in the reconstruction cost
        function)
        '''
        pre_sigmoid_activation = T.dot(hid, self.W.T) + T.dot(v_history, self.A) + self.vbias
        return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]

    def sample_v_given_h(self, h0_sample, v_history):
        ''' This function infers state of visible units given hidden units '''
        # compute the activation of the visible given the hidden sample
        #pre_sigmoid_v1, v1_mean = self.propdown(h0_sample)
        pre_sigmoid_v1, v1_mean = self.propdown(h0_sample, v_history)
        # get a sample of the visible given their activation
        # Note that theano_rng.binomial returns a symbolic sample of dtype
        # int64 by default. If we want to keep our computations in floatX
        # for the GPU we need to specify to return the dtype floatX
        #v1_sample = self.theano_rng.binomial(size=v1_mean.shape,
        #                                   n=1, p=v1_mean,
         #                                   dtype = theano.config.floatX)
        v1_sample = self.theano_rng.normal(size=v1_mean.shape, avg=v1_mean, std=0.01, dtype=theano.config.floatX)
        return [pre_sigmoid_v1, v1_mean, v1_sample]

    def gibbs_hvh(self, h0_sample, v_history):
        ''' This function implements one step of Gibbs sampling,
            starting from the hidden state'''
        pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h0_sample, v_history)
        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample, v_history)
        return [pre_sigmoid_v1, v1_mean, v1_sample, pre_sigmoid_h1, h1_mean, h1_sample]

    def gibbs_vhv(self, v0_sample, v_history):
        ''' This function implements one step of Gibbs sampling,
            starting from the visible state'''
        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v0_sample, v_history)
        pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h1_sample, v_history)
        return [pre_sigmoid_h1, h1_mean, h1_sample, pre_sigmoid_v1, v1_mean, v1_sample]

    def get_cost_updates(self, lr=0.1, k=1):
        """
        This functions implements one step of CD-k
        :param lr: learning rate used to train the RBM
        :param persistent: None for CD
        :param k: number of Gibbs steps to do in CD-k
        Returns a proxy for the cost and the updates dictionary. The
        dictionary contains the update rules for weights and biases but
        also an update of the shared variable used to store the persistent
        chain, if one is used.
        """
        # compute positive phase
        pre_sigmoid_ph, ph_mean, ph_sample = self.sample_h_given_v(self.input, self.input_history)

        # for CD, we use the newly generate hidden sample
        chain_start = ph_sample

        # perform actual negative phase
        # in order to implement CD-k we need to scan over the
        # function that implements one gibbs step k times.
        # Read Theano tutorial on scan for more information :
        # http://deeplearning.net/software/theano/library/scan.html
        # the scan will return the entire Gibbs chain
        # updates dictionary is important because it contains the updates
        # for the random number generator
        [pre_sigmoid_nvs, nv_means, nv_samples, pre_sigmoid_nhs, nh_means,
         nh_samples], updates = theano.scan(self.gibbs_hvh,
                    # the None are place holders, saying that
                    # chain_start is the initial state corresponding to the
                    # 5th output
                    outputs_info=[None, None, None, None, None, chain_start],
                    non_sequences=self.input_history,
                    n_steps=k)

        # determine gradients on CRBM parameters
        # not that we only need the sample at the end of the chain
        chain_end = nv_samples[-1]

        cost = T.mean(self.free_energy(self.input, self.input_history)) - T.mean(self.free_energy(chain_end, self.input_history))
        # We must not compute the gradient through the gibbs sampling
        gparams = T.grad(cost, self.params, consider_constant=[chain_end]) #consider_constant=[chain_end]

        # constructs the update dictionary
        for gparam, param in zip(gparams, self.params):
            # make sure that the learning rate is of the right dtype
            if param == self.A:
                # slow down autoregressive updates
                updates[param] = param - gparam * 0.01 * \
                                 T.cast(lr, dtype=theano.config.floatX)
            else:
                updates[param] = param - gparam * \
                                 T.cast(lr, dtype=theano.config.floatX)

        # reconstruction error is a better proxy for CD
        monitoring_cost = self.get_reconstruction_cost(updates, nv_means[-1])
        return monitoring_cost, updates

    def get_reconstruction_cost(self, updates, pre_sigmoid_nv):
        """Approximation to the reconstruction error
        """
        # sum over dimensions, mean over cases
        recon = T.mean(T.sum(T.sqr(self.input - pre_sigmoid_nv), axis=1))
        return recon

    def train(self, learning_rate=None, training_epochs=None,
                 dataset=None, seqlen= None, batch_size=None, full_seq=None):
        # compute number of minibatches for training, validation and testing
        #for each file, n_delay frames are used to initialized memory. So, these (delay*freq) frames not have to be count for frame minibatch division
        n_train_batches = (dataset.get_value(borrow=True).shape[0]-((self.delay+self.forward)*self.freq)*len(seqlen)) // batch_size
        #n_train_batches = (dataset.shape[0] - (self.delay * self.freq) * len(seqlen)) / batch_size
        # valid starting indices
        datasetindex = []
        last = 0
        if full_seq == True:
            for s in seqlen:
                datasetindex += range(last + (self.delay*self.freq), last + s-self.forward)
                last += s
            permindex = np.array(datasetindex)
        else:
            n_train_batches = int(np.sum(seqlen)/(self.delay+1)/batch_size)
            for s in seqlen:
                datasetindex += range(last + (self.delay * self.freq), last + s-self.forward,(self.delay * self.freq+1))
                last += s
            permindex = np.array(datasetindex)
        self.numpy_rng.shuffle(permindex)

        # allocate symbolic variables for the data
        index = T.lvector()    # index to a [mini]batch
        index_hist = T.lvector()  # index to history

        # get the cost and the gradient corresponding to one step of CD-15
        cost, updates = self.get_cost_updates(lr=learning_rate, k=5)

        #################################
        #     Training the CRBM         #
        #################################
        # the purpose of train_crbm is solely to update the CRBM parameters
        train_crbm = theano.function(
                [index, index_hist],
                cost,
                updates=updates,
                givens={self.input: dataset[index].reshape((batch_size,self.forward*self.n_visible)),
                        self.input_history: dataset[index_hist].reshape((batch_size, self.delay * self.n_visible))
                       },
                name='train_crbm')

        start_time = timeit.default_timer()
        cost_y = []

        # go through training epochs
        for epoch in range(training_epochs):

            # go through the training set
            mean_cost = []
            for batch_index in range(n_train_batches):

                # indexing is slightly complicated
                # build a linear index to the starting frames for this batch
                # (i.e. time t) gives a batch_size length array for data
                data_idx = permindex[batch_index * batch_size:(batch_index + 1) * batch_size]

                # now build a linear index to the frames at each delay tap
                # (i.e. time t-1 to t-delay)
                # gives a batch_size x delay array of indices for history
                #have also to include frequence of past visible layers actualization
                visi_idx = np.array([data_idx+(n*self.freq) for n in range(0, self.forward)]).T
                hist_idx = np.array([data_idx-(n*self.freq) for n in range(1, self.delay + 1)]).T
                this_cost = train_crbm(visi_idx.ravel(), hist_idx.ravel())
                #print batch_index, this_cost
                mean_cost += [this_cost]
            print('Training epoch %d, cost is ' % epoch, np.mean(mean_cost))
            cost_y.append(np.mean(mean_cost))
        end_time = timeit.default_timer()
        pretraining_time = (end_time - start_time)
        print ('Training took %f minutes' % (pretraining_time / 60.))
        return cost_y

    """
    def predict_hidden(self, dataset=None, batch_size=100):
        # compute number of minibatches for training, validation and testing
        n_test_batches = dataset.get_value(borrow=True).shape[0] / batch_size
        n_dim =  dataset.get_value(borrow=True).shape[1]

        # allocate symbolic variables for the data
        index = T.lvector()    # index to a [mini]batch
        index_hist = T.lvector()  # index to history

        #print hidden layer
        [pre_sigmoid_h1, h1_mean, h1_sample] = self.sample_h_given_v(self.input, self.input_history)

        print_hidden = theano.function(
            [index, index_hist],
            [h1_sample],
            givens={ self.input:dataset[index],
                     self.input_history:dataset[index_hist].reshape((batch_size, self.delay * self.n_visible))
                    },
            name='print_hidden'
            )

        # valid starting indices
        datasetindex = range(self.delay, dataset.get_value(borrow=True).shape[0])
        permindex = np.array(datasetindex)

        #For each frame in minibatch
        for batch_index in xrange(n_test_batches):
            data_idx = permindex[batch_index * batch_size : (batch_index + 1) * batch_size]
            hist_idx = np.array([data_idx - n for n in xrange(1, self.delay + 1)]).T
            for index_in_batch in range(batch_size) :
                print "Hidden CRBM (frame %d):" %(batch_index*batch_size+index_in_batch+1)
                print print_hidden(data_idx, hist_idx.ravel())[0][index_in_batch]
                print "-----------"
    """

def create_train_crbm(learning_rate=3e-3, training_epochs=1000,
             dataset=None, seqlen= None, batch_size=10,full_seq =False,forward=3,
             n_hidden=50, delay=6, freq=1):
    """
    Demonstrate how to train a CRBM.
    This is demonstrated on mocap data.
    :param learning_rate: learning rate used for training the CRBM
    :param training_epochs: number of epochs used for training
    :param dataset: path the the dataset (matlab format)
    :param batch_size: size of a batch used to train the RBM
    """

    n_dim =  dataset.get_value(borrow=True).shape[1]

    x = T.matrix('x')  # the data
    x_history = T.matrix('x_history')

    # construct the CRBM class
    crbm = CRBM(input=x, input_history=x_history, n_visible=n_dim, n_hidden=n_hidden, delay=delay, freq=freq,forward=forward)

    crbm.train(learning_rate=learning_rate,full_seq=full_seq,training_epochs=training_epochs, dataset=dataset, seqlen= seqlen, batch_size=batch_size)
    return crbm

if __name__ == '__main__':
    crbm, dataset = create_train_crbm(n_hidden=10)


from flask import Blueprint
from flask import render_template, redirect, flash, request, jsonify
from app.forms import LoginForm

routes_main = Blueprint('routes_main', __name__)

@routes_main.route('/')
def index():
    return render_template('landing/index.html', title='blog')

@routes_main.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm()

    if form.validate_on_submit():
        flash('Logging user {}'.format(form.username.data))
        return redirect('/')

    return render_template('landing/login.html', title='Sign In', form=form)

@routes_main.route('/ident')
def ident_request():
    headers = {k: v for k, v in request.headers}
    headers.update({
        'remote_addr': request.remote_addr
    })
    return jsonify(headers)


#Tony Johnson
#Requires 3 inputs: potentially contaminated fastq reads, host reference, false positive probability for bloom filter as a float x: 0 < x < 1
#Compares fastq reads to fasta reference and
#Outputs line by line reads in the fastq file that do not match with host reference reads into a CRs.txt file but has a false positive rate
#defined by user
import math
import mmh3
import sys
from bitarray import bitarray

#BloomFilter class from geeksforgeeks.org bloom filter introduction: https://www.geeksforgeeks.org/bloom-filters-introduction-and-python-implementation/
class BloomFilter(object):
    '''
    Class for Bloom filter, using murmur3 hash function
    '''

    def __init__(self, items_count, fp_prob):
        '''
        items_count : int
            Number of items expected to be stored in bloom filter
        fp_prob : float
            False Positive probability in decimal
        '''
        # False posible probability in decimal
        self.fp_prob = fp_prob

        # Size of bit array to use
        self.size = self.get_size(items_count, fp_prob)

        # number of hash functions to use
        self.hash_count = self.get_hash_count(self.size, items_count)

        # Bit array of given size
        self.bit_array = bitarray(self.size)

        # initialize all bits as 0
        self.bit_array.setall(0)

    def add(self, item):
        '''
        Add an item in the filter
        '''
        digests = []
        for i in range(self.hash_count):
            # create digest for given item.
            # i work as seed to mmh3.hash() function
            # With different seed, digest created is different
            digest = mmh3.hash(item, i) % self.size
            digests.append(digest)

            # set the bit True in bit_array
            self.bit_array[digest] = True

    def check(self, item):
        '''
        Check for existence of an item in filter
        '''
        for i in range(self.hash_count):
            digest = mmh3.hash(item, i) % self.size
            if self.bit_array[digest] == False:
                # if any of bit is False then,its not present
                # in filter
                # else there is probability that it exist
                return False
        return True

    @classmethod
    def get_size(self, n, p):
        '''
        Return the size of bit array(m) to be used using
        following formula
        m = -(n * lg(p)) / (lg(2)^2)
        n : int
            number of items expected to be stored in filter
        p : float
            False Positive probability in decimal
        '''
        m = -(n * math.log(p)) / (math.log(2) ** 2)
        return int(m)

    @classmethod
    def get_hash_count(self, m, n):
        '''
        Return the hash function(k) to be used using
        following formula
        k = (m/n) * lg(2)

        m : int
            size of bit array
        n : int
            number of items expected to be stored in filter
        '''
        k = (m / n) * math.log(2)
        return int(k)

def parse_fastq(host_and_contaminant_fastq_file):
#Parse FASTQ into a dictionary with keys being the name of read and values being the sequence of the read
#Assumes first read in fastq is representative of readlength of all sequences in fastq file
    if host_and_contaminant_fastq_file[-6:] != ".fastq":
        print("Host contaminant file not in .fastq format.")
    else:
        with open(host_and_contaminant_fastq_file) as myFile:
            inputLines = myFile.readlines()

        totalReads = int(len(inputLines) / 4)

        fa = {}
        for j, i in enumerate(range(totalReads)):
            nm = inputLines[i*4][1:] #Strip out the ’@’ character
            # Parse FASTQ record into name (nm) and sequence (seq)
            nm = nm.rstrip()
            seq = inputLines[(i*4)+1].rstrip().upper()
            fa[nm] = seq

            if j == 1:
                seq_read_length = len(seq)

    return fa, seq_read_length

def parse_fasta_bloom(host_reference_file, readlength, false_positive_probability):
    #parses fasta into a set of all possible reads with length specified
    if host_reference_file[-6:] != ".fasta" and host_reference_file[-3:] != ".fa":
        print("Host reference file not in .fasta or .fa format.")
    else:
        with open(host_reference_file) as myFile:
            line = myFile.readline()
            line = line.rstrip()
            #skip first line in fasta
            file = ''
            while line:
                line = myFile.readline()
                line = line.rstrip()
                file = file + line
            genome_length = int(len(file))
            host_set = {file[i:i + readlength] for i in range(0, len(file) - readlength + 1, 1)}

            #create bloomfilter from set of reads from host
            host_bloom = BloomFilter(genome_length, float(false_positive_probability))
            for sequence in host_set:
                host_bloom.add(sequence)

    return host_bloom

def main():
    #try and except block is to catch if CRs already exists.
    try:
        f = open("CRs.txt", "x")
        contaminant_dict, seq_read_length = parse_fastq(sys.argv[1])
        host_reads = parse_fasta_bloom(sys.argv[2], seq_read_length, sys.argv[3])

        #checking fastq reads against host bloom filter
        for sequence in contaminant_dict.values():
            if host_reads.check(sequence):
                continue
            else:
                f.write(sequence + '\n')

        f.close()

    except:
        print("CRs.txt already exists or reads are not in fastq or reference genome not in fasta format!")

if __name__ == '__main__':
	main()