import numpy as np
import scipy
import uproot
import awkward as ak
import pandas as pd
import math
from numpy import random
from numpy.matlib import repmat
from scipy.optimize import curve_fit
from scipy.stats import skewnorm
from sklearn import mixture

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from matplotlib.patches import Circle
from matplotlib.patches import Ellipse
from matplotlib.backends.backend_pdf import PdfPages
import glob
import os
import importlib
import sys
import csv
import json
import datetime
import time as timelib

##########################################
## Functional forms for scipy.curve_fit ##
##########################################

# single gaussian - free amplitude, mean, standard deviation
def gaussian(x, *p):

    A, mu, sigma = p

    return A*np.exp(-(x-mu)**2/(2.*sigma**2))

# constrained double gaussian - assumes second peak comes from sampling first peak twice
# can be used to fit single- and double-PE peaks in SPE calibration
def gaussian_2(x, *p):

    A1, A2, mu_1, sigma_1 = p

    p1 = [A1, mu_1, sigma_1]
    p2 = [A2, 2*mu_1, np.sqrt(2)*sigma_1] # standard deviations add in quadrature

    f = gaussian(x, *p1) + gaussian(x, *p2)

    return f

# constrained triple gaussian - assumes third peak comes from sampling second peak twice
# can be used to fit noise, single- and double-PE peaks in SPE calibration
def gaussian_3(x, *p):

    A0, A1, A2, mu_0, mu_1, sigma_0, sigma_1 = p

    p0 = [A0, mu_0, sigma_0]
    p1 = [A1, mu_1, sigma_1]
    p2 = [A2, 2*mu_1, np.sqrt(2)*sigma_1] # standard deviations add in quadrature

    f = gaussian(x, *p0) + gaussian(x, *p1) + gaussian(x, *p2)

    return f

# constrained quadruple gaussian - assumes higher peaks come from sampling second peak multiple times
# can be used to fit noise and 1/2/3 PE peaks in SPE calibration
def gaussian_4(x, *p):

    A0, A1, A2, A3, mu_0, mu_1, sigma_0, sigma_1 = p

    p0 = [A0, mu_0, sigma_0]
    p1 = [A1, mu_1, sigma_1]
    p2 = [A2, 2*mu_1, np.sqrt(2)*sigma_1] # standard deviations add in quadrature
    p3 = [A3, 3*mu_1, np.sqrt(3)*sigma_1]

    f = gaussian(x, *p0) + gaussian(x, *p1) + gaussian(x, *p2) + gaussian(x, *p3)

    return f

# poisson distribution - free amplitude and expected value
# can be used to check accuracy of multi-PE fits in SPE calibration
def poisson(x, *p):

    x = np.array(x)
    A, l = p

    return A*l**x/scipy.special.factorial(x)*np.exp(-l)

def linearPlusGaussian(x, *p):

    A, mu, sigma, m, b = p

    return A*np.exp(-(x-mu)**2/(2.*sigma**2)) + m*x + b

######################
## Unit conversions ##
######################

# ADC counts -> mV
def adcTomV(array, dVperAdc):

    # array - nD array of raw ADC values from digitzer
    # dVperAdc - voltage step per ADC count = 2.25/2**14 (voltage range/number of ADC values in 14-bit digitization)

    array = np.array(array)

    return array*(dVperAdc*1000)

# SPE area in (ADC Counts)*samples -> gain
def areaToGain(array, dVperAdc, resistance, samplingFreq):

    # array - nD array of pulse areas in units of (ADC Counts)*samples
    # dVperAdc - voltage step per ADC count
    # resistance - PMT HV/signal line resistance
    # samplingFreq - digitizer sampling frequency (100 MHz -> 1e8)

    array = np.array(array)
    fundamentalCharge = 1.602e-19 # coulomb charge per electron

    return array*(dVperAdc/resistance/samplingFreq/fundamentalCharge)

########################
## Analysis functions ##
########################

# compute the gain of each PMT using SPE sizes
def calcGains(voltages, gainFile):

    # gainFile - .csv file with gain fit parameters, generated from relative gain calibration

    voltages = np.array(voltages)
    params = np.genfromtxt(gainFile, delimiter = ',', skip_header = 1) # read fit params file
    gains = 10**(params[:, 1] + params[:, 2])*voltages**params[:, 0] # Gain = 10^(log(A) + offset)*V^p

    return gains

# calculate scale factors to convert ADC counts to PE/sample for each channel
def scaleToPE(dVperAdc, resistance, samplingFreq, gains):

    # dVperAdc - voltage step per ADC count
    # resistance - PMT HV/signal line resistance
    # samplingFreq - digitizer sampling frequency (100 MHz -> 1e8)
    # gains - gain for each PMT, computed by calcGains

    fundamentalCharge = 1.602e-19 # coulomb charge per electron
    PE = dVperAdc/(resistance*samplingFreq*fundamentalCharge*gains)

    return PE

# perform baseline subtraction for each event in a channel
def adjBaseline(array, numSamples):

    # array - 2D array, 10,000 events with microseconds*100 samples/event
    # numSamples - number of samples used to compute baseline

    adj = np.array(array - np.average(array[:, 0:numSamples], axis = 1)) # subtract baseline avg. for each event

    return adj

# compute 3-sample average for a channel
def threeSampleAvg(array):

    # array - 2D array, 10,000 events with microseconds*100 samples/event

    stacked = np.array([np.delete(array, [0, -1], 1), np.delete(array, [-2, -1], 1), np.delete(array, [0, 1], 1)]) # stack arrays shifted -1, 0, 1 samples forward
    avg = np.average(stacked, axis = 0) # calculate average for each sample

    return avg

# find boundaries between pulses for a given channel and event
def findPulses(pulseIndices, event, absoluteSpacing, relativeSpacing):

    # pulseIndices - parallel arrays, [0] contains event indices, [1] contains sample indices meeting PE/sample threshold
    # event - event in file to look at
    # absoluteSpacing - pulses must be separated by this many samples
    # relative Spacing - pulses must be seperated by this fraction of the summed length of all previous pulses

    cutIndices = pulseIndices[1][np.where(pulseIndices[0] == event)] # select sample indices corresponding to the desired event
    if len(cutIndices) < 2: # we are only interested in pulses >= 20 ns (2 samples)
        return None, None

    indexDiff = cutIndices[1:] - cutIndices[:-1] # calculate spacing between indices meeting PE/sample threshold
    absoluteBoundaries = np.where(indexDiff >= absoluteSpacing)[0] + 1 # define boundaries where spacing > absolute criterion

    if len(absoluteBoundaries) == 0: # if there is only one pulse in the event...
        pulseIntervals = [0, 3000] # look at all cutIndices for that event
    else:
        relativeSpaceCut = (indexDiff[indexDiff >= absoluteSpacing]/absoluteBoundaries > relativeSpacing).flatten() # check if boundaries also meet relative criterion
        pulseBreaks = absoluteBoundaries[relativeSpaceCut] # select boundaries that meet both criteria
        pulseIntervals = np.insert(pulseBreaks, [0, len(pulseBreaks)], [0, 3000]) # insert starting/ending bounds for first/last pulses

    return cutIndices, pulseIntervals

def constructRQArray(options):

    numRQs = np.array(options)*np.array([7, 3, 9, 3, 3, 3]) + 3
    outData = outData = [[] for i in range(len(numRQs))]

    return outData

def constructRQList(options):

    RQList = []

    if options[0] == True:
        RQList.extend(['ch0Area', 'ch1Area', 'ch2Area', 'ch3Area', 'topSumArea', 'botArea', 'totalArea'])

    if options[1] == True:
        RQList.extend(['topSumHeight', 'botHeight', 'totalHeight'])

    if options[2] == True:
        RQList.extend(['topSumWidth', 'botWidth', 'totalWidth'])
        RQList.extend(['topSum05', 'bot05', 'total05'])
        RQList.extend(['topSum95', 'bot95', 'total95'])

    if options[3] == True:
        RQList.extend(['topSumRMS', 'botRMS', 'totalRMS'])

    if options[4] == True:
        RQList.extend(['topSumFw1050', 'botFw1050', 'totalFw1050'])

    if options[5] == True:
        RQList.extend(['topSumPf200', 'botPf200', 'totalPf200'])

    return RQList

def calcRQs(outData, ch0Wf, ch1Wf, ch2Wf, ch3Wf, topSumWf, botWf, totalWf, options):

    return

def applyDataCut(dictionary, mask):

    newDict = dict(dictionary)
    for key in newDict.keys():

        newDict[key] = newDict[key][mask]

    return newDict

def selectNthPulses(dictionary, pulse):

    values, indices = np.unique(dictionary['eventNum'], return_index = True)
    indices = indices + (pulse - 1)
    nthPulses = applyDataCut(dictionary, indices)

    return nthPulses

def reconstructPosition(dictionary, cartCoor):

    XPulsesPos = cartCoor*(dictionary['ch1Area'] - dictionary['ch0Area'] - dictionary['ch3Area'] + \
                           dictionary['ch2Area'])/dictionary['topSumArea']
    YPulsesPos = cartCoor*(dictionary['ch1Area'] + dictionary['ch0Area'] - dictionary['ch3Area'] - \
                           dictionary['ch2Area'])/dictionary['topSumArea']

    return np.array(XPulsesPos), np.array(YPulsesPos)

def calcPositionMeanVals(dictionary, key, XPos, YPos, xBins, yBins, cartCoor):

    keyList = [[[] for i in range(len(xBins))] for j in range(len(yBins))]

    for event in range(len(dictionary[key])):

        xIndex = np.searchsorted(xBins, XPos[event], side = 'right') - 1
        yIndex = np.searchsorted(yBins, YPos[event], side = 'right') - 1

        keyList[yIndex][xIndex].append(dictionary[key][event])

    keyMeans = ak.mean(ak.Array(keyList), axis = -1)
    keyFilled = ak.fill_none(keyMeans, 0)

    return keyFilled

def drawEllipseSelection(ratio, theta):

    t = np.linspace(0, 2*np.pi, 100)

    xEllipse = ratio*np.cos(t)*np.cos(theta) + np.sin(t)*np.sin(theta)
    yEllipse = -ratio*np.cos(t)*np.sin(theta) + np.sin(t)*np.cos(theta)

    return xEllipse, yEllipse

def cutEllipseSelection(S1Dict, S2Dict, ratio, theta, xCenter, yCenter, xScale, yScale):

    ellipseMask = ((S1Dict['botArea'] - xCenter)/xScale*np.cos(-theta) + \
                   (S2Dict['topSumArea'] - yCenter)/yScale*np.sin(-theta))**2/ratio**2 + \
                  (-(S1Dict['botArea'] - xCenter)/xScale*np.sin(-theta) + \
                   (S2Dict['topSumArea'] - yCenter)/yScale*np.cos(-theta))**2 <= 1

    S1Ellipse = applyDataCut(S1Dict, ellipseMask)
    S2Ellipse = applyDataCut(S2Dict, ellipseMask)

    return S1Ellipse, S2Ellipse

def calcAnticorrelationParams(S1Dict, S2Dict):

    dataSet = np.transpose(np.vstack((S1Dict['botArea'], S2Dict['topSumArea'])))
    clf = mixture.GaussianMixture(n_components = 1, covariance_type = "full")
    clf.fit(dataSet)

    v, w = scipy.linalg.eigh(clf.covariances_[0])
    v = 2.0 * np.sqrt(2.0) * np.sqrt(v)
    u = w[0]/scipy.linalg.norm(w[0])

    theta = np.arctan(u[1]/u[0])
    xMean = clf.means_[0][0]
    yMean = clf.means_[0][1]

    return xMean, yMean, theta

def calcMicroG1G2(xIntercept, yIntercept, energy, W):

    quanta = energy/W

    g1 = xIntercept/quanta
    g2 = yIntercept/quanta

    return g1, g2

def calcEventEnergies(S1Dict, S2Dict, g1, g2, W):

    E = W*(S1Dict['botArea']/g1 + S2Dict['topSumArea']/g2)

    return E

def calcElectronLifetime(driftTimes, S2Dict, driftMin, driftMax, fitMin, fitMax):

    tS2Sizes = np.log(S2Dict['topSumArea'])
    rangeCut = (driftTimes >= driftMin) & (driftTimes <= driftMax)
    seltS2Sizes = tS2Sizes[rangeCut]

    driftFit = np.polyfit(driftTimes, seltS2Sizes, 1)
    xRange = np.arange(fitMin, fitMax, 0.1)
    yRange = driftFit[0]*xRange + driftFit[1]

    return driftFit, xRange, yRange

def prepChannelData(dataFile, gainFile, voltages, dVperAdc, resistance, samplingFreq, baselineSamples, PEThreshold):

    gains = calcGains(voltages, gainFile)
    scaleFactorsPE = scaleToPE(dVperAdc, resistance, samplingFreq, gains)

    rootFile = uproot.open(dataFile)

    ch0 = rootFile['Channel0/ADCValues'].array()
    ch0Adj = adjBaseline(ch0, baselineSamples)*scaleFactorsPE[0]
    ch1 = rootFile['Channel1/ADCValues'].array()
    ch1Adj = adjBaseline(ch1, baselineSamples)*scaleFactorsPE[1]
    ch2 = rootFile['Channel2/ADCValues'].array()
    ch2Adj = adjBaseline(ch2, baselineSamples)*scaleFactorsPE[2]
    ch3 = rootFile['Channel3/ADCValues'].array()
    ch3Adj = adjBaseline(ch3, baselineSamples)*scaleFactorsPE[3]
    ch4 = rootFile['Channel4/ADCValues'].array()
    ch4Adj = adjBaseline(ch4, baselineSamples)*scaleFactorsPE[4]

    botAdj = ch4Adj[:, 4:]
    topSumAdj = (ch0Adj + ch1Adj + ch2Adj + ch3Adj)[:, :-4]
    totalAdj = botAdj + topSumAdj

    ch0Avg = threeSampleAvg(ch0Adj[:, :-4])
    ch1Avg = threeSampleAvg(ch1Adj[:, :-4])
    ch2Avg = threeSampleAvg(ch2Adj[:, :-4])
    ch3Avg = threeSampleAvg(ch3Adj[:, :-4])
    botAvg = threeSampleAvg(botAdj)
    topSumAvg = threeSampleAvg(topSumAdj)
    totalAvg = threeSampleAvg(totalAdj)

    thresholdMask = totalAvg < -1*PEThreshold
    pulseIndices = np.nonzero(thresholdMask)

    return ch0Avg, ch1Avg, ch2Avg, ch3Avg, botAvg, topSumAvg, totalAvg, pulseIndices

##################
## File loading ##
##################

#return sorted file names of a specified type from a directory
def getFileList(filePath, fileType):

    # filePath - directory containing files to be sorted
    # fileType - file extension to look for, eg. '*.root'; for all files use '*'

    fileNames = sorted(filter(os.path.isfile, glob.glob(filePath + fileType)))

    return np.array(fileNames)

def getFileSizes(fileNames):

    fileSizes = [os.path.getsize(f) for f in fileNames]

    return np.array(fileSizes)

def loadData(selFiles, RQs):

    RQDict = {}
    fileNum = 0
    for file in selFiles:

        for RQ in RQs:

            RQVals = uproot.open(file + ':' + RQ)['Values'].array()
            if RQ == 'eventNum':
                RQVals = RQVals + 10000*fileNum

            if fileNum == 0:
                RQDict[RQ] = RQVals
            else:
                RQDict[RQ] = ak.concatenate([RQDict[RQ], RQVals])

        fileNum += 1

    return RQDict

def loadRedFile(file, RQs):
    file = file
    RQDict = {}
    f = uproot.open(file)
    for RQ in RQs:
        fileNum = 0
        RQVals = np.array(f['summary/' + RQ].array())
        if RQ == 'eventNum':
            RQVals = RQVals + 10000*fileNum
        if fileNum == 0:
            RQDict[RQ] = RQVals
        else:
            RQDict[RQ] = np.concatenate([RQDict[RQ], RQVals])
    return RQDict

def loadRawData(f,rawNames,scale):
    rawfile = uproot.open(rawNames[f])
    data0 = rawfile['Channel0/ADCValues'].array()
    data1 = rawfile['Channel1/ADCValues'].array()
    data2 = rawfile['Channel2/ADCValues'].array()
    data3 = rawfile['Channel3/ADCValues'].array()
    data4 = rawfile['Channel4/ADCValues'].array()

    data0_adj = data0 - np.average(data0[:, 0:100], axis = 1)
    data1_adj = data1 - np.average(data1[:, 0:100], axis = 1)
    data2_adj = data2 - np.average(data2[:, 0:100], axis = 1)
    data3_adj = data3- np.average(data3[:, 0:100], axis = 1)
    data4_adj = data4 - np.average(data4[:, 0:100], axis = 1)

    if type(scale) is str:
        d0 = data0_adj[:, :-4]
        d1 = data1_adj[:, :-4]
        d2 = data2_adj[:, :-4]
        d3 = data3_adj[:, :-4]

        topsum = d0 + d1 + d2 + d3
        bottom = data4_adj[:,4:]
        total = topsum+bottom
        x = np.arange(0, 2996, 1)
    elif np.isscalar(scale):
        d0 = data0_adj[:, :-4]*scale
        d1 = data1_adj[:, :-4]*scale
        d2 = data2_adj[:, :-4]*scale
        d3 = data3_adj[:, :-4]*scale

        topsum = d0 + d1 + d2 + d3
        bottom = data4_adj[:,4:]*scale
        total = topsum+bottom
        x = np.arange(0, 2996, 1)
    else:
        d0 = data0_adj[:, :-4]*scale[0]
        d1 = data1_adj[:, :-4]*scale[1]
        d2 = data2_adj[:, :-4]*scale[2]
        d3 = data3_adj[:, :-4]*scale[3]

        topsum = d0 + d1 + d2 + d3
        bottom = data4_adj[:,4:]*scale[4]
        total = topsum+bottom
        x = np.arange(0, 2996, 1)

    return x, np.array(d0), np.array(d1), np.array(d2), np.array(d3), np.array(topsum), np.array(bottom), np.array(total)

def ViewEvent(selFiles, rawFiles, RQs, fileNum, eventNum, channels = 'tot', scale='ADC', printRQs = False, xrange = [], yrange=[]):
    #load raw waveform and RQs
    x, d0, d1, d2, d3, topsum, bottom, total = loadRawData(fileNum,rawFiles,scale)
    RQDict = loadRedFile(selFiles[fileNum], RQs)
    #plot waveform
    if len(xrange)>0:
        plt.xlim(xrange[0],xrange[1])
    if len(yrange)>0:
        plt.ylim(yrange[0],yrange[1])
    if type(scale) is str:
        plt.xlabel('samples')
        plt.ylabel('ADC counts')
    elif np.isscalar(scale):
        plt.xlabel('samples')
        plt.ylabel('mV')
    else:
        plt.xlabel('samples')
        plt.ylabel('PE')
    if channels == 'all':
        plt.plot(x,d0[eventNum],label='tPMT0')
        plt.plot(x,d1[eventNum]-10,label='tPMT1')
        plt.plot(x,d2[eventNum]-20,label='tPMT2')
        plt.plot(x,d3[eventNum]-30,label='tPMT3')
        plt.plot(x,bottom[eventNum]-50,label='bPMT')
        plt.legend(fontsize=12)
        plt.show()
    elif channels == 'tb':
        plt.plot(x,topsum[eventNum],label='topSum')
        plt.plot(x,bottom[eventNum]-20,label='bPMT')
        plt.legend(fontsize=12)
        plt.show()
    else:
        plt.plot(x,total[eventNum])
        plt.show()
    #print RQs
    if printRQs == True:
        eventInds = np.where(np.in1d(RQDict['eventNum'],eventNum))[0]
        print('Pulses in event:',len(eventInds))
        if len(eventInds)==2:
            driftTime = RQDict['startTime'][eventInds[1]] - RQDict['endTime'][eventInds[0]]
            print('driftTime:',driftTime)
            print()
        for i in range(len(eventInds)):
            print('Pulse '+str(i+1)+':')
            for RQ in RQs:
                print(str(RQ)+':',RQDict[str(RQ)][eventInds[i]])
            print()

##########################################
## PMT calibration and characterization ##
##########################################

def LED_simple(fileNames, start, stop, channel):

    areas = []
    heights = []
    offsets = []
    for n in range(len(fileNames)):
        root_file = uproot.open(fileNames[n])

        PMT = root_file['Channel' + str(channel) + '/ADCValues'].array()
        PMT_adj = PMT - np.average(PMT[:, 0:100], axis = 1)

        for i in range(len(PMT_adj)):
            total_wf = np.array(PMT_adj[i][start:stop])
            area = -1*sum(total_wf)
            height = -1*np.amin(total_wf)
            offset = np.amax(total_wf)
            areas.append(area)
            heights.append(height)
            offsets.append(offset)

    return areas, heights, offsets

def compute_area(*p):
    A, mu, sigma = p
    x = np.arange(-5*sigma, 5*sigma, 1) + mu
    fit = gaussian(x, *p)
    return sum(fit)

def PMTPulseArea(fileNames, channel, start, cutoff, pulseArea):
    for fileName in fileNames:
        file = uproot.open(fileName)
        adc = file['Channel'+str(channel)+'/ADCValues'].array()
        ##Take the average of the first 100 adc values for each event, and shift everything by that
        ##This will make it easier to integrate later, since we can just sum the adjusted adc values
        adc_adj = adc - np.average(adc[:, 0:100], axis = 1)

        for event in range(len(adc_adj)):
            ##Find the first adc value before 1600 counts that is positive, indicating the end of the signal
            try:
                interval = np.where(adc_adj[event][cutoff:1600] >= 0)[0][0]
            except:
                print(fileName, event)
            end = cutoff + interval
            ##Integrate over the pulse area. Multiplying by -1 since the pulse is negative
            pulseArea.append(-1*sum(adc_adj[event][start:end]))
    return pulseArea

def count_pulses(fileNames, threshold, channel):

    total_pulses = []
    for fileName in fileNames:

        root_file = uproot.open(fileName)
        PMT = root_file['Channel' + str(channel) + '/ADCValues'].array()
        PMT_adj = PMT - np.average(PMT[:, 0:100], axis = 1)

        for event in PMT_adj:

            trigger = np.nonzero(event[0:1400] < threshold/(2.25/2**14*1000))[0]
            spaces = trigger[1:] - trigger[:-1]
            pulses = sum(spaces > 1)
            if len(trigger) > 0:
                pulses += 1
            total_pulses.append(pulses)

    return total_pulses

#################################
## Trigger rate and levelmeter ##
#################################

# remove unwanted trigger data before plotting
def prepTriggerData(triggerFile):

    # triggerFile - .txt file with trigger rates for each PMT + corresponding unix times
    # Note - trigger rates in the first and last second of each file need to be discarded because they do not represent
    #        a full second of data taking

    triggerData = np.genfromtxt(triggerFile, delimiter = ' ') # read trigger data
    goodVals = np.delete(triggerData, [0, len(triggerData) - 1], axis = 0) # cut first and last second from the data
    times = goodVals[:, 0] - np.amin(goodVals[:, 0]) # compute times since data-taking start

    for i in range(len(times) - 1, 0, -1): # start at the end of the file and work backwards
        if times[i] - times[i - 1] > 1: # if time spacing between data points > 1 second (aka new file)...
            goodVals = np.delete(goodVals, [i - 1, i], axis = 0) # delete data points on both sides

    adjTimes  = goodVals[:, 0] - np.amin(goodVals[:, 0]) # recompute times for selected data only

    return goodVals, adjTimes

# get the detector liquid level at the given unix times
def getLiquidLevels(timeStamps, levelFile, startUnixTime):

    # timeStamps - unix times at which liquid levels are calculated
    # levelData - 2D array of time in seconds *SINCE SLOW CONTROL DATABASE INITIALIZED* and level values at those times
    # startUnixTime - unix time of first value in levelData, computed manually

    levelData = np.genfromtxt(levelFile, delimiter = ',', skip_header = 1)[:, 1:] # read level data

    levelTimes = levelData[:, 0] # get database times
    levelVals = levelData[:, 1] # get level values
    adjLevelTimes = (levelTimes - levelTimes[0]) + startUnixTime # convert database times to unix times

    finalLevels = [] # initialize return array for liquid levels
    for timeStamp in timeStamps: # for each time stamp...

        index = np.searchsorted(adjLevelTimes, timeStamp, side = 'right') # adjLT[index - 1] <= timeStamp < adjLT[index]

        if timeStamp == adjLevelTimes[index - 1]: # check edge condition caused due to <= sign in np.searchsorted
            finalLevels.append(levelVals[index - 1])
        else:
            levelStep = levelVals[index] - levelVals[index - 1] # compute step between closest database values
            frac = (timeStamp - adjLevelTimes[index - 1])/(adjLevelTimes[index] - adjLevelTimes[index - 1]) # percent of the way to next database value
            compLevel = levelVals[index - 1] + levelStep*frac # linear interpolation between values
            finalLevels.append(compLevel)

    return np.array(finalLevels)

def getFileUnixTimes(fileNames, relative  = False):

    unixTimes = []
    for file in fileNames:

        dateIndex = file.find('MiX_')
        date = file[dateIndex + 4:dateIndex + 12]

        timeIndex = file.find('T', dateIndex)
        time = file[timeIndex + 1: timeIndex + 7]

        dateTime = datetime.datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
                                 int(time[0:2]), int(time[2:4]), int(time[4:6]))
        unixTime = timelib.mktime(dateTime.timetuple())
        unixTimes.append(unixTime)

    unixTimes = np.array(unixTimes)
    if relative:

        unixTimes = unixTimes - unixTimes[0]

    return unixTimes


import setuptools
import warnings

DISTNAME            = "scipydirect"
DESCRIPTION         = "Python wrapper to the DIRECT algorithm"
LONG_DESCRIPTION    ="""
DIRECT is a method to solve global bound constraint optimization problems and
was originally developed by D. R. Jones, C. D. Perttunen and B. E. Stuckmann.

The scipydirect is a fork of pydirect providing a scipy.optimize compatible
syntax. It uses the Fortran implementation of DIRECT written by
Joerg M Gablonsky, DIRECT Version 2.0.4.

For more info see the `documentation <http://scipydirect.readthedocs.io/en/latest/>`_ or the `source code <http://github.com/andim/scipydirect>`_.
"""
MAINTAINER          = "Andreas Mayer"
MAINTAINER_EMAIL    = "andimscience@gmail.com"
URL                 = "http://github.com/andim/scipydirect"
LICENSE             = "MIT"
VERSION             = "1.4"

classifiers =  ['Development Status :: 5 - Production/Stable',
                'Programming Language :: Python',
                "Programming Language :: Python :: 2",
                "Programming Language :: Python :: 2.7",
                "Programming Language :: Python :: 3",
                "Programming Language :: Python :: 3.8",
                'License :: OSI Approved :: MIT License',
                'Intended Audience :: Science/Research',
                'Topic :: Scientific/Engineering',
                'Topic :: Scientific/Engineering :: Mathematics',
                'Operating System :: OS Independent']

def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration(DISTNAME, parent_package, top_path,
                           version=VERSION,
                           maintainer=MAINTAINER,
                           maintainer_email=MAINTAINER_EMAIL,
                           description=DESCRIPTION,
                           license=LICENSE,
                           url=URL,
                           long_description=LONG_DESCRIPTION)

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    kwargs = dict(
            packages=setuptools.find_packages(),
            include_package_data=True,
            platforms=["any"],
            requires=["numpy"],
            install_requires=["numpy"],
            tests_require=['nose',],
            test_suite='nose.collector',
            zip_safe=True,
            classifiers=classifiers,
            )
    thiskwargs = kwargs.copy()
    config = configuration()
    config.add_extension('direct', sources=['src/direct.pyf', 'src/DIRect.f', 'src/DIRserial.f', 'src/DIRsubrout.f'])
    thiskwargs.update(config.todict())
    setup(**thiskwargs)


import os
from pathlib import Path
from typing import List

import cv2
import numpy as np

from variables import (
    IS_DOCKER,
    MOUNT_IMAGE_DEST,
    MOUNT_IMAGE_SRC,
    VIDEO_FILE_EXTENSIONS,
)
from utils.logger import logger

class IOVideoManager(object):
    """IOVideoManager class to work with IO"""

    def __init__(self, src_folder: Path, dest_folder: Path, silent=False) -> None:
        self._src_folder = src_folder
        self._dest_folder = dest_folder
        # The '_silent' variable is currently not used but it could be useful for logging
        # purposes
        self._silent = silent

    def _log_found_files(self, file_names: List[str]) -> None:
        """Helper function to log all the valid files found in the src folder"""

        logger.info(f"Found {len(file_names)} valid file(s).")
        for f in file_names:
            logger.info(f"   - {f}")

    def get_video_paths(self) -> List[dict]:
        """Gets the paths of the videos in the src folder"""

        path_objs = []
        abs_src_folder = self._src_folder.absolute()
        for p in os.listdir(abs_src_folder.as_posix()):
            if p.lower().endswith(tuple(VIDEO_FILE_EXTENSIONS)):
                full_path = ""
                if IS_DOCKER:
                    full_path = Path(os.path.join(MOUNT_IMAGE_SRC, p))
                else:
                    full_path = Path(os.path.join(abs_src_folder.as_posix(), p))

                tmp = {"name": p, "path": full_path}
                path_objs.append(tmp)

        self._log_found_files([p_obj["name"] for p_obj in path_objs])
        return path_objs

    def save_img(self, dest_path: Path, image: np.array) -> None:
        """Saves a np.array with cv2 to dest folder"""
        full_save_path = ""
        if IS_DOCKER:
            full_save_path = os.path.join(MOUNT_IMAGE_DEST, dest_path.as_posix())
        else:
            full_save_path = os.path.join(
                self._dest_folder.as_posix(), dest_path.as_posix()
            )

        # Create dirs if they do not exist already
        os.makedirs(os.path.dirname(full_save_path), exist_ok=True)
        cv2.imwrite(full_save_path, image)


import os, sys
import config, db_manage
from main import app

#check db exists else create
if not os.path.isfile(config.db_path):
    db_manage.db_create() #creates db if not exist
    db_manage.db_migrate() #creates first version migration - for rollbacks

#check db up to date else migrate
result = db_manage.compare_db_model()
if result and result != "No schema diffs":
    db_manage.db_migrate()

#run app
app.run(debug = True)


from binance.client import Client
import variables as var
from datetime import datetime

dollar = 1000
trading = 1000
position = 0

##Setup the client with your APIs stored in the variables.py folder
client = Client(var.API,var.APIKey)

klines = client.get_historical_klines(var.Pair, Client.KLINE_INTERVAL_1DAY, var.StartDate)

print("You are trading the",var.BigEMA,"vs",var.SmallEMA,"exponential moving average on"
      , var.Pair, " since ",var.StartDate, " and ", dollar , " dollars.")

old_direction = 'Null'

returns = []

for i in range(var.BigEMA,len(klines)):
    price_now = float(klines[i][4])
    if old_direction == 'Null':

        ##Calculate SMA for shorter time period.
        smallMA = 0
        for each in klines[i - var.SmallEMA:i]:
            smallMA += float(each[4])
        smallMA = smallMA / var.SmallEMA

        ##Calculate SMA for longer time period.
        bigMA = 0
        for each in klines[i - var.BigEMA:i]:
            bigMA += float(each[4])
        bigMA = bigMA / var.BigEMA

        SmallEMA = smallMA
        smallWeight = var.Smoothing / (var.SmallEMA + 1)
        BigEMA = bigMA
        bigWeight = var.Smoothing / (var.BigEMA + 1)
    elif old_direction != 'Null':
        ##Calculate EMA for shorter time period.
        SmallEMA = (price_now * smallWeight) + SmallEMA *(1-smallWeight)
        ##Calculate EMA for shorter time period.
        BigEMA = (price_now * bigWeight) + BigEMA *(1-bigWeight)

    if (BigEMA < SmallEMA) and (price_now > SmallEMA):
        direction = 'Positive'
    elif BigEMA < SmallEMA and old_direction == 'Null':
        direction = 'Positive'
    if (BigEMA > SmallEMA) and (price_now < BigEMA):
        direction = 'Negative'
    elif BigEMA > SmallEMA and old_direction == 'Null':
        direction = 'Negative'
    if old_direction != direction:
        if old_direction == 'Null':
            old_direction = direction
        elif old_direction != direction and i > 3*var.BigEMA:
            # Determine the date and time of the trade
            date = datetime.utcfromtimestamp(float(klines[i][6])/1000).strftime('%Y-%m-%d %H:%M:%S')
            if old_direction == 'Positive' and position != 0:
                print("Sell at " + str(klines[i][4]))
                dollar += position * price_now * (1-var.TakerFee)
                position = 0
                if price_now > buy:
                    gain = round((10000*(price_now-buy)/price_now))
                    print("Profit = " + str(price_now - buy) + " Or " + str(gain/100) + "% at " + date)
                    returns += [gain/100]
                else:
                    loss = round((10000*(price_now-buy)/price_now))
                    print("Loss = " + str(price_now - buy) + " Or " + str(loss/100) + "% at " + date)
                    returns += [loss/100]
                buy = 0
            elif old_direction != 'Positive':
                print("Buy at " + str(klines[i][4]))
                position += (trading/price_now) * (1-var.MakerFee)
                dollar -= trading
                buy = price_now
            old_direction = direction

if position != 0:
    print("Final sell at " + str(float(klines[len(klines)-1][4])))
    if float(klines[len(klines)-1][4]) > buy:
        print("Profit = " + str(float(klines[len(klines)-1][4]) - buy) + " Or " + str(
            (round((10000 * (float(klines[len(klines)-1][4]) - buy) / float(klines[len(klines)-1][4])))) / 100) + "%.")
    else:
        print("Loss = " + str(float(klines[len(klines)-1][4]) - buy) + "Or " + str(
            (round((10000 * (float(klines[len(klines)-1][4]) - buy) / float(klines[len(klines)-1][4])))) / 100) + "%.")

    dollar += position * float(klines[len(klines)-1][4])
    position = 0
print("Dollar is ",str(dollar))
print(returns)

ReturnsTest = var.Returns(returns)

print("Average is ",str(ReturnsTest.average()))
print("Variance is ",str(ReturnsTest.variance()))
print("Standard Deviation is ",str(ReturnsTest.standard_dev()))


import timm
import torch

def get_backbone_model(name, labels, **kwargs):
    model = timm.create_model(name, num_classes=len(labels), **kwargs)
    model.__arguments__ = {'name': name, 'labels': labels, **kwargs}
    model.__info__ = {}
    return model.float()

def save_model_to_ckpt(model, file):
    arguments = getattr(model, '__arguments__', {}) or {}
    info = getattr(model, '__info__', {}) or {}
    torch.save({
        'state_dict': model.state_dict(),
        'arguments': arguments,
        'info': info,
    }, file)

def load_model_from_ckpt(file):
    data = torch.load(file, map_location='cpu')
    arguments = data['arguments'].copy()
    name = arguments.pop('name')
    labels = arguments.pop('labels')
    model = get_backbone_model(name, labels, **arguments)
    existing_keys = set(model.state_dict())
    state_dict = {key: value for key, value in data['state_dict'].items() if key in existing_keys}
    model.load_state_dict(state_dict)
    model.__info__ = data['info']

    return model


#!/usr/bin/env python
# coding: utf-8

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
from skimage.io import imread
from skimage.transform import resize
import numpy as np
import skimage.transform as tf
from PIL import Image
from io import BytesIO
from images.utils import figure_to_PIL

def transform(corners):
    """
    This function is based in the jupyter notebook with the same name.
    Look at it for documentation.
    """
    image_path = corners["image"]
    image = imread(image_path, as_gray=True)
    image_shape = tuple(corners["image_shape"])
    image_resize = resize(image, image_shape, mode="symmetric", preserve_range=True)

    # Given that the new shape will be 300 by 300 this is the given points
    src = np.array([[0, 0], [0, 300], [300, 300], [300, 0]])

    corners["up_left"] = [float(x) for x in corners["up_left"]]
    corners["down_left"] = [float(x) for x in corners["down_left"]]
    corners["up_right"] = [float(x) for x in corners["up_right"]]
    corners["down_right"] = [float(x) for x in corners["down_right"]]

    dst = np.array(
        [
            corners["up_left"],
            corners["down_left"],
            corners["down_right"],
            corners["up_right"],
        ]
    )

    tform = tf.ProjectiveTransform()
    tform.estimate(src, dst)
    warped = tf.warp(image_resize, tform, output_shape=image_shape)

    new_image = warped[0:300, 0:300]

    fig, ax = plt.subplots()

    ax.imshow(new_image, cmap=plt.cm.gray)

    divisor = 299
    color = "#FF0000"

    for i in range(9):
        x = (divisor / 9) * (i + 1)
        y1 = 0
        y2 = 299
        linewidth = int((i + 1) % 3 == 0) * 3 or 1
        plt.plot([x, x], [y1, y2], "-", linewidth=linewidth, color=color)
        plt.plot([y1, y2], [x, x], "-", linewidth=linewidth, color=color)
    plt.plot([0, 0], [0, divisor], "-", linewidth=3, color=color)
    plt.plot([0, divisor], [0, 0], "-", linewidth=3, color=color)

    pil_image = figure_to_PIL(fig, ax, plt)

    response = {
        "image": pil_image,
        "transform_image": new_image
        }

    return response


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import smart_selects.db_fields

class Migration(migrations.Migration):

    dependencies = [
        ('baseline', '0006_auto_20150928_1711'),
    ]

    operations = [
        migrations.AlterField(
            model_name='baselineterm',
            name='follow_up_poll',
            field=models.ForeignKey(related_name='+', verbose_name='Observation Poll', to='polls.Poll'),
        ),
        migrations.AlterField(
            model_name='baselineterm',
            name='follow_up_question',
            field=smart_selects.db_fields.ChainedForeignKey(chained_model_field=b'poll', related_name='+', chained_field=b'follow_up_poll', verbose_name='Observation Question', auto_choose=True, to='polls.Question', help_text='Follow up poll responses over time to compare to the baseline values.'),
        ),
    ]


import torch
import matplotlib
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')

def save_model(epochs, model, optimizer, criterion):
    """
    Function to save the trained model to disk.
    """
    torch.save({
                'epoch': epochs,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'loss': criterion,
                }, 'outputs/model.pth')

def save_plots(train_acc, valid_acc, train_loss, valid_loss):
    """
    Function to save the loss and accuracy plots to disk.
    """
    # accuracy plots
    plt.figure(figsize=(10, 7))
    plt.plot(
        train_acc, color='green', linestyle='-',
        label='train accuracy'
    )
    plt.plot(
        valid_acc, color='blue', linestyle='-',
        label='validataion accuracy'
    )
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.legend()
    plt.savefig('outputs/accuracy.png')

    # loss plots
    plt.figure(figsize=(10, 7))
    plt.plot(
        train_loss, color='orange', linestyle='-',
        label='train loss'
    )
    plt.plot(
        valid_loss, color='red', linestyle='-',
        label='validataion loss'
    )
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()
    plt.savefig('outputs/loss.png')


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="TF3Dscan",
    version="0.0.1",
    author="Nigatu Ayele",
    author_email="naodm2006awet4@gmail.com",
    description="Transcription factor binding site prediction",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/pypa/example-project",
    packages=setuptools.find_packages(),
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
)


from logzero import logger
import torch
from src.genai.model.Generators.gen import Generator
from src.genai.model.discriminators.disc import Discriminator
from src.genai.dataoperations.dataoperations import DataLoadingOperations
import torch.optim as optim
import torch.nn as nn
from tqdm import tqdm
import wandb
import torchvision
import os

class TrainConfig:
    def __init__(self) -> None:
        pass

class GanTraining:
    def __init__(self, config) -> None:
        self.config = config

    def optimizer(self, params, model):
        if params["name"] == "adam":
            return optim.Adam(model.parameters(), lr=params["lr"])

    def criterion(self, params):
        if params["name"] == "bceloss":
            return nn.BCELoss()

    def train(self):
        model_info = self.config["gan"]

        if model_info["name"] == "simple_gan":
            trainconfig = self.config_simple_gan()
            self.train_simple_gan(trainconfig)

    def config_simple_gan(self):
        self.trainconfig = TrainConfig()
        self.trainconfig.data = 0
        self.trainconfig.device = self.config["operation"]["device"]
        self.trainconfig.disc = Discriminator(self.config["gan"]["image_dim"])
        self.trainconfig.gen = Generator(self.config["gan"]["z_dim"], self.config["gan"]["image_dim"])

        # print(self.trainconfig.gen)
        # print(self.trainconfig.disc)

        logger.debug("The Generator and Discriminator are built.")

        tfms = DataLoadingOperations.get_tfs()
        data = DataLoadingOperations.get_mnist(tfms)
        self.trainconfig.dataloader = DataLoadingOperations.dataloading(data)

        logger.debug("The dataloader is loaded.")

        gan_params = self.config["gan"]["gen"]["opt"]
        self.trainconfig.gen_opt = self.optimizer(gan_params, self.trainconfig.gen)

        disc_params = self.config["gan"]["disc"]["opt"]
        self.trainconfig.disc_opt = self.optimizer(disc_params, self.trainconfig.disc)

        logger.debug("The optimizers are init.")

        criterion_params = self.config["gan"]["criterion"]
        self.trainconfig.criterion = self.criterion(criterion_params)

        logger.debug("The criterion is set.")

        self.trainconfig.device = self.config["operation"]["device"]
        self.trainconfig.z_dim = self.config["gan"]["z_dim"]
        self.trainconfig.batch_size = self.config["gan"]["batch_size"]
        self.trainconfig.num_epochs = self.config["gan"]["num_epochs"]

        self.trainconfig.wandb_status = self.config["operation"]["wandb_flag"]
        freq = self.config["operation"]["wandb_model_freq"]
        if self.trainconfig.wandb_status:
            wandb.init(project="gan_training", config=self.config)
            wandb.watch(self.trainconfig.disc, log_freq=freq, log="all")
            wandb.watch(self.trainconfig.gen, log_freq=freq, log="all")

        return self.trainconfig

    def train_simple_gan(self, config):
        logger.debug("training a simple gan")

        # make sure the below are there configured in the config dic
        # before training.
        device = self.trainconfig.device
        z_dim = self.trainconfig.z_dim
        batch_size = self.trainconfig.batch_size
        num_epochs = self.trainconfig.num_epochs

        disc = config.disc
        gen = config.gen

        loader = config.dataloader
        opt_disc = config.disc_opt
        opt_gen = config.gen_opt

        criterion = config.criterion
        wandb_status = config.wandb_status

        # convert models to device
        disc = disc.to(device)
        gen = gen.to(device)

        fixed_noise = torch.randn((batch_size, z_dim)).to(device)
        step = 0

        for epoch in range(num_epochs):
            for batch_idx, (real, _) in enumerate(loader):
                real = real.view(-1, 784).to(device)
                batch_size = real.shape[0]

                ### Train Discriminator: max log(D(x)) + log(1 - D(G(z)))
                noise = torch.randn(batch_size, z_dim).to(device)
                fake = gen(noise)
                disc_real = disc(real).view(-1)
                lossD_real = criterion(disc_real, torch.ones_like(disc_real))
                disc_fake = disc(fake).view(-1)
                lossD_fake = criterion(disc_fake, torch.zeros_like(disc_fake))
                lossD = (lossD_real + lossD_fake) / 2
                disc.zero_grad()
                lossD.backward(retain_graph=True)
                opt_disc.step()

                ### Train Generator: min log(1 - D(G(z))) <-> max log(D(G(z))
                # where the second option of maximizing doesn't suffer from
                # saturating gradients
                output = disc(fake).view(-1)
                lossG = criterion(output, torch.ones_like(output))
                gen.zero_grad()
                lossG.backward()
                opt_gen.step()

                if wandb_status:
                    wandb_logs = {"lossD": lossD, "lossG": lossG, "epoch": epoch}
                    wandb.log(wandb_logs)

                if batch_idx == 0:
                    print("Epoch :", epoch)
                    print("Disc Loss :", lossD)
                    print("Gen Loss :", lossG)

                    with torch.no_grad():
                        fake = gen(fixed_noise).reshape(-1, 1, 28, 28)
                        data = real.reshape(-1, 1, 28, 28)
                        img_grid_fake = torchvision.utils.make_grid(fake, normalize=True)
                        img_grid_real = torchvision.utils.make_grid(data, normalize=True)
                        img_grid_fake = wandb.Image(
                            img_grid_fake,
                        )

                        wandb.log({"img_grid_fake": img_grid_fake})

                        img_grid_real = wandb.Image(
                            img_grid_real,
                        )

                        wandb.log({"img_grid_real": img_grid_real})

                    step += 1

        # save the model
        os.makedirs("./output", exist_ok=True)
        wandb.unwatch(gen)
        wandb.unwatch(disc)
        op = {
            "model_disc": disc.state_dict(),
            "model_gen": gen.state_dict(),
            "opt_disc": opt_disc.state_dict(),
            "opt_gen": opt_gen.state_dict(),
        }
        torch.save(op, "output/op.pt")
        logger.info("Saved the models of gen, disc.")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/11/9
# @Author  : RookieDay
# @Site    :
# @File    : callback_play
# @Github  : https://github.com/rookieday
# @Software: PyCharm Community Edition

import pyaudio
import wave
import os
import time

wav_file = os.path.join(os.path.dirname(__file__),'Alizee.wav')

wf = wave.open(wav_file,'rb')
p = pyaudio.PyAudio()

def callback(in_data, frame_count, time_info, status):
    data = wf.readframes(frame_count)
    # print(data)
    return (data,pyaudio.paContinue)

stream = p.open(format=p.get_format_from_width(wf.getsampwidth()),
                channels=wf.getnchannels(),
                rate=wf.getframerate(),
                output=True,
                stream_callback=callback)
stream.start_stream()

while stream.is_active():
    time.sleep(1)

stream.stop_stream()
stream.close()
wf.close()
p.terminate()


from flask import Flask, jsonify, request
from flask_cors import CORS
import utils

app = Flask(__name__)
CORS(app, supports_credentials=True)

@app.route("/get_total", methods=["GET"])
def get_total():
    return jsonify(count=utils.query_counts())

@app.route("/get_request/<int:id>", methods=["GET"])
def get_request_info(id):
    src_ip, dest_ip, datas = utils.query_request(id)
    return jsonify(src_ip=src_ip, dest_ip=dest_ip, datas=datas)

@app.route("/get_src_ip_rank", methods=["GET"])
def get_src_ip_rank():
    name, counts = utils.query_src_rank()
    return jsonify(name=name,counts=counts)

@app.route("/get_dest_ip_rank", methods=["GET"])
def get_dest_ip_rank():
    name, counts = utils.query_host_rank()
    return jsonify(name=name,counts=counts)

@app.route("/get_interests", methods=["GET"])
def get_interests():
    names, series = utils.analysis_interests()
    return jsonify(names=names,series=series)

@app.route("/get_gps_points", methods=["GET"])
def get_gps_points():
    points = utils.gen_gps_point()
    return jsonify(points=points)

if __name__ == '__main__':
    app.run('0.0.0.0')


#!/bin/python

'''
permutation test (one sided p-value)
following the description of: Ensemble-based Prediction of RNA Secondary Structures (Nima Aghaeepour and Holger H. Hoos)

Created on Dez 1, 2011

@author: Marius Schneider

Input: csv file
'''

import sys
import os
import math
import csv
import argparse
import random
import functools

def readIn(file,c1,c2,deli):
    Matrix = csv.reader(open(file, 'r'), delimiter=deli)
    col1 = []
    col2 = []
    for row in Matrix:
        try:
            col1.append(float(row[c1-1]))
            col2.append(float(row[c2-1]))
        except:
            pass
    if(len(col1) != len(col2)):
        print("WARNING: data have not the same length!")
    if(len(col1) == 0):
        print("WARNING: no data found!")
    return [col1,col2]

def randomSwap(col1,col2):
    n = len(col1)
    sc1 = []
    sc2 = []
    for i in range(0,n):
        if (random.random() > 0.5):
            sc1.append(col2[i])
            sc2.append(col1[i])
        else:
            sc1.append(col1[i])
            sc2.append(col2[i])
    avg1 = avg(sc1)
    avg2 = avg(sc2)
    return avg1-avg2    # betrag?

def getPValue(col1,col2,p,avgS):
    avgs = [avgS]
    for i in range(0,p-1):
        avgs.append(randomSwap(col1,col2))
    print("AVG Start : " +str(avgS))
    print("min AVG : "+str(min(avgs)))
    print("max AVG : "+str(max(avgs)))
    v = percentile(avgs,avgS)
    print("Percentile : "+str(v))
    pValue = v / 100
    return pValue

def percentile(N, value):
    ''' returns percentile of value on list N '''
    N.sort()
    #print(N)
    first = N.index(value)
    count = N.count(value)
    #return float(first+count-1)*100/len(N)
    return float(first)*100/len(N)

def avg(N):
    return (float(sum(N)) / len(N))

def filter(col1,col2,noise):
    for i in range(0,len(col1)):
        delta = math.sqrt(noise/2) * math.sqrt((col1[i]+col2[i])/2)
        avgI = (col1[i]+col2[i])/2
        #print("Delta : "+str(delta))
        #print("AVG : "+str(avgI))
        #print("cols :"+str(col1[i]) + " : "+str(col2[i]))
        if ((col1[i] < col2[i] and col1[i] >= (avgI-delta)) or (col2[i] <= col1[i] and col2[i] >= (avgI-delta))):
            col1[i] = col2[i]
            #print("Set equal")
    return [col1,col2]

def doTest(col1,col2,alpha,permutations,name1,name2,noise):
    if(noise != None):
        [col1,col2] = filter(col1,col2,float(noise))
    oriAVG = avg(col1) - avg(col2)
    switched = 1    # 1: not switched; -1: switched
    if (oriAVG > 0):
        print("Alternative Hypothesis: Coloumn "+str(name2) + " is better than Coloumn "+str(name1))
        tmp = col1
        col1 = col2
        col2 = tmp
        oriAVG *= -1
        switched = -1
    else:
        print("Alternative Hypothesis: Coloumn "+str(name1) + " is better than Coloumn "+str(name2))
    random.seed(1234)
    pValue = getPValue(col1,col2,permutations,oriAVG)
    if (pValue < alpha):
        print("Reject Null Hypothesis (p-value: "+str(pValue)+")")
        return [1,switched,pValue]
    else:
        print("Don't reject Null Hypothesis (p-value: "+str(pValue)+")")
        return [0,switched,pValue]

if __name__ == '__main__':

    description  = "usage: python -O permutationtest.py --csv=[file]  [options]"
    parser = argparse.ArgumentParser(prog="master.py",usage=description)
    req_group = parser.add_argument_group("Required Options")
    req_group.add_argument('--csv', dest='csv', action='store', default="", required=True,  help='csv file with instance in rows and configurations as columns')
    req_group.add_argument('--alpha', dest='alpha', action='store', default=0.05, required=True, type=float,  help='confidence level')

    opt_group = parser.add_argument_group("Misc Options")
    opt_group.add_argument('--coloumn1', dest='c1', action='store', default=1, type=int, help='coloumn 1')
    opt_group.add_argument('--coloumn2', dest='c2', action='store', default=2, type=int, help='coloumn 2')
    opt_group.add_argument('--delimeter', dest='deli', action='store', default=",", help='delimeter')
    opt_group.add_argument('--perm', dest='p', action='store', default=10000, type=int, help='number of permutations')
    opt_group.add_argument('--noise', dest='noise', action='store', default=None, help='noise to filter')

    args = parser.parse_args()

    if (os.path.isfile(args.csv) == False):
        sys.stderr.write("ERROR: CSV File was not found!\n")
        sys.exit()

    [col1, col2] = readIn(args.csv,args.c1,args.c2,args.deli)
    doTest(col1,col2,args.alpha,args.p,args.c1,args.c2,args.noise)


from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run
from prefect import flow, task
from prefect_snowflake.database import SnowflakeConnector
from prefect.states import Failed

snowflake_connector = SnowflakeConnector.load("snowflake-demo-connector")

@flow
def cloud_job(JOB_ID = 424466):
    dbt_cloud_credentials = DbtCloudCredentials.load("dbt-cloud-creds")
    trigger_dbt_cloud_job_run(dbt_cloud_credentials=dbt_cloud_credentials, job_id=JOB_ID)

@task
def count_recent_cc_records():
    result = snowflake_connector.fetch_one(
        "select cc_rec_start_date > current_date - 1 as is_fresh, max(cc_rec_start_date) as max_date from snowflake_sample_data.tpcds_sf10tcl.call_center group by 1"
    )
    return result

@flow(retries = 3, retry_delay_seconds = 3, log_prints = True)
def daily_job():
    fresh_data = count_recent_cc_records()
    if fresh_data[0]:
        cloud_job()
    else:
        return Failed(message = f'Stale data: most recent date is {str(fresh_data[1])}')

if __name__ == "__main__":
    daily_job()


#!/usr/bin/env python

import os
import sys

from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand

here = os.path.abspath(os.path.dirname(__file__))

with open(os.path.join(here, 'README.md')) as fp:
    README = fp.read()

with open(os.path.join(here, 'CHANGES.txt')) as fp:
    CHANGES = fp.read()

requires = ['pyramid>=1.4', 'webassets>=0.8', 'zope.interface', 'six>=1.4.1']

extras_require = {
    'bundles-yaml': 'PyYAML>=3.10',
}

class PyTest(TestCommand):
    user_options = [
        ('cov', None, "measure coverage")
    ]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.cov = None

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = ['pyramid_webassets/tests']
        if self.cov:
            self.test_args += ['--cov', 'pyramid_webassets']
        self.test_suite = True

    def run_tests(self):
        import pytest
        errno = pytest.main(self.test_args)
        sys.exit(errno)

setup(name='pyramid_webassets',
      version='0.10',
      description='pyramid_webassets',
      long_description=README + '\n\n' + CHANGES,
      classifiers=[
          "Programming Language :: Python",
          "Framework :: Pylons",
          "License :: OSI Approved :: MIT License",
          "Topic :: Internet :: WWW/HTTP",
          "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
      ],
      author='John Anderson',
      author_email='sontek@gmail.com',
      url='http://github.com/sontek/pyramid_webassets',
      keywords='web wsgi bfg pylons pyramid',
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      test_suite='pyramid_webassets',
      install_requires=requires,
      extras_require=extras_require,
      tests_require=['pytest', 'pytest-cov'],
      cmdclass={'test': PyTest},
      )


import collections
from random import randint,choice

def adj(G,u):
    a=[]
    for v in G[u]:
        if G[u][v]!=0:
            a.append(v)
    return a

def edges(G):
    e={}
    for u in G:
        for v in adj(G,u):
            if not (v,u) in e.keys():
                e[(u,v)]=G[u][v]
    return e

def NewAdjMatrix(G,copy=False):
    am={}
    for u in G:
        am[u]={}
        for v in G[u]:
            if copy:
                am[u][v]=G[u][v]
            else:
                am[u][v]=0
    return am

def CopyAdjMatrix(G):
    return NewAdjMatrix(G,True)

def RevPath(D,d):
    P=[d]
    while D[d]['p']!=None:
        d=D[d]['p']
        P.append(d)
    return P

def RevWFI(P,s,d):
    PR=[d]
    while P[s][d]!=0:
        d=P[s][d]
        PR.append(d)
    return PR

def FindCyclesMatrix(G):
    visited=[]
    cycs=[]
    S=collections.deque()
    for u0 in G:
        if u0 not in visited:
            P=NewAdjMatrix(G)
            FindCyclesMatrix_r(G,P,u0,visited,S,cycs)
    return cycs

def FindCyclesMatrix_r(G,P,u,visited,S,cycs):
    visited.append(u)
    S.append(u)
    n=[]
    for v in G[u]:
        if G[u][v]!=0: n.append(v)
    for v in n:
        if v not in S:
            P[u][v]=P[v][u]=1
            FindCyclesMatrix_r(G,P,v,visited,S,cycs)
        else:
            if P[u][v]!=1: # prevents turning back to the previous vertex in undirected graphs
                c = []
                for vx in reversed(S):
                    c.append(vx)
                    if vx is v:
                        break
                if c not in cycs:
                    cycs.append(c)
    S.pop()

def FindCycleWithWeightsMatrix(G):
    c=FindCyclesMatrix(G)
    E2={}
    E=edges(G)
    if len(c)>0:
        c=c[0]
        cp=[(c[-1],c[0])]
        for i in range(len(c)-1):
            cp.append((c[i],c[i+1]))
        for cpi in cp:
            E2=E2|dict(filter(lambda x:x[0][0] in cpi and x[0][1] in cpi,E.items()))
    return E2

def FindCycles(G):
    visited=[]
    cycs=[]
    S=collections.deque()
    for u0 in G:
        if u0 not in visited:
            FindCycles_r(G,u0,visited,S,cycs)
    return cycs

def FindCycles_r(G,u,visited,S,cycs):
    visited.append(u)
    S.append(u)
    for v in G[u]['adj']:
        if v not in S:
            G[v]['p']=u
            FindCycles_r(G,v,visited,S,cycs)
        else:
            if v is not G[u]['p']: # prevents turning back to the previous vertex in undirected graphs
                c = []
                for vx in reversed(S):
                    c.append(vx)
                    if vx is v:
                        break
                if c not in cycs:
                    cycs.append(c)
    S.pop()

def DetectCycle(G):
    visited=[]
    S=collections.deque()
    for u0 in G:
        if u0 not in visited:
            r=DetectCycle_r(G,u0,visited,S)
            if r: return True
    return False

def DetectCycle_r(G,u,visited,S):
    visited.append(u)
    S.append(u)
    for v in G[u]['adj']:
        if v not in S:
            G[v]['p']=u
            r=DetectCycle_r(G,v,visited,S)
            if r: return True
        else:
            if v is not G[u]['p']: # prevents turning back to the previous vertex in undirected graphs
                return True
    S.pop()
    return False

def MakeSet(F,x):
    if x not in F:
        F[x]={'p':x}
    return F

def Find(F,x):
    while F[x]['p']!=x:
        F[x]['p']=F[F[x]['p']]['p']
        x=F[x]['p']
    return x

def Union(F,x,y):
    x=Find(F,x)
    y=Find(F,y)
    F[x]['p']=y

def MatrixToTable(G):
    GR={}
    for v in G:
        GR[v]={'adj':adj(G,v)}
    return GR

def GenerateRandomMatrix(nodes:list, start:chr) -> dict:
    G={n1:{n2:0 for n2 in nodes} for n1 in nodes}
    svl=nodes.index(start)
    w1=[svl]
    while True:
        clds=randint(1,2)
        while clds+len(w1)>len(nodes): clds=randint(1,2)
        vls=[]
        stop=False
        while not stop:
            for c in range(clds):
                x=svl
                while x==svl or x in vls: x=randint(0,len(nodes)-1)
                vls.append(x)
            checks=0
            for tvl in vls:
                if tvl not in w1: checks=checks+1
            stop=checks==clds
            for tvl in vls:
                if G[nodes[svl]][nodes[tvl]]!=0 or G[nodes[tvl]][nodes[svl]]!=0: stop=False
            if not stop: vls.clear()
        if len(w1)>1:
            stop=False
            while not stop:
                s1=svl
                while s1==svl:
                    s1,t1=choice(w1),choice(vls)
                if G[nodes[s1]][nodes[t1]]==0 and G[nodes[t1]][nodes[s1]]==0: stop=True
            w=randint(-10,20)
            if w==0: w=21
            G[nodes[s1]][nodes[t1]]=w
        for tvl in vls:
            w=randint(-10,20)
            if w==0: w=21
            G[nodes[svl]][nodes[tvl]]=w
            if tvl not in w1: w1.append(tvl)
        svl=vls[0]
        if len(w1)==len(nodes): return G

def GenerateRandomCompleteUndirectedGraph(nodes:list) -> dict:
    G={n1:{n2:0 for n2 in nodes} for n1 in nodes}
    for u in G:
        for v in G[u]:
            ui,vi=nodes.index(u),nodes.index(v)
            if vi>ui: G[u][v]=G[v][u]=randint(0,20)
    return G

class StringPL:
    def __init__(self, s: str, p: int=0, l: int=None):
        self.s, self.p = s, p
        if l is None: self.l = len(s)
        else: self.l = l
        if self.s is not None and self.p>len(self.s): raise 'cloning position greater than the length of the string'
        if self.s is not None and self.p+self.l>len(self.s): raise 'cloning position+length greater than the length of the string'
    def substring(self) -> (str):
        return self.s[self.p:self.p + self.l]
    def indexSubstring(self, tp:int, tl:int) -> (str):
        return self.s[tp:tp+tl]
    def clone(self, tp:int, tl:int) -> (object):
        if self.s is None: return StringPL(None, 0, 0)
        if tp>len(self.s): raise 'cloning position greater than the length of the string'
        if tp+tl>len(self.s): raise 'cloning position+length greater than the length of the string'
        return StringPL(self.s, tp, tl)
    def __getitem__(self, item):
        if self.s is None: return None
        if isinstance(item, slice):
            return self.s[item.start+self.p:item.stop+self.p]
        else:
            if item>len(self): raise 'item position greater than the length of the string'
            return self.substring()[item]
    def __len__(self):
        return self.l
    def __str__(self):
        return self.substring()
    def __lt__(self, other):
        return self.substring() < other.substring()
    def lcp(self, other):
        l = 0
        ss = self.substring()
        if isinstance(other, str): os = other
        else: os = other.substring()
        for i in range(0, min(len(ss), len(os))):
            if ss[i] == os[i]:
                l = l + 1
            else:
                break
        return l
    def append(self, c: chr):
        self.s = self.s + c
        self.l = self.l + 1


import numpy as np
from matplotlib import pyplot as plt

nClasses = 3
nInpDim = 2
hiddenLayerSize = 4
nSamplesPerClass = 500

def classification_rate(Y, P):
	return (Y==P).tolist().count(True) * 1.0 / len(Y)

def generateData():
	X = np.array([])
	Y = np.array([])

	centroid = [ (-2,3), (2,2), (0,-1) ]

	for i in range(nClasses):
		x = np.random.randn(nSamplesPerClass, nInpDim) + np.array([ centroid[i][0], centroid[i][1] ])
		X = x if len(X)==0 else np.vstack( [X,x] )

		y = np.zeros( (nSamplesPerClass,nClasses), dtype=np.int )
		y[:,i] = 1
		Y = y if len(Y)==0 else np.vstack( [Y,y] )

	return X, Y

def softmax( x ):
	exp_x = np.exp(x)
	return exp_x / exp_x.sum(axis=1, keepdims=True)

def sigmoid( x ):
	return 1 / ( 1 + np.exp(-x) )

def gradient_sigmoid(x):
	return x * ( 1-x )

def main():
	X, T = generateData()
	t = np.array([0]*nSamplesPerClass+[1]*nSamplesPerClass+[2]*nSamplesPerClass)
	plt.scatter( X[:,0], X[:,1], c=t, alpha=0.5, s=100 )
	plt.show()

	# Add bias unit to X
	X = np.hstack([ np.ones((X.shape[0],1), dtype=np.float), X ])

	w_ji = np.random.randn( X.shape[1], hiddenLayerSize + 1 )	# +1 for bias in hidden layer
	w_kj = np.random.randn( hiddenLayerSize + 1, nClasses )		# +1 for bias in hidden layer

	learningRate = 10e-7
	costs = []

	for i in range(50000):
		net_ji = np.dot( X, w_ji )
		Y = sigmoid( net_ji )
		net_kj = np.dot( Y, w_kj )
		Z = softmax( net_kj )
		delta_w_kj = learningRate * Y.T.dot(T-Z)
		delta_w_ji = learningRate * X.T.dot( gradient_sigmoid(Y) * np.dot(T-Z, w_kj.T) )
		w_kj += delta_w_kj
		w_ji += delta_w_ji

		z = np.argmax( Z, axis=1 )
		cost = np.power(t-z ,2).sum()

		if i%100==0:
			costs.append( cost )
			cl_rate = classification_rate(t,z)
			print( "{}. Cost:{}  classification_rate:{}".format(i, cost,cl_rate) )
		if cost <= 10e-7:
			break

	plt.plot(costs)
	plt.show()

if __name__ == '__main__':
	main()


from math import pi, log
from functools import wraps

import torch
from torch import nn, einsum
import torch.nn.functional as F

from einops import rearrange, repeat

# helpers
def exists(val):
    return val is not None

def default(val, d):
    return val if exists(val) else d

def cache_fn(f):
    cache = None
    @wraps(f)
    def cached_fn(*args, _cache = True, **kwargs):
        if not _cache:
            return f(*args, **kwargs)
        nonlocal cache
        if cache is not None:
            return cache
        cache = f(*args, **kwargs)
        return cache
    return cached_fn

# helper classes

class PreNorm(nn.Module):
    def __init__(self, dim, fn, context_dim = None):
        super().__init__()
        self.fn = fn
        self.norm = nn.LayerNorm(dim)
        self.norm_context = nn.LayerNorm(context_dim) if exists(context_dim) else None

    '''
        x : 表示中间隐式数组
        kwargs : 多余参数
            'context' - 表示原数组
    '''
    def forward(self, x, **kwargs):
        x = self.norm(x)
        # 判断是否存在'context'，若不存在，则说明是self-attention，若存在，说明是cross-attention
        if exists(self.norm_context):
            context = kwargs['context']
            normed_context = self.norm_context(context)
            kwargs.update(context = normed_context)

        return self.fn(x, **kwargs)

class GEGLU(nn.Module):
    def forward(self, x):
        x, gates = x.chunk(2, dim = -1)
        return x * F.gelu(gates)

class FeedForward(nn.Module):
    def __init__(self, dim, mult = 4):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(dim, dim * mult * 2),
            GEGLU(),
            nn.Linear(dim * mult, dim)
        )

    def forward(self, x):
        return self.net(x)

class Attention(nn.Module):
    def __init__(self, query_dim, context_dim = None, heads = 8, dim_head = 64):
        super().__init__()
        inner_dim = dim_head * heads
        context_dim = default(context_dim, query_dim)
        self.scale = dim_head ** -0.5
        self.heads = heads
        self.to_q = nn.Linear(query_dim, inner_dim, bias = False)
        self.to_kv = nn.Linear(context_dim, inner_dim * 2, bias = False)
        self.to_out = nn.Linear(inner_dim, context_dim)

    def forward(self, x, context = None, mask = None):
        h = self.heads
        q = self.to_q(x)
        context = default(context, x)
        temp = self.to_kv(context)
        k,v = temp.chunk(2, dim = -1)
        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h = h), (q, k, v))

        sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
        attn = sim.softmax(dim = -1)
        out = einsum('b i j, b j d -> b i d', attn, v)
        out = rearrange(out, '(b h) n d -> b n (h d)', h = h)
        return self.to_out(out)

# Local Convolutional Position Encoding
class ConvPosEnc(nn.Module):
    def __init__(self, dim_q, dim_content, k=3):
        super(ConvPosEnc, self).__init__()
        self.proj_q = nn.Conv1d(
            in_channels=dim_q,
            out_channels=dim_q,
            kernel_size=k,
            stride=1,
            padding=k//2,
            groups=dim_q
        )

        self.proj_content = nn.Conv1d(
            in_channels=dim_content,
            out_channels=dim_content,
            kernel_size=k,
            stride=1,
            padding=k // 2,
            groups=dim_content
        )

    def forward(self,q,content):
        q = q.permute(0,2,1)
        q = self.proj_q(q) + q
        q = q.permute(0,2,1)

        # B,C,H,W = content.shape
        content = content.permute(0, 2, 1)
        content = self.proj_content(content) + content
        content = content.permute(0,2,1)

        return q,content

# main class
class FusionLayer(nn.Module):
    def __init__(
        self,
        depth,                                  #
        dim,                                    # image dim
        latent_dim = 512,                       # PC dim
        cross_heads = 1,                        # Encoder
        latent_heads = 8,                       # Process
        cross_dim_head = 64,                    # Encoder和Decoder
        latent_dim_head = 64,                   # Process
        weight_tie_layers = False,
        pe=False
    ):
        super().__init__()

        self.pe = pe
        if(pe):
            # position encoding
            self.cpe = ConvPosEnc(
                dim_q=latent_dim,
                dim_content=dim
            )

        self.cross_attend_blocks = nn.ModuleList([
            PreNorm(latent_dim, Attention(latent_dim, dim, heads = cross_heads, dim_head = cross_dim_head), context_dim = dim),
            PreNorm(latent_dim, FeedForward(latent_dim))
        ])

        get_latent_attn = lambda: PreNorm(latent_dim, Attention(latent_dim, heads = latent_heads, dim_head = latent_dim_head))
        get_latent_ff = lambda: PreNorm(latent_dim, FeedForward(latent_dim))
        get_latent_attn, get_latent_ff = map(cache_fn, (get_latent_attn, get_latent_ff))

        self.layers = nn.ModuleList([])
        cache_args = {'_cache': weight_tie_layers}

        for i in range(depth):
            self.layers.append(nn.ModuleList([
                get_latent_attn(**cache_args),
                get_latent_ff(**cache_args)
            ]))

    def forward(
        self,
        data,                           # Image
        mask = None,
        queries_encoder = None,         # PointCloud
    ):
        b, *_, device = *data.shape, data.device
        x = queries_encoder

        # lcpe
        if(self.pe):
            x,data = self.cpe(
                q=x,
                content=data,
            )

        # ---- Cross-Attention ----
        cross_attn, cross_ff = self.cross_attend_blocks
        x = cross_attn(x, context = data, mask = mask) + x
        x = cross_ff(x) + x
        # ---- Cross-Attention ----

        #  ---- Self-Attention ----
        for self_attn, self_ff in self.layers:
            x = self_attn(x) + x
            x = self_ff(x) + x
        # ---- Self-Attention ----

        return x


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import numpy as np
import pandas as pd

from ..utils import groupby_apply
from ..utils.exceptions import GgplotError
from ..scales.scale import scale_discrete
from .stat_summary import make_summary_fun
from .stat_bin_2d import bin_breaks
from .stat import stat

class stat_summary_bin(stat):
    REQUIRED_AES = {'x', 'y'}
    DEFAULT_PARAMS = {'geom': 'pointrange', 'position': 'identity',
                      'bins': 30, 'breaks': None, 'binwidth': None,
                      'origin': None, 'fun_data': None, 'fun_y': None,
                      'fun_ymin': None, 'fun_ymax': None,
                      'fun_args': dict()}

    def setup_params(self, data):
        keys = ('fun_data', 'fun_y', 'fun_ymin', 'fun_ymax')
        if not any(self.params[k] for k in keys):
            raise GgplotError('No summary function')

        return self.params

    @classmethod
    def compute_group(cls, data, scales, **params):
        bins = params['bins']
        breaks = params['breaks']
        binwidth = params['binwidth']
        origin = params['origin']

        func = make_summary_fun(params['fun_data'], params['fun_y'],
                                params['fun_ymin'], params['fun_ymax'],
                                params['fun_args'])

        breaks = bin_breaks(scales.x, breaks, origin, binwidth, bins)
        data['bin'] = pd.cut(data['x'], bins=breaks, labels=False,
                             include_lowest=True)

        def func_wrapper(data):
            res = func(data)
            res['bin'] = data['bin'].iloc[0]
            return res

        # This is a plyr::ddply
        out = groupby_apply(data, 'bin', func_wrapper)
        bin_centers = ((breaks[:-1] + breaks[1:])/2)[out['bin'].values]
        out['x'] = bin_centers
        out['bin'] += 1
        if isinstance(scales.x, scale_discrete):
            out['width'] = 0.9
        else:
            out['width'] = np.diff(breaks)[bins]

        return out


import kerastuner as kt
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_datasets as tfds
import os
import IPython
from kerastuner import HyperParameters

from absl import flags
from absl import logging
from absl import app

FLAGS = flags.FLAGS
# flag name, default value, explanation/help.
tf.compat.v1.flags.DEFINE_string('model_dir', 'default_model_dir', 'Directory or bucket for storing checkpoint model.')
tf.compat.v1.flags.DEFINE_bool('fine_tuning_choice', False, 'Retrain base parameters')
tf.compat.v1.flags.DEFINE_integer('train_batch_size', 32, 'Number of samples in a training batch')
tf.compat.v1.flags.DEFINE_integer('validation_batch_size', 40, 'Number of samples in a validation batch')
tf.compat.v1.flags.DEFINE_string('distribution_strategy', 'tpu', 'Distribution strategy for training.')
tf.compat.v1.flags.DEFINE_integer('train_epochs', 3, 'Number of epochs for training')
tf.compat.v1.flags.DEFINE_string('data_dir', 'gs://image-flowers/flower_photos/train', 'Location of training data')
tf.compat.v1.flags.DEFINE_integer('num_gpus', 4, 'Numer of GPU per worker')
tf.compat.v1.flags.DEFINE_string('tuner_type', 'Hyperband', 'Type of tuner. Default is hyperband')

def setup_keras_tuner_config():
    if 'TF_CONFIG' in os.environ:
        try:
            tf_config = json.loads(os.environ['TF_CONFIG'])
            cluster = tf_config['cluster']
            task = tf_config['task']
            chief_addr = cluster['chief'][0].split(':')
            chief_ip = socket.gethostbyname(chief_addr[0])
            chief_port = chief_addr[1]
            os.environ['KERASTUNER_ORACLE_IP'] = chief_ip
            #os.environ['KERASTUNER_ORACLE_IP'] = '0.0.0.0'
            os.environ['KERASTUNER_ORACLE_PORT'] = chief_port
            if task['type'] == 'chief':
                os.environ['KERASTUNER_TUNER_ID'] = 'chief'
            else:
                os.environ['KERASTUNER_TUNER_ID'] = 'tuner{}'.format(task['index'])

            print('set following environment arguments:')
            print('KERASTUNER_ORACLE_IP: %s' % os.environ['KERASTUNER_ORACLE_IP'])
            print('KERASTUNER_ORACLE_PORT: %s' % os.environ['KERASTUNER_ORACLE_PORT'])
            print('KERASTUNER_TUNER_ID: %s' % os.environ['KERASTUNER_TUNER_ID'])
        except Exception as ex:
            print('Error setting up keras tuner config: %s' % str(ex))

def model_builder(hp):
    os.environ["TFHUB_CACHE_DIR"] = "gs://ai-tpu-experiment/model-cache-dir/imagenet_resnet_v2_50_feature_vector_4"

    hp_units = hp.Int('units', min_value = 64, max_value = 256, step = 64)
    hp_activation = hp.Choice('dense_activation',
        values=['relu', 'tanh', 'sigmoid'])

    IMAGE_SIZE = (224, 224)
    model = tf.keras.Sequential([
    tf.keras.layers.InputLayer(input_shape=IMAGE_SIZE + (3,)),
    hub.KerasLayer("https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/4", trainable=False),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(units = hp_units, activation = hp_activation, kernel_initializer='glorot_uniform'),
    tf.keras.layers.Dense(5, activation='softmax', name = 'custom_class')
    ])

    model.build([None, 224, 224, 3])

    #hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-4])
    #hp_optimizer = hp.Choice('selected_optimizer', ['sgd', 'adam'])
    model.compile(
        optimizer=tf.keras.optimizers.SGD(lr=1e-2, momentum=0.5),
        #optimizer=hp_optimizer,
        loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
        metrics=['accuracy'])

    return model

class ClearTrainingOutput(tf.keras.callbacks.Callback):
        def on_train_end(*args, **kwargs):
            IPython.display.clear_output(wait = True)

def decode_and_resize(serialized_example):
    # resized image should be [224, 224, 3] and normalized to value range [0, 255]
    # label is integer index of class.

    parsed_features = tf.io.parse_single_example(
    serialized_example,
    features = {
    'image/channels' :  tf.io.FixedLenFeature([], tf.int64),
    'image/class/label' :  tf.io.FixedLenFeature([], tf.int64),
    'image/class/text' : tf.io.FixedLenFeature([], tf.string),
    'image/colorspace' : tf.io.FixedLenFeature([], tf.string),
    'image/encoded' : tf.io.FixedLenFeature([], tf.string),
    'image/filename' : tf.io.FixedLenFeature([], tf.string),
    'image/format' : tf.io.FixedLenFeature([], tf.string),
    'image/height' : tf.io.FixedLenFeature([], tf.int64),
    'image/width' : tf.io.FixedLenFeature([], tf.int64)
    })
    image = tf.io.decode_jpeg(parsed_features['image/encoded'], channels=3)
    label = tf.cast(parsed_features['image/class/label'], tf.int32)
    label_txt = tf.cast(parsed_features['image/class/text'], tf.string)
    label_one_hot = tf.one_hot(label, depth = 5)
    resized_image = tf.image.resize(image, [224, 224], method='nearest')
    return resized_image, label_one_hot

def normalize(image, label):
    #Convert `image` from [0, 255] -> [0, 1.0] floats
    image = tf.cast(image, tf.float32) / 255. + 0.5
    return image, label

def prepare_for_training(ds, cache=True, shuffle_buffer_size=1000):
    # This is a small dataset, only load it once, and keep it in memory.
    # use `.cache(filename)` to cache preprocessing work for datasets that don't
    # fit in memory.
    flags_obj = flags.FLAGS
    if cache:
        if isinstance(cache, str):
            ds = ds.cache(cache)
        else:
            ds = ds.cache()

    ds = ds.shuffle(buffer_size=shuffle_buffer_size)
    # Repeat forever
    ds = ds.repeat()
    ds = ds.batch(flags_obj.train_batch_size)
    # `prefetch` lets the dataset fetch batches in the background while the model
    # is training.
    AUTOTUNE = tf.data.experimental.AUTOTUNE
    ds = ds.prefetch(buffer_size=AUTOTUNE)
    return ds

def main(_):
    flags_obj = flags.FLAGS

    setup_keras_tuner_config()

    if flags_obj.distribution_strategy == 'tpu':
        resolver = tf.distribute.cluster_resolver.TPUClusterResolver()
        tf.config.experimental_connect_to_cluster(resolver)
        tf.tpu.experimental.initialize_tpu_system(resolver)
        strategy = tf.distribute.experimental.TPUStrategy(resolver)
        strategy_scope = strategy.scope()
        print("All devices: ", tf.config.list_logical_devices('TPU'))
    elif flags_obj.distribution_strategy == 'gpu':
        strategy = tf.distribute.MirroredStrategy()
        strategy_scope = strategy.scope()
        devices = ["device:GPU:%d" % i for i in range(flags_obj.num_gpus)]

    print('NUMBER OF DEVICES: ', strategy.num_replicas_in_sync)

    ## identify data paths and sources
    root_dir = flags_obj.data_dir # this is gs://<bucket>/folder where tfrecord are stored
    file_pattern = "{}/image_classification_builder-train*.tfrecord*".format(root_dir)
    val_file_pattern = "{}/image_classification_builder-validation*.tfrecord*".format(root_dir)

    file_list = tf.io.gfile.glob(file_pattern)
    all_files = tf.data.Dataset.list_files( tf.io.gfile.glob(file_pattern))

    val_file_list = tf.io.gfile.glob(val_file_pattern)
    val_all_files = tf.data.Dataset.list_files( tf.io.gfile.glob(val_file_pattern))

    train_all_ds = tf.data.TFRecordDataset(all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)
    val_all_ds = tf.data.TFRecordDataset(val_all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)

    # perform data engineering
    dataset = train_all_ds.map(decode_and_resize)
    val_dataset = val_all_ds.map(decode_and_resize)

    #
    BATCH_SIZE = flags_obj.train_batch_size
    VALIDATION_BATCH_SIZE = flags_obj.validation_batch_size
    dataset = dataset.map(normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    val_dataset = val_dataset.map(normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)

    val_ds = val_dataset.batch(VALIDATION_BATCH_SIZE)

    AUTOTUNE = tf.data.experimental.AUTOTUNE
    train_ds = prepare_for_training(dataset)

    FINE_TUNING_CHOICE = True
    NUM_CLASSES = 5
    IMAGE_SIZE = (224, 224)

    train_sample_size=0
    for raw_record in train_all_ds:
        train_sample_size += 1
    print('TRAIN_SAMPLE_SIZE = ', train_sample_size)
    validation_sample_size=0
    for raw_record in val_all_ds:
        validation_sample_size += 1
    print('VALIDATION_SAMPLE_SIZE = ', validation_sample_size)

    STEPS_PER_EPOCHS = train_sample_size // BATCH_SIZE
    VALIDATION_STEPS = validation_sample_size // VALIDATION_BATCH_SIZE

    """Runs the hyperparameter search."""
    if(flags_obj.tuner_type.lower() == 'BayesianOptimization'.lower()):
        tuner = kt.BayesianOptimization(
            hypermodel = model_builder,
            objective ='val_accuracy',
            tune_new_entries = True,
            allow_new_entries = True,
            max_trials = 5,
            directory = flags_obj.model_dir,
            project_name = 'hp_tune_bo',
            overwrite = True
            )
    elif (flags_obj.tuner_type.lower() == 'RandomSearch'.lower()):
        tuner = kt.RandomSearch(
            hypermodel = model_builder,
            objective='val_accuracy',
            tune_new_entries = True,
            allow_new_entries = True,
            max_trials = 5,
            directory = flags_obj.model_dir,
            project_name = 'hp_tune_rs',
            overwrite = True)
    else:
        tuner = kt.Hyperband(
            hypermodel = model_builder,
            objective = 'val_accuracy',
            max_epochs = 3,
            factor = 2,
            distribution_strategy=strategy,
            directory = flags_obj.model_dir,
            project_name = 'hp_tune_hb',
            overwrite = True)

    tuner.search(train_ds,
        steps_per_epoch=STEPS_PER_EPOCHS,
        validation_data=val_ds,
        validation_steps=VALIDATION_STEPS,
        epochs=3,
        callbacks=[tf.keras.callbacks.EarlyStopping('val_accuracy'), ClearTrainingOutput()])

    # Get the optimal hyperparameters
    best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]

    print(f"""
        The hyperparameter search is done.
        The best number of nodes in the dense layer is {best_hps.get('units')}.
        The best activation function in mid dense layer is {best_hps.get('dense_activation')}.
        """)

    # Build the model with the optimal hyperparameters and train it on the data
    model = tuner.hypermodel.build(best_hps)
    checkpoint_prefix = os.path.join(flags_obj.model_dir, "best_hp_train_ckpt_{epoch}")
    callbacks = [
    tf.keras.callbacks.TensorBoard(log_dir=os.path.join(flags_obj.model_dir, 'tensorboard_logs')),
    tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
                                       save_weights_only=True)]

    model.fit(
        train_ds,
        epochs=3, steps_per_epoch=STEPS_PER_EPOCHS,
        validation_data=val_ds,
        validation_steps=VALIDATION_STEPS,
        callbacks=callbacks)

    logging.info('INSIDE MAIN FUNCTION user input model_dir %s', flags_obj.model_dir)
    # Save model trained with chosen HP in user specified bucket location
    model_save_dir = os.path.join(flags_obj.model_dir, 'best_save_model')
    model.save(model_save_dir)

if __name__ == '__main__':
    app.run(main)


#!/usr/bin/python
import json
import foursquare
from chouse.metwit import *
from chouse.conf import *
from django.utils.encoding import smart_str
from hashlib import sha1

class Poi(object):
    def __init__(self, lat, lng):
        self.database = None
        self.id = sha1("%s%s" % (lat, lng)).hexdigest()

        self.coordinates = {"lat": lat, "lng": lng}
        self.cap = str()
        self.street = str()
        self.photos = list()
        self.price = int()

        self.title = str()
        self.description = str()
        self.link = str()
        self.energetic_class = str() # a - g

        self.green = int() # Value of "green" 1-100
        self.analysis_data = dict() # Output of analysis.

    def get_foursquare_vote(self, fsq):
        venues = fsq.venues.search({
                                    "ll":"%s,%s" % (self.coordinates['lat'], self.coordinates['lng']),
                                    "limit": 50,
                                    "radius": "800",
                                    "intent": "browse",
                                    })['venues']
        green = 25
        times = 0

        near_place = dict()
        near_place['location'] = dict()
        near_place['location']['distance'] = 1500
        near_place['referralId'] = None
        near_place['name'] = ""

        near_station = dict()
        near_station['location'] = dict()
        near_station['location']['distance'] = 1500
        near_station['referralId'] = None
        near_station['name'] = ""

        near_outdoor = dict()
        near_outdoor['location'] = dict()
        near_outdoor['location']['distance'] = 1500
        near_outdoor['referralId'] = None
        near_outdoor['name'] = ""

        near_plaza = dict()
        near_plaza['location'] = dict()
        near_plaza['location']['distance'] = 1500
        near_plaza['referralId'] = None
        near_plaza['name'] = ""

        for venue in venues:
            is_green = False
            for category in venue['categories']:
                s_category = smart_str(category['shortName']).lower()

                if "field" in s_category or "outdoor" in s_category: # EX: soccer field or outdoor
                    if int(near_outdoor['location']['distance']) > int(venue['location']['distance']):
                        self.analysis_data["Outdoor place"] = "%s ( %s  %sm )" % ( venue['name'], s_category, venue['location']['distance'] )
                        near_outdoor = venue
                    green += 80
                    times += 1
                    is_green = True

                elif "historyc" in s_category or "fountain" in s_category:
                    green += 60
                    times += 1
                    is_green = True

                elif "plaza" in s_category:
                    green += 60
                    times += 1
                    if int(near_plaza['location']['distance']) > int(venue['location']['distance']):
                        self.analysis_data["Plaza"] = "%s at %sm " % (venue['name'],  venue['location']['distance'])
                        near_outdoor = venue
                    is_green = True

                elif "station" in s_category or "bus" in s_category:
                    green += 50
                    times += 1
                    if int(near_station['location']['distance']) > int(venue['location']['distance']):
                        self.analysis_data["Public Transport"] = "%s at %sm - %s" % ( s_category, venue['location']['distance'], venue['name'] )
                        near_station = venue

                elif "subway" in s_category or "metro" in s_category.lower():
                    green += 50
                    times += 1
                    if int(near_station['location']['distance']) > int(venue['location']['distance']):
                        self.analysis_data["Public Transport"] = "%sm to %s" % ( venue['location']['distance'], venue['name'] )
                        near_station = venue

                elif "school" in s_category or "univer" in s_category.lower():
                    green += 40
                    times += 1

                elif "hospital" in s_category:
                    green += 30
                    times += 1

            if is_green:
                if int(near_place['location']['distance']) > int(venue['location']['distance']):
                    if venue["referralId"] != near_station['referralId'] and venue['referralId'] != near_outdoor['referralId'] and venue['referralID'] != near_plaza['referralId']:
                        near_place = venue

        if near_place['referralId']:
            self.analysis_data['Other'] = "%s at %sm: %s" % ( smart_str(near_place['categories'][0]['shortName']), near_place['location']['distance'], near_place['name'] )

        return int(green/times) # Media

    def get_nox_by_cap(self):
        if self.cap == "00145":
            self.analysis_data['Pollution level ( N02 )'] = "196 mcg/m3 ( Risky )"
            return 62
        elif self.cap == "00185":
            self.analysis_data['Pollution level ( N02 )'] = "110 mcg/m3 ( Average )"
            return 62

    def get_pm10_by_cap(self):
        if self.cap == "00145":
            self.analysis_data['Pollution level ( PM10 )'] = "33 mcg/m3 ( Acceptable )"
            return 30
        elif self.cap == "00185":
            self.analysis_data['Pollution level ( PM10 )'] = "37 mcg/m3 ( Acceptable )"
            return 20

    def get_grass_dencity_by_cap(self):
        if self.cap == "00145":
            self.analysis_data['Green density'] = "62,2% ( Above average )"
            self.analysis_data['Green per person'] = "216,7 m3 ( Into the wild )"
            return 62
        elif self.cap == "00185":
            self.analysis_data['Green density'] = "9,2% ( Very poor )"
            self.analysis_data['Green per person'] = "10 m3 ( Low )"
            return 9

    def get_value_by_energetic_class(self):
        self.analysis_data["Building Energetic Class"] = "%s" % self.energetic_class.upper()
        if self.energetic_class == "a":
            return 90
        elif self.energetic_class == "b":
            return 80
        elif self.energetic_class == "c":
            return 70
        elif self.energetic_class == "d":
            return 60
        elif self.energetic_class == "e":
            return 50
        elif self.energetic_class == "f":
            return 40
        elif self.energetic_class == "g":
            return 30
        else:
            return 20

    def calculate_value(self):
        self.green = 0
        total = 0

        foursquare_client = foursquare.Foursquare(
                client_id=FOURSQUARE_API_CLIENT_ID,
                client_secret=FOURSQUARE_API_CLIENT_SECRET)

        total += self.get_foursquare_vote(foursquare_client)
        total += self.get_nox_by_cap()
        total += self.get_pm10_by_cap()
        total += self.get_grass_dencity_by_cap()
        total += self.get_value_by_energetic_class()

        self.green = int(total/5)
        return self.green

    def load(self):
        search = self.database.houses.find_one({"id": self.id})
        if search:
           self.by_dictionary(search)
        return self

    def by_dictionary(self, dictionary):
        self.id = dictionary['id']

        self.coordinates = dictionary['coordinates']
        self.cap = dictionary['cap']
        self.street = dictionary['street']
        self.photos = dictionary['photos']
        self.price = int(dictionary['price'])

        self.title = dictionary['title']
        self.description = dictionary['description']
        self.link = dictionary['link']
        self.energetic_class = dictionary['energetic_class']

        self.green = int(dictionary['green'])
        self.analysis_data = dictionary['analysis_data']
        return self

    def save(self):
        search = self.database.houses.find_one({"id": self.id})
        dictionary = self.__dict__(old=search)
        self.database.houses.save(dictionary)

        return

    def __dict__(self, old=None):
        if not old:
            old = dict()

        if not self.id:
            self.id = sha1("%s%s" % ( self.coordinates['lat'], self.coordinates['lng'])).hexdigest()

        if not old.has_key("id"):
            old['id'] = self.id

        old['coordinates'] = self.coordinates
        old['cap'] = self.cap
        old['street'] = self.street
        old['photos'] = self.photos
        old['price'] = self.price

        old['title'] = self.title
        old['description'] = self.description
        old['link'] = self.link
        old['energetic_class'] = self.energetic_class

        old['green'] = self.green
        old['analysis_data'] = self.analysis_data

        return old


from sklearn import *
import sklearn
import numpy as np
import pandas as pd
import re

from multiprocessing import Pool, cpu_count
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
from nltk.tokenize import RegexpTokenizer
#import nltk
#nltk.download()

try:
    cpus = cpu_count()
except NotImplementedError:
    cpus = 2   # arbitrary default

paper_stop_words = [ "fig", "figure", "et", "al", "table",
            "data", "analysis", "analyze", "study",
            "method", "result", "conclusion", "author",
            "find", "found", "show", "perform",
            "demonstrate", "evaluate", "discuss"]

def clean_str(s):
    """Cleaning text """
    # s = re.sub("(\(\d*\))","",s)
    # s = re.sub(r"\/"," ",s)
    # s = re.sub(r">"," ",s)
    # s = re.sub(r"<"," ",s)
    # s = re.sub(r"-"," ",s)
    # s = re.sub(r"–"," ",s)
    # s = re.sub(r"\'s", " 's", s)
    # s = re.sub(r"\'ve", " 've", s)
    # s = re.sub(r"n\'t", " n't", s)
    # s = re.sub(r"\'re", " 're", s)
    # s = re.sub(r"\'d", " 'd", s)
    # s = re.sub(r"\'ll", " 'll", s)
    # s = re.sub(r" a "," ",s)
    # s = re.sub(r" the "," ",s)
    # s = re.sub(r"(\(Fig\..*\))"," ",s)
    # s = re.sub(r"(\(Table\..*\))"," ",s)
    # s = re.sub(r"Fig.\S+ "," ",s)
    # s = re.sub(r"Table "," ",s)
    # s = re.sub(r", "," ",s)
    # s = re.sub(r"[().?\[\]!;:+]", " ", s)
    # s = re.sub(r" '\S+", "", s)
    # s = re.sub(' +',' ',s)
    s = re.sub(r" \d*", " ", s)
    tokenizer = RegexpTokenizer(r'\w+')
    words = tokenizer.tokenize(s.lower())
    stemmer = SnowballStemmer("porter")
    words = [w for w in words if not w in stopwords.words("english")]
    words = [w for w in words if not w in paper_stop_words]
    words = [stemmer.stem(w) for w in words]
    s = " ".join(words)
    return s

def load_data_and_labels(variants_file,text_file):
    """Load text and Classes"""
    var = pd.read_csv(variants_file)
    txt = pd.read_csv(text_file, sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])
    txt_var = pd.merge(var, txt, how='left', on='ID')

    for index, row in txt_var.iterrows():
        if len(row['Text']) < 10:
            txt_var.drop(index,inplace=True)

    x_text = list(txt_var['Text'])
    print ("Cleaning data ...\n")

    # x_text = [clean_str(sent) for sent in x_text]

    pool = Pool(processes=cpus)
    x_text = pool.map(clean_str, x_text)
    pool.close()
    pool.join()

    # Map the actual classes to one hot labels
    try:
        labels = sorted(list(set(txt_var['Class'].tolist())))
        one_hot = np.zeros((len(labels), len(labels)), int)
        np.fill_diagonal(one_hot, 1)
        label_dict = dict(zip(labels, one_hot))
        y = txt_var['Class'].apply(lambda y: label_dict[y]).tolist()
    except:
        y = None
        labels = None
    return [x_text, y, labels]

class cust_regression_vals(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
    def fit(self, x, y=None):
        return self
    def transform(self, x):
        x = x.drop(['Gene', 'Variation','ID','Text'],axis=1).values
        return x

class cust_txt_col(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
    def __init__(self, key):
        self.key = key
    def fit(self, x, y=None):
        return self
    def transform(self, x):
        return x[self.key].apply(str)

def gene_related_features(variants_file,text_file):

    var = pd.read_csv(variants_file)
    txt = pd.read_csv(text_file, sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])

    train = pd.merge(var, txt, how='left', on='ID')

    for index, row in train.iterrows():
        if len(row['Text']) < 10:
            train.drop(index,inplace=True)

    train = train.drop(['Class'], axis=1)
    train['Gene_Share'] = train.apply(lambda r: sum([1 for w in r['Gene'].split(' ') if w in r['Text'].split(' ')]), axis=1)
    train['Variation_Share'] = train.apply(lambda r: sum([1 for w in r['Variation'].split(' ') if w in r['Text'].split(' ')]), axis=1)

    for i in range(56):
        train['Gene_'+str(i)] = train['Gene'].map(lambda x: str(x[i]) if len(x)>i else '')
        train['Variation'+str(i)] = train['Variation'].map(lambda x: str(x[i]) if len(x)>i else '')

    gen_var_lst = sorted(list(train.Gene.unique()) + list(train.Variation.unique()))
    print(len(gen_var_lst))
    gen_var_lst = [x for x in gen_var_lst if len(x.split(' '))==1]
    print(len(gen_var_lst))
    i_ = 0

    for gen_var_lst_itm in gen_var_lst:
       if i_ % 100 == 0: print(i_)
       train['GV_'+str(gen_var_lst_itm)] = train['Text'].map(lambda x: str(x).count(str(gen_var_lst_itm)))
       i_ += 1

    for c in train.columns:
        if train[c].dtype == 'object':
            if c in ['Gene','Variation']:
                lbl = preprocessing.LabelEncoder()
                train[c+'_lbl_enc'] = lbl.fit_transform(train[c].values)
                train[c+'_len'] = train[c].map(lambda x: len(str(x)))
                train[c+'_words'] = train[c].map(lambda x: len(str(x).split(' ')))
            elif c != 'Text':
                lbl = preprocessing.LabelEncoder()
                train[c] = lbl.fit_transform(train[c].values)
            if c=='Text':
                train[c+'_len'] = train[c].map(lambda x: len(str(x)))
                train[c+'_words'] = train[c].map(lambda x: len(str(x).split(' ')))

    # print('Pipeline...')

    # #Defining Transformers:

    # count_Gene  = feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))
    # tsvd1 = decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12)
    # count_Variation = feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))
    # tsvd2 = decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12)

    # pi1 = pipeline.Pipeline([('Gene', cust_txt_col('Gene')), ('count_Gene', count_Gene),('tsvd1',tsvd1)])
    # pi2 = pipeline.Pipeline([('Variation', cust_txt_col('Variation')), ('count_Variation',count_Variation), ('tsvd2',tsvd2)])
    # # pi3 = pipeline.Pipeline([('Text', cust_txt_col('Text')), ('tfidf_Text', feature_extraction.text.TfidfVectorizer(ngram_range=(1, 2))), ('tsvd3', decomposition.TruncatedSVD(n_components=50, n_iter=25, random_state=12))])

    # fp = pipeline.Pipeline([
    #     ('union', pipeline.FeatureUnion(
    #         n_jobs = -1,
    #         transformer_list = [
    #             ('standard', cust_regression_vals()),
    #             ('pi1', pi1),
    #             ('pi2', pi2),
    #             #commented for Kaggle Limits
    #             #('pi3', pi3)
    #         ])
    #     )])

    # train = fp.fit_transform(train).astype(int)
    # print(train.shape)

    return train

def batch_iter(data, batch_size, num_epochs, shuffle=True):
    """Iterate the data batch by batch"""

    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int((len(data)-1)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        if shuffle:
            shuffle_indices = np.random.permutation(np.arange(data_size))
            shuffled_data = data[shuffle_indices]
        else:
            shuffled_data = data
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]