# Elmo
import tensorflow_hub as hub
import tensorflow as tf

# GPT2
import numpy as np
import torch
from pytorch_pretrained_bert import GPT2Tokenizer, GPT2Model, GPT2LMHeadModel

from bert_serving.client import BertClient
from gensen.gensen import GenSen, GenSenSingle

#ELMO
elmo = None
graph = None
sess = None
# Glove
embeddings_index = None

# GPT2
tokenizer = None
model = None

# Bert
bc = None

# GPT2
tokenizer = None
model = None

# Bert
bc = None

#Gensen

gensen_1 = None
gensen_2 = None

def getGensenEmb(batch,needPad = False,padLength = 30):
	global gensen_1
	global gensen_2

	if gensen_1 == None:
		gensen_1 = GenSenSingle(model_folder='gensen/data/models',filename_prefix='nli_large_bothskip',pretrained_emb='gensen/data/embedding/glove.840B.300d.h5')
		gensen_2 = GenSenSingle(model_folder = 'gensen/data/models',filename_prefix ='nli_large',pretrained_emb='./data/embedding/glove.840B.300d.h5')
	output_1, sent_1 = gensen_1.get_representation(batch, pool='last', return_numpy=True, tokenize=False)
	output_2, sent_2 = gensen_1.get_representation(batch, pool='last', return_numpy=True, tokenize=False)
	maxLength = 0
	for i in range(len(output_1)):
		if len(output_1[i])> maxLength:
			maxLength = len(output_1[i])
	length = []
	output = []

	for i in range(len(output_1)):
		temp = np.concatenate((output_1[i],output_2[i]),axis = 1)
		length.append(len(temp[~np.all(temp==0,axis = 1)]))
		#print(length[-1])
		#if needPad:
		#	if needPad == -1:
		#		if len(temp) <= maxLength:
		#			temp = np.concatenate([temp,np.zeros([maxLength - len(temp),temp.shape[1]])])
		#		else:
		#			temp = temp[:maxLength]
		#	else:
		#		if len(temp) <= padLength:
		#			temp = np.concatenate([np.array(wordList),np.zeros([padLength - len(wordList),300])])
			#	else:
			#		temp = temp[:padLength]
		output.append(temp)

	return np.array(output),np.array(length)
def getGloveEmb(batch, needPad = False, padLength = 30, path = "glove.840B.300d.txt"):
	# Input format should be wordList (After tokenization)
	# padLength == -1, pad to the max length in this batch
	global embeddings_index

	length = []
	output = []
	maxLength = 0

	for sentence in batch:
		if len(sentence) > maxLength:
			maxLength  = len(sentence)

	if embeddings_index == None or len(embeddings_index) == 0:
		print("Load embedding")
		embeddings_index = dict()
		print(path)
		f = open(path  , 'r', errors = 'ignore', encoding='utf8')  # "word2vec.6B.50d.txt" for test
		for line in f:
			splitLine = line.split(' ')  # For 3B  splitLine = line.split()
			word = splitLine[0]
			coefs = np.array([float(val) for val in splitLine[1:]])
			embeddings_index[word] = coefs
		f.close()
		print("Finish loading")

	for sentence in batch:
		#print(sentence)
		wordList = []
		for vocab in sentence:
			try:
				wordList.append(embeddings_index[vocab])
			except:
				wordList.append(np.zeros(300))
		length.append(len(wordList))

		if needPad:
			if padLength == -1:
				if len(sentence) < maxLength:
					wordList = np.concatenate([np.array(wordList),np.zeros([maxLength - len(wordList),300])])
			else:
				if len(wordList) < padLength:
					wordList = np.concatenate([np.array(wordList),np.zeros([padLength - len(wordList),300])])
				else:
					wordList = np.array(wordList)[:padLength]

		output.append(wordList)

	return np.array(output),np.array(length)

def getElmoEmb(batch, needPad = False, padLength = 30):
	# Input format should be string without tokenization
	assert type(batch[0]) is str
	global elmo
	global graph

	if elmo == None:
		graph = tf.Graph()
		with tf.Session(graph = graph) as sess:
			elmo = hub.Module("https://tfhub.dev/google/elmo/2")
			embeddings = elmo(batch,as_dict=True)["elmo"]
			sess.run(tf.global_variables_initializer())
			output = embeddings.eval(session = sess)
	else:
		with tf.Session(graph = graph) as sess:
			embeddings = elmo(batch,as_dict=True)["elmo"]
			sess.run(tf.global_variables_initializer())
			output = embeddings.eval(session = sess)

	if needPad:
		output_temp = []
		for i,sent in enumerate(output):
			if len(sent) > padLength:
				output_temp.append(sent[:padLength,:])
			else:
				z = np.zeros([padLength-sent.shape[0],sent.shape[1]])
				output_temp.append(np.concatenate([sent,z]))
		output = np.array(output_temp)

	return output, [len(output[0])]*len(batch)

def getGPT2Emb(batch, needPad = False, padLength = 30):
	# Input format should be word List(After tokenization)
	#assert type(batch[0]) is not str
	output = []
	length = []

	global model
	global tokenizer
	if model == None:
		tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
		model = GPT2Model.from_pretrained('gpt2')

	for text in batch:
		indexed_tokens_1 = tokenizer.encode(text)
		tokens_tensor_1 = torch.tensor([indexed_tokens_1])
		with torch.no_grad():
			hidden_states_1, past = model(tokens_tensor_1)
			length.append(len(hidden_states_1[0]))
			output.append(hidden_states_1[0].numpy())

	if needPad:
		output_temp = [ ]
		maxLength = np.array(length).max()

		if padLength == -1 :
			for i,hidden_states_1 in enumerate(output):
				if hidden_states_1.shape[0] < maxLength:
					output_temp.append(np.concatenate([hidden_states_1,np.zeros([maxLength-hidden_states_1.shape[0],hidden_states_1.shape[1]])]))
				else:
					output_temp.append(hidden_states_1[:maxLength,:])
		else:
			for i,hidden_states_1 in enumerate(output):
				if hidden_states_1.shape[0] < padLength:
					output_temp.append(np.concatenate([hidden_states_1,np.zeros([padLength-hidden_states_1.shape[0],hidden_states_1.shape[1]])]))
				else:
					output_temp.append(hidden_states_1[:padLength,:])

		output = np.array(output_temp)

	return np.array(output),np.array(length)

def getBertEmb(batch,needPad = False, padLength = 30):
	global bc
	if bc == None:
		bc = BertClient()
	output = bc.encode(batch)
	length = []
	output1 = []

	for i,emb in enumerate(output):
		output1.append(emb[~np.all(emb == 0, axis=1)])
		#print("output_1:",len(output1[-1]))
		length.append(len(output1[i]))
	return np.array(output),np.array(length)


''' Plot script WC parameter LogLikelihood
'''

# Standard imports
import sys
import ROOT
import imp
import pickle
import ctypes
import numpy as np

from math import sqrt
# turn off graphics
ROOT.gROOT.SetBatch( True )

# RootTools
from RootTools.core.standard import *

from plot_helpers import getUncertaintyValue, getObservationValue
from multiprocessing import Pool

# Logger
import TTXPheno.Tools.logger as logger
import RootTools.core.logger as logger_rt
logger    = logger.get_logger(   'DEBUG', logFile = None)
logger_rt = logger_rt.get_logger('INFO', logFile = None)

# TTXPheno
from TTXPheno.samples.benchmarks import *
from TTXPheno.Tools.user import plot_directory, cardfileLocation

# get the reweighting function
from TTXPheno.Tools.WeightInfo import WeightInfo

ROOT.gStyle.SetNumberContours(255)

# Arguments
import argparse

argParser = argparse.ArgumentParser(description = "Argument parser")
argParser.add_argument('--version',            action='store',     default='test', help='Appendix to plot directory')
argParser.add_argument('--process',            action='store',     default='ttZ_3l', nargs='?', choices=['ttZ_3l', 'ttZ_4l', 'ttgamma_1l', 'ttgamma_2l'], help="which process to calculate?")
argParser.add_argument('--bestFit',            action='store_true', help='Run combine with bestFit scenario (wide r ranges)')
argParser.add_argument('--removeCardFiles',    action='store_true', help='remove cardfiles after calculation?')
argParser.add_argument('--useCombine',         action='store_true', help='use Higgs Combine Tool?')
argParser.add_argument('--fitOnly',            action='store_true', help='do you already have the cardfiles?')
argParser.add_argument('--sample',             action='store',     default='fwlite_ttZ_ll_LO_order2_15weights_ref', help='Sample name specified in sample/python/benchmarks.py, e.g. fwlite_ttZ_ll_LO_order2_15weights_ref')
argParser.add_argument('--order',              action='store',     default=2, help='Polynomial order of weight string (e.g. 2)')
argParser.add_argument('--selection',          action='store',     default='lepSel3-onZ-njet3p-nbjet1p-Zpt0', help="Specify cut.")
argParser.add_argument('--small',              action='store_true', help='Run only on a small subset of the data?')
argParser.add_argument('--contours',           action='store_true', help='draw 1sigma and 2sigma contour line?')
argParser.add_argument('--smooth',             action='store_true', help='smooth histogram?')
argParser.add_argument('--level',              action='store',     default='reco', nargs='?', choices=['reco', 'gen'], help='Which level of reconstruction? reco, gen')
argParser.add_argument('--variables' ,         action='store',     default = ['ctZ', 'ctZI'], type=str, nargs=2, help = "argument plotting variables")
argParser.add_argument('--binning',            action='store',     default = [1, -2, 2, 1, -2, 2], type=float, nargs=6, help = "argument parameters")
argParser.add_argument('--zRange',             action='store',     default = [None, None], type=float, nargs=2, help = "argument parameters")
argParser.add_argument('--luminosity',         action='store',     default=150, type=int, help='Luminosity for weighting the plots')
argParser.add_argument('--scale',              action='store',     default=None, help='Luminosity for weighting the plots')
argParser.add_argument('--cores',              action='store',     default=8, type=int, help='number of cpu cores for multicore processing')
argParser.add_argument('--overwrite',          action='store_true', help='overwrite datafile?')
argParser.add_argument('--binMultiplier',      action='store',     default=3, type=int, help='bin multiplication factor')
argParser.add_argument('--detector',           action='store',     default='CMS', nargs='?', choices=['CMS', 'ATLAS', 'phase2_CMS'], help='Which Delphes detector simulation?')
argParser.add_argument('--scale14TeV',         action='store_true', help='scale 13 TeV cross-sections to 14 Tev?')
argParser.add_argument('--additionalCardFile', action='store',     default='TopEFTCardFile.txt', help='Cardfile where additional uncertainties are taken from')
argParser.add_argument('--addNonPrompt',       action='store_true', help='add nonPrompt?')
argParser.add_argument('--addUncertainties',   action='store',     default = ['trigger_2016','scale','scale_sig','PDF','PartonShower','nonprompt','WZ_xsec', 'ttX'], type=str, help = "add additional uncertainties from cardFile")
argParser.add_argument('--addBinNumberShift',  action='store',     default = 0, type=int, help = "which bin number does the region start in the additional card file?")
argParser.add_argument('--uncertaintyScale',   action='store',     default = 0.5, type=float, help = "scale factor for additional uncertainties")
argParser.add_argument('--statOnly',           action='store_true', help='use only statistical uncertainties')
argParser.add_argument('--noExpUnc',          action='store_true', help='use only statistical and theory uncertainties')

args = argParser.parse_args()

for unc in ['trigger_2016', 'nonprompt']:
    if unc in args.addUncertainties and args.noExpUnc: args.addUncertainties.remove(unc)

if args.level == 'gen':
    if 'ttZ' in args.process.split('_'):
        if args.small: from TTXPheno.Analysis.regions import genttZRegionsSmall as regions
        else:          from TTXPheno.Analysis.regions import genttZRegions as regions
    elif 'ttgamma' in args.process.split('_'):
        if args.small: from TTXPheno.Analysis.regions import genttgammaRegionsSmall as regions
        else:          from TTXPheno.Analysis.regions import genttgammaRegions as regions

    # Import additional functions/classes specified for the level of reconstruction
    from TTXPheno.Tools.cutInterpreterGen import cutInterpreter

elif args.level == 'reco':
    if 'ttZ' in args.process.split('_'):
        if args.small: from TTXPheno.Analysis.regions import recottZRegionsSmall as regions
        else:          from TTXPheno.Analysis.regions import recottZRegions as regions
    elif 'ttgamma' in args.process.split('_'):
        if args.small: from TTXPheno.Analysis.regions import recottgammaRegionsSmall as regions
        else:          from TTXPheno.Analysis.regions import recottgammaRegions as regions

    # Import additional functions/classes specified for the level of reconstruction
    from TTXPheno.Tools.cutInterpreterReco import cutInterpreter

#binning range
binningX = args.binning[:3]
binningY = args.binning[3:]

if binningX[0] > 1:
    xRange = np.linspace( binningX[1], binningX[2], int(binningX[0]), endpoint=False)
    xRange = [ el + 0.5 * ( xRange[1] - xRange[0] ) for el in xRange ]
else:
    xRange = [ 0.5 * ( binningX[1] + binningX[2] ) ]

if binningY[0] > 1:
    yRange = np.linspace( binningY[1], binningY[2], int(binningY[0]), endpoint=False)
    yRange = [ el + 0.5 * ( yRange[1] - yRange[0] ) for el in yRange ]
else:
    yRange = [ 0.5 * ( binningY[1] + binningY[2] ) ]

addon = []
if args.statOnly: addon += ["statOnly"]
if args.noExpUnc: addon += ["noExpUnc"]
#save data file
filename = '_'.join( ['nll', args.detector ] + args.sample.split('_')[1:3] + args.variables + map( str, args.binning ) + [ args.selection, str(args.luminosity), "14TeV" if args.scale14TeV else "13TeV" ] + addon ) + '.data'

#do the calculation
if not os.path.isfile('dat/' + filename) or args.overwrite:

    if not args.fitOnly:

        # Import samples
        sample_file     = "$CMSSW_BASE/python/TTXPheno/samples/benchmarks.py"
        loadedSamples   = imp.load_source( "samples", os.path.expandvars( sample_file ) )

        ttXSample       = getattr( loadedSamples, args.sample + '_%s' %args.detector )
        WZSample        = getattr( loadedSamples, 'fwlite_WZ_lep_LO_order2_15weights_%s' %args.detector )
    #    ttSample        = getattr( loadedSamples, 'fwlite_tt_full_LO_order2_15weights_%s' %args.detector )
    #    tWSample        = getattr( loadedSamples, 'fwlite_tW_LO_order2_15weights_%s' %args.detector )
        tWZSample       = getattr( loadedSamples, 'fwlite_tWZ_LO_order2_15weights_%s' %args.detector )
        tZqSample       = getattr( loadedSamples, 'fwlite_tZq_LO_order2_15weights_%s' %args.detector )
    #    ZgammaSample    = getattr( loadedSamples, 'fwlite_Zgamma_LO_order2_15weights_%s' %args.detector )
        ttgammaSample   = getattr( loadedSamples, 'fwlite_ttgamma_bg_LO_order2_15weights_%s' %args.detector )

        #if args.process.split('_')[0] == 'ttgamma':
        #    ttgammaIsrSample  = copy.deepcopy( ttXSample ) #select ttgamma events with isolated gamma from ISR (cat a2)
        #    ttgammaIsrSample.name = 'fwlite_ttgamma_ISR_LO_order2_15weights_ref'

        if args.process == 'ttZ_3l': bg = [ WZSample, tWZSample, tZqSample, ttgammaSample ]
    #    if args.process == 'ttZ_3l': bg = [ WZSample, tWZSample, tZqSample ]
        elif args.process == 'ttZ_4l': bg = [ WZSample, tWZSample, tZqSample, ttgammaSample ]
        elif args.process == 'ttgamma_1l': bg = [ ttSample, tWSample, tWZSample, tZqSample, ZgammaSample ]
        elif args.process == 'ttgamma_2l': bg = [ ttSample, tWSample, tWZSample, tZqSample, ZgammaSample ]

        def checkReferencePoint( sample ):
            ''' check if sample is simulated with a reference point
            '''
            return pickle.load(file(sample.reweight_pkl))['ref_point'] != {}

        # set selection string
        selectionString      = cutInterpreter.cutString(args.selection)
        selectionString_up   = selectionString.replace('nBTag','nBTag_JEC_up').replace('nrecoJet','nrecoJets_JEC_up')
        selectionString_down = selectionString.replace('nBTag','nBTag_JEC_down').replace('nrecoJet','nrecoJets_JEC_down')

        # somehow has to be separate from the next loop
        if args.small:
            for s in [ttXSample] + bg:
                s.reduceFiles( to = 15 )

        # configure samples
        for s in [ttXSample] + bg:

            s.event_factor = s.nEvents / float( s.chain.GetEntries() )
            s.xsecScaleFactor = s.xsec14 / s.xsec if args.scale14TeV else 1.
            s.weightInfo = WeightInfo( s.reweight_pkl )
            s.weightInfo.set_order( args.order )
            s.setSelectionString( selectionString )

            if checkReferencePoint( s ):
                s.setWeightString( 'ref_lumiweight1fb*(%s)*(%s)*(%s)'%( str(args.luminosity), str(s.event_factor), str(s.xsecScaleFactor) ) )
            else:
                s.setWeightString( 'lumiweight1fb*(%s)*(%s)*(%s)'%( str(args.luminosity), str(s.event_factor), str(s.xsecScaleFactor) ) )

        # overlap removal
        if args.process.split('_')[0] == 'ttgamma':
            ttXSample.addSelectionString( "(nonIsoPhoton!=1)" )
            ttSample.addSelectionString(  "(nonIsoPhoton==1)" )

        for var in args.variables:
            if var not in ttXSample.weightInfo.variables and not (args.variables[0] == 'cuB' and args.variables[1] == 'cuW'):
                raise ValueError('Input variable not in gridpack: %s' %var)

        observation                    = {}

        signal_btagging_uncertainty    = {}
        signal_mistagging_uncertainty  = {}
        signal_jes_uncertainty         = {}
        signal_electronId_uncertainty  = {}
        signal_muonId_uncertainty      = {}

        ttX_SM_rate                  = {}
        ttX_coeffList                = {}

        ttX_coeffList_reweighted_btagging   = {}
        ttX_coeffList_reweighted_mistagging = {}
        ttX_coeffList_reweighted_muonId     = {}
        ttX_coeffList_reweighted_electronId = {}
        ttX_coeffList_reweighted_jes_up     = {}
        ttX_coeffList_reweighted_jes_down   = {}

        background_rate                   = {}
        background_btagging_uncertainty   = {}
        background_mistagging_uncertainty = {}
        background_jes_uncertainty        = {}
        background_electronId_uncertainty = {}
        background_muonId_uncertainty     = {}

        nonPromptObservation              = {}

        for i_region, region in enumerate(regions):
            # compute signal yield for this region (this is the final code)

            logger.info( "At region %s", region )

            # ttX SM
            ttX_coeffList[region]  = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString() )
            ttX_SM_rate[region]    = ttXSample.weightInfo.get_weight_yield( ttX_coeffList[region] )

            if not args.statOnly:
                # uncertainty coeffLists
                ttX_coeffList_reweighted_btagging[region]   = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString(), weightString="reweight_BTag_B" )
                ttX_coeffList_reweighted_mistagging[region] = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString(), weightString="reweight_BTag_L" )
                ttX_coeffList_reweighted_muonId[region]     = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString(), weightString="reweight_id_mu" )
                ttX_coeffList_reweighted_electronId[region] = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString(), weightString="reweight_id_ele" )

                ttXSample.setSelectionString( selectionString_up )
                ttX_coeffList_reweighted_jes_up[region]     = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString() )

                ttXSample.setSelectionString( selectionString_down )
                ttX_coeffList_reweighted_jes_down[region]   = ttXSample.weightInfo.getCoeffListFromDraw( ttXSample, selectionString = region.cutString() )

                # reset selectionstring
                ttXSample.setSelectionString( selectionString )

            background_rate[region]                   = {}
            background_btagging_uncertainty[region]   = {}
            background_mistagging_uncertainty[region] = {}
            background_jes_uncertainty[region]        = {}
            background_muonId_uncertainty[region]     = {}
            background_electronId_uncertainty[region] = {}

            for i_background, background in enumerate(bg):
                # compute bg yield for this region (this is the final code)

                background_rate                 [region][background.name] = background.getYieldFromDraw( selectionString=region.cutString() )['val']

                if not args.statOnly and not args.noExpUnc:
                    #calculate btagging uncert.
                    background_rate_reweighted                                = background.getYieldFromDraw( selectionString=region.cutString(), weightString="reweight_BTag_B" )['val']
                    background_btagging_uncertainty [region][background.name] = 1 + (( background_rate_reweighted - background_rate[region][background.name] ) / background_rate[region][background.name]) if background_rate[region][background.name] > 0 else 1.

                    #calculate mistagging uncert.
                    background_rate_reweighted                                = background.getYieldFromDraw( selectionString=region.cutString(), weightString="reweight_BTag_L" )['val']
                    background_mistagging_uncertainty [region][background.name] = 1 + (( background_rate_reweighted - background_rate[region][background.name] ) / background_rate[region][background.name]) if background_rate[region][background.name] > 0 else 1.

                    #calculate muon Id uncert.
                    background_rate_reweighted                                = background.getYieldFromDraw( selectionString=region.cutString(), weightString="reweight_id_mu" )['val']
                    background_muonId_uncertainty [region][background.name] = 1 + (( background_rate_reweighted - background_rate[region][background.name] ) / background_rate[region][background.name]) if background_rate[region][background.name] > 0 else 1.

                    background_rate_reweighted                                = background.getYieldFromDraw( selectionString=region.cutString(), weightString="reweight_id_ele" )['val']
                    #calculate electron Id uncert.
                    background_electronId_uncertainty [region][background.name] = 1 + (( background_rate_reweighted - background_rate[region][background.name] ) / background_rate[region][background.name]) if background_rate[region][background.name] > 0 else 1.

                    # set selectionstring to JES_up
                    background.setSelectionString( selectionString_up )
                    background_rate_reweighted_up                             = background.getYieldFromDraw( selectionString=region.cutString() )['val']
                    # set selectionstring to JES_up
                    background.setSelectionString( selectionString_down )
                    background_rate_reweighted_down                           = background.getYieldFromDraw( selectionString=region.cutString() )['val']
                    # reset selectionstring
                    background.setSelectionString( selectionString )
                    #calculate JES uncert.
                    background_jes_uncertainty      [region][background.name] = 1 + (( background_rate_reweighted_up - background_rate_reweighted_down ) / (2*background_rate[region][background.name])) if background_rate[region][background.name] > 0 else 1.

            nonPromptObservation[region] = 0.
            if args.addNonPrompt:
                # scale nonprompt observation value from Run2 to args.luminosity
                nonPromptObservation[region] = getObservationValue( args.additionalCardFile, args.addBinNumberShift + i_region, 'nonPromptDD' ) * float(args.luminosity) / 35.9

            # Our expected observation :-)
            # add nonPrompt observation to total observation
            observation[region] = int( round( sum( background_rate[region].values() ) + ttX_SM_rate[region] + nonPromptObservation[region] ) )

    # Write temporary card file
    from TTXPheno.Tools.cardFileWriter import cardFileWriter
#    c = cardFileWriter.cardFileWriter()
    if args.useCombine:
        from TTXPheno.Tools.user import combineReleaseLocation
#        c.releaseLocation = combineReleaseLocation
    else:
        # non CMS NLL plot
        from TTXPheno.Analysis.ProfiledLoglikelihoodFit import ProfiledLoglikelihoodFit

    def cuBWtoctWZ( cuB, cuW ):
        ''' transforms C_tZ and C_tW to C_uB and C_uW
            C_tZ = Re( -sW*C_uB + cW*C_uW )
            C_tW = Re( C_uW )
            arXiv: 1802.07237
        '''
        sW=0.4715
        cW=0.8819

        ctW = cuW
        ctZ = -sW*cuB + cW*cuW

        return ctZ, ctW

    def calculation( variables ):
    #def calculation( var1, var2 ):

        if args.variables[0] == 'cuB' and args.variables[1] == 'cuW':
            var1, var2 = variables #cuB cuW
            ctZ, ctW = cuBWtoctWZ( var1, var2 )
            kwargs = { 'ctZ':ctZ, 'ctW':ctW }
        else:
            var1, var2 = variables
            kwargs = { args.variables[0]:var1, args.variables[1]:var2 }

        nameList = args.sample.split('_')[1:3] + args.variables + args.binning + [ args.level, args.version, args.order, args.luminosity, "14TeV" if args.scale14TeV else "13TeV", args.selection, 'small' if args.small else 'full', 'statOnly' if args.statOnly else 'fullUnc' if not args.noExpUnc else 'noExpUnc', var1, var2 ]
        cardname = '%s_nll_card'%'_'.join( map( str, nameList ) )
        cardFilePath = os.path.join( cardfileLocation, cardname + '.txt' )

        c = cardFileWriter.cardFileWriter()
        if args.useCombine:
            c.releaseLocation = combineReleaseLocation

        if not args.fitOnly:
#            print 'run cardfile'

            # uncertainties
            c.reset()
            if not args.statOnly:
                if not args.noExpUnc:
                    c.addUncertainty('lumi',        'lnN')
                    c.addUncertainty('JES',         'lnN')
                    c.addUncertainty('btagging',    'lnN')
                    c.addUncertainty('mistagging',  'lnN')
                    c.addUncertainty('muonId',      'lnN')
                    c.addUncertainty('electronId',  'lnN')
                for unc in args.addUncertainties:
                    c.addUncertainty(unc,  'lnN')

            signal_rate                  = {}
            for i_region, region in enumerate(regions):

                signal_rate[region] = ttXSample.weightInfo.get_weight_yield( ttX_coeffList[region], **kwargs)

                if not args.statOnly and not args.noExpUnc:
                    # signal uncertainties
                    # btagging
                    signal_rate_reweighted   = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_btagging[region], **kwargs )
                    signal_btagging_uncertainty [region] = 1 + (( signal_rate_reweighted - signal_rate[region] ) / signal_rate[region]) if signal_rate[region] > 0 else 1.

                    # mistagging
                    signal_rate_reweighted   = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_mistagging[region], **kwargs )
                    signal_mistagging_uncertainty [region] = 1 + (( signal_rate_reweighted - signal_rate[region] ) / signal_rate[region]) if signal_rate[region] > 0 else 1.

                    # muonId
                    signal_rate_reweighted   = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_muonId[region], **kwargs )
                    signal_muonId_uncertainty [region] = 1 + (( signal_rate_reweighted - signal_rate[region] ) / signal_rate[region]) if signal_rate[region] > 0 else 1.

                    # electronId
                    signal_rate_reweighted   = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_electronId[region], **kwargs )
                    signal_electronId_uncertainty [region] = 1 + (( signal_rate_reweighted - signal_rate[region] ) / signal_rate[region]) if signal_rate[region] > 0 else 1.

                    # JES
                    signal_rate_reweighted_JES_up   = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_jes_up[region], **kwargs )
                    signal_rate_reweighted_JES_down = ttXSample.weightInfo.get_weight_yield( ttX_coeffList_reweighted_jes_down[region], **kwargs )
                    signal_jes_uncertainty[region] = 1 + (( signal_rate_reweighted_JES_up - signal_rate_reweighted_JES_down ) / (2*signal_rate[region])) if signal_rate[region] > 0 else 1.

                bin_name = "Region_%i" % i_region
                nice_name = region.__str__()
                c.addBin(bin_name, ['_'.join(s.name.split('_')[1:3]) for s in bg] + ['nonPrompt'] if args.addNonPrompt else ['_'.join(s.name.split('_')[1:3]) for s in bg], nice_name)

                c.specifyObservation( bin_name, observation[region] )

                c.specifyExpectation( bin_name, 'signal', signal_rate[region]                                 )

                if not args.statOnly:
                    if not args.noExpUnc:
                        c.specifyFlatUncertainty( 'lumi', 1.01 )
                        c.specifyUncertainty( 'JES',        bin_name, 'signal', signal_jes_uncertainty[region]        )
                        c.specifyUncertainty( 'btagging',   bin_name, 'signal', signal_btagging_uncertainty[region]   )
                        c.specifyUncertainty( 'mistagging', bin_name, 'signal', signal_mistagging_uncertainty[region] )
                        c.specifyUncertainty( 'muonId',     bin_name, 'signal', signal_muonId_uncertainty[region]     )
                        c.specifyUncertainty( 'electronId', bin_name, 'signal', signal_electronId_uncertainty[region] )

                    for unc in args.addUncertainties:
                        c.specifyUncertainty( unc,      bin_name, 'signal', 1+(getUncertaintyValue( args.additionalCardFile, args.addBinNumberShift + i_region, 'signal', unc )-1)*args.uncertaintyScale )

                if args.addNonPrompt:
                    # for nonpromt only nonpromt uncertainty is important
                    c.specifyExpectation( bin_name, 'nonPrompt', nonPromptObservation[region] )
                    if not args.statOnly: c.specifyUncertainty( 'nonprompt',      bin_name, 'nonPrompt', 1+(getUncertaintyValue( args.additionalCardFile, args.addBinNumberShift + i_region, 'nonPromptDD', 'nonprompt' )-1)*args.uncertaintyScale )

                #c.specifyExpectation( bin_name, 'ttX_SM', ttX_SM_rate[region] )
                #c.specifyUncertainty( 'JES', bin_name, 'ttX_SM', ttX_SM_jes_uncertainty[region])
                #c.specifyUncertainty( 'btagging',bin_name, 'ttX_SM', ttX_SM_btagging_uncertainty[region])

                for background in bg:
                    c.specifyExpectation( bin_name, '_'.join( background.name.split('_')[1:3] ), background_rate[region][background.name] )
                    if not args.statOnly:
                        if not args.noExpUnc:
                            c.specifyUncertainty( 'JES',        bin_name, '_'.join( background.name.split('_')[1:3] ), background_jes_uncertainty[region][background.name])
                            c.specifyUncertainty( 'btagging',   bin_name, '_'.join( background.name.split('_')[1:3] ), background_btagging_uncertainty[region][background.name])
                            c.specifyUncertainty( 'mistagging', bin_name, '_'.join( background.name.split('_')[1:3] ), background_mistagging_uncertainty[region][background.name])
                            c.specifyUncertainty( 'muonId',     bin_name, '_'.join( background.name.split('_')[1:3] ), background_muonId_uncertainty[region][background.name])
                            c.specifyUncertainty( 'electronId', bin_name, '_'.join( background.name.split('_')[1:3] ), background_electronId_uncertainty[region][background.name])
                        for unc in args.addUncertainties:
                            if 'tZq' in background.name.split('_') or 'ttgamma' in background.name.split('_') or 'tWZ' in background.name.split('_'): proc = 'TTX'
                            elif 'WZ' in background.name.split('_'): proc = 'WZ'
                            else: raise ValueError('Background not found: %s' %background.name)
                            c.specifyUncertainty( unc,      bin_name, '_'.join( background.name.split('_')[1:3] ), 1+(getUncertaintyValue( args.additionalCardFile, args.addBinNumberShift + i_region, proc, unc )-1)*args.uncertaintyScale )

            c.writeToFile( cardFilePath )

        else:
            logger.info( "Running only NLL Fit with given CardFile %s"%cardFilePath)

        if not os.path.isfile( cardFilePath ):
            raise ValueError('CardFiles not found! Run script without --fitOnly!')

        if args.useCombine:
            # use the official cms combine tool
#                c.calcNuisances( cardFilePath, bestFit=args.bestFit )
            nll = c.calcNLL( cardFilePath, bestFit=args.bestFit )
#            nll = nll['nll0'] #pre-fit
            nll = nll['nll_abs'] #post-fit

            if args.removeCardFiles:
                for file in os.listdir( cardfileLocation ):
                    if file.startswith( cardname ):
                        os.remove( os.path.join( cardfileLocation, file ) )

        else:
            if args.bestFit: r = (0.99, 1.01)
            else: r = (0., 2.)

            profiledLoglikelihoodFit = ProfiledLoglikelihoodFit( cardFilePath )
            profiledLoglikelihoodFit.make_workspace(rmin=r[0], rmax=r[1])
            nll = profiledLoglikelihoodFit.likelihoodTest()
            profiledLoglikelihoodFit.cleanup(removeFiles=args.removeCardFiles)
            del profiledLoglikelihoodFit

        logger.info( "NLL: %f", nll)
        ROOT.gDirectory.Clear()

        # in very large WC regions, the fit fails, not relevant for the interesting regions
        if nll is None or abs(nll) > 10000 or abs(nll) < 1: nll = 999

        del c

        return var1, var2, nll

    results = []

    SM = calculation( (0, 0) )

    for varX in xRange:
        # do not run all calc in one pool, memory leak!!!
        pool = Pool( processes = args.cores )
        results += pool.map( calculation, [ (varX, varY) for varY in yRange ] )
        pool.close()
        del pool

    with open('tmp/'+filename, 'w') as f:
        for item in [SM]+results:
            f.write( "%s\n" % ','.join( map( str, list(item) ) ) )

else:
    with open('dat/'+filename, 'r') as f:
        data = f.readlines()

    results = []
    for i, line in enumerate(data):
        vals = map( float, line.split('\n')[0].split(',') )
        if args.scale is not None: vals[2] = vals[2]*float(args.scale)/float(args.luminosity)/2
        if i == 0:
            if vals[0] != 0 or vals[1] != 0:
                raise ValueError('SM Point in data file is not valid!')
            SM = tuple( vals )
        else: results.append( tuple( vals ) )

#Plot

#scale to SM
results.sort( key = lambda res: ( abs(res[0]), abs(res[1]), res[2] ) )
nll_SM = SM[2]

results = [ (x, y, 2*(result - nll_SM)) for x, y, result in results ]

def toGraph2D( name, title, data ):
    result = ROOT.TGraph2D( len(data) )
    debug = ROOT.TGraph()
    result.SetName( name )
    result.SetTitle( title )
    for i, datapoint in enumerate(data):
        x, y, val = datapoint
        result.SetPoint(i, x, y, val)
        debug.SetPoint(i, x, y)
    c = ROOT.TCanvas()
    result.Draw()
    debug.Draw()
    del c
    #res = ROOT.TGraphDelaunay(result)
    return result, debug

def convDipolesAnomalousCoupling( c2V, c2A ):
    ctZ  = c2V / 0.103
    ctZI = c2A / 0.103
    return round(ctZ,6), round(ctZI,6)

def convVectorAnomalousCoupling( c1V, c1A ):
    cpQM = ( c1V - c1A - 0.849 ) / ( -0.073 )
    cpt  = ( c1V + c1A + 0.357 ) / ( -0.073 )
    return round(cpQM,6), round(cpt,6)

def convVectorCoupling( cpQM, cpt ):
    c1V =  0.244 - 0.0365 * (cpQM + cpt)
    c1A = -0.601 + 0.0365 * (cpQM - cpt)
    return c1V, c1A

def convDipoles( ctZ, ctZI ):
    c2V = 0.103 * ctZ
    c2A = 0.103 * ctZI
    return c2V, c2A

def convertToAnomalousCouplings( res ):
    if "cpQM" in args.variables and "cpt" in args.variables:
        conversion = convVectorCoupling
        reversed = args.variables[0] == "cpt"

    elif "ctZ"  in args.variables and "ctZI" in args.variables:
        conversion = convDipoles
        reversed = args.variables[0] == "ctZI"

    else:
        raise ValueError('Anomalous Coupling conversion not implemented for %s'% " and ".join(args.variables))

    res_conv = []
    for x, y, nll in res:
        if reversed:
            y_conv, x_conv = conversion( y, x )
        else:
            x_conv, y_conv = conversion( x, y )

        res_conv.append( (x_conv, y_conv, nll) )

    return res_conv

#get TGraph2D from results list
a, debug = toGraph2D( args.process, args.process, results )#res_dic)
resultsAC = convertToAnomalousCouplings(results)
allX = [x for x, y, nll in resultsAC ]
allY = [y for x, y, nll in resultsAC ]
resultsAC.append( ( min(allX),min(allY),999) )
resultsAC.append( ( min(allX),max(allY),999) )
resultsAC.append( ( max(allX),min(allY),999) )
resultsAC.append( ( max(allX),max(allY),999) )
if "cpQM" in args.variables and "cpt" in args.variables:
    resultsAC.append( ( -0.47, -0.53, 32) )
    resultsAC.append( ( -0.48, -0.55, 32) )
    resultsAC.append( ( -0.48, -0.58, 32) )
    resultsAC.append( (  0.61, -0.58, 32) )
    resultsAC.append( (  0.61, -0.52, 32) )
    resultsAC.append( (  0.58, -0.5, 32) )

ac, debug = toGraph2D( args.process + "ac", args.process + "ac", resultsAC )#res_dic)
nxbins   = max(1, min(500, int(binningX[0])*int(args.binMultiplier)))
nybins   = max(1, min(500, int(binningY[0])*int(args.binMultiplier)))

#re-bin
hist = a.GetHistogram().Clone()
a.SetNpx(nxbins)
a.SetNpy(nybins)
hist = a.GetHistogram().Clone()

histac = ac.GetHistogram().Clone()
ac.SetNpx(nxbins)
ac.SetNpy(nybins)
histac = ac.GetHistogram().Clone()

#smoothing
if args.smooth:
    hist.Smooth()
    histac.Smooth()

cans = ROOT.TCanvas("can_%s"%args.process,"",500,500)

#calculate contour lines (1sigma, 2sigma) for 2D
contours = {'ttZ_3l': [2.28, 5.99]}
if args.contours:
    histsForCont = hist.Clone()
    c_contlist = ((ctypes.c_double)*(len(contours[args.process])))(*contours[args.process])
    histsForCont.SetContour(len(c_contlist),c_contlist)
    histsForCont.Draw("contzlist")
    cans.Update()
    conts = ROOT.gROOT.GetListOfSpecials().FindObject("contours")
    #cont_m2 = conts.At(0).Clone()
    #cont_m1 = conts.At(1).Clone()
    cont_p1 = conts.At(0).Clone()
    cont_p2 = conts.At(1).Clone()

pads = ROOT.TPad("pad_%s"%args.process,"",0.,0.,1.,1.)
pads.SetRightMargin(0.20)
pads.SetLeftMargin(0.14)
pads.SetTopMargin(0.11)
pads.Draw()
pads.cd()

hist.Draw("colz")

#draw contour lines
if args.contours:
    for conts in [cont_p2]:
        for cont in conts:
            cont.SetLineColor(ROOT.kOrange+7)
            cont.SetLineWidth(3)
#            cont.SetLineStyle(7)
            cont.Draw("same")
    for conts in [cont_p1]:
        for cont in conts:
            cont.SetLineColor(ROOT.kSpring-1)
            cont.SetLineWidth(3)
#            cont.SetLineStyle(7)
            cont.Draw("same")

hist.GetZaxis().SetTitle("-2 #Delta ln L")

if not None in args.zRange:
    hist.GetZaxis().SetRangeUser( args.zRange[0], args.zRange[1] )
#    hist.GetXaxis().SetRangeUser( -0.3 , 0.3 )
#    hist.GetYaxis().SetRangeUser( -0.3 , 0.3 )
#    hist.GetXaxis().SetRangeUser( -1 , 1 )
#    hist.GetYaxis().SetRangeUser( -1 , 1 )
#    hist.GetXaxis().SetRangeUser( -8 , 12 )
#    hist.GetYaxis().SetRangeUser( -8 , 12 )

if args.variables[0] == 'cuB' and args.variables[1] == 'cuW':
    hist.GetXaxis().SetTitle('C^{(33)}_{uB} (#Lambda/TeV)^{2}' )
    hist.GetYaxis().SetTitle('C^{(33)}_{uW} (#Lambda/TeV)^{2}' )
else:
    xTitle = args.variables[0].replace('c','C_{').replace('p','#phi').replace('M','') + '}'
    if 'I' in xTitle: xTitle = xTitle.replace('I','') + '^{[Im]}'
    yTitle = args.variables[1].replace('c','C_{').replace('p','#phi').replace('M','') + '}'
    if 'I' in yTitle: yTitle = yTitle.replace('I','') + '^{[Im]}'
    hist.GetXaxis().SetTitle( xTitle + ' (#Lambda/TeV)^{2}' )
    hist.GetYaxis().SetTitle( yTitle + ' (#Lambda/TeV)^{2}' )

hist.GetXaxis().SetTitleFont(42)
hist.GetYaxis().SetTitleFont(42)
hist.GetZaxis().SetTitleFont(42)
hist.GetXaxis().SetLabelFont(42)
hist.GetYaxis().SetLabelFont(42)
hist.GetZaxis().SetLabelFont(42)

hist.GetXaxis().SetTitleOffset(1.15)
hist.GetYaxis().SetTitleOffset(1.25)

hist.GetXaxis().SetTitleSize(0.045)
hist.GetYaxis().SetTitleSize(0.045)
hist.GetZaxis().SetTitleSize(0.042)
hist.GetXaxis().SetLabelSize(0.04)
hist.GetYaxis().SetLabelSize(0.04)
hist.GetZaxis().SetLabelSize(0.04)

latex1 = ROOT.TLatex()
latex1.SetNDC()
latex1.SetTextSize(0.04)
latex1.SetTextFont(42)
latex1.SetTextAlign(11)

latex1.DrawLatex(0.03, 0.92, '#bf{CMS Phase-2} #it{Simulation Preliminary}'),
#latex1.DrawLatex(0.15, 0.95, '#bf{CMS Phase-2} #it{Simulation Preliminary}'),
latex1.DrawLatex(0.68, 0.92, '%i ab{}^{-1} (%s TeV)' % (int(args.luminosity/1000.), "14" if args.scale14TeV else "13"))

latex2 = ROOT.TLatex()
latex2.SetNDC()
latex2.SetTextSize(0.04)
latex2.SetTextFont(42)
latex2.SetTextAlign(11)

#latex2.DrawLatex(0.15, 0.9, 'with Stat. uncert. only' if args.statOnly else 'with YR18 syst. uncert.' if not args.noExpUnc else 'w/o Exp. uncert.'),
if (args.noExpUnc or args.statOnly): latex2.DrawLatex(0.17, 0.82, 'Stat. uncert. only' if args.statOnly else 'w/o Exp. uncert.'),

#latex1.DrawLatex(0.15, 0.92, ' '.join(args.process.split('_')[:2]) + ' (' + args.detector + ')')
#latex1.DrawLatex(0.55, 0.92, '%3.1f fb{}^{-1} @ 13 TeV'%(float(args.luminosity) if args.scale is None else float(args.scale)) )

plot_directory_ = os.path.join(\
    plot_directory,
    '%s_%s'%(args.level, args.version),
    args.detector,
    args.sample,
    'backgrounds',
    'nll_small' if args.small else 'nll',
    args.selection)

if not os.path.isdir( plot_directory_ ):
    os.makedirs( plot_directory_ )

for e in [".png",".pdf",".root"]:
    cans.Print( plot_directory_ + '/' + '_'.join(args.variables + ['lumi'+str(args.luminosity) if args.scale is None else 'lumi'+str(args.scale), "14TeV" if args.scale14TeV else "13TeV", "CMScombine" if args.useCombine else "privateFit", "bestFit" if args.bestFit else "r1", 'statOnly' if args.statOnly else 'fullUnc' if not args.noExpUnc else 'noExpUnc']) + e)

del cans
del hist
# plot anomalous couplings

args.variables[0] = args.variables[0].replace('pQM','1V').replace('pt','1A').replace('tZ','2V').replace('2VI','2A')
args.variables[1] = args.variables[1].replace('pQM','1V').replace('pt','1A').replace('tZ','2V').replace('2VI','2A')

cans = ROOT.TCanvas("can_%s"%args.process,"",500,500)

#calculate contour lines (1sigma, 2sigma) for 2D
contours = {'ttZ_3l': [2.28, 5.99]}
if args.contours:
    histsForCont = histac.Clone()
    c_contlist = ((ctypes.c_double)*(len(contours[args.process])))(*contours[args.process])
    histsForCont.SetContour(len(c_contlist),c_contlist)
    histsForCont.Draw("contzlist")
    cans.Update()
    conts = ROOT.gROOT.GetListOfSpecials().FindObject("contours")
    #cont_m2 = conts.At(0).Clone()
    #cont_m1 = conts.At(1).Clone()
    cont_p1 = conts.At(0).Clone()
    cont_p2 = conts.At(1).Clone()

pads = ROOT.TPad("pad_%s"%args.process,"",0.,0.,1.,1.)
pads.SetRightMargin(0.20)
pads.SetLeftMargin(0.14)
pads.SetTopMargin(0.11)
pads.Draw()
pads.cd()

histac.Draw("colz")

#draw contour lines
if args.contours:
    for conts in [cont_p2]:
        for cont in conts:
            cont.SetLineColor(ROOT.kOrange+7)
            cont.SetLineWidth(3)
#            cont.SetLineStyle(7)
            cont.Draw("same")
    for conts in [cont_p1]:
        for cont in conts:
            cont.SetLineColor(ROOT.kSpring-1)
            cont.SetLineWidth(3)
#            cont.SetLineStyle(7)
            cont.Draw("same")

histac.GetZaxis().SetTitle("-2 #Delta ln L")

if not None in args.zRange:
    histac.GetZaxis().SetRangeUser( args.zRange[0], args.zRange[1] )
#    if "c1V" in args.variables and "c1A" in args.variables:
#        histac.GetYaxis().SetRangeUser( -0.8 , -0.4 )
#    histac.GetXaxis().SetRangeUser( -1 , 1 )
#    histac.GetYaxis().SetRangeUser( -1 , 1 )
#    histac.GetXaxis().SetRangeUser( -8 , 12 )
#    histac.GetYaxis().SetRangeUser( -8 , 12 )

xTitle = args.variables[0].replace('c','C_{') + '}'
yTitle = args.variables[1].replace('c','C_{') + '}'

histac.GetXaxis().SetTitle( xTitle )
histac.GetYaxis().SetTitle( yTitle )

histac.GetXaxis().SetTitleFont(42)
histac.GetYaxis().SetTitleFont(42)
histac.GetZaxis().SetTitleFont(42)
histac.GetXaxis().SetLabelFont(42)
histac.GetYaxis().SetLabelFont(42)
histac.GetZaxis().SetLabelFont(42)

histac.GetXaxis().SetTitleOffset(1.15)
histac.GetYaxis().SetTitleOffset(1.25)

histac.GetXaxis().SetTitleSize(0.045)
histac.GetYaxis().SetTitleSize(0.045)
histac.GetZaxis().SetTitleSize(0.042)
histac.GetXaxis().SetLabelSize(0.04)
histac.GetYaxis().SetLabelSize(0.04)
histac.GetZaxis().SetLabelSize(0.04)

latex1 = ROOT.TLatex()
latex1.SetNDC()
latex1.SetTextSize(0.04)
latex1.SetTextFont(42)
latex1.SetTextAlign(11)

latex1.DrawLatex(0.03, 0.92, '#bf{CMS Phase-2} #it{Simulation Preliminary}'),
#latex1.DrawLatex(0.15, 0.95, '#bf{CMS Phase-2} #it{Simulation Preliminary}'),
latex1.DrawLatex(0.68, 0.92, '%i ab{}^{-1} (%s TeV)' % (int(args.luminosity/1000.), "14" if args.scale14TeV else "13"))

latex2 = ROOT.TLatex()
latex2.SetNDC()
latex2.SetTextSize(0.04)
latex2.SetTextFont(42)
latex2.SetTextAlign(11)

#latex2.DrawLatex(0.15, 0.9, 'with Stat. uncert. only' if args.statOnly else 'with YR18 syst. uncert.' if not args.noExpUnc else 'w/o Exp. uncert.'),
if (args.noExpUnc or args.statOnly): latex2.DrawLatex(0.17, 0.82, 'Stat. uncert. only' if args.statOnly else 'w/o Exp. uncert.'),

#latex1.DrawLatex(0.15, 0.92, ' '.join(args.process.split('_')[:2]) + ' (' + args.detector + ')')
#latex1.DrawLatex(0.55, 0.92, '%3.1f fb{}^{-1} @ 13 TeV'%(float(args.luminosity) if args.scale is None else float(args.scale)) )

plot_directory_ = os.path.join(\
    plot_directory,
    '%s_%s'%(args.level, args.version),
    args.detector,
    args.sample,
    'backgrounds',
    'nll_small' if args.small else 'nll',
    args.selection)

if not os.path.isdir( plot_directory_ ):
    os.makedirs( plot_directory_ )

for e in [".png",".pdf",".root"]:
    cans.Print( plot_directory_ + '/' + '_'.join(args.variables + ['lumi'+str(args.luminosity) if args.scale is None else 'lumi'+str(args.scale), "14TeV" if args.scale14TeV else "13TeV", "CMScombine" if args.useCombine else "privateFit", "bestFit" if args.bestFit else "r1", 'statOnly' if args.statOnly else 'fullUnc' if not args.noExpUnc else 'noExpUnc']) + e)


# -*- coding: utf-8 -*-
"""
flup
================
This module provides upload support for Flask. The basic pattern is to set up
an `UploadSet` object and upload your files to it.
"""
import os.path
import posixpath
from flask import current_app, Blueprint, send_from_directory, abort, url_for
from itertools import chain
from werkzeug import secure_filename, FileStorage, LocalProxy

_flup = LocalProxy(lambda: current_app.extensions['flup'])

class AllExcept(object):
    def __init__(self, items):
        self.items = items

    def __contains__(self, item):
        return item not in self.items

class All(object):
    def __contains__(self, item):
        return True

TEXT = ('txt',)
DOCUMENTS = tuple('rtf odf ods gnumeric abw doc docx xls xlsx'.split())
IMAGES = tuple('jpg jpe jpeg png gif svg bmp'.split())
AUDIO = tuple('wav mp3 aac ogg oga flac'.split())
DATA = tuple('csv ini json plist xml yaml yml'.split())
SCRIPTS = tuple('js php pl py rb sh'.split())
ARCHIVES = tuple('gz bz2 zip tar tgz txz 7z'.split())
EXECUTABLES = tuple('so exe dll'.split())
ALL = All()
DEFAULTS = TEXT + DOCUMENTS + IMAGES + DATA

class UploadNotAllowed(Exception):
    pass

def tuple_from(*iters):
    return tuple(itertools.chain(*iters))

def extension(filename):
    return filename.rsplit('.', 1)[-1]

def lowercase_ext(filename):
    if '.' in filename:
        main, ext = filename.rsplit('.', 1)
        return main + '.' + ext.lower()
    else:
        return filename.lower()

def addslash(url):
    if url.endswith('/'):
        return url
    return url + '/'

class UploadConfiguration(object):
    def __init__(self, destination, base_url=None, allow=(), deny=()):
        self.destination = destination
        self.base_url = base_url
        self.allow = allow
        self.deny = deny

    @property
    def tuple(self):
        return (self.destination, self.base_url, self.allow, self.deny)

    def __eq__(self, other):
        return self.tuple == other.tuple

class UploadSet:
    def __init__(self, name='files', extensions=DEFAULTS):
        if not name.isalnum():
            raise ValueError("Name must be alphanumeric (no underscores)")
        self.name = name
        self.extensions = extensions
        self._config = None

    @property
    def config(self):
        if self._config is not None:
            return self._config
        try:
            return _flup.upload_sets_config[self.name]
        except AttributeError:
            raise RuntimeError("cannot access configuration outside request")

    def url(self, filename):
        base = self.config.base_url
        if base is None:
            return url_for('_uploads.uploaded_file', setname=self.name,
                           filename=filename, _external=True)
        else:
            return base + filename

    def path(self, filename, folder=None):
        if folder:
            target_folder = os.path.join(self.config.destination, folder)
        else:
            target_folder = self.config.destination
        return os.path.join(target_folder, filename)

    def file_allowed(self, storage, basename):
        return self.extension_allowed(extension(basename))

    def extension_allowed(self, ext):
        return ((ext in self.config.allow) or
                (ext in self.extensions and ext not in self.config.deny))

    def save(self, storage, folder=None, name=None):
        if not isinstance(storage, FileStorage):
            raise TypeError("storage must be a werkzeug.FileStorage")

        if folder is None and name is not None and "/" in name:
            folder, name = name.rsplit("/", 1)

        basename = lowercase_ext(secure_filename(storage.filename))

        if name:
            if name.endswith('.'):
                basename = name + extension(basename)
            else:
                basename = name

        if not self.file_allowed(storage, basename):
            raise UploadNotAllowed()

        if folder:
            target_folder = os.path.join(self.config.destination, folder)
        else:
            target_folder = self.config.destination
        if not os.path.exists(target_folder):
            os.makedirs(target_folder)
        if os.path.exists(os.path.join(target_folder, basename)):
            basename = self.resolve_conflict(target_folder, basename)

        target = os.path.join(target_folder, basename)
        storage.save(target)
        if folder:
            return posixpath.join(folder, basename)
        else:
            return basename

    def resolve_conflict(self, target_folder, basename):
        name, ext = basename.rsplit('.', 1)
        count = 0
        while True:
            count = count + 1
            newname = '{}_{:d}.{}'.format(name, count, ext)
            if not os.path.exists(os.path.join(target_folder, newname)):
                return newname

class TestingFileStorage(FileStorage):
    def __init__(self, stream=None, filename=None, name=None,
                 content_type='application/octet-stream', content_length=-1,
                 headers=None):
        FileStorage.__init__(self,
                             stream,
                             filename,
                             name=name,
                             content_type=content_type,
                             content_length=content_length,
                             headers=None)
        self.saved = None

    def save(self, dst, buffer_size=16384):
        if isinstance(dst, str):
            self.saved = dst
        else:
            self.saved = dst.name

class Flup(object):
    """
    :param app:         The application
    :param upload_sets: A list of instances of UploadSets
    """
    def __init__(self,
                 app=None,
                 upload_sets=None):
        self.app = app
        self.upload_sets = upload_sets
        self.upload_sets_config = {}

        if app is not None:
            self.app = app
            self.init_app(self.app)
        else:
            self.app = None

    def init_app(self, app):
        self.register_upload_sets(app, self.upload_sets)

        should_serve = any(s.base_url is None
                           for s in iter(self.upload_sets_config.values()))

        if '_uploads' not in app.blueprints and should_serve:
            app.register_blueprint(self._blueprint)

        app.extensions['flup'] = self

    def register_upload_sets(self, app, upload_sets):
        for uset in upload_sets:
            uset_config = self.config_for_set(uset, app)
            self.upload_sets_config[uset.name] = uset_config

    def config_for_set(self, uset, app):
        app_config = app.config
        prefix = 'UPLOADED_{}_'.format(uset.name.upper())
        using_defaults = False

        app_default_dest = app_config.get('UPLOADS_DEFAULT_DEST', None)
        app_default_url = app_config.get('UPLOADS_DEFAULT_URL', None)

        allow_extns = tuple(app_config.get('{}{}'.format(prefix, 'ALLOW'), ()))
        deny_extns = tuple(app_config.get('{}{}'.format(prefix, 'DENY'), ()))
        destination = app_config.get('{}{}'.format(prefix, 'DEST'))
        base_url = app_config.get('{}{}'.format(prefix, 'URL'))

        if destination is None:
            if app_default_dest:
                destination = os.path.join(app_default_dest, uset.name)
                using_defaults = True

        if destination is None:
            raise RuntimeError("""
                               no destination for set designated '{}' as {}\n
                               no application config var for '{}'\n
                               """.format(uset.name,
                                          '{}{}'.format(prefix, 'DEST'),
                                          'UPLOADS_DEFAULT_DEST')
                               )

        if base_url is None and using_defaults and app_default_url:
            base_url = addslash(app_default_url) + uset.name + '/'

        return UploadConfiguration(destination, base_url,
                                   allow_extns,
                                   deny_extns)

    @property
    def _blueprint(self):
        uploads_blueprint = Blueprint('_uploads',
                                      __name__,
                                      url_prefix='/_uploads')

        def uploaded_file(setname, filename):
            config = _flup.upload_sets_config.get(setname, None)
            if config is None:
                abort(404)
            return send_from_directory(config.destination, filename)

        uploads_blueprint.add_url_rule('/<setname>/<path:filename>',
                                       view_func=uploaded_file)

        return uploads_blueprint


# -*- coding: utf-8 -*-

u"""编译阶段"""

import subprocess
import logging
import glob
import os
import sys
import string
import codecs

from buildexp import BuildException
import butil

class CompilePhase(object):

    def __init__(self, build_prop):
        self._logger = logging.getLogger(__name__)
        self._build_prop = build_prop
        self._back_dir = os.path.join(self._build_prop.path.dist_dir, "bak")

        self._tasks = []
        self._determine_task()

    def _determine_task(self):
        build_type = self._build_prop.share.build_type

        if "debug" in build_type:
            self._tasks.append(DebugCompileTask(self._build_prop))

        if "release" in build_type:
            self._tasks.append(ReleaseCompileTask(self._build_prop))

    def compile_(self):
        u"""编译阶段"""
        self._init_dist_dir()
        self._init_backup_dir()
        self._generate_string_translator()
        self._generate_qm()

        if self._build_prop.package.inject_version:
            self._backup_version_relative_files()
            try:
                self._generate_version_relative_files()
                self._execute_all_task()
            except:
                raise
            finally:
                self._restore_version_relative_files()
        else:
            self._execute_all_task()

    def _init_dist_dir(self):
        u"""初始化dist目录。这个目录用于存放构建中生成中间文件（除了编译所生成的）
        这个目录需要在编译开始前创建，不能早也不能晚
        """
        butil.mkdir(self._build_prop.path.dist_dir)

    def _init_backup_dir(self):
        butil.mkdir(self._back_dir)

    def _generate_string_translator(self):
        u"""生成与国际化相关的代码文件"""
        returncode = subprocess.call(["python",
            "script/xmlstringtranslator.py",
            "bin/release/layout",
            "bin/release/addons/addonsmanager/resource",
            "src/yyapp/xml_string_translator.h",
            "src/yyapp/yyapp.pro",
            "HEADERS += xml_string_translator.h"],
            cwd=self._build_prop.path.app_dir)
        if returncode != 0:
            raise BuildException("String translatation failed")

    def _generate_qm(self):
        process = subprocess.Popen([
            "lrelease",
            "-compress",
            "../framework/src/duifw/duifw_zh.ts",
            "src/yycommon/yycommon_zh.ts",
            "src/yyapp/yyapp_zh.ts",
            "src/yymainframe/yymainframe_zh.ts",
            "src/yychannel/yychannel_zh.ts",
            "src/yyim/yyim_zh.ts",
            "src/yyhistroymsg/yymsg_zh.ts",
            "src/yygroup/yygroup_zh.ts",
            "src/yggroup/yggroup_zh.ts",
            "src/yyplugin/yyplugin_zh.ts",
            "src/addonfw/addonfw_zh.ts",
            "-qm",
            "src/yyapp/yyapp_zh.qm",
            ],
            cwd=self._build_prop.path.app_dir,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        process.wait()

        if process.returncode != 0:
            output_normalize = process.communicate()[1].replace("\r\n", "\n")
            self._logger.debug("\n%s" % output_normalize)
            sys.stderr.write(output_normalize)
            raise BuildException("Generate .qm file failed")

        butil.copy_file_to_dir("app/bin/release/lang", "app/src/yyapp",
            ["yyapp_zh.qm"])

    def _backup_version_relative_files(self):
        self._backup_productinfo()
        self._backup_protocol_module_version()

    def _backup_productinfo(self):
        butil.copy_file_to_dir(self._back_dir,
            "version",
            ["productinfo.cpp"])

    def _backup_protocol_module_version(self):
        butil.copy_file_to_dir(self._back_dir,
            "mw/protocol4/protocol",
            ["ProtocolModuleVersion.h"])

    def _generate_version_relative_files(self):
        self._generate_productinfo()
        self._generate_protocol_module_version()

    def _generate_productinfo(self):
        file_path = "version/productinfo.cpp"
        file_ = codecs.open(file_path, mode="r", encoding="GBK")

        template = string.Template(file_.read())
        result = template.substitute(self._build_prop.version.about.__dict__)
        file_.close()

        file_ = codecs.open(file_path, mode="w", encoding="GBK")
        file_.write(result)
        file_.close()

        butil.copy_file(file_path,
            os.path.join(self._back_dir, "%s.modify" % os.path.basename(file_path)))

    def _generate_protocol_module_version(self):
        file_path = "mw/protocol4/protocol/ProtocolModuleVersion.h"

        rev_string = self._build_prop.version.about.last_commit_rev
        if rev_string != '[rev]' and len(rev_string) > 1:
            rev = int(rev_string[1:])
        else:
            rev = 0

        if rev != 0:
            lines = []
            with codecs.open(file_path, mode="r", encoding="GBK") as f_read:
                lines = f_read.readlines()
                for index, line in enumerate(lines):
                    if line.startswith('#define REVISION_VERSION'):
                        lines[index] = '#define REVISION_VERSION %d\r\n' % rev
            with codecs.open(file_path, mode="w", encoding="GBK") as f_write:
                f_write.write(''.join(lines))

        butil.copy_file(file_path,
            os.path.join(self._back_dir, "%s.modify" % os.path.basename(file_path)))

    def _execute_all_task(self):
        for task in self._tasks:
            task.execute()

    def _restore_version_relative_files(self):
        self._restore_productinfo()
        self._restore_protocol_module_version()

    def _restore_productinfo(self):
        butil.copy_file_to_dir("version",
            self._back_dir,
            ["productinfo.cpp"])

    def _restore_protocol_module_version(self):
        butil.copy_file_to_dir("mw/protocol4/protocol",
            self._back_dir,
            ["ProtocolModuleVersion.h"])

class AbstractCompileTask(object):
    u"""提供编译任务的模板方法"""

    def __init__(self, build_prop):
        self._logger = logging.getLogger(__name__)
        self._build_prop = build_prop
        self._projects = [
            butil.Project("misc",
                "yymisc.sln",
                os.path.join(self._build_prop.path.tools_dir, "yymisc"),
                True),
            butil.Project("yy",
                "yy.sln",
                ".",
                True),
            butil.Project("crashreport",
                "crashreport.sln",
                self._build_prop.path.crash_report_dir,
                True),
            butil.Project("yy_checker",
                "yy_checker.sln",
                os.path.join(self._build_prop.path.tools_dir, "yy_checker"),
                True),
            butil.Project("duospeak",
                "duospeak.sln",
                os.path.join(self._build_prop.path.tools_dir, "duospeak"),
                True),
            butil.Project("yylauncher",
                "yylauncher.sln",
                self._build_prop.path.tools_dir,
                True),
        ]

    def execute(self):
        self._logger.info("execute compile task [%s]" % self._get_task_name())

        self._generate_pro_files()

        if self._build_prop.compile.recompile:
            self._clean_by_devenv(self._projects, self._get_vc_config_name())

        self._compile_by_devenv(self._projects, self._get_vc_config_name())
        self._unit_test()

    def _get_task_name(self):
        u"""供子类使用，定义用于标识task的名字，如Debug"""
        raise BuildException("_get_task_name must be overrided")

    def _generate_pro_files(self):
        u"""调用qmake，生成.sln和.vcproj文件"""
        command_list = ["qmake", "-tp", "vc", "-r"]

        if self._build_prop.share.official_build:
            command_list.append("DEFINES+=OFFICIAL_BUILD")

        if self._build_prop.share.coverage:
            command_list.append("DEFINES+=CODE_COVERAGE")
            command_list.append("QMAKE_LFLAGS_RELEASE+=/debugtype:cv,fixup")

        self._get_addtional_qmake_arguments(command_list)

        returncode = subprocess.call(command_list)

        if returncode != 0:
            raise BuildException("Invoke qmake failed")

    def _get_addtional_qmake_arguments(self, qmake_command_list):
        u"""供子类使用，添加附加的qmake参数"""
        pass

    def _get_vc_config_name(self):
        u"""供子类使用，定义vc的config名，用于调用devenv时使用"""
        raise BuildException("_get_vc_config_name must be overrided")

    def _clean_by_devenv(self, project_list, config):
        for project in project_list:
            devenv = butil.Devenv(self._build_prop)
            devenv(project,
                "clean",
                config)

    def _compile_by_devenv(self, project_list, config):
        for project in project_list:
            devenv = butil.Devenv(self._build_prop)
            devenv(project,
                "build",
                config)

    def _unit_test(self):
        u"""供子类使用，单元测试"""
        raise BuildException("_unit_test must be overrided")

class DebugCompileTask(AbstractCompileTask):
    u"""编译Debug版"""
    def _get_task_name(self):
        return "Debug"

    def _get_vc_config_name(self):
        return "Debug|Win32"

    def _unit_test(self):
        unittest_dir = "/app/bin/debug"
        returncode = subprocess.call([
            "app/bin/debug/dwutest.exe",
            unittest_dir,
            "--gtest_output=xml:log/unittest/dwutestd.xml",
        ])

        if returncode == 0:
            self._logger.info("%s: unittest success" % unittest_dir)
        else:
            raise BuildException("%s: unittest failed, returncode: %d"
                % (unittest_dir, returncode))

class ReleaseCompileTask(AbstractCompileTask):
    u"""编译Release版"""
    def _get_task_name(self):
        return "Release"

    def _get_vc_config_name(self):
        return "Release|Win32"

    def _unit_test(self):
        unittest_dir = "/app/bin/release"
        returncode = subprocess.call([
            "app/bin/release/dwutest.exe",
            unittest_dir,
            "--gtest_output=xml:log/unittest/dwutest.xml",
        ])

        if returncode == 0:
            self._logger.info("%s: unittest success" % unittest_dir)
        else:
            raise BuildException("%s: unittest failed, returncode: %d"
                % (unittest_dir, returncode))


import json
import pandas
import numpy
import os
from itertools import chain
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.base import BaseEstimator, TransformerMixin

class ForecastTransformer(BaseEstimator, TransformerMixin):

    def __init__(self, n_time_steps=8, prediction_steps=1):
        self.n = n_time_steps
        self.pred_steps = prediction_steps
        self.tuple_index = None

    def fit(self, X, y=None):
        """

        Args:
            X (pandas.DataFrame):
            y (pandas.Series, numpy.array):

        Returns:

        """
        top_columns = [['t_' + str(self.n - y) for x in range(self.n) for y in [x]*len(X.columns)] + ['t_0'],
                       list(X.columns)*self.n + ['target']]
        self.tuple_index = list(zip(*top_columns))
        return self

    def transform(self, X, y=None, live=False):
        data = []
        index = []
        for i, sub_df in X.groupby(numpy.arange(len(X))//self.n):

            target = sub_df['price'].values[-1]
            index.append(sub_df.index[-1])
            new_row = list(chain.from_iterable(sub_df.values.tolist())) + [target]
            data.append(new_row)

            if len(new_row) != len(self.tuple_index):
                continue

        forecasting = pandas.DataFrame(data, columns=pandas.MultiIndex.from_tuples(self.tuple_index), index=index)
        forecasting[('t_0', 'target')] = forecasting[('t_0', 'target')].shift(-self.pred_steps)
        if len(forecasting) > 1:
            forecasting = forecasting.dropna()
            return forecasting[:-self.pred_steps].values
        else:
            forecasting = forecasting.fillna(1)
            return forecasting.values

class TimeEmbedder(BaseEstimator, TransformerMixin):

    def __init__(self, inital_dims):
        """
        Transform to a time-embedded array from a single long array. I.e., group features into their
        buckets so that they actually represent data from related timestamps

        Args:
            inital_dims (int): initial dimensions of array
        """
        self.initial_dims = inital_dims
        self.middle_shape = None

    def fit(self, X, y=None):
        self.middle_shape = (X.shape[1]-1)//self.initial_dims
        return self

    def transform(self, X):
        y = X[:, -1]
        X = X[:, :-1]
        return numpy.reshape(X, (X.shape[0], self.middle_shape, self.initial_dims)), y

def get_data(source, remote=False, keep_keys=list(['ts']),
             categorical=list(['side'])):
    file_path = '/media/carlo/HDD/kafka_local/'
    file_path = os.path.join(file_path, source + '.txt')
    rows = []
    with open(file_path) as inf:
        for i, row in enumerate(inf):
            if remote:
                if i >= 8:
                    break
            row_dict = json.loads(row)
            rows.append({k: v for k, v in row_dict.items() if k in keep_keys})

    df = pandas.DataFrame(rows)
    df.index = pandas.to_datetime(df['ts'])
    df['ts'] = pandas.to_datetime(df['ts'])
    df['time_diff'] = df['ts'].diff().dt.seconds.div(1, fill_value=0)
    if categorical:
        df = one_hot_encode(df, categorical)
    df = df.drop('ts', 1)
    return df

def one_hot_encode(df, categ_vars):
    ohe = []
    for categ in categ_vars:
        ohe.append(pandas.get_dummies(df[categ]))
        df = df.drop(categ, 1)
    return pandas.concat([df] + ohe, 1)

if __name__ == '__main__':

    local_df = get_data('gdax', remote=False, keep_keys=['ts', 'price', 'volume_24h',
                                                         'spread', 'side'])  # type: pandas.DataFrame

    remote_df = get_data('gdax', remote=True, keep_keys=['ts', 'price', 'volume_24h',
                                                         'spread', 'side'])  # type: pandas.DataFrame

    pipe = Pipeline([
        ('tr', ForecastTransformer()),
        ('scaler', MinMaxScaler()),
        ('time', TimeEmbedder(inital_dims=len(local_df.columns)))
    ])

    x, y = pipe.fit_transform(local_df)


#coding:utf-8
from numpy import *
import pandas as pd

def cutsearch(str):
    import jieba
    seg_list=jieba.cut(str)
    return seg_list

def loaduselesswords():   #把停用词读取到字典里，一是因为字典查找效率高，而是因为字典不会（破坏原来的中文字符串）
    useless={}
    f = open('useless.txt', 'r')
    # f = codecs.open('test.txt', encoding='UTF-8')
    for word in f.readlines():
        word.decode(encoding='utf-8').encode(encoding='utf-8')
        word = word.strip()
        useless[word]=1
        # print word
        # print repr(word)
    f.close()
    return useless

def loadTrainData(train_sum,test_sum):#train_sum为训练集的大小,test_sum是测试集的大小，两个集是从2W中连续取的
    j=0
    f=open('user_tag_query.2W.TRAIN')#there is 20000 records
    train_fullwords=[]
    train_searchList=[] #存储训练集里所有出现的词，每个用户为一行
    train_ageList=[] #得到的标签，每个元素是整型元素，而不是字符串
    train_GenderList=[]#记录性别标签
    train_EducationList=[]#记录学历标签

    test_searchList=[]
    test_ageList=[]
    test_GenderList=[]
    test_EducationList=[]
    for user in f.readlines():
        if j<train_sum:
            userdata=user.decode(encoding='gb18030').encode(encoding='utf-8')
            # userdata=userdata.encode(encoding='utf-8')
            data=userdata.strip().split('\t')
            train_ageList.append(data[1])
            train_GenderList.append(data[2])
            train_EducationList.append(data[3])
            singlesearch = []
            for i in range(4,len(data)):
                singlesearch.extend(cutsearch(data[i]))
            train_searchList.append(singlesearch)
            train_fullwords.extend(singlesearch)

        else:
            userdata = user.decode(encoding='gb18030').encode(encoding='utf-8')
            data = userdata.strip().split('\t')
            test_ageList.append(data[1])
            test_GenderList.append(data[2])
            test_EducationList.append(data[3])
            singlesearch = []
            for i in range(4, len(data)):
                singlesearch.extend(cutsearch(data[i]))
            test_searchList.append(singlesearch)

        j += 1  # 不要忘记自增
        if j == train_sum + test_sum: break
    f.close()
    return train_fullwords,train_searchList,train_ageList,train_GenderList,train_EducationList,\
             test_searchList,test_ageList,test_GenderList,test_EducationList

def loadTrainData_(train_sum,test_sum):#train_sum为训练集的大小,test_sum是测试集的大小，两个集是从2W中连续取的
    j=0
    f=open('user_tag_query.2W.TRAIN')#there is 20000 records
    train_fullwords = []
    train_searchList=[] #存储训练集里所有出现的词，每个用户为一行
    train_ageList=[] #得到的标签，每个元素是整型元素，而不是字符串
    train_GenderList=[]#记录性别标签
    train_EducationList=[]#记录学历标签

    test_searchList=[]
    test_ageList=[]
    test_GenderList=[]
    test_EducationList=[]
    for user in f.readlines():
        if j<test_sum:
            userdata=user.decode(encoding='gb18030').encode(encoding='utf-8')
            # userdata=userdata.encode(encoding='utf-8')
            data=userdata.strip().split('\t')
            train_ageList.append(data[1])
            train_GenderList.append(data[2])
            train_EducationList.append(data[3])
            singlesearch = []
            for i in range(4,len(data)):
                singlesearch.extend(cutsearch(data[i]))
            train_searchList.append(singlesearch)
            train_fullwords.extend(singlesearch)

            test_ageList.append(data[1])
            test_GenderList.append(data[2])
            test_EducationList.append(data[3])
            singlesearch = []
            for i in range(4, len(data)):
                singlesearch.extend(cutsearch(data[i]))
            test_searchList.append(singlesearch)

        else:
            userdata = user.decode(encoding='gb18030').encode(encoding='utf-8')
            data = userdata.strip().split('\t')
            train_ageList.append(data[1])
            train_GenderList.append(data[2])
            train_EducationList.append(data[3])
            singlesearch = []
            for i in range(4, len(data)):
                singlesearch.extend(cutsearch(data[i]))
            train_searchList.append(singlesearch)
            train_fullwords.extend(singlesearch)

        j += 1  # 不要忘记自增
        if j == train_sum: break
    f.close()
    return train_fullwords,train_searchList,train_ageList,train_GenderList,train_EducationList,\
             test_searchList,test_ageList,test_GenderList,test_EducationList

def loadTestData():
    f = open('user_tag_query.2W.TEST')  # there is 20000 records
    Test_Id=[]
    Test_Search=[]
    # j=0
    for user in f.readlines():
        userdata=user.decode(encoding='gb18030').encode(encoding='utf-8')
        data=userdata.strip().split('\t')
        Test_Id.append(data[0])
        singlesearch = []
        for i in range(1,len(data)):
            singlesearch.extend(cutsearch(data[i]))
        Test_Search.append(singlesearch)
        # if j==200:break
        # j+=1
    return Test_Id,Test_Search

def count_IntofList(target,list):#计算标签列表中出现target标签的次数，target是字符形式的
    i=0
    for k in list:
        if k==target:i+=1
    return i

def trainNB_Gender(trainMatrix,trainCategory):#朴素贝叶斯分类器训练函数
    numTrainSearcher=len(trainMatrix)#一共多少个搜索者
    numWords=len(trainMatrix[0])#一个搜索者所拥有的词向量长度
    p0=count_IntofList('0',trainCategory)/float(numTrainSearcher)#计算标签中各种性别的比例
    p1=count_IntofList('1',trainCategory)/float(numTrainSearcher)
    p2=count_IntofList('2',trainCategory)/float(numTrainSearcher)
    p0Num=ones(numWords);p1Num=ones(numWords);p2Num=ones(numWords) #词计数向量，向量的大小与词集的大小相等
    for i in range(numWords):
        p0Num[i]=0.01
        p1Num[i]=0.01
        p2Num[i]=0.01
    # print 'p0Num',p0Num
    p0Denom=1;p1Denom=1;p2Denom=1   #标签计算变量，记录各种标签的总数目
    for i in range(numTrainSearcher):
        if trainCategory[i]=='0':
            p0Num+=trainMatrix[i]
            p0Denom+=sum(trainMatrix[i])
        elif trainCategory[i]=='1':
            p1Num+=trainMatrix[i]
            p1Denom+=sum(trainMatrix[i])
        else:
            p2Num+=trainMatrix[i]
            p2Denom+=sum(trainMatrix[i])
    p0Vect=log(p0Num/float(p0Denom))
    p1Vect=log(p1Num/float(p1Denom))
    p2Vect=log(p2Num/float(p2Denom))
    return p0Vect,p1Vect,p2Vect,p0,p1,p2

def trainNB_Age(trainMatrix,trainCategory):
    numTrainSearcher = len(trainMatrix)  # 一共多少个搜索者
    numWords = len(trainMatrix[0])  # 一个搜索者所拥有的词向量长度,即词集的大小
    p0 = count_IntofList('0', trainCategory) / float(numTrainSearcher)  # 计算标签中各种性别的比例
    p1 = count_IntofList('1', trainCategory) / float(numTrainSearcher)
    p2 = count_IntofList('2', trainCategory) / float(numTrainSearcher)
    p3 = count_IntofList('3', trainCategory) / float(numTrainSearcher)
    p4 = count_IntofList('4', trainCategory) / float(numTrainSearcher)
    p5 = count_IntofList('5', trainCategory) / float(numTrainSearcher)
    p6 = count_IntofList('6', trainCategory) / float(numTrainSearcher)
    p0Num = ones(numWords)
    p1Num = ones(numWords)
    p2Num = ones(numWords)  # 词计数向量，向量的大小与词集的大小相等
    p3Num = ones(numWords)
    p4Num = ones(numWords)
    p5Num = ones(numWords)
    p6Num = ones(numWords)
    for i in range(numWords):
        p0Num[i] = 0.01
        p1Num[i] = 0.01
        p2Num[i] = 0.01
        p3Num[i] = 0.01
        p4Num[i] = 0.01
        p5Num[i] = 0.01
        p6Num[i] = 0.01
    # print 'p0Num',p0Num
    p0Denom = 1
    p1Denom = 1
    p2Denom = 1  # 标签计算变量，记录各种标签的总数目
    p3Denom = 1
    p4Denom = 1
    p5Denom = 1
    p6Denom = 1
    for i in range(numTrainSearcher):
        if trainCategory[i] == '0':
            p0Num += trainMatrix[i]
            p0Denom += sum(trainMatrix[i])
        elif trainCategory[i] == '1':
            p1Num += trainMatrix[i]
            p1Denom += sum(trainMatrix[i])
        elif trainCategory[i] == '2':
            p2Num += trainMatrix[i]
            p2Denom += sum(trainMatrix[i])
        elif trainCategory[i] == '3':
            p3Num += trainMatrix[i]
            p3Denom += sum(trainMatrix[i])
        elif trainCategory[i] == '4':
            p4Num += trainMatrix[i]
            p4Denom += sum(trainMatrix[i])
        elif trainCategory[i] == '5':
            p5Num += trainMatrix[i]
            p5Denom += sum(trainMatrix[i])
        else:
            p6Num += trainMatrix[i]
            p6Denom += sum(trainMatrix[i])
    p0Vect = log(p0Num / float(p0Denom))
    p1Vect = log(p1Num / float(p1Denom))
    p2Vect = log(p2Num / float(p2Denom))
    p3Vect = log(p3Num / float(p3Denom))
    p4Vect = log(p4Num / float(p4Denom))
    p5Vect = log(p5Num / float(p5Denom))
    p6Vect = log(p6Num / float(p6Denom))
    return p0Vect, p1Vect, p2Vect,p3Vect, p4Vect, p5Vect,p6Vect, p0, p1, p2, p3, p4, p5, p6

def classifyNB_Age(vec2Classify,p0Vect, p1Vect, p2Vect,p3Vect, p4Vect, p5Vect,p6Vect, p0_, p1_, p2_, p3_, p4_, p5_, p6_): #根据贝叶斯公式计算某个用户向量属于某个标签的概率，并选出概率最大的标签作为输出
    p0=sum(vec2Classify*p0Vect)+log(p0_)
    p1=sum(vec2Classify*p1Vect)+log(p1_)
    p2=sum(vec2Classify*p2Vect)+log(p2_)
    p3 = sum(vec2Classify * p3Vect) + log(p3_)
    p4 = sum(vec2Classify * p4Vect) + log(p4_)
    p5 = sum(vec2Classify * p5Vect) + log(p5_)
    p6 = sum(vec2Classify * p6Vect) + log(p6_)
    a=[p0,p1,p2,p3,p4,p5,p6]
    return str(argsort(a)[-1])

def classifyNB_Gender(vec2Classify,p0Vect,p1Vect,p2Vect,p0_,p1_,p2_): #根据贝叶斯公式计算某个用户向量属于某个标签的概率，并选出概率最大的标签作为输出
    # print 'start-------------------start'
    s0=sum(vec2Classify*p0Vect)
    s1=sum(vec2Classify*p1Vect)
    s2=sum(vec2Classify*p2Vect)
    p0=s0+log(p0_)
    p1=s1+log(p1_)
    p2=s2+log(p2_)
    # print s0,' ',s1,' ',s2
    # print p0,' ',p1,' ',p2
    # print 'end--------------------end'
    return (('0' if p0>p2 else '2') if p0>p1 else ('1' if p1>p2 else '2'))

def new_wordsfreq(fulltext):
    freqDict={}
    i=0
    for token in fulltext:
        if freqDict.has_key(token):freqDict[token]+=1
        else:freqDict[token]=1
    return freqDict

def decrease_vocabset(setandfreq,useless):#传入的是字典，把频次大于为1的词组成一个新的列表,返回的也写成字典比较好
    vocabset={}                 #返回的也是字典，因为之后查找效率会比较高
    for word in setandfreq.keys():
        # if useless.has_key(word):i=0
        # elif setandfreq[word]>5 :vocabset.extend(word)
        if not useless.has_key(word):
            if setandfreq[word]>2:
                vocabset[word]=1
    return vocabset

fullwords,train_searchList,train_AgeList,train_GenderList,train_EducationList,\
    test_searchList,test_AgeList,test_GenderList,test_EducationList=loadTrainData(2000,200) #得到训练和测试的词矩阵

# Test_Id,Test_Search=loadTestData()

useless = loaduselesswords()

setandfreq=new_wordsfreq(fullwords)
train_vocabset=decrease_vocabset(setandfreq,useless)
setandfreq.clear()         #清理内存
jia=0
for word in train_vocabset.keys():
    train_vocabset[word]=jia   #在这个地方之后各个词的频数已经没有用了，所以用values记录键的位置
    jia+=1
# print 'len',len(train_vocabset)

def bagofwords2vec(inpuset):#把输入的一行词转化为一个向量，向量的大小为词集合的大小，即把每个用户输入的搜索词条转为向量形式
    returnvec = [0]*len(train_vocabset.keys())
    for word in inpuset:
        if train_vocabset.has_key(word):
            returnvec[train_vocabset[word]] += 1
    return returnvec

train_Mat = []  # 把原来的训练矩阵由中文词转化为向量
for i in range(len(train_searchList)):
    train_Mat.append(bagofwords2vec(train_searchList[i]))

def algortithm_test_Gender():
    p0Vect,p1Vect,p2Vect,p0,p1,p2=trainNB_Gender(array(train_Mat),array(train_GenderList)) #得到分类器所需的数据
    errorcount=0
    # out_gender=[]
    for i in range(len(test_searchList)):
        wordvector=bagofwords2vec(test_searchList[i])
        gender=classifyNB_Gender(array(wordvector),p0Vect,p1Vect,p2Vect,p0,p1,p2)
        # out_gender.append(gender)
        if gender!=test_GenderList[i]:
            errorcount+=1
    print('the gender error rate is:',float(errorcount)/len(test_searchList))
    # return out_gender

def algortithm_test_Age():
    p0Vect,p1Vect,p2Vect,p3Vect,p4Vect,p5Vect,p6Vect,p0,p1,p2,p3,p4,p5,p6=trainNB_Age(array(train_Mat),array(train_AgeList)) #得到分类器所需的数据
    errorcount=0
    # out_age=[]
    for i in range(len(test_searchList)):
        wordvector=bagofwords2vec(test_searchList[i])
        age=classifyNB_Age(array(wordvector),p0Vect,p1Vect,p2Vect,p3Vect,p4Vect,p5Vect,p6Vect,p0,p1,p2,p3,p4,p5,p6)
        # out_age.append(age)
        if age!=test_AgeList[i]:
            errorcount+=1
    print('the age error rate is:',float(errorcount)/len(test_searchList))
    # return out_age

def algortithm_test_Education():
    p0Vect,p1Vect,p2Vect,p3Vect,p4Vect,p5Vect,p6Vect,p0,p1,p2,p3,p4,p5,p6=trainNB_Age(array(train_Mat),array(train_EducationList)) #得到分类器所需的数据
    errorcount=0
    # out_education=[]
    for i in range(len(test_searchList)):
        wordvector=bagofwords2vec(test_searchList[i])
        education=classifyNB_Age(array(wordvector),p0Vect,p1Vect,p2Vect,p3Vect,p4Vect,p5Vect,p6Vect,p0,p1,p2,p3,p4,p5,p6)
        # out_education.append(education)
        if education!=test_EducationList[i]:
            errorcount+=1
    print('the education error rate is:',float(errorcount)/len(test_searchList))
    # return out_education

algortithm_test_Gender()
algortithm_test_Age()
algortithm_test_Education()

# data = {'ID':Test_Id,'age':out_age,'gender':out_gender,'education':out_education}
# df = pd.DataFrame(data)
# df.to_csv('result.csv',sep=' ',columns=["ID",'age',"gender",'education'],index=False,header=False,encoding="gbk")


from django.db import models

"""
	Le model qui permet de creer les salles
"""
class Room(models.Model):
	appelation = models.CharField(max_length=200)
	notes = models.TextField(u'Commentaires')

	def __str__(self):
		return self.appelation

	class Meta:
		verbose_name = u'Room'

"""
	Le model qui permet de creer les salles
"""
class Motif(models.Model):
	appelation = models.CharField(max_length=200)
	notes = models.TextField(u'Commentaires')

	def __str__(self):
		return self.appelation

	class Meta:
		verbose_name = u'Motif'

"""
	Ce model contient l'evenement en lui meme (la reservation)
"""
class Event(models.Model):
	day_start = models.DateField(u'Jour debut (jj/mm/aaaa)')
	time_start = models.TimeField(u'Heure de debut (hh:mm)')
	day_end = models.DateField(u'Jour fin (jj/mm/aaaa)')
	time_end = models.TimeField(u'Heure de fin (hh:mm)')
	room_id = models.ForeignKey(Room, on_delete=models.CASCADE)
	visio = models.BooleanField(default=True)
	motif_id = models.ForeignKey(Motif, on_delete=models.CASCADE)
	notes = models.TextField(u'Theme / Commentaires')

	def __str__(self):
		return self.notes

	class Meta:
		verbose_name = u'Event'

	def check_overlap(self,fixed_start, fixed_end, new_start, new_end):
			overlap = False
			if new_start == fixed_end or new_end == fixed_start:
				overlap = False
			elif (new_start >= fixed_start and new_start <= fixed_end) or (new_end >= fixed_start and new_end <= fixed_end):
				overlap = True
			elif new_start <= fixed_start and new_end >= fixed_end:
				overlap = True
			return overlap

	def clean(self):
		if self.day_end <= self.day_start:
			raise ValidationError ('Ending time must be after starting time')

		# events = Event.objects.filter(day=self.day)
		# if events.exists():
		# 	for event in events:
		# 		if self.check_overlap(event.start_time, event.end_time, self.start_time, self.end_time):
		# 			raise ValidationError ('There is an overlap with another event: ' + str(event.day) + ', '+ str(event.start_time) + '-' + str(event.end_time))


#!/usr/bin/env python3

import argparse
import os
from subprocess import call
from wand.image import Image

__version__     = '0.2.0'
__author__      = 'Shawn McElroy'
__description__ = 'Convert text based files to other formats.'
__homepage__    = 'http://github.com/skiftio/txt2other'
__email__       = 'hey@skift.io'
__license__     = 'MIT'

parser = argparse.ArgumentParser(description='convert code to other format (ps, pdf, jpg).')
parser.add_argument('-r','--recursive', help='scan files recursively',                      default=False, action='store_true')
parser.add_argument('-f','--force',     help='overwrite any already generated files'      , default=False, action='store_true')
parser.add_argument('-d','--directory', help='top level directory to be scanned',           default='.',   required=False)
parser.add_argument('-i','--input file',help='file to be scanned',                          default=False, required=False)
parser.add_argument('-t','--type',      help='format to be converted to',                   default='pdf', required=False)
parser.add_argument('-o','--out',       help='location where the output will be generated', default='output', required=False)
parser.add_argument('-y','--only',      help='only convert files with these extensions. Comma seperated list (php,py,txt)',
                    default=False, required=False)
args = vars(parser.parse_args())

class txt2jpg():

    def __init__(self, out, recr, ftype, force, only=False):

        self.out   = out
        self.recr  = recr
        self.ftype = ftype
        self.force = force

        self.ignorePaths = (
            '__pycache__',
            'cache',
            'config',
            'console',
            'git',
            'idea',
            'lib',
            'locale',
            'node_modules',
            'out',
            'output',
            'plugin',
            'test',
            'tmp',
            'uploads',
            'vendor',
            'bootstrap',
            'sass',
            'font',
            'logs'
        )

        self.ignoreFiles = [
            'code2jpg.iml',
            'gruntfile',
            'composer',
            'bootstrap.css',
            'bootstrap.js',
            'jquery',
            'test',
            '.min.',
            'travis.yml'
        ]

        self.ignoreExtensions = (
            'jpg',
            'ps',
            'pdf',
            'gitignore',
            'md',
            'iml',
            'gitignore',
            'htaccess',
            'gitattributes',
            'project',
            'json'
        )

        if only is not False:
            self.onlyExtensions = only
            ign = list(self.ignoreExtensions)
            [ign.remove(ext) for ext in self.onlyExtensions if ext in self.ignoreExtensions]
            self.ignoreExtensions = tuple(ign)

        else:
            self.onlyExtensions = (
                'php',
                'js',
                'css',
                'ctp',
                'py'
            )

    def parseFile(self, file, fold, curPath, depth=0):
        if self.ignoreFile(file, curPath):
            return

        if fold == '/':
            output  = self.out + '/'
        else:
            output = self.out + fold + '/'

        self.processFiles(file, output, curPath)

    def parseDir(self, fold, curPath, depth=0):
        # print('')

        if self.ignoreDir(fold):
            return

        # print('Folder:[' + fold + ']')
        # print('Path  :[' + curPath + ']')

        # make sure path exists
        if not os.path.isdir(curPath):
            self.badPath(fold)

        # we need to make sure this dir is in the output dir
        if not os.path.isdir(self.out):
            self.cmd(['mkdir', '-p', self.out])

        newOutput = self.out + fold + '/'
        print('Output    :[' + newOutput + ']')
        if not os.path.isdir(newOutput):
            self.cmd(['mkdir', '-p', newOutput])

        currentItems = os.listdir(curPath)

        dirs  = []
        files = []
        other = []
        for item in currentItems:
            curr = curPath + item
            if os.path.isdir(curr) and not self.ignoreDir(item):
                dirs.append(item)
            elif os.path.isfile(curr) and not self.ignoreFile(item, curPath):
                files.append(item)
            else:
                other.append(item)

        # print('dir items:')
        # print(dirs)
        # print('file item:')
        # print(files)
        # print('other item:')
        # print(other)

        # first lest process the current files
        if len(files) > 0:
            [self.parseFile(newFile, fold, curPath, depth) for newFile in files]

        # parse directories, only if recursive is true
        if self.recr is True and len(dirs) > 0:
            [self.parseDir(fold + newDir + '/', curPath + newDir + '/', depth+1) for newDir in dirs]

    def ignoreFile(self, file, path):
        name = file.lower()

        if len(self.onlyExtensions) > 0 and not name.endswith(self.onlyExtensions):
            # if we have an only extension list, lets use that
            return True

        if name.endswith(self.ignoreExtensions):
            # check for extension
            return True

        for ignore in self.ignoreFiles:
            if ignore in name or name == ignore:
                return True

        filepath = path + name
        if not os.path.isfile(filepath) or os.path.islink(filepath):
            # check if is real file
            return True

        return False

    def ignoreDir(self, fold):
        fold = fold.lower()
        for ignore in self.ignorePaths:
            if ignore in fold or ignore == fold:
                # make sure were not ignoring this
                return True

        return False

    def processFiles(self, file, outPath, curPath):
        # files
        curFile = curPath + file
        outFile = outPath + file
        psFile  = outFile + '.ps'
        pdfFile = outFile + '.pdf'
        jpgFile = outFile + '.jpg'

        # start generating
        print('converting:[' + curFile + '] -> [' + outFile + '.' + self.ftype + ']')

        # check if file was already generated. No need to if already there.
        if self.force is False and \
            (os.path.isfile(outFile + '.' + self.ftype)
             or os.path.isfile(outFile + '-0.' + self.ftype)):
            return

        if self.ftype == 'ps':
            self.makePs(curFile, psFile)

        if self.ftype == 'pdf':
            self.makePdf(curFile, psFile, pdfFile)

        if self.ftype == 'jpg':
            altFile = outFile + '-0' + '.jpg'
            self.makeJpg(curFile, psFile, pdfFile, jpgFile)

        # remove ps file
        if self.ftype != 'ps': self.cmd(['rm', psFile])

        # remove ps file
        if self.ftype != 'pdf': self.cmd(['rm', pdfFile])

        # remove pdf file
        if self.ftype != 'jpg': self.cmd(['rm', jpgFile])

    def makePs(self, curFile, psFile):
        # easiest file to make
        self.cmd(['enscript', curFile, '-p', psFile])

    def makePdf(self, curFile, psFile, pdfFile):
        # we need a ps file first
        self.makePs(curFile, psFile,)
        self.cmd(['ps2pdf', psFile, pdfFile])

    def makeJpg(self, curFile, psFile, pdfFile, jpgFile):
        # first we need a pdf file
        self.makePdf(curFile, psFile, pdfFile)
        with Image(filename=pdfFile, resolution=300) as img:
            img.save(filename=jpgFile)

    def cmd(self, cmd):
        with open(os.devnull, "w") as fnull:
            return call(cmd, stdout=fnull, stderr=fnull)

    def badPath(self, fold):
        ## for lols mostly
        from random import choice
        comments = [
            'Bad path sir',
            'Did you spell that right',
            'Dude, really? That doesn\'t exist',
            'Not a real path',
            'No dice. No Folder',
            'PEBKAC Error. Folder does not exist'
        ]
        print(choice(comments) + ':[' + fold + ']')
        exit()

def main():
    # print(args)
    recr  = args['recursive']
    fold  = args['directory']
    file  = args['file']
    ftype = args['type']
    out   = args['out']
    force = args['force']
    only  = args['only']

    # make sure output dir is properly setup
    if not os.path.isdir(out):
        os.mkdir(out)
    out = os.path.realpath(out)

    # make sure current folder is correct
    if not os.path.isdir(fold):
        print('Not a directory:[' + fold + ']')

    if only is not True and 'str' in str(type(only)):
        only = tuple([x.strip() for x in only.split(',')])

    curPath = os.path.realpath(fold) + '/'

    t2j = txt2jpg(out=out, recr=recr, ftype=ftype, force=force, only=only)

    if file is not False:
        t2j.parseFile(file, '/', curPath=curPath)
    else:
        t2j.parseDir('/', curPath=curPath)

if __name__ == '__main__':
    main()


#!/usr/bin/python3

import json
import os
import subprocess
import sys

from contextlib import contextmanager

@contextmanager
def change_directory(directory):
    original = os.path.abspath(os.getcwd())

    os.chdir(directory)
    yield
    os.chdir(original)

def build(directory, tags, options):
    with change_directory(directory):
        subprocess.run([
            'docker', 'build',
                *options,
                *[f'--tag={tag}' for tag in tags],
                '.'], check=True)

    for tag in tags:
        subprocess.run(['docker', 'push', tag], check=True)

if __name__ == '__main__':
    build_options = sys.argv[1:]

    with open('config.json', 'r', encoding='utf-8') as input_file:
        config = json.load(input_file)

    for item in config['builds']:
        build(item['dockerfile'], item['tags'], build_options)