import operator
import os

from param import FeatParams, XPParams, Paths
import logging


class Corpus:
    """
        a class used to encapsulate all the information of the corpus
    """

    mweTokenDic, mweDictionary, mwtDictionary, mwtDictionaryWithSent = {}, {}, {}, {}

    def __init__(self, langName):
        """
            an initializer of the corpus, responsible of creating a structure of objects encapsulating all the information
            of the corpus, its sentences, tokens and MWEs.

            This function iterate over the lines of corpus document to create the precedent ontology
        """
        logging.warn('Language name: {0}'.format(langName))
        self.trainingSents, self.testingSents, self.trainDataSet = [], [], []
        path = os.path.join(Paths.corporaPath, langName)
        mweFile = os.path.join(path, 'train.parsemetsv')
        testMweFile = os.path.join(path, 'test.parsemetsv')
        conlluFile, testConllu = self.getTrainAndTestConlluPath(path)
        testFile = os.path.join(path, 'test.parsemetsv')

        if conlluFile and testConllu:
            self.trainDataSet = Corpus.readConlluFile(conlluFile)
            Corpus.readMweFile(mweFile, self.trainDataSet)

            for sent in self.trainDataSet:
                sent.recognizeEmbedded()
                sent.recognizeInterleavingVMWEs()
                sent.recognizeContinouosandSingleVMWEs()
            if XPParams.realExper:
                self.testDataSet = Corpus.readConlluFile(testConllu)
                Corpus.readMweFile(testMweFile, self.testDataSet)
        else:
            self.trainDataSet = Corpus.readSentences(mweFile)
            self.testDataSet = Corpus.readSentences(testFile, forTest=True)
            for sent in self.trainDataSet:
                sent.recognizeEmbedded()
                sent.recognizeInterleavingVMWEs()
                sent.recognizeContinouosandSingleVMWEs()
        self.getTrainAndTestSents()
        if XPParams.useCrossValidation:
            self.testRange, self.trainRange = self.getRangs()
        else:
            self.testRange, self.trainRange = None, None

    def getTrainAndTestConlluPath(self, path):
        conlluFile, testConllu = None, None
        if XPParams.useAutoGeneratedPOS and XPParams.useAutoGeneratedDEP and os.path.isfile(
                os.path.join(path, 'train.conllu.autoPOS.autoDep')):
            conlluFile = os.path.join(path, 'train.conllu.autoPOS.autoDep')
            if os.path.isfile(os.path.join(path, 'test.conllu.autoPOS.autoDep')):
                testConllu = os.path.join(path, 'test.conllu.autoPOS.autoDep')
        elif XPParams.useAutoGeneratedPOS and os.path.isfile(os.path.join(path, 'train.conllu.autoPOS')):
            conlluFile = os.path.join(path, 'train.conllu.autoPOS')
            if os.path.isfile(os.path.join(path, 'test.conllu.autoPOS')):
                testConllu = os.path.join(path, 'test.conllu.autoPOS')
        elif os.path.isfile(os.path.join(path, 'train.conllu')):
            conlluFile = os.path.join(path, 'train.conllu')
            if os.path.isfile(os.path.join(path, 'test.conllu')):
                testConllu = os.path.join(path, 'test.conllu')
        return conlluFile, testConllu

    @staticmethod
    def readConlluFile(conlluFile):
        sentences = []
        with open(conlluFile) as corpusFile:
            # Read the corpus file
            lines = corpusFile.readlines()
            sent = None
            senIdx = 0
            sentId = ''

            lineNum = 0
            missedUnTag = 0
            missedExTag = 0

            for line in lines:
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('# sentid:'):
                    sentId = line.split('# sentid:')[1].strip()


                elif line.startswith('# sentence-text:'):
                    continue

                elif line.startswith('1\t'):
                    if sentId.strip() != '':
                        sent = Sentence(senIdx, sentid=sentId)
                    else:
                        sent = Sentence(senIdx)
                    senIdx += 1
                    sentences.append(sent)

                if not line.startswith('#'):
                    lineParts = line.split('\t')

                    if len(lineParts) != 10 or '-' in lineParts[0]:
                        continue

                    lineNum += 1
                    if lineParts[3] == '_':
                        missedUnTag += 1
                    if lineParts[4] == '_':
                        missedExTag += 1

                    morpho = ''
                    if lineParts[5] != '_':
                        morpho = lineParts[5].split('|')
                    if lineParts[6] != '_':
                        token = Token(lineParts[0], lineParts[1].lower(), lemma=lineParts[2],
                                      abstractPosTag=lineParts[3], morphologicalInfo=morpho,
                                      dependencyParent=int(lineParts[6]),
                                      dependencyLabel=lineParts[7])
                    else:
                        token = Token(lineParts[0], lineParts[1].lower(), lemma=lineParts[2],
                                      abstractPosTag=lineParts[3], morphologicalInfo=morpho,
                                      dependencyLabel=lineParts[7])
                    if XPParams.useUniversalPOSTag:
                        token.posTag = lineParts[3]
                    else:
                        if lineParts[4] != '_':
                            token.posTag = lineParts[4]
                        else:
                            token.posTag = lineParts[3]
                    # Associate the token with the sentence
                    sent.tokens.append(token)
                    sent.text += token.text + ' '
        return sentences

    @staticmethod
    def readMweFile(mweFile, sentences):
        mweNum = 0
        with open(mweFile) as corpusFile:

            # Read the corpus file
            lines = corpusFile.readlines()
            noSentToAssign = False
            sentIdx = 0
            for line in lines:
                if line == '\n' or line.startswith('# sentence-text:') or (
                        line.startswith('# sentid:') and noSentToAssign):
                    continue
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('1\t'):
                    sent = sentences[sentIdx]
                    sentIdx += 1
                lineParts = line.split('\t')
                if '-' in lineParts[0]:
                    continue
                if lineParts is not None and len(lineParts) == 4 and lineParts[3] != '_':

                    token = sent.tokens[int(lineParts[0]) - 1]
                    vMWEids = lineParts[3].split(';')
                    for vMWEid in vMWEids:
                        id = int(vMWEid.split(':')[0])
                        # New MWE captured
                        if id not in sent.getWMWEIds():
                            if len(vMWEid.split(':')) > 1:
                                type = str(vMWEid.split(':')[1])
                                vMWE = VMWE(id, token, type)
                            else:
                                vMWE = VMWE(id, token)
                            mweNum += 1
                            sent.vMWEs.append(vMWE)
                        # Another token of an under-processing MWE
                        else:
                            vMWE = sent.getVMWE(id)
                            if vMWE is not None:
                                vMWE.addToken(token)
                        # associate the token with the MWE
                        token.setParent(vMWE)
        return mweNum

    @staticmethod
    def readSentences(mweFile, forTest=False):
        sentences = []
        sentNum, mweNum = 0, 0
        with open(mweFile) as corpusFile:
            # Read the corpus file
            lines = corpusFile.readlines()
            sent = None
            senIdx = 1
            for line in lines:
                if len(line) > 0 and line.endswith('\n'):
                    line = line[:-1]
                if line.startswith('1\t'):
                    # sentId = line.split('# sentid:')[1]
                    if sent is not None:
                        # Represent the sentence as a sequece of tokens and POS tags
                        sent.setTextandPOS()
                        # if not forTest:
                        sent.recognizeEmbedded()
                        sent.recognizeInterleavingVMWEs()

                    sent = Sentence(senIdx)
                    senIdx += 1
                    sentences.append(sent)

                elif line.startswith('# sentence-text:'):
                    if len(line.split(':')) > 1:
                        sent.text = line.split('# sentence-text:')[1]

                lineParts = line.split('\t')

                # Empty line or lines of the form: "8-9	can't	_	_"
                if len(lineParts) != 4 or '-' in lineParts[0]:
                    continue
                token = Token(lineParts[0], lineParts[1])
                # Trait the MWE
                # if not forTest and lineParts[3] != '_':
                if lineParts[3] != '_':
                    vMWEids = lineParts[3].split(';')
                    for vMWEid in vMWEids:
                        id = int(vMWEid.split(':')[0])
                        # New MWE captured
                        if id not in sent.getWMWEIds():
                            type = str(vMWEid.split(':')[1])
                            vMWE = VMWE(id, token, type)
                            mweNum += 1
                            sent.vMWEs.append(vMWE)
                        # Another token of an under-processing MWE
                        else:
                            vMWE = sent.getVMWE(id)
                            if vMWE is not None:
                                vMWE.addToken(token)
                        # associate the token with the MWE
                        token.setParent(vMWE)
                # Associate the token with the sentence
                sent.tokens.append(token)
            sentNum = len(sentences)
            return sentences

    @staticmethod
    def getMWEDic(sents):
        mweDictionary, mweTokenDictionary, mwtDictionary = {}, {}, {}
        for sent in sents:
            for mwe in sent.vMWEs:
                lemmaString = mwe.getLemmaString()
                if len(mwe.tokens) == 1:
                    if lemmaString not in mwtDictionary:
                        mwtDictionary[lemmaString] = mwe.type
                    if lemmaString not in Corpus.mwtDictionaryWithSent:
                        Corpus.mwtDictionaryWithSent[lemmaString] = [sent]
                    elif lemmaString in Corpus.mwtDictionaryWithSent and Corpus.mwtDictionaryWithSent[
                        lemmaString] is not None:
                        Corpus.mwtDictionaryWithSent[lemmaString] = Corpus.mwtDictionaryWithSent[lemmaString].append(
                            sent)
                if lemmaString in mweDictionary:
                    mweDictionary[lemmaString] += 1
                    for token in mwe.tokens:
                        if token.lemma.strip() != '':
                            mweTokenDictionary[token.lemma] = 1
                        else:
                            mweTokenDictionary[token.text] = 1
                else:
                    mweDictionary[lemmaString] = 1
                    for token in mwe.tokens:
                        if token.lemma.strip() != '':
                            mweTokenDictionary[token.lemma] = 1
                        else:
                            mweTokenDictionary[token.text] = 1
        if FeatParams.usePreciseDictionary:
            for key1 in mweDictionary.keys():
                for key2 in mweDictionary.keys():
                    if key1 != key2:
                        if key1 in key2:
                            mweDictionary.pop(key1, None)
                        elif key2 in key1:
                            mweDictionary.pop(key2, None)
        return mweDictionary, mweTokenDictionary, mwtDictionary

    def initializeSents(self, training=True):
        # Erasing each effect of the previous iteration
        sents = self.trainingSents
        if not training:
            sents = self.testingSents

        for sent in sents:
            sent.identifiedVMWEs = []
            sent.initialTransition = None
            sent.featuresInfo = []
            sent.blackMergeNum = 0
            for mwe in sent.vMWEs:
                mwe.isInTrainingCorpus = 0

    def getTrainAndTestSents(self):

        if XPParams.debug:
            self.trainingSents = self.trainDataSet[:500]
            self.testingSents = self.testDataSet[:200]
        elif XPParams.realExper:
            self.trainingSents = self.trainDataSet
            self.testingSents = self.testDataSet

        elif XPParams.useCrossValidation:
            self.trainDataSet = self.trainDataSet
            self.testDataSet = []
            return [], []



        if len(self.trainingSents) <= 0:
            idx = 0
            self.trainingSents, self.testingSents = [], []
            for sent in self.trainDataSet:
                if idx % 5 == 0:
                    self.testingSents.append(sent)
                else:
                    self.trainingSents.append(sent)
                idx += 1

        return self.trainingSents, self.testingSents

    def getRangs(self):
        sents = self.trainDataSet
        testNum = int(len(sents) * 0.2)
        testRanges = [[0, testNum], [testNum, 2 * testNum], [2 * testNum, 3 * testNum], [3 * testNum, 4 * testNum],
                      [4 * testNum, len(sents)]]

        trainRanges = [[testNum, len(sents)], [0, testNum, 2 * testNum, len(sents)],
                       [0, 2 * testNum, 3 * testNum, len(sents)], [0, 3 * testNum, 4 * testNum, len(sents)],
                       [0, 4 * testNum]]

        return testRanges, trainRanges

    def divideSents(self):
        x = XPParams.currentIteration
        if self.testRange is None or self.trainRange is None:
            return
        self.testingSents = self.trainDataSet[self.testRange[x][0]: self.testRange[x][1]]
        if len(self.trainRange[x]) == 2:
            self.trainingSents = self.trainDataSet[self.trainRange[x][0]: self.trainRange[x][1]]
        else:
            self.trainingSents = self.trainDataSet[self.trainRange[x][0]: self.trainRange[x][1]] + \
                                 self.trainDataSet[self.trainRange[x][2]: self.trainRange[x][3]]

    def update(self):
        if XPParams.useCrossValidation:
            self.divideSents()
            self.initializeSents()
        Corpus.mweDictionary, Corpus.mweTokenDic, Corpus.mwtDictionary = Corpus.getMWEDic(self.trainingSents)

    @staticmethod
    def getNewIdentifiedMWE(testingSents):

        idenMWEs = 0
        newIdenMWEs = 0
        semiNewIdenMWEs = 0
        for sent in testingSents:
            for mwe in sent.vMWEs:
                if mwe.getLemmaString() not in Corpus.mweDictionary.keys():
                    idenMWEs += 1
            for mwe in sent.identifiedVMWEs:
                if mwe.getLemmaString() not in Corpus.mweDictionary.keys():
                    for vmw1 in sent.vMWEs:
                        if vmw1.getLemmaString() == mwe.getLemmaString():
                            newIdenMWEs += 1
                            break
                elif mwe.getLemmaString() in Corpus.mweDictionary.keys() and \
                        Corpus.mweDictionary[mwe.getLemmaString()] < 5:
                    semiNewIdenMWEs += 1

        return float(newIdenMWEs) / idenMWEs, float(semiNewIdenMWEs) / idenMWEs

    def __iter__(self):
        for sent in self.trainingSents:
            yield sent

    def __str__(self):
        res = ''
        for sent in self.testingSents:
            tokenList = []
            for token in sent.tokens:
                tokenList.append(token.text.strip())
            labels = ['_'] * len(tokenList)
            for mwe in sent.identifiedVMWEs:
                for token in mwe.tokens:
                    if labels[token.position - 1] == '_':
                        labels[token.position - 1] = str(mwe.id)
                    else:
                        labels[token.position - 1] += ';' + str(mwe.id)
                    if mwe.tokens[0] == token and mwe.type:
                        labels[token.position - 1] += ':' + mwe.type

            for i in range(len(tokenList)):
                res += '{0}\t{1}\t{2}\t{3}\n'.format(i + 1, tokenList[i], '_', labels[i])
            res += '\n'
        return res

class Sentence:
    """
       a class used to encapsulate all the information of a sentence
    """

    def __init__(self, id, sentid=''):

        self.sentid = sentid
        self.id = id
        self.tokens = []
        self.vMWEs = []
        self.identifiedVMWEs = []
        self.text = ''
        self.initialTransition = None
        self.featuresInfo = []
        self.containsEmbedding = False
        self.containsInterleaving = False
        self.containsDistributedEmbedding = False
        self.withRandomSelection = False
        self.blackMergeNum, self.interleavingNum, self.embeddedNum, self.distributedEmbeddingNum = 0, 0, 0, 0

    def getWMWEs(self):
        return self.vMWEs

    def getWMWEIds(self):
        result = []
        for vMWE in self.vMWEs:
            result.append(vMWE.getId())
        return result

    def getVMWE(self, id):

        for vMWE in self.vMWEs:
            if vMWE.getId() == int(id):
                return vMWE
        return None

    def setTextandPOS(self):

        tokensTextList = []
        for token in self.tokens:
            self.text += token.text + ' '
            tokensTextList.append(token.text)
        self.text = self.text.strip()

    def recognizeEmbedded(self, recognizeIdentified=False):
        if recognizeIdentified:
            vmws = self.identifiedVMWEs
        else:
            vmws = self.vMWEs

        if len(vmws) <= 1:
            return 0
        result = 0
        # [x1; x2; x3
        for vMwe1 in vmws:
            if vMwe1.isEmbedded:
                continue
            for vMwe2 in vmws:
                if vMwe1 is not vMwe2 and len(vMwe1.tokens) < len(vMwe2.tokens):
                    if vMwe1.getString() in vMwe2.getString():
                        vMwe1.isEmbedded = True
                        if not recognizeIdentified:
                            self.embeddedNum += 1
                            self.containsEmbedding = True
                        result += 1
                    else:
                        isEmbedded = True
                        vMwe2Lemma = vMwe2.getLemmaString()
                        for token in vMwe1.tokens:
                            if token.getLemma() not in vMwe2Lemma:
                                isEmbedded = False
                                break
                        if isEmbedded:
                            vMwe1.isDistributedEmbedding = True
                            vMwe1.isEmbedded = True
                            if not recognizeIdentified:
                                self.containsDistributedEmbedding = True
                                self.embeddedNum += 1
                                self.distributedEmbeddingNum += 1
                                self.containsEmbedding = True
                            result += 1
        if not recognizeIdentified:
            self.getDirectParents()
        return result

    def recognizeContinouosandSingleVMWEs(self):
        singleWordExp, continousExp = 0, 0
        for mwe in self.vMWEs:
            if len(mwe.tokens) == 1:
                mwe.isSingleWordExp = True
                mwe.isContinousExp = True
                singleWordExp += 1
                continousExp += 1
            else:
                if self.isContinousMwe(mwe):
                    continousExp += 1
        return singleWordExp, continousExp

    def isContinousMwe(self, mwe):
        idxs = []
        for token in mwe.tokens:
            idxs.append(self.tokens.index(token))
        range = xrange(min(idxs), max(idxs))
        mwe.isContinousExp = True
        for i in range:
            if i not in idxs:
                mwe.isContinousExp = False
        return mwe.isContinousExp

    def recognizeInterleavingVMWEs(self):
        if len(self.vMWEs) <= 1:
            return 0
        result = 0
        for vmwe in self.vMWEs:
            if vmwe.isEmbedded or vmwe.isInterleaving:
                continue
            for token in vmwe.tokens:
                if len(token.parentMWEs) > 1:
                    for parent in token.parentMWEs:
                        if parent is not vmwe:
                            if parent.isEmbedded or parent.isInterleaving:
                                continue
                            if len(parent.tokens) <= len(vmwe.tokens):
                                parent.isInterleaving = True
                            else:
                                vmwe.isInterleaving = True
                            self.containsInterleaving = True
                            self.interleavingNum += 1
                            result += 1
        return result

    def getCorpusText(self, gold=True):
        if gold:
            mwes = self.vMWEs
        else:
            mwes = self.identifiedVMWEs
        lines = ''
        idx = 1
        for token in self.tokens:
            line = str(idx) + '\t' + token.text + '\t_\t'
            idx += 1
            for mwe in mwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += str(mwe.id)
                    else:
                        line += ';' + str(mwe.id)
                    if token == mwe.tokens[0]:
                        line += ':' + str(mwe.type)
            if line.endswith('\t'):
                line += '_'
            lines += line + '\n'
        return lines

    def getCorpusTextWithPlus(self):
        goldMwes = self.vMWEs
        predMwes = self.identifiedVMWEs
        lines = ''
        idx = 1
        for token in self.tokens:
            line = str(idx) + '\t' + token.text + '\t_\t'
            idx += 1
            for mwe in goldMwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += '+'
                        break

            if line.endswith('\t'):
                line += '_\t'
            else:
                line += '\t'
            for mwe in predMwes:
                if token in mwe.tokens:
                    if line.endswith('\t'):
                        line += '+'
                        break
            if line.endswith('\t'):
                line += '_'
            lines += line + '\n'
        return lines

    def isPrintable(self):
        if len(self.vMWEs) > 2:
            return True
        # if not PrintParams.printSentsWithEmbeddedMWEs:
        #     return
        # for mwe in self.vMWEs:
        #     if mwe.isEmbedded:
        #         return True
        return False

        # if PrintParams.printSentsWithEmbeddedMWEs and len(self.vMWEs) > 2:
        #     for mwe in self.vMWEs:
        #         if mwe.isEmbedded:
        #             return True
        # return False

    def getDirectParents(self):

        for token in self.tokens:
            token.getDirectParent()

    @staticmethod
    def getTokens(elemlist):
        if isinstance(elemlist, Token):
            return [elemlist]
        if isinstance(elemlist, list):
            result = []
            for elem in elemlist:
                if isinstance(elem, Token):
                    result.append(elem)
                elif isinstance(elem, list):
                    result.extend(Sentence.getTokens(elem))
            return result
        return [elemlist]

    @staticmethod
    def getTokenLemmas(tokens):
        text = ''
        tokens = Sentence.getTokens(tokens)
        for token in tokens:
            if token.lemma != '':
                text += token.lemma + ' '
            else:
                text += token.text + ' '
        return text.strip()

    def printSummary(self):
        vMWEText = ''
        for vMWE in self.vMWEs:
            vMWEText += str(vMWE) + '\n'
        if len(self.identifiedVMWEs) > 0:
            identifiedMWE = '### Identified MWEs: \n'
            for mwe in self.identifiedVMWEs:
                identifiedMWE += str(mwe) + '\n'
        else:
            identifiedMWE = ''

        return '## Sentence No. ' + str(self.id) + ' - ' + self.sentid + '\n' + self.text + \
               '\n### Existing MWEs: \n' + vMWEText + identifiedMWE

    def __str__(self):

        vMWEText = ''
        for vMWE in self.vMWEs:
            vMWEText += str(vMWE) + '\n\n'
        if len(self.identifiedVMWEs) > 0:
            identifiedMWE = '### Identified MWEs: \n'
            for mwe in self.identifiedVMWEs:
                identifiedMWE += str(mwe) + '\n\n'
        else:
            identifiedMWE = ''
        # featuresInfo = ''

        result = ''
        transition = self.initialTransition
        idx = 0
        tab = '&nbsp;'
        while True:
            if transition is not None:
                if transition.type is not None:
                    type = transition.type.name
                else:
                    type = tab * 8
                configuration = str(transition.configuration)
                if type.startswith('MERGE') or type.startswith('WHITE'):
                    type = '**' + type + '**' + tab * 3
                if len(type) == 'SHIFT':
                    type = type + tab * 3
                result += '\n\n' + str(
                    transition.id) + '- ' + type + tab * 3 + '>' + tab * 3 + configuration + '\n\n'
                if transition.next is None:
                    break
                transition = transition.next
                idx += 1
            else:  # result += str(self.featuresInfo[1][idx]) + '\n\n'
                break
        text = ''
        for token in self.tokens:
            if token.parentMWEs is not None and len(token.parentMWEs) > 0:
                text += '**' + token.text + '**' + ' '
            else:
                text += token.text + ' '

        return '## Sentence No. ' + str(self.id) + ' - ' + self.sentid + '\n' + text + \
               '\n### Existing MWEs: \n' + vMWEText + identifiedMWE  # + 'black Merge Num : ' + str(self.blackMergeNum) + ' Interleaving Num: ' + str(self.interleavingNum) \
        # + '\n' + result #+ str(self.initialTransition) + '\n### Features: \n' + featuresInfo

    def __iter__(self):
        for vmwe in self.vMWEs:
            yield vmwe


class VMWE:
    """
        A class used to encapsulate the information of a verbal multi-word expression
    """

    def __init__(self, id, token=None, type='', isEmbedded=False, isInterleaving=False, isDistributedEmbedding=False,
                 isInTrainingCorpus=0):
        self.id = int(id)
        self.isInTrainingCorpus = isInTrainingCorpus
        self.tokens = []
        self.isSingleWordExp = False
        self.isContinousExp = False
        if token is not None:
            self.tokens.append(token)
        self.type = type
        self.isEmbedded = isEmbedded
        self.isDistributedEmbedding = isDistributedEmbedding
        self.isInterleaving = isInterleaving
        self.isVerbal = True
        self.directParent = None

    def getId(self):
        return self.id

    def addToken(self, token):
        self.tokens.append(token)

    @staticmethod
    def getVMWENumber(tokens):
        result = 0
        for token in tokens:
            if isinstance(token, VMWE):
                result += 1
        return result

    @staticmethod
    def haveSameParents(tokens):
        # Do they have all a parent?
        for token in tokens:
            if not token.parentMWEs:
                return None
        # Get all parents of tokens
        parents = set()
        for token in tokens:
            for parent in token.parentMWEs:
                parents.add(parent)
        if len(parents) == 1:
            return list(parents)

        selectedParents = list(parents)
        for parent in parents:
            for token in tokens:
                if parent not in token.parentMWEs:
                    if parent in selectedParents:
                        selectedParents.remove(parent)

        for parent in list(selectedParents):
            if parent.isInterleaving or parent.isDistributedEmbedding:
                selectedParents.remove(parent)
        return selectedParents

    # @staticmethod
    # def haveSameDirectParents(s0, s1):
    #     if isinstance(s0, Token) and isinstance(s1, Token):
    #         return s0.directParent == s1.directParent
    # if isinstance(s0, list):

    @staticmethod
    def getParents(tokens, type=None):
        if len(tokens) == 1:
            if tokens[0].parentMWEs:
                for vmwe in tokens[0].parentMWEs:
                    if len(vmwe.tokens) == 1:  # and vmwe.type.lower() != type:
                        if type is not None:
                            if vmwe.type.lower() == type.lower():
                                return [vmwe]
                            else:
                                return None
                        else:
                            return [vmwe]

        # Do they have all a parent?
        for token in tokens:
            if len(token.parentMWEs) == 0:
                return None

        # Get all parents of tokens
        parents = set()
        for token in tokens:
            for parent in token.parentMWEs:
                parents.add(parent)
        selectedParents = list(parents)
        for parent in parents:
            if len(parent.tokens) != len(tokens):
                if parent in selectedParents:
                    selectedParents.remove(parent)
                continue
            for token in tokens:
                if parent not in token.parentMWEs:
                    if parent in selectedParents:
                        selectedParents.remove(parent)
        for parent in list(selectedParents):
            if parent.isInterleaving or parent.isDistributedEmbedding:
                selectedParents.remove(parent)
        if type is not None:
            for parent in list(selectedParents):
                if parent.type.lower() != type:
                    selectedParents.remove(parent)
        return selectedParents

    # def getDirectParent(self):
    #     self.directParent = None
    #     if self.parentMWEs is not None and len(self.parentMWEs)> 0:
    #         if len(self.parentMWEs) == 1:
    #             self.directParent = self.parentMWEs[0]
    #         else:
    #             parents = sorted(self.parentMWEs, key=lambda VMWE : (VMWE.isInterleaving, VMWE.isEmbedded, len(VMWE.tokens) ))
    #             for parent in parents:
    #                 if not parent.isInterleaving:
    #                     self.directParent = parent
    #                     break
    #     return self.directParent

    def __str__(self):
        tokensStr = ''
        for token in self.tokens:
            tokensStr += token.text + ' '
        tokensStr = tokensStr.strip()
        isInterleaving = ''
        if self.isInterleaving:
            isInterleaving = ', Interleaving '
        isEmbedded = ''
        if self.isEmbedded:
            if self.isDistributedEmbedding:
                isEmbedded = ', DistributedEmbedding '
            else:
                isEmbedded = ', Embedded '
            # isContinousExp =''
            # if self.isContinousExp:
            # isContinousExp = 'Continous'
        type = ''
        if self.type != '':
            type = '(' + self.type
            if self.isInTrainingCorpus != 0:
                type += ', ' + str(self.isInTrainingCorpus) + ')'
            else:
                type += ')'
        return str(self.id) + '- ' + '**' + tokensStr + '** ' + type + isEmbedded + isInterleaving

    def __iter__(self):
        for t in self.tokens:
            yield t

    def getString(self):
        result = ''
        for token in self.tokens:
            result += token.text + ' '
        return result[:-1].lower()

    def getLemmaString(self):
        result = ''
        for token in self.tokens:
            if token.lemma.strip() != '':
                result += token.lemma + ' '
            else:
                result += token.text + ' '
        return result[:-1].lower()

    def In(self, vmwes):

        for vmwe in vmwes:
            if vmwe.getString() == self.getString():
                return True

        return False

    def __eq__(self, other):
        if not isinstance(other, VMWE):
            raise TypeError()
        if self.getLemmaString() == other.getLemmaString():
            return True
        return False

    def __hash__(self):
        return hash(self.getLemmaString())

    def __contains__(self, vmwe):
        if not isinstance(vmwe, VMWE):
            raise TypeError()
        if vmwe is self or vmwe.getLemmaString() == self.getLemmaString():
            return False
        if vmwe.getLemmaString() in self.getLemmaString():
            return True
        for token in vmwe.tokens:
            if token.getLemma() not in self.getLemmaString():
                return False
        return True


class Token:
    """
        a class used to encapsulate all the information of a sentence tokens
    """

    def __init__(self, position, txt, lemma='', posTag='', abstractPosTag='', morphologicalInfo=[], dependencyParent=-1,
                 dependencyLabel=''):
        self.position = int(position)
        self.text = txt
        self.lemma = lemma
        self.abstractPosTag = abstractPosTag
        self.posTag = posTag
        self.morphologicalInfo = morphologicalInfo
        self.dependencyParent = dependencyParent
        self.dependencyLabel = dependencyLabel
        self.parentMWEs = []
        self.directParent = None

    def setParent(self, vMWE):
        self.parentMWEs.append(vMWE)

    def getLemma(self):
        if self.lemma != '':
            return self.lemma.strip()
        return self.text.strip()

    def getDirectParent(self):
        self.directParent = None
        if self.parentMWEs is not None and len(self.parentMWEs) > 0:
            if len(self.parentMWEs) == 1:
                if not self.parentMWEs[0].isInterleaving:
                    self.directParent = self.parentMWEs[0]
            else:
                parents = sorted(self.parentMWEs,
                                 key=lambda VMWE: (VMWE.isInterleaving, VMWE.isEmbedded, len(VMWE.tokens)),
                                 reverse=True)
                for parent in parents:
                    if not parent.isInterleaving:
                        self.directParent = parent
                        break
        return self.directParent

    def In(self, vmwe):
        for token in vmwe.tokens:
            if token.text.lower() == self.text.lower() and token.position == self.position:
                return True
        return False

    def isMWT(self):
        if self.parentMWEs:
            for vmw in self.parentMWEs:
                if len(vmw.tokens) == 1:
                    return vmw
        return None

    def __str__(self):
        parentTxt = ''
        if len(self.parentMWEs) != 0:
            for parent in self.parentMWEs:
                parentTxt += str(parent) + '\n'

        return str(self.position) + ' : ' + self.text + ' : ' + self.posTag + '\n' + 'parent VMWEs\n' + parentTxt


def getTokens(elemlist):
    if str(elemlist.__class__) == 'corpus.Token':  # isinstance(elemlist, Token):
        return [elemlist]
    if isinstance(elemlist, list):
        result = []
        for elem in elemlist:
            if str(elem.__class__) == 'corpus.Token':
                result.append(elem)
            elif isinstance(elem, list) and len(elem) == 1 and isinstance(elem[0], list):
                result.extend(getTokens(elem[0]))
            elif isinstance(elem, list) and len(elem):
                result.extend(getTokens(elem))
        return result
    return [elemlist]