import sys, re, os
from ucscGb.gbData.ordereddict import OrderedDict
from ucscGb.encode import encodeUtils, track
from ucscGb.externalData.geo import submission

cvDetails = {
    'cell':    [ 'organism', 'description', 'karyotype', 'lineage', 'sex' ],
    'antibody': [ 'antibodyDescription', 'targetDescription', 'vendorName', 'vendorId' ]
}

#if the term appears in the mdb and must overriding the value in the cv
cvOverride = [ 'sex' ]

#talk to Venkat lol
cvPretend = { 'antibody Input': 'control' }

#if its not in cvDetails, which things should we check by default
cvDefaults = [ 'description' ]

mdbWhitelist = [
    'age',
    'bioRep',
    'control',
    'controlId',
    'fragSize',
    'labExpId',
    'labVersion',
    'mapAlgorithm',
    'obtainedBy',
    'phase',
    'readType',
    'region',
    'replicate',
    'restrictionEnzyme',
    'run',
    'softwareVersion',
    'spikeInPool',
    'strain'
]
    
def isRawFile(file):
    return (file.extension == 'fastq' or file.extension == 'csfasta' or file.extension == 'csqual')
    
def isSupplementaryFile(file):
    return (not isRawFile(file)) and file.extension != 'fasta'
    
def sampleTitle(stanza, expVars, warn=False, rep=False):
    concat = stanza[expVars[0]].replace('-m', '')
    for expVar in expVars[1:len(expVars)]:
        if expVar in stanza and stanza[expVar] != 'None':
            concat += '_' + stanza[expVar]
        elif warn:
            print 'warning: %s is None or not in %s' % (expVar, stanza.name)
    if rep:
        concat += 'Rep' + stanza['replicate']
    return concat
    
def linkName(file, track):
    return '%s_%s' % (track.database, file.name)
    
# This function is deprecated. It is specific to ENCODE 2 and should not be used in ENCODE 3
def createMappings(metadb, all=False, rep=False):
    expIds = dict()
    geoMapping = dict()
    expVars = None
    series = None
    datatype = None
    
    for stanza in metadb.itervalues():
        
        if 'objType' in stanza and stanza['objType'] == 'composite':
            series = stanza
            expVars = stanza['expVars'].split(',')
            continue

        if 'expId' not in stanza:
            print stanza.name + ': no expId'
            continue

        if 'objStatus' in stanza:
            print stanza.name + ': skipping because ' + stanza['objStatus']
            continue
            
        if 'geoSampleAccession' not in stanza or all:
            if stanza['fileName'].endswith('bam') or stanza['fileName'].endswith('bai') and 'Splices' not in stanza['fileName']:
                continue
            # if this hasn't been submitted to GEO yet, we'll add it to the submission list
            if rep:
                if stanza['expId'] + '_' + stanza['replicate'] not in expIds:
                    expIds[stanza['expId'] + '_' + stanza['replicate']] = list()
                expIds[stanza['expId'] + '_' + stanza['replicate']].append(stanza)
            else:
                if stanza['expId'] not in expIds:
                    expIds[stanza['expId']] = list()
                expIds[stanza['expId']].append(stanza)
        
        else:
            # otherwise we keep track of the geo number for partially submitted samples
            if rep:
                varname = stanza['expId'] + '_' + stanza['replicate']
                if varname not in geoMapping:
                    geoMapping[varname] = stanza['geoSampleAccession']
                    print varname + ': ' + stanza['geoSampleAccession']
                elif geoMapping[varname] != 'Inconsistent' and geoMapping[varname] != stanza['geoSampleAccession']:
                    geoMapping[varname] = 'Inconsistent'
                    print stanza.name + ': inconsistent geo mapping'
            else:
                if stanza['expId'] not in geoMapping:
                    geoMapping[stanza['expId']] = stanza['geoSampleAccession']
                elif geoMapping[stanza['expId']] != 'Inconsistent' and geoMapping[stanza['expId']] != stanza['geoSampleAccession']:
                    geoMapping[stanza['expId']] = 'Inconsistent'
                    print stanza.name + ': inconsistent geo mapping'
        
        if datatype == None and 'dataType' in stanza:
            datatype = stanza['dataType']
        elif datatype != None and 'dataType' in stanza and datatype != stanza['dataType']:
            raise KeyError(stanza.name + ': inconsistent data type') 

    try:
        dt = datatype
        datatype = encodeUtils.dataTypes[dt]
        datatype.name = dt
    except KeyError:
        raise KeyError(datatype)
    
    return expIds, expVars, geoMapping, series, datatype
    
# This function is deprecated. It is specific to ENCODE 2 and should not be used in ENCODE 3
def createSeries(softfile, compositeTrack, expIds, expVars, geoMapping, series, datatype, replace, audit, argseries, all=False, rep=False):
    
    if 'geoSeriesAccession' in series and not all:
        print 'Existing series ' + series['composite'] + ' using geoSeriesAccession ' + series['geoSeriesAccession']
        return
        
    print 'Writing series ' + series['composite']
    
    seriesStanza = SeriesStanza(softfile)
    seriesStanza['^SERIES'] = series['composite']
    seriesStanza['!Series_title'] = compositeTrack.trackDb[compositeTrack.name]['longLabel'] #STILL INCORRECT
    
    if '!Series_summary' in replace:
        seriesStanza['!Series_summary'] = replace['!Series_summary']
    else:
        print 'warning: no series summary found. Please include in replace file.'
        seriesStanza['!Series_summary'] = '[REPLACE]'
        if audit:
            print seriesStanza.name + ': no summary'
        
    if '!Series_overall_design' in replace:
        seriesStanza['!Series_overall_design'] = replace['!Series_overall_design']
    else:
        print 'no series overall design found. Please include in replace file.'
        seriesStanza['!Series_overall_design'] = '[REPLACE]'
        if audit:
            print seriesStanza.name + ': no overall design'
            
    seriesStanza['!Series_web_link'] = [ compositeTrack.url, 'https://www.ncbi.nlm.nih.gov/geo/info/ENCODE.html' ]
    
    if '!Series_contributor' in replace:
        seriesStanza['!Series_contributor'] = replace['!Series_contributor']
    else:
        seriesStanza['!Series_contributor'] = '[REPLACE]'
        if audit:
            print seriesStanza.name + ': no contributor'
        
    seriesStanza['!Series_gp_id'] = encodeUtils.gpIds[compositeTrack.organism + ' ' + datatype.source]
    
    # could use !Series_variable_* and !Series_repeats_*
    
    if not argseries:
        seriesStanza['!Series_sample_id'] = list()
        
        for idNum in expIds.iterkeys():
            if idNum in geoMapping and geoMapping[idNum] != 'Inconsistent':
                seriesStanza['!Series_sample_id'].append(geoMapping[idNum])
            else:
                seriesStanza['!Series_sample_id'].append(sampleTitle(expIds[idNum][0], expVars, False, rep))
        
    softfile[series['composite']] = seriesStanza

# This function is deprecated. It is specific to ENCODE 2 and should not be used in ENCODE 3
def createHighThroughputSoftFile(compositeTrack, cv, expIds, expVars, geoMapping, series, datatype, replace, audit, tarpath, argseries, all=False, rep=False):
    
    print 'Creating HighThroughput soft file'

    softfile = HighThroughputSoftFile()
    fileList = list()
    
    createSeries(softfile, compositeTrack, expIds, expVars, geoMapping, series, datatype, replace, audit, argseries, all)
    
    if argseries:
        return softfile, fileList
    
    for idNum in expIds.iterkeys():
        
        expId = expIds[idNum]
        firstStanza = expId[0]
        if not all: print 'Writing sample ' + firstStanza['metaObject'] + ' (' + idNum + ')'
        sample = HighThroughputSampleStanza(softfile)

        sample['^SAMPLE'] = sampleTitle(firstStanza, expVars, 1, rep)
        sample['!Sample_type'] = 'SRA'
        sample['!Sample_title'] = sample['^SAMPLE']
        
        if 'geoSeriesAccession' in series:
            sample['!Sample_series_id'] = series['geoSeriesAccession']
            
        count = 1
        
        #figure out if the instrument model is consistent across the entire sample
        instrumentModel = None
        for stanza in expId:    
            if 'seqPlatform' in stanza:
                if instrumentModel == None:
                    instrumentModel = submission.instrumentModels[stanza['seqPlatform']]
                else:
                    if instrumentModel != submission.instrumentModels[stanza['seqPlatform']]:
                        instrumentModel = None
                        if audit:
                            print 'expId' + str(expId) + ': inconsistent instrument model'
                        break
        
        for stanza in expId:
        
            for fname in stanza['fileName'].split(','):
              
                file = compositeTrack.files[fname]
                filelist = list()
                
                if file.extension == 'fasta':
                    print 'WARNING: fastas detected!!!'
                
                if isRawFile(file):
                
                    if all:
                        continue
                
                    if file.name.endswith('.tgz') or file.name.endswith('.tar.gz'):
                    
                        if tarpath == None:
                            raise IOError('this track contains tarred fastqs. Please specify a path through the -z option')
                        dirname = tarpath + file.name.split('.')[0] + '/'
                        if os.path.exists(dirname):
                            print dirname + ' already exists, so not unzipping'
                        else:
                            print 'creating ' + dirname + '...'
                            os.mkdir(dirname)
                            os.system('tar -xf %s -C %s' % (file.path + file.name, dirname))
                        
                        for root, dirnames, filenames in os.walk(dirname):
                            for filename in filenames:
                                if 'reject' in filename or 'md5sum' in filename:
                                    continue
                                if filename.endswith('.fastq') or filename.endswith('.txt'):
                                    print 'gzipping ' + filename
                                    os.system('gzip %s' % (root + '/' + filename))
                        
                        for root, dirnames, filenames in os.walk(dirname):
                        
                            rootmd5s = None
                            if os.path.isfile(root + '/md5sum.txt'):
                                rootmd5s = encodeUtils.readMd5sums(root + '/md5sum.txt')
                            
                            for filename in filenames:
                                if 'reject' in filename or 'md5sum' in filename:
                                    continue
                                
                                print root + '/' + filename
                                
                                if rootmd5s != None and filename in rootmd5s:
                                    newmd5 = rootmd5s[filename]
                                else:
                                    newmd5 = encodeUtils.hashFile(root + '/' + filename)
                                    encodeUtils.writeMd5sums(root + '/md5sum.txt', filename, newmd5)
                                newfile = track.TrackFile(root + '/' + filename, newmd5)
                                
                                filelist.append(newfile)

                    else:
                        filelist.append(file)
                        
                    for f in filelist:
                        
                        sample['!Sample_raw_file_' + str(count)] = linkName(f, compositeTrack)
                        if f.extension == 'txt':
                            sample['!Sample_raw_file_type_' + str(count)] = 'fastq'
                        elif f.extension == 'csfasta':
                            sample['!Sample_raw_file_type_' + str(count)] = 'SOLiD_native_csfasta'
                        elif f.extension == 'csqual':
                            sample['!Sample_raw_file_type_' + str(count)] = 'SOLiD_native_qual'
                        else:
                            sample['!Sample_raw_file_type_' + str(count)] = f.extension
                        
                        sample['!Sample_raw_file_checksum_' + str(count)] = f.md5sum

                        if instrumentModel == None and 'seqPlatform' in stanza:
                            sample['!Sample_raw_file_instrument_model_' + str(count)] = submission.instrumentModels[stanza['seqPlatform']]
                            
                        fileList.append(f)    
                        count = count + 1
            
        count = 1

        pooledStanza = dict()
        
        for stanza in expId:
        
            for fname in stanza['fileName'].split(','):
                file = compositeTrack.files[fname]
        
                if isSupplementaryFile(file):
                    sample['!Sample_supplementary_file_' + str(count)] = linkName(file, compositeTrack)
                    
                    if not all:
                        if file.md5sum != None:
                            sample['!Sample_supplementary_file_checksum_' + str(count)] = file.md5sum
                    
                    sample['!Sample_supplementary_file_build_' + str(count)] = compositeTrack.database
                    
                    if instrumentModel == None and 'seqPlatform' in stanza:
                        sample['!Sample_supplementary_file_instrument_model_' + str(count)] = submission.instrumentModels[stanza['seqPlatform']]
                    
                    fileList.append(file)
                    count = count + 1
                    
            if 'objStatus' in stanza:
                continue
            for k in stanza.iterkeys():
                if k not in pooledStanza:
                    pooledStanza[k] = set()
                pooledStanza[k].add(stanza[k])
        for k in pooledStanza.iterkeys():
            pooledStanza[k] = ','.join(pooledStanza[k])
            
        
        if (idNum in geoMapping and geoMapping[idNum] != 'Inconsistent'):
            sample['!Sample_geo_accession'] = geoMapping[idNum]
        else:
        
            if all and 'geoSampleAccession' in pooledStanza:
                sample['!Sample_geo_accession'] = pooledStanza['geoSampleAccession']
        
            sample['!Sample_source_name'] = pooledStanza['cell']
            sample['!Sample_organism'] = compositeTrack.organism
            
            sample['!Sample_characteristics'] = list()
            allVars = expVars + mdbWhitelist
            
            for var in allVars:
                if var in pooledStanza:
                    foobar = var
                    sample['!Sample_characteristics'].append(var + ': ' + pooledStanza[var])
                    for pretend in cvPretend.iterkeys():
                        if var + ' ' + pooledStanza[var] == pretend:
                            foobar = cvPretend[pretend]
                    if foobar in cvDetails:
                        for cvVar in cvDetails[foobar]:
                            if cvVar in cvOverride and cvVar in pooledStanza:
                                sample['!Sample_characteristics'].append(var + ' ' + cvVar + ': ' + pooledStanza[cvVar])
                            elif cvVar in cv[pooledStanza[var]]:
                                sample['!Sample_characteristics'].append(var + ' ' + cvVar + ': ' + cv[pooledStanza[var]][cvVar])
                    else:
                        for cvVar in cvDefaults:
                            if pooledStanza[var] in cv and cvVar in cv[pooledStanza[var]]:
                                sample['!Sample_characteristics'].append(var + ' ' +  cvVar + ': ' + cv[pooledStanza[var]][cvVar])
                    
            sample['!Sample_biomaterial_provider'] = cv[pooledStanza['cell']]['vendorName']
            
            if 'treatment' in pooledStanza:
                sample['!Sample_treatment_protocol'] = pooledStanza['treatment']
            
            if 'protocol' in cv[pooledStanza['cell']]:
                for protocol in cv[pooledStanza['cell']]['protocol'].split(' '):
                        if protocol == 'missing':
                            continue
                        if ':' not in protocol:
                            raise KeyError(protocol + ' is not valid')
                        key, val = protocol.split(':')
                        if key == 'ENCODE' or key == cv[pooledStanza['lab']]['labPi']:
                            sample['!Sample_growth_protocol'] = val
            
            if datatype.molecule == 'RNA':
                if 'rnaExtract' not in pooledStanza:
                    sample['!Sample_molecule'] = 'total RNA'
                elif pooledStanza['rnaExtract'] in submission.rnaExtractMapping:
                    sample['!Sample_molecule'] = submission.rnaExtractMapping[pooledStanza['rnaExtract']]
                elif pooledStanza['localization'] in submission.localizationMapping:
                    sample['!Sample_molecule'] = submission.localizationMapping[pooledStanza['localization']]
                    
            else:
                sample['!Sample_molecule'] = datatype.molecule
                
            if '!Sample_instrument_model' in replace and replace['!Sample_instrument_model'][0] == 'Unknown':
                sample['!Sample_extract_protocol'] = 'Instrument model unknown. ("%s" specified by default). For more information, see %s' % (submission.instrumentModels[replace['!Sample_instrument_model'][0]], compositeTrack.url)
            else:
                sample['!Sample_extract_protocol'] = compositeTrack.url
            sample['!Sample_library_strategy'] = datatype.strategy
            sample['!Sample_library_source'] = datatype.source
            sample['!Sample_library_selection'] = datatype.selection
            
            # if the instrumentModel is consistent, just use that
            # otherwise take the first seqPlatform value from metadata
            # if that still fails, check the replacement file
            # finally just make it say [REPLACE]
            if instrumentModel != None:
                sample['!Sample_instrument_model'] = instrumentModel
            else:
                for stanza in expId:    
                    if 'seqPlatform' in stanza:
                        sample['!Sample_instrument_model'] = submission.instrumentModels[stanza['seqPlatform']]
                        break
                if '!Sample_instrument_model' not in sample:
                    if '!Sample_instrument_model' in replace:
                        sample['!Sample_instrument_model'] = submission.instrumentModels[replace['!Sample_instrument_model'][0]]
                if '!Sample_instrument_model' not in sample:
                    sample['!Sample_instrument_model'] = '[REPLACE]'
                    if audit:
                        print stanza.name + ': no instrument'
                    
            sample['!Sample_data_processing'] = compositeTrack.url
            
        softfile[sample['^SAMPLE']] = sample
        
    return softfile, fileList
        
# This function is deprecated. It is specific to ENCODE 2 and should not be used in ENCODE 3        
def createMicroArraySoftFile(compositeTrack, cv, expIds, expVars, geoMapping, series, datatype, replace, audit, tarpath, argseries, all=False):
    
    print 'Creating HighThroughput soft file'

    softfile = HighThroughputSoftFile()
    fileList = list()
    
    createSeries(softfile, compositeTrack, expIds, expVars, geoMapping, series, datatype, replace, audit, argseries, all)
    
    if argseries:
        return softfile, fileList
    
    for idNum in expIds.iterkeys():
        
        # sample['!Sample_table'] = KeyOptional # CEL file
        # sample['!Sample_source_name_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_organism_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_characteristics_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_biomaterial_provider_ch'] = KeyZeroPlusNumbered
        # sample['!Sample_treatment_protocol_ch'] = KeyZeroPlusNumbered
        # sample['!Sample_growth_protocol_ch'] = KeyZeroPlusNumbered
        # sample['!Sample_molecule_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_extract_protocol_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_label_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_label_protocol_ch_1'] = '[REPLACE]' #KeyOnePlusNumbered
        # sample['!Sample_hyb_protocol'] = '[REPLACE]' #KeyOnePlus
        # sample['!Sample_scan_protocol'] = '[REPLACE]' #KeyOnePlus
        # sample['!Sample_data_processing'] = '[REPLACE]' #KeyOnePlus
        # sample['!Sample_description'] = '[REPLACE]' #KeyZeroPlus
        # sample['!Sample_platform_id'] = '[REPLACE]'
        # sample['!Sample_table_begin'] = ''
        # sample['!Sample_table_end'] = ''
        
        expId = expIds[idNum]
        firstStanza = expId[0]
        print 'Writing sample ' + firstStanza['metaObject'] + ' (' + idNum + ')'
        sample = HighThroughputSampleStanza(softfile)

        sample['^SAMPLE'] = sampleTitle(firstStanza, expVars, 1)
        sample['!Sample_title'] = sample['^SAMPLE']
        
        if 'geoSeriesAccession' in series:
            sample['!Sample_series_id'] = series['geoSeriesAccession']
        
        for stanza in expId:
            for fname in stanza['fileName'].split(','):
                file = compositeTrack.files[fname]
                if isRawFile(file):
                    print 'ERROR: RAW FILES IN MICROARRAY SUBMISSION DETECTED'
            
        count = 1
            
        for stanza in expId:
            for fname in stanza['fileName'].split(','):
                file = compositeTrack.files[fname]
        
                if isSupplementaryFile(file):
                    sample['!Sample_supplementary_file_' + str(count)] = linkName(file, compositeTrack)
                    
                    if file.md5sum != None:
                        sample['!Sample_supplementary_file_checksum_' + str(count)] = file.md5sum
                        
                    fileList.append(file)
                    count = count + 1
                    
        print idNum
        if idNum in geoMapping:
            print geoMapping[idNum]
        
        if idNum in geoMapping and geoMapping[idNum] != 'Inconsistent':
            sample['!Sample_geo_accession'] = geoMapping[idNum]
        else:
        
            sample['!Sample_source_name_ch1'] = firstStanza['cell']
            sample['!Sample_organism_ch1'] = compositeTrack.organism
            
            sample['!Sample_characteristics_ch1'] = list()
            allVars = expVars + mdbWhitelist
            
            for var in allVars:
                if var in firstStanza:
                    foobar = var
                    sample['!Sample_characteristics_ch1'].append(var + ': ' + firstStanza[var])
                    for pretend in cvPretend.iterkeys():
                        if var + ' ' + firstStanza[var] == pretend:
                            foobar = cvPretend[pretend]
                    if foobar in cvDetails:
                        for cvVar in cvDetails[foobar]:
                            if cvVar in cvOverride and cvVar in firstStanza:
                                sample['!Sample_characteristics_ch1'].append(var + ' ' + cvVar + ': ' + firstStanza[cvVar])
                            elif cvVar in cv[firstStanza[var]]:
                                sample['!Sample_characteristics_ch1'].append(var + ' ' + cvVar + ': ' + cv[firstStanza[var]][cvVar])
                    else:
                        for cvVar in cvDefaults:
                            if firstStanza[var] in cv and cvVar in cv[firstStanza[var]]:
                                sample['!Sample_characteristics_ch1'].append(var + ' ' +  cvVar + ': ' + cv[firstStanza[var]][cvVar])
                    
            sample['!Sample_biomaterial_provider_ch1'] = cv[firstStanza['cell']]['vendorName']
            
            if 'treatment' in firstStanza:
                sample['!Sample_treatment_protocol_ch1'] = firstStanza['treatment']
            elif '!Sample_treatment_protocol_ch1' in replace:
                sample['!Sample_treatment_protocol_ch1'] = replace['!Sample_treatment_protocol_ch1']
            else:
                sample['!Sample_treatment_protocol_ch1'] = 'Unknown'
            
            if 'protocol' in cv[firstStanza['cell']]:
                for protocol in cv[firstStanza['cell']]['protocol'].split(' '):
                        if protocol == 'missing':
                            continue
                        if ':' not in protocol:
                            raise KeyError(protocol + ' is not valid')
                        key, val = protocol.split(':')
                        if key == 'ENCODE' or key == cv[firstStanza['lab']]['labPi']:
                            sample['!Sample_growth_protocol_ch1'] = val
            
            if datatype.molecule == 'RNA':
                if 'rnaExtract' not in firstStanza:
                    sample['!Sample_molecule_ch1'] = 'total RNA'
                elif firstStanza['rnaExtract'] in submission.rnaExtractMapping:
                    sample['!Sample_molecule_ch1'] = submission.rnaExtractMapping[firstStanza['rnaExtract']]
                elif firstStanza['localization'] in submission.localizationMapping:
                    sample['!Sample_molecule_ch1'] = submission.localizationMapping[firstStanza['localization']]
                    
            else:
                sample['!Sample_molecule_ch1'] = datatype.molecule

            sample['!Sample_extract_protocol_ch1'] = compositeTrack.url
            
            sample['!Sample_label_ch_1'] = '[REPLACE]'
            if '!Sample_label_ch_1' in replace:
                sample['!Sample_label_ch_1'] = replace['!Sample_label_ch_1'] #KeyOnePlusNumbered
                
            sample['!Sample_label_protocol_ch_1'] = compositeTrack.url #KeyOnePlusNumbered
            
            # are all these just links to the trackDB too?
            sample['!Sample_hyb_protocol'] = compositeTrack.url #KeyOnePlus
            sample['!Sample_scan_protocol'] = compositeTrack.url #KeyOnePlus
            sample['!Sample_data_processing'] = compositeTrack.url #KeyOnePlus
            sample['!Sample_description'] = compositeTrack.url #KeyZeroPlus
            
        softfile[firstStanza['metaObject']] = sample
        
    return softfile, fileList
    
class SoftFile(OrderedDict):

    """
    Stores an Ra file in a set of entries, one for each stanza in the file.
    """

    def __init__(self, filePath=''):
        OrderedDict.__init__(self)
        if filePath != '':
            self.read(filePath) 
    
    def read(self, filePath):
        file = open(filePath, 'r')
        self.readStream(file)
        file.close()
    
    def readStream(self, stream):
        """
        Reads an SoftFile stanza by stanza, and internalizes it.
        """

        stanza = list()

        for line in stream:
 
            line = line.strip()

            if line.startswith('^') and stanza != []:
                name, entry = self.readStanza(stanza)
                #print 'hit: ' + name
                if entry != None:
                    if name in self:
                        raise KeyError('Duplicate Key ' + name)
                    self[name] = entry
                
                stanza = list()

            #print 'appending: ' + line
            stanza.append(line)

        #stream.close()
        
        name, entry = self.readStanza(stanza)
        #print 'hit: ' + name
        if entry != None:
            if name in self:
                raise KeyError('Duplicate Key ' + name)
            self[name] = entry


    def readStanza(self, stanza):

        if stanza[0].startswith('^SAMPLE'):
            entry = HighThroughputSampleStanza(self) #WILL HAVE TO CHANGE
        elif stanza[0].startswith('^SERIES'):
            entry = SeriesStanza(self)
        elif stanza[0].startswith('^PLATFORM'):
            entry = PlatformStanza(self)
        else:
            raise KeyError(stanza[0])

        val = entry.readStanza(stanza)
        return val, entry


    def iter(self):
        for item in self._OrderedDict__ordering:
            yield item


    def iterkeys(self):
        for item in self._OrderedDict__ordering:
            yield item


    def itervalues(self):
        for item in self._OrderedDict__ordering:
            yield self[item]


    def iteritems(self):
        for item in self._OrderedDict__ordering:
            yield [item]


    def __str__(self):
        str = ''
        for item in self.iterkeys():
            str += self[item].__str__()
            
        return str
        
    def diff(self, other):
        result = dict()
        for key in self.iterkeys():
            if key not in other:
                result[key] = list()
                result[key].append(key)
                result[key].append(None)
            elif self[key] != other[key]:
                result[key] = list()
                result[key].append(self[key].diff(other[key]))
        for key in other.iterkeys():
            if key not in self:
                result[key] = list()
                result[key].append(None)
                result[key].append(key)
        return result
            
class HighThroughputSoftFile(SoftFile):

    def __init__(self, filePath=''):
        SoftFile.__init__(self, filePath)

    def readStanza(self, stanza):
        if stanza[0].startswith('^SAMPLE'):
            entry = HighThroughputSampleStanza(self)
        elif stanza[0].startswith('^SERIES'):
            entry = SeriesStanza(self)
        else:
            raise KeyError(stanza[0])

        val = entry.readStanza(stanza)
        return val, entry
        
        
class MicroArraySoftFile(SoftFile):

    def __init__(self, filePath=''):
        SoftFile.__init__(self, filePath)
        
    def readStanza(self, stanza):
        if stanza[0].startswith('^SAMPLE'):
            entry = MicroArraySampleStanza(self)
        elif stanza[0].startswith('^SERIES'):
            entry = SeriesStanza(self)
        elif stanza[0].startswith('^PLATFORM'):
            entry = PlatformStanza(self)
        else:
            raise KeyError(stanza[0])

        val = entry.readStanza(stanza)
        return val, entry
        
            
class KeyRequired(object):
    pass
    
class KeyOptional(object):
    pass
    
class KeyZeroPlus(object):
    pass
    
class KeyOnePlus(object):
    pass
    
class KeyZeroPlusNumbered(object):
    pass
    
class KeyOnePlusNumbered(object):
    pass
    
class KeyZeroPlusChannel(object):
    pass
    
class KeyOnePlusChannel(object):
    pass
    

class SoftStanza(OrderedDict):
    """
    Holds an individual entry in the RaFile.
    """

    def __init__(self, keys, parent):
        self._name = ''
        self._keys = keys
        self.parent = parent
        OrderedDict.__init__(self)
        
    @property 
    def name(self):
        return self._name
        
    def setName(self, newname):
        for k in self._keys:
            if k.startswith('^'):
                self[k] = newname
                break
        self.parent[newname] = self.parent[self._name]
        del self.parent[self._name]
        self._name = newname
        
    def readStanza(self, stanza):
        """
        Populates this entry from a single stanza
        """

        for line in stanza:
            self.__readLine(line)

        return self.__readName(stanza[0])


    def __readName(self, line):
        """
        Extracts the Stanza's name from the value of the first line of the
        stanza.
        """

        if len(line.split('=', 1)) != 2:
            raise ValueError()

        self._name = line.split('=', 1)[1].strip()
        return self._name

    def __readLine(self, line):
        """
        Reads a single line from the stanza, extracting the key-value pair
        """ 
        key = line.split('=', 1)[0].strip()
        val = ''
        if (len(line.split('=', 1)) == 2):
            val = line.split('=', 1)[1].strip()
        
        #split on the last underscore to determine if we're using a numbered key or not
        splitkey = key.rsplit('_', 1)[0]
        channelkey = splitkey + '_ch'
        #if the key is a numbered key
        if splitkey in self._keys and (self._keys[splitkey] == KeyZeroPlusNumbered or self._keys[splitkey] == KeyOnePlusNumbered):
            self[key] = val
        
        #this is for channel data in MicroArraySamples
        elif channelkey in self._keys and (self._keys[channelkey] == KeyZeroPlusChannel or self._keys[channelkey] == KeyOnePlusChannel):
            self[key] = val
        
        #if its a single value (ie 0 or 1 allowed entries)
        elif key in self._keys and (self._keys[key] == KeyRequired or self._keys[key] == KeyOptional):
            self[key] = val

        else:
        
            #if key not in self.keys:
            #    print splitkey
            #    raise KeyError(self._name + ': invalid key: ' + key)
            
            #if (self.keys[key] == KeyRequired or self.keys[key] == KeyOptional) and key in self:
            #    raise KeyError(self._name + ': too many of key: ' + key)
                
            if key not in self:
                self[key] = list()
            self[key].append(val)


    def iter(self):
        yield iterkeys(self)


    def iterkeys(self):
        for item in self._OrderedDict__ordering:
            yield item


    def itervalues(self):
        for item in self._OrderedDict__ordering:
            yield self[item]


    def iteritems(self):
        for item in self._OrderedDict__ordering:
            yield item, self[item]


    def __str__(self):
        str = ''
        for key in self:
            if isinstance(self[key], basestring):
                str += key + ' = ' + self[key] + '\n'
            else:
                for val in self[key]:
                    str += key + ' = ' + val + '\n'

        return str
        
    def write(self, filename):
        #check for absence of required vars
        file = open(filename, 'r')
        file.write(self.__str__())
        file.close()
        
    def diff(self, other):
        result = dict()
        for key in self.iterkeys():
            if key not in other:
                result[key] = list()
                result[key].append(self[key])
                result[key].append(None)
            else:
                val1 = self[key]
                val2 = other[key]
                if val1 == None:
                    val1 = 'None'
                if val2 == None:
                    val2 = 'None'
                if isinstance(val1, list) and len(val1) == 1:
                    val1 = val1[0]
                if isinstance(val2, list) and len(val2) == 1:
                    val2 = val2[0]
                if val1 != val2:
                    if isinstance(val1, list) and isinstance(val2, list):
                        toremove = list()
                        for i in val1:
                            if i in val2:
                                toremove.append(i)
                        for r in toremove:
                            val1.remove(r)
                            val2.remove(r)
                    if (val1 != val2):
                        result[key] = list()
                        result[key].append(val1)
                        result[key].append(val2)
        for key in other.iterkeys():
            if key not in self:
                result[key] = list()
                result[key].append(None)
                result[key].append(other[key])
        return result


class MicroArrayPlatformStanza(SoftStanza):

    def __init__(self, parent):
    
        allowedkeys = { 
            '^PLATFORM': KeyRequired,
            '!Platform_title': KeyRequired,
            '!Platform_distribution': KeyRequired,
            '!Platform_technology': KeyRequired,
            '!Platform_organism': KeyOnePlus,
            '!Platform_manufacturer': KeyRequired,
            '!Platform_manufacture_protocol': KeyOnePlus,
            '!Platform_catalog_number': KeyZeroPlus,
            '!Platform_web_link': KeyZeroPlus,
            '!Platform_support': KeyOptional,
            '!Platform_coating': KeyOptional,
            '!Platform_description': KeyZeroPlus,
            '!Platform_contributor': KeyZeroPlus,
            '!Platform_pubmed_id': KeyZeroPlus,
            '!Platform_geo_accession': KeyOptional,
            '!Platform_table_begin': KeyRequired,
            '!Platform_table_end': KeyRequired
        }
        
        SoftStanza.__init__(self, allowedkeys, parent)
        
        
class MicroArraySampleStanza(SoftStanza):

    def __init__(self, parent):
    
        allowedkeys = { 
            '^SAMPLE': KeyRequired,
            '!Sample_title': KeyRequired,
            '!Sample_supplementary_file': KeyOnePlus,
            '!Sample_table': KeyOptional,
            '!Sample_source_name_ch': KeyOnePlusNumbered,
            '!Sample_organism_ch': KeyOnePlusNumbered,
            '!Sample_characteristics_ch': KeyOnePlusNumbered,
            '!Sample_biomaterial_provider_ch': KeyZeroPlusNumbered,
            '!Sample_treatment_protocol_ch': KeyZeroPlusNumbered,
            '!Sample_growth_protocol_ch': KeyZeroPlusNumbered,
            '!Sample_molecule_ch': KeyOnePlusNumbered,
            '!Sample_extract_protocol_ch': KeyOnePlusNumbered,
            '!Sample_label_ch': KeyOnePlusNumbered,
            '!Sample_label_protocol_ch': KeyOnePlusNumbered,
            '!Sample_hyb_protocol': KeyOnePlus,
            '!Sample_scan_protocol': KeyOnePlus,
            '!Sample_data_processing': KeyOnePlus,
            '!Sample_description': KeyZeroPlus,
            '!Sample_platform_id': KeyRequired,
            '!Sample_geo_accession': KeyOptional,
            '!Sample_anchor': KeyRequired,
            '!Sample_type': KeyRequired,
            '!Sample_tag_count': KeyRequired,
            '!Sample_tag_length': KeyRequired,
            '!Sample_table_begin': KeyRequired,
            '!Sample_table_end': KeyRequired
        }
        
        SoftStanza.__init__(self, allowedkeys, parent)        
        
        
class SeriesStanza(SoftStanza):
    
    def __init__(self, parent):
    
        allowedkeys = { 
            '^SERIES': KeyRequired,
            '!Series_title': KeyRequired,
            '!Series_summary': KeyOnePlus,
            '!Series_overall_design': KeyRequired,
            '!Series_pubmed_id': KeyZeroPlus,
            '!Series_web_link': KeyZeroPlus,
            '!Series_contributor': KeyZeroPlus,
            '!Series_variable': KeyZeroPlusNumbered,
            '!Series_variable_description': KeyZeroPlusNumbered,
            '!Series_variable_sample_list': KeyZeroPlusNumbered,
            '!Series_repeats': KeyZeroPlusNumbered,
            '!Series_repeats_sample_list': KeyZeroPlusNumbered,
            '!Series_sample_id': KeyOnePlus,
            '!Series_geo_accession': KeyOptional,
            '!Series_gp_id': KeyOptional
        }
                
        SoftStanza.__init__(self, allowedkeys, parent)

        
class HighThroughputSampleStanza(SoftStanza):

    def __init__(self, parent):
    
        allowedkeys = {
            '^SAMPLE': KeyRequired,
            '!Sample_type': KeyRequired,
            '!Sample_title': KeyRequired,
            '!Sample_supplementary_file': KeyOnePlusNumbered,
            '!Sample_supplementary_file_checksum': KeyZeroPlusNumbered,
            '!Sample_supplementary_file_build': KeyZeroPlusNumbered,
            '!Sample_raw_file': KeyOnePlusNumbered,
            '!Sample_raw_file_type': KeyOnePlusNumbered,
            '!Sample_raw_file_checksum': KeyZeroPlusNumbered,
            '!Sample_source_name': KeyRequired,
            '!Sample_organism': KeyOnePlus,
            '!Sample_characteristics': KeyOnePlus,
            '!Sample_biomaterial_provider': KeyZeroPlus,
            '!Sample_treatment_protocol': KeyZeroPlus,
            '!Sample_growth_protocol': KeyZeroPlus,
            '!Sample_molecule': KeyRequired,
            '!Sample_extract_protocol': KeyOnePlus,
            '!Sample_library_strategy': KeyOnePlus,
            '!Sample_library_source': KeyOnePlus,
            '!Sample_library_selection': KeyOnePlus,
            '!Sample_instrument_model': KeyOnePlus,
            '!Sample_data_processing': KeyRequired,
            '!Sample_barcode': KeyOptional,
            '!Sample_description': KeyZeroPlus,
            '!Sample_geo_accession': KeyOptional,
            '!Sample_table_begin': KeyOptional,
            '!Sample_table': KeyOptional,
            '!Sample_table_end': KeyOptional
        }
        
        SoftStanza.__init__(self, allowedkeys, parent)
        
