'''
Created on Jan 8, 2014

@author: mkiyer
'''
import os
import sys
import argparse
import logging
import subprocess
import itertools
import shutil
from multiprocessing import Process, Queue

import pysam

from assemblyline.lib.base import which
from assemblyline.lib.seq import DNA_reverse_complement
from assemblyline.lib.transcript import parse_gtf, strand_int_to_str, strand_str_to_int, NEG_STRAND, NO_STRAND, POS_STRAND, Exon
from assemblyline.lib.batch_sort import batch_sort

SIGNALP_HEADER = ['name', 'Cmax', 'Cpos', 'Ymax', 'Ypos', 'Smax', 'Spos',
                  'Smean', 'D', '?', 'Dmaxcut', 'Networksused']
PFAM_HEADER = ['seq_id', 'alignment_start', 'alignment_end',
               'envelope_start', 'envelope_end', 'hmm_acc', 'hmm_name',
               'type', 'hmm_start', 'hmm_end', 'hmm_length', 'bit_score',
               'e_value', 'significance', 'clan']

# batch sort configuration
SORT_BUFFER_SIZE = 32000

PFAM_FILES = ['Pfam-A.hmm',
              'Pfam-A.hmm.dat',
              'Pfam-A.hmm.h3f',
              'Pfam-A.hmm.h3i',
              'Pfam-A.hmm.h3m',
              'Pfam-A.hmm.h3p',
              'Pfam-B.hmm',
              'Pfam-B.hmm.dat',
              'Pfam-B.hmm.h3f',
              'Pfam-B.hmm.h3i',
              'Pfam-B.hmm.h3m',
              'Pfam-B.hmm.h3p']

def check_pfam_dir(path):
    for pfam_file in PFAM_FILES:
        if not os.path.exists(os.path.join(path, pfam_file)):
            return False
    return True

def to_fasta(name, seq):
    return '>%s\n%s' % (name, seq)

def parse_fasta_file(fasta_file):
    with open(fasta_file) as f:
        while True:
            lines = list(itertools.islice(f,2))
            if lines:
                yield lines
            else:
                break

def split_fasta_file(fasta_file, n, prefix):
    # open files
    filehandles = []
    for x in xrange(n):
        filename = '%s%d.fasta' % (prefix, x)
        filehandles.append(open(filename))
    x = 0
    for lines in parse_fasta_file(fasta_file):
        filehandles[x].writelines(lines)
        x  = (x + 1) % n
    # close files
    for f in filehandles:
        f.close()

GENETIC_CODE = {
    'ATA':'I', 'ATC':'I', 'ATT':'I', 'ATG':'M',
    'ACA':'T', 'ACC':'T', 'ACG':'T', 'ACT':'T',
    'AAC':'N', 'AAT':'N', 'AAA':'K', 'AAG':'K',
    'AGC':'S', 'AGT':'S', 'AGA':'R', 'AGG':'R',
    'CTA':'L', 'CTC':'L', 'CTG':'L', 'CTT':'L',
    'CCA':'P', 'CCC':'P', 'CCG':'P', 'CCT':'P',
    'CAC':'H', 'CAT':'H', 'CAA':'Q', 'CAG':'Q',
    'CGA':'R', 'CGC':'R', 'CGG':'R', 'CGT':'R',
    'GTA':'V', 'GTC':'V', 'GTG':'V', 'GTT':'V',
    'GCA':'A', 'GCC':'A', 'GCG':'A', 'GCT':'A',
    'GAC':'D', 'GAT':'D', 'GAA':'E', 'GAG':'E',
    'GGA':'G', 'GGC':'G', 'GGG':'G', 'GGT':'G',
    'TCA':'S', 'TCC':'S', 'TCG':'S', 'TCT':'S',
    'TTC':'F', 'TTT':'F', 'TTA':'L', 'TTG':'L',
    'TAC':'Y', 'TAT':'Y', 'TAA':'*', 'TAG':'*',
    'TGC':'C', 'TGT':'C', 'TGA':'*', 'TGG':'W',
}

class ORFInfo(object):
    FIELDS = ['transcript_id', 'gene_id', 'orf_id', 'frame',
              'chrom', 'start', 'end', 'strand', 'exons', 'seq']
    SEQ_COL_NUM = 11

    @staticmethod
    def make_orf_id(x):
        return 'ORF%09d' % (x)
    @staticmethod
    def cmp_orf_id(x, y):
        x = int(x.split('ORF')[1])
        y = int(y.split('ORF')[1])
        return cmp(x, y)

    def __init__(self):
        for field in ORFInfo.FIELDS:
            setattr(self, field, None)

    def to_table(self):
        return [self.transcript_id,
                self.gene_id,
                str(self.orf_id),
                str(self.frame),
                str(len(self.seq)),
                self.chrom,
                str(self.start),
                str(self.end),
                self.strand,
                ','.join(map(str, (e.start for e in self.exons))),
                ','.join(map(str, (e.end for e in self.exons))),
                self.seq]

    @staticmethod
    def from_table(line):
        fields = line.strip().split('\t')
        self = ORFInfo()
        self.transcript_id = fields[0]
        self.gene_id = fields[1]
        self.orf_id = fields[2]
        self.frame = int(fields[3])
        self.chrom = fields[5]
        self.start = int(fields[6])
        self.end = int(fields[7])
        self.strand = fields[8]
        exon_starts = map(int, fields[9].split(','))
        exon_ends = map(int, fields[10].split(','))
        self.exons = [Exon(x,y) for x,y in zip(exon_starts, exon_ends)]
        self.seq = fields[ORFInfo.SEQ_COL_NUM]
        return self

    def to_bed(self, name=None):
        assert all(self.exons[0].start < x.start for x in self.exons[1:])
        assert all(self.exons[-1].end > x.end for x in self.exons[:-1])
        assert self.start == self.exons[0].start
        assert self.end == self.exons[-1].end
        block_sizes = []
        block_starts = []
        for e in self.exons:
            block_starts.append(e.start - self.start)
            block_sizes.append(e.end - e.start)
        if name is None:
            name = '%s|%s|frame=%d|length=%d' % (self.transcript_id, self.gene_id, self.frame, len(self.seq))
        # make bed fields
        fields = [self.chrom,
                  str(self.start),
                  str(self.end),
                  name,
                  '0',
                  self.strand,
                  str(self.start),
                  str(self.end),
                  '0',
                  str(len(self.exons)),
                  ','.join(map(str,block_sizes)) + ',',
                  ','.join(map(str,block_starts)) + ',']
        return fields

def get_transcript_dna_sequence(t, ref_fa):
    exon_seqs = []
    error_occurred = False
    for e in t.exons:
        seq = ref_fa.fetch(t.chrom, e.start, e.end)
        if (not seq) or (len(seq) < (e.end - e.start)):
            logging.warning("transcript %s exon %s:%d-%d not found in reference" %
                            (t.attrs['transcript_id'], t.chrom, e.start, e.end))
            error_occurred = True
            break
        exon_seqs.append(seq)
    if error_occurred:
        return None
    # make fasta record
    seq = ''.join(exon_seqs).upper()
    # reverse complement negative stranded sequences
    if t.strand == NEG_STRAND:
        seq = DNA_reverse_complement(seq)
    return seq

def translate_dna_3frames(seq):
    proteins = [[] for x in xrange(3)]
    for n in range(0,len(seq),3):
        for frame in xrange(3):
            codon = seq[n+frame:n+frame+3]
            proteins[frame].append(GENETIC_CODE.get(codon, '.'))
    for frame in xrange(3):
        proteins[frame] = ''.join(proteins[frame]).rstrip('.')
    return proteins

def find_orfs(seq, start_codon='M', stop_codon='*'):
    '''
    start is inclusive and end is exclusive
    includes stop codon
    '''
    # find first start codon
    start = seq.find(start_codon)
    while (start >= 0):
        end = seq.find(stop_codon, start)
        if end == -1:
            yield start, len(seq), seq[start:]
            return
        yield start, end+1, seq[start:end+1]
        # find next start codon
        start = seq.find(start_codon, end+1)

def transcript_to_genome_pos(t, pos, reverse=False):
    reverse = (t.strand == NEG_STRAND) or reverse
    if reverse:
        exon_iter = xrange(len(t.exons)-1, -1, -1)
    else:
        exon_iter = xrange(0, len(t.exons))
    offset = 0
    for i in exon_iter:
        exon = t.exons[i]
        exon_size = exon.end - exon.start
        if pos < offset + exon_size:
            if reverse:
                return exon.start + exon_size - (pos - offset) - 1
            else:
                return exon.start + (pos - offset)
        offset += exon_size
    if reverse:
        return t.start
    else:
        return (t.end-1)

def genome_interval_to_exons(start, end, t):
    assert start >= t.start
    assert end <= t.end
    assert start < end
    i = 0
    while t.exons[i].end < start:
        i += 1
    start_exon = i
    while t.exons[i].end < end:
        i += 1
    end_exon = i
    newexons = []
    if start_exon == end_exon:
        newexons.append(Exon(start,end))
    else:
        newexons.append(Exon(start, t.exons[start_exon].end))
        for i in xrange(start_exon+1, end_exon):
            newexons.append(t.exons[i])
        newexons.append(Exon(t.exons[end_exon].start, end))
    return newexons

def orf_to_genome(t, frame, aa_start, aa_end):
    t_orf_start = frame + (3 * aa_start)
    t_orf_end = frame + (3 * aa_end) - 1 # make end position inclusive
    reverse = ((t.strand == NEG_STRAND) or (frame >= 3))
    g_orf_start = transcript_to_genome_pos(t, t_orf_start, reverse)
    g_orf_end = transcript_to_genome_pos(t, t_orf_end, reverse)
    if reverse:
        g_orf_start, g_orf_end = g_orf_end, g_orf_start
    g_orf_end += 1 # end position is inclusive so adjust
    orf_exons = genome_interval_to_exons(g_orf_start, g_orf_end, t)
    return g_orf_start, g_orf_end, orf_exons

def translate_transcript(t, ref_fa):
    # get DNA sequence of transcript
    seq = get_transcript_dna_sequence(t, ref_fa)
    # translate on all reading frames
    if t.strand == NO_STRAND:
        aa_seqs = translate_dna_3frames(seq)
        aa_seqs.extend(translate_dna_3frames(DNA_reverse_complement(seq)))
    else:
        aa_seqs = translate_dna_3frames(seq)
    return aa_seqs

def get_all_transcript_orfs(t, ref_fa, min_orf_length):
    # get amino acid sequences in all reading frames
    aa_seqs = translate_transcript(t, ref_fa)
    # get ORFs
    for frame, aa_seq in enumerate(aa_seqs):
        for aa_start, aa_end, orf_seq in find_orfs(aa_seq):
            if len(orf_seq) < min_orf_length:
                continue
            orf_start, orf_end, orf_exons = \
                orf_to_genome(t, frame, aa_start, aa_end)
            orf = ORFInfo()
            orf.transcript_id = t.attrs['transcript_id']
            orf.gene_id = t.attrs['gene_id']
            orf.frame = frame
            orf.chrom = t.chrom
            orf.start = orf_start
            orf.end = orf_end
            orf.strand = strand_int_to_str(t.strand)
            orf.exons = orf_exons
            orf.seq = orf_seq
            yield orf

def get_first_transcript_orf(t, ref_fa):
    def first_orf_to_genome(t, start, end):
        reverse = (t.strand == NEG_STRAND)
        g_orf_start = transcript_to_genome_pos(t, start, reverse)
        g_orf_end = transcript_to_genome_pos(t, end, reverse)
        if reverse:
            g_orf_start, g_orf_end = g_orf_end, g_orf_start
        g_orf_end += 1 # end position is inclusive so adjust
        orf_exons = genome_interval_to_exons(g_orf_start, g_orf_end, t)
        return g_orf_start, g_orf_end, orf_exons

    def translate_orf(seq):
        aa_seq = []
        for n in range(0, len(seq), 3):
            codon = seq[n:n+3]
            aa = GENETIC_CODE.get(codon, '.')
            aa_seq.append(aa)
            if aa == '*':
                break
        aa_seq = ''.join(aa_seq).rstrip('.')
        return aa_seq

    def find_first_orf(t, ref_fa):
        orf = ORFInfo()
        orf.transcript_id = t.attrs['transcript_id']
        orf.gene_id = t.attrs['gene_id']
        orf.chrom = t.chrom
        # get transcript sequence
        seq = get_transcript_dna_sequence(t, ref_fa)
        # find first ATG in sequence
        start = seq.find('ATG')
        if start == -1:
            orf.start = t.start
            orf.end = t.start
            orf.strand = '.'
            orf.exons = []
            orf.seq = ''
            orf.frame = 0
        else:
            aa_seq = translate_orf(seq[start:])
            end = start + 3 * len(aa_seq) - 1
            orf_start, orf_end, orf_exons = \
                first_orf_to_genome(t, start, end)
            orf.start = orf_start
            orf.end = orf_end
            orf.strand = strand_int_to_str(t.strand)
            orf.exons = orf_exons
            orf.seq = aa_seq
            orf.frame = (start % 3)
        return orf

    if t.strand == NO_STRAND:
        t.strand = POS_STRAND
        orfpos = find_first_orf(t, ref_fa)
        t.strand = NEG_STRAND
        orfneg = find_first_orf(t, ref_fa)
        if len(orfpos.seq) >= len(orfneg.seq):
            return orfpos
        else:
            return orfneg
    else:
        return find_first_orf(t, ref_fa)

def group_unique_orfs(line_iter):
    cur_orfs = []
    num_unique_orfs = 1
    cur_orf_id = ORFInfo.make_orf_id(num_unique_orfs)
    for line in line_iter:
        orf = ORFInfo.from_table(line)
        if (len(cur_orfs) > 0) and (cur_orfs[0].seq != orf.seq):
            yield cur_orfs
            cur_orfs = []
            num_unique_orfs += 1
            cur_orf_id = ORFInfo.make_orf_id(num_unique_orfs)
        orf.orf_id = cur_orf_id
        cur_orfs.append(orf)
    if len(cur_orfs) > 0:
        yield cur_orfs

def run_signalp(fasta_files, output_file, tmp_dir):
    def _consumer(worker_index, worker_fasta_file, output_queue):
        # process with signalp
        logging.debug('[Worker %d] Running SignalP' % (worker_index))
        args = ['signalp', '-f', 'short', '-t', 'euk', worker_fasta_file]
        tmp_output_file = os.path.join(tmp_dir, 'worker%d.signalp.txt' % (worker_index))
        with open(tmp_output_file, 'w') as outfileh:
            retcode = subprocess.call(args, stdout=outfileh)
            if retcode != 0:
                logging.error('[Worker %d] Error running SignalP' % (worker_index))
        # put results into output queue
        with open(tmp_output_file) as f:
            for line in f:
                if not line:
                    continue
                line = line.strip()
                if not line:
                    continue
                if line.startswith('#'):
                    continue
                output_queue.put(line)
        output_queue.put(None)
        logging.debug('[Worker %d] Finished' % (worker_index))
    # create multiprocessing queue for passing data
    output_queue = Queue(maxsize=len(fasta_files)*4)
    # start consumer processes
    procs = []
    for i in xrange(len(fasta_files)):
        p = Process(target=_consumer, args=(i, fasta_files[i], output_queue))
        p.start()
        procs.append(p)
    # get results from consumers
    num_alive = len(procs)
    tmp_output_file = os.path.join(tmp_dir, 'signalp.merge.txt')
    with open(tmp_output_file, 'w') as f:
        while num_alive > 0:
            result = output_queue.get()
            if result is None:
                num_alive -= 1
                logging.debug("Main process detected worker finished, %d still alive" % (num_alive))
            else:
                print >>f, result
    logging.debug("Joining all processes")
    # wait for consumers to finish
    for p in procs:
        p.join()
    # sort
    logging.debug('Sorting signal peptide results')
    def sort_signalp(line):
        return line.split(None,1)[0]
    sort_tmp_dir = os.path.join(tmp_dir, 'sort_tmp')
    os.makedirs(sort_tmp_dir)
    batch_sort(input=tmp_output_file,
               output=output_file,
               key=sort_signalp,
               buffer_size=SORT_BUFFER_SIZE,
               tempdirs=[sort_tmp_dir])
    # cleanup temporary files
    shutil.rmtree(sort_tmp_dir)
    return 0

def run_pfam(fasta_files, pfam_dir, output_file, tmp_dir):
    def _consumer(worker_index, worker_fasta_file, output_queue):
        # process with pfam
        logging.debug('[Worker %d] Running pfam_scan.pl' % (worker_index))
        tmp_output_file = os.path.join(tmp_dir, 'worker%d.pfam.txt' % (worker_index))
        args = ['pfam_scan.pl',
                '-cpu', '1',
                '-pfamB',
                '-fasta', worker_fasta_file,
                '-dir', pfam_dir,
                '-outfile', tmp_output_file]
        retcode = subprocess.call(args)
        if retcode != 0:
            logging.error('[Worker %d] Error running pfam_scan.pl' % (worker_index))
        # put results into output queue
        with open(tmp_output_file) as f:
            for line in f:
                if not line:
                    continue
                line = line.strip()
                if not line:
                    continue
                if line.startswith('#'):
                    continue
                output_queue.put(line)
        output_queue.put(None)
        logging.debug('[Worker %d] Finished' % (worker_index))
    # create multiprocessing queues for passing data
    output_queue = Queue(maxsize=len(fasta_files)*4)
    # start consumer processes
    procs = []
    for i in xrange(len(fasta_files)):
        p = Process(target=_consumer, args=(i, fasta_files[i], output_queue))
        p.start()
        procs.append(p)
    # get results from consumers
    num_alive = len(procs)
    tmp_output_file = os.path.join(tmp_dir, 'pfam.merge.txt')
    with open(tmp_output_file, 'w') as f:
        while num_alive > 0:
            result = output_queue.get()
            if result is None:
                num_alive -= 1
                logging.debug("Main process detected worker finished, %d still alive" % (num_alive))
            else:
                print >>f, result
    logging.debug("Joining all processes")
    # wait for consumers to finish
    for p in procs:
        p.join()
    # sort
    logging.debug('Sorting pfam results')
    def sort_pfam(line):
        return line.split(None,1)[0]
    sort_tmp_dir = os.path.join(tmp_dir, 'sort_tmp')
    os.makedirs(sort_tmp_dir)
    batch_sort(input=tmp_output_file,
               output=output_file,
               key=sort_pfam,
               buffer_size=SORT_BUFFER_SIZE,
               tempdirs=[sort_tmp_dir])
    # cleanup temporary files
    shutil.rmtree(sort_tmp_dir)
    return 0

def merge_results(orf_table_file, signalp_file, pfam_file, output_file):
    class SortedFileParser:
        def __init__(self, fileh, key_col=0, key_cmp=ORFInfo.cmp_orf_id, sep=None):
            self.fileh = fileh
            self.key_col = key_col
            self.key_cmp = key_cmp
            self.fields = None
            self.sep = sep
        def seek(self, orf_id):
            try:
                while True:
                    if self.fields is None:
                        c = -1
                    else:
                        c = self.key_cmp(self.fields[self.key_col], orf_id)
                    if c == 0:
                        # found matching line
                        return self.fields
                    elif c == 1:
                        # larger than current id so no match
                        return None
                    # smaller than current id so keep seeking
                    self.fields = self.fileh.next().strip().split(self.sep)
            except StopIteration:
                pass
    with open(output_file, 'w') as outfile:
        with open(orf_table_file) as infile:
            pfam_parser = SortedFileParser(open(pfam_file), key_col=0)
            signalp_parser = SortedFileParser(open(signalp_file), key_col=0)
            for line in infile:
                orf = ORFInfo.from_table(line)
                signalp_fields = signalp_parser.seek(orf.orf_id)
                if signalp_fields is None:
                    signalp_fields = ['None'] * len(SIGNALP_HEADER)
                pfam_fields = pfam_parser.seek(orf.orf_id)
                if pfam_fields is None:
                    pfam_fields = ['None'] * len(PFAM_HEADER)
                fields = orf.to_table()
                fields.extend(signalp_fields)
                fields.extend(pfam_fields)
                print >>outfile, '\t'.join(fields)

def orf_analysis(gtf_file, genome_fasta_file, pfam_dir,
                 output_dir, min_orf_length, first_orf_only,
                 num_processes):
    #
    # extract transcript DNA sequences, translate to protein, and
    # search for ORFs
    #
    logging.debug('Finding ORFs in transcript sequences')
    # output files
    tmp_dir = os.path.join(output_dir, 'tmp')
    if not os.path.exists(tmp_dir):
        logging.info("Creating tmp directory '%s'" % (tmp_dir))
        os.makedirs(tmp_dir)
    orf_bed_file = os.path.join(output_dir, 'transcript_orfs.bed')
    unique_orf_file = os.path.join(output_dir, 'unique_orfs.txt')
    unique_orf_bed_file = os.path.join(output_dir, 'unique_orfs.bed')
    orf_file = os.path.join(tmp_dir, 'transcript_orfs.no_ids.txt')
    sorted_orf_file = os.path.join(tmp_dir, 'transcript_orfs.no_ids.sortbyorf.txt')
    sorted_orf_id_file = os.path.join(tmp_dir, 'transcript_orfs.sortbyorf.txt')
    signalp_file = os.path.join(output_dir, 'signalp.txt')
    pfam_file = os.path.join(output_dir, 'pfam.txt')
    merged_orf_id_file = os.path.join(output_dir, 'transcript_orfs.sortbyorf.merged.txt')
    sorted_merged_orf_id_file = os.path.join(output_dir, 'transcript_orfs.sortbytranscript.merged.txt')
    # open output files
    orf_fileh = open(orf_file, 'w')
    orf_bed_fileh = open(orf_bed_file, 'w')
    # open genome fasta file
    ref_fa = pysam.Fastafile(genome_fasta_file)
    num_finished = 1
    for locus_transcripts in parse_gtf(open(gtf_file)):
        for t in locus_transcripts:
            if first_orf_only:
                # get first ORF
                orf = get_first_transcript_orf(t, ref_fa)
                if len(orf.seq) >= min_orf_length:
                    print >>orf_fileh, '\t'.join(orf.to_table())
                    print >>orf_bed_fileh, '\t'.join(orf.to_bed())
            else:
                # get all ORFs
                for orf in get_all_transcript_orfs(t, ref_fa, min_orf_length):
                    print >>orf_fileh, '\t'.join(orf.to_table())
                    print >>orf_bed_fileh, '\t'.join(orf.to_bed())
            if (num_finished % 10000) == 0:
                logging.debug('Processed %d transcripts' % (num_finished))
            num_finished += 1
    # cleanup
    orf_fileh.close()
    orf_bed_fileh.close()
    #
    # sort ORF table by ORF amino acid sequence to group identical ORFs
    # together
    #
    logging.debug('Sorting ORFs by amino acid sequence')
    def sort_by_seq(line):
        '''comparison function for batch_sort'''
        fields = line.strip().split('\t')
        return fields[ORFInfo.SEQ_COL_NUM]
    sort_tmp_dir = os.path.join(tmp_dir, 'sort_tmp')
    os.makedirs(sort_tmp_dir)
    batch_sort(input=orf_file,
               output=sorted_orf_file,
               key=sort_by_seq,
               buffer_size=SORT_BUFFER_SIZE,
               tempdirs=[sort_tmp_dir])
    shutil.rmtree(sort_tmp_dir)
    #
    # assign each ORF a unique id and write to FASTA file
    #
    logging.debug('Determining unique ORFs')
    orf_fasta_prefix = os.path.join(tmp_dir, 'orf')
    orf_fasta_files = []
    orf_fasta_sizes = []
    for i in xrange(num_processes):
        orf_fasta_files.append(open('%s%d.fasta' % (orf_fasta_prefix, i), 'w'))
        orf_fasta_sizes.append(0)
    orf_file_index = 0
    outfileh = open(sorted_orf_id_file, 'w')
    unique_orf_fileh = open(unique_orf_file, 'w')
    print >>unique_orf_fileh, '\t'.join(['orf_id', 'orf_length', 'total_occurrences', 'unique_genomic_occurrences'])
    unique_orf_bed_fileh = open(unique_orf_bed_file, 'w')
    with open(sorted_orf_file) as infileh:
        for orfs in group_unique_orfs(infileh):
            # write to master transcript/ORF table
            for orf in orfs:
                print >>outfileh, '\t'.join(orf.to_table())
            # write ORF to fasta file
            lines = to_fasta(orfs[0].orf_id, orfs[0].seq.strip('*'))
            print >>orf_fasta_files[orf_file_index], lines
            orf_fasta_sizes[orf_file_index] += 1
            # advance to next fasta file
            orf_file_index = (orf_file_index + 1) % (num_processes)
            # group by genomic position and write ORFs to BED file
            unique_genome_orfs = {}
            for orf in orfs:
                k = (orf.chrom, orf.strand, tuple(orf.exons))
                if k in unique_genome_orfs:
                    continue
                unique_genome_orfs[k] = orf
            for orf in unique_genome_orfs.itervalues():
                print >>unique_orf_bed_fileh, '\t'.join(orf.to_bed(orf.orf_id))
            # write unique ORF to tab-delimited text file
            fields = [orfs[0].orf_id, len(orfs[0].seq), len(orfs), len(unique_genome_orfs)]
            print >>unique_orf_fileh, '\t'.join(map(str,fields))
    # cleanup
    unique_orf_bed_fileh.close()
    outfileh.close()
    # get fasta files with lines written
    orf_fasta_file_names = []
    for i in xrange(len(orf_fasta_files)):
        orf_fasta_files[i].close()
        if orf_fasta_sizes[i] > 0:
            orf_fasta_file_names.append(orf_fasta_files[i].name)
    #
    # search FASTA file against signalp
    #
    logging.debug('Searching for signal peptides')
    retcode = run_signalp(orf_fasta_file_names, signalp_file, tmp_dir)
    if retcode != 0:
        logging.error('Error searching for signal peptides')
        return 1
    #
    # search FASTA file against Pfam
    #
    logging.error('Scanning for Pfam domains')
    retcode = run_pfam(orf_fasta_file_names, pfam_dir, pfam_file, tmp_dir)
    if retcode != 0:
        logging.error('Error running pfam_scan.pl')
    #
    # merge results from Pfam and signalp
    #
    logging.debug('Merging SignalP and Pfam results')
    merge_results(sorted_orf_id_file, signalp_file, pfam_file, merged_orf_id_file)
    #
    # sort by transcript id
    #
    logging.debug('Sorting ORFs by transcript ID')
    def sort_by_transcript_id(line):
        return line.split('\t', 1)[0]
    sort_tmp_dir = os.path.join(tmp_dir, 'sort_tmp')
    os.makedirs(sort_tmp_dir)
    batch_sort(input=merged_orf_id_file,
               output=sorted_merged_orf_id_file,
               key=sort_by_transcript_id,
               buffer_size=SORT_BUFFER_SIZE,
               tempdirs=[sort_tmp_dir])
    shutil.rmtree(sort_tmp_dir)
    # cleanup
    ref_fa.close()
    return 0

def full_transcript_analysis(gtf_file, genome_fasta_file, pfam_dir,
                             output_dir, num_processes):
    # output files
    pfam_file = os.path.join(output_dir, 'full_length_pfam.txt')
    # open genome fasta file
    ref_fa = pysam.Fastafile(genome_fasta_file)
    # convert transcripts to amino acid sequences and write to fasta file
    logging.debug('Writing transcript amino acid sequences to FASTA file(s)')
    tmp_dir = os.path.join(output_dir, 'tmp')
    os.makedirs(tmp_dir)
    fasta_prefix = os.path.join(tmp_dir, 'full')
    fasta_files = []
    fasta_sizes = []
    for i in xrange(num_processes):
        fasta_files.append(open('%s%d.fasta' % (fasta_prefix, i), 'w'))
        fasta_sizes.append(0)
    num_finished = 1
    fasta_file_index = 0
    for locus_transcripts in parse_gtf(open(gtf_file)):
        for t in locus_transcripts:
            # get amino acid sequences in all reading frames
            aa_seqs = translate_transcript(t, ref_fa)
            for frame, aa_seq in enumerate(aa_seqs):
                lines = to_fasta('%s|frame=%d' % (t.attrs['transcript_id'], frame), aa_seq)
                print >>fasta_files[fasta_file_index], lines
            fasta_sizes[fasta_file_index] += 1
            fasta_file_index = (fasta_file_index + 1) % num_processes
            if (num_finished % 10000) == 0:
                logging.debug('Processed %d transcripts' % (num_finished))
            num_finished += 1
    # get fasta files with lines written
    fasta_file_names = []
    for i in xrange(len(fasta_files)):
        fasta_files[i].close()
        if fasta_sizes[i] > 0:
            fasta_file_names.append(fasta_files[i].name)
    # cleanup
    ref_fa.close()
    #
    # search FASTA file against Pfam
    #
    logging.debug('Scanning for Pfam domains')
    retcode = run_pfam(fasta_file_names, pfam_dir, pfam_file, tmp_dir)
    if retcode != 0:
        logging.error('Error running pfam_scan.pl')
        return retcode
    return 0

def main():
    logging.basicConfig(level=logging.DEBUG,
                        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
    parser = argparse.ArgumentParser()
    parser.add_argument('--pfam', dest='pfam_dir')
    parser.add_argument('--genome-fasta', dest='genome_fasta_file')
    parser.add_argument('--min-orf-length', dest='min_orf_length', type=int, default=30)
    parser.add_argument('-o', '--output-dir', dest='output_dir', default='out')
    parser.add_argument('-p', '--num-processes', dest='num_processes', type=int, default=1)
    parser.add_argument('--mode', dest='mode', choices=['orf', 'first_orf', 'full'], default='orf')
    parser.add_argument('gtf_file')
    args = parser.parse_args()
    # get args
    pfam_dir = args.pfam_dir
    genome_fasta_file = args.genome_fasta_file
    gtf_file = args.gtf_file
    min_orf_length = args.min_orf_length
    num_processes = args.num_processes
    output_dir = args.output_dir
    mode = args.mode
    # check command line parameters
    if which('pfam_scan.pl') is None:
        parser.error("'pfam_scan.pl' not found in PATH")
    if which('hmmscan') is None:
        parser.error("'hmmscan' not found in PATH")
    if which('signalp') is None:
        parser.error("'signalp' not found in PATH")
    if not check_pfam_dir(pfam_dir):
        parser.error("Required Pfam-A and Pfam-B files not found at '%s'" % (pfam_dir))
    if not os.path.exists(genome_fasta_file):
        parser.error("Genome FASTA file '%s' not found" % (genome_fasta_file))
    if not os.path.exists(gtf_file):
        parser.error("GTF file '%s' not found" % (gtf_file))
    #if os.path.exists(output_dir):
    #    parser.error("Output directory '%s' already exists" % (output_dir))
    # create output dir
    if not os.path.exists(output_dir):
        logging.info("Creating output directory '%s'" % (output_dir))
        os.makedirs(output_dir)
    if mode == 'full':
        return full_transcript_analysis(gtf_file, genome_fasta_file,
                                        pfam_dir, output_dir, num_processes)
    else:
        first_orf_only = (mode == 'first_orf')
        return orf_analysis(gtf_file, genome_fasta_file, pfam_dir,
                            output_dir, min_orf_length, first_orf_only,
                            num_processes)
    return 0

if __name__ == '__main__':
    sys.exit(main())


import numpy as np
import pytest
from shapely.geometry import box, Point

from dask_geomodeling import raster
from dask_geomodeling.utils import shapely_transform
from dask_geomodeling.raster.sources import MemorySource

@pytest.fixture
def source():
    yield MemorySource(
        data=np.full((1, 10, 10), 7, dtype=np.uint8),
        no_data_value=255,
        projection="EPSG:28992",
        pixel_size=10,
        pixel_origin=(135000, 456000),
    )

@pytest.fixture(scope="module", params=["exact", "zoomed_in", "zoomed_out"])
def vals_request(request):
    if request.param == "exact":
        bbox = (0, 0, 100, 80)
    elif request.param == "zoomed_in":
        bbox = (0, 0, 50, 40)
    elif request.param == "zoomed_out":
        bbox = (0, 0, 200, 160)
    return dict(
        mode="vals",
        bbox=bbox,
        projection="EPSG:28992",
        width=int(bbox[2] / 10),
        height=int(bbox[3] / 10),
    )

@pytest.fixture
def empty():
    yield MemorySource(
        data=np.full((0, 0, 0), 7, dtype=np.uint8),
        no_data_value=255,
        projection="EPSG:28992",
        pixel_size=20,
        pixel_origin=(0, 0),
    )

@pytest.fixture
def center():
    # the source raster has data at x 135000..135100 and y 456000..455900
    yield 135050, 455950

@pytest.fixture
def center_epsg3857(center):
    yield shapely_transform(Point(*center), "EPSG:28992", "EPSG:3857").coords[0]

def test_place_attrs(source, center):
    place = raster.Place(source, "EPSG:28992", center, [(50, 50)])

    # Place should not adapt these attributes:
    assert place.period == source.period
    assert place.timedelta == source.timedelta
    assert place.dtype == source.dtype
    assert place.fillvalue == source.fillvalue

    # If the place projection equals the store projection:
    assert place.projection == source.projection
    assert place.geo_transform == source.geo_transform

    extent_epsg28992 = (0, 0, 100, 100)
    extent_epsg4326 = shapely_transform(
        box(*extent_epsg28992), "EPSG:28992", "EPSG:4326"
    ).bounds
    x1, x2, y1, y2 = place.geometry.GetEnvelope()
    assert (x1, y1, x2, y2) == extent_epsg28992
    assert place.extent == pytest.approx(extent_epsg4326, rel=1e-4)

def test_place_invalid_statistic(source, center):
    with pytest.raises(ValueError):
        raster.Place(source, "EPSG:28992", center, [(50, 50)], statistic="nonexisting")

def test_place_attrs_reproject(source, center_epsg3857):
    # Place should adapt the spatial attributes (extent & geometry)
    place = raster.Place(
        source, "EPSG:3857", center_epsg3857, [(572050, 6812050), (570050, 6811050)]
    )
    # The native projection != store projection:
    assert place.projection is None
    assert place.geo_transform is None

    extent_epsg3857 = 570000, 6811000, 572100, 6812100
    extent_epsg4326 = shapely_transform(
        box(*extent_epsg3857), "EPSG:3857", "EPSG:4326"
    ).bounds
    x1, x2, y1, y2 = place.geometry.GetEnvelope()
    assert (x1, y1, x2, y2) == pytest.approx(extent_epsg3857, rel=1e-4)
    assert place.extent == pytest.approx(extent_epsg4326, rel=1e-4)

def test_place_empty(empty, center, vals_request):
    place = raster.Place(empty, "EPSG:28992", center, [(50, 50)])

    assert place.geometry is None
    assert place.extent is None
    assert place.get_data(**vals_request) is None

def test_place_no_coords(source, center, vals_request):
    place = raster.Place(source, "EPSG:28992", center, [])
    values = place.get_data(**vals_request)["values"]
    assert (values[:, :10, :10] == source.fillvalue).all()

def test_place_exact(source, center, vals_request):
    place = raster.Place(source, "EPSG:28992", center, [(50, 50)])
    values = place.get_data(**vals_request)["values"]
    # swap Y axis for readable test
    values = values[:, ::-1, :]
    assert (values[:, :10, :10] == 7).all()

def test_place_reproject(source, center_epsg3857):
    target = 572050, 6812050  # EPSG3857 coords somewhere inside RD validity
    place = raster.Place(source, "EPSG:3857", center_epsg3857, [target])
    x, y = shapely_transform(Point(*target), "EPSG:3857", "EPSG:28992").coords[0]
    values = place.get_data(
        mode="vals",
        bbox=(x - 40, y - 40, x + 40, y + 40),
        projection="EPSG:28992",
        width=8,
        height=8,
    )["values"]
    assert (values == 7).all()

def test_place_horizontal_shift(source, center, vals_request):
    # shift 1 cell (10 meters) to the right
    place = raster.Place(source, "EPSG:28992", center, [(60, 50)])
    values = place.get_data(**vals_request)["values"]
    # swap Y axis for readable test
    values = values[:, ::-1, :]
    assert (values[:, :10, 1:11] == 7).all()
    assert (values[:, :, 0] == 255).all()

def test_place_vertical_shift(source, center, vals_request):
    # shift 1 cell (10 meters) up
    place = raster.Place(source, "EPSG:28992", center, [(50, 60)])
    values = place.get_data(**vals_request)["values"]
    # swap Y axis for readable test
    values = values[:, ::-1, :]
    assert (values[:, 1:11, :10] == 7).all()
    assert (values[:, 0, :] == 255).all()

@pytest.mark.parametrize(
    "statistic,expected",
    [
        ("first", (255, 7, 7, 7)),  # (no features, first, second, both)
        ("last", (255, 7, 7, 7)),  # the default
        ("count", (0, 1, 1, 2)),
        ("sum", (0, 7, 7, 14)),
        ("mean", (255, 7, 7, 7)),
        ("min", (255, 7, 7, 7)),
        ("max", (255, 7, 7, 7)),
        ("argmin", (255, 0, 1, 0)),
        ("argmax", (255, 0, 1, 0)),
        ("std", (255, 0, 0, 0)),
        ("var", (255, 0, 0, 0)),
        ("median", (255, 7, 7, 7)),
        ("p99", (255, 7, 7, 7)),
    ],
)
def test_place_multiple(source, center, vals_request, statistic, expected):
    # place such that only the left and bottom ridges have values
    place = raster.Place(
        source, "EPSG:28992", center, [(-40, 50), (50, -40)], statistic
    )
    values = place.get_data(**vals_request)["values"]
    # swap Y axis for readable test
    values = values[:, ::-1, :]
    # zero features
    assert (values[:, 1:, 1:] == expected[0]).all()
    # one feature
    assert (values[:, 1:10, 0] == expected[1]).all()
    assert (values[:, 0, 1:10] == expected[2]).all()
    # two features
    assert (values[:, 0, 0] == expected[3]).all()

def test_place_outside(source, center, vals_request):
    x1, y1, x2, y2 = vals_request["bbox"]
    coordinates = [(x1 - 50, y1), (x1, y1 - 50), (x2 + 50, y2), (x2, y2 + 50)]
    place = raster.Place(source, "EPSG:28992", center, coordinates)
    values = place.get_data(**vals_request)["values"]
    assert (values[:, :, :] == 255).all()

def test_place_time_request(source, center):
    place = raster.Place(source, "EPSG:28992", center, [(150, 50)])
    assert source.get_data(mode="time") == place.get_data(mode="time")

def test_place_meta_request(source, center):
    place = raster.Place(source, "EPSG:28992", center, [(150, 50)])
    assert source.get_data(mode="meta") == place.get_data(mode="meta")

@pytest.mark.parametrize(
    "point,expected",
    [
        ((5, 15), 7),  # zone 1
        ((15, 15), 255),  # zone 2
        ((5, 5), 255),  # zone 3
        ((15, 5), 7),  # zone 4
        ((10, 15), 255),  # line 1-2
        ((5, 10), 255),  # line 1-3
        ((15, 10), 7),  # line 2-4
        ((10, 5), 7),  # line 3-4
        ((10, 10), 7),  # center
        ((1000, 1000), 255),  # outside
    ],
)
def test_place_point_request(source, center, point, expected):
    # For point requests, edges are important. Let's do a drawing:
    # 20  _______
    #    |       |
    #    |   1   |   2
    # 10 |_______|_______
    #            |       |
    #        3   |   4   |
    #  0         |_______|
    #    0       10     20
    # - zone 1 and 4 are filled; zone 2 and 3 are empty (see below coordinates)
    # A pixel includes its topleft corner and top and left edges
    # - line between 2-4 and 3-4 are filled; line 1-2 and 1-3 are empty
    # - center point at (10, 10) is filled
    coordinates = [(60, -40), (-40, 60)]
    place = raster.Place(source, "EPSG:28992", anchor=center, coordinates=coordinates)
    point_request = dict(
        mode="vals", bbox=point * 2, projection="EPSG:28992", width=1, height=1
    )
    values = place.get_data(**point_request)["values"]
    assert values.shape == (1, 1, 1)
    assert values.item() == expected


import os
os.environ.setdefault('PLANTMONITOR_MODULE_SETTINGS', 'conf.settings.testing')

import unittest

from pony import orm

import datetime

from ORM.pony_manager import PonyManager

from plantmonitor.storage import PonyMetricStorage, ApiMetricStorage

from yamlns import namespace as ns
import datetime

# Test against a fully functioning api
from multiprocessing import Process
import uvicorn
import time
import requests

class ApiClient_Test(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        from conf import envinfo
        assert envinfo.SETTINGS_MODULE == 'conf.settings.testing'

        cls.pony = PonyManager(envinfo.DB_CONF)

        cls.pony.define_all_models()
        cls.pony.binddb(create_tables=True)

        from api_server.plantmonitor_api import api

        cls.api = api

    @classmethod
    def tearDownClass(cls):
        cls.pony.db.drop_all_tables(with_all_data=True)
        cls.pony.db.disconnect()

    def setUp(self):

        from conf import envinfo
        self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')

        self.pony.db.drop_all_tables(with_all_data=True)

        self.pony.db.create_tables()

        orm.db_session.__enter__()

        # log_level = "debug"
        log_level = "warning"

        # create api and launch local test server
        self.proc = Process(target=uvicorn.run,
                    args=(self.api,),
                    kwargs={
                        "host": "127.0.0.1",
                        "port": self.apiPort(),
                        "log_level": log_level},
                    daemon=True)
        self.proc.start()
        time.sleep(0.1)

    def tearDown(self):
        orm.rollback()
        orm.db_session.__exit__()
        self.proc.terminate()

    def apiPort(self):
        return 5000

    def createApiClient(self):
        #TODO configfile
        config = {
            "api_url":"http://localhost:{}".format(self.apiPort()),
            "version":"1.0",
        }
        return ApiMetricStorage(config)

    def createPlantDict(self):
        plant_name = 'SomEnergia_Alibaba'
        inverter_name = 'Mary'
        metrics = ns([
            ('daily_energy_h_wh', 0),
            ('daily_energy_l_wh', 17556),
            ('e_total_h_wh', 566),
            ('e_total_l_wh', 49213),
            ('h_total_h_h', 0),
            ('h_total_l_h', 18827),
            ('pac_r_w', 0),
            ('pac_s_w', 0),
            ('pac_t_w', 0),
            ('powerreactive_t_v', 0),
            ('powerreactive_r_v', 0),
            ('powerreactive_s_v', 0),
            ('temp_inv_dc', 320),
            ('time', datetime.datetime.now(datetime.timezone.utc)),
        ])

        return plant_name, inverter_name, metrics

    def samplePlantNS(self):
        plantNS = ns.loads("""\
                name: alibaba
                codename: SCSOM04
                description: la bonica planta
                inverters:
                - inverter:
                    name: '5555'
                temperatureAmbientSensors:
                - temperatureAmbientSensor:
                    name: benjami""")
        return plantNS

    def samplePlantData(self, time):
        plantData = {
              "plant": "alibaba",
              "devices":
                sorted([{
                    "id": "Meter:1234578",
                    "readings": [{
                        "time": time,
                        "export_energy_wh": 1,
                        "import_energy_wh": 123,
                        "r1_VArh": 1234,
                        "r2_VArh": 124,
                        "r3_VArh": 1234,
                        "r4_VArh": 124,
                    }],
                },
                {
                    "id": "SensorTemperatureAmbient:5555",
                    "readings": [{
                        "time": time,
                        "temperature_dc": 170,
                    }]
                }],key=lambda d: d['id']),
              }
        return plantData

    def createPlant(self):
        plantNS = self.samplePlantNS()

        alibaba = self.pony.db.Plant(name=plantNS.name, codename=plantNS.codename, description=plantNS.description)
        alibaba = alibaba.importPlant(plantNS)
        orm.flush()
        # do we need to add some data?
        # time = datetime.datetime.now(datetime.timezone.utc)
        # plant_data = self.samplePlantData(time)

        # storage = PonyMetricStorage()
        # storage.insertPlantData(plant_data)

    def test__api_hello_version(self):
        response = requests.get("http://localhost:{}/version".format(self.apiPort()))
        self.assertEqual(response.status_code, 200)
        self.assertDictEqual(response.json(),{"version":"1.0"})

    def test_Environment(self):
        #TODO will it be too late if the config is misconfigured?
        from conf import envinfo
        self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')

    def test__ApiMetricStorage__storeInverterMeasures(self):
        plant_name, inverter_name, inverter_registers = self.createPlantDict()

        apiClient = self.createApiClient()
        result = apiClient.storeInverterMeasures(plant_name, inverter_name, inverter_registers)

        self.assertTrue(result)

    def test__ApiMetricStorage_insertPlantData__empty(self):
        plant_data = {
            "plant": "Alibaba",
            "version": "1.0",
        }

        apiClient = self.createApiClient()
        result = apiClient.insertPlantData(plant_data)

        expectedError = {"detail":[{"loc":["body","time"],"msg":"field required","type":"value_error.missing"},{"loc":["body","devices"],"msg":"field required","type":"value_error.missing"}]}

        self.assertDictEqual(result, expectedError)

    def test__ApiMetricStorage_insertPlantData__someData(self):
        self.createPlant()
        time = datetime.datetime.now(datetime.timezone.utc)

        plant_data = {
            "plant": "alibaba",
            "version": "1.0",
            "time": time.isoformat(),
            "devices":
            [{
                "id": "SensorTemperatureAmbient:benjami",
                "readings":
                [{
                        "temperature_dc": 120,
                        "time": time.isoformat(),
                }]
            }]
        }

        apiClient = self.createApiClient()
        result = apiClient.insertPlantData(plant_data)
        print(result)
        # expectedResponse = {"detail":[{"loc":["body","time"],"msg":"OK","type":"None"},{"loc":["body","devices"],"msg":"OK","type":"None"}]}

        self.assertDictEqual(result, plant_data)

class Storage_Test(unittest.TestCase):

    from yamlns.testutils import assertNsEqual

    def setUp(self):

        from conf import envinfo
        self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')

        orm.rollback()
        self.pony = PonyManager(envinfo.DB_CONF)

        self.pony.define_all_models()
        self.pony.binddb(create_tables=True)

        self.pony.db.drop_all_tables(with_all_data=True)

        self.storage = PonyMetricStorage(self.pony.db)

        self.maxDiff=None
        # orm.set_sql_debug(True)

        self.pony.db.create_tables()

        # database.generate_mapping(create_tables=True)
        orm.db_session.__enter__()

    def tearDown(self):
        orm.rollback()
        orm.db_session.__exit__()
        self.pony.db.drop_all_tables(with_all_data=True)
        self.pony.db.disconnect()

    def samplePlantData(self, time):
        plantData = {
            "plant": "alibaba",
            "devices":
                sorted([{
                    "id": "Meter:1234578",
                    "readings": [{
                        "time": time,
                        "export_energy_wh": 1,
                        "import_energy_wh": 123,
                        "r1_VArh": 1234,
                        "r2_VArh": 124,
                        "r3_VArh": 1234,
                        "r4_VArh": 124,
                    }],
                },
                {
                    "id": "SensorTemperatureAmbient:5555",
                    "readings": [{
                        "time": time,
                        "temperature_dc": 170,
                    }]
                }],key=lambda d: d['id']),
            }
        return plantData

    def test_Environment(self):
        #TODO will it be too late if the config is misconfigured?
        from conf import envinfo
        self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')

    def test_connection(self):
        with orm.db_session:
            self.assertEqual(self.pony.db.get_connection().status, 1)

    def test__datetimeToStr(self):
        time = datetime.datetime.now(datetime.timezone.utc)
        plant_data = self.samplePlantData(time)
        ApiMetricStorage.datetimeToStr(plant_data)
        expected_plant_data = self.samplePlantData(time.isoformat())

        self.assertDictEqual(plant_data, expected_plant_data)

    #TODO deprecated as soon as we switch plant_data
    # test instead registries_to_plant_data in task.py
    def __test_PublishOrmOneInverterRegistry(self):
        plant_name = 'SomEnergia_Alibaba'
        inverter_name = 'Mary'
        with orm.db_session:
            alcolea = self.pony.db.Plant(name=plant_name,  codename='SOMSC01', description='descripción de planta')
            inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
            metrics = ns([
                    ('daily_energy_h_wh', 0),
                    ('daily_energy_l_wh', 17556),
                    ('e_total_h_wh', 566),
                    ('e_total_l_wh', 49213),
                    ('h_total_h_h', 0),
                    ('h_total_l_h', 18827),
                    ('pac_r_w', 0),
                    ('pac_s_w', 0),
                    ('pac_t_w', 0),
                    ('powerreactive_t_v', 0),
                    ('powerreactive_r_v', 0),
                    ('powerreactive_s_v', 0),
                    ('temp_inv_dc', 320),
                    ('time', datetime.datetime.now(datetime.timezone.utc)),
                    # Sensors registers  obtained from inverters
                    ('probe1value', 443),
                    ('probe2value', 220),
                    ('probe3value', 0),
                    ('probe4value', 0),
                    ])

            self.storage.storeInverterMeasures(
                plant_name, inverter_name, metrics)

            metrics.pop('probe1value')
            metrics.pop('probe2value')
            metrics.pop('probe3value')
            metrics.pop('probe4value')

            expectedRegistry = dict(metrics)
            expectedRegistry['inverter'] = inverter.id

            inverterReadings = self.storage.inverterReadings()
            self.assertEqual(inverterReadings, [expectedRegistry])

    def __test_PublishOrmIfInverterNotExist(self):
        inverter_name = 'Alice'
        plant_name = 'SomEnergia_Alibaba'
        with orm.db_session:
            alcolea = self.pony.db.Plant(name=plant_name,  codename='SOMSC01', description='descripción de planta')
            inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
            metrics = ns([
                ('daily_energy_h_wh', 0),
                ('daily_energy_l_wh', 17556),
                ('e_total_h_wh', 566),
                ('e_total_l_wh', 49213),
                ('h_total_h_h', 0),
                ('h_total_l_h', 18827),
                ('pac_r_w', 0),
                ('pac_s_w', 0),
                ('pac_t_w', 0),
                ('powerreactive_t_v', 0),
                ('powerreactive_r_v', 0),
                ('powerreactive_s_v', 0),
                ('temp_inv_dc', 320),
                ('time', datetime.datetime.now(datetime.timezone.utc)),
                # Sensors registers  obtained from inverters
                ('probe1value', 443),
                ('probe2value', 220),
                ('probe3value', 0),
                ('probe4value', 0),
                ])
            self.storage.storeInverterMeasures(
                plant_name, "UnknownInverter", metrics)

            self.assertListEqual(self.storage.inverterReadings(), [])

    def __test_PublishOrmIfPlantNotExist(self):
        inverter_name = 'Alice'
        plant_name = 'SomEnergia_Alibaba'
        with orm.db_session:
            alcolea = self.pony.db.Plant(name=plant_name,  codename='SOMSC01', description='descripción de planta')
            inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
            metrics = ns([
                ('daily_energy_h_wh', 0),
                ('daily_energy_l_wh', 17556),
                ('e_total_h_wh', 566),
                ('e_total_l_wh', 49213),
                ('h_total_h_h', 0),
                ('h_total_l_h', 18827),
                ('pac_r_w', 0),
                ('pac_s_w', 0),
                ('pac_t_w', 0),
                ('powerreactive_t_v', 0),
                ('powerreactive_r_v', 0),
                ('powerreactive_s_v', 0),
                ('temp_inv_dc', 320),
                ('time', datetime.datetime.now(datetime.timezone.utc)),
                # Sensors registers  obtained from inverters
                ('probe1value', 443),
                ('probe2value', 220),
                ('probe3value', 0),
                ('probe4value', 0),
                ])
            self.storage.storeInverterMeasures(
                'UnknownPlant', inverter_name, metrics)

            self.assertListEqual(self.storage.inverterReadings(), [])

    def test__PonyMetricStorage_insertPlantData__storeTemperatureSensor(self):
        sensor_name = 'Alice'
        plant_name = 'SomEnergia_Alibaba'
        time = datetime.datetime.now(datetime.timezone.utc)

        with orm.db_session:
            alcolea = self.pony.db.Plant(name=plant_name,  codename='SOMSC01', description='descripción de planta')
            sensor = self.pony.db.SensorTemperatureAmbient(name=sensor_name, plant=alcolea)
            orm.flush()
            plant_data = {
                "plant": plant_name,
                "version": "1.0",
                "time": time.isoformat(), #consider using fastapi.jsonable_encoder
                "devices":
                [{
                    "id": "SensorTemperatureAmbient:Alice",
                    "readings":
                    [{
                        "temperature_dc": 120,
                        "time": time,
                    }]
                }]
            }
            self.storage.insertPlantData(plant_data)
            orm.flush()

            expected_plant_data = {
                'plant': plant_name,
                'devices':
                [{
                    'id': 'SensorTemperatureAmbient:Alice',
                    'readings':
                    [{
                        "temperature_dc": 120,
                        "time": time,
                    }]
                }],
            }
            self.assertDictEqual(self.storage.plantData(plant_name), expected_plant_data)

    def test__PonyMetricStorage_insertPlantData__storeTemperatureSensorWithoutTemperature(self):
        sensor_name = 'Alice'
        plant_name = 'SomEnergia_Alibaba'
        time = datetime.datetime.now(datetime.timezone.utc)

        with orm.db_session:
            alcolea = self.pony.db.Plant(name=plant_name,  codename='SOMSC01', description='descripción de planta')
            sensor = self.pony.db.SensorTemperatureAmbient(name=sensor_name, plant=alcolea)
            orm.flush()
            plant_data = {
                "plant": plant_name,
                "version": "1.0",
                "time": time.isoformat(), #consider using fastapi.jsonable_encoder
                "devices":
                [{
                    "id": "SensorTemperatureAmbient:Alice",
                    "readings":
                    [{
                        "temperature_dc": None,
                        "time": time,
                    }]
                }]
            }
            self.storage.insertPlantData(plant_data)
            orm.flush()

            expected_plant_data = {
                'plant': plant_name,
                'devices': [{'id': 'SensorTemperatureAmbient:Alice', 'readings':
                    [{
                        "temperature_dc": None,
                        "time": time,
                    }]
                }],
            }

            self.assertDictEqual(self.storage.plantData(plant_name), expected_plant_data)

# vim: et ts=4 sw=4


import re
import operator
import urwid

from ..helper import string_decode
from ..settings.const import settings
from ..db.attachment import Attachment
from ..errors import CompletionError

class AttachmentWidget(urwid.WidgetWrap):
    """
    one-line summary of an :class:`~alot.db.attachment.Attachment`.
    """

    def __init__(self, attachment, selectable=True):
        self._selectable = selectable
        self.attachment = attachment
        if not isinstance(attachment, Attachment):
            self.attachment = Attachment(self.attachment)
        att = settings.get_theming_attribute('thread', 'attachment')
        focus_att = settings.get_theming_attribute('thread',
                                                   'attachment_focus')
        widget = urwid.AttrMap(urwid.Text(self.attachment.__str__()),
                               att, focus_att)
        urwid.WidgetWrap.__init__(self, widget)

    def get_attachment(self):
        return self.attachment

    def selectable(self):
        return self._selectable

    def keypress(self, size, key):
        return key

class ChoiceWidget(urwid.Text):
    def __init__(self, choices, callback, cancel=None, select=None,
                 separator=' ', choices_to_return=None):
        self.choices = choices
        self.choices_to_return = choices_to_return or {}
        self.callback = callback
        self.cancel = cancel
        self.select = select
        self.separator = separator

        items = []
        for k, v in choices.items():
            if v == select and select is not None:
                items += ['[', k, ']:', v]
            else:
                items += ['(', k, '):', v]
            items += [self.separator]
        urwid.Text.__init__(self, items)

    def selectable(self):
        return True

    def keypress(self, size, key):
        if key == 'enter' and self.select is not None:
            self.callback(self.select)
        elif key == 'esc' and self.cancel is not None:
            self.callback(self.cancel)
        elif key in self.choices_to_return:
            self.callback(self.choices_to_return[key])
        elif key in self.choices:
            self.callback(self.choices[key])
        else:
            return key

class CompleteEdit(urwid.Edit):
    """
    This is a vamped-up :class:`urwid.Edit` widget that allows for
    tab-completion using :class:`~alot.completion.Completer` objects

    These widgets are meant to be used as user input prompts and hence
    react to 'return' key presses by calling a 'on_exit' callback
    that processes the current text value.

    The interpretation of some keypresses is hard-wired:
        :enter: calls 'on_exit' callback with current value
        :esc/ctrl g: calls 'on_exit' with value `None`, which can be
                     interpreted as cancellation
        :tab: calls the completer and tabs forward in the result list
        :shift tab: tabs backward in the result list
        :up/down: move in the local input history
        :ctrl f/b: moves curser one character to the right/left
        :meta f/b shift right/left: moves the cursor one word to the right/left
        :ctrl a/e: moves curser to the beginning/end of the input
        :ctrl d: deletes the character under the cursor
        :meta d: deletes everything from the cursor to the end of the next word
        :meta delete/backspace ctrl w: deletes everything from the cursor to
                                       the beginning of the current word
        :ctrl k: deletes everything from the cursor to the end of the input
        :ctrl u: deletes everything from the cursor to the beginning of the
                 input
    """

    def __init__(self, completer, on_exit,
                 on_error=None,
                 edit_text='',
                 history=None,
                 **kwargs):
        """
        :param completer: completer to use
        :type completer: alot.completion.Completer
        :param on_exit: "enter"-callback that interprets the input (str)
        :type on_exit: callable
        :param on_error: callback that handles
                         :class:`alot.errors.CompletionErrors`
        :type on_error: callback
        :param edit_text: initial text
        :type edit_text: str
        :param history: initial command history
        :type history: list or str
        """
        self.completer = completer
        self.on_exit = on_exit
        self.on_error = on_error
        self.history = list(history)  # we temporarily add stuff here
        self.historypos = None
        self.focus_in_clist = 0

        if not isinstance(edit_text, str):
            edit_text = string_decode(edit_text)
        self.start_completion_pos = len(edit_text)
        self.completions = None
        urwid.Edit.__init__(self, edit_text=edit_text, **kwargs)

    def keypress(self, size, key):
        # if we tabcomplete
        if key in ['tab', 'shift tab'] and self.completer:
            # if not already in completion mode
            if self.completions is None:
                self.completions = [(self.edit_text, self.edit_pos)]
                try:
                    self.completions += self.completer.complete(self.edit_text,
                                                                self.edit_pos)
                    self.focus_in_clist = 1
                except CompletionError as e:
                    if self.on_error is not None:
                        self.on_error(e)

            else:  # otherwise tab through results
                if key == 'tab':
                    self.focus_in_clist += 1
                else:
                    self.focus_in_clist -= 1
            if len(self.completions) > 1:
                ctext, cpos = self.completions[self.focus_in_clist %
                                               len(self.completions)]
                self.set_edit_text(ctext)
                self.set_edit_pos(cpos)
            else:
                self.completions = None
        elif key in ['up', 'down']:
            if self.history:
                if self.historypos is None:
                    self.history.append(self.edit_text)
                    self.historypos = len(self.history) - 1
                if key == 'up':
                    self.historypos = (self.historypos - 1) % len(self.history)
                else:
                    self.historypos = (self.historypos + 1) % len(self.history)
                self.set_edit_text(self.history[self.historypos])
        elif key == 'enter':
            self.on_exit(self.edit_text)
        elif key in ('ctrl g', 'esc'):
            self.on_exit(None)
        elif key == 'ctrl a':
            self.set_edit_pos(0)
        elif key == 'ctrl e':
            self.set_edit_pos(len(self.edit_text))
        elif key == 'ctrl f':
            self.set_edit_pos(min(self.edit_pos+1, len(self.edit_text)))
        elif key == 'ctrl b':
            self.set_edit_pos(max(self.edit_pos-1, 0))
        elif key == 'ctrl k':
            self.edit_text = self.edit_text[:self.edit_pos]
        elif key == 'ctrl u':
            self.edit_text = self.edit_text[self.edit_pos:]
            self.set_edit_pos(0)
        elif key == 'ctrl d':
            self.edit_text = (self.edit_text[:self.edit_pos] +
                              self.edit_text[self.edit_pos+1:])
        elif key in ('meta f', 'shift right'):
            self.move_to_next_word(forward=True)
        elif key in ('meta b', 'shift left'):
            self.move_to_next_word(forward=False)
        elif key == 'meta d':
            start_pos = self.edit_pos
            end_pos = self.move_to_next_word(forward=True)
            if end_pos is not None:
                self.edit_text = (self.edit_text[:start_pos] +
                                  self.edit_text[end_pos:])
                self.set_edit_pos(start_pos)
        elif key in ('meta delete', 'meta backspace', 'ctrl w'):
            end_pos = self.edit_pos
            start_pos = self.move_to_next_word(forward=False)
            if start_pos is not None:
                self.edit_text = (self.edit_text[:start_pos] +
                                  self.edit_text[end_pos:])
                self.set_edit_pos(start_pos)
        else:
            result = urwid.Edit.keypress(self, size, key)
            self.completions = None
            return result

    def move_to_next_word(self, forward=True):
        if forward:
            match_iterator = re.finditer(r'(\b\W+|$)', self.edit_text,
                                         flags=re.UNICODE)
            match_positions = [m.start() for m in match_iterator]
            op = operator.gt
        else:
            match_iterator = re.finditer(r'(\w+\b|^)', self.edit_text,
                                         flags=re.UNICODE)
            match_positions = reversed([m.start() for m in match_iterator])
            op = operator.lt
        for pos in match_positions:
            if op(pos, self.edit_pos):
                self.set_edit_pos(pos)
                return pos

class HeadersList(urwid.WidgetWrap):
    """ renders a pile of header values as key/value list """

    def __init__(self, headerslist, key_attr, value_attr, gaps_attr=None):
        """
        :param headerslist: list of key/value pairs to display
        :type headerslist: list of (str, str)
        :param key_attr: theming attribute to use for keys
        :type key_attr: urwid.AttrSpec
        :param value_attr: theming attribute to use for values
        :type value_attr: urwid.AttrSpec
        :param gaps_attr: theming attribute to wrap lines in
        :type gaps_attr: urwid.AttrSpec
        """
        self.headers = headerslist
        self.key_attr = key_attr
        self.value_attr = value_attr
        pile = urwid.Pile(self._build_lines(headerslist))
        if gaps_attr is None:
            gaps_attr = key_attr
        pile = urwid.AttrMap(pile, gaps_attr)
        urwid.WidgetWrap.__init__(self, pile)

    def __str__(self):
        return str(self.headers)

    def _build_lines(self, lines):
        max_key_len = 1
        headerlines = []
        # calc max length of key-string
        for key, value in lines:
            if len(key) > max_key_len:
                max_key_len = len(key)
        for key, value in lines:
            # todo : even/odd
            keyw = ('fixed', max_key_len + 1,
                    urwid.Text((self.key_attr, key)))
            valuew = urwid.Text((self.value_attr, value))
            line = urwid.Columns([keyw, valuew])
            headerlines.append(line)
        return headerlines

class TagWidget(urwid.AttrMap):
    """
    text widget that renders a tagstring.

    It looks up the string it displays in the `tags` section
    of the config as well as custom theme settings for its tag.

    Attributes that should be considered publicly readable:
        :attr tag: the notmuch tag
        :type tag: str
    """

    def __init__(self, tag, fallback_normal=None, fallback_focus=None):
        self.tag = tag
        representation = settings.get_tagstring_representation(tag,
                                                               fallback_normal,
                                                               fallback_focus)
        self.translated = representation['translated']
        self.hidden = self.translated == ''
        self.txt = urwid.Text(self.translated, wrap='clip')
        self.__hash = hash((self.translated, self.txt))
        normal_att = representation['normal']
        focus_att = representation['focussed']
        self.attmaps = {'normal': normal_att, 'focus': focus_att}
        urwid.AttrMap.__init__(self, self.txt, normal_att, focus_att)

    def set_map(self, attrstring):
        self.set_attr_map({None: self.attmaps[attrstring]})

    def width(self):
        # evil voodoo hotfix for double width chars that may
        # lead e.g. to strings with length 1 that need width 2
        return self.txt.pack()[0]

    def selectable(self):
        return True

    def keypress(self, size, key):
        return key

    def set_focussed(self):
        self.set_attr_map(self.attmaps['focus'])

    def set_unfocussed(self):
        self.set_attr_map(self.attmaps['normal'])

    def __cmp(self, other, comparitor):
        """Shared comparison method."""
        if not isinstance(other, TagWidget):
            return NotImplemented

        self_len = len(self.translated)
        oth_len = len(other.translated)

        if (self_len == 1) is not (oth_len == 1):
            return comparitor(self_len, oth_len)
        return comparitor(self.translated.lower(), other.translated.lower())

    def __lt__(self, other):
        """Groups tags of 1 character first, then alphabetically.

        This groups tags unicode characters at the begnining.
        """
        return self.__cmp(other, operator.lt)

    def __gt__(self, other):
        return self.__cmp(other, operator.gt)

    def __ge__(self, other):
        return self.__cmp(other, operator.ge)

    def __le__(self, other):
        return self.__cmp(other, operator.le)

    def __eq__(self, other):
        if not isinstance(other, TagWidget):
            return NotImplemented
        if len(self.translated) != len(other.translated):
            return False
        return self.translated.lower() == other.translated.lower()

    def __ne__(self, other):
        if not isinstance(other, TagWidget):
            return NotImplemented
        return self.translated.lower() != other.translated.lower()

    def __hash__(self):
        return self.__hash


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri May  4 19:07:34 2018

@author: abinaya
"""

import numpy as np
import matplotlib.pyplot as plt
import random

class Traveling_Salesmen_problem():

    def __init__(self, no_cities, T, cooling_function, max_iter):
        self.no_cities = no_cities
        self.T = T
        self.cooling_function = cooling_function
        self.max_iter = max_iter

    def generate_cities(self):
        return [random.sample(range(0,1000), 2) for x in range(self.no_cities)]

    def generate_initial_random_tour(self):
        return random.sample(range(self.no_cities),self.no_cities)

    def travel_cost(self, tour):
        cost = 0
        for i in range(0, self.no_cities-1):
            from_city_index = tour[i]
            to_city_index = tour[i+1]
            from_city = self.cities[from_city_index]
            to_city = self.cities[to_city_index]
            cost += np.linalg.norm(np.array(from_city) - np.array(to_city))
        return cost

    def return_iterTemp(self,n):
        if self.cooling_function == "Polynomial":
            return self.T * pow(n+1, -0.751)
        elif self.cooling_function == "Logarithmic":
            return self.T / np.log(n+1)
        elif self.cooling_function == "Exponential":
            return self.T / np.exp(n+1)
        elif self.cooling_function == "Other":
            return self.T * 0.99

    def generate_acceptance_probability(self, tour_cost, swapped_tour_cost, iterT):
        return np.exp((tour_cost - swapped_tour_cost) / iterT)

    def solve_problem(self):
        self.cities = self.generate_cities()
        tour = self.generate_initial_random_tour()
        initial_tour = tour[:]

        cost_to_plot = []

        for i in range(0,self.max_iter):
            swap_positions =  random.sample(range(self.no_cities),2)
            swapped_tour = tour[:]
            swapped_tour[swap_positions[0]] = tour[swap_positions[1]]
            swapped_tour[swap_positions[1]] = tour[swap_positions[0]]

            tour_cost = self.travel_cost(tour)
            swapped_tour_cost = self.travel_cost(swapped_tour)

            iterT = self.return_iterTemp(i)

            acceptance_probability = self.generate_acceptance_probability(tour_cost, swapped_tour_cost, iterT)
            if(np.random.uniform() <= acceptance_probability):
                tour = swapped_tour[:]
                cost_to_plot.append(swapped_tour_cost)

        return initial_tour,tour, cost_to_plot

no_cities = 400
tr_sa_pr = Traveling_Salesmen_problem(no_cities, 1, "Other", 5000)
initial_tour,tour, cost_to_plot = tr_sa_pr.solve_problem()
oldx = zip(*[tr_sa_pr.cities[initial_tour[i % no_cities]] for i in range(no_cities+1)])[0]
oldy = zip(*[tr_sa_pr.cities[initial_tour[i % no_cities]] for i in range(no_cities+1)])[1]
newx = zip(*[tr_sa_pr.cities[tour[i % no_cities]] for i in range(no_cities+1)])[0]
newy = zip(*[tr_sa_pr.cities[tour[i % no_cities]] for i in range(no_cities+1)])[1]

plt.figure()
plt.plot(oldx, oldy, marker='o', color='r')
plt.grid('on')
plt.title('Initial Random Tour: Number of Cities = '+str(no_cities))

plt.figure()
plt.plot(newx, newy, color='g', marker='*')
plt.grid('on')
plt.title('Optmized Tour: Number of Cities = '+str(no_cities))

plt.figure()
plt.plot(cost_to_plot)
plt.xlabel('Iteration')
plt.ylabel('Cost')
plt.title('Number of Cities = '+str(no_cities))


# for mesuring time
import time
# for plotting
import matplotlib.pyplot as plt

def fibonacci(n):
    fib = [0, 1]
    for i in range(2, n+1):
        fib.append(fib[i-1] + fib[i-2])
    return fib[n]

xdp=[]
for i in range(350):
    xdp.append(i+1)

ydp=[]

for i in range(350):
    start_time_dp = time.time()
    fibonacci(i)
    end_time_dp = time.time()
    ydp.append(end_time_dp - start_time_dp)

plt.plot(xdp, ydp, color='b', label='D.P')
plt.show()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('add', views.newRecipe, name='addNew'),
    path('get', views.getAllRecipe, name='getRecipe'),
    path('search', views.search, name='searchRecipe'),
    path('getId/<int:id>', views.getRecipeById, name='searchRecipeById'),
    path('getIngredients/<int:id>', views.getIngredientsByRecipe, name='getIngredients'),
    path('getSteps/<int:id>', views.getStepsByRecipe, name='getSteps'),
    path('deleteRecipe', views.deleteRecipe, name='deleteRecipe')
]


from httpx import post, put
from pyogg import VorbisFile
from mutagen import File

class lol:
    def __init__(gay, token: str) -> None:
        gay.Token = token

    def attachment(gay, channel: int, filePath: str) -> dict:
        try:
            z = post(
                f'https://discord.com/api/v9/channels/{channel}/attachments',
                headers={'Authorization': gay.Token},
                json={
                      "files": [
                        {
                          "filename": "voice-message.ogg",
                          "file_size": VorbisFile(filePath).buffer_length,
                          "id": "6"
                        }
                      ]
                    }
            )
            if z.status_code == 200:
                return z.json()['attachments'][0]
            else:
                exit(z.text)
        except Exception as err:
            exit(err)

    def uploadToCloud(gay, upload_url: str, filePath: str) -> bool:
        try:
            with open(filePath, 'rb') as fobj:
                x = put(
                    upload_url,
                    data=fobj.read(),
                    headers={'Content-Type': 'audio/ogg'}
                    )
                if x.status_code == 200:
                    return True
                else:
                    exit(x.text)
        except Exception as err:
            exit(err)

    def uploadToDiscord(gay, channel: int, filePath: str) -> tuple:
        try:
            z = a.attachment(channel, filePath)
            n = gay.uploadToCloud(z['upload_url'], filePath)
            if n:
                y = post(
                    f'https://discord.com/api/v9/channels/{channel}/messages',
                    headers={
                        'Authorization': gay.Token,
                        'x-super-properties': 'eyJvcyI6IkFuZHJvaWQiLCJicm93c2VyIjoiRGlzY29yZCBBbmRyb2lkIiwiZGV2aWNlIjoiZHJlYW0ybHRla3MiLCJzeXN0ZW1fbG9jYWxlIjoiZnItRlIiLCJjbGllbnRfdmVyc2lvbiI6IjE3My4yMyAtIHJuIiwicmVsZWFzZV9jaGFubmVsIjoiZ29vZ2xlUmVsZWFzZSIsImRldmljZV92ZW5kb3JfaWQiOiI5M2RjMDRiOS04ODhkLTQ1NjMtYmI0OC1iMzA4NTNhYjNjOWMiLCJicm93c2VyX3VzZXJfYWdlbnQiOiIiLCJicm93c2VyX3ZlcnNpb24iOiIiLCJvc192ZXJzaW9uIjoiMjUiLCJjbGllbnRfYnVpbGRfbnVtYmVyIjoxNzMwMjMsImNsaWVudF9ldmVudF9zb3VyY2UiOm51bGwsImRlc2lnbl9pZCI6MH0=',
                        'user-agent': 'Discord-Android/173023;RNA',
                        'content-type': 'application/json'
                        },
                    json={
                          "content": "",
                          "channel_id": f"{channel}",
                          "type": 0,
                          "flags": 8192,
                          "attachments": [
                            {
                              "id": z['id'],
                              "filename": z['upload_filename'].split('/')[1],
                              "uploaded_filename": z['upload_filename'],
                              "duration_secs": File(filePath).info.length,
                              "waveform": "KBkKDRAKGBIWcnZ+o1xub6GUX6CITm0b"
                            }
                          ]
                        }
                )
                if y.status_code == 200:
                    return True, 'Successfully uploaded audio file !'
                else:
                    return False, y.json()
        except Exception as err:
            exit(err)

a = lol(
    input('Token >')
)

b = a.uploadToDiscord(
    int(input('Channel ID >')),
    input('.ogg audio file path > ') # './audio.ogg'
    )
print(b[1])


from flask import abort

from ...services.orga_presence import service as orga_presence_service
from ...services.party import service as party_service
from ...util.framework.blueprint import create_blueprint
from ...util.templating import templated

from ..authorization.decorators import permission_required
from ..authorization.registry import permission_registry

from .authorization import OrgaPresencePermission

blueprint = create_blueprint('orga_presence', __name__)

permission_registry.register_enum(OrgaPresencePermission)

@blueprint.route('/<party_id>')
@permission_required(OrgaPresencePermission.list)
@templated
def view(party_id):
    """List orga presence and task time slots for that party."""
    party = party_service.find_party(party_id)

    if party is None:
        abort(404)

    presences = orga_presence_service.get_presences(party.id)
    tasks = orga_presence_service.get_tasks(party.id)

    hour_ranges = list(orga_presence_service.get_hour_ranges(party, tasks))

    days_and_hour_totals = list(orga_presence_service \
        .get_days_and_hour_totals(hour_ranges))

    return {
        'party': party,
        'days_and_hour_totals': days_and_hour_totals,
        'hour_ranges': hour_ranges,
        'presences': presences,
        'tasks': tasks,
    }


import matplotlib.pyplot as plt
import numpy as np

import os
import json

dir = '/home/buens/Desktop/MA_Evaluation/e4e_encoder_original/webcam_images/results/corrected_original_without_background'
lpips = []
ssim = []
fsim = []

for root, dirs, files in sorted(os.walk(dir)):
    for name in files:
        if name == 'metrics.json':
            id = sorted(dirs)[0]
            with open(os.path.join(dir, id, name)) as f:
                data = json.load(f)
                lpips.append(data['LPIPS'])
                ssim.append(data['SSIM'])
                fsim.append(data['FSIM'])

x = np.arange(1, 101, 1)

# Plotting both the curves simultaneously
plt.plot(x, lpips, color='r', label='LPIPS')
plt.plot(x, ssim, color='g', label='SSIM')
plt.plot(x, fsim, color='b', label='FSIM')

# Naming the x-axis, y-axis and the whole graph
plt.xlabel("Index")
plt.ylabel("Wert der Metrik")
plt.title("Beleuchtungskorrektur auf Webcam-Aufnahmen")

# Adding legend, which helps us recognize the curve according to it's color
plt.legend()

# To load the display window
plt.show()

lpips_arr = np.array(lpips)
outliers = np.where(lpips_arr > 0.10)
print(outliers)


import os

import cv2
import tensorflow as tf

from args import opt

def _bytes_feature(value):
    """string / byte 型から byte_list を返す"""
    if isinstance(value, str):
        value = value.encode()

    if isinstance(value, type(tf.constant(0))):
        value = value.numpy()  # BytesList won't unpack a string from an EagerTensor.
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _float_feature(value):
    """float / double 型から float_list を返す"""
    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

def _int64_feature(value):
    """bool / enum / int / uint 型から Int64_list を返す"""
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def image_encode(path):
    image = cv2.imread(path)
    if path.endswith('.jpg'):
        encode_image = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tobytes()
    elif path.endswith('.png'):
        encode_image = cv2.imencode('.png', image)[1].tobytes()
    return _bytes_feature(encode_image)

def make_serialize(path, label, filename, classname):
    image = cv2.imread(path)
    height, width = image.shape[:2]
    feature = {
        "image": image_encode(path),
        "width": _int64_feature(int(width)),
        "height": _int64_feature(int(height)),
        "label": _int64_feature(label),
        "filename": _bytes_feature(filename),
        "classname": _bytes_feature(classname),
    }
    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
    return example_proto.SerializeToString()

def main(args):
    print(args)
    for phase in ("train", "val"):
        tfrecord_path = f"{args.path2db}/{phase}.tfrecord"
        with tf.io.TFRecordWriter(tfrecord_path) as writer:
            classnames = os.listdir(os.path.join(args.path2db, phase))
            classnames.sort()
            for label, classname in enumerate(classnames):
                filenames = os.listdir(os.path.join(args.path2db, phase, classname))
                for i, filename in enumerate(filenames):
                    if i == 0:
                        print("{:03d}: {}".format(label, filename))
                    image_path = os.path.join(args.path2db, phase, classname, filename)
                    example = make_serialize(image_path, label, filename, classname)
                    writer.write(example)

if __name__ == '__main__':
    args = opt()
    main(args)


import argparse
import os
import json
from colorama import Fore, Style, init
import traceback
from pytube import YouTube
from tqdm import tqdm
import datetime
import cv2
import argparse

# Setting argparse tools
parse = argparse.ArgumentParser(description='Download Youtube Video and cut to frame.')
parse.add_argument('--url', type=str, required=True, help='Paste Youtube URL')

# The center of path
main_path = os.path.dirname(os.path.abspath(__file__))

# Defualt setting
storage_folder_name = 'videos'
frame_folder_name = 'frames'
frame_path = os.path.join(main_path, frame_folder_name)
storage_path = os.path.join(main_path, storage_folder_name)
storage_files_by_videos = None
storage_folders_by_frames = None

youtube_record = 'youtube_record.json'
record_data = None

# setting colorama
init(autoreset=True)

def init_factory():
    global storage_files_by_videos
    global record_data
    global storage_folders_by_frames

    # Check videos storage path
    if os.path.isdir(storage_path):
        print('|- \"{}\" folder is exist !'.format(storage_folder_name))
        contents = os.listdir(storage_path)

        storage_files_by_videos = list(filter(lambda f: os.path.isfile(os.path.join(storage_path, f)), contents))

        print(Fore.CYAN+'|- {} video files'.format(len(storage_files_by_videos)))

    else:
        try:
            print(Fore.YELLOW+'|- \"{}\" folder did not exists. Make it right now....'.format(storage_folder_name))
            os.mkdir(storage_path)

            if os.path.isdir(storage_path):
                print(Fore.GREEN+'|- Done !')
        except:
            print(traceback.format_exc())

    # Check videos per frame storage path
    if os.path.isdir(frame_path):
        print('|- \"{}\" folder is exist !'.format(frame_folder_name))
        contents = os.listdir(frame_path)
        storage_folders_by_frames = list(filter(lambda f: os.path.isdir(os.path.join(frame_path, f)), contents))

        print(Fore.CYAN+'|- {} frames folders'.format(len(storage_folders_by_frames)))

    else:
        try:
            print(Fore.YELLOW+'|- \"{}\" folder did not exists. Make it right now....'.format(frame_folder_name))
            os.mkdir(frame_path)

            if os.path.isdir(frame_path):
                print(Fore.GREEN, '|- Done !')
        except:
            print(traceback.format_exc())

    print("-"*100)
    # Checking record file if exist
    if os.path.isfile(os.path.join(main_path, youtube_record)):
        print('|- Loading record data \"{}\"....'.format(youtube_record))
        with open(youtube_record, 'r') as jsFile:
            record_data = json.load(jsFile)

def file_name_factory():
    global record_data
    print('|- via naming factory')

    start_number = 0

    if record_data:
        sort_list = sorted(record_data.keys())
        return str(int(sort_list[-1])+1)
    else:
        return str(start_number)

def write_record_data(yt_title, yt_url, file_name):
    global record_data

    data = {}
    data.update({file_name : {'title': yt_title,
                              'url': yt_url}})

    print('|- Saving record....')
    if record_data:
        record_data.update(data)
        with open(youtube_record, 'w') as jfile:
            json.dump(record_data, jfile)
    else:
        record_data = data
        with open(youtube_record, 'w') as jfile:
            json.dump(record_data, jfile)

    print(Fore.GREEN,'|- Done!')

def download_youtube(yt_url, target_folder, file_name):
    try:
        print('|- Connecting Youtube ... ')
        yt = YouTube(yt_url)
        video = yt.streams.first()
        file_size = video.filesize
        yt_title = video.title

        print('-'*100+'|')
        print('|-Title : {} '.format(yt_title))
        print('-'*100+'|')
        print('|-URL : {} '.format(yt_url))
        print('-'*100+'|')
        print('|-File Size : {} KB'.format(file_size//(1024*1024)))
        print('-'*100+'|')
        print('|-Storage Path : {}'.format(target_folder))
        print('-'*100+'|')
        print('|-File Name : {}.mp4'.format(file_name))
        print('-'*100+'|')

        for _ in tqdm(yt.streams):
            video.download(target_folder, file_name)

        # save metadata
        write_record_data(yt_title, yt_url, file_name)

        now = datetime.datetime.now()
        print('-'*100+'|')
        print(Fore.GREEN+'|- {}.{}.{} {}:{} download successful!! '.format(now.year, now.month, now.day, now.hour, now.minute))

    except:
        print(traceback.format_exc())

def cut_video2frame(src_path, video_name):
    sec = 0
    frameRate = 1

    #FIXME : Check format
    video_format = '.mp4'
    frame_folder_name = 'video{}'.format(video_name)

    frames_folder = os.path.join(frame_path, frame_folder_name)

    if os.path.isdir(frames_folder):
        raise Exception('The frame by video folder already exist. Please check the target path({}) or the folder name({})'.format(frames_folder, frame_folder_name))
    else:
        print("|- Create \"{}\" folder ...".format(frame_folder_name))
        os.mkdir(frames_folder)

    def getFrame(sec):
        image_name = 'video{}_{}.jpg'.format(video_name, str(sec))
        vidcap.set(cv2.CAP_PROP_POS_MSEC, sec*1000)
        hasFrames, image = vidcap.read()
        if hasFrames:
            cv2.imwrite(os.path.join(frames_folder, image_name), image)
        return hasFrames

    print(Fore.GREEN+"|- Capture video {}".format(video_name+video_format))
    vidcap = cv2.VideoCapture(os.path.join(src_path, video_name+video_format))
    success = getFrame(sec)
    while success:
        sec = sec + frameRate
        success = getFrame(sec)

    print("|- Cutting video successful!!")

def main():
    args = parse.parse_args()
    yt_url = args.url

    init_factory()
    file_name = file_name_factory()
    download_youtube(yt_url, storage_path, file_name)
    cut_video2frame(storage_path, file_name)

if __name__ == '__main__':
    main()


from pyspark.sql import SparkSession
from pyspark.sql import functions as func
from pyspark.sql.types import StructField, StructType, IntegerType, StringType

spark = (
    SparkSession
    .builder
    .appName('MarvelSuperHeroe')
    .getOrCreate()
)

schema = StructType(
    [
        StructField('id', IntegerType(), True),
        StructField('name', StringType(), True),
    ]
)

names = (
    spark
    .read
    .schema(schema)
    .option('sep', ' ')
    .csv('./marvel-super-hero/data/Marvel+Names')
)

lines = spark.read.text('./marvel-super-hero/data/Marvel+Graph')

connections = (
    lines
    .withColumn('id', func.split(func.col('value'), ' ')[0])
    .withColumn('connections', func.size(func.split(func.col('value'), ' ')) - 1)
    .groupBy('id')
    .agg(func.sum('connections').alias('connections'))
)

hero_only_one_conn = (
    connections
    .filter(func.col('connections') == 1)
)
hero_only_one_conn_name = (
    hero_only_one_conn
    .join(names, 'id')
    .select('name', 'connections')
)
hero_only_one_conn_name.show(hero_only_one_conn_name.count())

min_connection = (
    connections
    .agg(func.min(func.col('connections')))
    .first()
)

print(f'the minimun number of connections is {min_connection[0]} connections')

spark.stop()


from typing import TYPE_CHECKING, Any, Dict, Type, TypeVar, Union

import attr

from ..types import UNSET, Unset

if TYPE_CHECKING:
    from ..models.codat_data_contracts_datasets_account_ref import CodatDataContractsDatasetsAccountRef
    from ..models.codat_data_contracts_datasets_tracking import CodatDataContractsDatasetsTracking

T = TypeVar("T", bound="CodatDataContractsDatasetsJournalLine")

@attr.s(auto_attribs=True)
class CodatDataContractsDatasetsJournalLine:
    """
    Attributes:
        net_amount (float):
        description (Union[Unset, None, str]):
        currency (Union[Unset, None, str]):
        account_ref (Union[Unset, CodatDataContractsDatasetsAccountRef]):
        tracking (Union[Unset, CodatDataContractsDatasetsTracking]):
    """

    net_amount: float
    description: Union[Unset, None, str] = UNSET
    currency: Union[Unset, None, str] = UNSET
    account_ref: Union[Unset, "CodatDataContractsDatasetsAccountRef"] = UNSET
    tracking: Union[Unset, "CodatDataContractsDatasetsTracking"] = UNSET

    def to_dict(self) -> Dict[str, Any]:
        net_amount = self.net_amount
        description = self.description
        currency = self.currency
        account_ref: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.account_ref, Unset):
            account_ref = self.account_ref.to_dict()

        tracking: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.tracking, Unset):
            tracking = self.tracking.to_dict()

        field_dict: Dict[str, Any] = {}
        field_dict.update(
            {
                "netAmount": net_amount,
            }
        )
        if description is not UNSET:
            field_dict["description"] = description
        if currency is not UNSET:
            field_dict["currency"] = currency
        if account_ref is not UNSET:
            field_dict["accountRef"] = account_ref
        if tracking is not UNSET:
            field_dict["tracking"] = tracking

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        from ..models.codat_data_contracts_datasets_account_ref import CodatDataContractsDatasetsAccountRef
        from ..models.codat_data_contracts_datasets_tracking import CodatDataContractsDatasetsTracking

        d = src_dict.copy()
        net_amount = d.pop("netAmount")

        description = d.pop("description", UNSET)

        currency = d.pop("currency", UNSET)

        _account_ref = d.pop("accountRef", UNSET)
        account_ref: Union[Unset, CodatDataContractsDatasetsAccountRef]
        if isinstance(_account_ref, Unset):
            account_ref = UNSET
        else:
            account_ref = CodatDataContractsDatasetsAccountRef.from_dict(_account_ref)

        _tracking = d.pop("tracking", UNSET)
        tracking: Union[Unset, CodatDataContractsDatasetsTracking]
        if isinstance(_tracking, Unset):
            tracking = UNSET
        else:
            tracking = CodatDataContractsDatasetsTracking.from_dict(_tracking)

        codat_data_contracts_datasets_journal_line = cls(
            net_amount=net_amount,
            description=description,
            currency=currency,
            account_ref=account_ref,
            tracking=tracking,
        )

        return codat_data_contracts_datasets_journal_line


import discord
from discord.ext import commands
import aiohttp
import datetime
from PIL import Image, ImageDraw, ImageFont

from functools import partial

from io import BytesIO

from discord_components import (
    Button,
    ButtonStyle,
)

class ImageCog(commands.Cog):
    def __init__(self, bot: commands.Bot):
        self.bot = bot
        self.session = aiohttp.ClientSession(loop=bot.loop)

    # getting the album cover
    async def get_image(self, url) -> BytesIO:
        async with self.session.get(url) as response:
            image_bytes = await response.read()

        return image_bytes

    @staticmethod
    def imgen(delta1, delta2, title: str, artist: str, album: str, image_bytes: bytes) -> BytesIO:

        myFont3 = ImageFont.truetype("assets/Roboto-Regular.ttf", 19)

        if title.isascii==False or album.isascii==False or artist.isascii==False:
            myFont2 = ImageFont.truetype("assets/code2000.ttf", 28)
            myFont1 = ImageFont.truetype("assets/code2000.ttf", 41)
        else:
            myFont1 = ImageFont.truetype("assets/Roboto-Regular.ttf", 41)
            myFont2 = ImageFont.truetype("assets/Roboto-Regular.ttf", 28)

        m1 = divmod(delta1.seconds, 60)
        m2 = divmod(delta2.seconds, 60)

        # we must use BytesIO to load the image here as PIL expects a stream instead of
        # just raw bytes.
        with Image.open(BytesIO(image_bytes)) as im:
            with Image.open("assets/template.jpg") as background:
                img2 = im.resize((210, 210))
                background.paste(img2, (36, 36))
                draw = ImageDraw.Draw(background)

                # title
                draw.text((303, 36), "{}".format(
                    title[:23]), font=myFont1, fill="#befe00")

                #artist and album
                draw.text((303, 99), "{}".format(artist[:33]),
                          font=myFont2, fill="#FFFFFF")
                draw.text((303, 147), "{}".format(
                    album[:33]), font=myFont2, fill="#FFFFFF")

                # timestamps

                draw.text((303, 229), "{}:{}".format(str(m1[0]).zfill(
                    2), str(m1[1]).zfill(2)), font=myFont3, fill="#FFFFFF")
                draw.text((855, 229), "{}:{}".format(str(m2[0]).zfill(
                    2), str(m2[1]).zfill(2)), font=myFont3, fill="#FFFFFF")

                # total progress bar
                draw.rounded_rectangle(
                    (303, 211, 897, 221), radius=7, fill='#FFFFFF', outline=None, width=10
                )

                # current progress
                # some math
                xbar = int((delta1.seconds/delta2.seconds)*594)
                draw.rounded_rectangle(
                    (303, 211, 303+xbar, 221), radius=7, fill='#befe00', outline=None, width=10
                )
                # prepare the stream to save this image into
                final_buffer = BytesIO()

                # save into the stream, using png format.
                background.save(final_buffer, "png")

        # seek back to the start of the stream
        final_buffer.seek(0)

        return final_buffer

    @commands.command(name='spotify', aliases=['spot'])
    async def spotify(self, ctx, user: discord.Member = None):

        user = user or ctx.author
        if user.activities:
            for activity in user.activities:

                if isinstance(activity, discord.Spotify):

                    delta1 = datetime.datetime.utcnow() - activity.start
                    delta2 = activity.end - activity.start
                    title = activity.title
                    artist = activity.artist
                    album = activity.album
                    url = activity.album_cover_url
                    songid = activity.track_id
                    spoturl = 'https://open.spotify.com/track/{}'.format(
                        songid)
                    image_bytes = await self.get_image(url)

                    # create partial function so we don't have to stack the args in run_in_executor
                    fn = partial(self.imgen, delta1, delta2,
                                 title, artist, album, image_bytes)

                    final_buffer = await self.bot.loop.run_in_executor(None, fn)
                    file = discord.File(
                        filename="spotify.png", fp=final_buffer)

                    await ctx.reply(file=file,
                        components=[Button(
                            style=ButtonStyle.URL,
                            label="Play On Spotify",
                            url=spoturl)])
                    break
            else:

                spotify = discord.Embed(
                    title="Uh Oh!",
                description="Looks like {} isn't listening to **Spotify <:spotify:887426366818115585>** right now. \nPerhaps [give it a shot?](https://www.spotify.com/)".format(user),
                colour=discord.Colour.gold(),
            ).set_footer(
                text="Q-Loggr Bot"
            )

                await ctx.reply(embed=spotify)

        else:

            spotify = discord.Embed(
                title="Uh Oh!",
                description="Looks like {} isn't listening to **Spotify <:spotify:887426366818115585>** right now. \nPerhaps [give it a shot?](https://www.spotify.com/)".format(user),
                colour=discord.Colour.gold(),
            ).set_footer(
                text="Q-Loggr Bot"
            )

            await ctx.reply(embed=spotify)

# setup function so this can be loaded as an extension
def setup(bot: commands.Bot):
    bot.add_cog(ImageCog(bot))