#!/usr/bin/env python

import argparse
import csv
import datetime
import functools
import os
import sys
from collections import defaultdict
from multiprocessing import Pool, Pipe
from multiprocessing import cpu_count
from multiprocessing.context import TimeoutError

# from tqdm import tqdm
import numpy as np
import pysam
from interlap import InterLap
from rich import print
from rich.progress import (
    BarColumn,
    TimeRemainingColumn,
    Progress,
)

progress = Progress(
    "[progress.description]{task.description}",
    BarColumn(),
    "[progress.percentage]{task.percentage:>3.0f}%",
    "ETA:",
    TimeRemainingColumn()
)

progressupdates_recv, progressupdates_send = Pipe()

defaultchroms = list(map(str, range(1, 720))) + ['X', 'Y', '2L', '2R', '3L', '3R', 'Z', 'W', 'I', 'II', 'III', 'IV',
                                                 'V', 'VI',
                                                 'VII', 'VIII', 'IX', 'X', 'XI', 'XII', 'XIII', 'XIV', 'XV', 'XVI']

def exon2intron(listOfExons, min_intron_length):
    """
    exons must be zero-based, start excluded
    :param listOfExons:
    :param min_intron_length:
    :return: are zero-based, start excluded
    """

    minus_strand = listOfExons[0][3] == "-"
    listOfExons.sort(key=lambda exon: exon[1])
    numberExons = len(listOfExons)
    intron = []
    if numberExons > 1:
        strandrange = range(1, numberExons) if not minus_strand else reversed(range(1, numberExons))
        id = 0
        for i in strandrange:
            exon0 = listOfExons[i - 1]
            exon1 = listOfExons[i]
            id += 1
            thisintron = (exon0[0], exon0[2], exon1[1], exon1[3], id)
            if (thisintron[2] - thisintron[1] - 1 >= min_intron_length):
                intron.append(thisintron)
    return intron

def get_sjCoordinates(GTFfile, min_intron_length=30, keep_masked_introns=False, filterSJs=False, intron_file=None,
                      exon_file=None, chroms=defaultchroms):
    # gets transcript per gene
    iso2gene = {}
    # lists exons per transcript (tcx)
    iso2exons = defaultdict(list)
    exon_by_chr = defaultdict(dict)

    for line in GTFfile:
        splitGTFline = line.split('\t')
        chrom = splitGTFline[0]
        if len(chrom) > 3 and chrom[:3] == "chr":
            chrom = chrom[3:]

        if chrom != '#' and chrom in chroms:
            if splitGTFline[2] == 'exon':
                gene = line.split('gene_id')[1].split('"')[1]
                tcx = line.split('transcript_id')[1].split('"')[1]
                iso2gene[tcx] = gene
                start1 = int(splitGTFline[3])
                end1 = int(splitGTFline[4])
                strand = splitGTFline[6]
                newExon = (chrom, start1 - 1, end1, strand)
                iso2exons[tcx].append(newExon)
                if not keep_masked_introns or filterSJs:
                    if chrom not in exon_by_chr:
                        exon_by_chr[chrom] = defaultdict(InterLap)
                    exon_by_chr[chrom][strand].add((start1, end1, {'g': gene}))
                if exon_file is not None and not exon_file.closed:
                    exon_file.write("\t".join([chrom, str(start1 - 1), splitGTFline[4], gene + "_" + tcx, "0",
                                               strand]) + "\n")  # bed structure

    GTFfile.close()

    # calculates intron location based on exons for each tcx
    introns = defaultdict(list)
    for tcx, listOfExons in iso2exons.items():
        # gets list of introns (no UTR)
        listOfIntrons = exon2intron(listOfExons, min_intron_length)
        for intron in listOfIntrons:
            if keep_masked_introns:
                introns[tcx].append(intron)
            else:
                # Only if this intron interval is not in the list of exons intervals for this chr-strand pair: add it
                if not (intron[1] + 1, intron[2]) in exon_by_chr[intron[0]][intron[3]]:
                    introns[tcx].append(intron)

    sjIntervals = defaultdict(dict)
    intronIDtoTcxGeneNr = defaultdict()
    intronID = 0
    for tcx, listOfIntrons in introns.items():
        for intron in listOfIntrons:
            gene = iso2gene[tcx]

            if intron_file is not None and not intron_file.closed:
                joined = "\t".join([intron[0], str(intron[1]), str(intron[2]), gene + "_" + tcx, "0", intron[3]]) + "\n"
                intron_file.write(joined)

            if intron[3] == '+':
                if filterSJs:
                    overlap5 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[1], intron[1] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap5 = True
                            break

                    overlap3 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[2], intron[2] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap3 = True
                            break

                    if not overlap3 and not overlap5:
                        if intron[0] not in sjIntervals:
                            sjIntervals[intron[0]] = defaultdict(InterLap)
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[1], intron[1] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[2], intron[2] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))

                        intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                        intronID += 1
                else:
                    if intron[0] not in sjIntervals:
                        sjIntervals[intron[0]] = defaultdict(InterLap)
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[1], intron[1] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[2], intron[2] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))
                    intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                    intronID += 1
            else:
                if filterSJs:
                    overlap5 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[2], intron[2] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap5 = True
                            break

                    overlap3 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[1], intron[1] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap3 = True
                            break

                    if not overlap3 and not overlap5:
                        if intron[0] not in sjIntervals:
                            sjIntervals[intron[0]] = defaultdict(InterLap)
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[2], intron[2] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[1], intron[1] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))

                        intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                        intronID += 1
                else:
                    if intron[0] not in sjIntervals:
                        sjIntervals[intron[0]] = defaultdict(InterLap)
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[2], intron[2] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[1], intron[1] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))
                    intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                    intronID += 1

    return sjIntervals, intronID, intronIDtoTcxGeneNr

def processBamEntry(chromSJ, bamfile, verbose, read_qual, chr_prefix):
    with open(bamfile, 'r') as bamf:

        chrom, sjintervals = chromSJ
        contig = chrom
        if chr_prefix:
            contig = "chr" + chrom
        if verbose:
            print("Processing chromosome " + chrom + " ...")

        bam = pysam.AlignmentFile(bamf, 'rb')
        for read in bam.fetch(contig=contig):
            if read.mapping_quality > read_qual:
                strand = "+"
                if read.is_reverse:
                    strand = "-"

                # zero-based left-most coordinate. In the Interval tree they are stored one-based.
                readStart = read.reference_start + 1
                # readStart = read.reference_start
                # pysam reports the end non-inclusive
                readEnd = read.reference_end - 1

                countIdx = 6
                key = 'us'  # nonsplit
                for cigaroperation, length in read.cigartuples:
                    if cigaroperation == 3:  # if any M present
                        countIdx = 7
                        key = 's'  # split
                        break

                if countIdx == 6:  # nonsplit reads
                    for mapped_sj_interval in sjintervals[strand].find((readStart, readEnd)):
                        if (readEnd >= mapped_sj_interval[1] and
                                readStart <= mapped_sj_interval[0]):
                            mapped_sj_interval[2][key] += 1
                else:  # split reads

                    for blockstart, blockend in read.get_blocks():
                        for mapped_sj_interval in sjintervals[strand].find((blockstart, blockend)):
                            if ((blockstart + 1) == mapped_sj_interval[1]) or (blockend == mapped_sj_interval[0]):
                                mapped_sj_interval[2][key] += 1

        if verbose:
            print("Processing chromosome " + chrom + " done.")

        bam.close()
        return chrom, sjintervals

## For use if we support multiple Bam files.
def resetSJDict(sjintervals):
    for chrom, strands in sjintervals.items():
        for strand, intervals in strands.items():
            for interval in intervals:
                interval[2]['us'] = 0
                interval[2]['s'] = 0

######## -------------------------  IER ----------- #############

def calcScore(introncov, leftexoncov, rightexoncov):
    exonsumm = (leftexoncov + rightexoncov)
    if exonsumm == 0:
        return 1.
    else:
        return 1. - min(1, (introncov / (0.5 * exonsumm)))

def sameChrom(tcxExons, chromNr):
    tcx, listExons = tcxExons
    return chromNr == listExons[0][0]

def splitIntoChroms(tcxExons, chroms):
    res = [] * len(chroms)
    i = 0
    for chrom in chroms:
        for tcxexon in tcxExons:
            if sameChrom(tcxexon, chrom):
                res[i].append(tcxexon)
        i += 1
    return res

def sjOverlaps(intron, exon_by_chr, gene):
    for overlappingExon in exon_by_chr[(intron[0], intron[3])].find((intron[1], intron[1] + 1)):
        if overlappingExon[2]['g'] != gene:
            return True

    for overlappingExon in exon_by_chr[(intron[0], intron[3])].find((intron[2], intron[2] + 1)):
        if overlappingExon[2]['g'] != gene:
            return True

def getCoverage(bam, chrom, start, end, minus_strand, min_quality):
    getCoverage.cache = getattr(getCoverage, 'cache', tuple((defaultdict(), defaultdict())))
    if (start, end) in getCoverage.cache[minus_strand]:
        return getCoverage.cache[minus_strand][(start, end)]

    if minus_strand:
        filterRead = lambda r: r.is_reverse and r.mapq > min_quality
    else:
        filterRead = lambda r: not r.is_reverse and r.mapq > min_quality

    ## sum up coverages per nucleotide
    cov_array = functools.reduce(np.add,
                                 bam.count_coverage(contig='chr' + str(chrom), start=start,
                                                    stop=end, quality_threshold=0,
                                                    read_callback=filterRead))
    ## return median coverage for this genomic feature/interval
    result = np.median(cov_array)
    getCoverage.cache[minus_strand][(start, end)] = result
    return result

def getSJCoverage(bam, chrom, start, end, minus_strand, min_quality, debug):
    """
    Gets the coverage for split and non-split reads in the bam file

    @type  start: int
    @param start: 0-based inclusive start position of SJ
    @type  end: int
    @param end: 0-based exclusive start position of SJ
    @rtype:   tuple(int,int)
    @return:  number split, number non-split reads
    """
    getSJCoverage.cache = getattr(getSJCoverage, 'cache', tuple((defaultdict(), defaultdict())))
    if (start, end) in getSJCoverage.cache[minus_strand]:
        return getSJCoverage.cache[minus_strand][(start, end)]

    def filterReadSplit(read):
        # read quality also covers "uniquely mapped" etc. otherwise we could separately check flags
        if read.mapping_quality <= min_quality:
            return False
        if read.is_reverse == minus_strand:
            for cigaroperation, length in read.cigartuples:
                if cigaroperation == 3:
                    return read.get_overlap(start, end) == 1
            return False
        else:
            return False

    def filterReadUnsplit(read):
        if read.mapq <= min_quality:
            return False
        if read.is_reverse == minus_strand:
            for cigaroperation, length in read.cigartuples:
                if cigaroperation == 3:
                    return False
            # Full overlap
            return read.aend > end and read.pos <= start
        else:
            return False

    array = bam.count_coverage(contig='chr' + str(chrom), start=start,
                               stop=end, quality_threshold=0,
                               read_callback=filterReadSplit)

    cov_split_array = functools.reduce(np.add, array)

    cov_unsplit_array = functools.reduce(np.add,
                                         bam.count_coverage(contig='chr' + str(chrom), start=start,
                                                            stop=end, quality_threshold=0,
                                                            read_callback=filterReadUnsplit))

    result = np.max(cov_split_array), np.max(cov_unsplit_array)
    getSJCoverage.cache[minus_strand][(start, end)] = result
    return result

def worker_main(itemsexonschrom, bamfile, chr_prefix, verbose, min_intron_length, min_coverage, min_quality):
    items, exons, chrom, chromnr, task_id = itemsexonschrom

    if chr_prefix:
        contig = "chr" + chrom
    else:
        contig = chrom

    bam = pysam.AlignmentFile(bamfile, 'rb')
    n = len(items.items())
    # print(n, "transcripts on chromosome ", contig)
    resultrows = []

    progressupdates_send.send((task_id, True, n))

    updateinterval = 100
    c = 0
    # descr = "Chromosome " + str(chrom)
    # with tqdm(total=n, desc=descr, position=chromnr, leave=False) as pbar:
    for tcxExons in items.items():
        c += 1
        if c % updateinterval == 0:
            progressupdates_send.send((task_id, False, updateinterval))
        # pbar.update(1)
        tcx, listOfExons = tcxExons

        numberExons = len(listOfExons)

        listOfExons.sort(key=lambda exon: exon[1])
        minus_strand = listOfExons[0][3] == "-"

        strandrange = range(1, numberExons) if not minus_strand else reversed(range(1, numberExons))
        intronid = 0

        exoncov3p = None
        chrom = listOfExons[0][0]
        strand = listOfExons[0][3]

        for i in strandrange:

            exoncov5p = exoncov3p
            exoncov3p = None

            if minus_strand:
                exon5p = listOfExons[i]
                exon3p = listOfExons[i - 1]
            else:
                exon5p = listOfExons[i - 1]
                exon3p = listOfExons[i]

            intronid += 1
            debug = False

            thisintron = (listOfExons[i - 1][0], listOfExons[i - 1][2], listOfExons[i][1], listOfExons[i][3], intronid)

            # filter by intron length
            if (thisintron[2] - thisintron[1] - 1 >= min_intron_length):
                # filter by overlap with other exons (i.e. level 3)
                if (thisintron[1] + 1, thisintron[2]) not in exons[strand]:
                    if exoncov5p is None:
                        exoncov5p = getCoverage(bam, chrom, exon5p[1], exon5p[2], minus_strand, min_quality)

                    exoncov3p = getCoverage(bam, chrom, exon3p[1], exon3p[2], minus_strand, min_quality)

                    introncov = getCoverage(bam, chrom, thisintron[1], thisintron[2], minus_strand, min_quality)
                    sjcov5p_s, sjcov5p_us = getSJCoverage(bam, chrom,
                                                          exon5p[2 - int(minus_strand)] - 1,
                                                          exon5p[2 - int(minus_strand)] + 1,
                                                          minus_strand, min_quality, debug)
                    sjcov3p_s, sjcov3p_us = getSJCoverage(bam, chrom,
                                                          exon3p[1 + int(minus_strand)] - 1,
                                                          exon3p[1 + int(minus_strand)] + 1,
                                                          minus_strand, min_quality, debug)

                    if (sjcov5p_s + sjcov5p_us >= min_coverage and
                            sjcov3p_s + sjcov3p_us >= min_coverage):
                        resultrows.append([thisintron[0], thisintron[1], thisintron[2],
                                           thisintron[3], thisintron[4],
                                           None, tcx, exoncov5p, sjcov5p_s, sjcov5p_us,
                                           introncov, sjcov3p_s,
                                           sjcov3p_us, exoncov3p,
                                           calcScore(introncov, exoncov5p, exoncov3p)])

    progressupdates_send.send(
        (task_id, False, updateinterval))  # send another batch to definitely set the progress to 100%
    return resultrows

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-g', '--gtffile', metavar='', type=argparse.FileType('r'), nargs=1, required=True,
                        help="GTFfile provided by GENCODE or Ensembl.")
    parser.add_argument('-b', '--bamfile', metavar='', type=argparse.FileType('r'), nargs=1, required=True,
                        help="Sorted BAM file.")
    parser.add_argument('-o', '--outfile', metavar='', type=str,
                        help="Output file name.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="Enable quiet mode (only shows warnings and errors).")
    parser.add_argument('-i', '--IERatio', action='store_true',
                        help="Running mode that additionally outputs the Inverse Intron Expression Ratio (IER). Takes longer.")
    parser.add_argument('-f', '--filterLevel', type=int, choices=range(1, 4), metavar='', default=3,
                        help="Levels:"
                             "1) Keeps all introns in the genome. "
                             "2) Keeps only introns whose SJs do not overlap with exons in different genes. "
                             "3) Keeps only introns that do not overlap with exons in other parts of the genome (Default).")
    parser.add_argument('-l', '--MinIntronLength', type=int, metavar='', default=30,
                        help="Minimum intron length. (Default = 30)")
    parser.add_argument('-r', '--MinReadQuality', type=int, metavar='', default=10,
                        help="Minimum read quality. (Default > 10)")
    parser.add_argument('-c', '--MinCoverage', type=int, metavar='', default=10,
                        help="Minimum coverage of each splice junctions. (Default = 10)")
    parser.add_argument('-p', '--NrProcesses', type=int, metavar='', default=4,
                        help="Number of processes spawned to process chromosomes in parallel. 0 spawns as many as processors are available.")
    parser.add_argument('-x', '--ChromsList', type=str, metavar='', default='',
                        help="List of Chromosome names separated by comma (without spaces). Without this parameter a default set of common chromosomes is used.")

    args = parser.parse_args()

    global defaultchroms
    if not args.ChromsList == '':
        defaultchroms = args.ChromsList.split(',')

    bamfiles = args.bamfile
    for bamfile in bamfiles:
        bamfile_name = bamfile.name

        if not os.path.isfile(bamfile_name + ".bai") or os.stat(bamfile_name).st_mtime >= os.stat(
                bamfile_name + ".bai").st_mtime:
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Generating index file ' + bamfile_name + ".bai" + "...")
            bai = str(bamfile_name + ".bai")
            pysam.index(bamfile_name, bai)
        else:
            print("Index available")
            continue

    if args.IERatio:
        if not args.filterLevel == 3:
            print("Warning: using IER is restricted to --filterLevel 3. Ignoring user input for -f/--filterLevel.")
        main_coverage(parser)
    else:
        main_nocoverage(parser)

def main_nocoverage(parser):
    args = parser.parse_args()
    verbose = not args.quiet
    gtffile = args.gtffile[0]
    bamfiles = args.bamfile
    if gtffile == None or bamfiles == None:
        print("ERROR: Check input file arguments!", file=sys.stderr)
        parser.print_help()
        exit(1)

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Parsing splice junctions...')

    keepMaskedIntrons = args.filterLevel <= 2
    filterSJs = args.filterLevel == 2
    sjintervals, nrIntrons, intronIDtoTcxGeneNr = get_sjCoordinates(gtffile, min_intron_length=args.MinIntronLength,
                                                                    keep_masked_introns=keepMaskedIntrons,
                                                                    filterSJs=filterSJs,
                                                                    intron_file=None,
                                                                    exon_file=None,
                                                                    chroms=defaultchroms)

    nrProcesses = cpu_count()
    if args.NrProcesses > 0:
        nrProcesses = args.NrProcesses

    for bamfile in bamfiles:
        # Open bam quickly to check chromosomes to be extracted
        name = bamfile.name
        bam = pysam.AlignmentFile(bamfile, 'rb')
        bamRefs = set()
        chr_prefix = False

        for r in bam.references:
            if r[:3] == "chr":
                chr_prefix = True
                bamRefs.add(r[3:])
            else:
                bamRefs.add(r)
        foundChroms = set(defaultchroms).intersection(bamRefs)
        bam.close()

        if nrProcesses > len(foundChroms):
            nrProcesses = len(foundChroms)
            print("Warning: Number of requested processes is higher than the number of chromosomes to "
                  "parallelize over. Reducing to " + str(nrProcesses))

        if verbose:
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Calculating coverage for ' + name + '...')

        read_qual = args.MinReadQuality

        processBamEntryWSettings = functools.partial(processBamEntry, bamfile=name, verbose=verbose,
                                                     read_qual=read_qual, chr_prefix=chr_prefix)
        p = Pool(processes=nrProcesses)
        results = p.map(processBamEntryWSettings, [(x, sjintervals[x]) for x in foundChroms if x in sjintervals])
        p.close()
        p.join()

    # Initialize output table
    outfilename = args.outfile
    if outfilename is None or len(outfilename) == 0:
        outfilename = "splicing-efficiency-score.tsv"

    # rows: nrIntrons
    # cols: 13, see header line below
    lines = [[None for x in range(14)] for y in range(nrIntrons)]

    # process file...
    # Fill the entries at the row corresponding to the ID that was saved for the SJ
    for chrom, strands in results:
        for strand, intervals in strands.items():
            for interval in intervals:
                currentIntronID = interval[2]['id']
                entry = lines[currentIntronID]
                if interval[2]['sj3']:
                    entry[0] = chrom
                    entry[1] = strand
                    idColumn = intronIDtoTcxGeneNr[currentIntronID].split("_")
                    entry[2] = idColumn[0]
                    entry[3] = idColumn[1]
                    entry[4] = idColumn[2]
                    offset = 9
                else:
                    offset = 5
                entry[offset] = interval[0]
                entry[offset + 1] = interval[1]
                entry[offset + 2] = interval[2]['us']
                entry[offset + 3] = interval[2]['s']

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Calculation splicing efficiencies...')

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Writing output...')

    skipped = 0
    min_coverage_param = args.MinCoverage
    min_coverage = max(1, min_coverage_param)
    with open(outfilename, 'w') as f:
        # header
        f.write("\t".join(
            ["chr", "strand", "gene_ID", "transcript_ID", "intron_ID",
             "sj5start", "sj5end", "sj5_cov_nonsplit", "sj5_cov_split",
             "sj3start", "sj3end", "sj3_cov_nonsplit", "sj3_cov_split", "score"]) + "\n")
        for line in lines:
            # skip filtered entries
            if line[7] is None or line[8] is None or line[11] is None or line[12] is None:
                skipped = skipped + 1
                continue

            sj5_cov = float(line[7] + line[8])
            sj3_cov = float(line[11] + line[12])

            if sj5_cov >= min_coverage and sj3_cov >= min_coverage:
                score = (float(line[8] + line[12])) / (sj5_cov + sj3_cov)
                line[13] = str(score)
                f.write("\t".join(str(x) for x in line) + "\n")
            # else skip this intron completely
    f.close()

    if verbose:
        if skipped > 0:
            "Warning: Skipped " + str(skipped) + " introns (probably due to missing chromosome/contig in bam file)."
        print("Done!")

def progress_checking(recv):
    # for task_id, rec in recv:
    if recv.poll():
        task_id, init, inc = recv.recv()
        if init:
            progress.update(task_id, total=inc)
            progress.start_task(task_id)
        else:
            progress.advance(task_id, inc)

def main_coverage(parser):
    args = parser.parse_args()

    verbose = not args.quiet
    GTFfile = args.gtffile[0]
    bamfile = args.bamfile[0]
    if GTFfile is None or bamfile is None:
        print("ERROR: Check input file arguments!", file=sys.stderr)
        parser.print_help()
        exit(1)

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Parsing splice junctions...')

    min_quality = args.MinReadQuality

    nrProcesses = cpu_count()
    if args.NrProcesses > 0:
        nrProcesses = args.NrProcesses

    min_intron_length = args.MinIntronLength
    min_coverage_param = args.MinCoverage
    min_coverage = max(1, min_coverage_param)

    chr_prefix = False

    bam = pysam.AlignmentFile(bamfile.name, 'rb')
    bamRefs = set()

    for r in bam.references:
        if r[:3] == "chr":
            chr_prefix = True
            bamRefs.add(r[3:])
        else:
            bamRefs.add(r)
    foundChroms = set(defaultchroms).intersection(bamRefs)
    bam.close()

    # gets transcript per gene
    iso2gene = {}
    # lists exons per transcript (tcx)
    chrom2iso2exons = defaultdict(defaultdict)
    exon_by_chr = defaultdict(dict)
    # tcxpos = defaultdict()

    # get exons from GTF
    for line in GTFfile:
        splitGTFline = line.split('\t')
        chrom = splitGTFline[0]
        if len(chrom) > 3 and chrom[:3] == "chr":
            chrom = chrom[3:]

        if chrom != '#' and chrom in defaultchroms:
            if splitGTFline[2] == 'exon':
                gene = line.split('gene_id')[1].split('"')[1]
                tcx = line.split('transcript_id')[1].split('"')[1]
                iso2gene[tcx] = gene
                start1 = int(splitGTFline[3])
                end1 = int(splitGTFline[4])
                strand = splitGTFline[6]
                newExon = (chrom, start1 - 1, end1, strand)
                if chrom not in chrom2iso2exons:
                    chrom2iso2exons[chrom] = defaultdict(list)
                chrom2iso2exons[chrom][tcx].append(newExon)
                if chrom not in exon_by_chr:
                    exon_by_chr[chrom] = defaultdict(InterLap)
                exon_by_chr[chrom][strand].add((start1, end1, {'g': gene}))

    GTFfile.close()

    # Initialize output table

    outfilename = args.outfile
    if len(outfilename) == 0:
        outfilename = "EIratio.tsv"

    # with Pool(processes=nrProcesses, initargs=(RLock(),), initializer=tqdm.set_lock) as p:
    with Pool(processes=nrProcesses) as p:
        with open(outfilename, 'w+') as f:
            writer = csv.writer(f, lineterminator='\n', delimiter="\t")
            writer.writerow(["chr", "IStart", "IEnd", "strand", "intron_ID", "gene_ID", "transcript_ID",
                             "exon5_cov", "sj5_cov_split",
                             "sj5_cov_nonsplit", "intron_cov", "sj3_cov_split", "sj3_cov_nonsplit",
                             "exon3_cov", "IER"])
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Calculating coverage...')
            chromtotask = {}
            for chrom in foundChroms:
                chromtotask[chrom] = progress.add_task("Chromosome " + str(chrom), start=False)

            progress.start()
            processtcxwsettings = functools.partial(worker_main, bamfile=bamfile.name, chr_prefix=chr_prefix,
                                                    verbose=verbose, min_intron_length=min_intron_length,
                                                    min_coverage=min_coverage, min_quality=min_quality)

            it = p.imap_unordered(processtcxwsettings,
                                  [(chrom2iso2exons[chrom], exon_by_chr[chrom], chrom, i, chromtotask[chrom]) for
                                   i, chrom in enumerate(foundChroms)])
            while True:
                try:
                    result = it.next(timeout=0.1)
                except TimeoutError:
                    progress_checking(progressupdates_recv)
                except StopIteration:
                    break
                except IndexError:
                    break
                else:
                    if result is not None:
                        for row in result:
                            row[5] = iso2gene[row[6]]
                            writer.writerow(row)
            progress.stop()
            f.close()

    now = datetime.datetime.now()
    print(now.strftime("%D   %H:%M") + '   Done!')

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
import streamlit as st
import pivottablejs as pt
st.set_page_config(page_title="Analisis Efectividad Essalud", page_icon=":bar_chart:", layout="wide")

# Supongamos que tienes un DataFrame llamado df que contiene tus datos
# Si aún no tienes un DataFrame, puedes cargar tus datos desde un archivo CSV o Excel.
df = pd.read_csv("https://docs.google.com/spreadsheets/d/e/2PACX-1vTnluyDGAkS8o-IezNn9dAEK9KKJa8ZkdbsLM6ijhyhlnm6-b5FEiXdfVPMsEL1qg/pub?gid=2015743116&single=true&output=csv")

# Elimina todas las columnas que contienen "CIM" en su nombre
df_cant = df.copy()
columnas_cim = [col for col in df.columns if "CIM" in col]
df = df.drop(columns=columnas_cim)

# Lista de las palabras clave que deseas buscar en los nombres de las columnas
palabras_clave = ["Interpretación"]

# Filtra las columnas que contienen las palabras clave
columnas_seleccionadas = [col for col in df.columns if any(palabra in col for palabra in palabras_clave)]

# Crea un nuevo DataFrame con todas las otras columnas y las seleccionadas en formato vertical
df_vertical = pd.melt(df, id_vars=[col for col in df.columns if col not in columnas_seleccionadas],
                      value_vars=columnas_seleccionadas, var_name="Nombre de la Columna", value_name="Valor")

# Cambiar el tipo de datos de la columna "Fecha de nacimiento" a datetime
df_vertical['Fecha de nacimiento'] = pd.to_datetime(df_vertical['Fecha de nacimiento'], errors='coerce')

# Renombrar las columnas
df_vertical = df_vertical.rename(columns={'Nombre de la Columna': 'Medicamento_Antibiotico', 'Valor': 'Interpretacion_Origen'})

df_vertical['Interpretacion_Futuro'] = df_vertical['Interpretacion_Origen']

# Realizar las transformaciones
df_vertical['Interpretacion_Futuro'].replace({'S*': 'S', 'R*': 'R', 'I': 'I', 'IB': 'I', np.nan: '-'}, inplace=True)
df_vertical['Medicamento_Antibiotico'] = df_vertical['Medicamento_Antibiotico'].str.replace(' Interpretación', '')
# df_vertical.to_clipboard()
tabla_dinamica = pd.pivot_table(df_vertical,
                                 values='Fecha de muestra',  # Puedes seleccionar cualquier columna numérica
                                 index=['Origen','Microorganismo', 'Interpretacion_Futuro', 'Medicamento_Antibiotico'],
                                 aggfunc='count')

# Resetear el índice para que los índices se conviertan en columnas
tabla_dinamica = tabla_dinamica.reset_index()
tabla_dinamica = tabla_dinamica.rename(columns={'Fecha de muestra': 'Recuento'})
tabla_dinamica = tabla_dinamica[tabla_dinamica["Interpretacion_Futuro"]!="-"]
df_filtered_v = tabla_dinamica
# df_filtered_v.to_clipboard()
# st.sidebar.header("Aplique los filtros aqui:")
# origen = st.sidebar.multiselect(
#         "Seleccione el origen:",
#         options=df_filtered_v["Origen"].unique()
#     )
# medicamento = st.sidebar.multiselect(
#         "Seleccione el medicamento:",
#         options=df_filtered_v["Medicamento_Antibiotico"].unique()
#     )
# microorganismo = st.sidebar.multiselect(
#         "Seleccione el microorganismo:",
#         options=df_filtered_v["Microorganismo"].unique()
#     )
# interpretacion = st.sidebar.multiselect(
#         "Seleccione la interpretacion:",
#         options=df_filtered_v["Interpretacion_Futuro"].unique()
#     )

# if origen:
#         df = df[df["Origen"].isin(origen)]

with st.container():
    st.title("ESSALUD - MICROBIOLOGIA")

    # Crear un contenedor con dos columnas

    st.header("", divider="grey")
    st.title("Analisis por Tipo Origen - Bacterias mas Frecuentes")
    st.write("Clasificacion R: RESISTENTE - S: SENSIBLE")
    top3_activate = st.checkbox("¿Desea activar el TOP3 Bacterias mas frecuentes?")
    origen_fil = st.multiselect(
        "Seleccione el ORIGEN:",
        options=df_filtered_v["Origen"].unique(),key="Origen_Frecuente"
    )
    df_selection_fil = df_cant
    if origen_fil:
        df_selection_fil = df_selection_fil[df_selection_fil["Origen"].isin(origen_fil)]
        df_cant = df_cant[df_cant["Origen"].isin(origen_fil)]

    st.write(f"Cantidad de registros para el ORIGEN :                   **{df_cant.shape[0]}** muestras", unsafe_allow_html=True)
    microorganismo_fil = st.multiselect(
        "Seleccione el MICROORGANISMO:",
        options=df_filtered_v["Microorganismo"].unique(),key="Microorganismo_Frecuente"
    )
    if microorganismo_fil:
        df_selection_fil = df_selection_fil[df_selection_fil["Microorganismo"].isin(microorganismo_fil)]

#     # Checkbox para mostrar/ocultar el DataFrame general
#     # mostrar_dataframe = st.checkbox("Mostrar Tabla General de Datos", value=False)

#     # if mostrar_dataframe:

#     #     ordenar_por = st.selectbox("Ordenar por:", df_vertical.columns)
#     #     df_sorted = df_vertical.sort_values(by=[ordenar_por])

#     #     # Mostrar la tabla en Streamlit
#     #     st.dataframe(df_sorted)
#     # Si el checkbox está seleccionado, muestra el DataFrame
#     # filtro = st.text_input("Filtrar por Medicamento Antibiótico:", "")
#     # filtro_microorganismo = st.text_input("Filtrar por Microorganismo", "")
#     # filtro_interpretacion = st.text_input("Filtrar por Interpretacion", "")

#     # Aplicar filtros secuencialmente a la tabla dinámica

#     # if filtro:
#     #     df_filtered_v = df_filtered_v[df_filtered_v['Medicamento_Antibiotico'].str.contains(filtro, case=False)]

#     # if filtro_microorganismo:
#     #     df_filtered_v = df_filtered_v[df_filtered_v['Microorganismo'].str.contains(filtro_microorganismo, case=False)]

#     # if filtro_interpretacion:
#     #     df_filtered_v = df_filtered_v[df_filtered_v['Interpretacion_Futuro'].str.contains(filtro_interpretacion, case=False)]

#     # # Ordenar la tabla dinámica por el recuento en orden descendente
#     # df_filtered_v = df_filtered_v.sort_values(by=['Recuento'], ascending=False)

#     # # Mostrar la tabla dinámica filtrada y ordenada
#     # st.write("Tabla Dinámica de Recuento (Ordenada por Recuento Descendente)")
#     # st.dataframe(df_filtered_v)

#     df_selection = df_filtered_v
#     if origen:
#         df_selection = df_selection[df_selection["Origen"].isin(origen)]
#     if medicamento:
#         df_selection = df_selection[df_selection["Medicamento_Antibiotico"].isin(medicamento)]
#     if microorganismo:
#         df_selection = df_selection[df_selection["Microorganismo"].isin(microorganismo)]
#     if interpretacion:
#         df_selection = df_selection[df_selection["Interpretacion_Futuro"].isin(interpretacion)]

#     df_selection = df_selection.sort_values(by=['Recuento','Interpretacion_Futuro'], ascending=[False, False])
#     total_recuento = df_selection["Recuento"].sum()

# # Calcular el porcentaje y agregarlo como una nueva columna
#     df_selection["Porcentaje"] = ((df_selection["Recuento"] / total_recuento) * 100).apply(lambda x: f"{x:.2f}%")

#     st.dataframe(df_selection)
#     df_sens_resis = df_selection.copy()
#     df_sens_resis = df_sens_resis[(df_sens_resis["Interpretacion_Futuro"] == "R") | (df_sens_resis["Interpretacion_Futuro"] == "S")]
#     df_sensible = df_sens_resis[df_sens_resis["Interpretacion_Futuro"]=="S"]
#     df_resistente = df_sens_resis[df_sens_resis["Interpretacion_Futuro"]=="R"]
#     col1, col2 = st.columns(2)
#     try:
#         valor_mas_sensible=df_sensible.iloc[0,3]
#     except:
#         valor_mas_sensible = "No se tienen datos para medicamentos sensibles"

#     try:
#         valor_mas_resistente=df_resistente.iloc[0,3]
#     except:
#         valor_mas_resistente = "No se tienen datos para medicamentos resistentes"
#     # Columna 1
#     with col1:
#         st.header("MEDICAMENTO MAS SENSIBLE")
#         st.write(f"EL MEDICAMENTO MAS SENSIBLE ES: {valor_mas_sensible}")

#     # Columna 2
#     with col2:
#         st.header("MEDICAMENTO MAS RESISTENTE")
#         st.write(f"EL MEDICAMENTO MAS RESISTENTE ES: {valor_mas_resistente}")

    st.title("")

    tabla_conteo = df_cant.groupby(["Origen", "Microorganismo"]).size().reset_index(name="Conteo")
    tabla_conteo = tabla_conteo.sort_values(by="Conteo", ascending=False)
    total_filas = tabla_conteo["Conteo"].sum()
    tabla_conteo["Porcentaje"] = (tabla_conteo["Conteo"] / total_filas * 100).apply(lambda x: f'{x:.2f}%')
    top3 = tabla_conteo.nlargest(3, 'Conteo')
    if top3_activate:
        tabla_conteo = tabla_conteo[tabla_conteo['Conteo'].isin(top3['Conteo'])]
        lista_top3 = top3.values.tolist()
        total_filas = tabla_conteo["Conteo"].sum()
        tabla_conteo["Porcentaje"] = (tabla_conteo["Conteo"] / total_filas * 100).apply(lambda x: f'{x:.2f}%')
        top3 = tabla_conteo.nlargest(3, 'Conteo')
        total_conteo = tabla_conteo["Conteo"].sum()
        total_porcentaje = tabla_conteo["Porcentaje"].str.rstrip('%').astype(float).sum()

    # Crear un DataFrame para los totales
        totales = pd.DataFrame({'Conteo': [total_conteo], 'Porcentaje': [f'{total_porcentaje:.2f}%']})

    # Concatenar el DataFrame de totales al DataFrame original
        tabla_conteo = pd.concat([tabla_conteo, totales], ignore_index=True)

        st.dataframe(tabla_conteo) #modificar tabla

    else:
        total_filas = tabla_conteo["Conteo"].sum()
        tabla_conteo["Porcentaje"] = (tabla_conteo["Conteo"] / total_filas * 100).apply(lambda x: f'{x:.2f}%')
        top3 = tabla_conteo.nlargest(3, 'Conteo')
        total_conteo = tabla_conteo["Conteo"].sum()
        total_porcentaje = tabla_conteo["Porcentaje"].str.rstrip('%').astype(float).sum()

    # Crear un DataFrame para los totales
        totales = pd.DataFrame({'Conteo': [total_conteo], 'Porcentaje': [f'{total_porcentaje:.2f}%']})

    # Concatenar el DataFrame de totales al DataFrame original
        tabla_conteo = pd.concat([tabla_conteo, totales], ignore_index=True)

        st.dataframe(tabla_conteo)
    df_selection_fil = df_vertical
    if top3_activate:
        df_selection_fil = df_selection_fil[df_selection_fil["Microorganismo"].isin(tabla_conteo["Microorganismo"].to_list())]
    else:
        df_selection_fil = df_selection_fil

    # df_selection_fil.to_clipboard()

    filas = st.multiselect("Selecciona las filas:", ["Medicamento_Antibiotico","Fecha de muestra","Area","Muestra","Sexo"])
    df_selection_fil = df_selection_fil[df_selection_fil["Interpretacion_Futuro"].isin(["R","S"])]
    df_selection_fil["Recuento"] = 1
    if origen_fil:
        df_selection_fil = df_selection_fil[df_selection_fil["Origen"].isin(origen_fil)]
    if microorganismo_fil:
        df_selection_fil = df_selection_fil[df_selection_fil["Microorganismo"].isin(microorganismo_fil)]
    # columnas = st.multiselect("Selecciona las columnas:", df_selection_fil.columns)
    # valor = st.selectbox("Selecciona un valor:", df_selection_fil.columns)

    if 1:
                tabla_dinamica = pd.pivot_table(df_selection_fil, values="Recuento", index= filas, columns="Interpretacion_Futuro", aggfunc='count',fill_value=0)
                total_general = tabla_dinamica.sum().sum()

    # Calcular los porcentajes y agregarlos a la tabla dinámica
                tabla_porcentajes = tabla_dinamica.applymap(lambda x: f'{(x / total_general * 100):.2f}%' if x != 0 else '')
                nuevos_nombres_columnas = {
        'R': '% Resistente',
        'S': '% Sensible'
    }

                tabla_porcentajes = tabla_porcentajes.rename(columns=nuevos_nombres_columnas)

    # Concatenar la tabla de porcentajes a la tabla dinámica original
                tabla_dinamica = pd.concat([tabla_dinamica, tabla_porcentajes], axis=1)

                try:
                    tabla_dinamica = tabla_dinamica.sort_values(by='S', ascending=False)
                except:
                     pass
                def highlight_max(s):
                    is_max = s == s.max()
                    background_color = 'background-color: darkred'
                    text_color = 'color: white'
                    styles = [background_color if v else '' for v in is_max]
                    styles[0] += f'; {text_color}'  # Aplicar estilo de texto blanco al primer valor
                    return styles

# Aplicar el estilo condicional a las columnas donde deseas resaltar los valores más altos
                columnas_resaltar = ['% Resistente', '% Sensible']  # Aquí puedes especificar las columnas que deseas resaltar
                if ('% Resistente' in (tabla_dinamica.columns.tolist())) and ('% Sensible' in (tabla_dinamica.columns.tolist())):
                    st.write(tabla_dinamica.style.apply(highlight_max, subset=columnas_resaltar))
                elif "% Resistente" in (tabla_dinamica.columns.tolist()):
                    st.write(tabla_dinamica.style.apply(highlight_max, subset=["% Resistente"]))
                elif "% Sensible" in (tabla_dinamica.columns.tolist()):
                    st.write(tabla_dinamica.style.apply(highlight_max, subset=["% Sensible"]))

    # st.write("La tabla dinámica solicitada no funciono")

hide_st_style = """
            <style>
            #MainMenu {visibility: hidden;}
            footer {visibility: hidden;}
            header {visibility: hidden;}
            </style>
            """
st.markdown(hide_st_style, unsafe_allow_html=True)


from collections import defaultdict

from ..model.room import Room

class RoomService:
    def __init__(self):
        self._rooms = {}
        self._players = {}
        self._player_id_to_room_id = {}
        self._room_id_to_player_ids = defaultdict(set)

    def get_room_details_response(self, room_id):
        print("get_room_details_response", room_id, self._get_players_for_room_id(room_id))
        room = self._rooms[room_id]
        return dict(players=self._get_players_for_room_id(room_id), **self._rooms[room_id].to_dict(),)

    def get_all_rooms_response(self):
        return list(
            filter(
                lambda room: room["is_public"] and room["num_players"],
                [
                    dict(**room.to_dict(), num_players=len(self._get_players_for_room_id(room.room_id)),)
                    for room in self._rooms.values()
                ],
            )
        )

    def is_player_in_room(self, player_id, room_id):
        return self._player_id_to_room_id.get(player_id) == room_id

    def insert_player_in_room(self, player, room_id):
        self._add_player(player)
        old_room_id = self._set_player_room(player["id"], room_id)
        self._sanity_check()
        return old_room_id

    def delete_player(self, player_id):
        if not player_id in self._players:
            return None
        old_room_id = self._player_id_to_room_id[player_id]
        del self._player_id_to_room_id[player_id]
        del self._players[player_id]
        self._room_id_to_player_ids[old_room_id].remove(player_id)
        return old_room_id

    def update_player(self, player):
        if player["id"] not in self._players:
            raise KeyError(f"Player {player['id']} not recognized.")
        self._players[player["id"]] = player
        return self._player_id_to_room_id[player["id"]]

    def player_can_join_room(self, player_id, room_id):
        return True

    def _get_players_for_room_id(self, room_id):
        return [self._players[pid] for pid in self._room_id_to_player_ids[room_id]]

    def _add_player(self, player):
        self._players[player["id"]] = player

    def _set_player_room(self, player_id, room_id):
        if not room_id in self._rooms:
            self._create_room_for_player(room_id, player_id)

        old_room_id = self._player_id_to_room_id.get(player_id)
        if old_room_id:
            self._room_id_to_player_ids[old_room_id].remove(player_id)
        if old_room_id != room_id:
            self._room_id_to_player_ids[room_id].add(player_id)
            self._player_id_to_room_id[player_id] = room_id
            return old_room_id
        return None

    def _create_room_for_player(self, room_id, player_id):
        self._rooms[room_id] = Room(room_id, room_id != player_id)

    def _sanity_check(self):
        if not set(self._rooms) == set(self._room_id_to_player_ids):
            print("rooms", self._rooms)
            print("room id to player ids", self._room_id_to_player_ids)
            raise RuntimeError("Rooms does not match room ids to player ids")
        if not set(self._players) == set(self._player_id_to_room_id):
            print(self._players)
            print(self._player_id_to_room_id)
            raise RuntimeError("Player ids do not match player to room ids")
        for room_id, player_ids in self._room_id_to_player_ids.items():
            for player_id in player_ids:
                if not self._player_id_to_room_id.get(player_id) == room_id:
                    raise RuntimeError("Room id does not match")
        for player_id, room_id in self._player_id_to_room_id.items():
            if not player_id in self._room_id_to_player_ids.get(room_id, {}):
                raise RuntimeError("Room id does not match in player id -> room id")


import logging
import os

from cros.factory.device import device_utils
from cros.factory.test import test_case
from cros.factory.utils.arg_utils import Arg
from cros.factory.utils import pygpt

class PartitionTableTest(test_case.TestCase):
  ARGS = [
      Arg('min_usage_pct', (int, float),
          'Percentage of the storage device that must be before the end of the '
          'stateful partition.  For example, if this is 95%, then the stateful '
          'partition must end at a sector that is >=95% of the total number of '
          'sectors on the device.',
          default=95),
      Arg('expand_stateful', bool,
          'Repair partition headers and tables and expand stateful partition '
          'to all available free space',
          default=True)
  ]

  def _ShowGPTTable(self, path):
    show_cmd = pygpt.GPTCommands.Show()
    show_cmd.ExecuteCommandLine(path)

  def setUp(self):
    self.dut = device_utils.CreateDUTInterface()
    self.gpt = None

  def runTest(self):
    self.assertTrue(self.dut.link.IsLocal(),
                    'This test only support local device')
    dev = self.dut.storage.GetMainStorageDevice()
    self.gpt = pygpt.GPT.LoadFromFile(dev)
    stateful_no = self.dut.partitions.STATEFUL.index
    stateful_part = self.gpt.GetPartition(stateful_no)
    minios_a_no = self.dut.partitions.MINIOS_A.index
    minios_a_part = self.gpt.GetPartition(minios_a_no)
    minios_b_no = self.dut.partitions.MINIOS_B.index
    start_sector = stateful_part.FirstLBA
    sector_count = stateful_part.blocks
    end_sector = start_sector + sector_count
    sector_size = self.gpt.block_size

    # Linux always considers sectors to be 512 bytes long independently of the
    # devices real block size.
    device_size = 512 * int(
        self.dut.ReadFile('/sys/class/block/%s/size' % os.path.basename(dev)))

    pct_used = end_sector * sector_size * 100 / device_size

    logging.info(
        'start_sector=%d, sector_count=%d, end_sector=%d, device_size=%d',
        start_sector, sector_count, end_sector, device_size)
    logging.info('Stateful partition extends to %.3f%% of storage',
                 pct_used)
    if pct_used < self.args.min_usage_pct:
      if not self.args.expand_stateful:
        self.FailTask('Stateful partition does not cover enough of storage '
                      'device')

      # Repair partition headers and tables
      self.gpt.Resize(pygpt.GPT.GetImageSize(dev))

      # In disk_layout_v3, minios_b is the last partition.
      # We have to remove it or we cannot expand the stateful partition.
      has_minios_b = self.gpt.IsLastPartition(minios_b_no)
      # Calculate the size of minios_a and reserve space when expanding
      # stateful partition.
      reserved_blocks = minios_a_part.blocks if has_minios_b else 0

      if has_minios_b:
        self.gpt.WriteToFile(dev)
        pygpt.RemovePartition(dev, minios_b_no)
        # Reload gpt table if we remove partition minios_b.
        self.gpt = pygpt.GPT.LoadFromFile(dev)

      _, new_blocks = self.gpt.ExpandPartition(stateful_no, reserved_blocks)
      # Write back GPT table.
      self.gpt.WriteToFile(dev)

      if not has_minios_b:
        self._ShowGPTTable(dev)
        return

      # Add back partition minios_b.
      add_cmd = pygpt.GPTCommands.Add()
      add_cmd.ExecuteCommandLine('-i', str(minios_b_no), '-t', 'minios', '-b',
                                 str(start_sector + new_blocks), '-s',
                                 str(reserved_blocks), '-l', 'MINIOS-B', dev)

      # Write the content of minios_a to minios_b_part.
      src = '%sp%d' % (dev, minios_a_no)
      dst = '%sp%d' % (dev, minios_b_no)
      self.dut.CheckCall([
          'dd', 'bs=1048576',
          'if=%s' % src,
          'of=%s' % dst, 'iflag=fullblock', 'oflag=dsync'
      ], log=True)
      self._ShowGPTTable(dev)


import  logging
import  sys

class LogHelper:

    def __init__(self,name='LogHelper', setLevel = logging.DEBUG):

        self.logger = logging.getLogger(name)
        self.formatter = logging.Formatter('%(asctime)s  %(levelname)s  %(filename)s:%(lineno)d  %(message)s')

        #文件日志
        self.file_handler = logging.FileHandler(name +".log")
        self.file_handler.setFormatter(self.formatter)

        #终端日志
        self.consle_handler = logging.StreamHandler(sys.stdout)
        self.consle_handler.setFormatter(self.formatter)

        #日志级别
        self.logger.setLevel(setLevel)

        #添加到日志
        self.logger.addHandler(self.file_handler)
        self.logger.addHandler(self.consle_handler)

    #写日志
    def writeLog(self,info,level = 'debug'):
        if level == "critail":
            self.logger.critical(info)
        elif level == "error":
            self.logger.error(info)

        elif level == "warning":
            self.logger.warning(info)

        elif level == "info":
            self.logger.info(info)

        else:
            self.logger.debug(info)

    #删除日志
    def removeLog(self):
        self.logger.removeFilter(self.file_handler)
        self.logger.removeFilter(self.consle_handler)

logger = LogHelper()


from brave.inputs.input import Input
from gi.repository import Gst
import brave.config as config
import brave.exceptions

class TcpClientInput(Input):
    '''
    Allows an an input by receiving from another server via TCP.
    Basically using the `tcpclientsrc` element.
    '''

    def permitted_props(self):
        return {
            **super().permitted_props(),
            'host': {
                'type': 'str',
                'required': True
            },
            'port': {
                'type': 'int',
                'required': True
            },
            'width': {
                'type': 'int'
            },
            'height': {
                'type': 'int'
            },
            'container': {
                'type': 'str',
                'default': 'mpeg',
                'permitted_values': {
                    'mpeg': 'MPEG',
                    'ogg': 'OGG'
                }
            }
        }

    def create_elements(self):
        '''
        Creates the pipeline with elements needed to accept a TCP connection.
        '''

        # We support ogg and mpeg containers; the right demuxer must be used:
        demux_element = 'oggdemux' if self.container == 'ogg' else 'tsdemux'

        # We start with tcpclientsrc, and immediately demux it into audio and video.
        pipeline_string = 'tcpclientsrc name=tcpclientsrc ! %s name=demux ' % demux_element

        if self.has_video():
            # We need to decode the video:
            pipeline_string += (' queue2 max-size-time=3000000000 name=demux_to_video_queue ! '
                                'decodebin name=video_decodebin ')
            # This then can be connected to the common bit of the pipeline:
            pipeline_string += self.default_video_pipeline_string_end()

        if self.has_audio():
            # The audio part also needs a decodebin:
            pipeline_string += (' queue2 max-size-time=3000000000 name=demux_to_audio_queue'
                                ' ! decodebin name=audio_decodebin')

            # We will then connect this decodebin to aan audio convert/resample
            pipeline_string += ' audioconvert name=audioconvert ! audioresample ! ' + \
                config.default_audio_caps() + \
                self.default_audio_pipeline_string_end()

        self.create_pipeline_from_string(pipeline_string)
        tcpclientsrc = self.pipeline.get_by_name('tcpclientsrc')
        tcpclientsrc.set_property('host', self.host)
        tcpclientsrc.set_property('port', self.port)

        if self.has_video():
            self.final_video_tee = self.pipeline.get_by_name('final_video_tee')
            self.demux_to_video_queue = self.pipeline.get_by_name('demux_to_video_queue')
            self.video_element_after_demux = self.pipeline.get_by_name('video_output_queue')
            video_decodebin = self.pipeline.get_by_name('video_decodebin')
            video_decodebin.connect('pad-added', self._on_decodebin_pad_added)

        if self.has_audio():
            self.final_audio_tee = self.pipeline.get_by_name('final_audio_tee')
            self.demux_to_audio_queue = self.pipeline.get_by_name('demux_to_audio_queue')
            self.audio_element_after_demux = self.pipeline.get_by_name('audioconvert')
            audio_decodebin = self.pipeline.get_by_name('audio_decodebin')
            audio_decodebin.connect('pad-added', self._on_decodebin_pad_added)

        demux = self.pipeline.get_by_name('demux')
        demux.connect('pad-added', self._on_demux_pad_added)

    def _on_demux_pad_added(self, _, pad):
        '''
        Demux creates new pads every time the stream starts.
        This handles the creation of a new pad by linking it to the relevant element.
        The pipeline is:
        - For video: demuxer --> demux_to_video_queue --> decodebin --> ...
        - For audio: demuxer --> demux_to_audio_queue --> decodebin --> ...
        '''
        if not pad.has_current_caps():
            return

        caps = pad.get_current_caps()
        structure = caps.get_structure(0)
        name = structure.get_name()
        element_to_connect_to = None
        if name.startswith('video'):
            if hasattr(self, 'demux_to_video_queue'):
                element_to_connect_to = self.demux_to_video_queue

        elif name.startswith('audio'):
            if hasattr(self, 'demux_to_audio_queue'):
                element_to_connect_to = self.demux_to_audio_queue

        else:
            raise brave.exceptions.PipelineFailure('TCP Client: Unexpected pad name %s' % name)

        if not element_to_connect_to:
            return

        pad_to_connect_to = element_to_connect_to.get_static_pad('sink')
        if pad.link(pad_to_connect_to) is not Gst.PadLinkReturn.OK:
            self.logger.warning('Unable to connect demuxer to queue')

    def _on_decodebin_pad_added(self, _, pad):
        '''
        Like demux, decodebin creates new pads every time the stream starts.
        This handles the creation of a new pad by linking it to the relevant element.
        The pipeline is:
        - For video: decodebin --> video_output_queue --> tee --> ...
        - For audio: decodebin --> audioconvert --> audioresample --> ...
        '''
        if not pad.has_current_caps():
            return

        caps = pad.get_current_caps()
        structure = caps.get_structure(0)
        name = structure.get_name()
        element_to_connect_to = None
        if name.startswith('video'):
            element_to_connect_to = self.video_element_after_demux
        elif name.startswith('audio'):
            element_to_connect_to = self.audio_element_after_demux
        else:
            raise brave.exceptions.PipelineFailure('TCP Client: Unexpected decodebin pad name %s' % name)

        pad_to_connect_to = element_to_connect_to.get_static_pad('sink')
        if pad.link(pad_to_connect_to) is not Gst.PadLinkReturn.OK:
            self.logger.warning('Unable to connect demuxer to queue')


import os, yaml, argparse, torch

from module.data import load_dataloader
from module.model import load_generator, load_discriminator
from module.pretrain import GenTrainer, DisTrainer
from module.train import Trainer
from module.test import Tester

from module.generate import generate

from transformers import set_seed, AutoTokenizer

class Config(object):
    def __init__(self, args):

        with open('config.yaml', 'r') as f:
            params = yaml.load(f, Loader=yaml.FullLoader)
            for group in params.keys():
                for key, val in params[group].items():
                    setattr(self, key, val)

        self.mode = args.mode
        self.search_method = args.search

        use_cuda = torch.cuda.is_available()
        self.device_type = 'cuda' \
                           if use_cuda and self.mode != 'inference' \
                           else 'cpu'
        self.device = torch.device(self.device_type)

        self.g_ckpt = 'ckpt/generator.pt'
        self.d_ckpt = 'ckpt/discriminator.pt'
        self.tokenizer_path = f'data/{self.task}/tokenizer.json'

    def print_attr(self):
        for attribute, value in self.__dict__.items():
            print(f"* {attribute}: {value}")

def load_tokenizer(config):
    return

def pretrain(config, g_model, d_model, tokenizer):

    ###PreTrain Generator with Character Dataset
    g_train_dataloader = load_dataloader(config, tokenizer, 'train')
    g_valid_dataloader = load_dataloader(config, tokenizer, 'valid')

    g_trainer = GenTrainer(config, g_model, g_train_dataloader, g_valid_dataloader)
    g_trainer.train()

    ###Generate Samples to PreTrain Discriminator
    generate(config, g_model, tokenizer)

    ###PreTrain Discriminator
    config.model_type = 'discriminator'
    d_train_dataloader = load_dataloader(config, tokenizer, 'train')
    d_valid_dataloader = load_dataloader(config, tokenizer, 'valid')

    d_trainer = DisTrainer(config, d_model, d_train_dataloader, d_valid_dataloader)
    d_trainer.train()

def train(config, g_model, d_model, tokenizer):
    train_dataloader = load_dataloader(config, tokenizer, 'train')
    valid_dataloader = load_dataloader(config, tokenizer, 'valid')

    trainer = Trainer(config, g_model, d_model, tokenizer,
                      train_dataloader, valid_dataloader)
    trainer.train()

def test(config, g_model, d_model, tokenizer):
    test_dataloader = load_dataloader(config, 'test')
    tester = Tester(config, g_model, d_model, tokenizer, test_dataloader)
    tester.test()

def inference(g_model, tokenizer):
    g_model.eval()
    print(f'--- Inference Process Started! ---')
    print('[ Type "quit" on user input to stop the Process ]')

    while True:
        input_seq = input('\nUser Input Sequence >> ').lower()

        #End Condition
        if input_seq == 'quit':
            print('\n--- Inference Process has terminated! ---')
            break

        #convert user input_seq into model input_ids
        input_ids = tokenizer(input_seq, return_tensors='pt')['input_ids']
        output_ids = g_model.generate(input_ids, max_new_tokens=128, use_cache=True)
        output_seq = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]

        #Search Output Sequence
        print(f"Model Out Sequence >> {output_seq}")

def main(args):
    set_seed(42)
    config = Config(args)
    tokenizer = load_tokenizer(config)

    g_model = load_generator(config)
    d_model = load_discriminator(config)

    if config.mode == 'pretrain':
        pretrain(config, g_model, d_model, tokenizer)
    elif config.mode == 'train':
        train(config, g_model, d_model, tokenizer)
    elif config.mode == 'test':
        test(config, g_model, d_model, tokenizer)
    elif config.mode == 'inference':
        inference(g_model, tokenizer)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-mode', required=True)

    args = parser.parse_args()
    assert args.mode.lower() in ['pretrain', 'train', 'test', 'inference']

    if args.mode != 'pretrain':
        assert os.path.exists('ckpt/generator.pt')
        assert os.path.exists('ckpt/discriminator.pt')

    main(args)


from network import Net
import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
from PIL import Image
import random
import os
import getdata

dataset_dir = './data/test/'
model_file = './model/model.pth'
N = 10

device = torch.device("cuda:0" if (torch.cuda.is_available()) else "cpu")

def test():

    model = Net()
    model.to(device)
    model.load_state_dict(torch.load(model_file))
    model.eval()
    files = random.sample(os.listdir(dataset_dir), N)
    imgs = []
    imgs_data = []
    for file in files:
        img = Image.open(dataset_dir + file)
        img_data = getdata.dataTransform(img).to(device)
        imgs.append(img)
        imgs_data.append(img_data)
    imgs_data = torch.stack(imgs_data)
    out = model(imgs_data)
    out = F.softmax(out, dim=1)
    out = out.data.cpu().numpy()
    def del_file(path_data):
        for i in os.listdir(path_data):
            file_data = path_data + "\\" + i
            if os.path.isfile(file_data) == True:
                os.remove(file_data)
            else:
                del_file(file_data)

    path_data = r"./test_result"
    del_file(path_data)
    for idx in range(N):
        plt.figure()
        if out[idx, 0] > out[idx, 1]:
            plt.suptitle('cat:{:.1%},dog:{:.1%}'.format(out[idx, 0], out[idx, 1]))
        else:
            plt.suptitle('dog:{:.1%},cat:{:.1%}'.format(out[idx, 1], out[idx, 0]))
        plt.imshow(imgs[idx])
        plt.savefig("./test_result/result{}.png".format(idx))

if __name__ == '__main__':
    test()


from flask import Blueprint, request
from . import DB
import json
from ..workers import scan

scan_buffer = Blueprint('scan_buffer', __name__)

base_Coll = DB.getBaseColl()

from rq.job import Job
from redis import Redis
from rq import Queue
redis_conn = Redis()
q = Queue(connection=redis_conn)

from flask import jsonify

@scan_buffer.route('/scanBuffer',methods=["POST"])
def scanBufferPost():

      data = request.get_json()
      #id_client = data['id_client']  # use id_client for logging
      job = q.enqueue(scan.detect,data)
      return jsonify({'id': job.id})

@scan_buffer.route('/scanBuffer/<path:id>', methods=["GET"])
def scanBufferGet(id):

    job2 = Job.fetch(id, connection=redis_conn)
    response = {}

    if job2.get_status() == 'started':
        response['status'] = 'pending'

    elif job2.get_status() == 'finished':
        response['status'] = 'ready'

            # for file in job2.result:
            #   base_Coll.insert_one({'md5': file['md5'], 'status': file['status']})

        response['result'] = job2.result

    elif job2.get_status() == 'failed':
        response['status'] = 'failed'

    return jsonify(response)


from multiprocessing.connection import wait
import os
import csv
import time
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler

def firebase_to_csv(data_directory, csv_destination_directory):
    data_files = os.listdir(data_directory)

    columns = ['ind','Activity','Accel. X','Accel. Y','Accel. Z','Gyro. X','Gyro. Y','Gyro. Z']

    if not os.path.exists(csv_destination_directory):
        os.makedirs(csv_destination_directory)

    f_csv = open(f'{csv_destination_directory}/data.csv','w')
    writer = csv.writer(f_csv)
    writer.writerow(columns)
    i=1

    for aux_f in data_files:
        f = open(f'{data_directory}/{aux_f}', 'r')
        lines = f.readlines()
        activity = lines[0].split()[0] #split é para remover o \n

        for line in lines[1::]:
            row = []
            row.append(i)
            i+=1
            row.append(activity)
            aux_row = line.split(',')
            for n in aux_row:
                row.append(n.split()[0])

            writer.writerow(row)
        f.close()

    f_csv.close()

def preprocess_csvdata(data_dataframe):

    ###############################################################################################################
    # Now that the csv file win un-processed data has been created, we can read it as a Pandas Dataframe and make some changes to the data
    FRAME_SIZE = 200

    # Remove the index column
    data_dataframe = data_dataframe.drop('ind', axis=1, inplace=False)

    # Split data into X/Input y/output/Target sets
    X = data_dataframe.drop('Activity', axis=1, inplace=False)
    y = pd.DataFrame(data_dataframe['Activity'])

    # Encode Target labels ('corrida', 'andar', etc...)
    y = pd.get_dummies(y)

    # Scale input values (the axis values)

    scaler = MinMaxScaler(feature_range=(-1, 1))
    X = pd.DataFrame(scaler.fit_transform(X=X))

    '''
        By default, each frame has 1000 entries.
        We'll split each one in half, so that each frame/instance contains only 200 entries.
        This has to be done on both input and ouput data sets.
        Each frame will have 200 rows and 6 columns (6 sensor axis).
        Each frame will be associated with a vector of length 4 indicating its corresponding activity/label.
    '''
    print(f"Splitting X and y of shapes {X.shape} and {y.shape}")
    X, y = split_frames(X, y, frame_size=FRAME_SIZE, input_features=6, output_classes=4)

    return X, y

def split_frames(X, y, frame_size, input_features, output_classes):
    model_input = []
    model_output = []

    # Model input are (frame_size x input_features) matrices
    instances = X.shape[0]

    for index in range(0, instances, frame_size):

        # Get 'frame_size' entries of input data
        input_frame_data = X.iloc[index:index + frame_size].to_numpy(copy=True)
        model_input.append(input_frame_data.reshape(
            frame_size, input_features))

        # For each input frame, we have a label/activity to match.
        output_frame_label = y.iloc[[index]].to_numpy(copy=True)
        model_output.append(output_frame_label.reshape(output_classes))

    model_input = np.array(model_input)
    model_output = np.array(model_output)
    return model_input, model_output

def augment_running_data(dataframe: pd.DataFrame):

    original_dataframe = dataframe.copy(deep=True)

    # Get rows of activity 'running'
    running_rows = original_dataframe[original_dataframe['Activity'] == 'correr']

    # Apply symmetry transformation
    transformed_running_rows = running_rows
    transformed_running_rows['Accel. X'] = -transformed_running_rows['Accel. X']
    transformed_running_rows['Accel. Z'] = -transformed_running_rows['Accel. Z']
    transformed_running_rows['Gyro. X'] = -transformed_running_rows['Gyro. X']
    transformed_running_rows['Gyro. Z'] = -transformed_running_rows['Gyro. Z']

    # Insert these transformed rows ino the dataframe
    original_dataframe = pd.concat([original_dataframe, transformed_running_rows], ignore_index=True)
    return original_dataframe


from itertools import permutations, combinations

n, r = input().split()
arr = list(map(int,input().split()))
comi = combinations(arr, 3)
# perm = permutations(arr, 3)

count = 0
for i in list(comi):
    if int(i[1]/i[0]) == int(i[2]/ i[1]) == int(r):
        count += 1
print(count)
# print(list(comi))
# print(list(perm))


from ceilometer.api.controllers.v2 import base
from ceilometer.i18n import _
from ceilometer import pipeline as pipeline_t
from ceilometer.publisher import csvfile
import os
from oslo_log import log
import pecan
from pecan import rest
from stevedore import extension
import subprocess
import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
import yaml

LOG = log.getLogger(__name__)

P_TYPE = {
    'name': 'sample',
    'pipeline': pipeline_t.SamplePipeline,
    'source': pipeline_t.SampleSource,
    'sink': pipeline_t.SampleSink
}

class Pipeline(base.Base):
    """An externally defined object for pipeline. Corresponds to a SINK

    """

    name = wtypes.text
    enabled = bool
    location = wtypes.text
    max_bytes = int
    backup_count = int
    compress = bool

    "The unique name for the pipeline"

    def __init__(self, rule=None, **kwargs):
        super(Pipeline, self).__init__(**kwargs)

    def __repr__(self):
        # for logging calls
        return "Pipeline Sink(%s)" % self.name

    @staticmethod
    def validate(pipeline):
        valid_fields = ['name',
                        'enabled',
                        'location',
                        'max_bytes',
                        'backup_count',
                        'compress']
        for field in valid_fields:
            if getattr(pipeline, field) is None:
                error = _("%s is mandatory") % field
                pecan.response.translatable_error = error
                raise wsme.exc.ClientSideError(unicode(error))

        if pecan.request.cfg.publisher_csvfile.csv_location_strict:
            # Use abspath to expand sneaky things like ../ in the path
            loc_val = os.path.abspath(getattr(pipeline, 'location'))
            if not loc_val.startswith(pecan.request.cfg.
                                      publisher_csvfile.csv_location):
                fmt = _("%s violates the location constraint in config")
                error = fmt % loc_val
                pecan.response.translatable_error = error
                raise wsme.exc.ClientSideError(unicode(error))

        # we cannot have negative values for max_bytes
        if getattr(pipeline, 'max_bytes') < 0:
            error = _("max_bytes must be >= 0.")
            pecan.response.translatable_error = error
            raise wsme.exc.ClientSideError(unicode(error))

        return pipeline

    @classmethod
    def from_dict(cls, some_dict):
        return cls(**(some_dict))

    @classmethod
    def sample(cls):
        return cls(name='Sample')

class PipelinesController(rest.RestController):
    """Works on pipelines.

         Caveat: we do not support MULTIPLE CSV per sink
    """

    def format_as_dict(self, sink, csvpublisher):
        some_dict = {}
        some_dict['name'] = sink.name
        some_dict['enabled'] = csvpublisher.is_enabled
        some_dict['location'] = csvpublisher.location
        some_dict['max_bytes'] = csvpublisher.max_bytes
        some_dict['backup_count'] = csvpublisher.backup_count
        some_dict['compress'] = csvpublisher.compress
        return some_dict

    def get_pipelines_from_manager(self):
        pipeline_cfg = {}
        pipelines = pecan.request.pipeline_manager.pipelines
        for p in pipelines:
            for pb in p.sink.publishers:
                if isinstance(pb, csvfile.CSVFilePublisher):
                    # If it is the same sink, we can just overwrite it
                    pipeline_cfg[p.sink.name] = self.format_as_dict(p.sink, pb)
        return pipeline_cfg.values()

    def get_pipelines_from_cfg(self, conf, cfg):
        # This code is based on the constructor in pipeline.py
        transformer_manager = extension.ExtensionManager(
            'ceilometer.transformer')
        publisher_manager = pipeline_t.PublisherManager(
            conf, P_TYPE['name'])
        new_pipelines = []
        if 'sources' in cfg or 'sinks' in cfg:
            if not ('sources' in cfg and 'sinks' in cfg):
                raise pipeline_t.PipelineException(
                    "Both sources & sinks are required", cfg)
            sources = [P_TYPE['source'](s) for s in cfg.get('sources', [])]
            sinks = dict((s['name'], P_TYPE['sink'](conf, s,
                         transformer_manager, publisher_manager))
                         for s in cfg.get('sinks', []))
            for source in sources:
                source.check_sinks(sinks)
                for target in source.sinks:
                    new_pipelines.append(
                        P_TYPE['pipeline'](conf, source, sinks[target]))
        else:
            for pipedef in cfg:
                source = P_TYPE['source'](pipedef)
                sink = P_TYPE['sink'](conf, pipedef,
                                      transformer_manager, publisher_manager)
                new_pipelines.append(P_TYPE['pipeline'](conf, source, sink))
        return new_pipelines

    def get_pipelines_from_yaml(self):
        pipeline_cfg = {}

        yaml_cfg = []
        conf = pecan.request.cfg
        cfg_file = conf.pipeline_cfg_file
        if not os.path.exists(cfg_file):
            cfg_file = conf.find_file(cfg_file)
        with open(cfg_file) as fap:
            filedata = fap.read()
            yaml_cfg = yaml.safe_load(filedata)

        # Parse and convert to Pipeline objects
        pipelines = self.get_pipelines_from_cfg(conf, yaml_cfg)
        for p in pipelines:
            for pb in p.sink.publishers:
                if isinstance(pb, csvfile.CSVFilePublisher):
                    # If it is the same sink, we can just overwrite it
                    pipeline_cfg[p.sink.name] = self.format_as_dict(p.sink, pb)
        return pipeline_cfg.values()

    def get_pipelines(self):
        return self.get_pipelines_from_yaml()

    @wsme_pecan.wsexpose(Pipeline, unicode)
    def get_one(self, pipeline_name):
        """Retrieve details about one pipeline.

        :param pipeline_name: The name of the pipeline.
        """
        # authorized_project = acl.get_limited_to_project(
        #                           pecan.request.headers)
        pipelines = self.get_pipelines()
        for p in pipelines:
            # The pipeline name HERE is the same as the sink name
            if p.get('name', None) == pipeline_name:
                return Pipeline.from_dict(p)
        raise base.EntityNotFound(_('Pipeline'), pipeline_name)

    @wsme_pecan.wsexpose([Pipeline], [base.Query])
    def get_all(self, q=[]):
        """Retrieve definitions of all of the pipelines.

        :param q: Filter rules for the pipelines to be returned.   unused
        """
        # authorized_project = acl.get_limited_to_project(
        #                           pecan.request.headers)
        # kwargs =
        # _query_to_kwargs(q, pecan.request.storage_conn.get_resources)
        pipelines = self.get_pipelines()
        pipeline_list = [Pipeline.from_dict(p) for p in pipelines]
        return pipeline_list

    @wsme.validate(Pipeline)
    @wsme_pecan.wsexpose(Pipeline, wtypes.text, body=Pipeline)
    def put(self, data):
        """Modify this pipeline."""
        # authorized_project = acl.get_limited_to_project(
        #                           pecan.request.headers)
        # note(sileht): workaround for
        # https://bugs.launchpad.net/wsme/+bug/1220678
        Pipeline.validate(data)

        # Get the matching csv publisher for the pipeline.
        yaml_cfg = []
        conf = pecan.request.cfg
        cfg_file = conf.pipeline_cfg_file
        if not os.path.exists(cfg_file):
            cfg_file = conf.find_file(cfg_file)
        with open(cfg_file) as fap:
            filedata = fap.read()
            yaml_cfg = yaml.safe_load(filedata)

        # Parse and convert to Pipeline objects
        pipelines = self.get_pipelines_from_cfg(conf, yaml_cfg)

        csv = None
        for p in pipelines:
            if p.sink.name == data.name:
                for pb in p.sink.publishers:
                    if isinstance(pb, csvfile.CSVFilePublisher):
                        csv = pb
                        break

        if not csv:
            raise base.EntityNotFound(_('Pipeline'), data.name)

        # Alter the settings.
        csv.change_settings(enabled=data.enabled,
                            location=data.location,
                            max_bytes=data.max_bytes,
                            backup_count=data.backup_count,
                            compress=data.compress)
        new_csv = csv.as_yaml()

        # Set new_csv as the matching csvfile publisher for the sink
        sinks = yaml_cfg.get('sinks', [])
        for sink in sinks:
            if sink.get('name') == data.name:
                publishers = sink.get('publishers', [])
                for index, item in enumerate(publishers):
                    if item.strip().lower().startswith('csvfile:'):
                        publishers[index] = new_csv

        # Re-process in-memory version of yaml to prove it is still good
        # Should show throw an exception if syntax became invalid

        pipelines = self.get_pipelines_from_cfg(conf, yaml_cfg)

        # Must be outputted as a single yaml (ie: use dump ,not dump_all)
        stream = file(cfg_file, 'w')
        # Write a YAML representation of data to 'document.yaml'.
        yaml.safe_dump(yaml_cfg,
                       stream,
                       default_flow_style=False)
        stream.close()

        # Emit SIGHUP to all ceilometer processes to re-read config
        hup_list = ['ceilometer-collector',
                    'ceilometer-agent-notification']
        for h in hup_list:
            p1 = subprocess.Popen(['pgrep', '-f', h], stdout=subprocess.PIPE)
            subprocess.Popen(['xargs', '--no-run-if-empty', 'kill', '-HUP'],
                             stdin=p1.stdout, stdout=subprocess.PIPE)
        pecan.request.pipeline_manager = \
            pipeline_t.setup_pipeline(pecan.request.cfg)
        # Re-parse and return value from file
        # This protects us if the file was not stored properly
        pipelines = self.get_pipelines()
        for p in pipelines:
            if p.get('name', None) == data.name:
                return Pipeline.from_dict(p)
        # If we are here, the yaml got corrupted somehow
        raise wsme.exc.ClientSideError(unicode('pipeline yaml is corrupted'))


# coding: utf-8

from konlpy.tag import Twitter
from hoover.models import Hoover, Review
from hoover import search
from collections import defaultdict

def get_keyword(doc):
    keywords = []
    pos_tagger = Twitter()
    tokens = pos_tagger.pos(doc, norm=True, stem=True)
    for token in tokens:
        if token[1] == 'Noun':
            keywords.append(token[0])
    return keywords

def get_recommended_hoover(keywords):
    hoover_ids = Hoover.objects.values_list('id', flat=True)
    hoover_scores = {}
    for hoover_id in hoover_ids:
        hoover_scores[hoover_id] = get_hoover_score(hoover_id, keywords)
    return hoover_scores

def get_hoover_score(hoover_id, keywords):
    hoover_score = 0
    pos_tagger = Twitter()
    reviews = Review.objects.filter(product_id_id=hoover_id)
    for review in reviews:
        review_nouns = set(pos_tagger.nouns(review.content))
        for keyword in keywords:
            if keyword in review_nouns:
                hoover_score += review.rating - 3
    return hoover_score

def get_recommended_hoover_by_dict(keywords):
    hoover_scores = {}
    for keyword in keywords:
        if len(keyword) > 1:
            res = search.search(keyword)
            review_ids = res['hits']['hits'][0]['_source']['review_ids']
            hoover_scores = get_hoover_score_by_dict(review_ids, hoover_scores)
        else:
            pass
    return hoover_scores

def get_hoover_score_by_dict(review_ids, hoover_scores):
    reviews = Review.objects.filter(id__in=review_ids)
    for review in reviews:
        if review.product_id_id in hoover_scores.keys():
            hoover_scores[review.product_id_id] += review.rating - 3
        else:
            hoover_scores[review.product_id_id] = review.rating - 3
    return hoover_scores


from tqdm import tqdm
import argparse
import cv2
import os, sys
import argparse

sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))

from json_maker import json_maker

if __name__ == '__main__' :
    parser=argparse.ArgumentParser()
    parser.add_argument('--json_path', default = '/data3/sap/frcnn_keras/result/result-sv_messytable_cam3_resume_model_110/test_3cam/svdet+asnet.json', help = 'Path to decide how to crop')
    parser.add_argument('--save_dir', default= '/data3/sap/mvcnn/svdet+asnet',help = 'Path(parent directory) to images for saving')

    parser.add_argument('--image_dir', default = '/data1/sap/MessyTable/images', help = 'Path to images for classification')
    parser.add_argument('--num_cam', type=int, default = 3)

    args = parser.parse_args()

    json = json_maker([], args.json_path, args.num_cam)
    json.load()
    all_scene_nums = json.get_all_scenes()

    os.makedirs(args.save_dir, exist_ok=True)
    for scene_num in tqdm(all_scene_nums) :
        scene = json.get_scene(scene_num)
        cam_ids = json.get_all_cams(scene)
        instance_summary = json.get_instance_summary(scene_num)

        for cam_id in cam_ids :
            if int(cam_id) > args.num_cam : break
            cam = scene['cameras'][cam_id]
            img_name, instances = json.get_all_inst(cam)

            img = cv2.imread(os.path.join(args.image_dir, img_name))

            for inst in instances:
                inst_id = inst['inst_id']
                x1, y1, x2, y2 = map(int, inst['pos'])

                #save_name = '%s-%d-%d_%d.jpg' %(scene_num, int(cam_id), int(inst_id), int(subcls))
                save_name = '%s-%d-%d_10.jpg' %(scene_num, int(cam_id), int(inst_id))
                save_path = os.path.join(args.save_dir, save_name)

                cropped_img = img[y1: y2, x1: x2]
                cropped_img = cv2.resize(cropped_img,(256,256))

                cv2.imwrite(save_path, cropped_img)


import cv2
import os
from os import path
from pathlib import Path

# filepath = '../../../projects/vode/CCD/Normal/000004.mp4'
filepath = '/Users/kimminchae/Desktop/CNN/k400/'
# vidcap = cv2.VideoCapture(filepath)

file_names = os.listdir('/Users/kimminchae/Desktop/CNN/k400/')
file_list = list(file_names)

for file in file_list:
    video_names = os.listdir('/Users/kimminchae/Desktop/CNN/k400/' + file + '/')
    video_list = list(video_names)
    for video in video_list:
        count = 0
        file_num = 0
        str = video.split('.')
        vidcap = cv2.VideoCapture(filepath + file + '/' + video)
        print(filepath + file + '/' + video)
        while(vidcap.isOpend()):

            ret, image = vidcap.read()

            if count % 2 == 0:
                dir_path = Path("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file)
                if dir_path.is_dir() == True:
                    video_path = Path("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0])
                    if video_path.is_dir() == True:
                        cv2.imwrite("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0] + "/" + "%d.jpg" %file_num, image)
                        file_num += 1
                    else:
                        os.makedirs("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0])
                        cv2.imwrite("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0] + "/" + "%d.jpg" %file_num, image)
                        file_num += 1
                else:
                    os.makedirs("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file)
                    os.makedirs("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0])
                    cv2.imwrite("../../../projects/vode/data/video_segment/kinetics400_100/test/" + file + '/' + str[0] + "/" + "%d.jpg" %file_num, image)
                    file_num += 1
            count += 1

            if file_num == 100:
                print("Saved " + file + "Done")
                break


from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('<int:mall_id>/mall/', views.shop, name='shop'),
    path('<int:shop_id>/shop/', views.product, name='product'),
    path('<int:product_id>/product/', views.detail, name='detail'),
    path('<int:mall_id>/rate_mall/', views.rate_mall, name='rate_mall'),
    path('<int:shop_id>/rate_shop/', views.rate_shop, name='rate_shop'),
    path('<int:product_id>/rate_product/', views.rate_product, name='rate_product'),
]


from django.contrib import admin
from .models import City, Location, Driver, Customer, GasStation, Service
# Register your models here.

class CityAdmin(admin.ModelAdmin):
    fields = ('name', 'width', 'height', 'velocity')
    list_display = ('name', 'width', 'height', 'velocity')

class LocationAdmin(admin.ModelAdmin):
    fields = ('pos_x', 'pos_y')
    list_display = ('id', 'pos_x', 'pos_y')

class DriverAdmin(admin.ModelAdmin):
    fields = ('name', 'identification', 'is_busy', 'location')
    list_display = ('name', 'identification', 'is_busy', 'location')

class CustomerAdmin(admin.ModelAdmin):
    fields = ('identification', 'location')
    readonly_fields = ('created_at', 'updated_at')
    list_display = ('identification', 'location')

class GasStationAdmin(admin.ModelAdmin):
    fields = ('name', 'has_gasoline', 'nearby_driver_distance', 'location',
              'nearby_driver')
    list_display = ('name', 'has_gasoline', 'nearby_driver_distance',
                    'location', 'nearby_driver')

class ServiceAdmin(admin.ModelAdmin):
    fields = ('requesting_client', 'responsible_driver',
              'nearby_gas_station', 'status', 'distance')
    readonly_fields = ('date',)
    list_display = ('id', 'date', 'requesting_client', 'responsible_driver',
                    'nearby_gas_station', 'status', 'distance')

admin.site.register(City, CityAdmin)
admin.site.register(Location, LocationAdmin)
admin.site.register(Driver, DriverAdmin)
admin.site.register(Customer, CustomerAdmin)
admin.site.register(GasStation, GasStationAdmin)
admin.site.register(Service, ServiceAdmin)


#!/usr/bin/env python3

import numpy as np
from matplotlib import pyplot as plt
import collections

def PolyCoefficients(x, coeffs):
    """ Returns a polynomial for ``x`` values for the ``coeffs`` provided.

    The coefficients must be in ascending order (``x**0`` to ``x**o``).
    """
    o = len(coeffs)
    # print('This is a polynomial of order' + str(o))
    y = 0
    for i in range(o):
        y += coeffs[i] * x ** i
    return y

def readSplineDataFromFile(filename):
    d = {}
    with open(filename) as file:
        for line in file:
            words = line.split()
            name = words[0].replace("'", "")
            i = 1
            polies = []
            while i < len(words):
                min = float(words[i])
                max = float(words[i + 1])
                nr_coef = int(words[i + 2])
                coefs = []
                for j in range(0, nr_coef):
                    coefs.append(float(words[i + j + 3]))
                polies.append((min, max, nr_coef, coefs))
                i += 3 + nr_coef
            d[name] = polies
    print(d)
    print("\n\n")
    return d

data = readSplineDataFromFile("/tmp/spline_export")
oderedData = collections.OrderedDict(sorted(data.items()))
position = 1
for name in oderedData:
    if (not "foot_" in name) and (not "trunk_" in name):
        continue

    plt.subplot(4, 3, position)
    for poly in data[name]:
        min = poly[0]
        max = poly[1]
        nr_coef = poly[2]
        coeffs = poly[3]
        x = np.linspace(min, max)

        plt.plot(x, PolyCoefficients(x, coeffs))
    plt.title(name)
    position += 1

plt.show()