#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
***********************************************************
* @File    : 03.bamtrans.py
* @Author  : Zhou Wei                                     *
* @Date    : 2020/10/10 11:41:07                          *
* @E-mail  : welljoea@gmail.com                           *
* @Version : --                                           *
* You are using the program scripted by Zhou Wei.         *
* If you find some bugs, please                           *
* Please let me know and acknowledge in your publication. *
* Thank you!                                              *
* Best wishes!                                            *
***********************************************************
'''
import pysam
import pybedtools as bt
from Bio.Seq import Seq
import os
from joblib import Parallel, delayed
import glob
import pandas as pd
pd.set_option('display.max_rows', 1000)
pd.set_option('display.max_columns', 100000)
pd.set_option('display.width', 10000000)
import numpy  as np
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
import matplotlib.pyplot as plt
import seaborn as sns

class GetSimu():
    def __init__(self):
        pass
    def simuget(self, reffa, region, ratio, overlap):
        Type = region[-1]
        if Type=='a':
            regions= reffa.fetch(*region[0:3])
            regionn= '%s:%s_%s-r%s-v%s-t%s'%( tuple(region[0:3] + [ratio, overlap, Type]) )
        elif Type=='b':
            regions= reffa.fetch(*region[0:3]) + reffa.fetch(*region[3:6])
            regionn= '%s:%s_%s_%s:%s_%s-r%s-v%s-t%s'%( tuple(region[0:6] + [ratio, overlap, Type]) )
        elif Type=='c':
            regionf= reffa.fetch(*region[0:3])
            regionr= str(Seq(regionf).reverse_complement())[: int((region[2]-region[1])*region[3]) ]
            regions= regionf + regionr
            regionn= '%s:%s_%s_R%s-r%s-v%s-t%s'%( tuple(region[0:4] + [ratio, overlap, Type]) )

        regionb= int(len(regions)*ratio)
        regionq= regions[regionb-overlap:] + regions[:regionb+overlap]
        return '@%s\n%s\n+\n%s'%(regionn, regionq, '~'*len(regionq) )

    def simufa(self):
        hg38_ens='/share/home/share/Repository/GenomeDB/Reference/Homo_Sapiens/ENSEMBL/Homo_sapiens.GRCh38.dna.primary_assembly.fa'
        hg38_ens=pysam.FastaFile(hg38_ens)
        simatx=[
            ['10', 13070000, 13090000, 'a'],
            ['11', 13800000, 13830000, 'a'],
            ['1' , 13610000, 13615000, '3', 13710000, 13715000, 'b'],
            ['2' , 67250000, 67251000, '2', 67800000, 67810000, 'b'],
            ['3' , 80950000, 80970000, 0.2, 'c'],
            ['3' , 80950000, 80970000, 0.5, 'c'],
            ['3' , 80950000, 80970000, 0.7, 'c'],
            ['3' , 80950000, 80970000, 1.0, 'c'],
            ['4' , 21500000, 21520000, 0.2, 'c'],
            ['4' , 21500000, 21520000, 0.5, 'c'],
            ['4' , 21500000, 21520000, 0.7, 'c'],
            ['4' , 21500000, 21520000, 1.0, 'c'],
            #['12', 13830000, 13800000, 'a1'],
            #['13', 13830000, 13800000, 'a2'],
            #['3' , 67250000, 67251000, '2', 67800000, 67810000, 'b1'],
            #['4' , 67250000, 67251000, '2', 67800000, 67810000, 'b2'],
        ]
        ratios=[0.1, 0.3, 0.5, 0.7, 0.9]
        ovlaps=[0, 100, -100, -500]
        allfas=[ self.simuget(hg38_ens, _s, _r, _o) for _s in simatx for _r in ratios for _o in ovlaps]
        allfas='\n'.join(allfas)
        f=open('./simulate.fq','w')
        f.write(allfas)
        f.close()
#GetSimu().simufa()

class Visal():
    def __init__(self):
        pass
    def query_length(self, _indf, out, X='query_length', Dup='query_name', log=False, title=''):
        if not _indf.empty:
            indef = _indf.copy()
            if Dup:
                indef = indef[[Dup, X]].drop_duplicates(keep='first')

            indef[X] = indef[X].astype(int)
            dp = sns.displot(data=indef, x=X, kde=True, log_scale=log)
            dp.set_xticklabels(rotation=270)

            if title:
                plt.title(title)

            plt.tight_layout()
            plt.savefig( out )
            plt.close()

class Utilities():
    def __init__(self ):
        self.GTFBED = '/share/home/share/Repository/GenomeDB/Reference/Homo_Sapiens/ENSEMBL/Homo_sapiens.GRCh38.100.gtf.gene.bed'
        self.annotatepeak='/share/home/share/software/Homer/bin/annotatePeaks.pl'
        self.hg38gtf='/share/home/share/Repository/GenomeDB/Reference/Homo_Sapiens/ENSEMBL/Homo_sapiens.GRCh38.100.gtf'

    def emptyfile(_func):
        def wrapper(self, *args, **kargs):
            file = args[0]
            if os.path.exists(file):
                sz = os.path.getsize(file)
                if not sz:
                    print(file, " is empty!")
                else:
                    _func(self, *args, **kargs)
            else:
                print(file, " is not exists!")
        return wrapper

    def readgtf(self, gtffile):
        def splitinfo(_l):
            _l = _l.strip(';').split('; ')
            _k = ['']*3
            for _ll in _l:
                if  _ll.split(' ')[0]=='gene_id':
                    _k[0] = _ll.split(' ')[1].strip('"')
                if  _ll.split(' ')[0]=='gene_name':
                    _k[1] = _ll.split(' ')[1].strip('"')
                if  _ll.split(' ')[0]=='gene_biotype':
                    _k[2] = _ll.split(' ')[1].strip('"')
            return _k
        with open(gtffile, 'r') as  f:
            gtf = [ i.strip().split('\t')[:-1] + splitinfo(i.strip().split('\t')[-1])
                    for i in f.readlines()
                    if (not i.startswith('#')) and (i.split('\t')[2]=='gene') ]
        gtf = pd.DataFrame(gtf, columns=['#chrom', 'db', 'gtype', 'start', 'end', 'U1', 'forword',
                                            'U2', 'gene_id', 'gene_name', 'gene_biotype'])
        gtf['length'] = gtf.end.astype(int) - gtf.start.astype(int) + 1
        gtf = gtf[['#chrom', 'start', 'end', 'gtype', 'length', 'forword', 'gene_name', 'gene_id', 'gene_biotype']]
        gtf.to_csv('./Homo_sapiens.GRCh38.100.gtf.gene.bed', sep='\t',index=False)

    def annogene(self, _inbed, minoverA=30, minoverB=30,
                    biotype=['miRNA','lncRNA', 'protein_coding', 'IG_C_gene', 'IG_D_gene',
                            'IG_J_gene', 'IG_V_gene', 'TR_C_gene', 'TR_D_gene', 'TR_J_gene']
                            #'TR_V_gene', 'rRNA', 'scaRNA', 'scRNA', 'snoRNA', 'snRNA', 'sRNA'] ):
                ):

        GTFBED = pd.read_csv(self.GTFBED, sep='\t', header=0)
        GTFBED = GTFBED[(GTFBED.gene_biotype.isin(biotype))]
        annobed = bt.BedTool.from_dataframe(GTFBED)
        Rcol = ['#chrom', 'start', 'end', 'length', 'support_sum',
                '#chrom_g', 'start_g', 'end_g', 'gtype', 'length_g', 'forword_g',  'gene_name', 'gene_id', 'gene_biotype']
        Fcol = ['#chrom', 'start', 'end', 'length', 'support_sum',
                'head_dist', 'tail_dist', 'gene_name', 'gene_id', 'gene_biotype', 'overlen', 'overfre']

        def eachinter(_df):
            inbt = bt.BedTool.from_dataframe( _df )
            _minoverA = minoverA/_df.length.max()

            over = inbt.intersect(annobed, s=False, S=False, loj=True, f=_minoverA )\
                        .to_dataframe(disable_auto_names=True,  header=None, names=Rcol)
            over['start_m']   = over[['start', 'start_g']].max(1)
            over['end_m']     = over[['end'  , 'end_g']].min(1)
            over['overlen']   = over['end_m'] - over['start_m']
            over['overfre']   = over['overlen']/over['length'].astype(int)

            df=_df.copy()
            if over.iloc[0]['gene_name'] != '.':
                df['head_dist'] = over.iloc[0]['start_g'] - over.iloc[0]['start']
                df['tail_dist'] = over.iloc[-1]['end']    - over.iloc[-1]['end_g']
                df['gene_name'] = over.gene_name.str.cat(sep=';')
                df['gene_id']   = over.gene_id.str.cat(sep=';')
                df['gene_biotype'] = over.gene_biotype.str.cat(sep=';')
                df['overlen'] = over.overlen.astype(str).str.cat(sep=';')
                df['overfre'] = over.overfre.round(4).astype(str).str.cat(sep=';')
            else:
                df['head_dist'] = -1
                df['tail_dist'] = -1
                df['gene_name'] = '.'
                df['gene_id']   = '.'
                df['gene_biotype'] = '.'
                df['overlen'] = -1
                df['overfre'] = -1
            return df[Fcol]

        inbed = Parallel( n_jobs=-1, backend='loky')( delayed( eachinter )(_g)
                    for _r, _g in _inbed.groupby(by=['#chrom', 'start', 'end'],sort=True) )
        inbed = pd.concat(inbed, axis=0,sort=False)
        return inbed

    @emptyfile
    def Peakannotate(self, inbed):
        cmd = '''
        INBED={inbed}
        GTF={hg38gtf}
        {annotatepeak} \
            $INBED \
            hg38 \
            -gtf $GTF \
            -gid \
            -cpu 10 \
            -go $INBED.go \
            -genomeOntology  $INBED.go.detail \
            1> $INBED.annotate.txt
            2> $INBED.annotate.log'''.format(inbed=inbed, hg38gtf=self.hg38gtf, annotatepeak=self.annotatepeak)
        print(cmd)
        os.system(cmd)

class Mapping():
    def __init__(self, fq1, inid, outdir):
        self.fq1 = fq1
        self.inid = inid
        self.outdir=outdir + '/' + inid
        os.makedirs(self.outdir, exist_ok=True)

    def emptyfile(_func):
        def wrapper(self, *args, **kargs):
            file = self.fq1
            if os.path.exists(file):
                sz = os.path.getsize(file)
                if not sz:
                    print(file, " is empty!")
                else:
                    _func(self, *args, **kargs)
            else:
                print(file, " is not exists!")
        return wrapper

    @emptyfile
    def SoftMINI(self,
                 samtls='/share/home/share/software/samtools-1.10/bin/samtools',
                 minip2='/share/home/share/software/minimap2-2.17_x64-linux/minimap2',
                 REF='/share/home/share/Repository/GenomeDB/Index/Homo_Sapiens/MinimapDB/ENSEMBL_GRch38_PRIM_MINIM/Homo_sapiens.GRCh38.dna.primary_assembly.mmi'):
        cmd = '''
        mkdir -p {outdir}
        {minimap2} \
            -ax asm20 \
            -t 20 \
            -k 19  -w 10 -H -O 5,56 -E 4,1 -A 2 -B 5 -z 400,50 -r 2000 -g 5000 --lj-min-ratio 0.5 \
            --cs --MD -Y \
            --secondary no \
            {REF} \
            {fq1} | \
            {samtools} view -bS -@ 10  - \
            > {outdir}/{ID}.bam
        {samtools} sort -@ 20 -o {outdir}/{ID}.sorted.bam {outdir}/{ID}.bam
        {samtools} index -@ 20  {outdir}/{ID}.sorted.bam
        '''.format(minimap2=minip2, REF=REF, fq1=self.fq1, samtools=samtls,outdir=self.outdir,ID=self.inid)
        print(cmd)
        os.system(cmd)

class SoftFetch():
    def __init__(self, inbam, inid, outdir):
        self.inbam=inbam
        self.inid = inid
        self.outdir=outdir + '/' + inid
        self.chrs=[str(i) for i in range(1,23)] + ['MT','X','Y'] #remove
        self.minsoflt = 5
        self.mapq     = 5
        os.makedirs(self.outdir, exist_ok=True)

    def cigarmerge(self, _ct, indel=100000, skip=1000000000, hard=100000, pad=1000000000, match='Q'):
        ct  = [ (0, i[1]) if (i[0] in [7,8]) else i for i in _ct ]
        ctN = ct[:1]
        for i in ct[1:]:
            if match=='Q':
                if ( i[1] <=indel and i[0] ==1 ):
                    i = (0, i[1])
                elif ( i[1] <=indel and i[0] ==2 ):
                    i = (0, 0)
                elif ( i[1] <=skip and i[0] ==3 ):
                    i = (0, 0)
                elif ( i[1] <=hard and i[0] ==5 ):
                    i = (0, 0)
                elif ( i[1] <=pad and i[0] ==6 ):
                    i = (0, 0)
                elif ( i[0] in [7, 8] ):
                    i = (0, i[1])
                else:
                    i = i

            elif match=='R':
                if ( i[1] <=indel and i[0] ==1 ):
                    i = (0, 0)
                elif ( i[1] <=indel and i[0] ==2 ):
                    i = (0, i[1])
                elif ( i[1] <=skip and i[0] ==3 ):
                    i = (0, i[1])
                elif ( i[1] <=hard and i[0] ==5 ):
                    i = (0, 0)
                elif ( i[1] <=pad and i[0] ==6 ):
                    i = (0, 0)
                elif ( i[0] in [7, 8] ):
                    i = (0, i[1])
                else:
                    i = i

            if ( ctN[-1][0]==0 and i[0]==0 ):
                ctN[-1] = (ctN[-1][0] + i[0], ctN[-1][1] + i[1])
            else:
                ctN.append(i)
        return ctN

    def bamcigarsoft(self):
        samfile = pysam.AlignmentFile(self.inbam, "rb")
        #samidx = pysam.IndexedReads(samfile)
        #samidx.build()
        sampd = []
        for read in samfile.fetch():
            tags = dict(read.tags)
            raw_cigarstring = read.cigarstring
            raw_cigartuples = read.cigartuples
            reference_name = read.reference_name
            softlens = [ i[1] for i in raw_cigartuples if (i[0]==4) and (i[1] >self.minsoflt) ]

            if (('SA' in tags) \
                    and ('S' in raw_cigarstring) \
                    and len(softlens) >0) \
                    and read.mapping_quality >=self.mapq :
                    #and (reference_name in self.chrs) \
                #TP = tags['tp'] if 'tp' in tags else ''
                SA = tags['SA']
                query_name = read.query_name
                flag = read.flag
                mapping_quality  = read.mapping_quality
                reference_start  = read.reference_start
                read.cigartuples = self.cigarmerge(raw_cigartuples, match='Q')
                cigartuples_ref  = self.cigarmerge(raw_cigartuples, match='R')
                query_sequence   = read.query_sequence
                #query_qualities  = pysam.array_to_qualitystring(read.query_qualities)
                is_reverse = '-' if read.is_reverse else '+'
                sampd.append([self.inid, query_name, flag, reference_name, reference_start, mapping_quality, read.cigartuples, cigartuples_ref,
                              is_reverse, len(query_sequence), SA, read.cigarstring ,raw_cigarstring, query_sequence])

        sampd = pd.DataFrame(sampd, columns=['SID', 'query_name', 'flag', 'reference_name', 'reference_start', 'mapping_quality',
                                            'cigartuples', 'cigartuples_ref', 'is_reverse', 'query_length', 'SA',
                                            'cigarstring', 'raw_cigar', 'query_sequence'])
        sampd = sampd.merge(sampd.groupby('query_name')['query_name'].size().reset_index(name='query_counts'),
                            on='query_name', how='outer')
        sampd = sampd[(sampd.query_counts>1)]

        #sampd.sort_values(by=['query_name', 'reference_name', 'reference_start', 'flag'], ascending=[True, True, True, True], inplace=True)
        sampd.to_csv('%s/%s.chimeric.txt'%(self.outdir, self.inid), sep='\t', index=False)
        return sampd

    def getsoftfq(self, INsoft, softminlen=100):
        f=open( '%s/%s.chimeric.fq'%(self.outdir, self.inid),'w')
        for _n, _l in INsoft.iterrows():
            cigt  = eval(_l.cigartuples) if type(_l.cigartuples)==str else _l.cigartuples
            start = 0
            SEQs  = []
            for  n,i in enumerate(cigt):
                end = int(start) + int(i[1])
                if (i[0] ==4 and i[1]>=softminlen):
                    seq = _l.query_sequence[start:end]
                    #qul = _l.query_qualities[start:end]
                    qul = '~'*len(seq)
                    name= '@%s_soft%s-%s_%s_%s'%(_l.query_name, _l.reference_name, _l.reference_start, n, i[1])
                    SEQs.append('%s\n%s\n+\n%s'%(name, seq, qul) )
                start += int(i[1])
            f.write('\n'.join(SEQs))
        f.close()

    def getsoftregion(self, _g):
        start  = int(_g.reference_start)
        cigref = eval(_g.cigartuples_ref) if type(_g.cigartuples_ref)==str else _g.cigartuples_ref
        cigartuples = eval(_g.cigartuples) if type(_g.cigartuples)==str else _g.cigartuples
        match_type  = ''.join([ 'S' if i[0]==4 else 'M' for i in cigartuples ])

        if _g.is_reverse=='-':
            cigartuples = cigartuples[::-1]

        cigarpos = [(0, cigartuples[0][1])]
        for i in cigartuples[1:]:
            cigarpos.append( (cigarpos[-1][1], cigarpos[-1][1]+i[1]) )
        if _g.is_reverse=='-':
            cigarpos = cigarpos[::-1]

        Regs  = []
        for  n,i in enumerate(cigref):
            if (i[0] !=4 ):
                bed = [ _g.reference_name, start, start + int(i[1]-1), _g.SID, i[1], _g.is_reverse, _g.query_name,
                        _g.flag, _g.cigarstring, cigarpos, cigarpos[n], _g.query_length, cigref, _g.query_counts]
                Regs.append(bed)
                start += int(i[1]-1)
        return Regs

    def getsoft(self, softfq=False, researh=True):
        print('start finding soft signal: ' + self.inid)
        if researh:
            INsoft = self.bamcigarsoft()
        else:
            INsoft = pd.read_csv('%s/%s.chimeric.txt'%(self.outdir, self.inid), sep='\t', low_memory=False)

        if INsoft.empty:
            print(self.inid +' donot find soft signal...')

        BEDS = Parallel( n_jobs=-1, backend='loky')( delayed( self.getsoftregion )(_l) for _n, _l in INsoft.iterrows() )

        if softfq:
            self.getsoftfq(INsoft)
        del(INsoft)

        colm = ['#chrom', 'start', 'end',  'SID', 'length', 'forword', 'query_name', 'flag',
                'cigarstring', 'cigarpos',  'cigarreg' , 'query_length', 'cigarreffilt', 'query_counts']
        BEDS = pd.DataFrame( np.reshape( np.array(BEDS),(-1,len(colm))), columns=colm)
        BEDS.sort_values(by=['#chrom', 'start', 'end', 'query_name', 'cigarreg'],
                            ascending=[True]*5, inplace=True)
        BEDS.to_csv('%s/%s.chimeric.bed'%(self.outdir, self.inid), header=True, index=False, sep='\t')
        Visal().query_length(BEDS , '%s/%s.chimeric.query_length.pdf'%(self.outdir, self.inid))

        print('finish finding soft signal: ' + self.inid)

class SearchType():
    def __init__(self, inbed, outpre, Chrom=False ):
        self._inbed= inbed
        self.outpre= outpre
        self.Chrom = Chrom
        self.chrs=[str(i) for i in range(1,23)] + ['MT','X','Y'] \
                    + ['2x35S-eYGFPuv-T878-p73', '2x35S-LbCpf1-pQD', '380B-eYGFPuv-d11-d15', '380K-eYGFPuv',
                        '380K-eYGFPuv-d123456', '5P2T-pKGW7', 'A10-pg-p221', 'Cas9-U6-sgRNA-pQD', 'd2A-E9t-v4',
                        'HD-T878-UBQ10', 'HG-F2A-pQD', 'Lat52-grim-TE-MC9-prk6-pKGW7', 'Lat52-RG-HTR10-1-GFP-pBGW7',
                        'myb98-genomic-nsc-TOPO', 'pB2CGW', 'pHDzCGW', 'pQD-in', 'pro18-Mal480-d1S-E9t',
                        'SunTag-CRISPRi', 'V7-MC-HG-FA']
        if self.Chrom:
            self.outpre += '.OnlyChrom'

    def _getbeddb(self):
        self.inbed = pd.read_csv( self._inbed, sep='\t')
        if self.inbed.empty:
            self.inbed=pd.DataFrame( columns = self.inbed.columns.tolist() + ['beds', 'raw_order', 'links', 'type'])
        else:
            #self.inbed[['Type', 'beds', 'links']] = pd.DataFrame([['',[],[]]]*self.inbed.shape[0])
            self.inbed['beds'] = self.inbed[['#chrom', 'start', 'end']].apply(lambda x: [tuple(x)], axis=1)
            self.inbed[['start', 'end']]  = self.inbed[['start', 'end']].astype(int)
            self.inbed['#chrom']  = self.inbed['#chrom'].astype(str)
            self.inbed['cigarreg'] = self.inbed.cigarreg.apply(lambda x: tuple([int(i) for i in eval(x)]) )
            self.inbed['raw_order'] = 'raw_order_0'
            self.inbed['links'] = ''
            self.inbed['type']  = 'DROP'
            self.inbed.sort_values(by=['query_name', 'cigarreg', '#chrom', 'start', 'end' ], ascending=[True]*5, inplace=True)
        self.COLs = ['#chrom', 'start', 'end',  'SID', 'length', 'forword', 'query_name',
                     'query_length', 'type', 'raw_order','query_counts', 'cigarstring',  'cigarreg' , 'beds', 'links']
        self.inbed = self.inbed[self.COLs]
        if self.Chrom:
            self.inbed  = self.inbed[ (self.inbed['#chrom'].isin(self.chrs) )]
        return self

    def maxbeddistance(self, _inbed):
        inbed = _inbed.copy().reset_index(drop=True)
        k=[]
        if not inbed.empty:
            for _n, _l in inbed.iterrows() :
                for _m, _j in inbed.loc[_n+1:].iterrows():
                    if (_l['#chrom'] == _j['#chrom']) and (_l['forword'] == _j['forword']):
                        k.append( [_n ,_m, _m - _n, inbed.loc[_n:_m,'length'].sum() ])
        if k:
            k = pd.DataFrame(k, columns=['s','e','d','l'])\
                    .sort_values(by=['l','d'],ascending=[False, False])\
                    .iloc[0,:]
            return inbed.loc[k.s : k.e, :]
        else:
            return inbed

    def dropcigarinclude(self, _inbed, errors=30, dropcigarover=True):
        if (not dropcigarover) or (_inbed.shape[0]<2):
            return _inbed, pd.DataFrame(columns=_inbed.columns)
        else:
            inbed = _inbed.copy().reset_index(drop=True)
            for _n, _l in inbed.iterrows():
                for _m, _k in inbed.loc[_n+1 :,:].iterrows():
                    _s1 = _l.cigarreg[0]
                    _e1 = _l.cigarreg[1]
                    _f1 = _l.forword
                    _s2 = _k.cigarreg[0]
                    _e2 = _k.cigarreg[1]
                    _f2 = _k.forword
                    if (_s1 <= _s2) and (_e1 >= _e2 ) and (_f1==_f2):
                        inbed.loc[_m, 'type'] ='OVER'
                    elif  (_s1 >= _s2 ) and (_e1 <= _e2 ) and (_f1==_f2):
                        inbed.loc[_n, 'type'] ='OVER'
            return inbed[(inbed.type != 'OVER')], inbed[(inbed.type == 'OVER')]

    def mergeHeadTail(self, _inbed, oriant=True, maxhtdistance=10000000):
        inbed = _inbed.reset_index(drop=True).copy()
        if inbed.shape[0] >=2:
            _H = inbed.iloc[0,:]
            _T = inbed.iloc[-1,:]
            _S = min([_H['start'], _T['start']])
            _E = max([_H['end'],   _T['end']])
            _L = _E - _S + 1
            if (_H['#chrom'] == _T['#chrom']) \
                and (_H['forword'] == _T['forword']):
                #if _L > maxhtdistance:
                #    print('warning: the ecDNA breakpoint lenght is large than %s:\n%s'%(_L, inbed))
                _H['start']  = _S
                _H['end']    = _E
                _H['length'] = _L
                _H['beds'].append((_T['#chrom'], _T['start'], _T['end']) )
                _H['cigarreg']   = _H['cigarreg']  + _T['cigarreg']
                _H['cigarstring'] = '%s;%s'%(_H['cigarstring'], _T['cigarstring'] )
                inbed.iloc[0,:] = _H
                inbed = inbed.iloc[:-1,:].reset_index(drop=True)

        inbed['raw_order'] = 'raw_order_' + inbed.index.astype(str)
        inbed['links']= [inbed[['#chrom','start', 'end']].to_numpy().tolist()]*inbed.shape[0]
        return inbed

    def mergeNeighb(self, _inbed, maxdistance=200, maxsimilar=50, maxreg=True, oriant=True, dropcigarover=False):
        SortL = ['query_name', 'cigarreg', '#chrom', 'start', 'end' ]
        inbed = _inbed.sort_values(by=SortL, ascending=[True]*len(SortL))
        inbed['cigarreg'] = inbed['cigarreg'].apply(lambda x: [x])

        if dropcigarover:
            inbed = inbed[(inbed.type!='OVER')]
            overd = inbed[(inbed.type=='OVER')]

        if inbed.shape[0] <2:
            overm = inbed
        else:
            overm = [ inbed.iloc[0,:].copy() ]
            for _n, _l in inbed.iloc[1:,:].iterrows():
                _L = overm[-1].copy()
                R  = _l['#chrom'] == _L['#chrom']
                S1 = (_L['start'] - maxdistance) <= _l['start'] <= (_L['start'] + maxdistance)
                E1 = (_L['end'] - maxdistance)   <= _l['end']   <= (_L['end'] + maxdistance)
                S2 = (_L['start'] - maxsimilar)  <= _l['start'] <= (_L['start'] + maxsimilar)
                E2 = (_L['end'] - maxsimilar)    <= _l['end']   <= (_L['end'] + maxsimilar)
                if R \
                    and ( not(oriant and (_L['forword'] !=_l['forword'])) ) \
                    and ( (S1 and E1) or (S2 or E2) ):
                    if maxreg:
                        _L['start'] = min([_L['start'], _l['start']])
                        _L['end']   = max([_L['end'],   _l['end']]  )
                    else:
                        _L['start'] = max([_L['start'], _l['start']])
                        _L['end']   = min([_L['end'],   _l['end']]  )
                    _L['length']     = _L['end'] - _L['start'] + 1
                    _L['forword']    = ''.join(sorted( set(list(_L['forword']) + list(_l['forword'])) ))
                    _L['cigarreg']   = _L['cigarreg']  + _l['cigarreg']
                    _L['cigarstring'] = '%s;%s'%(_L['cigarstring'], _l['cigarstring'] )
                    _L['beds'].append((_l['#chrom'], _l['start'], _l['end']) )
                    overm[-1] = _L
                else:
                    overm.append(_l)
            overm = pd.concat(overm, axis=1).T

        if dropcigarover:
            overm = pd.concat([overm, overd],axis=0, sort=False)
        return overm

    def typeBase1(self, indf, maxmis=0.1, maxoverlap=300):
        BaseMark, BaseKeep = [],[]

        for (_s, _q,), _g in indf\
                        .sort_values(by=['query_name', 'cigarreg', '#chrom', 'start', 'end' ],
                                    ascending=[True]*5)\
                        .groupby(by=['SID', 'query_name'], sort=False):

            _g = self.maxbeddistance(_g)
            _g, _d = self.dropcigarinclude(_g)
            _g = self.mergeNeighb(_g)

            if _g.shape[0]>=2:
                BreakF = _g.iloc[ 0,:]
                BreakL = _g.iloc[-1,:]
                C  = BreakF['#chrom'] == BreakL['#chrom']
                F  = BreakF.forword == BreakL.forword

                QF = BreakF.cigarreg[ 0][0] <= BreakF.query_length*maxmis
                QL = BreakL.cigarreg[-1][1] >= BreakF.query_length*(1-maxmis)
                S1 = BreakF.start > BreakL.start
                E1 = BreakF.end   > BreakL.end
                X1 = BreakF.start > BreakL.end - maxoverlap

                S2 = BreakF.start < BreakL.start
                E2 = BreakF.end   < BreakL.end
                X2 = BreakF.start < BreakL.end - maxoverlap

                if (C and F and QF and QL):
                    if (S1 and E1 and X1) :
                        _g['type'] ='KEEP'
                        BaseKeep.append(self.mergeHeadTail(_g))
                    else:
                        _g['type'] ='DISS'
            BaseMark.append(_g)
            BaseMark.append(_d)

        if BaseMark:
            BaseMark = pd.concat(BaseMark,axis=0, sort=False)
        else:
            BaseMark = pd.DataFrame(columns=indf.columns)
        if BaseKeep:
            BaseKeep = pd.concat(BaseKeep,axis=0, sort=False)
        else:
            BaseKeep = pd.DataFrame(columns=indf.columns)
        BaseMark.to_csv(self.outpre+'.Mark', sep='\t', index=False)
        BaseKeep.to_csv(self.outpre+'.Keep', sep='\t', index=False)
        return BaseKeep

    def typemark(self, _G, maxmis=0.1, maxoverlap=300):
        _g = self.maxbeddistance(_G)
        _g, _d = self.dropcigarinclude(_g)
        _g = self.mergeNeighb(_g)
        _k = pd.DataFrame()
        if _g.shape[0]>=2:
            BreakF = _g.iloc[ 0,:]
            BreakL = _g.iloc[-1,:]
            C  = BreakF['#chrom'] == BreakL['#chrom']
            F  = BreakF.forword == BreakL.forword

            QF = BreakF.cigarreg[ 0][0] <= BreakF.query_length*maxmis
            QL = BreakL.cigarreg[-1][1] >= BreakF.query_length*(1-maxmis)
            S1 = BreakF.start > BreakL.start
            E1 = BreakF.end   > BreakL.end
            X1 = BreakF.start > BreakL.end - maxoverlap

            S2 = BreakF.start < BreakL.start
            E2 = BreakF.end   < BreakL.end
            X2 = BreakL.start > BreakF.end - maxoverlap

            if (C and F and QF and QL):
                if (S1 and E1 and X1 and BreakF.forword=='+' ) \
                    or (S2 and E2 and X2 and BreakF.forword=='-' ):
                    _g['type'] ='KEEP'
                    _k = self.mergeHeadTail(_g)
                else:
                    _g['type'] ='DISS'
        _d['cigarreg'] = _d['cigarreg'].apply(lambda x:[x])
        _g = pd.concat( [_g,_d], axis=0, sort=False )
        return (_g, _k)

    def typeBase(self, indf, maxmis=0.1, maxoverlap=300):
        SortN = ['query_name', 'cigarreg', '#chrom', 'start', 'end' ]
        BaseCat = Parallel( n_jobs= -1, backend='loky')\
                    ( delayed( self.typemark )(_g)
                        for _, _g in indf\
                            .sort_values(by=SortN, ascending=[True]*len(SortN))\
                            .groupby(by=['SID', 'query_name'], sort=False))
        BaseMark = [ i[0] for i in BaseCat ]
        BaseKeep = [ i[1] for i in BaseCat if not i[1].empty]
        del(BaseCat)

        if BaseMark:
            BaseMark = pd.concat(BaseMark,axis=0, sort=False)
        else:
            BaseMark = pd.DataFrame(columns=indf.columns)
        BaseMark.to_csv(self.outpre+'.Mark', sep='\t', index=False)
        del(BaseMark)

        if BaseKeep:
            BaseKeep = pd.concat(BaseKeep,axis=0, sort=False)
        else:
            BaseKeep = pd.DataFrame(columns=indf.columns)
        BaseKeep.to_csv(self.outpre+'.Keep', sep='\t', index=False)

        return BaseKeep

    def mergeMap(self, _inbed, maxdistance=500, maxreg=True, oriant=False):
        _mergeB= _inbed[['#chrom', 'start', 'end', 'SID', 'query_name', 'forword']].drop_duplicates(keep='first').copy()
        _mergeB[['start', 'end']]  = _mergeB[['start', 'end']].astype(int)
        _mergeB['Beds']       = _mergeB[['#chrom', 'start', 'end']].apply(lambda x: [tuple(x)], axis=1)
        _mergeB['query_name'] = _mergeB['query_name'].apply(lambda x: [x])
        _mergeB['SID']        = _mergeB['SID'].apply(lambda x: [x])
        _mergeB[['#chrom_', 'start_', 'end_']] = _mergeB[['#chrom', 'start', 'end']]

        if oriant:
            _mergeB.sort_values(by=['#chrom', 'forword', 'start', 'end'], ascending=[True]*4, inplace=True)
        else:
            _mergeB.sort_values(by=['#chrom', 'start', 'end', 'forword'], ascending=[True]*4, inplace=True)

        if _mergeB.shape[0] <2:
            mergeE = _mergeB
        else:
            mergeB = [_mergeB.iloc[0,:] ]
            for _n, _l in _mergeB.iloc[1:,:].iterrows():
                _L = mergeB[-1].copy()
                R = _l['#chrom_'] == _L['#chrom_']
                S = (_L['start_'] - maxdistance) <= _l['start_'] <= (_L['start_'] + maxdistance)
                E = (_L['end_'] - maxdistance)   <= _l['end_']   <= (_L['end_'] + maxdistance)
                if (R and S and E) and ( not(oriant and (_L['forword'] !=_l['forword'])) ):
                    if maxreg:
                        _L['start_'] = min([_L['start_'], _l['start_']])
                        _L['end_']   = max([_L['end_'],   _l['end_']]  )
                    else:
                        _L['start_'] = max([_L['start_'], _l['start_']])
                        _L['end_']   = min([_L['end_'],   _l['end_']]  )
                    _L['forword']   = _L['forword'] + _l['forword']
                    _L['query_name']= _L['query_name'] + _l['query_name']
                    _L['SID']       = _L['SID']        + _l['SID']
                    _L['Beds'].append((_l['#chrom_'], _l['start_'], _l['end_']) )
                    mergeB[-1] = _L
                else:
                    mergeB.append(_l)
            mergeE = []
            for _m in mergeB:
                for _n, _r in enumerate(_m.Beds):
                    _x = _m.copy()
                    _x['query_name'] = _m['query_name'][_n]
                    _x['SID']        = _m['SID'][_n]
                    _x['forword']    = _m['forword'][_n]
                    _x[['#chrom', 'start', 'end']]= _r
                    mergeE.append(_x)
            mergeE = pd.concat(mergeE, axis=1, sort=False).T
        return mergeE[['#chrom', 'start', 'end', 'SID', 'query_name', 'forword', '#chrom_', 'start_', 'end_']]

    def mergeLinks(self, _inbed):
        mergeE = self.mergeMap(_inbed)
        inbed  = _inbed.merge(mergeE, on=[ '#chrom', 'start', 'end', 'SID', 'query_name', 'forword'], how='left')
        inbed[['#chrom_raw', 'start_raw', 'end_raw']] = inbed[['#chrom', 'start', 'end']]
        inbed[['#chrom', 'start', 'end']] = inbed[['#chrom_', 'start_', 'end_']]
        inbed['length'] = inbed['end'] - inbed['start'] + 1
        inbed['links_sorted'] = inbed['links']
        inbed['Type']  = 'Mono'
        inbed['new_order']  = 'new_order_0'
        inbed.drop(['#chrom_', 'start_', 'end_'], axis=1,inplace=True)

        uplinks=[]
        for _r, _g in inbed\
                        .sort_values(by=['SID', 'query_name', 'raw_order'], ascending=[True, True, True])\
                        .groupby( by= ['SID', 'query_name'], sort=False):
            _p = _g.copy()
            _p['links'] = [_g[['#chrom','start', 'end']].apply(lambda x: '%s:%s-%s'%tuple(x),axis=1).str.cat(sep=';')]*_g.shape[0]
            if _p.shape[0] >=2:
                _p['Type']  = 'Multi'
                #_S = _p.sort_values(by=['#chrom','start', 'end']).iloc[0].name
                #_p = pd.concat((_p.loc[_S:], _p.loc[:_S]),axis=0, sort=False).iloc[:-1]
                # need update order
                _p = _p.sort_values(by=['#chrom','start', 'end'])
                _p['new_order'] = ['new_order_%s'%i for i in range(_p.shape[0])]
            _p['links_sorted']  = [_p[['#chrom','start', 'end']].apply(lambda x: '%s:%s-%s'%tuple(x),axis=1).str.cat(sep=';')]*_p.shape[0]
            uplinks.append(_p)

        uplinks= pd.concat(uplinks, axis=0, sort=False)
        uplinks.sort_values(by=['links', 'SID', 'query_name', 'raw_order'],inplace=True)

        return uplinks

    def mergeReads(self, _inbed):
        uplinks = self.mergeLinks(_inbed)
        uplinks.to_csv(self.outpre+'.Uplinks', sep='\t', index=False)

        uplinks = uplinks.sort_values(by=['links_sorted', 'new_order'])
        uplinks = uplinks.merge(
                    uplinks.groupby(by=['SID', 'links_sorted'], sort=False)['query_name'].unique().to_frame('support_reads').reset_index(),
                    on=['SID', 'links_sorted'], how='outer')
        uplinks['forword'] = '*'
        uplinks['support_number'] = uplinks.support_reads.apply(lambda x: len(x))
        uplinks['support_reads']  = uplinks.support_reads.apply(lambda x: ';'.join(x))
        COLS = ['#chrom', 'start', 'end', 'SID', 'length', 'forword', 'support_number', 'Type', 'links_sorted', 'new_order', 'support_reads']
        uplinks = uplinks[COLS]\
                    .drop_duplicates(keep='first')\
                    .sort_values(by=['Type','links_sorted', 'new_order', '#chrom', 'start', 'end','SID'])

        Bedregn = uplinks[['#chrom', 'start', 'end', 'support_number', 'length' ]].copy()
        Bedregn = Bedregn.merge(
                        Bedregn.groupby(by=['#chrom', 'start', 'end'])['support_number'].sum().to_frame('support_sum').reset_index(),
                        on=['#chrom', 'start', 'end'], how='outer')\
                    .drop('support_number',axis=1).drop_duplicates(keep='first')
        Bedregn = Utilities().annogene(Bedregn)
        Bedregn.to_csv(self.outpre+'.BedAnnotate', sep='\t', index=False)

        uplinks = uplinks\
                    .merge(Bedregn, on=['#chrom',  'start',  'end', 'length'], how='outer')\
                    .sort_values(by=['Type', 'links_sorted', 'new_order', '#chrom', 'start', 'end',  'SID'])
        uplinks.to_csv(self.outpre+'.UpMerge', sep='\t', index=False)

        uplinks =uplinks.sort_values(by=['Type', 'new_order',  '#chrom', 'start', 'end', 'links_sorted', 'SID'])
        uplinks.to_csv(self.outpre+'.UpMerge_sort', sep='\t', index=False)
        Visal().query_length(uplinks, self.outpre+'.UpMerge.length.pdf', X='length', Dup='', log=True, title='breakpoint length' )

    def EachEcDNA(self):
        print('start searching breakpoin region: ' + self._inbed)
        self._getbeddb()
        if not self.inbed.empty:
            kmerge = self.typeBase( self.inbed )
            #kmerge = pd.read_csv(self.outpre+'.Keep', sep='\t', low_memory=False)
        else:
            kmerge = pd.DataFrame()
            print('cannot find any circle region singal: ' + self._inbed)

        if not kmerge.empty:
            self.mergeReads(kmerge)
        else:
            print('cannot find any circle region type: ' + self._inbed)
        print('finish searching breakpoin region: ' + self._inbed)

    def linkParall(self, _g):
        COLS = ['#chrom', 'start', 'end', 'Type', 'length', 'forword', 'support_number', 'links_sorted',
                    'new_order', 'support_read_num', 'support_ID_num', 'support_IDs']
        _G = _g.copy()
        _K = _g[(_g['new_order'] =='new_order_0')]\
                .groupby(by='SID',sort=False)['query_name'].size().to_frame(name='SUP').reset_index()
        _G['forword']        = '*'
        _G['support_number'] = _g['query_name'].unique().size
        _G['support_reads']  = ';'.join( _g['query_name'].unique() )
        _G['support_IDs']    =  _K['SID'].str.cat(sep=';')
        _G['support_ID_num']   =  _K['SID'].size
        _G['support_read_num'] =  _K['SUP'].astype(str).str.cat(sep=';')
        _G = _G[COLS].drop_duplicates(keep='first')
        return _G

    def mergeSamples(self, _inbed):
        uplinks = self.mergeLinks(_inbed)
        uplinks.to_csv(self.outpre+'.Uplinks', sep='\t', index=False)

        #uplinks = pd.read_csv(self.outpre+'.Uplinks', sep='\t')
        UpLinks = Parallel( n_jobs=-1)( delayed( self.linkParall )(_g) for _l, _g in uplinks.groupby(by='links_sorted', sort=False) )
        UpLinks = pd.concat(UpLinks, axis=0, sort=False)

        Bedregn = UpLinks[['#chrom', 'start', 'end', 'support_number', 'length' ]].copy()
        Bedregn = Bedregn.merge(
                        Bedregn.groupby(by=['#chrom', 'start', 'end'])['support_number'].sum().to_frame('support_sum').reset_index(),
                        on=['#chrom', 'start', 'end'], how='outer')\
                    .drop('support_number',axis=1).drop_duplicates(keep='first')
        Bedregn = Utilities().annogene(Bedregn)
        Bedregn.to_csv(self.outpre+'.BedAnnotate', sep='\t', index=False)

        UpLinks = UpLinks\
                    .merge(Bedregn, on=['#chrom',  'start',  'end', 'length'], how='outer')\
                    .sort_values(by=['Type', 'links_sorted', 'new_order', '#chrom', 'start', 'end'])
        UpLinks.to_csv(self.outpre+'.UpMerge', sep='\t', index=False)

        UpLinks =UpLinks.sort_values(by=['Type', 'new_order',  '#chrom', 'start', 'end', 'links_sorted'])
        UpLinks.to_csv(self.outpre+'.UpMerge_sort', sep='\t', index=False)
        Visal().query_length(UpLinks, self.outpre+'.UpMerge.length.pdf', X='length', Dup='', log=True, title='breakpoint length' )

    def AllEcDNA(self):
        self.mergeSamples(self._inbed)

class ecDNA():
    'The pipeline used for machine learning models'
    def __init__(self, arg, log,  *array, **dicts):
        self.arg = arg
        self.log = log
        self.array  = array
        self.dicts  = dicts

        import multiprocessing
        CORES = multiprocessing.cpu_count()*0.8 if multiprocessing.cpu_count() >8 else 8
        os.environ['NUMEXPR_MAX_THREADS'] = '1000' #str(int(CORES))
        os.environ['PATH'] += ':' + self.arg.bedtools

        #bt.helpers.set_bedtools_path(elf.arg.bedtools)
        import importlib
        importlib.reload(bt)

        self.arg.BamDir     = '%s/%s'%(self.arg.indir, self.arg.infetch)
        self.arg.FetchDir   = '%s/%s'%(self.arg.outdir, self.arg.outfetch)
        self.arg.SearchDir  = '%s/%s'%(self.arg.outdir, self.arg.outsearch)
        self.arg.AllTypeDir = '%s/%s'%(self.arg.outdir, self.arg.outregion)

        os.makedirs(self.arg.FetchDir , exist_ok=True)
        os.makedirs(self.arg.SearchDir, exist_ok=True)
        os.makedirs(self.arg.AllTypeDir   , exist_ok=True)

    def eachwork(self, _l ):
        if self.arg.commands in ['Fetch', 'Auto']:
            SoftFetch(  '%s/%s.sorted.bam'%(self.arg.BamDir, _l.sampleid),
                        _l.sampleid,
                        self.arg.FetchDir).getsoft()

        if self.arg.commands in ['Search', 'Auto']:
            SearchType('{0}/{1}/{1}.chimeric.bed'.format(self.arg.FetchDir , _l.sampleid),
                        '{0}/{1}/{1}.bed'.format(self.arg.SearchDir, _l.sampleid),
                        Chrom=self.arg.chroms).EachEcDNA()

    def allwork(self, _INdf):
        AllMerge = []
        for _n, _l in _INdf.iterrows():
            if self.arg.chroms:
                EachMerge = '{0}/{1}/{1}.bed.OnlyChrom.Keep'.format(self.arg.SearchDir, _l.sampleid)
            else:
                EachMerge = '{0}/{1}/{1}.bed.Keep'.format(self.arg.SearchDir, _l.sampleid)
            if os.path.exists(  EachMerge ):
                AllMerge.append( pd.read_csv(EachMerge, sep='\t', header=0) )
        if AllMerge:
            AllMerge = pd.concat(AllMerge, axis=0,sort=False)
            SearchType(AllMerge, self.arg.AllTypeDir  + '/All.basedon.Keep', Chrom=self.arg.chroms).AllEcDNA()
        else:
            self.log.CI('cannot find the valid files.')

    def FetchW(self):
        SoftFetch(  '%s/%s.sorted.bam'%(self.arg.BamDir, _l.sampleid),
                    _l.sampleid,
                    self.arg.FetchDir).getsoft()
    def SearchW(self):
        SearchType('{0}/{1}/{1}.chimeric.bed'.format(self.arg.FetchDir , _l.sampleid),
                    '{0}/{1}/{1}.bed'.format(self.arg.SearchDir, _l.sampleid),
                    Chrom=self.arg.chroms).EachEcDNA()
    def Region(self):
        AllMerge = []
        for _n, _l in _INdf.iterrows():
            if self.arg.chroms:
                EachMerge = '{0}/{1}/{1}.bed.OnlyChrom.Keep'.format(self.arg.SearchDir, _l.sampleid)
            else:
                EachMerge = '{0}/{1}/{1}.bed.Keep'.format(self.arg.SearchDir, _l.sampleid)
            if os.path.exists(  EachMerge ):
                AllMerge.append( pd.read_csv(EachMerge, sep='\t', header=0) )
        if AllMerge:
            AllMerge = pd.concat(AllMerge, axis=0,sort=False)
            SearchType(AllMerge, self.arg.AllTypeDir  + '/All.basedon.Keep', Chrom=self.arg.chroms).AllEcDNA()
        else:
            self.log.CI('cannot find the valid files.')

    def Pipeline(self):
        if os.path.exists(self.arg.infile):
            INdf = pd.read_csv(self.arg.infile, sep='\t', comment='#')
        else:
            INdf = pd.DataFrame( {'sampleid' : self.arg.infile.split(',')} )

        if self.arg.commands in ['Fetch', 'Search', 'Region', 'Auto']:
            Parallel( n_jobs=self.arg.njob)( delayed( self.eachwork )(_l) for _n, _l in INdf.iterrows() )
            #for _n, _l in INdf.iterrows():
            #    if _l.sampleid in [ 'simulate', 'HEK293T.CCS.lima.BC1--BC1', 'HEK293T.CCS.lima.BC5--BC5']:
            #        self.eachwork(_l)
        if self.arg.commands in ['Region', 'Auto']:
            self.allwork(INdf)

import argparse
def Args():
    parser = argparse.ArgumentParser(
                formatter_class=argparse.RawDescriptionHelpFormatter,
                prefix_chars='-+',
                conflict_handler='resolve',
                description="",
                epilog="")

    parser.add_argument('-V','--version',action ='version',
                version='EcDNA version 0.1')

    subparsers = parser.add_subparsers(dest="commands",
                    help='models help.')
    P_Common = subparsers.add_parser('Common',conflict_handler='resolve', #add_help=False,
                    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                    help='The common parameters used for other models.')
    P_Common.add_argument("-f", "--infile",type=str,
                    help='''the input file or input number split by ",".''')
    P_Common.add_argument("-i", "--indir",type=str,
                    help='''the input directory.''')
    P_Common.add_argument("-o", "--outdir",type=str,default=os.getcwd(),
                    help="output file dir, default=current dir.")
    P_Common.add_argument("-p", "--prefix",type=str,default='',
                    help="output file header, default=None.")
    P_Common.add_argument("-n", "--njob",type=int,default=5,
        help="The maximum number of concurrently running jobs.")
    P_Common.add_argument("-uf", "--outfetch", type=str, default='03.SoftMap',
                    help="out directory for fetch")
    P_Common.add_argument("-if", "--infetch", type=str, default='02.MiniMap',
                    help="input bam directory for fetch ")
    P_Common.add_argument("-us", "--outsearch", type=str, default='03.SoftMap',
                    help="out directory for search")
    P_Common.add_argument("-ur", "--outregion", type=str, default='04.AllRegion',
                    help="out directory for search")
    P_Common.add_argument("-bt", "--bedtools", type=str, default='/share/home/share/software/bedtools2/bin/',
                    help="bedtools path")
    P_Common.add_argument("-ch", "--chroms", action='store_true', default=True,
                    help='''only keep the 23 chromsomes.''')

    P_fetch = subparsers.add_parser('Fetch', conflict_handler='resolve', add_help=False)
    P_Fetch = subparsers.add_parser('Fetch',conflict_handler='resolve',
                    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                    parents=[P_Common,P_fetch],
                    help='fatch reads information from bam file.')

    P_search = subparsers.add_parser('Search', conflict_handler='resolve', add_help=False)
    P_Search = subparsers.add_parser('Search',conflict_handler='resolve',
                    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                    parents=[P_Common,P_search],
                    help='search breakpoint  region from bed file.')

    P_region = subparsers.add_parser('Region', conflict_handler='resolve', add_help=False)
    P_Region = subparsers.add_parser('Region',conflict_handler='resolve',
                    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                    parents=[P_Common,P_region],
                    help='merge all breakpoint region in all samples.')

    P_Autopipe = subparsers.add_parser('Auto', conflict_handler='resolve', prefix_chars='-+',
                    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                    parents=[P_Common, P_fetch, P_search, P_region],
                    help='the auto-processing for all.')
    P_Autopipe.add_argument("+P", "++pipeline",nargs='+',
                    help="the auto-processing: standardization, feature selection, Fitting and/or Prediction.")
    P_Autopipe.add_argument('+M','++MODEL' , nargs='+', type=str, default=['Standard'],
                    help='''Chose more the one models from Standard, Fselect,Fitting and Predict used for DIY pipline.''')
    args  = parser.parse_args()
    return args

import logging
class DispatchingFormatter:
    def __init__(self, formatters, default_formatter):
        self._formatters = formatters
        self._default_formatter = default_formatter

    def format(self, record):
        formatter = self._formatters.get(record.name, self._default_formatter)
        return formatter.format(record)

class Logger:
    level_dict = {
        'NOTSET'  : logging.NOTSET,
        'DEBUG'   : logging.DEBUG,
        'INFO'    : logging.INFO,
        'WARNING' : logging.WARNING,
        'ERROR'   : logging.ERROR,
        'CRITICAL': logging.CRITICAL,
    }

    ChangeFrom = DispatchingFormatter(
            { 'c' : logging.Formatter( '[%(asctime)s] [%(levelname)-4s]: %(message)s', '%Y-%m-%d %H:%M:%S'),
              'p' : logging.Formatter( '[%(levelname)-4s]: %(message)s'),
              'n' : logging.Formatter( '%(message)s' ),
            },
            logging.Formatter('%(message)s')
     )

    def __init__(self, outpath, filemode='w',  clevel = 'INFO', Flevel = 'INFO'):

        logging.basicConfig(
            level    = Logger.level_dict[clevel] ,
            format   = '[%(asctime)s] [%(levelname)-4s]: %(message)s',
            datefmt  = '%Y-%m-%d %H:%M:%S',
            filename = None,
        )

        File = logging.FileHandler(outpath,  mode= filemode)
        File.setLevel(Logger.level_dict[Flevel])
        File.setFormatter(Logger.ChangeFrom)
        logging.getLogger().addHandler(File)

        self.R = logging
        self.C = logging.getLogger('c')
        self.P = logging.getLogger('p')
        self.N = logging.getLogger('n')
        self.CI = logging.getLogger('c').info
        self.NI = logging.getLogger('n').info
        self.CW = logging.getLogger('c').warning
        self.NW = logging.getLogger('n').warning

import os
import time
import traceback
def Commands():
    info ='''
***********************************************************
* Author : Zhou Wei                                       *
* Date   : %s                       *
* E-mail : welljoea@gmail.com                             *
* You are using The scripted by Zhou Wei.                 *
* If you find some bugs, please email to me.              *
* Please let me know and acknowledge in your publication. *
* Sincerely                                               *
* Best wishes!                                            *
***********************************************************
'''%(time.strftime("%a %b %d %H:%M:%S %Y", time.localtime()))

    args = Args()
    os.makedirs( args.outdir, exist_ok=True)
    Log = Logger( '%s/%s_log.log'%(args.outdir, args.commands) )

    Log.NI(info.strip())
    Log.NI("The argument you have set as follows:".center(59, '*'))
    for i,k in enumerate(vars(args),start=1):
        Log.NI('**%s|%-13s: %s'%(str(i).zfill(2), k, str(getattr(args, k))) )
    Log.NI(59 * '*')

    try:
        ecDNA(args, Log).Pipeline()
        Log.CI('Success!!!')
    except Exception:
        Log.CW('Failed!!!')
        traceback.print_exc()
    finally:
        Log.CI('You can check your progress in log file.')
Commands()


from meetup.models import Group
from meetup.mixins import SetErrorMixin
from meetup.custom_form import RequestModelForm
from django.forms.util import ErrorList
from django.forms import widgets

class GroupForm(RequestModelForm, SetErrorMixin):
    class Meta:
        model = Group
        fields = ['name', 'home', 'members', 'description']
        widgets = {
            'description': widgets.Textarea
        }

    def clean(self):
        if self.cleaned_data.get('name', None):
            # Make sure the creator doesn't already have a group with this name
            existing_groups = Group.objects.filter(creator__id=self.request.user.id,
                name=self.cleaned_data['name'])

            if getattr(self.instance, 'pk', None):
                existing_groups = existing_groups.exclude(pk=self.instance.pk)

        # Make sure that the creator is himself a member of the group.
        if self.cleaned_data.get('members', None):
            if self.request.user not in self.cleaned_data['members']:
                errors = self._errors.setdefault("members", ErrorList())
                errors.append(u'You have to add yourself as a member of the group')

        return self.cleaned_data


# Recursive Circus
# Advent of Code 2017 - Day 7
#
# Part 1:
# The programs form a tree. All we need to do when parsing is note
# each program's parent. When we finish building the full tree, we can
# select a program at random and scale the tree back down to the bottom
# program
#
# Part 2:
# To find the unbalanced program, we recursively check if each program's
# plate is balanced correctly. If not, we calculate the fixed weight and then
# short-circuit our way back up to the root of the recursion.

from collections import defaultdict
import re

class Program:
    def __init__(self):
        self.weight = 0
        self.parent = None
        self.children = []

def build_tree(programs):
    tree = defaultdict(Program)
    for program in programs:
        name, weight, children = program[0], int(program[1]), program[2:]
        tree[name].weight = weight
        for c in filter(lambda c: len(c) > 0, children):
            tree[c].parent = name
            tree[name].children.append(c)
    return tree

def get_bottom_program(tree):
    # Just grab any program
    program = list(tree)[0]
    while tree[program].parent:
        program = tree[program].parent
    return program

def correct_out_of_balance_program(tree, program):
    subprogram_weights = defaultdict(list)
    for c in tree[program].children:
        subprogram_weight, weight_fix = correct_out_of_balance_program(tree, c)
        if weight_fix is not None:
            return None, weight_fix
        subprogram_weights[subprogram_weight].append(c)

    # Check if all subprograms are balanced. If not, calculate the fixed weight.
    program_weight = tree[program].weight
    weight_fix = None
    if len(subprogram_weights) == 2:
        p1, p2 = tuple(subprogram_weights.items())
        weight_difference = abs(p1[0] - p2[0])
        imbalanced_child = p1[1] if len(p1[1]) == 1 else p2[1]
        weight_fix = tree[imbalanced_child[0]].weight - weight_difference
    elif len(subprogram_weights) == 1:
        subprogram_weight, subprograms = tuple(subprogram_weights.items())[0]
        program_weight += subprogram_weight * len(subprograms)
    return program_weight, weight_fix

if __name__ == "__main__":
    with open("inputs/recursive_circus.txt") as f:
        programs = [re.split(" \(|\) -> |\)|, ", p.strip()) for p in f.readlines()]
    tree = build_tree(programs)

    # Part 1
    bottom_program = get_bottom_program(tree)
    print("The bottom program is {}".format(bottom_program))

    # Part 2
    _, weight_fix = correct_out_of_balance_program(tree, bottom_program)
    print("The correct weight of the out of balance program is {}".format(weight_fix))


# chat/consumers.py
from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer
import json
from django.contrib.auth.models import User
from requests import request

from chat.models import Message
from user.redis import Redis
# from user.views import ajax_token
from user.decorators import login_decorator

red=Redis()

class ChatConsumer(WebsocketConsumer):
    """
     this function is used for connecting to the websocket
    """

    def connect(self):
        """
        :return: will connect the user with the websocket
        """
        self.room_name = self.scope['url_route']['kwargs']['room_name']
        self.room_group_name = 'chat_%s' % self.room_name

        # Join room group
        async_to_sync(self.channel_layer.group_add)(
            self.room_group_name,
            self.channel_name
        )
        self.accept()
        return self.room_name

    def disconnect(self, close_code):
        """
        :param close_code: if websocket is disconnected then it will give error
        :return: will disconnect user
        """
        # Leave room group
        async_to_sync(self.channel_layer.group_discard)(
            self.room_group_name,
            self.channel_name
        )

    # Receive message from WebSocket

    def receive(self, text_data):
        """
        :param text_data: data is passed
        :return: will save the message
        """
        text_data_json = json.loads(text_data)
        message = text_data_json['message']

    # Send message to room group
        async_to_sync(self.channel_layer.group_send)(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message': message
            }
        )

    # Receive message from room group

    def chat_message(self, event):
        """
        :param event: messages receive will be checked
        :return: will dispatch chat messages
        """
        room=self.room_name
        message = event['message']

        mess = Message.objects.create(messages=message,indentifier_message_number=room)
        # Message.objects.all().filter(room)
        # json.dumps(mess)
        self.send(text_data=json.dumps({

            'message': message
        }))

    def rating(self):
        pass


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Artist',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=80)),
            ],
            options={
                'ordering': ('name',),
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Fan',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('user', models.OneToOneField(related_name=b'profile', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Favorite',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('rank', models.IntegerField()),
                ('created', models.DateTimeField(auto_now_add=True)),
                ('modified', models.DateTimeField(auto_now=True)),
                ('is_archived', models.BooleanField(default=False)),
                ('artist', models.ForeignKey(to='artists.Artist')),
                ('fan', models.ForeignKey(related_name=b'favorites', to='artists.Fan')),
            ],
            options={
                'ordering': ('fan', 'artist', 'rank'),
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='FavoriteHistory',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('rank', models.IntegerField()),
                ('date', models.DateTimeField(auto_now=True)),
                ('favorite', models.ForeignKey(related_name=b'history', to='artists.Favorite')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Genre',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=80)),
                ('description', models.CharField(max_length=5000)),
                ('parent', models.ForeignKey(to='artists.Genre', null=True)),
            ],
            options={
                'ordering': ('name',),
            },
            bases=(models.Model,),
        ),
        migrations.AddField(
            model_name='artist',
            name='genres',
            field=models.ManyToManyField(to='artists.Genre', null=True),
            preserve_default=True,
        ),
    ]


'''
TODO: train the model
        This model takes too much memory and time to train.
        I will train the model when I have time or more computational power...
'''

import itertools

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision.utils import save_image

from .model import Generator, Discriminator

from dataset import DanbooruAutoPair, to_loader

def train(
    epochs,
    dataset,
    G_A2B, G_B2A,
    local_DA, local_DB,
    global_DA, global_DB,
    optimizer_G,
    optimizer_D,
    L1Loss,
    MSELoss,
    BCELoss,
    device,
    verbose_interval=1000,
    save_interval=1000
):

    Tensor = torch.FloatTensor
    global_patch = (1,  6,  6)
    local_patch  = (1, 30, 30)

    batches_done = 0

    for epoch in range(epochs):
        for index, (A, B) in enumerate(dataset):
            A = A.type(Tensor).to(device)
            B  = B.type(Tensor).to(device)

            # label smoothing
            local_real = ((1.0 - 0.7) * torch.rand(A.size(0), *local_patch) + 0.7).to(device)
            local_fake = ((0.3 - 0.0) * torch.rand(A.size(0), *local_patch) + 0.0).to(device)
            global_real = ((1.0 - 0.7) * torch.rand(A.size(0), *global_patch) + 0.7).to(device)
            global_fake = ((0.3 - 0.0) * torch.rand(A.size(0), *global_patch) + 0.0).to(device)

            '''
            Discriminator
            '''

            # gen image
            A2B, _ = G_A2B(A)
            B2A, _ = G_B2A(B)

            # discriminate
            # real
            A_local,  A_local_cam  = local_DA(A)
            A_global, A_global_cam = global_DA(A)
            B_local,  B_local_cam  = local_DB(B)
            B_global, B_global_cam = global_DB(B)
            # fake
            B2A_local,  B2A_local_cam  = local_DA(B2A)
            B2A_global, B2A_global_cam = global_DA(B2A)
            A2B_local,  A2B_local_cam  = local_DB(A2B)
            A2B_global, A2B_global_cam = global_DB(A2B)

            # cam label
            cam_local_real = ((1.0 - 0.7) * torch.rand(*A_local_cam.size()) + 0.7).to(device)
            cam_local_fake = ((0.3 - 0.0) * torch.rand(*A_local_cam.size()) + 0.0).to(device)
            cam_global_real = ((1.0 - 0.7) * torch.rand(*A_global_cam.size()) + 0.7).to(device)
            cam_global_fake = ((0.3 - 0.0) * torch.rand(*A_global_cam.size()) + 0.0).to(device)

            # losses
            A_local_loss      = MSELoss(A_local,      local_real)      + MSELoss(B2A_local,      local_fake)
            A_local_cam_loss  = MSELoss(A_local_cam,  cam_local_real)  + MSELoss(B2A_local_cam,  cam_local_fake)
            A_global_loss     = MSELoss(A_global,     global_real)     + MSELoss(B2A_global,     global_fake)
            A_global_cam_loss = MSELoss(A_global_cam, cam_global_real) + MSELoss(B2A_global_cam, cam_global_fake)
            B_local_loss      = MSELoss(B_local,      local_real)      + MSELoss(A2B_local,      local_fake)
            B_local_cam_loss  = MSELoss(B_local_cam,  cam_local_real)  + MSELoss(A2B_local_cam,  cam_local_fake)
            B_global_loss     = MSELoss(B_global,     global_real)     + MSELoss(A2B_global,     global_fake)
            B_global_cam_loss = MSELoss(B_global_cam, cam_global_real) + MSELoss(A2B_global_cam, cam_global_fake)

            A_loss = A_local_loss + A_local_cam_loss + A_global_loss + A_global_cam_loss
            B_loss = B_local_loss + B_local_cam_loss + B_global_loss + B_global_cam_loss
            D_loss = A_loss + B_loss

            optimizer_D.zero_grad()
            D_loss.backward()
            optimizer_D.step()

            '''
            Generator
            '''

            # gen image
            A2B, A2B_cam = G_A2B(A)
            B2A, B2A_cam = G_B2A(B)

            # re-gen image
            A2B2A, _ = G_B2A(A2B)
            B2A2B, _ = G_A2B(B2A)

            # identity
            A2A, A2A_cam = G_B2A(A)
            B2B, B2B_cam = G_A2B(B)

            # fool discriminator
            B2A_local,  B2A_local_cam  = local_DA(B2A)
            B2A_global, B2A_global_cam = global_DA(B2A)
            A2B_local,  A2B_local_cam  = local_DB(A2B)
            A2B_global, A2B_global_cam = global_DB(A2B)

            # generator cam label
            cam_real = ((1.0 - 0.7) * torch.rand(*A2B_cam.size()) + 0.7).to(device)
            cam_fake = ((0.3 - 0.0) * torch.rand(*A2B_cam.size()) + 0.0).to(device)

            # losses
            # adversarial
            A_local_loss      = MSELoss(B2A_local,      local_real)
            A_local_cam_loss  = MSELoss(B2A_local_cam,  cam_local_real)
            A_global_loss     = MSELoss(B2A_global,     global_real)
            A_global_cam_loss = MSELoss(B2A_global_cam, cam_global_real)
            B_local_loss      = MSELoss(A2B_local,      local_real)
            B_local_cam_loss  = MSELoss(A2B_local_cam,  cam_local_real)
            B_global_loss     = MSELoss(A2B_global,     global_real)
            B_global_cam_loss = MSELoss(A2B_global_cam, cam_global_real)

            # cycle
            A2B2A_loss = L1Loss(A2B2A, A)
            B2A2B_loss = L1Loss(B2A2B, B)

            # identity
            A2A_loss = L1Loss(A2A, A)
            B2B_loss = L1Loss(B2B, B)

            A2A_cam_loss = BCELoss(B2A_cam, cam_real) + BCELoss(A2A_cam, cam_fake)
            B2B_cam_loss = BCELoss(A2B_cam, cam_real) + BCELoss(B2B_cam, cam_fake)

            A_loss = A_local_loss + A_local_cam_loss + A_global_loss + A_global_cam_loss + 10 * A2B2A_loss + 10 * A2A_loss + 1000 * A2A_cam_loss
            B_loss = B_local_loss + B_local_cam_loss + B_global_loss + B_global_cam_loss + 10 * B2A2B_loss + 10 * B2B_loss + 1000 * B2B_cam_loss
            G_loss = A_loss + B_loss

            optimizer_G.zero_grad()
            G_loss.backward()
            optimizer_G.step()

            if batches_done % verbose_interval == 0:
                print('{:6} G Loss : {:.5f} D Loss : {:.5f}'.format(batches_done, G_loss.item(), D_loss.item()))

            if batches_done % save_interval == 0:
                image_grid = make_grid(A, A2B, B, n_image=999)
                save_image(image_grid, './implementations/UGATIT/result/{}.png'.format(batches_done), nrow=9, normalize=True)

            batches_done += 1

def make_grid(A, A2B, B, n_image=999):
    b = A.size(0)
    split_A = A.chunk(b)
    split_A2B = A2B.chunk(b)
    split_B = B.chunk(b)

    images = []
    for index, (a, a2b, b) in enumerate(zip(split_A, split_A2B, split_B)):
        images += [a, a2b, b]
        if index == n_image:
            break

    image_grid = torch.cat(images, dim=0)
    return image_grid

from PIL import ImageFilter

class SoftMozaic(object):
    def __init__(self, linear_scale, radius):
        self.linear_scale = linear_scale
        self.radius = radius

    def __call__(self, sample):
        sample = sample.filter(ImageFilter.GaussianBlur(self.radius))
        return sample.resize([int(x / self.linear_scale) for x in sample.size]).resize(sample.size)

def main(parser):
    import warnings
    warnings.warn('Reimplement. This is too old!')
    exit()

    import torchvision.transforms as T

    epochs = 5

    lr = 0.0001
    betas = (0.5, 0.999)
    batch_size = 5

    pair_trasform = SoftMozaic(1.3, 0.4)

    dataset = DanbooruAutoPair(image_size, pair_trasform)
    dataset.co_transform_head = T.Compose([
        T.CenterCrop((350, 350)),
        T.Resize(128)
    ])
    dataset = to_loader(dataset, batch_size)

    device = torch.device('cpu')
    # device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    G_A2B = Generator(n_blocks=6)
    G_B2A = Generator(n_blocks=6)
    local_DA = Discriminator(n_layers=3)
    local_DB = Discriminator(n_layers=3)
    global_DA = Discriminator(n_layers=5)
    global_DB = Discriminator(n_layers=5)

    G_A2B.to(device)
    G_B2A.to(device)
    local_DA.to(device)
    local_DB.to(device)
    global_DA.to(device)
    global_DB.to(device)

    optimizer_G = optim.Adam(itertools.chain(G_A2B.parameters(), G_B2A.parameters()), lr=lr, betas=betas)
    optimizer_D = optim.Adam(itertools.chain(local_DA.parameters(), local_DB.parameters(), global_DA.parameters(), global_DB.parameters()), lr=lr, betas=betas)

    MSELoss = nn.MSELoss()
    L1Loss = nn.L1Loss()
    BCELoss = nn.BCEWithLogitsLoss()

    torch.cuda.empty_cache()

    train(
        epochs, dataset, G_A2B, G_B2A,
        local_DA, local_DB, global_DA, global_DB,
        optimizer_G, optimizer_D,
        L1Loss, MSELoss, BCELoss,
        device, 1000, 1000
    )


"""iHub URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, re_path
import iHub_site.views as site_view

urlpatterns = [
    path('admin/', admin.site.urls),
    path('login/', site_view.login),
    path('logout/', site_view.logout),
    path('my/', site_view.my),
    path('register/', site_view.register),
    path('start_plan/', site_view.start_plan),
    path('plans/', site_view.plans),
    path('take_part/', site_view.take_part),
    path('my_plans/', site_view.my_plans),
    path('cancel_plan/', site_view.cancel_plan),
    path('quit_plan/', site_view.quit_plan),
    path('carpool_index/', site_view.carpool_index),

    path('shop_home/', site_view.order_bf),
    re_path(r'^shop_home/(\w+)/$', site_view.shop_menu),
    path('shop_cart/', site_view.shop_cart),
    path('shop_my/', site_view.shop_my),
]


import argparse
from ast import parse
from feeder.skeleton_helper import random_move
import os
import random
import shutil
import time
import warnings
import logging

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
from tensorboardX import SummaryWriter

from model.SF_GCN import SF_GCN
from feeder.skeleton_dataset import Skeleton
from utils import *

parser = argparse.ArgumentParser(description="Pytorch SF_GCN Training")
parser.add_argument('--gpu_idx', default=[0,1,2,3], type=list, help='list of used gpus index')
parser.add_argument('--base_lr', default=0.1, type=float, help='base learning_rate')
parser.add_argument('--batch_size', default=64, type=int, help='number of batch size')
parser.add_argument('--epoch', default=100, type=int, help='number of training epoch')

parser.add_argument('--train_data_path', default='data/NTU-RGB-D/xsub/train_data.npy', type=str ,help='path of training original data file')
parser.add_argument('--train_label_path', default='data/NTU-RGB-D/xsub/train_label.pkl', type=str ,help='path of training original label file')
parser.add_argument('--val_data_path', default='data/NTU-RGB-D/xsub/val_data.npy', type=str ,help='path of valing original data file')
parser.add_argument('--val_label_path', default='data/NTU-RGB-D/xsub/val_label.pkl', type=str ,help='path of valing original label file')

parser.add_argument('--alpha', default=8, type=int, help='alpha times temporal resolution')
parser.add_argument('--beta_inv', default=8, type=int, help='beta times channel dimension')

parser.add_argument('--save_model', default='checkpoints/best_model.pth', type=str, help='model save path')
parser.add_argument('--log_dir', default='log/{}.log'.format(time.strftime(r"%Y-%m-%d-%H_%M_%S", time.localtime())), type=str, help='path of log file')
parser.add_argument('--print-freq', default=10, type=int, metavar='N', help='print frequency (default: 10)')
parser.add_argument('--seed', default=int(time.time() * 256), type=int, help='seed for initializing training. ')
args = parser.parse_args()

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv1d') != -1:
        m.weight.data.normal_(0.0, 0.02)
        if m.bias is not None:
            m.bias.data.fill_(0)
    elif classname.find('Conv2d') != -1:
        m.weight.data.normal_(0.0, 0.02)
        if m.bias is not None:
            m.bias.data.fill_(0)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

def main(args):
    best_acc = 0.

    train_dataset = Skeleton(args,
                            args.train_data_path,
                            args.train_label_path,
                            random_choose=True,
                            random_move=True,
                            window_size=150,
                            debug=True
                            )
    val_dataset = Skeleton(args,
                            args.val_data_path,
                            args.val_label_path,
                            random_choose=True,
                            random_move=True,
                            window_size=150,
                            debug=True
                            )
    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size * len(args.gpu_idx), shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size * len(args.gpu_idx), shuffle=True)

    # initialize model
    model = SF_GCN(
                    num_pathways=2,
                    in_channels=3,
                    max_frame=150,
                    num_class=80,
                    graph_args={'layout':'ntu-rgb+d', 'strategy':'spatial'},
                    beta_inv=args.beta_inv
                    )
    model_name = model._get_name()
    model.apply(weights_init)
    model = nn.DataParallel(model, device_ids=args.gpu_idx)
    optimizer = torch.optim.SGD(model.parameters(), lr=args.base_lr, momentum=0.9, weight_decay=1e-7)
    criterion = nn.CrossEntropyLoss()

    # get logger
    logger = get_logger(args)
    logger.info('Parameter:{ Batch_size:' + str(args.batch_size) +
                '\tbase_lr:' + str(args.base_lr) +
                '\tModel:' + model_name + '}')

    # tesorboard writer
    writer = SummaryWriter()

    for epoch in range(args.epoch):
        print('Train Phase:')
        train(args, epoch, train_dataloader, model, criterion, optimizer, logger, writer)
        print('Val Phase:')
        acc1, acc5 = validation(model, val_dataloader, criterion, logger)

        # remember best acc@1 and save checkpoint
        is_best = acc1 > best_acc
        best_acc = max(acc1, best_acc)

        if is_best:
            torch.save(
                {
                    'epoch': epoch + 1,
                    'state_dict': model.module.state_dict(),
                    'best_acc1': best_acc,
                }, args.save_model
                )

    print("Train finished! to see train infomation in log/, to see train result in runs/ by using tensorboard command")

def train(args, epoch, train_dataloader, model, criterion, optimizer, logger, writer):
    model.train()

    for iter, traindata in enumerate(train_dataloader):
        start = time.time()
        data, label = traindata
        if isinstance(data, (list,)):
            for i in range(len(data)):
                data[i] = data[i].cuda()
        else:
            data = data.cuda()
        label.cuda()

        output = model(data)
        loss = criterion(output, label)
        adjust_lr(optimizer, epoch, args.base_lr)

        acc1 = accuracytop1(output, label, (1, ))
        acc5 = accuracytop5(output, label, (5, ))

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        end = time.time()
        if iter % args.print_freq == 0:
            logger.info('epoch:{} [{}/{}] loss:{:.4f} | acc@1:{:.4f} | acc@5:{:.4f} | time:{:.4f}s'.format(epoch, iter, len(train_dataloader), loss, acc1, acc5, end-start))

        writer.add_scalar('Loss', loss, epoch)
        writer.add_scalar('Acc@1', acc1, epoch)
        writer.add_scalar('Acc@5', acc5, epoch)
        writer.flush()
        writer.close()

def validation(model, val_dataloader, criterion, logger):
    model.eval()

    with torch.no_grad():
        for iter, valdata in enumerate(val_dataloader):
            data, label = valdata
            if isinstance(data, (list,)):
                for i in range(len(data)):
                    data[i] = data[i].cuda()
            else:
                data = data.cuda()
            label.cuda()
            output = model(data)
            loss = criterion(output, label)
            acc1 = accuracytop1(output, label, (1, ))
            acc5 = accuracytop5(output, label, (5, ))

            if iter % args.print_freq == 0:
                logger.info('[{}/{}] | loss:{:.4f} | acc@1:{:.4f} | acc@5:{:.4f}'.format(iter, len(val_dataloader), loss, acc1, acc5))

if __name__ == '__main__':
    main(args)


import requests
from datetime import date, timedelta

def get_trending_repositories(top_size):
    one_week_ago_date = date.today() - timedelta(days=7)
    git_hub_url = 'https://api.github.com/search/repositories'
    search_params = {
        'q': 'created:>{}'.format(one_week_ago_date),
        'page': '1',
        'per_page': top_size,
        'sort': 'stars',
        'order': 'desc'
    }
    response = requests.get(git_hub_url, search_params)
    repository_list = response.json()['items']
    return repository_list

def get_open_issues_amount(repo_owner, repo_name):
    issues_url = "https://api.github.com/repos/{owner}/{repo}/issues".format(
        owner=repo_owner, repo=repo_name)
    response = requests.get(issues_url)
    return len(response.json())

if __name__ == '__main__':
    top_count = 20
    trending_repositories = get_trending_repositories(top_count)
    for repo in trending_repositories:
        repo_open_issues = get_open_issues_amount(repo['owner']['login'],
                                                  repo['name'])
        print('--------------------------------------------------------------')
        print(
            "Repository owner: {owner}\n"
            "Repository name: {name}\n"
            "URL: {repo_url}\n"
            "Open issues: {issues}".format(
                owner=repo['owner']['login'],
                name=repo['name'],
                repo_url='{owners_html_url}/{repo_name}'.format(
                    owners_html_url=repo['owner']['html_url'],
                    repo_name=repo['name']
                ),
                issues=repo_open_issues
            )
        )


import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm
import pandas as pd
import seaborn as sns
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn import neighbors, datasets
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_blobs
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import GradientBoostingClassifier
from scipy.spatial import ConvexHull
from tqdm import tqdm
import random
import pickle
from sklearn import tree
from sklearn.tree import export_graphviz
from joblib import dump, load

DEBUG=False

def pickleModel(models,threshold=0.87,filename='model.pkl',verbose=True):
    '''
        save trained model set
    '''
    MODELS=[]
    for key,mds in models.items():
        if key >= threshold:
            mds_=[i[0] for i in mds]
            MODELS.extend(mds_)
    if verbose:
        print("number of models (tests):", len(MODELS))
        FS=getCoverage(MODELS,verbose=True)
        print("Item Use Fraction:", FS.size/(len(MODELS)+0.0))
    dump(MODELS, filename)
    return

def loadModel(filename):
    '''
        load models
    '''
    return load(filename)

def drawTrees(model,PREF='../'):
    '''
        draw the estimators (trees)
        in a single model
    '''
    N=len(model.estimators_)

    for count in range(N):
        estimator = model.estimators_[count]

        export_graphviz(estimator, out_file=PREF+'tree.dot',
                        #feature_names = iris.feature_names,
                        #class_names = iris.target_names,
                        rounded = True, proportion = False,
                        precision = 2, filled = True)

        from subprocess import call
        call(['dot', '-Tpng',  PREF+'tree.dot', '-o', PREF+'tree'+str(count)+'.png', '-Gdpi=600'])
        from IPython.display import Image
        Image(filename = PREF+'tree'+str(count)+'.png')

def getCoverage(model,verbose=True):
    '''
        return how many distinct items (questions)
        are used in the model set.
        This includes the set of questions being
        covered by all forms that may be
        generated by the model set
    '''
    FS=[]
    for m in model:
        for count in range(len(m.estimators_)):
            clf=m.estimators_[count]
            fs=clf.tree_.feature[clf.tree_.feature>0]
            FS=np.array(list(set(np.append(FS,fs))))
    if verbose:
        print("Number of items used: ", FS.size)
    return FS

def getAuc(X,y,test_size=0.25,max_depth=None,n_estimators=100,
           minsplit=4,FPR=[],TPR=[],VERBOSE=False, USE_ONLY=None):
    '''
        get AUC given training data X, with target labels y
    '''
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
    CLASSIFIERS=[DecisionTreeClassifier(max_depth=max_depth, min_samples_split=minsplit),
                RandomForestClassifier(n_estimators=n_estimators,
                                       max_depth=max_depth,min_samples_split=minsplit),
                ExtraTreesClassifier(n_estimators=n_estimators,
                                     max_depth=max_depth,min_samples_split=minsplit),
                AdaBoostClassifier(n_estimators=n_estimators),
                GradientBoostingClassifier(n_estimators=n_estimators,max_depth=max_depth),
                svm.SVC(kernel='rbf',gamma='scale',class_weight='balanced',probability=True)]

    if USE_ONLY is not None:
        if isinstance(USE_ONLY, (list,)):
            CLASSIFIERS=[CLASSIFIERS[i] for i in USE_ONLY]
        if isinstance(USE_ONLY, (int,)):
            CLASSIFIERS=CLASSIFIERS[USE_ONLY]

    for clf in CLASSIFIERS:
        clf.fit(X_train,y_train)
        y_pred=clf.predict_proba(X_test)
        if len(y_pred[0,:]) == 2:
            fpr, tpr, thresholds = metrics.roc_curve(y_test,y_pred[:,1], pos_label=1)
            auc=metrics.auc(fpr, tpr)
            if VERBOSE:
                print(auc)

            FPR=np.append(FPR,fpr)
            TPR=np.append(TPR,tpr)

        else:
            fpr, tpr, thresholds = metrics.roc_curve(y_test,y_pred[:,1], pos_label=1)
            fpr1, tpr1, thresholds1 = metrics.roc_curve(y_test,y_pred[:,2], pos_label=2)
            auc1=metrics.auc(fpr1, tpr1)
            auc=metrics.auc(fpr, tpr)

            if VERBOSE:
                print(auc,auc1)

            FPR=np.append(FPR,fpr)
            TPR=np.append(TPR,tpr)
            FPR=np.append(FPR,fpr1)
            TPR=np.append(TPR,tpr1)

    points=np.array([[a[0],a[1]] for a in zip(FPR,TPR)])
    hull = ConvexHull(points)
    x=np.argsort(points[hull.vertices,:][:,0])
    auc=metrics.auc(points[hull.vertices,:][x,0],points[hull.vertices,:][x,1])
    return auc,CLASSIFIERS

#test model
def getModel(P,X,y,THRESHOLD=0.9):
    '''
        Select only models with minimum AUC
    '''
    Pgood=[model for (auc,model) in zip(P[::2],P[1::2]) if auc > THRESHOLD]
    AUC=[]
    if len(Pgood)==0:
        return Pgood,len(Pgood),0,0,0,[]
    for i in tqdm(range(1000)):
        random_choice=random.randint(0,len(Pgood)-1)
        clf=Pgood[random_choice][0]
        # pretend as if we have not sen any of this data before
        # but we have!
        # need to only use test data here
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8)
        y_pred=clf.predict_proba(X_test)
        fpr, tpr, thresholds = metrics.roc_curve(y_test,y_pred[:,1], pos_label=1)
        auc=metrics.auc(fpr, tpr)
        AUC=np.append(AUC,auc)
    DEPTH=Pgood[0][0].max_depth
    N_ESTIMATORS=Pgood[0][0].n_estimators

    NITEMS=DEPTH*N_ESTIMATORS
    VARIATIONS=len(Pgood)#2*DEPTH*len(Pgood)
    return Pgood,len(Pgood),np.median(AUC),NITEMS,VARIATIONS,AUC

def getSystem(X,y,max_depth=2,n_estimators=3):
    '''
        get model set with training data X and target labels y
        -> calls getAUC, and getModel
    '''
    P1=[]
    for i in tqdm(range(100)):
        #USE_ONLY=2 implies ExtraTreesClassifier is used only
        P1=np.append(P1,getAuc(X,y,minsplit=2,max_depth=max_depth,
                               n_estimators=n_estimators,USE_ONLY=[2]))
    PERF=[]
    DPERF={}
    MODELS={}
    for threshold in np.arange(0.8,0.95,0.01):
        Pgood,nmodels,auc_,NITEMS,VARIATIONS,AUC=getModel(P1,X,y,threshold)
        if len(Pgood) > 0:
            PERF=np.append(PERF,[auc_,NITEMS,VARIATIONS])
            DPERF[VARIATIONS]=AUC
            MODELS[auc_]=Pgood
    PERF=PERF.reshape(int(len(PERF)/3),3)
    return PERF,DPERF,MODELS,NITEMS

def PLOT(Dperf,Nitems,N=1000,dn='',pdfname='perf.pdf'):
    '''
        Plots the achieved AUC along with
        confidence bounds against the
        number of different forms
        generated.
    '''
    NUMQ='No. of Items Per Subject: '+str(Nitems)
    Df=pd.DataFrame(Dperf)
    dfs=Df.std()
    dfm=Df.mean()
    plt.figure(figsize=[8,6])
    dfm.plot(marker='o',color='r',ms=10,markeredgecolor='w',markerfacecolor='k',lw=2)
    (dfm+2.62*(dfs/np.sqrt(N))).plot(ls='--',color='.5')
    (dfm-2.62*(dfs/np.sqrt(N))).plot(ls='--',color='.5')
    plt.xlabel('No. of different question sets')
    plt.ylabel('mean AUC')
    plt.title('AUC vs Test Variation (99% CB)',fontsize=12,fontweight='bold')
    plt.text(0.55,0.9,NUMQ,transform=plt.gca().transAxes,fontweight='bold',
             fontsize=12,bbox=dict(facecolor='k', alpha=0.4),color='w')

    plt.savefig(pdfname,dpi=300,bbox_inches='tight',pad_inches=0,transparent=False)
    return

def chooseForm(modelset,index=None):
    if index is None:
        index=cx.random.randint(0,len(modelset)-1)
    return modelset[index]

def ask(item,B=0,E=5):
    response=random.randint(B,E)
    return {item:response}

def getNodeid(model,nodeid=None,responses=None,ask=ask,DEBUG=DEBUG):
    if responses is None:
        responses={}
    if nodeid is None:
        nodeid=np.zeros(model.n_estimators).astype(int)

    items=[model.estimators_[i].tree_.feature[nodeid[i]]
              for i in range(model.n_estimators)]

    for i in np.arange(len(items)):
        if nodeid[i] < 0:
            continue
        if items[i] not in responses:
            responses.update(ask(items[i]))
        if responses[items[i]] < model.estimators_[i].tree_.threshold[nodeid[i]]:
            nodeid[i]=model.estimators_[i].tree_.children_left[nodeid[i]]
        else:
            nodeid[i]=model.estimators_[i].tree_.children_right[nodeid[i]]

    responses.pop(-2,None)
    if DEBUG:
        print(nodeid)
    return nodeid,responses

def runCAD(model,nfeatures,ask=ask):
    nodeid=np.zeros(model.n_estimators).astype(int)
    responses={}
    while not all(nodeid<0):
        [nodeid,responses]=getNodeid(model,nodeid=nodeid,responses=responses,ask=ask)

    Xs=np.zeros(nfeatures)
    for key in responses.keys():
        Xs[key]=responses[key]
    prd=model.predict_proba(Xs.reshape(1, -1))

    return responses,prd

def chooseForm(modelset,index=None):
    if index is None:
        index=random.randint(0,len(modelset)-1)
    return modelset[index]


import os
import sys

import sh
import click
from flask import current_app

from utils import path_to

@click.command()
def debug():
    sh.sudo.docker.build(
        '--tag', 'codeguild', path_to() or '.',
        _out=sys.stdout, _err=sys.stderr)
    args = [
        '--rm',
        '--interactive',
        '--tty',
        '--publish=80:5000',
        '--volume=/vagrant:/usr/src/app',
        '--name=codeguild'
    ]
    for k, v in current_app.config.iteritems():
        if k in os.environ:
            args += ['--env', '%s=%s' % (k, str(v))]
    args += [
        'codeguild',
        'python', 'wsgi.py']
    try:
        sh.sudo.docker.run(
            *args, _out=sys.stdout, _err=sys.stderr, _tty_in=True)
    finally:
        try:
            sh.sudo.docker.stop('codeguild')
        except:
            pass