from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import argparse
import datetime

import backtrader as bt

class St(bt.Strategy):
    params = dict(
        ma=bt.ind.SMA,
        p1=5,
        p2=15,
        limit=0.005,
        limdays=3,
        limdays2=1000,
        hold=10,
        usetarget=False,  # use order_target_size
        switchp1p2=False,  # switch prices of order1 and order2
        oco1oco2=False,  # False - use order1 as oco for order3, else order2
        do_oco=True,  # use oco or not
    )

    def notify_order(self, order):
        print('{}: Order ref: {} / Type {} / Status {}'.format(
            self.data.datetime.date(0),
            order.ref, 'Buy' * order.isbuy() or 'Sell',
            order.getstatusname()))

        if order.status == order.Completed:
            self.holdstart = len(self)

        if not order.alive() and order.ref in self.orefs:
            self.orefs.remove(order.ref)

    def __init__(self):
        ma1, ma2 = self.p.ma(period=self.p.p1), self.p.ma(period=self.p.p2)
        self.cross = bt.ind.CrossOver(ma1, ma2)

        self.orefs = list()

        if self.p.usetarget:
            print('-' * 5, 'Using order_target_size')
            self._dobuy = self.order_target_size
            self._doclose = self.order_target_size
        else:
            self._dobuy = self.buy
            self._doclose = self.close

    def next(self):
        if self.orefs:
            return  # pending orders do nothing

        if not self.position:
            if self.cross > 0.0:  # crossing up

                p1 = self.data.close[0] * (1.0 - self.p.limit)
                p2 = self.data.close[0] * (1.0 - 2 * 2 * self.p.limit)
                p3 = self.data.close[0] * (1.0 - 3 * 3 * self.p.limit)

                valid1 = datetime.timedelta(self.p.limdays)
                valid2 = valid3 = datetime.timedelta(self.p.limdays2)

                if self.p.switchp1p2:
                    p1, p2 = p2, p1
                    valid1, valid2 = valid2, valid1

                print('valid1 is:', valid1)

                kargs = dict(exectype=bt.Order.Limit)
                kargs[('target' * self.p.usetarget) or 'size'] = 1

                o1 = self._dobuy(price=p1, valid=valid1, **kargs)
                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o1.ref, p1))

                oco2 = o1 if self.p.do_oco else None
                o2 = self._dobuy(price=p2, valid=valid2, oco=oco2, **kargs)

                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o2.ref, p2))

                if self.p.do_oco:
                    oco3 = o1 if not self.p.oco1oco2 else oco2
                else:
                    oco3 = None

                o3 = self._dobuy(price=p3, valid=valid3, oco=oco3, **kargs)

                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o3.ref, p3))

                self.orefs = [o1.ref, o2.ref, o3.ref]

        else:  # in the market
            if (len(self) - self.holdstart) >= self.p.hold:
                self._doclose()

def runstrat(args=None):
    args = parse_args(args)

    cerebro = bt.Cerebro()

    # Data feed kwargs
    kwargs = dict()

    # Parse from/to-date
    dtfmt, tmfmt = '%Y-%m-%d', 'T%H:%M:%S'
    for a, d in ((getattr(args, x), x) for x in ['fromdate', 'todate']):
        if a:
            strpfmt = dtfmt + tmfmt * ('T' in a)
            kwargs[d] = datetime.datetime.strptime(a, strpfmt)

    # Data feed
    data0 = bt.feeds.BacktraderCSVData(dataname=args.data0, **kwargs)
    cerebro.adddata(data0)

    # Broker
    cerebro.broker = bt.brokers.BackBroker(**eval('dict(' + args.broker + ')'))

    # Sizer
    cerebro.addsizer(bt.sizers.FixedSize, **eval('dict(' + args.sizer + ')'))

    # Strategy
    cerebro.addstrategy(St, **eval('dict(' + args.strat + ')'))

    # Execute
    cerebro.run(**eval('dict(' + args.cerebro + ')'))

    if args.plot:  # Plot if requested to
        cerebro.plot(**eval('dict(' + args.plot + ')'))

def parse_args(pargs=None):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description=(
            'Sample Skeleton'
        )
    )

    parser.add_argument('--data0', default='../../datas/2005-2006-day-001.txt',
                        required=False, help='Data to read in')

    # Defaults for dates
    parser.add_argument('--fromdate', required=False, default='',
                        help='Date[time] in YYYY-MM-DD[THH:MM:SS] format')

    parser.add_argument('--todate', required=False, default='',
                        help='Date[time] in YYYY-MM-DD[THH:MM:SS] format')

    parser.add_argument('--cerebro', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--broker', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--sizer', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--strat', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--plot', required=False, default='',
                        nargs='?', const='{}',
                        metavar='kwargs', help='kwargs in key=value format')

    return parser.parse_args(pargs)

if __name__ == '__main__':
    runstrat()


import math
import scipy.stats as stats
import pandas as pd
import numpy as np
import backtrader as bt
import datetime
import psycopg2
import json
import os

class TemplateStrategy(bt.Strategy):

    params = {
        'trade_freq': None,
        'indicator': None,
    }

    def __init__(self):
        self.dataclose = self.datas[0].close
        self.order = None
        self.buyprice = None

        self.indicator = self.params.indicator()
        self.days = 0

    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))

    def notify_order(self, order):
        if order.status in [order.Submitted, order.Accepted]:
            return

        if order.status in [order.Completed]:
            if order.isbuy():
                self.log(
                    'BUY EXECUTED, PRICE: %.2f, Cost: %.2f, Comm %.2f' % (
                        order.executed.price,
                        order.executed.value,
                        order.executed.comm,
                    )
                )
            else:
                self.log(
                    'SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (
                        order.executed.price,
                        order.executed.value,
                        order.executed.comm,
                    )
                )
            self.bar_executed = len(self)

        self.order = None

    def notify_trade(self, trade):
        if not trade.isclosed:
            return

        self.log(
            'OPERATION PROFIT, GROSS %.2f, NET %.2f' % (
                trade.pnl,
                trade.pnlcomm,
            )
        )

    def next(self):
        self.days += 1

        if self.order:
            return

        if self.days > self.params.trade_freq - 1:
            self.days = 0
            if not self.position:
                if self.indicator > 0:
                    self.log('BUY CREATE, %.2f' % self.dataclose[0])
                    self.order = self.buy()
            elif self.indicator < 0:
                self.log('SELL CREATE, %.2f' % self.dataclose[0])
                self.order = self.sell()

class SimpleMovingAverageIndicator(bt.Indicator):
    lines = ('sma_trend',)
    params = {
        'long_ma': 30,
        'short_ma': 10,
    }

    def __init__(self):
        self.addminperiod(max(self.params.long_ma, self.params.short_ma))
        self.long_period = max(self.params.long_ma, self.params.short_ma)
        self.short_period = min(self.params.long_ma, self.params.short_ma)

    def next(self):
        sma_long = np.mean(self.data.get(size = self.long_period))
        sma_short = np.mean(self.data.get(size = self.short_period))
        self.lines.sma_trend[0] = sma_short - sma_long

if __name__ == '__main__':
    # Connect db
    conn = psycopg2.connect('dbname=securities user=postgres')
    cur = conn.cursor()

    # Query and format data
    cur.execute('SELECT * FROM prices WHERE symbol = \'SPY\' ORDER BY datetime ASC')
    data = pd.DataFrame(cur.fetchall(), columns = ['symbol', 'date', 'open', 'high', 'low', 'close'])
    data.loc[:, 'date'] = pd.to_datetime(data['date'])
    data = data.set_index('date')

    # Create backtest engine and add data/strategies
    cerebro = bt.Cerebro()
    cerebro_data = bt.feeds.PandasData(dataname = data, datetime = -1)
    cerebro.adddata(cerebro_data)
    cerebro.addstrategy(SimpleMovingAverageStrategy, trade_freq = 10, indicator = SimpleMovingAverageIndicator)
    cerebro.addsizer(bt.sizers.FixedSize, stake=10)

    # Run backtest and evaluate strategies
    print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.run()
    print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())

    # Close database connections
    cur.close()
    conn.close()


import random
import re
from copy import deepcopy

import torch
from myModel import MyModel
import torchvision
from matplotlib import pyplot
from torch import nn
from torch.utils.data.dataloader import DataLoader
from torch.utils.tensorboard import SummaryWriter

from Loader import *

def outWrongMatCsv(WrongMat:np.ndarray,label:dict):
  label = [chr(a) for a in label.values()]
  sumOfRow = np.sum(WrongMat,axis=1)
  WrongMat = np.where(sumOfRow == 0,0,WrongMat/sumOfRow)
  with open("WrongMat.csv","w",newline="") as WMfp:
    writer = csv.writer(WMfp)
    writer.writerow([""]+label)
    for i in range(len(label)):
      writer.writerow([label[i]]+WrongMat[i].tolist())

def charTest(setName = "Test"):
  #参数
  miniBatch = 100   #批大小
  #部署GPU
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  #数据
  if(setName == "Test"):
    testSet = TestDataSet("Test",prePorcess=False)
  else:
    testSet = CharDataSet(setName,prePorcess=False)
  testLoader = DataLoader(testSet,miniBatch,shuffle=False,drop_last=False)
  testLen = len(testSet)

  #模型
  myModel = MyModel()
  myModel.classifier[-1] = nn.Linear(256,71)
  # myModel = torchvision.models.vgg16()
  # myModel.classifier[6] = nn.Linear(4096,71)
  #将模型中的ReLU换成mish
  # for i in range(len(myModel.features)):
  #   if isinstance(myModel.features[i],nn.ReLU):
  #     myModel.features[i] = nn.Mish(inplace=True)
  # for i in range(len(myModel.classifier)):
  #   if isinstance(myModel.classifier[i],nn.ReLU):
  #     myModel.classifier[i] = nn.Mish(inplace=True)
  myModel.load_state_dict(torch.load("Models/myModel.pth"))
  myModel.train(False)
  myModel = myModel.to(device)

  #代价函数
  lossFn = nn.CrossEntropyLoss()
  lossFn = lossFn.to(device)

  #评估指标
  testLoss = 0
  rightCount = 0

  #错误图
  wrongImgs = [[]for i in range(71)]
  shutil.rmtree("TestLogs",ignore_errors=True)
  logWriter = SummaryWriter("TestLogs")
  wrongResult = [[]for i in range(71)]

  #错误矩阵
  wrongMat = np.zeros((71,71),dtype="uint8")

  testLoss = 0     #总损失
  AvgLoss = 0       #平均损失
  rightRate = 0     #正确率
  rightCount = 0

  #计算精确率和召回率的数组
  testLen = len(testSet)
  TP = [0]*71
  FP = [0]*71
  FN = [0]*71
  TN = [0]*71

  with torch.no_grad():
    batchCount = testLen // miniBatch + 1
    batchi = 0        #用来输出测试进度
    for data in testLoader:
      imgs,targets = data
      imgs = imgs.to(device)
      targets = targets.to(device)
      outputs = myModel(imgs)                     #跑验证集
      result = torch.argmax(outputs,1)            #取最大值
      rightCount += torch.sum(result==targets)    #和标签比较

      #写入错误图片和错误矩阵
      for i in range(len(result)):
        if result[i] != targets[i]:
          wrongImgs[result[i]].append(imgs[i].cpu().numpy().tolist())
          wrongResult[result[i]].append(targets[i].item())
        wrongMat[targets[i]][result[i]] += 1

      loss = lossFn(outputs,targets)              #代价函数
      testLoss+=loss
      batchi += 1
      print(f"\r测试中，进度{(batchi / batchCount)*100:.2f}%",flush=True,end="")

      #统计TP/FP/FN/TN
      for i in range(len(targets)):
        if(result[i]==targets[i]):
          TP[targets[i]]+=1
        else:
          FP[result[i]]+=1
          FN[targets[i]]+=1
    print()
    rightRate = rightCount/testLen*100
    AvgLoss = testLoss/(testLen/miniBatch)

    TP = np.array(TP)
    FP = np.array(FP)
    FN = np.array(FN)
    TN = np.array(TN)
    TN = testLen-TP-FP-FN

    #计算正确率，精确率，召回率，F1分数
    rightRate = rightCount/testLen
    accRate = (TP / (TP+FP))
    recallRate = (TP / (TP+FN))
    F1Score = (2*accRate*recallRate)/(accRate+recallRate)

  #控制台输出测试结果
  print(f"测试损失:{AvgLoss:.3f},总正确率:{rightRate*100:.3f}%")
  print(f"平均精确率：{np.nanmean(accRate)*100:.2f}%，最大精确率：{np.nanmax(accRate)*100:.2f}%,最小精确率：{np.nanmin(accRate)*100:.2f}%")
  print(f"平均召回率：{np.nanmean(recallRate)*100:.2f}%，最大召回率：{np.nanmax(recallRate)*100:.2f}%,最小召回率：{np.nanmin(recallRate)*100:.2f}%")
  print(f"平均F1分数：{np.nanmean(F1Score):.4f}，最大F1分数：{np.nanmax(F1Score):.4f},最小F1分数：{np.nanmin(F1Score):.4f}")

  #写入tensorboard
  for i in range(len(wrongImgs)):
    imgs = torch.from_numpy(np.asarray(wrongImgs[i])).type(torch.uint8)
    if(len(imgs)!=0):
      logWriter.add_images(f"result:{chr(testSet.rCharDict[i])},target:{[chr(testSet.rCharDict[item]) for item in wrongResult[i]]}",imgs)

  logWriter.close()
  outWrongMatCsv(wrongMat,testSet.rCharDict)

class sentenceLoader(TestDataSet):    #重写TestDataSet使其支持读取句子
  def __init__(self,path,prePorcess=False):
    super().__init__(path,prePorcess)
    self.makeSentence()

  def makeSentence(self):
    self.sentence = {}
    with open(f"{self.rootDir}/data.csv",mode="r",encoding="utf8") as csvFile:
      reader = csv.reader(csvFile)
      next(reader)            #读掉表头
      for row in reader:
        if(row[5] == 'FALSE'):    #不忽略文本
          row = [int(i) for i in row[0:5]] + row[5:]
          if((row[0],row[1],row[2]) not in self.sentence.keys()):
            self.sentence[(row[0],row[1],row[2])] = []
          self.sentence[(row[0],row[1],row[2])].append((row[3],row[4],row[6]))

      for key in self.sentence.keys():
        self.sentence[key].sort()

    self.keys = list(self.sentence.keys())
    self.lenFileList = len(self.keys)

  def __getitem__(self, index):
    imgs = []
    targets = []
    key = self.keys[index]
    for val in self.sentence[key]:
      img = cv2.imread(f"Temp/{self.rootDir}/data/{key[0]}_{key[1]}_{key[2]}_{val[0]}_{val[1]}.png",flags=0)
      img = np.repeat(img[np.newaxis,:,:],3,axis=0)
      imgs.append(img)
      targets.append(val[2])

    imgs = torch.from_numpy(np.asarray(imgs)).type(torch.float)
    targets = [self.charDict[ord(c)] for c in targets]
    targets = torch.from_numpy(np.asarray(targets)).type(torch.int)
    return imgs,targets

  def __len__(self):
    return self.lenFileList

def sentenceTest():
  #部署GPU
  device = torch.device("cuda")
  #数据
  testSet = sentenceLoader("Test",prePorcess=False)

  #模型
  myModel = MyModel()
  myModel.classifier[-1] = nn.Linear(256,71)
  # myModel = torchvision.models.vgg16()
  # myModel.classifier[6] = nn.Linear(4096,71)
  #将模型中的ReLU换成mish
  # for i in range(len(myModel.features)):
  #   if isinstance(myModel.features[i],nn.ReLU):
  #     myModel.features[i] = nn.Mish(inplace=True)
  # for i in range(len(myModel.classifier)):
  #   if isinstance(myModel.classifier[i],nn.ReLU):
  #     myModel.classifier[i] = nn.Mish(inplace=True)
  myModel.load_state_dict(torch.load("Models/myModel.pth"))
  myModel.train(False)
  myModel = myModel.to(device)

  #代价函数
  lossFn = nn.CrossEntropyLoss()
  lossFn = lossFn.to(device)

  #评估指标
  rightCount = 0

  #错误图
  wrongImgs = [[]for i in range(71)]
  shutil.rmtree("TestLogs",ignore_errors=True)
  logWriter = SummaryWriter("TestLogs")
  wrongResult = [[]for i in range(71)]

  #正确率
  rightRate = 0     #正确率
  rightCount = 0
  totalCount = 0

  #结果标记硬编码
  alpAndDigConfuse = list("0Oo1Il")  # 数字字母易混
  alp2DigMapper = {"O": "0", "I": "1"}           # 字母转数字
  dig2AlpMapper = {"0": "O", "1": "I"}            # 数字转字母
  terms = {"KPA":"kPa","KG":"kg","K9":"kg","IBS":"lbs"}

  def fixResult(result, resultCf):
    # 统计连续数字混淆的个数，1个只有相邻才转换，2个以上直接转换
    # 最后修正专用名词，比如kPa
    resultMark = []         #标记结果

    # 标记
    for i in range(len(result)):
      if(result[i] in alpAndDigConfuse):
        resultMark.append("D")
      else:
        resultMark.append("*")

    # 统一转换大写
    for i in range(len(result)):
      if(result[i] == "l"):   #I和l特殊处理
        result[i] = "I"
      else:
        result[i] = str.upper(result[i])
      if(resultMark[i] != "D"):
        resultMark[i] = "*"

    # 数字修正
    L,R = -1,-1
    for i in range(len(result)):
      if(resultMark[i] == "D"):   #统计数字长度
        if(L == -1): L,R = i,i
        R+=1
      else:         #转换数字
        if(L != -1 and R - L >= 3):   #长数字，可以转换
          for j in range(L,R):
            if(j == L and result[j] in ["0","O"] and L-1 >= 0 and not str.isdigit(result[L-1])):
              result[j] = "O"         #0不可能出现在最前面，除非前面有数字
            elif(result[j] in alp2DigMapper.keys()):
              result[j] = alp2DigMapper[result[j]]
            resultMark[j] = "*"

        elif(L != -1 and R - L < 3):
          if(L - 1 >= 0 and str.isdigit(result[L-1]) or R < len(result) and str.isdigit(result[R])):
            for j in range(L,R):  #太短，但是左右有数字，可以转换
              if(j == L and result[j] in ["0","O"] and L-1 >= 0 and not str.isdigit(result[L-1])):
                result[j] = "O"
              elif(result[j] in alp2DigMapper.keys()):
                result[j] = alp2DigMapper[result[j]]
              resultMark[j] = "*"
          else:     #太短，不能转换
            for j in range(L,R):
              if(result[j] in dig2AlpMapper.keys()):
                result[j] = dig2AlpMapper[result[j]]
              resultMark[j] = "*"
        L,R = -1,-1

    # 专用名词
    resStr = ""
    resStr = str.join(resStr,result)
    for t in terms.keys():
      resStr = resStr.replace(t,terms[t])
    result = list(resStr)
    return result

  #计算精确率和召回率的数组
  TP = [0]*71
  FP = [0]*71
  FN = [0]*71
  TN = [0]*71

  with torch.no_grad():
    for dataIdx in range(len(testSet)):
      data = testSet[dataIdx]
      imgs,targets = data
      imgs = imgs.to(device)
      targets = targets.to(device)
      outputs = myModel(imgs)                     #跑测试集
      outputs = torch.softmax(outputs,dim=1)
      resultCf = torch.max(outputs,dim=1)             #取置信度
      result = torch.argmax(outputs,dim=1)            #取类别
      result = [chr(testSet.rCharDict[i.item()]) for i in result]   #类别转文本
      targets = [chr(testSet.rCharDict[i.item()]) for i in targets]
      #print(str.join("",result))
      result = fixResult(result, resultCf[0])  #修正结果

      #统计正确数量
      result = [testSet.charDict[ord(item)]for item in result]
      targets = [testSet.charDict[ord(item)]for item in targets]

      result = np.array(result)
      targets = np.array(targets)
      rightCount += np.sum(result == targets)
      totalCount += len(result)

      #写入错误图片
      for i in range(len(result)):
        if result[i] != targets[i]:
          wrongImgs[result[i]].append(imgs[i].cpu().numpy().tolist())
          wrongResult[result[i]].append(targets[i])

      #统计TP/FP/FN/TN
      for i in range(len(targets)):
        if(result[i]==targets[i]):
          TP[targets[i]]+=1
        else:
          FP[result[i]]+=1
          FN[targets[i]]+=1

    testLen = totalCount
    TP = np.array(TP)
    FP = np.array(FP)
    FN = np.array(FN)
    TN = np.array(TN)
    TN = testLen-TP-FP-FN

    #计算正确率，精确率，召回率，F1分数
    rightRate = rightCount/testLen
    accRate = (TP / (TP+FP))
    recallRate = (TP / (TP+FN))
    F1Score = (2*accRate*recallRate)/(accRate+recallRate)

  #控制台输出测试结果
  print(f"总正确率:{rightRate*100:.3f}%")
  print(f"平均精确率：{np.nanmean(accRate)*100:.2f}%，最大精确率：{np.nanmax(accRate)*100:.2f}%,最小精确率：{np.nanmin(accRate)*100:.2f}%")
  print(f"平均召回率：{np.nanmean(recallRate)*100:.2f}%，最大召回率：{np.nanmax(recallRate)*100:.2f}%,最小召回率：{np.nanmin(recallRate)*100:.2f}%")
  print(f"平均F1分数：{np.nanmean(F1Score):.4f}，最大F1分数：{np.nanmax(F1Score):.4f},最小F1分数：{np.nanmin(F1Score):.4f}")

  #写入tensorboard
  for i in range(len(wrongImgs)):
    imgs = torch.from_numpy(np.asarray(wrongImgs[i])).type(torch.uint8)
    if(len(imgs)!=0):
      logWriter.add_images(f"result:{chr(testSet.rCharDict[i])},target:{[chr(testSet.rCharDict[item]) for item in wrongResult[i]]}",imgs)

if __name__ == '__main__':
  charTest()
  sentenceTest()


"""
Create sample level metrics to be passed to the CGD.  Metrics are passed as a json dump.

VERSION HISTORY
0.8.11
    Add QIAseq_XP_RNA_STP
0.8.10
    Switched before and after picard metric source in _pumi
0.8.9
    Add uniformity_of_coverage, cnv_median_segment_mad_cn metrics
0.8.8
    Add QIAseq_V4_STP4
    Revert xy_check to bio_sex_check to maintain CGD compatibility
0.8.7
    Rename metric bio_sex_check to y_ploidy_check for AgilentCRE_V1 and TruSightOneV2_5
    Rename metric bio_sex_check to xy_check for QIAseq_V4_MINI
0.8.5
    Change forced calls metric to json
0.8.3
    Add entry for json metric bio_sex_check
0.8.0
    Pass forced calls above and below background metric
0.7.0
    Now calculate percentOnTarget from FastQC tatal sequences and collectalignmentmetrics on sorted bwa bam
"""

import argparse
import json
# User libraries
from inputs import ProbeQcRead, PerLocusRead, AlignSummaryMetrics, GatkDepthOfCoverageRead, GatkCountReads, MsiSensor, SamReader, GatkCollectRnaSeqMetrics
from inputs import FastQcRead

VERSION = '0.8.10'

def supply_args():
    """
    Populate args.
    https://docs.python.org/2.7/library/argparse.html
    """
    parser = argparse.ArgumentParser(description='')
    # Input files that will be parsed for data.
    parser.add_argument('--probeqc_after', required=False, type=ProbeQcRead,
                        help='Probe coverage QC after UMI deduplication metrics.')
    parser.add_argument('--probeqc_before', required=False, type=ProbeQcRead,
                        help='Probe coverage QC before UMI deduplication metrics.')

    parser.add_argument('--fastqc_r1', type=FastQcRead, help='FastQC stats for read 1.')
    parser.add_argument('--fastqc_r2', type=FastQcRead, help='FastQC stats for read 2.')

    parser.add_argument('--picard_summary', type=AlignSummaryMetrics,
                        help='Picard alignment summary metrics file.')
    parser.add_argument('--picard_summary_umi', type=AlignSummaryMetrics,
                        help='Picard alignment summary metrics file taken after umi deduplication.')

    parser.add_argument('--gatk_depth_cov_prop', type=GatkDepthOfCoverageRead,
                        help='GATK DepthOfCoverage file.')
    parser.add_argument('--gatk_depth_cov_cnts', type=PerLocusRead,
                        help='GATK DepthOfCoverage locus file.')
    parser.add_argument('--gatk_coll_rnaseq_mets', type=GatkCollectRnaSeqMetrics,
                        help='GATK CollectRnaSeqMetrics file.')
    parser.add_argument('--gatk_count_reads_total', type=GatkCountReads,
                        help='Output from GATK4 CountReads with total read count.')
    parser.add_argument('--gatk_count_reads_ints', type=GatkCountReads,
                        help='Output from GATK4 CountReads with read count for reads overlapping targets.')

    parser.add_argument('--msi', type=MsiSensor, help='TSV file containing MSI results')

    parser.add_argument('--primers_bam', help='BAM file to calculate primer reads on target.')
    parser.add_argument('--primers_bed', help='BED file containing primer coordinates only.')

    parser.add_argument('--blia_pre', help='JSON from Ding correlation subtyping, pre-normalization.')
    parser.add_argument('--blia_post', help='JSON from Ding correlation subtyping, post-normalization.')

    # These just get attached to the final json output as-is.
    parser.add_argument('--json_in', nargs='*',
                        help='Arbitrary number of files to be included in sample metrics that are in json format.')

    parser.add_argument('--outfile', help='Output file with json string.')
    parser.add_argument('--outfile_new', help='Output file with new style json string.')
    parser.add_argument('--outfile_txt', help='Output file in human readable text format.')
    parser.add_argument('--workflow', help='Pass the Galaxy workflow name, if applicable.')
    parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION)
    args = parser.parse_args()

    # Check to make sure if one gatk_count_reads option is set, both are set.
    if args.gatk_count_reads_total and not args.gatk_count_reads_ints:
        parser.error("Argument gatk_count_reads_total requires gatk_count_reads_ints.")
    if args.gatk_count_reads_ints and not args.gatk_count_reads_total:
        parser.error("Argument gatk_count_reads_ints requires gatk_count_reads_total.")

    # Check to make sure if one primers option is set, both are set.
    if args.primers_bam and not args.primers_bed:
        parser.error("Argument primers_bam requires primers_bed.")
    if args.primers_bed and not args.primers_bam:
        parser.error("Argument primers_bed requires primers_bam.")

    # Ensure at least one output option has been selected.
    if not args.outfile and not args.outfile_txt and not args.outfile_new:
        parser.error("You must specify one of outfile, outfile_new, or outfile_txt.")

    return args

class RawMetricCollector:
    """
    Gather all of the metrics from each input source, and allow it to be accessible from here.
    gatk_cr_total = total number of reads, as given by GATK4 CountReads
    gatk_cr_ints = total number of reads crossing target intervals, as given by GATK4 CountReads
    msi = from MSIsensor, TSV output file
    """
    def __init__(self, args):
        if args.gatk_count_reads_total:
            self.gatk_cr_total = args.gatk_count_reads_total.count
            self.gatk_cr_ints = args.gatk_count_reads_ints.count
        else:
            self.gatk_cr_total = None
            self.gatk_cr_ints = None

        if args.msi:
            self.msi = args.msi.msi
        else:
            self.msi = None

        if args.picard_summary:
            self.picard_summary = args.picard_summary.metrics
        else:
            self.picard_summary = None

        if args.picard_summary_umi:
            self.picard_summary_umi = args.picard_summary_umi.metrics
        else:
            self.picard_summary_umi = None

        if args.gatk_depth_cov_prop:
            self.gatk_depth_cov_prop = args.gatk_depth_cov_prop
        else:
            self.gatk_depth_cov_prop = None

        if args.gatk_depth_cov_cnts:
            self.gatk_depth_cov_cnts = args.gatk_depth_cov_cnts
        else:
            self.gatk_depth_cov_cnts = None

        if args.gatk_coll_rnaseq_mets:
            self.gatk_coll_rnaseq_mets = args.gatk_coll_rnaseq_mets.metrics
        else:
            self.gatk_coll_rnaseq_mets = None

        if args.fastqc_r1:
            self.gc_pct_1 = args.fastqc_r1.gc_pct
            self.fastqc_seq_count = args.fastqc_r1.seq_cnt
        else:
            self.gc_pct_1 = None
            self.fastqc_seq_count = None

        if args.fastqc_r2:
            self.gc_pct_2 = args.fastqc_r2.gc_pct
        else:
            self.gc_pct_2 = None

        if args.probeqc_before:
            self.probeqc_before = args.probeqc_before.probeqc
            self.probeqc_header_before = args.probeqc_before.headers
        else:
            self.probeqc_before = None
            self.probeqc_header_before = None

        if args.probeqc_after:
            self.probeqc_after = args.probeqc_after.probeqc
            self.probeqc_header_after = args.probeqc_after.headers
        else:
            self.probeqc_after = None
            self.probeqc_header_after = None

        if args.workflow:
            self.wf = args.workflow
        else:
            self.wf = None

        if args.primers_bam:
            self.primers_bam = SamReader(args.primers_bam, args.primers_bed).count
        else:
            self.primers_bam = None

        if args.blia_pre:
            self.blia_pre = self._json_in([args.blia_pre])
        else:
            self.blia_pre = {'blia': None, 'blis': None, 'lar': None, 'mes': None}

        if args.blia_post:
            self.blia_post = self._json_in([args.blia_post])
        else:
            self.blia_post = {'blia': None, 'blis': None, 'lar': None, 'mes': None}

        if args.json_in:
            self.json_mets = self._json_in(args.json_in)
        else:
            self.json_mets = None

    @staticmethod
    def _json_in(json_in):
        """
        Get all of the incoming JSON metrics.
        :return:
        """
        json_mets = {}
        for filename in json_in:
            with open(filename, 'r') as myfile:
                for line in myfile:
                    try:
                        for k, v in json.loads(line).items():
                            json_mets[str(k)] = v
                    except AttributeError:
                        for k, v in json.loads(line)[0].items():
                            json_mets[str(k)] = v
        return json_mets

    def _params_stdout(self):
        """
        Print variable contents to stdout.
        :return:
        """
        print("MSI: {0}".format(self.msi))
        print("ProbeQC Before: {0}".format(self.probeqc_before))
        print("ProbeQC After: {0}".format(self.probeqc_after))
        print("FastQC R1 GC: {0}".format(self.gc_pct_1))
        print("FastQC R2 GC: {0}".format(self.gc_pct_2))
        print("GATK CountReads Total: {0}".format(self.gatk_cr_total))
        print("GATK CountReads Intervals: {0}".format(self.gatk_cr_ints))
        print("Picard: {0}".format(self.picard_summary))
        print("JSON Metrics: {0}".format(self.json_mets))
        print("Primers BAM: {0}".format(self.primers_bam))

class SampleMetrics:
    def __init__(self, raw_mets):
        self.raw_mets = raw_mets
        self.probeqc_before = self.raw_mets.probeqc_before
        if self.probeqc_before:
            self.total_cov_before = self._calc_cov(self.probeqc_before, 'AVGD')
            self.total_bp_before = self._calc_total_bp(self.probeqc_before)
            self.header_mets_before = self._metrics_from_probeqc_header(self.raw_mets.probeqc_header_before[5:],
                                                                        self.raw_mets.probeqc_before,
                                                                        self.total_bp_before)
        else:
            self.total_cov_before = None
            self.total_bp_before = None
            self.header_mets_before = None

        self.probeqc_after = self.raw_mets.probeqc_after
        if self.probeqc_after:
            self.total_cov_after = self._calc_cov(self.probeqc_after, 'AVGD')
            self.total_bp_after = self._calc_total_bp(self.probeqc_after)
            self.header_mets_after = self._metrics_from_probeqc_header(self.raw_mets.probeqc_header_after[5:],
                                                                       self.raw_mets.probeqc_after,
                                                                       self.total_bp_after)
        else:
            self.total_cov_after = None
            self.total_bp_after = None
            self.header_mets_before = None

        try:
            self.gatk_depth_cov_prop = self.raw_mets.gatk_depth_cov_prop
        except:
            self.gatk_depth_cov_prop = None

        try:
            self.gatk_depth_cov_cnts = self.raw_mets.gatk_depth_cov_cnts
        except:
            self.gatk_depth_cov_cnts = None

        try:
            self.pumi = self._pumi()
        except:
            self.pumi = None
        try:
            self.on_target = self._add_on_target(self.raw_mets.picard_summary, self.raw_mets.fastqc_seq_count,
                                                 total_lbl='PF_READS_ALIGNED')
        except:
            self.on_target = None
        try:
            self.on_primer_frag_count = self.raw_mets.primers_bam
        except:
            self.on_primer_frag_count = None
        try:
            self.on_primer_frag_count_pct = self._add_on_target(self.raw_mets.picard_summary,
                                                                self.on_primer_frag_count,
                                                                'PF_HQ_ALIGNED_READS')
        except:
            self.on_primer_frag_count_pct = None
        try:
            self.gatk_cr_on_target = self._gatk_cr_on_target()
        except:
            self.gatk_cr_on_target = None

        try:
            self.gatk_pct_mrna_bases = self.raw_mets.gatk_coll_rnaseq_mets['PCT_MRNA_BASES']
            self.gatk_pct_correct_strand_reads = self.raw_mets.gatk_coll_rnaseq_mets['PCT_CORRECT_STRAND_READS']
        except:
            self.gatk_pct_mrna_bases = None
            self.gatk_pct_correct_strand_reads = None

        try:
            self.blia_pre_mets = self._top_two_diff(self.raw_mets.blia_pre)
            self.blia_post_mets = self._top_two_diff(self.raw_mets.blia_post)
            self.assess_blia = self._assess_blia()
        except:
            self.blia_pre_mets = {'best': None, 'second': None, 'diff': None}
            self.blia_post_mets = {'best': None, 'second': None, 'diff': None}
            self.assess_blia = None

    def _blia_blis_map(self, name):
        """
        Map between numbers and strings.
        :return:
        """
        mapping = {'blia': '0',
                   'blis': '1',
                   'lar': '2',
                   'mes': '3'}
        return mapping[name]

    def _assess_blia(self, thresh=0.1):
        """
        If the difference between the top two scores is >=0.1 for both pre and post, and the two
        top types are the same, this is a reportable subtyping.
        :return:
        """
        if (self.blia_pre_mets['diff'] >= thresh
                and self.blia_post_mets['diff'] >= thresh
                and self.blia_pre_mets['best'] == self.blia_post_mets['best']
                and self.blia_pre_mets['second'] == self.blia_post_mets['second']
                and (self.blia_post_mets['best'] == '0' or self.blia_post_mets['best']) == '1'):
            return '1'
        return '0'

    def _top_two_diff(self, scores):
        """
        Get the difference between the top two correlation scores.
        :return:
        """
        sort_scores = dict(sorted((val, key) for (key, val) in scores.items()))
        diff = list(sort_scores.keys())[-1] - list(sort_scores.keys())[-2]
        best = list(sort_scores.values())[-1]
        second = list(sort_scores.values())[-2]
        return {'diff': diff,
                'best': self._blia_blis_map(best),
                'second': self._blia_blis_map(second)
                }

    def _gatk_cr_on_target(self):
        """
        Use GATK4 CountReads or CountReadsSpark to estimate on-target pct.
        :return:
        """
        if self.raw_mets.gatk_cr_total and self.raw_mets.gatk_cr_ints:
            return float(self.raw_mets.gatk_cr_ints) / (float(self.raw_mets.gatk_cr_total) + 0.0)

    def _pumi(self):
        """
        Create the PUMI metric, if applicable.
        :return:
        """
        if self.raw_mets.picard_summary and self.raw_mets.picard_summary_umi:
            before = int(self.raw_mets.picard_summary['FIRST_OF_PAIR']['PF_ALIGNED_BASES'])
            after = int(self.raw_mets.picard_summary_umi['FIRST_OF_PAIR']['PF_ALIGNED_BASES'])
            return self._calc_metric(before, (after * 100))

    def _metrics_from_probeqc_header(self, headers, probeqc, total_bp):
        """
        :param headers:
        :param probeqc:
        :param total_bp:
        :return:
        """
        sample_metrics = {}
        for label in headers:
            this_cov = self._calc_cov(probeqc, label)
            sample_metrics[label] = self._calc_metric(total_bp, this_cov)
        return sample_metrics

    @staticmethod
    def _calc_total_bp(probeqc):
        """
        Get the total number of base pairs covered by your targeted region set.
        :return:
        """
        total_bp = 0
        for line in probeqc.values():
            try:
                curr = int(line['STOP']) - int(line['START']) + 1.0
                total_bp += curr
            except ValueError:
                pass

        return total_bp

    @staticmethod
    def _calc_cov(probeqc, metric):
        """
        Calculate total coverage across sample.
        :param probeqc:
        :param metric:
        :return:
        """
        total_cov = 0
        for line in probeqc.values():
            try:
                curr_bp = int(line['STOP']) - int(line['START']) + 1.0
                curr_cov = curr_bp * float(line[metric])
                total_cov += curr_cov
            except ValueError:
                pass

        return total_cov

    @staticmethod
    def _calc_metric(bp, cov):
        """
        Calculate the sample level AVGD from total bp anc coverage.
        :param bp:
        :param cov:
        :return:
        """
        return '{:0.1f}'.format(cov / bp)

    @staticmethod
    def _add_on_target(picard, total_cov, total_lbl='PF_ALIGNED_BASES'):
        """
        Include the percent on target reads metric, mainly for amplicon assays.
        Also include the on primer frag count percentage.  Rename variables...
        :return:
        """
        if 'FIRST_OF_PAIR' in picard:
            pf_bases_aligned = int(picard['FIRST_OF_PAIR'][total_lbl])
        elif 'UNPAIRED' in picard:
            pf_bases_aligned = int(picard['UNPAIRED'][total_lbl])
        else:
            pf_bases_aligned = None

        if pf_bases_aligned and total_cov:
            if total_lbl == 'PF_HQ_ALIGNED_READS':
                on_target = str("{:.4}".format((int(total_cov) / pf_bases_aligned) * 100.0))
            else:
                on_target = str("{:.4}".format(pf_bases_aligned / (int(total_cov)) * 100.0))
        else:
            on_target = None

        return on_target

class MetricPrep(SampleMetrics):
    """
    Get the metrics we want from SampleMetrics, and prepare them for being written.
    """
    def __init__(self, raw_mets):
        super(MetricPrep, self).__init__(raw_mets)
        self.mets = self._gather_metrics()
        self.req_old = self._req_old()
        self.req_new = self._req_new()

    def _gather_metrics(self):
        """
        Old style metrics.  Will need to handle special json metrics separately.
        :return:
        'parentage_sites', 'parentage_disc', 'parentage_binom', 'parentage_confirmed'
        {"COUNT_0": 21916, "COUNT_1": 15287, "COUNT_10": 11012, "COUNT_100": 3505, "COUNT_1000": 187,
        "COUNT_10000": 9, "COUNT_100000": 0, "TPM_0": 22296, "TPM_0.01": 15366, "TPM_0.1": 15360, "TPM_1": 14323,
        "TPM_10": 9791, "TPM_100": 1505, "TPM_1000": 98}
        """
        mets = {'qthirty': self._get_avg_probeqc('Q30'),
                'averageDepth': self._get_avg_probeqc('AVGD'),
                'depthTen': self._get_avg_probeqc('D10'),
                'depthTwenty': self._get_avg_probeqc('D20'),
                'depthFifty': self._get_avg_probeqc('D50'),
                'depthOneHundred': self._get_avg_probeqc('D100'),
                'depthTwoHundredFifty': self._get_avg_probeqc('D250'),
                'depthFiveHundred': self._get_avg_probeqc('D500'),
                'depthSevenHundred': self._get_avg_probeqc('D700'),
                'depthOneThousand': self._get_avg_probeqc('D1000'),
                'depthTwelveHundredFifty': self._get_avg_probeqc('D1250'),
                'depthTwoThousand': self._get_avg_probeqc('D2000'),
                'uniformity_of_coverage': self._get_cov_uniformity(self._get_avg_probeqc('AVGD')),
                'allele_balance': self._reduce_sig(self._add_json_mets(lookin=self.raw_mets.json_mets,
                                                                       metric='allele_balance')),
                'allele_balance_het_count': self._add_json_mets(lookin=self.raw_mets.json_mets,
                                                                metric='allele_balance_het_count'),
                'gatk_cr_on_target': self._reduce_sig_pct(self.gatk_cr_on_target),
                'gatk_cr_total': self.raw_mets.gatk_cr_total,
                'gatk_cr_ints': self.raw_mets.gatk_cr_ints,
                'gatk_pct_mrna_bases': self._reduce_sig_pct(self.gatk_pct_mrna_bases),
                'gatk_pct_correct_strand_reads': self._reduce_sig_pct(self.gatk_pct_correct_strand_reads),
                'gc_pct_r1': self.raw_mets.gc_pct_1,
                'gc_pct_r2': self.raw_mets.gc_pct_2,
                'bio_sex_check': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='bio_sex_check'),
                'homozygosity_flag': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='homozygosity_flag'),
                'parentage_binom': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='parentage_binom'),
                'parentage_disc': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='parentage_disc'),
                'parentage_confirmed': self._add_json_mets(lookin=self.raw_mets.json_mets,
                                                           metric='parentage_confirmed'),
                'parentage_sites': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='parentage_sites'),
                'percentOnTarget': self._reduce_sig(self.on_target),
                'percentUmi': self.pumi,
                'rna_count_zero': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_0'),
                'rna_count_one': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_1'),
                'rna_count_ten': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_10'),
                'rna_count_onehundred': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_100'),
                'rna_count_onethousand': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_1000'),
                'rna_count_tenthousand': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_10000'),
                'rna_count_hundredthousand': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='COUNT_100000'),
                'rna_tpm_zero': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_0'),
                'rna_tpm_hundredth': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_0.01'),
                'rna_tpm_tenth': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_0.1'),
                'rna_tpm_one': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_1'),
                'rna_tpm_ten': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_10'),
                'rna_tpm_onehundred': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_100'),
                'rna_tpm_onethousand': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='TPM_1000'),
                'tmb': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='tmb'),
                'msi_pct': self._add_json_mets(lookin=self.raw_mets.msi, metric='somatic_pct'),
                'msi_sites': self._add_json_mets(lookin=self.raw_mets.msi, metric='total_sites'),
                'msi_somatic_sites': self._add_json_mets(lookin=self.raw_mets.msi, metric='somatic_sites'),
                'blia_pre_best': self.blia_pre_mets['best'],
                'blia_pre_second': self.blia_pre_mets['second'],
                'blia_pre_diff': self._reduce_sig(self.blia_pre_mets['diff']),
                'blia_post_best': self.blia_post_mets['best'],
                'blia_post_second': self.blia_post_mets['second'],
                'blia_post_diff': self._reduce_sig(self.blia_post_mets['diff']),
                'blia_reportable': self.assess_blia,
                'blia_raw_pre': self.raw_mets.blia_pre['blia'],
                'blis_raw_pre': self.raw_mets.blia_pre['blis'],
                'lar_raw_pre': self.raw_mets.blia_pre['lar'],
                'mes_raw_pre': self.raw_mets.blia_pre['mes'],
                'blia_raw_post': self.raw_mets.blia_post['blia'],
                'blis_raw_post': self.raw_mets.blia_post['blis'],
                'lar_raw_post': self.raw_mets.blia_post['lar'],
                'mes_raw_post': self.raw_mets.blia_post['mes'],
                'total_on_target_transcripts': self.on_primer_frag_count,
                'total_on_target_transcripts_pct': self.on_primer_frag_count_pct,
                'forced_calls_above': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='forced_calls_above'),
                'forced_calls_below': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='forced_calls_below'),
                'y_ploidy_check': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='y_ploidy_check'),
                'cnv_median_segment_mad_cn': self._add_json_mets(lookin=self.raw_mets.json_mets, metric='cnv_median_segment_mad_cn')
                }
        return mets

    @staticmethod
    def _add_json_mets(lookin, metric):
        """
        Currently the following metrics are sent in standalone.  Need to think about ways to do this better.
        allele_balance
        tmb
        :return:
        """
        if lookin:
            if metric in lookin:
                return str(lookin[metric])
        return None

    @staticmethod
    def _reduce_sig(metric):
        """
        Get rid of digits that are not significant.
        :return:
        """
        if metric:
            return "{:.2f}".format(float(metric))

    @staticmethod
    def _reduce_sig_pct(metric):
        """
        Get rid of digits that are not significant, and provide in percent format.
        :return:
        """
        if metric:
            return "{:.2f}".format(float(metric)*100)

    def _get_avg_probeqc(self, label):
        """
        Retrieve average metric from the probeqc_after dict.  All ProbeQCs are after, unless they
        are being run on UMI-containing assays.  If this is the case, the probeqc_before dict
        captures metrics before UMI deduplication occurs.
        :return:
        """
        try:
            this_cov = self._calc_cov(self.probeqc_after, label)
            return self._calc_metric(self.total_bp_after, this_cov)
        except:
            return None

    def _get_cov_uniformity(self, average_depth):
        """
        Calculate the uniformity of coverage
        The percentage of targeted base positions in which the read depth is greater than 0.2 times the mean region target coverage depth.
        :return:
        """
        try:
            # return float(self.gatk_depth_cov_prop.sample_mets['gte_'+str(round(float(average_depth)*0.2))])*100
            # calculate using perlocusread instead to get decimal places
            return "{:.1f}".format(float(sum(float(i) > float(average_depth)*0.2 for i in list(self.gatk_depth_cov_cnts.perlocus.values())) / len(list(self.gatk_depth_cov_cnts.perlocus.values())) * 100))
        except:
            return None

    @staticmethod
    def _req_old():
        """
        Based on test name, list which metrics should be provided.
        :return:
        """
        return {'QIAseq_V3_RNA': ['qthirty', 'averageDepth', 'percentUmi'],
                'QIAseq_XP_RNA_HEME': ['qthirty', 'averageDepth', 'percentUmi'],
                'QIAseq_XP_RNA_STP': ['qthirty', 'averageDepth', 'percentUmi'],
                'TruSightOne': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty'],
                'TruSightOneV2_5': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                    'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty'],
                'AgilentCRE_V1': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                  'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty'],
                'QIAseq_V3_HEME2': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwelveHundredFifty',
                                    'depthOneHundred', 'percentOnTarget', 'depthSevenHundred', 'percentUmi'],
                'QIAseq_V4_MINI': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwelveHundredFifty',
                                   'depthOneHundred', 'percentOnTarget', 'depthSevenHundred', 'percentUmi'],
                'QIAseq_V3_STP3': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwelveHundredFifty',
                                   'depthOneHundred', 'percentOnTarget', 'depthSevenHundred', 'percentUmi'],
                'QIAseq_V4_STP4': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwelveHundredFifty',
                                   'depthOneHundred', 'percentOnTarget', 'depthSevenHundred', 'percentUmi'],
                'TruSeq_RNA_Exome_V1-2': ['qthirty'],
                'QIAseq_V3_HOP': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                  'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty', 'percentUmi'],
                'QIAseq_V3_HOP2': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                   'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty', 'percentUmi'],
                'QIAseq_V3_HOP3': ['qthirty', 'averageDepth', 'depthTwoHundredFifty', 'depthTwenty',
                                   'depthOneHundred', 'percentOnTarget', 'depthTen', 'depthFifty', 'percentUmi']
                }

    @staticmethod
    def _req_new():
        """
        Based on test name, list which metrics should be provided.
        :return:
        """
        return {'QIAseq_V3_RNA': ['total_on_target_transcripts', 'total_on_target_transcripts_pct'],
                'QIAseq_XP_RNA_HEME': ['total_on_target_transcripts', 'total_on_target_transcripts_pct'],
                'QIAseq_XP_RNA_STP': ['total_on_target_transcripts', 'total_on_target_transcripts_pct'],
                'TruSightOne': ['gc_pct_r1', 'gc_pct_r2', 'y_ploidy_check'],
                'TruSightOneV2_5': ['gc_pct_r1', 'gc_pct_r2', 'y_ploidy_check', 'homozygosity_flag'],
                'AgilentCRE_V1': ['parentage_sites', 'parentage_disc', 'parentage_binom', 'parentage_confirmed',
                                  'gc_pct_r1', 'gc_pct_r2', 'homozygosity_flag', 'y_ploidy_check'],
                'QIAseq_V3_HEME2': [],
                'QIAseq_V4_MINI': ['forced_calls_above', 'forced_calls_below', 'bio_sex_check'],
                'QIAseq_V3_STP3': ['msi_sites', 'msi_somatic_sites', 'msi_pct', 'tmb'],
                'QIAseq_V4_STP4': ['msi_sites', 'msi_somatic_sites', 'msi_pct', 'tmb', 'bio_sex_check',
                                   'uniformity_of_coverage', 'cnv_median_segment_mad_cn'],
                'TruSeq_RNA_Exome_V1-2': ['total_on_target_transcripts', 'gatk_pct_mrna_bases',
                                          'gatk_pct_correct_strand_reads'],
                'QIAseq_V3_HOP': ['allele_balance', 'allele_balance_het_count'],
                'QIAseq_V3_HOP2': ['allele_balance', 'allele_balance_het_count'],
                'QIAseq_V3_HOP3': ['allele_balance', 'allele_balance_het_count']
                }

class Writer:
    def __init__(self, mets):
        self.mets = mets
        self.wf = self.mets.raw_mets.wf

    def write_cgd_new(self, filename):
        """
        Provide new metric style for CGD import.
        {
        "sampleRunMetrics": [
            {
                "metric": "total_on_target_reads",
                "value": 1230411
            },
            {
                "metric": "percent_on_target_reads",
                "value": 0.91
            }
        ],
        "geneMetrics": [
            {
                "gene": "ASXL1",
                "metric": "total_on_target_reads",
                "value": 469012
            },
            {
                "gene": "BRCA1",
                "metric": "total_on_target_reads",
                "value": 362330
            }
        ]
        }
        :return:
        """
        to_write = {'sampleRunMetrics': [], 'geneMetrics': []}
        for metric, val in self.mets.mets.items():
            if metric in self.mets.req_new[self.wf]:
                if val:
                    metric_dict = {'metric': str(metric), 'value': str(val)}
                    to_write['sampleRunMetrics'].append(metric_dict)

        with open(filename, 'w') as jwrite:
            json.dump(to_write, jwrite)

    def write_cgd_old(self, filename):
        """
        Provide old metric style for CGD import.
        {
        "depthSevenHundred": "95.6",
        "depthTwelveHundredFifty": "66.8",
        "percentOnTarget": "61.03"
        }
        """
        to_write = {}
        for metric, val in self.mets.mets.items():
            if metric in self.mets.req_old[self.wf]:
                to_write[str(metric)] = str(val)

        with open(filename, 'w') as jwrite:
            json.dump(to_write, jwrite)

    def write_to_text(self, filename):
        """
        Write metrics to a text file, mainly to be viewed in Galaxy.
        :return:
        """
        with open(filename, 'w') as to_write:
            for metric, val in self.mets.mets.items():
                to_write.write("{}: {}\n".format(metric, val))

def main():
    args = supply_args()
    raw_mets = RawMetricCollector(args)
    samp_mets = MetricPrep(raw_mets)
    writer = Writer(samp_mets)
    if args.outfile_txt:
        writer.write_to_text(args.outfile_txt)
    if args.outfile:
        writer.write_cgd_old(args.outfile)
    if args.outfile_new:
        writer.write_cgd_new(args.outfile_new)

if __name__ == "__main__":
    main()


import sys
import pandas as pd
import numpy as np
import sqlite3
from sqlalchemy import create_engine
import nltk
import re
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
import pickle
nltk.download(['punkt', 'wordnet'])
nltk.download('stopwords')

def load_data(database_filepath):
    """Load data from the database filepath
    & store them in X and y variable
    args:
        database_filepath: path to the database file
    returns:
        X, y & the list of categories
    """
    engine = create_engine('sqlite:///{}'.format(database_filepath))
    df = pd.read_sql_table('DisasterResponse', con=engine)
    X = df['message']
    Y = df.drop(['id', 'message', 'original', 'genre'], axis=1)
    category_names = Y.columns

    return X,Y,category_names

def tokenize(text):
    """
    args:
        Input value
    returns:
        returns the tokenize version of the input
    """

    url_re = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'

    # Detect and replace urls
    detected_urls = re.findall(url_re, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    # tokenize sentences
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    # save cleaned tokens
    clean_tokens = [lemmatizer.lemmatize(tok).lower().strip() for tok in tokens]

    # remove stopwords
    STOPWORDS = list(set(stopwords.words('english')))
    clean_tokens = [token for token in clean_tokens if token not in STOPWORDS]

    return clean_tokens

def build_model():
    """
      build NLP pipeline - count words, tf-idf, multiple output classifier,
      grid search the best parameters
    Args:
        None
    Returns:
        cross validated classifier object
    """
    pipeline = Pipeline([
        ('vec', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', MultiOutputClassifier(RandomForestClassifier(n_estimators = 100, n_jobs = -1)))
    ])

    parameters = {
    'clf__estimator__n_estimators':[50,100]
    }

    cv = GridSearchCV(pipeline, param_grid=parameters, n_jobs= -1)

    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """
        Evaluate the model performances, in terms of f1-score, precison and recall
    Args:
        model: the model to be evaluated
        X_test: X_test dataframe
        Y_test: Y_test dataframe
        category_names: category names list defined in load data
    Returns:
        perfomances (DataFrame)
    """
    Y_pred = model.predict(X_test)

    # build classification report on every column
    performances = []
    for i in range(len(category_names)):
        performances.append([f1_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro'),
                             precision_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro'),
                             recall_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro')])
    # build dataframe
    performances = pd.DataFrame(performances, columns=['f1 score', 'precision', 'recall'],index = Y_test.columns)

    return performances

def save_model(model, model_filepath):
    """
        Save model to pickle
    """
    pickle.dump(model, open('model_filepath', 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


import json
import rospkg

hotelGoal_dict = {
    # 'room_id': [x, y, z, w, Ori_Release, Position_tolerance]
    'Station': [3.45, 3.87, 0.7, 0.7,  False, 0.1],
    'Station_out': [2.11, 6.25, 0.7, 0.7, False, 0.1],
    'Lobby': [2.11, 6.27, 0, 1,  False, 0.1],
    '101':   [2.94, -3.77, 0, 1,  True, 0.1],
    '102':   [3.45, 3.87, 0, 1,  True, 0.1],
    '404':   [-0.72, 3.33, 0.7, 0.7,  True, 0.1],
    'EVW1':  [3.06, 7.24, 1, 0,  False, 0.1],
    'EVW4':  [3.06, 7.24, 1, 0,  False, 0.1],
    'EVW1S': [-0.72, 3.33, 0.7, 0.7,  True, 0.1],
    'EVW4S': [-0.72, 3.33, 0.7, 0.7,  True, 0.1]
}

print(str(hotelGoal_dict))

ros_package = rospkg.RosPack()
param_path = ros_package.get_path('robot_unique_parameters')
param_path += '/service_setting/hotelGoal.txt'

mydatafile = "hotelGoal.txt"

with open(param_path, 'w') as f:
    json.dump(hotelGoal_dict, f)

print("END")

"""
with open(mydatafile) as f:
    a = json.load(f)
    for key in a.keys():
        temp = key.encode('UTF8')
        a[temp] = a.pop(key)
    print a

for key in hotelGoal_dict.keys():
    print key
    print hotelGoal_dict[key]
"""


#!/usr/bin/env python3
"""
---
name: timer_test.py
description: Test Timer package
people:
  developers:
  - name: Marcio Pessoa
    email: marcio.pessoa@gmail.com
"""

import time
from unittest import TestCase
from tools.timer import Timer

class TestTimerMethods(TestCase):
    """
    description:
    """

    def test_get_period(self):
        """
        description: get() method must return defined time period.
        """
        period = 1000
        timer = Timer(period)
        self.assertEqual(timer.period, period)

    def test_status(self):
        """
        status() method must return current time, so it is a number between 0
        and defined period.
        """
        period = 10
        timer = Timer(period)
        self.assertLessEqual(timer.status(), period)

    def test_set(self):
        """
        description: set() method must configure a new time period.
        """
        period = 10
        timer = Timer(period)
        period = 20
        timer.period = period
        self.assertEqual(timer.period, period)

    def test_reset(self):
        """
        description: reset() is used to reset (obviously) a timer counter.
        """
        period = 100
        timer = Timer(period)
        time.sleep((period - 10) / 1000)
        timer.reset()
        self.assertLessEqual(timer.status(), 1)

    def test_countdown(self):
        """
        description:
        """
        period = 100
        timer = Timer(period, 'COUNTDOWN')
        time.sleep(period / 1000)
        self.assertGreaterEqual(timer.status(), period)

    def test_loop(self):
        """
        description: Test a loop timer
        """
        period = 10
        timer = Timer(period)
        time.sleep(period / 2 / 1000)
        self.assertEqual(timer.check(), False)
        time.sleep(period / 2 / 1000)
        self.assertEqual(timer.check(), True)

    def test_stopwatch(self):
        """
        description:
        """
        period = 100
        timer = Timer(period, 'STOPWATCH')
        time.sleep(period / 1000)
        self.assertGreaterEqual(timer.check(), period)

    def test_disable(self):
        """
        description: disable() is used to timer always return False
        """
        period = 100
        timer = Timer(period)
        timer.disable()
        time.sleep(period / 1000)
        self.assertEqual(timer.check(), False)

    def test_enable(self):
        """
        description: enable() is used to turn on a timer
        """
        period = 100
        timer = Timer(period)
        timer.disable()
        timer.enable()
        time.sleep(period / 1000)
        self.assertEqual(timer.check(), True)


from .imports import *
from collections import defaultdict

class LinearDecoder(nn.Module):
    def __init__(self, encoder, output_size, dropout_p=0.1, weights=None):
        super(LinearDecoder, self).__init__()
        self.encoder = encoder
        self.dropout = nn.Dropout(dropout_p)
        self.hidden_to_tag = nn.Linear(ENCODER_OUTPUT_DIM, output_size)
        self.activation = nn.LogSoftmax(dim=2)
        self.loss_f = nn.NLLLoss(reduction="none", weight=weights)

    def forward(self, input_seq, output_seq, mask):
        """
        Returns predictions and loss
        """
        encoded_states = self.dropout(self.encoder(input_seq)[0][2])
        predictions = self.activation(self.hidden_to_tag(encoded_states))
        losses = list()
        for i, (pred, active, out) in enumerate(zip(predictions, mask, output_seq)):
            loss = self.loss_f(pred, out)
            masked_loss = loss * active
            losses.append(torch.sum(masked_loss))
        losses = torch.stack(losses)
        total_loss = torch.sum(losses)

        return predictions, total_loss

def get_encoder(itos):
    emb_sz=400
    lin_ftrs = [50]
    layers = [emb_sz*3] + lin_ftrs + [2]

    drops = np.array([0.4, 0.1])

    model = get_rnn_classifier(bptt=70,
                               max_seq=70*20,
                               vocab_sz=len(itos),
                               emb_sz=400,
                               n_hid=1150,
                               n_layers=3,
                               pad_token=1,
                               layers=layers,
                               drops=drops,
                               qrnn=False
                               )
    enc = model[0]
    enc.eval()
    return enc

def load_tag_model(path, device, itos):
    encoder = get_encoder(itos)
    model = LinearDecoder(encoder, DECODER_OUTPUT_DIM)
    state_dict = torch.load(path, map_location=device)
    model.load_state_dict(state_dict)
    model.eval()
    return model

def predict_binary(classifier):
    preds = classifier.get_preds(ordered=True, ds_type=DatasetType.Test)
    return [l.argmax().item() for l in preds[0]]

def predict_labels(model, data, device, thresh=0.0):
    flatten = lambda x: [item for sublist in x for item in sublist]
    model.eval()
    total_loss = 0

    all_labels = list()
    predicted_labels = list()
    predicted_proba = list()

    with torch.no_grad():
        for inputs, outputs, masks in data:
            inputs, outputs, masks = inputs.to(device), outputs.to(device), masks.to(device)
            predictions, loss = model(inputs, outputs, masks)
            total_loss += loss.item()

            outputs_ = outputs.cpu().detach().numpy()
            masks_ = masks.cpu().detach().numpy()
            predictions_ = predictions.cpu().detach().numpy()

            for output_, mask_, pred_ in zip(outputs_, masks_, predictions_):
                predicted = defaultdict(list)
                true_labels = output_[mask_.astype(bool)]
                preds = np.argmax(pred_, axis=1)[mask_.astype(bool)]
                preds_probas = np.exp(np.max(pred_, axis=1)[mask_.astype(bool)])
                thresholded = list()
                for i, (l, p) in enumerate(zip(preds, preds_probas)):
                    if p > thresh:
                        thresholded.append(l)
                        predicted[l].append((i, p))
                    else:
                        thresholded.append(out_stoi["O"])
                if len(predicted[out_stoi["A"]]) < len(predicted[out_stoi["P"]]):
                    less_probable = list(sorted(predicted[out_stoi["P"]], key=lambda x: x[1]))
                    for i, _ in less_probable[:-len(predicted[out_stoi["A"]])]:
                        thresholded[i] = out_stoi["O"]
                predicted_labels.append(thresholded)
                all_labels.extend(true_labels)
                predicted_proba.append(preds_probas)

    return total_loss, predicted_labels, predicted_proba


import datetime
from pyfolio import timeseries
from collections import OrderedDict
from functools import wraps
from pyfolio.utils import APPROX_BDAYS_PER_MONTH, APPROX_BDAYS_PER_YEAR
import pandas as pd
import numpy as np
def get_perf_stats(returns, factor_returns=None, positions=None,
                    transactions=None, turnover_denom='AGB',
                    live_start_date=None, bootstrap=False,
                    header_rows=None):
    """
    Prints some performance metrics of the strategy.
    - Shows amount of time the strategy has been run in backtest and
      out-of-sample (in live trading).
    - Shows Omega ratio, max drawdown, Calmar ratio, annual return,
      stability, Sharpe ratio, annual volatility, alpha, and beta.
    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
    positions : pd.DataFrame, optional
        Daily net position values.
         - See full explanation in create_full_tear_sheet.
    transactions : pd.DataFrame, optional
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet
    turnover_denom : str, optional
        Either AGB or portfolio_value, default AGB.
        - See full explanation in txn.get_turnover.
    live_start_date : datetime, optional
        The point in time when the strategy began live trading, after
        its backtest period.
    bootstrap : boolean, optional
        Whether to perform bootstrap analysis for the performance
        metrics.
         - For more information, see timeseries.perf_stats_bootstrap
    header_rows : dict or OrderedDict, optional
        Extra rows to display at the top of the displayed table.
    """

    if bootstrap:
        perf_func = timeseries.perf_stats_bootstrap
    else:
        perf_func = timeseries.perf_stats

    perf_stats_all = perf_func(
        returns,
        factor_returns=factor_returns,
        positions=positions,
        transactions=transactions,
        turnover_denom=turnover_denom)

    date_rows = OrderedDict()
    if len(returns.index) > 0:
        date_rows['Start date'] = returns.index[0].strftime('%Y-%m-%d')
        date_rows['End date'] = returns.index[-1].strftime('%Y-%m-%d')

    if live_start_date is not None:
        live_start_date = ep.utils.get_utc_timestamp(live_start_date)
        returns_is = returns[returns.index < live_start_date]
        returns_oos = returns[returns.index >= live_start_date]

        positions_is = None
        positions_oos = None
        transactions_is = None
        transactions_oos = None

        if positions is not None:
            positions_is = positions[positions.index < live_start_date]
            positions_oos = positions[positions.index >= live_start_date]
            if transactions is not None:
                transactions_is = transactions[(transactions.index <
                                                live_start_date)]
                transactions_oos = transactions[(transactions.index >
                                                 live_start_date)]

        perf_stats_is = perf_func(
            returns_is,
            factor_returns=factor_returns,
            positions=positions_is,
            transactions=transactions_is,
            turnover_denom=turnover_denom)

        perf_stats_oos = perf_func(
            returns_oos,
            factor_returns=factor_returns,
            positions=positions_oos,
            transactions=transactions_oos,
            turnover_denom=turnover_denom)
        if len(returns.index) > 0:
            date_rows['In-sample months'] = int(len(returns_is) /
                                                APPROX_BDAYS_PER_MONTH)
            date_rows['Out-of-sample months'] = int(len(returns_oos) /
                                                    APPROX_BDAYS_PER_MONTH)

        perf_stats = pd.concat(OrderedDict([
            ('In-sample', perf_stats_is),
            ('Out-of-sample', perf_stats_oos),
            ('All', perf_stats_all),
        ]), axis=1)
    else:
        if len(returns.index) > 0:
            date_rows['Total months'] = int(len(returns) /
                                            APPROX_BDAYS_PER_MONTH)
        perf_stats = pd.DataFrame(perf_stats_all, columns=['Backtest'])
    STAT_FUNCS_PCT = [
        'Annual return',
        'Cumulative returns',
        'Annual volatility',
        'Max drawdown',
        'Daily value at risk',
        'Daily turnover'
    ]
    for column in perf_stats.columns:
        for stat, value in perf_stats[column].iteritems():
            if stat in STAT_FUNCS_PCT:
                perf_stats.loc[stat, column] = str(np.round(value * 100,
                                                            3)) + '%'
    return perf_stats

def stof(x):
    if type(x) == type(3.14): return x
    elif 'inf' in x : return 0
    else: return float(x.strip("%"))/100
#show_perf_stats(return_ser['return'])


def consistent_search(user_str: str, find_str: str) -> int:
    for i in range(len(user_str) - len(find_str)):
        if user_str[i:i + len(find_str)] == find_str:
            return i + 1

    return - 1

def kmp_logic(user_str: str, find_el: str) -> list:
    status = set_status(find_el)
    key_set = set(status.keys())
    match = []
    # Havent found good name for var j
    j = 0
    for i in range(len(user_str)):
        j = status[user_str[i]][j] if user_str[i] in key_set else 0
        if j == len(find_el):
            match.append(i - len(find_el) + 1)
            j = 0
    return match

def set_status(find_el: str) -> dict:
    """
    Checking, might be that find_el contains same letters
    for example in "apple" heres 2 'p'
    """
    status = {el: [0 for _ in range(len(find_el))] for el in set(find_el)}
    for i in range(len(find_el)):
        for j in range(i + 1):
            if find_el[i - j:i] == find_el[0:j]:
                status[find_el[j]][i] = j + 1
    return status

from typing import List

def letters_jump(user_str: str, find_el: str) -> list:
    """
    numeric each letter in find_el (in reversed find_el)
    vpo -> [[o, 0], [p, 1], [v, 2]]
    rest letters in ASCII will have [letter, len(find_el)]
    """
    # creating this [letter, len(find_el)]
    prefix = [[i, len(find_el)] for i in range(255)]
    # numeric this vpo
    find_el_index = [[ord(find_el[::-1][i]), i] for i in range(len(find_el))]

    # if letter was before, it must have same index as last one
    find_el_index = same_letters_find_el(find_el_index)
    # replacing this vpo stuff into prefix
    # so, for vpo [v, len(find_el)] ([v, 3]) => [v, 2]
    for i in range(len(find_el_index)):
        prefix[find_el_index[i][0]][1] = find_el_index[i][1]

    return prefix

def same_letters_find_el(find_el_index: list) -> list:
    """
    [p, 1] -> separating [i][j] to 2 diff lists
    if we found p and its already in our list[i]
    we should take his el from list[j] (so here j == i)
    so our [p, 4] must change to [p, 1] in total
    """
    total_letter: List[list] = []
    letters_ascii_letter: List[int] = []
    letters_word_jump: List[int] = []
    for i in range(len(find_el_index)):
        if find_el_index[i][0] not in letters_ascii_letter:
            letters_ascii_letter.append(find_el_index[i][0])
            letters_word_jump.append(find_el_index[i][1])
            total_letter.append([find_el_index[i][0], find_el_index[i][1]])
        else:
            for j in range(len(find_el_index)):
                if letters_ascii_letter[j] == find_el_index[i][0]:
                    total_letter.append([find_el_index[i][0], letters_word_jump[j]])
                    break

    return total_letter

def boiera_mura_search(user_str: str, find_el: str) -> list:
    """
    by this prefix we will make jump
    lets say that [user_str = 'asdkjvpoijqvpoqego[qf', find_el = 'vpo']
    in prefix [a, 3], so we go from index 0 to index 3 in user_str
    same for [k, 3] to 6
    user_str[6] is p, which is [p, 1] in prefix, so go to 7
    in prefix [o, 0] so we check, does {
      user_str[user_str_index - len(find_el) + 1: user_str_index + 1] == find_el
    }
    after we decide does that equal, we make += len(find_el)
    and like that to the end
    """
    # making this prefix
    prefix = letters_jump(user_str, find_el)
    user_str_index = 0
    result: List[int] = []
    # by user_str_index we will make a jump
    while user_str_index < len(user_str):
        if prefix[ord(user_str[user_str_index])][1] != 0:
            # Adding this for jump
            user_str_index += prefix[ord(user_str[user_str_index])][1]
        else:
            # Here our jump equal to 0
            if user_str[user_str_index - len(find_el) + 1: user_str_index + 1] == find_el:
                result.append(user_str_index - len(find_el) + 1)
            user_str_index += len(find_el)

    return result

if __name__ == '__main__':
    print('You cant run this file as main')


# import the necessary packages
from tkinter import *
from tkinter.filedialog import askopenfilename
from PIL import Image
from PIL import ImageTk
from random import randint
import os
import time

def select_astore():
    filename = askopenfilename(title = "Select Astore",filetypes = (("Astore Models","*.astore"),("all files","*.*")))
    change_astore(filename)
    return

def change_astore(path):
    return

def popupmsg():
    popup = tkinter.Tk()
    popup.wm_title("Changing Astore")
    label = ttk.Label(popup, text=msg, font=NORM_FONT)
    label.pack(side="top", fill="x", pady=10)
    B1 = ttk.Button(popup, text="Okay", command = popup.destroy)
    B1.pack()
    popup.mainloop()

root = Tk()
root.title("SAS Astore Switcher")
root.iconbitmap('icon.ico')
header = Canvas(root, width=700, height=125)
header.grid(row=0, column=0, columnspan=5)
logo = ImageTk.PhotoImage(Image.open("sasbluelogo.png"))
header.create_image(0,0,anchor=NW,image=logo)

title = Canvas(root, width=700, height=25)
title.grid(row=1, column=0, columnspan=5)
title.create_text(30,0, anchor=NW, fill="darkblue", font="Times 18 bold", text = "Change Astore to new model:")

mining      = ImageTk.PhotoImage(Image.open("mining.png"))
insurance   = ImageTk.PhotoImage(Image.open("insurance.png"))
banking     = ImageTk.PhotoImage(Image.open("banking.png"))
government  = ImageTk.PhotoImage(Image.open("police.png"))
health      = ImageTk.PhotoImage(Image.open("health.png"))
unknown     = ImageTk.PhotoImage(Image.open("unknown.png"))

Button(root, text="Mining & Utilities", font="Times 12 bold", image=mining, command= lambda: change_astore("")).grid(row=2, column=0)
Button(root, text="Insurance", font="Times 12 bold", image=insurance, command= lambda: change_astore("")).grid(row=2, column=1)
Button(root, text="Banking", font="Times 12 bold", image=banking, command= lambda: change_astore("")).grid(row=2, column=2)
Button(root, text="Government", font="Times 12 bold", image=government, command= lambda: change_astore("")).grid(row=4, column=0)
Button(root, text="Health", font="Times 12 bold", image=health, command= lambda: change_astore("")).grid(row=4, column=1)
Button(root, text="Create New", font="Times 12 bold", image=unknown, command= select_astore).grid(row=4, column=2)

labels = Canvas(root, width=700, height=25)
labels.grid(row=3, column=0, columnspan=5)
labels.create_text(50,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Mining & Utilities")
labels.create_text(300,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Insurance")
labels.create_text(520,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Banking")

labels = Canvas(root, width=700, height=25)
labels.grid(row=5, column=0, columnspan=5)
labels.create_text(50,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Government")
labels.create_text(300,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Health")
labels.create_text(520,0, anchor=NW, fill="darkblue", font="Times 12 bold", text = "Select New Astore")

footer = Canvas(root, width=700, height=25)
footer.grid(row=6, column=0, columnspan=5)
ftr = ImageTk.PhotoImage(Image.open("bluestrip.png"))
footer.create_image(0,0,anchor=NW,image=ftr)

mainloop()


import sys
import os
from PySide6.QtCore import QSize
from PySide6.QtGui import QAction
from PySide6.QtWidgets import *

class MenuBar(QMainWindow):
    def __init__(self, app):
        super().__init__()
        self.resize(500, 300)
        self.setWindowTitle("Menu Bar")
        self.app = app
        menu_bar = self.menuBar()
        file_menu = menu_bar.addMenu("&File")
        quit_action = file_menu.addAction("Quit")
        quit_action.triggered.connect(lambda: sys.exit())

        edit_menu = menu_bar.addMenu("&Edit")
        edit_menu.addAction("Copy")
        edit_menu.addAction("Cut")
        edit_menu.addAction("Paste")
        edit_menu.addAction("Undo")
        edit_menu.addAction("Redo")

        menu_bar.addMenu("&Window")
        settings = menu_bar.addMenu("&Settings")
        dir = settings.addAction("Dir")
        dir.triggered.connect(lambda: print(os.system("dir")))
        menu_bar.addMenu("&Help")

        toolbar = QToolBar("This is my first toolbar")
        toolbar.setIconSize(QSize(16, 16))
        toolbar.addAction(quit_action)
        self.addToolBar(toolbar)

        action = QAction("First Action", self)
        action.setStatusTip("This is my status message")
        action.triggered.connect(lambda: print("This is QAction"))
        toolbar.addAction(action)

def main():
    app = QApplication(sys.argv)
    window = MenuBar(app)
    window.show()
    app.exec()

if __name__ == "__main__":
    main()


import scrapy
from scrapy.shell import inspect_response

class TheAtlanticInfoSpider(scrapy.Spider):
    name = "the_atlantic_info"
    allowed_domains = ["theatlantic.com"]
    start_urls = ["https://www.theatlantic.com/magazine/archive/2021/09/twenty-years-gone-911-bobby-mcilvaine/619490//"]

    def parse(self, response):

        title = response.xpath('//*[@id="main-content"]/article/header/div[2]/h1/text()').extract()
        author = response.xpath('//*[@id="byline"]/a').extract()
        date_altered = response.xpath(
            '//*[@id="main-content"]/article/header/div[3]/time/attribute::datetime'
        ).extract()
        content = response.xpath('//*[@id="main-content"]/article/section/p/text()').getall()
        data = zip(title, author, date_altered, content)
        for item in data:
            info = {"title": item[0], "author": item[1], "date_altered": item[2], "content": item[3]}
        yield info
        for i in response.xpath("//p[contains(@class, 'ArticleParagraph_root__2QM08')]/text()"):
            print(i.get().lstrip())


import cv2
import numpy as np

dataset = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)

data = list()

while True:
    ret,img = cap.read()
    if ret:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = dataset.detectMultiScale(gray)

        for x,y,w,h in faces:
            cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),3)

            face_comp = img[y:y + h, x:x + w, :]
            fc = cv2.resize(face_comp, (50,50))

            if x % 10 == 0 or len(data) < 20:
                data.append(fc)

        cv2.imshow('result',img)
        if cv2.waitKey(33) == 27 or len(data) >= 20:
            break
    else:
        print("There is some error")

data = np.asarray(data)
np.save('face_1', data)

cap.release()
cv2.destroyAllWindows()


from datetime import datetime
from gpiozero import CPUTemperature
import time
# from subprocess import call

# find common / short name in species label
def short_name(birdname):
    start = birdname.find('(')
    end = birdname.find(')')
    return birdname[start + 1:end] if start >= 0 and end >= 0 else birdname

class DailyChores:

    def __init__(self, tweeter_obj, birdpop, city_weather, output_class=None, maxcpu_c_temp=86):
        self.curr_day = datetime.now().day
        self.curr_hr = datetime.now().hour
        self.starttime = datetime.now()
        self.weather_reported = False
        self.pop_reported = False
        self.tweeter = tweeter_obj
        self.cityweather = city_weather
        self.birdpop = birdpop
        self.output_class = output_class  # take an arguement of class Controller from output_stream.py
        self.output_func = output_class.message if output_class is not None else print
        self.maxcpu_c_temp = maxcpu_c_temp

    # end of process report
    def end_report(self):
        self.tweeter.post_status(f'Ending process at {datetime.now().strftime("%I:%M:%S %P")}.  Run time was ' +
                                 f'{divmod((datetime.now() - self.starttime).total_seconds(), 60)[0]} minutes')

    # check current cpu temp, print, shutdown if overheated
    def check_cpu_temp(self):
        cpu = CPUTemperature()
        self.output_func(f'***hourly temp check. cpu temp is: {cpu.temperature:.1f}C'
                         f' {(cpu.temperature * 9 / 5) + 32:.1f}F')
        try:
            if int(cpu.temperature) >= self.maxcpu_c_temp:  # limit is 85 C
                self.tweeter.post_status(f'***sleeping for 30 minutes. temp: {cpu.temperature}C'
                                         f' {(cpu.temperature * 9 / 5) + 32:.1f}F')
                time.sleep(1800)  # 1800 seconds is 30 min, allow CPU to cool
                # call("sudo shutdown -poweroff")
        except Exception as e:
            self.output_func('Error in temp shutdown protection:', e)
            pass  # uncharted territory....
        return

    # post weather conditions
    def weather_report(self):
        self.tweeter.post_status(f'Current time and weather for {self.cityweather.city} '
                                 f'{datetime.now().strftime("%I:%M:%S %P")}, ' +
                                 f'{self.cityweather.weatherdescription} ' +
                                 f'with {self.cityweather.skycondition}% cloud cover. Visibility of ' +
                                 f'{self.cityweather.visibility} ft.' +
                                 f' Temp is currently {self.cityweather.temp}F with ' +
                                 f'wind speeds of {self.cityweather.windspeed} MPH.' +
                                 f'Sunrise was at {self.cityweather.sunrise: %H:%m}. ' +
                                 f'Sunset is at {self.cityweather.sunset: %H:%m}.')
        return

    # post a report of the top population for birds
    # takes in a list of tuple objects
    # position 0 in tuple is bird name, 1 is count, 2 is last seen
    def top_pop_report(self):
        try:
            observed = self.birdpop.get_census_by_count()
            post_txt = f'Top birds for day {str(self.curr_day)} - '
            for index, birdkey in enumerate(observed):  # bird pop is list of tuples with 0th item species name
                bird_txt = str(f'#{str(index + 1)}: {observed[birdkey][0]} {short_name(birdkey)}, ') \
                    if observed[birdkey][0] > 0 else ''  # post observed bird if count > 0 else keep prior txt
                post_txt = post_txt + bird_txt if len(bird_txt) + len(post_txt) < 280 else post_txt
            self.tweeter.post_status(post_txt[0:279])  # grab full text up to 280 characters
        except Exception as e:
            self.output_func('Error in daily population report:', e)
            pass  # just keep going...
        return

    # housekeeping for day and hour
    # takes a pointer to the population tracking object
    def hourly_and_daily(self, filename='', report_pop=False):
        # post the weather once during the daytime and while the sun is rising
        if self.weather_reported is False and self.cityweather.is_daytime() and \
                self.cityweather.is_dawn():
            self.weather_report()
            self.weather_reported = True

        if report_pop:  # process is ending report populations observations
            self.pop_reported = True
            self.top_pop_report()
            self.birdpop.clear()  # clear count for new day

        if self.curr_hr != datetime.now().hour:  # check weather and CPU temp hourly
            self.check_cpu_temp()
            if self.output_class is not None:  # output is not being sent to the default print function
                self.output_class.occurrences(self.birdpop.get_occurrences())
                self.output_class.message(message=f'Seed and camera position check. ', image_name=filename)
                self.output_class.flush()

        self.curr_hr = datetime.now().hour
        self.curr_day = datetime.now().day
        return


import sys
sys.path.append('../../')
import os
from utils import progressBar

##############################
# ##### Spike encoding ##### #
##############################
subsets = [('subset1', 'A,B,G,H,P,R'), ('subset2', 'G,H,I,J,K,L,M')]
filterbanks = ['butterworth', 'gammatone']
channels = [4, 8, 16]
for subsetName, subsetLabels in subsets:
    for filterbank in filterbanks:
        for channel in channels:
            os.system(f'python encodingSpike.py -d=datasetWatch -n={subsetName} -s={subsetLabels} -f={filterbank} -c={channel}')

###################################
# ##### Sonogram generation ##### #
###################################
##### Encoding algorithm selected #####
encodings = ['RATE', 'TBR', 'SF', 'MW', 'PFM', 'HSA', 'MHSA', 'BSA', 'PHASE', 'TTFS', 'BURST']

##### Binning settings #####
configurations = [
    ('butterworth', 4, 1000/24),  # (32, 70)big bins  (32, 10)small bins
    ('gammatone', 4, 1000/24),

    ('butterworth', 8, 1000/18),
    ('gammatone', 8, 1000/18),

    ('butterworth', 16, 1000/18),
    ('gammatone', 16, 1000/18),
]

for subsetName, _ in subsets:
    for encoding in encodings:
        progressBar(0, len(configurations), f'{subsetName}, Encoding: {encoding}')
        p = 1
        for configuration in configurations:
            filterbank, channel, binsWindow = configuration
            os.system(f'python encodingSonogram.py -n={subsetName} -e={encoding} -f={filterbank} -c={channel} -b={binsWindow}')
            progressBar(p, len(configurations), f'{subsetName}, Encoding: {encoding}')
            p += 1