##########################################
# To parse log files generated by abelian.
# Author: Gurbinder Gill
# Email: gurbinder533@gmail.com
#########################################

import re
import os
import sys, getopt
import csv
import numpy
import subprocess

######## NOTES:
# All time values are in sec by default.


def match_timers(fileName, benchmark, forHost, numRuns, numThreads, time_unit, total_hosts, partition):

  mean_time = 0.0;
  recvNum_total = 0
  recvBytes_total = 0
  sendNum_total = 0
  sendBytes_total = 0
  sync_pull_avg_time_total = 0.0;
  extract_avg_time_total = 0.0;
  set_avg_time_total = 0.0;
  sync_push_avg_time_total = 0.0;
  graph_init_time = 0
  hg_init_time = 0
  total_time = 0

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  if(benchmark == "pagerank"):
    benchmark = "PageRank"

  if (time_unit == 'seconds'):
    divisor = 1000
  else:
    divisor = 1

  log_data = open(fileName).read()

  timer_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_(\d*),(\d*),0,(\d*)')
  timers = re.findall(timer_regex, log_data)
  #print timers

  time = []

  for i in range(int(total_hosts)):
    time.append(0)

  #overall slowest host for all 3 runs
  slowest_host = int(0);
  for timer in timers:
    host = int(timer[1])
    #print host ," : ", timer[2]
    host_time = float(timer[2])
    time[host] += host_time

  print time
  for i, val in enumerate(time):
    if(time[slowest_host] < val):
      slowest_host = i

  print slowest_host
  print time[slowest_host]

  if(len(time) > 0):
    mean_time = time[slowest_host] / int(numRuns)
  mean_time /= divisor
  mean_time = round(mean_time, 3)
  print "Mean time: ", mean_time

  forHost = slowest_host 


  rep_regex = re.compile(r'.*,\(NULL\),0\s,\sREPLICATION_FACTOR_0_0,(\d*),\d*,(.*)')

  rep_search = rep_regex.search(log_data)
  rep_factor = 0;
  if rep_search is not None:
    rep_factor = rep_search.group(2)
    rep_factor = round(float(rep_factor), 3)
    if (rep_factor == 1.0):
      rep_regex = re.compile(r'.*,\(NULL\),0\s,\sREPLICATION_FACTOR_0_0,(\d*),\d*,(.*)')
      rep_search = rep_regex.search(log_data)
      rep_factor = rep_search.group(2)
      rep_factor = round(float(rep_factor), 3)

    print ("Replication factor  : ", rep_factor)


  #Finding mean, variance and sd for first iteration in compute time
  do_all_regex = re.compile(r'.*,\(NULL\),0\s,\sDO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_0_0'  +r',.*' + r',\d*,(\d*)')
  do_all_all_hosts = re.findall(do_all_regex, log_data)
  num_arr = numpy.array(map(int,do_all_all_hosts))

  if(num_arr.size < total_hosts):
    #for i in range(1, int(total_hosts)):
    for i in range(1, 3):
      do_all_regex = re.compile(r'.*,\(NULL\),0\s,\sDO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_0_' + re.escape(str(i)) +r',.*' + r',\d*,(\d*)')
      do_all_all_hosts = re.findall(do_all_regex, log_data)
      num_arr = numpy.array(map(int,do_all_all_hosts))
      if(num_arr.size == total_hosts):
        break;

  print num_arr
  sd_do_all = round(numpy.std(num_arr, axis=0),3)
  mean_do_all = round(numpy.mean(num_arr, axis=0),3)
  var_do_all = round(numpy.var(num_arr, axis=0),3)
  min_do_all = num_arr.min(axis=0)
  max_do_all = num_arr.max(axis=0)


  print "sd_do_all", sd_do_all
  print "mean_do_all", mean_do_all
  print "min_do_all", min_do_all
  print "max_do_all", max_do_all


  #Finding mean, variance and sd for first iteration in sync time
  sync_pull_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PULL_(?i)' + re.escape(benchmark) + r'_0_0'  +r',.*' + r',\d*,(\d*)')
  sync_pull_all_hosts = re.findall(sync_pull_regex, log_data)
  num_arr = numpy.array(map(int,sync_pull_all_hosts))
  if(num_arr.size == 0):
    sync_pull_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PULL_FirstItr_(?i)' + re.escape(benchmark) + r'_0_0'  +r',.*' + r',\d*,(\d*)')
    sync_pull_all_hosts = re.findall(sync_pull_regex, log_data)
    num_arr = numpy.array(map(int,sync_pull_all_hosts))

  if(num_arr.size < total_hosts):
    #for i in range(1, int(total_hosts)):
    for i in range(1, 3):
      sync_pull_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PULL_(?i)' + re.escape(benchmark) + r'_0_' + re.escape(str(i)) +r',.*' + r',\d*,(\d*)')
      sync_pull_all_hosts = re.findall(sync_pull_regex, log_data)
      try:
        num_arr = numpy.array(map(int,sync_pull_all_hosts))
      except ValueError:
        pass
      if(num_arr.size == total_hosts):
        break;

  print "SYNC_PULL" , num_arr
  sd_sync_pull = 0.0
  mean_sync_pull = 0.0
  var_sync_pull = 0.0
  min_sync_pull = 0.0
  max_sync_pull = 0.0
  if(num_arr.size > 0):
    try:
      sd_sync_pull = round(numpy.std(num_arr, axis=0),3)
      mean_sync_pull = round(numpy.mean(num_arr, axis=0),3)
      var_sync_pull = round(numpy.var(num_arr, axis=0),3)
      min_sync_pull = num_arr.min(axis=0)
      max_sync_pull = num_arr.max(axis=0)
    except ValueError:  #raised if `num_arr` is empty.
      pass

  print "sd_sync_pull", sd_sync_pull
  print "mean_sync_pull", mean_sync_pull
  print "min_sync_pull", min_sync_pull
  print "max_sync_pull", max_sync_pull

  sync_push_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PUSH_(?i)' + re.escape(benchmark) + r'_0_0'  +r',.*' + r',\d*,(\d*)')
  sync_push_all_hosts = re.findall(sync_push_regex, log_data)
  num_arr = numpy.array(map(int,sync_push_all_hosts))
  if(num_arr.size == 0):
    sync_push_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PUSH_FirstItr_(?i)' + re.escape(benchmark) + r'_0_0'  +r',.*' + r',\d*,(\d*)')
    sync_push_all_hosts = re.findall(sync_push_regex, log_data)
    num_arr = numpy.array(map(int,sync_push_all_hosts))


  if(num_arr.size < total_hosts):
    #for i in range(1, int(total_hosts)):
    for i in range(1, 3):
      sync_push_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PUSH_(?i)' + re.escape(benchmark) + r'_0_' + re.escape(str(i)) + r',.*' + r',\d*,(\d*)')
      sync_push_all_hosts = re.findall(sync_push_regex, log_data)
      num_arr = numpy.array(map(int,sync_push_all_hosts))
      if(num_arr.size == total_hosts):
        break;

  print "SYNC_PUSH" , num_arr

  sd_sync_push = 0.0
  mean_sync_push = 0.0
  var_sync_push = 0.0
  min_sync_push = 0.0
  max_sync_push = 0.0
  if(num_arr.size > 0):
    try:
      sd_sync_push = round(numpy.std(num_arr, axis=0),3)
      mean_sync_push = round(numpy.mean(num_arr, axis=0),3)
      var_sync_push = round(numpy.var(num_arr, axis=0),3)
      min_sync_push = num_arr.min(axis=0)
      max_sync_push = num_arr.max(axis=0)
    except ValueError:  #raised if `num_arr` is empty.
      pass
  print "sd_sync_push", sd_sync_push
  print "mean_sync_push", mean_sync_push
  print "min_sync_push", min_sync_push
  print "max_sync_push", max_sync_push



  ## Get Graph_init, HG_init, total
  #81a5b117-8054-46af-9a23-1f28e5ed1bba,(NULL),0 , TIMER_GRAPH_INIT,0,0,306
  timer_graph_init_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_GRAPH_INIT,\d*,\d*,(\d*)')
  timer_hg_init_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_HG_INIT' + r',\d*,\d*,(\d*)')
  timer_total_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_TOTAL' + r',\d*,\d*,(\d*)')


  timer_graph_init = timer_graph_init_regex.search(log_data)
  timer_hg_init = timer_hg_init_regex.search(log_data)
  timer_total = timer_total_regex.search(log_data)

  if timer_graph_init is not None:
    graph_init_time = float(timer_graph_init.group(1))
    graph_init_time /= divisor
    graph_init_time = round(graph_init_time, 3)

  if timer_hg_init is not None:
    hg_init_time = float(timer_hg_init.group(1))
    hg_init_time /= divisor
    hg_init_time = round(hg_init_time, 3)

  if timer_total is not None:
    total_time = float(timer_total.group(1))
    total_time /= divisor
    total_time = round(total_time, 3)

  print graph_init_time
  print hg_init_time
  print total_time

  return mean_time,rep_factor,mean_do_all,sd_do_all,var_do_all,min_do_all,max_do_all,mean_sync_pull,sd_sync_pull,var_sync_pull,min_sync_pull,max_sync_pull,mean_sync_push,sd_sync_push,var_sync_push,min_sync_push,max_sync_push,graph_init_time,hg_init_time,total_time

def get_basicInfo(fileName):

  hostNum_regex = re.compile(r'.*,\(NULL\),0\s,\sHosts,0,0,(\d*)')
  cmdLine_regex = re.compile(r'.*,\(NULL\),0\s,\sCommandLine,0,0,(.*)')
  threads_regex = re.compile(r'.*,\(NULL\),0\s,\sThreads,0,0,(\d*)')
  runs_regex = re.compile(r'.*,\(NULL\),0\s,\sRuns,0,0,(\d*)')

  log_data = open(fileName).read()

  hostNum    = ''
  cmdLine    = ''
  threads    = ''
  runs       = ''
  benchmark  = ''
  algo_type  = ''
  cut_type   = ''
  input_graph = ''

  hostNum_search = hostNum_regex.search(log_data)
  if hostNum_search is not None:
    hostNum = hostNum_search.group(1)

  cmdLine_search = cmdLine_regex.search(log_data)
  if cmdLine_search is not None:
    cmdLine = cmdLine_search.group(1)

  threads_search = threads_regex.search(log_data)
  if threads_search is not None:
    threads = threads_search.group(1)

  runs_search    = runs_regex.search(log_data)
  if runs_search is not None:
    runs = runs_search.group(1)
  if runs == "":
    runs = "3"

  print cmdLine
  split_cmdLine_algo = cmdLine.split()[0].split("/")[-1].split("_")
  benchmark, algo_type = split_cmdLine_algo

  split_cmdLine_input = cmdLine.split()[1].split("/")
  input_graph_name = split_cmdLine_input[-1]
  input_graph = input_graph_name.split(".")[0]

  print cmdLine
  split_cmdLine = cmdLine.split()
  print split_cmdLine
  cut_type = "edge-cut"
  for index in range(0, len(split_cmdLine)):
    if split_cmdLine[index] == "-enableVertexCut=1":
      cut_type = "vertex-cut"
      break
    elif split_cmdLine[index] == "-enableVertexCut":
         cut_type = "vertex-cut"
         break
    elif split_cmdLine[index] == "-enableVertexCut=0":
         cut_type = "edge-cut"
         break


  devices = str(hostNum) + " CPU"
  deviceKind = "CPU"
  for index in range(2, len(cmdLine.split())):
    split_cmdLine_devices = cmdLine.split()[index].split("=")
    if split_cmdLine_devices[0] == '-pset':
      devices_str = split_cmdLine_devices[-1]
      cpus = devices_str.count('c')
      gpus = devices_str.count('g')
      if str(cpus + gpus) == hostNum and gpus > 0:
        if cpus == 0:
          devices = str(gpus) + " GPU"
          deviceKind = "GPU"
        else:
          devices = str(cpus) + " CPU + " + str(gpus) + " GPU"
          deviceKind = "CPU+GPU"
          hostNum = str(int(hostNum) - cpus)
      break

  return hostNum, cmdLine, threads, runs, benchmark, algo_type, cut_type, input_graph, devices, deviceKind

def format_str(col):
  max_len = 0
  for c in col:
    if max_len < len(str(c)):
      max_len = len(str(c))
  return max_len

def main(argv):
  inputFile = ''
  forHost = ''
  outputFile = 'LOG_output.csv'
  time_unit = 'seconds'
  try:
    opts, args = getopt.getopt(argv,"hi:n:o:md",["ifile=","node=","ofile=","milliseconds"])
  except getopt.GetoptError:
    print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)
  for opt, arg in opts:
    if opt == '-h':
      print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
      sys.exit()
    elif opt in ("-i", "--ifile"):
      inputFile = arg
    elif opt in ("-n", "--node"):
      forHost = arg
    elif opt in ("-o", "--ofile"):
      outputFile = arg
    elif opt in ("-m", "--milliseconds"):
      time_unit = 'milliseconds'

  if inputFile == '':
    print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)

  print 'Input file is : ', inputFile
  print 'Output file is : ', outputFile
  print 'Data for host : ', forHost

  if forHost == '':
    print 'Find the slowest host and calculating everything for that host'

  hostNum, cmdLine, threads, runs, benchmark, algo_type, cut_type, input_graph, devices, deviceKind = get_basicInfo(inputFile)

  #shorten the graph names:
  if input_graph == "twitter-ICWSM10-component_withRandomWeights" or input_graph == "twitter-ICWSM10-component-transpose" or input_graph == "twitter-ICWSM10-component":
    input_graph = "twitter-50"
  elif input_graph == "twitter-WWW10-component_withRandomWeights" or input_graph == "twitter-WWW10-component-transpose" or input_graph == "twitter-WWW10-component":
    input_graph = "twitter-40"

  print 'Hosts : ', hostNum , ' CmdLine : ', cmdLine, ' Threads : ', threads , ' Runs : ', runs, ' benchmark :' , benchmark , ' algo_type :', algo_type, ' cut_type : ', cut_type, ' input_graph : ', input_graph
  print 'Devices : ', devices
  data = match_timers(inputFile, benchmark, forHost, runs, threads, time_unit, hostNum, cut_type)

  print data

  output_str = benchmark + ',' + 'abelian' + ',' + hostNum  + ',' + threads  + ','
  output_str += deviceKind  + ',' + devices  + ','
  output_str += input_graph  + ',' + algo_type  + ',' + cut_type
  print output_str


  header_csv_str = "benchmark,platform,host,threads,"
  header_csv_str += "deviceKind,devices,"
  header_csv_str += "input,variant,partition,mean_time,rep_factor,mean_do_all,sd_do_all,var_do_all,min_do_all,max_do_all,mean_sync_pull,sd_sync_pull,var_sync_pull,min_sync_pull,max_sync_pull,mean_sync_push,sd_sync_push,var_sync_push,min_sync_push,max_sync_push,graph_init_time,hg_init_time,total_time"
  

  header_csv_list = header_csv_str.split(',')
  try:
    if os.path.isfile(outputFile) is False:
      fd_outputFile = open(outputFile, 'wb')
      wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
      wr.writerow(header_csv_list)
      fd_outputFile.close()
      print "Adding header to the empty file."
    else:
      print "outputFile : ", outputFile, " exists, results will be appended to it."
  except OSError:
    print "Error in outfile opening\n"

  data_list = list(data) #[data] #list(data)
  complete_data = output_str.split(",") + data_list
  fd_outputFile = open(outputFile, 'a')
  wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
  wr.writerow(complete_data)
  fd_outputFile.close()

'''
  ## Write ghost and slave nodes to a file.
  ghost_array = build_master_ghost_matrix(inputFile, benchmark, cut_type, hostNum, runs, threads)
  ghostNodes_file = outputFile + "_" + cut_type
  fd_ghostNodes_file = open(ghostNodes_file, 'ab')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.write("\nHosts : " + hostNum + "\nInputFile : "+ inputFile + "\nBenchmark: " + benchmark + "\nPartition: " + cut_type + "\n\n")
  numpy.savetxt(fd_ghostNodes_file, ghost_array, delimiter=',', fmt='%d')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.close()
'''

if __name__ == "__main__":
  main(sys.argv[1:])

