########################################################################
# $HeadURL$
# Author : Andrei Tsaregorodtsev
########################################################################
"""
  Utilities for managing DIRAC configuration:

  getCEsFromCS
  getUnusedGridCEs
  getUnusedGridSEs
  getSiteUpdates
  getSEUpdates
"""
__RCSID__ = "$Id$"

import re
import types
from DIRAC import gConfig, gLogger, S_OK
from DIRAC.Core.Utilities import List
from DIRAC.Core.Utilities.Grid import getBdiiCEInfo, getBdiiSEInfo, ldapService
from DIRAC.Core.Utilities.SitesDIRACGOCDBmapping import getDIRACSiteName, getDIRACSesForSRM
from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
from DIRAC.Core.Utilities.Pfn import pfnparse
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOs, getVOOption

def getGridVOs():
  """ Get all the VOMS VO names served by this DIRAC service
  """
  voNames = []
  result = getVOs()
  if not result['OK']:
    return result
  else:
    vos = result['Value']
    for vo in vos:
      vomsVO = getVOOption( vo, "VOMSName" )
      if vomsVO:
        voNames.append( vomsVO )
  return S_OK( voNames )

def getCEsFromCS():
  """ Get all the CEs defined in the CS
  """

  knownCEs = []
  result = gConfig.getSections( '/Resources/Sites' )
  if not result['OK']:
    return result
  grids = result['Value']

  for grid in grids:
    result = gConfig.getSections( '/Resources/Sites/%s' % grid )
    if not result['OK']:
      return result
    sites = result['Value']

    for site in sites:
      opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
      ces = List.fromChar( opt.get( 'CE', '' ) )
      knownCEs += ces

  return S_OK( knownCEs )

def getSEsFromCS( protocol = 'srm' ):
  """ Get all the SEs defined in the CS
  """
  knownSEs = {}
  result = gConfig.getSections( '/Resources/StorageElements' )
  if not result['OK']:
    return result
  ses = result['Value']
  for se in ses:
    seSection = '/Resources/StorageElements/%s' % se
    result = gConfig.getSections( seSection )
    if not result['OK']:
      continue
    accesses = result['Value']
    for access in accesses:
      seProtocol = gConfig.getValue( cfgPath( seSection, access, 'Protocol' ), '' )
      if seProtocol.lower() == protocol.lower() or protocol == 'any':
        host = gConfig.getValue( cfgPath( seSection, access, 'Host' ), '' )
        knownSEs.setdefault( host, [] )
        knownSEs[host].append( se )
      else:
        continue

  return S_OK( knownSEs )

def getGridCEs( vo, bdiiInfo = None, ceBlackList = None ):
  """ Get all the CEs available for a given VO and having queues in Production state
  """
  knownCEs = set()
  if ceBlackList is not None:
    knownCEs = knownCEs.union( set( ceBlackList ) )

  ceBdiiDict = bdiiInfo
  if bdiiInfo is None:
    result = getBdiiCEInfo( vo )
    if not result['OK']:
      return result
    ceBdiiDict = result['Value']

  siteDict = {}
  for site in ceBdiiDict:
    siteCEs = set( ceBdiiDict[site]['CEs'].keys() )
    newCEs = siteCEs - knownCEs
    if not newCEs:
      continue

    ceFullDict = {}
    for ce in newCEs:
      ceDict = {}
      ceInfo = ceBdiiDict[site]['CEs'][ce]
      ceType = 'Unknown'
      ceDict['Queues'] = []
      for queue in ceInfo['Queues']:
        queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
        if 'production' in queueStatus.lower():
          ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
          ceDict['Queues'].append( queue )
      if not ceDict['Queues']:
        continue

      ceDict['CEType'] = ceType
      ceDict['GOCSite'] = site
      ceDict['CEID'] = ce
      systemName = ceInfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
      systemVersion = ceInfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
      systemRelease = ceInfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
      ceDict['System'] = ( systemName, systemVersion, systemRelease )

      ceFullDict[ce] = ceDict

    siteDict[site] = ceFullDict

  result = S_OK( siteDict )
  result['BdiiInfo'] = ceBdiiDict
  return result

def getSiteUpdates( vo, bdiiInfo = None, log = None ):
  """ Get all the necessary updates for the already defined sites and CEs
  """

  def addToChangeSet( entry, changeSet ):
    _section, _option, value, new_value = entry
    if new_value and new_value != value:
      changeSet.add( entry )

  if log is None:
    log = gLogger

  ceBdiiDict = bdiiInfo
  if bdiiInfo is None:
    result = getBdiiCEInfo( vo )
    if not result['OK']:
      return result
    ceBdiiDict = result['Value']

  changeSet = set()
  gConfig.forceRefresh()

  for site in ceBdiiDict:
    result = getDIRACSiteName( site )
    if not result['OK']:
      continue
    siteNames = result['Value']
    for siteName in siteNames:
      siteSection = cfgPath( '/Resources', 'Sites', siteName.split('.')[0], siteName )
      result = gConfig.getOptionsDict( siteSection )
      if not result['OK']:
        continue
      siteDict = result['Value']
      # Current CS values
      coor = siteDict.get( 'Coordinates', 'Unknown' )
      mail = siteDict.get( 'Mail', 'Unknown' ).replace( ' ','' )
      description = siteDict.get( 'Description', 'Unknown' )

      longitude = ceBdiiDict[site].get( 'GlueSiteLongitude', '' ).strip()
      latitude = ceBdiiDict[site].get( 'GlueSiteLatitude', '' ).strip()

      # Current BDII value
      newcoor = ''
      if longitude and latitude:
        newcoor = "%s:%s" % ( longitude, latitude )
      newmail = ceBdiiDict[site].get( 'GlueSiteSysAdminContact', '' ).replace( 'mailto:', '' ).strip()
      newdescription = ceBdiiDict[site].get( 'GlueSiteDescription', '' ).strip()
      # Adding site data to the changes list
      addToChangeSet( ( siteSection, 'Coordinates', coor, newcoor ), changeSet )
      addToChangeSet( ( siteSection, 'Mail', mail, newmail ), changeSet )
      addToChangeSet( ( siteSection, 'Description', description, newdescription ), changeSet )

      ces = gConfig.getValue( cfgPath( siteSection, 'CE' ), [] )
      for ce in ces:
        ceSection = cfgPath( siteSection, 'CEs', ce )
        ceDict = {}
        result = gConfig.getOptionsDict( ceSection )
        if result['OK']:
          ceDict = result['Value']
        else:
          if ceBdiiDict[site]['CEs'].get( ce, None ):
            log.notice( "Adding new CE %s to site %s/%s" % (ce, siteName, site) )
        ceInfo = ceBdiiDict[site]['CEs'].get( ce, None )
        if ceInfo is None:
          ceType = ceDict.get( 'CEType', '')
          continue

        # Current CS CE info
        arch = ceDict.get( 'architecture', 'Unknown' )
        OS = ceDict.get( 'OS', 'Unknown' )
        si00 = ceDict.get( 'SI00', 'Unknown' )
        ceType = ceDict.get( 'CEType', 'Unknown' )
        ram = ceDict.get( 'HostRAM', 'Unknown' )
        submissionMode = ceDict.get( 'SubmissionMode', 'Unknown' )

        # Current BDII CE info
        newarch = ceBdiiDict[site]['CEs'][ce].get( 'GlueHostArchitecturePlatformType', '' ).strip()
        systemName = ceInfo.get( 'GlueHostOperatingSystemName', '' ).strip()
        systemVersion = ceInfo.get( 'GlueHostOperatingSystemVersion', '' ).strip()
        systemRelease = ceInfo.get( 'GlueHostOperatingSystemRelease', '' ).strip()
        newOS = ''
        if systemName and systemVersion and systemRelease:
          newOS = '_'.join( ( systemName, systemVersion, systemRelease ) )
        newsi00 = ceInfo.get( 'GlueHostBenchmarkSI00', '' ).strip()
        newCEType = 'Unknown'
        for queue in ceInfo['Queues']:
          queueDict = ceInfo['Queues'][queue]
          newCEType = queueDict.get( 'GlueCEImplementationName', '' ).strip()
          if newCEType:
            break
        if newCEType=='ARC-CE':
          newCEType = 'ARC'

        newSubmissionMode = None
        if newCEType in ['ARC','CREAM']:
          newSubmissionMode = "Direct"
        newRAM = ceInfo.get( 'GlueHostMainMemoryRAMSize', '' ).strip()

        # Adding CE data to the change list
        addToChangeSet( ( ceSection, 'architecture', arch, newarch ), changeSet )
        addToChangeSet( ( ceSection, 'OS', OS, newOS ), changeSet )
        addToChangeSet( ( ceSection, 'SI00', si00, newsi00 ), changeSet )
        addToChangeSet( ( ceSection, 'CEType', ceType, newCEType ), changeSet )
        addToChangeSet( ( ceSection, 'HostRAM', ram, newRAM ), changeSet )
        if submissionMode == "Unknown" and newSubmissionMode:
          addToChangeSet( ( ceSection, 'SubmissionMode', submissionMode, newSubmissionMode ), changeSet )

        queues = ceInfo['Queues'].keys()
        for queue in queues:
          queueSection = cfgPath( ceSection, 'Queues', queue )
          queueDict = {}
          result = gConfig.getOptionsDict( queueSection )
          if result['OK']:
            queueDict = result['Value']
          else:
            log.notice( "Adding new queue %s to CE %s" % (queue, ce) )
          queueInfo = ceInfo['Queues'][queue]
          queueStatus = queueInfo['GlueCEStateStatus']
          if queueStatus.lower() != "production":
            continue

          # Current CS queue info
          maxCPUTime = queueDict.get( 'maxCPUTime', 'Unknown' )
          si00 = queueDict.get( 'SI00', 'Unknown' )
          maxTotalJobs = queueDict.get( 'MaxTotalJobs', 'Unknown' )

          # Current BDII queue info
          newMaxCPUTime = queueInfo.get( 'GlueCEPolicyMaxCPUTime', '' )
          newSI00 = ''
          caps = queueInfo['GlueCECapability']
          if type( caps ) == type( '' ):
            caps = [caps]
          for cap in caps:
            if 'CPUScalingReferenceSI00' in cap:
              newSI00 = cap.split( '=' )[-1]

          # Adding queue info to the CS
          addToChangeSet( ( queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime ), changeSet )
          addToChangeSet( ( queueSection, 'SI00', si00, newSI00 ), changeSet )
          if maxTotalJobs == "Unknown":
            newTotalJobs =  min( 1000, int( int( queueInfo.get( 'GlueCEInfoTotalCPUs', 0 ) )/2 ) )
            newWaitingJobs =  max( 2, int( newTotalJobs * 0.1 ) )
            newTotalJobs = str( newTotalJobs )
            newWaitingJobs = str( newWaitingJobs )
            addToChangeSet( ( queueSection, 'MaxTotalJobs', '', newTotalJobs ), changeSet )
            addToChangeSet( ( queueSection, 'MaxWaitingJobs', '', newWaitingJobs ), changeSet )

          # Updating eligible VO list
          VOs = set()
          if queueDict.get( 'VO', '' ):
            VOs = set( [ q.strip() for q in queueDict.get( 'VO', '' ).split( ',' ) if q ] )
          if not vo in VOs:
            VOs.add( vo )
            VOs = list( VOs )
            newVOs = ','.join( VOs )
            addToChangeSet( ( queueSection, 'VO', '', newVOs ), changeSet )

  return S_OK( changeSet )

def getGridSEs( vo, bdiiInfo = None, seBlackList = None ):
  """ Get all the SEs available for a given VO
  """
  seBdiiDict = bdiiInfo
  if bdiiInfo is None:
    result = getBdiiSEInfo( vo )
    if not result['OK']:
      return result
    seBdiiDict = result['Value']

  knownSEs = set()
  if seBlackList is not None:
    knownSEs = knownSEs.union( set( seBlackList ) )

  siteDict = {}
  for site in seBdiiDict:
    for gridSE in seBdiiDict[site]['SEs']:
      seDict = seBdiiDict[site]['SEs'][gridSE]

      #if "lhcb" in seDict['GlueSAName']:
      #  print '+'*80
      #  print gridSE
      #  for k,v in seDict.items():
      #    print k,'\t',v

      if not gridSE in knownSEs:
        siteDict.setdefault( site, {} )
        if type( seDict['GlueSAAccessControlBaseRule'] ) == types.ListType:
          voList = [ re.sub( '^VO:', '', s ) for s in seDict['GlueSAAccessControlBaseRule'] ]
        else:
          voList = [ re.sub( '^VO:', '', seDict['GlueSAAccessControlBaseRule'] ) ]
        siteDict[site][gridSE] = { 'GridSite': seDict['GlueSiteUniqueID'],
                                   'BackendType': seDict['GlueSEImplementationName'],
                                   'Description': seDict.get( 'GlueSEName', '-' ),
                                   'VOs': voList
                                 }

  result = S_OK( siteDict )
  result['BdiiInfo'] = seBdiiDict
  return result

def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ):

  result = ldapService( serviceType = 'SRM', vo = vo )
  if not result['OK']:
    return result
  srmBdiiDict = result['Value']

  knownSRMs = set()
  if srmBlackList is not None:
    knownSRMs = knownSRMs.union( set( srmBlackList ) )

  siteSRMDict = {}
  for srm in srmBdiiDict:

    endPoint = srm.get( 'GlueServiceEndpoint', '')
    srmHost = ''
    if endPoint:
      result = pfnparse( endPoint )
      if not result['OK']:
        continue
      srmHost = result['Value']['Host']
    if not srmHost:
      continue

    if srmHost in knownSRMs:
      continue

    if unUsed:
      result = getDIRACSesForSRM( srmHost )
      if not result['OK']:
        return result
      diracSEs = result['Value']
      if diracSEs:
        # If it is a known SRM and only new SRMs are requested, continue
        continue
    site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' )
    siteSRMDict.setdefault( site, {} )
    siteSRMDict[site][srmHost] = srm

  if bdiiInfo is None:
    result = getBdiiSEInfo( vo )
    if not result['OK']:
      return result
    seBdiiDict = result['Value']
  else:
    seBdiiDict = bdiiInfo

  srmSeDict = {}
  for site in siteSRMDict:
    srms = siteSRMDict[site].keys()
    for srm in srms:
      if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ):
        srmSeDict.setdefault( site, {} )
        srmSeDict[site].setdefault( srm, {} )
        srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
        srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]

  return S_OK( srmSeDict )

def getSRMUpdates( vo, bdiiInfo = None ):

  changeSet = set()

  def addToChangeSet( entry, changeSet ):
    _section, _option, value, new_value = entry
    if new_value and new_value != value:
      changeSet.add( entry )

  result = getGridSRMs( vo, bdiiInfo = bdiiInfo )
  if not result['OK']:
    return result
  srmBdiiDict = result['Value']

  result = getSEsFromCS()
  if not result['OK']:
    return result
  seDict = result['Value']

  result = getVOs()
  if result['OK']:
    csVOs = set( result['Value'] )
  else:
    csVOs = set( [vo] )

  for seHost, diracSE in seDict.items():
    seSection = '/Resources/StorageElements/%s' % diracSE[0]
    # Look up existing values first
    description = gConfig.getValue( cfgPath( seSection, 'Description'), 'Unknown' )
    backend = gConfig.getValue( cfgPath( seSection, 'BackendType'), 'Unknown' )
    vos = gConfig.getValue( cfgPath( seSection, 'VO'), 'Unknown' ).replace( ' ','' )
    size = gConfig.getValue( cfgPath( seSection, 'TotalSize'), 'Unknown' )
    # Look up current BDII values
    srmDict = {}
    seBdiiDict = {}
    for site in srmBdiiDict:
      if seHost in srmBdiiDict[site]:
        srmDict = srmBdiiDict[site][seHost]['SRM']
        seBdiiDict = srmBdiiDict[site][seHost]['SE']
        break

    if not srmDict or not seBdiiDict:
      continue

    newDescription = seBdiiDict.get( 'GlueSEName', 'Unknown' )
    newBackend = seBdiiDict.get( 'GlueSEImplementationName', 'Unknown' )
    newSize = seBdiiDict.get( 'GlueSESizeTotal', 'Unknown' )
    addToChangeSet( ( seSection, 'Description', description, newDescription ), changeSet )
    addToChangeSet( ( seSection, 'BackendType', backend, newBackend ), changeSet )
    addToChangeSet( ( seSection, 'TotalSize', size, newSize ), changeSet )

    # Evaluate VOs if no space token defined, otherwise this is VO specific
    spaceToken = ''
    for i in range( 1, 10 ):
      protocol = gConfig.getValue( cfgPath( seSection, 'AccessProtocol.%d' % i, 'Protocol' ), '' )
      if protocol.lower() == 'srm':
        spaceToken = gConfig.getValue( cfgPath( seSection, 'AccessProtocol.%d' % i, 'SpaceToken' ), '' )
        break
    if not spaceToken:
      bdiiVOs = srmDict.get( 'GlueServiceAccessControlBaseRule', [] )
      bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in bdiiVOs ] )
      seVOs = csVOs.intersection( bdiiVOs )
      newVOs = ','.join( seVOs )
      addToChangeSet( ( seSection, 'VO', vos, newVOs ), changeSet )

  return S_OK( changeSet )


import igraph
from igraph import Graph, plot
import numpy as np
from typing import List, Tuple
from typing import Optional
import config
from api import get_friends
from api_models import User
from api import get_friends

def get_network(users_ids, as_edgelist=True):
    vertices = set()
    surnames = set()

    for user_id in users_ids:
        vertices.add(user_id)

        friends = get_friends(user_id, '')
        surnames_getting = get_friends(user_id, 'last_name')
        for i in surnames_getting['response']['items']:
            surname = i['last_name']
            surnames.add(surname)

        if not 'error' in friends:
            for user_friend in friends['response']['items']:
                vertices.add(user_friend)
    surnames = [sur for sur in surnames]

    vertices = [vert for vert in vertices]

    if as_edgelist:
        edges = []

        for user_id in vertices:
            friends = get_friends(user_id, '')

            if not 'error' in friends:
                for user_friend in friends['response']['items']:
                    if user_friend in vertices:
                        edges.append((vertices.index(user_id), vertices.index(user_friend)))

        return surnames, edges

    else:
        matrix = [[0 for _ in range(len(vertices))] for _ in range(len(vertices))]

        for user_id in vertices:
            friends = get_friends(user_id)

            if not 'error' in friends:
                for user_friend in friends['response']['items']:
                    if user_friend in vertices:
                        matrix[vertices.index(user_id)][vertices.index(user_friend)] = 1

def plot_graph(graph):
    vertices = graph[0]

    edges = graph[1]

    # Создание графа
    g = Graph(vertex_attrs={"label": vertices},
              edges=edges, directed=False)

    # Задаем стиль отображения графа
    N = len(vertices)
    visual_style = {}
    visual_style["layout"] = g.layout_fruchterman_reingold(
        maxiter=1000,
        area=N ** 3,
        repulserad=N ** 3)

    # Отрисовываем граф
    plot(g, **visual_style)

    communities = g.community_edge_betweenness(directed=False)
    clusters = communities.as_clustering()
    print(clusters)
    pal = igraph.drawing.colors.ClusterColoringPalette(len(clusters))
    g.vs['color'] = pal.get_many(clusters.membership)

user_ids = [150572566]
graph = get_network(user_ids)
plot_graph(graph)


# -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod
from collections import namedtuple
import os
import shutil
import subprocess
from tempfile import NamedTemporaryFile
from Bio import SeqIO
import itertools

class GeneFinder(object):
    __metaclass__ = ABCMeta

    executable = "gene_finder"
    name = "gene_finder"

    def __init__(self, data_dir, lib_dir):
        self.lib_dir = lib_dir
        self.data_dir = data_dir
        self.install()

    def execute(self, genome_name):
        self.genome_name = genome_name
        genome_path = os.path.join(self.data_dir, genome_name + ".fasta")
        args = self.get_args(genome_path)
        devnull = open(os.devnull, "w")

        subprocess.check_call(
            [os.path.join(self.lib_dir, self.executable)] + args,
            stdout=devnull, stderr=devnull)
        parsed_result_path = os.path.join(self.data_dir,
            "{0}_{1}_genes.fasta".format(genome_name, self.name))

        def rename(gene):
            gene.id = gene.name = gene.description = self.name
        result = itertools.imap(rename, self.parse_result())
        SeqIO.write(self.parse_result(), parsed_result_path, "fasta")
        return parsed_result_path

    __call__ = execute

    @abstractmethod
    def install(self):
        pass

    @abstractmethod
    def get_args(self, _genome_path):
        pass

    @abstractmethod
    def parse_result(self):
        pass

class GeneMarkCommonGC(GeneFinder):
    """
    ./gm -rn -m heuristic_path genome_path
    """
    executable = "gm"
    name = "gm_commonGC"

    def install(self):
        gm_key_path = os.path.join(self.lib_dir, "gm_key")
        if not os.path.isfile(os.path.expanduser("~/.gm_key")):
            shutil.copyfile(gm_key_path, os.path.expanduser("~/.gm_key"))

    def get_args(self, genome_path):
        gc = common_gc_content(SeqIO.parse(genome_path, "fasta"))
        gc = min(70, max(30, int(gc)))
        heuristic_dir = os.path.join(self.lib_dir, "heuristic_mat")
        heuristic_filename = os.path.join(heuristic_dir, "heu_11_{0}.mat".format(gc))
        return ["-rn", "-m", heuristic_filename, genome_path]

    def parse_result(self):
        result_path = os.path.join(self.data_dir, self.genome_name + ".fasta.rgn")
        if not os.path.isfile(os.path.expanduser(result_path)):
            return

        gm_result = SeqIO.parse(result_path, "fasta")

        for seq_record in gm_result:
            seq_record.description = ""
            yield seq_record

class GeneMarkEveryGC(GeneMarkCommonGC):
    """
   For every GC:
   ./gm -rn -m heuristic_for_the_gc_path contigs_with_the_gc_path
   """
    executable = "gm"
    name = "gm_everyGC"

    def execute(self, genome_name):
        genome_path = os.path.join(self.data_dir, genome_name + ".fasta")

        counter = [0]
        def rename(gene):
            gene.id = gene.description = gene.name = self.name + "_" + str(counter[0])
            counter[0] += 1
            return gene

        genes = []
        execute = super(GeneMarkEveryGC, self).execute
        for seq_record in SeqIO.parse(genome_path, "fasta"):
            if len(seq_record.seq) > 200:
                with NamedTemporaryFile(prefix=seq_record.id,
                    suffix=".fasta") as f:
                    SeqIO.write(seq_record, f, "fasta")
                    f.flush()
                    current_genes = SeqIO.parse(execute(f.name.replace(".fasta", "")),
                        "fasta")
                    genes.append(itertools.imap(rename, current_genes))

        parsed_result_path = os.path.join(self.data_dir,
            "{0}_{1}_genes.fasta".format(genome_name, self.name))
        SeqIO.write(itertools.chain(*genes), parsed_result_path, "fasta")
        return parsed_result_path

def common_gc_content(sequences):
    """
    Example:
    s = "AGCCT"
    gc_content = count_gc_content(s)
    print "GC content is %.2f%%"%gc_content

    Result:
    GC content is 60.00%
    """
    GC_count, length = 0, 0
    for seq_record in sequences:
        sequence = seq_record.seq
        GC_count += sequence.count("G") + sequence.count("C")
        length += len(sequence) - sequence.count("N")
    return round((1.0 * GC_count)/length * 100, 2)

def make_sam_out(predict_path, genes_path, genome_name, data_dir, method_name):
    devnull = open(os.devnull, "w")
    subprocess.call(['bwa', 'index', '-a', 'bwtsw', predict_path],
        stdout=devnull, stderr=devnull)
    subprocess.call(['bwa', 'index', '-a', 'bwtsw', genes_path],
        stdout=devnull, stderr=devnull)
    out_path_prefix = os.path.join(data_dir, genome_name) + '_'
    with open(out_path_prefix + method_name + '_on.sam', 'w') as fout:
        subprocess.call(['bwa', 'bwasw', predict_path, genes_path],
            stdout=fout, stderr=devnull)
    with open(out_path_prefix + 'on_' + method_name + '.sam', 'w') as fout:
        subprocess.call(['bwa', 'bwasw', genes_path, predict_path],
            stdout=fout, stderr=devnull)
    return out_path_prefix

def get_sam_map(sam_path):
    res = {}
    keys = set()
    with open(sam_path) as first:
        for line in first:
            if line.startswith('@'):
                pass
            else:
                sl = line.split()
                qname, rname, mapq = sl[0], sl[2], int(sl[4])
                keys.add(qname)
                if mapq >= 200: #mapq = -10*log(10, Pr{mapp. pos. is wrong}), if mapq = 200, Pr{..} = 10**(-20)
                    res[qname] = rname
    return res, len(keys)

def count_metrics(genes_sam_path, predict_sam_path):
    predict_on_genes, predict_count = get_sam_map(predict_sam_path)
    genes_on_predict, genes_count = get_sam_map(genes_sam_path)
    H = namedtuple('H', ['name', 'TP', 'FP', 'FN'])
    TP = len(predict_on_genes)
    H_predict_is_gene = H('predict is gene', TP, predict_count - TP,
        genes_count - len(set(predict_on_genes.values())))
    TP = len(genes_on_predict)
    H_gene_was_predicted = H('gene was predicted', TP, genes_count - TP,
        predict_count - len(set(genes_on_predict.values())))
    return H_predict_is_gene, H_gene_was_predicted

def compare_data(tools, genomes, data_dir):
    for genome_name, genes_path in genomes.iteritems():
        for tool in tools:
            path_result = tool.execute(genome_name)
            path_prefix = make_sam_out(path_result, genes_path, genome_name,
                data_dir, tool.name)
            hypothesises = count_metrics(path_prefix + tool.name + '_on.sam',
                path_prefix + 'on_' + tool.name + '.sam')
            for hypothesis in hypothesises:
                yield hypothesis.name, genome_name, tool.name, hypothesis.TP, \
                      hypothesis.FP, hypothesis.FN

if __name__ == "__main__":
    data_dir = 'Data'
    lib_dir = 'genemark_suite_linux_64/gmsuite'
    genomes = {'ECO2_seq': 'Data/eco_genes.fasta', 'ECO3_seq': 'Data/eco_genes.fasta',
               'ECO6_seq': 'Data/eco_genes.fasta', 'ECO7_seq': 'Data/eco_genes.fasta',
               'MRU5_seq': 'Data/mru_genes.fasta', 'MRU6_seq': 'Data/mru_genes.fasta',
               'MRU9_seq': 'Data/mru_genes.fasta', 'PHE4_seq': 'Data/phe_genes.fasta',
               'PHE6_seq': 'Data/phe_genes.fasta', 'PHE7_seq': 'Data/phe_genes.fasta'}
    gm_common_gc = GeneMarkCommonGC(data_dir, lib_dir)
    gm_every_gc = GeneMarkEveryGC(data_dir, lib_dir)
    tools = [gm_common_gc, gm_every_gc]
    with open('compare_out', 'w') as out:
        for data in compare_data(tools, genomes, data_dir):
            out.write(','.join([str(x) for x in data]) + '\n')


#!/usr/bin/python3

import sys
from argparse import ArgumentParser

class grammar():

    rules = []
    max_length = 0
    count_operations = 0

    def __init__(self, verbose=False, filename=''):
        self.verbose = verbose
        self.filename = filename
        self.template = '{:>8} -> {:<12} | {:>%d} -> {:<%d}\n'
        if self.verbose:
            open(self.filename, 'w').close()
        self.read_rules()

    def read_rules(self):
        self.rules = []
        with open(self.__class__.__name__ + '.txt', 'r') as f_in:
            for line in f_in:
                line = line[:-1]
                if line == '':
                    continue
                else:
                    head, _, body = line.split(' ')
                    self.rules.append((head, body))

    def parse(self, number):
        pass

    def apply_rule(self, rule, string):
        new = string.replace(rule[0], rule[1], 1)

        if new != string:
            self.count_operations += 1
            if self.verbose:
                with open(self.filename, 'a') as f_out:
                    f_out.write((self.template % (self.max_length, self.max_length)).format(rule[0], rule[1], string, new))
            string = new

        return string

class grammar0(grammar):

    def parse(self, number):
        self.count_operations = 0
        self.max_length = 2 + number * 2
        string  = '[init]'

        string = self.apply_rule(self.rules[0], string)
        while (self.count_operations < number - 1):
            string = self.apply_rule(self.rules[-1], string)
        done = False
        while not done:
            done = True

            for rule in self.rules:
                    new = self.apply_rule(rule, string)
                    if (new != string):
                        done = False
                        string = new
                        break

        return all(symbol == '0' for symbol in string)

class grammar1(grammar):

    def parse(self, number):
        self.count_operations = 0
        self.max_length = 7 + number
        string = '[init]'

        if (number == 2):
            string = self.apply_rule(self.rules[0], string)
        else:
            string = self.apply_rule(self.rules[1], string)

            while (self.count_operations < number - 1):
                string = self.apply_rule(self.rules[-1], string)
            done = False
            while not done:
                done = True

                for rule in self.rules:
                    new = self.apply_rule(rule, string)
                    if (new != string):
                        done = False
                        string = new
                        break

        return all(symbol == '0' for symbol in string)

grammars = {
    0 : grammar0,
    1 : grammar1
}

def main():
    parser = ArgumentParser(prog = 'prime_grammar.py', description = 'Free (21 rules) and context sensitive (31 rules) grammars for prime numbers.\nParsing and derive.')
    parser.add_argument('-n', '--number', type = int, help = 'number to check for prime', default = -1)
    parser.add_argument('-t', '--type', type = int, help = 'grammar\'s type: 0 -- free; 1 -- context sensitive', default = 1)
    parser.add_argument('-o', '--output', type=str, help='file for output', default='log.txt')
    parser.add_argument('-v', '--verbose', help = 'verbose derive', action='store_true')

    args = parser.parse_args()

    g = grammars[args.type](args.verbose, args.output)

    if (args.number > 1):
        if (g.parse(args.number)):
            answer = 'is prime'
        else:
            answer = 'is not prime'
        print('{} {} | count of operations: {}'.format(args.number, answer, g.count_operations))
    else:
        number = 2
        while True:
            if (g.parse(number)):
                print('{} | count of operations: {}'.format(number, g.count_operations))
            number += 1

if (__name__ == "__main__"):
    main()


"""empty message

Revision ID: b56e01bac4ee
Revises: 477bbbfb48d6
Create Date: 2018-12-28 11:14:36.963527

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'b56e01bac4ee'
down_revision = '477bbbfb48d6'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('uage', sa.Integer(), nullable=True))
    op.add_column('user', sa.Column('uemail', sa.String(length=200), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('user', 'uemail')
    op.drop_column('user', 'uage')
    # ### end Alembic commands ###


import re

import discord

from discord_impl.permissions import Permissions, get_permissions
from discord_impl.channel_type import ChannelType, get_channel_type
from challonge_impl.accounts import ChallongeAccess, get as get_account
from challonge_impl.utils import validate_tournament_state
from database.core import db
from utils import print_array
from log import log_commands_core
from profiling import Profiler, Scope

from const import (T_ValidateCommandContext_BadParameters, T_ValidateCommandContext_BadChannel,
                   T_ValidateCommandContext_BadPrivileges, T_ValidateCommandContext_BadTournamentState,
                   T_Log_ValidatedCommand)

commandFormat = '| {0:16} | {1:15} | {2:12} | {3:17} | {4:17} | {5:18} | {6:13} |'

class MissingParameters(Exception):
    def __init__(self, req, given):
        self.req = req
        self.given = given

    def __str__(self):
        return T_ValidateCommandContext_BadParameters.format(self.req, self.given)

class WrongChannel(Exception):
    def __str__(self):
        return T_ValidateCommandContext_BadChannel

class InsufficientPrivileges(Exception):
    def __str__(self):
        return T_ValidateCommandContext_BadPrivileges

class BadTournamentState(Exception):
    def __str__(self):
        return T_ValidateCommandContext_BadTournamentState

class Attributes:
    def __init__(self, **kwargs):
        self.minPermissions = kwargs.get('minPermissions', Permissions.User)
        self.channelRestrictions = kwargs.get('channelRestrictions', ChannelType.Other)
        self.challongeAccess = kwargs.get('challongeAccess', ChallongeAccess.NotRequired)
        self.tournamentState = kwargs.get('tournamentState', None)

class Command:
    def __init__(self, name, cb, attributes=None):
        self.name = name
        self.cb = cb
        self.attributes = attributes
        self.aliases = []
        self.reqParams = []
        self.optParams = []
        self.helpers = []

    def __repr__(self):
        return '[Command:%s]' % self.name

    def add_required_params(self, *args):
        self.reqParams = args
        return self

    def add_optional_params(self, *args):
        self.optParams = args
        return self

    def add_aliases(self, *args):
        self.aliases = args
        return self

    def add_helpers(self, *args):
        self.helpers = args
        return self

    async def validate_context(self, client, message, postCommand, context_cache):
        if context_cache['permissions'] < self.attributes.minPermissions:
            return False, InsufficientPrivileges()

        if not context_cache['channel_type'] & self.attributes.channelRestrictions:
            return False, WrongChannel()

        if self.attributes.challongeAccess == ChallongeAccess.RequiredForAuthor:
            acc, exc = await get_account(message.author.id)
            if exc:
                return False, exc
        elif self.attributes.challongeAccess == ChallongeAccess.RequiredForHost and context_cache['db_tournament']:
            acc, exc = await get_account(context_cache['db_tournament'].host_id)
            if exc:
                return False, exc
            if acc and self.attributes.tournamentState:
                if not await validate_tournament_state(acc, context_cache['db_tournament'].challonge_id, self.attributes.tournamentState):  # can raise
                    return False, BadTournamentState()

        reqParamsExpected = len(self.reqParams)
        givenParams = len(postCommand)
        if givenParams < reqParamsExpected:
            return False, MissingParameters(reqParamsExpected, givenParams)

        return True, None

    def validate_name(self, name):
        if self.name == name:
            return True
        if self.aliases is not None:
            return name in self.aliases
        return False

    async def _fetch_helpers(self, message, postCommand, context_cache):
        kwargs = {}
        for x in self.helpers:
            if x == 'account':
                if self.attributes.challongeAccess == ChallongeAccess.RequiredForAuthor:
                    kwargs[x], exc = await get_account(message.author.id)
                else:
                    kwargs[x], exc = await get_account(context_cache['db_tournament'].host_id)
            elif x == 'tournament_id':
                kwargs[x] = context_cache['db_tournament'].challonge_id
            elif x == 'tournament_role':
                roleid = context_cache['db_tournament'].role_id
                kwargs[x] = discord.utils.find(lambda r: r.id == roleid, message.server.roles)
            elif x == 'tournament_channel':
                channelid = context_cache['db_tournament'].channel_id
                kwargs[x] = discord.utils.find(lambda c: c.id == channelid, message.server.channels)
            elif x == 'participant_username':
                kwargs[x] = db.get_user(message.author.id).challonge_user_name
            elif x == 'announcement':
                kwargs[x] = ' '.join(postCommand)

        return kwargs

    def _fetch_args(self, postCommand):
        kwargs = {}

        for count, x in enumerate(self.reqParams):
            kwargs[x] = postCommand[count]
        offset = len(self.reqParams)

        for count, x in enumerate(self.optParams):
            if count + offset < len(postCommand):
                kwargs[x] = postCommand[count + offset]

        return kwargs

    async def execute(self, client, message, postCommand, context_cache):
        kwargs = {}
        kwargs.update(self._fetch_args(postCommand))
        kwargs.update(await self._fetch_helpers(message, postCommand, context_cache))
        await self.cb(client, message, **kwargs)

    def pretty_print(self):
        return self.simple_print() + '\n```{0}{1}```'.format('' if self.cb.__doc__ is None else self.cb.__doc__,
                                                             'No aliases' if len(self.aliases) == 0 else 'Aliases: ' + ' / '.join(self.aliases))

    def simple_print(self):
        return '`{0}` {1}{2} -- *{3}*'.format(self.name,
                                              '' if len(self.reqParams) == 0 else ' '.join(['[' + p + ']' for p in self.reqParams]),
                                              '' if len(self.optParams) == 0 else ' '.join(['{' + p + '}' for p in self.optParams]),
                                              'No description available' if self.cb.__doc__ is None else self.cb.__doc__.splitlines()[0])

class CommandsHandler:
    simple_word = r"""[!,\-\+\w@<>]+"""
    separated_words = r"""(?:{0}\s*)""".format(simple_word)
    argument = r"""\s*({0}|'{1}+')?""".format(simple_word, separated_words)
    base_regex = r"""(\w+){0}{0}{0}{0}""".format(argument)
    base_compiled_re = re.compile(base_regex, re.IGNORECASE)

    def __init__(self):
        self._commands = []

    def _add(self, command):
        self._commands.append(command)
        return command

    def find(self, name):
        for command in self._commands:
            if command.validate_name(name):
                return command
        return None

    def register(self, **attributes):
        def decorator(func):
            async def wrapper(client, message, **postCommand):
                # choose only those that are most likely arguments but not the api key (could be Account...)
                args = ' '.join([v for k, v in postCommand.items() if isinstance(v, str) and k != 'key'])
                # server for profiling info
                server = 0 if message.channel.is_private else message.channel.server.id
                with Profiler(Scope.Command, name=func.__name__, args=args, server=server):
                    await func(client, message, **postCommand)
            wrapper.__doc__ = func.__doc__
            return self._add(Command(func.__name__, wrapper, Attributes(**attributes)))
        return decorator

    def _get_command_and_postcommand(self, client, message, context_cache):
        if not message.channel.is_private:
            commandTrigger = context_cache['db_server'].trigger
            regex = '(?:%s\s?|<@!?%s>\s)%s' % (commandTrigger, client.user.id, CommandsHandler.base_regex)
            r = re.compile(regex, re.IGNORECASE)
        else:
            r = CommandsHandler.base_compiled_re

        m = r.match(message.content)
        if m:
            return self.find(m.groups()[0]), [v for i, v in enumerate(m.groups()) if i > 0 and v is not None]
        return None, None

    def get_context_cache_update(self, context_cache, message):
        db_tournament = db.get_tournament(message.channel) if message.server else None
        context_cache.update({'permissions': get_permissions(message.author, message.channel),
                              'channel_type': get_channel_type(message.channel, context_cache['db_server'], db_tournament),
                              'db_tournament': db_tournament})
        return context_cache

    async def try_execute(self, client, message):
        context_cache = {'db_server': db.get_server(message.server) if message.server else None}
        command, postCommand = self._get_command_and_postcommand(client, message, context_cache)
        if command:
            # caching a few other things
            context_cache = self.get_context_cache_update(context_cache, message)
            validated, exc = await command.validate_context(client, message, postCommand, context_cache)
            if exc:
                await client.send_message(message.channel, exc)
            elif validated:
                await command.execute(client, message, postCommand, context_cache)
                log_commands_core.info(T_Log_ValidatedCommand.format(command.name,
                                                                     '' if len(postCommand) == 0 else ' ' + ' '.join(postCommand),
                                                                     message,
                                                                     'PM' if message.channel.is_private else '{0.channel.server.name}/#{0.channel.name}'.format(message)))

    def dump(self):
        return print_array('Commands Registered',
                           commandFormat.format('Name', 'Min Permissions', 'Channel Type', 'Challonge', 'Aliases', 'Required Args', 'Optional Args'),
                           self._commands,
                           lambda c: commandFormat.format(c.name,
                                                          c.attributes.minPermissions.name,
                                                          c.attributes.channelRestrictions.name,
                                                          c.attributes.challongeAccess.name,
                                                          '-' if len(c.aliases) == 0 else '/'.join(c.aliases),
                                                          '-' if len(c.reqParams) == 0 else '/'.join(c.reqParams),
                                                          '-' if len(c.optParams) == 0 else '/'.join(c.optParams)))

cmds = CommandsHandler()

class AuthorizedCommandsWrapper:
    def __init__(self, client, message):
        self._client = client
        self._message = message
        self._commands = iter(cmds._commands)
        self._context_cache = {'db_server': db.get_server(message.server) if message.server else None}
        self._context_cache = cmds.get_context_cache_update(self._context_cache, message)

    async def __aiter__(self):
        return self

    async def __anext__(self):
        try:
            command = next(self._commands)
        except StopIteration:
            raise StopAsyncIteration

        validated, exc = await command.validate_context(self._client, self._message, [], self._context_cache)
        if validated or isinstance(exc, MissingParameters):
            return command.simple_print()
        else:
            return await self.__anext__()

def required_args(*args):
    def decorator(func):
        return func.add_required_params(*args)
    return decorator

def optional_args(*args):
    def decorator(func):
        return func.add_optional_params(*args)
    return decorator

def aliases(*args):
    def decorator(func):
        return func.add_aliases(*args)
    return decorator

def helpers(*args):
    def decorator(func):
        return func.add_helpers(*args)
    return decorator

import commands.definitions.admin  # needed to preload commands
import commands.definitions.management  # needed to preload commands
import commands.definitions.challonge  # needed to preload commands


from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',

    # url(r'^$', 'fishsite.views.home', name='home'),
    # Main sites
    url(r'^$', include('fishtraits.urls')),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    url(r'^download/attr/$','fishtraits.views.download_attr'),
    url(r'^download/key/$',     'fishtraits.views.download_key'),
    url(r'^download/taxo/$','fishtraits.views.download_taxo'),

	url(r'^fishtraits/', include('fishtraits.urls', namespace="fishtraits")),
    url(r'^geodist/(?P<trait_id>\d+)/$', 'fishtraits.views.geodist'),
	url(r'^keyword_search/$', 'fishtraits.views.keyword_search', name='keyword_search'),

    # Load initial dataset
    # url(r'^import$', 'fishtraits.views.index'),

	url(r'^range/$', 'fishtraits.views.range'),

    url(r'^search/attr/$', 'fishtraits.views.search_attr', name='search_attr'),
    url(r'^search/key/$', 'fishtraits.views.search_key', name='search_key'),
    url(r'^search_lookup/$', 'fishtraits.views.search_lookup'),
    url(r'^search/map/$', 'fishtraits.views.search_map'),
    url(r'^search/taxo/$', 'fishtraits.views.search_taxo', name='search_taxo'),

	url(r'^result/attr/$',  'fishtraits.views.result_attr', name='result_attr'),
	url(r'^result/key/$',   'fishtraits.views.result_key',  name='result_key'),
    url(r'^result/taxo/$',  'fishtraits.views.result_taxo', name='result_taxo'),

    url(r'^taxonomy/$', 'fishtraits.views.taxonomy'),

	# django_publications app
	url(r'^publications/', include('publications.urls')),
    url(r'^admin/publications/publication/import_bibtex/$', 'publications.admin_views.import_bibtex'),

)


from django.urls import include, path
from rest_framework.routers import SimpleRouter
from survey.views import OperatingSystemViewSet, SurveyResultViewSet

app_name = 'survey'

router = SimpleRouter()
router.register('survey', SurveyResultViewSet, basename='survey')
router.register('os', OperatingSystemViewSet, basename='os')

urlpatterns = [
    path('', include((router.urls))),
]


# send and receive numpy arrays on ZeroMQ
import zmq
import numpy
import torch
import json

def send_array_and_str(socket, A, sentence, flags=0, copy=True, track=False):
    """send a numpy array with metadata"""
    md = dict(dtype = str(A.dtype),shape = A.shape)
    socket.send_string(sentence, flags|zmq.SNDMORE)
    socket.send_json(md, flags|zmq.SNDMORE)
    return socket.send(A, flags, copy=copy, track=track)

def recv_array_and_str(socket, flags=0, copy=True, track=False):
    """recv a numpy array and sentence"""
    sentence = socket.recv_string(flags=flags)
    md = socket.recv_json(flags=flags)
    msg = socket.recv(flags=flags, copy=copy, track=track)
    buf = memoryview(msg)
    A = numpy.frombuffer(buf, dtype=md['dtype'])
    return sentence, A.reshape(md['shape'])

def preprocess(text, tokenizer):
    '''tokenize text into subwords and convert to indices
    :param text str: text to be preprocessed
    :param tokenizer: BertTokenizer object
    :output: torch tensor of vocab ids
    '''
    tokenized_text = tokenizer.tokenize(text)
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    tokens_tensor = torch.tensor([indexed_tokens]).numpy()
    return tokens_tensor


#FCN_16s
import cv2, os
import random
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from sklearn import preprocessing

from sklearn.model_selection import train_test_split, StratifiedKFold
## Import usual libraries
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
import keras, sys, time, warnings
from keras.models import *
from keras.layers import *
from sklearn.utils import shuffle
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import KFold

# Saving and loading model and weights
from keras.models import model_from_json
from keras.models import load_model

import pydensecrf.densecrf as dcrf

from pydensecrf.utils import compute_unary, create_pairwise_bilateral, \
    create_pairwise_gaussian, softmax_to_unary, unary_from_softmax

def give_color_to_seg_img(seg,n_classes):
    '''
    seg : (input_width,input_height,3)
    '''

    if len(seg.shape)==3:
        seg = seg[:,:,0]
    seg_img = np.zeros( (seg.shape[0],seg.shape[1],3) ).astype('float')
    colors = sns.color_palette("hls", n_classes)

    for c in range(n_classes):
        segc = (seg == c)
        seg_img[:,:,0] += (segc*( colors[c][0] ))
        seg_img[:,:,1] += (segc*( colors[c][1] ))
        seg_img[:,:,2] += (segc*( colors[c][2] ))

    return(seg_img)

def loadImage( path , width , height ):
    image = cv2.imread(path, 1)
    image = cv2.resize(image, ( width , height ))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    return image

def loadLabel( path , nClasses ,  width , height  ):
    label = np.zeros((height, width , nClasses ))
    image = cv2.imread(path, 1)
    image = cv2.resize(image, ( width,height ))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image_red = image[:, : , 0] # [width,height,channel]
    image_green = image[:, : , 1] # [width,height,channel]
    image_blue = image[:, : , 2] # [width,height,channel]
    for channel in range(nClasses):
        if channel == 0:
            label[: , : , channel ] = ((image_red == 255) & (image_green == 255) & (image_blue == 255)).astype(int) #name="generalface"
        elif channel == 1:
            label[: , : , channel ] = ((image_red == 0) & (image_green == 0) & (image_blue == 0)).astype(int) #name="background"
        else:
            continue
    return label

def Contigency(nClasses,labels, predicts):
    matrixs = []
    for channel in range(nClasses):
        TP = np.sum( (labels == channel) & (predicts == channel) )
        TN = np.sum( (labels != channel) & (predicts != channel) )
        FP = np.sum( (labels != channel) & (predicts == channel) )
        FN = np.sum( (labels == channel) & (predicts != channel) )

        matrix = []
        matrix.append(TP)
        matrix.append(TN)
        matrix.append(FP)
        matrix.append(FN)

        matrixs.append(matrix)

    return matrixs

def Validate(matrixs):
    ## mean Intersection over Union
    ## Mean IoU = TP/(FN + TP + FP)
    IoUs = []
    channel = 0
    for matrix in matrixs:
        TP = matrix[0]
        TN = matrix[1]
        FP = matrix[2]
        FN = matrix[3]
        IoU = TP/float(TP + FP + FN)
        acc = (TP+TN)/float(TP + TN + FP + FN)
        prec = TP/float(TP + FP)
        rec = TP/float(TP+FN)
        F1 = (2*prec*rec)/float(prec + rec)
        print("____________________________________________________________________")
        print("class {:02.0f}: #TP={:6.0f}, #TN={:6.0f}, #FP={:6.0f}, #FN={:5.0f}".format(channel,TP,TN,FP,FN))
        print("acc={:4.3f}, prec={:4.3f}, rec={:4.3f}, F1={:4.3f}, IoU={:4.3f}".format(acc,prec,rec,F1,IoU))
        channel = channel + 1
        IoUs.append(IoU)
    mIoU = np.mean(IoUs)
    print("_________________")
    print("Mean IoU: {:4.3f}".format(mIoU))

def CRFs(images, y_preds, n_classes):
    X_i = images.astype(np.uint8)
    Y_i = y_preds
    transpose_matrix = np.array(Y_i).transpose((2, 0, 1))
    unary = unary_from_softmax(transpose_matrix, 0.7)
    unary = np.ascontiguousarray(unary)
    d = dcrf.DenseCRF2D(X_i.shape[0], X_i.shape[1], n_classes)
    d.setUnaryEnergy(unary)
    d.addPairwiseGaussian(sxy=3, compat=3)
    d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=X_i, compat=10)
    Q = d.inference(20)
    arr = np.argmax(Q, axis=0)
    arr = arr.ravel().reshape((X_i.shape[0], X_i.shape[1]))
    result = np.zeros((X_i.shape[0], X_i.shape[1], n_classes ))
    for i in range(n_classes):
        result[:,:,i] = (arr==i).astype(int)

    return result

# load json and create model
json_file = open('/content/drive/My Drive/app/F7/output16/model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
global model
model = model_from_json(loaded_model_json)

# this is key : save the graph after loading the model
global graph
graph = tf.get_default_graph()

# load weights into new model
model.load_weights("/content/drive/My Drive/app/F7/output16/model.h5")
print("Loaded model from disk")

##### Read images in folder
location_image = "/content/drive/My Drive/app/F6/dataset1/images_prepped_test/"
location_segmentation = "/content/drive/My Drive/app/F6/dataset1/annotations_prepped_test/"

image_direction = os.listdir(location_image)
image_direction.sort()
segmentation_direction = os.listdir(location_segmentation)
segmentation_direction.sort()

##### Load and augment dataset
n_classes = 2
input_width = 224
input_height = 224

X = []
Y = []

for real_pic , seg_pic in zip(image_direction,segmentation_direction) :
    pre_X = loadImage(location_image + real_pic, input_width, input_height)
    pre_Y = loadLabel(location_segmentation + seg_pic, n_classes, input_width , input_height)
    X.append(pre_X)
    Y.append(pre_Y)

train_rate = 0.85
##### Separate data
X, Y = np.array(X) , np.array(Y)
X_val, y_val = shuffle(X,Y)

##### Validation
print ("=================================================")
print ("Validation before CRFs")
print ("-------------------------")
print ("X_test.shape = ", X_val.shape)
print ("y_test.shape = ", y_val.shape)

print ("-------------------------")
# Validate by contigency
y_pred = model.predict(X_val)
y_predi = np.argmax(y_pred, axis=3)
sh = give_color_to_seg_img(y_predi[0],n_classes)
plt.imshow(sh)
plt.show()
y_testi = np.argmax(y_val, axis=3)
matrixs = Contigency(n_classes, y_testi, y_predi)
Validate(matrixs)

print ("-------------------------")
print ("=================================================")
print ("Validation after CRFs")
print ("-------------------------")
results = []
for i in range(len(X_val)):
    images = X_val[i]
    labels = y_pred[i]
    result = CRFs(images,labels,n_classes)
    results.append(result)

results = np.asarray(results)
resultsi = np.argmax(results, axis=3)
sh = give_color_to_seg_img(resultsi[0],n_classes)
plt.imshow(sh)
plt.show()
matrixs = Contigency(n_classes, y_testi, resultsi)
Validate(matrixs)

print ("-------------------------")
print ("=================================================")


# coding=utf-8

import tool as t
from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, String, Boolean, DateTime, Unicode
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.expression import update

engine = create_engine('mysql+mysqldb://root:laso2401@localhost:3306/fanboflask')

Base = declarative_base()

class zhubo(Base):
    __tablename__ = 'zhubo'
    idzhubo = Column(Integer, primary_key=True)
    room = Column(String(128), unique=True)
    nick_name = Column(Unicode(128))
    avatar = Column(String(1024))
    room_url = Column(String(1024))
    cover_picture = Column(String(1024))
    is_online = Column(Boolean)
    update_time = Column(DateTime)
    last_kaibo_time = Column(DateTime)

Base.metadata.create_all(engine)

smaker = sessionmaker(bind=engine)
session = smaker()
conn = engine.connect()

def commit_change(model_obj):
    session.add(model_obj)
    session.commit()

def find_zhubo(room):
    try:
        zb = session.query(zhubo).filter(zhubo.room==room).first()
        if zb is None:
            zb = zhubo(room=room)
            commit_change(zb)
        return zb
    except Exception as ex:
        t.err()
        t.log('can not find or create zhubo, room number: %s' % room)
        return None

def set_zhubo_info(room, **kwargs):
    zb = find_zhubo(room)
    if zb is None:
        return
    for key, value in kwargs.items():
        if hasattr(zb, key):
            setattr(zb, key, value)
    zb.update_time = t.now()
    commit_change(zb)

def update_other_invalid(time_before):
    tupdate = update(zhubo).where(zhubo.update_time<time_before).values(is_online=False)
    conn.execute(tupdate)


from torch.nn.modules.batchnorm import _BatchNorm
from torch.nn.parameter import Parameter
from torch.nn import functional as F

import torch
import torch.nn as nn

class SAN_V3(_BatchNorm):
    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, batch=-1):
        super().__init__(num_features, eps, momentum, affine)
        self.batch = batch

        z_dim = 8

        self.mean_cfc_1 = Parameter(torch.Tensor(num_features, 3, z_dim))
        self.mean_cfc_2 = Parameter(torch.Tensor(num_features, z_dim+1, 1))
        self.mean_cfc_1.data.fill_(0)
        self.mean_cfc_2.data.fill_(0)

        self.std_cfc_1 = Parameter(torch.Tensor(num_features, 3, z_dim))
        self.std_cfc_2 = Parameter(torch.Tensor(num_features, z_dim+1, 1))
        self.std_cfc_1.data.fill_(0)
        self.std_cfc_2.data.fill_(0)

        self.activation = nn.Sigmoid()
        setattr(self.mean_cfc_1, 'affine_gate', True)
        setattr(self.mean_cfc_2, 'affine_gate', True)
        setattr(self.std_cfc_1, 'affine_gate', True)
        setattr(self.std_cfc_2, 'affine_gate', True)

    def get_feat(self, feat):
        B, C, H, W = feat.size()

        feat = feat.view(B, C, -1)
        mean_i = feat.mean(dim=2, keepdim=True)
        var_i = feat.var(dim=2, keepdim=True)
        std_i = var_i.sqrt()

        feat_ones = torch.ones(B, C, 1).cuda()

        mean_feat = torch.cat((mean_i, std_i, feat_ones), dim=2)

        return torch.transpose(mean_feat, 0, 1)

    def forward(self, input):
        B, C, H, W = input.size()
        feat_ones = torch.ones(C, B, 1).cuda()

        feat = self.get_feat(input)

        mean_att_z = torch.bmm(feat, self.mean_cfc_1)
        mean_att_z = torch.cat((mean_att_z, feat_ones), dim=2)
        mean_att = torch.bmm(mean_att_z, self.mean_cfc_2)
        mean_att = torch.transpose(mean_att, 0, 1).squeeze()
        mean_att = self.activation(mean_att)

        std_att_z = torch.bmm(feat, self.std_cfc_1)
        std_att_z = torch.cat((std_att_z, feat_ones), dim=2)
        std_att = torch.bmm(std_att_z, self.std_cfc_2)
        std_att = torch.transpose(std_att, 0, 1).squeeze()
        std_att = self.activation(std_att)

        # Compute BN (B x C x None)
        if self.batch == -1 or self.batch == B:
            out = F.batch_norm(
                    input, self.running_mean, self.running_var, None, None,
                    self.training, self.momentum, self.eps)
        else:
            out = input.new(input.size())
            for i in range(0, B, self.batch):
                j = min(B, i + self.batch)
                out[i:j, :, :, :] = F.batch_norm(
                    input[i:j, :, :, :], self.running_mean, self.running_var, None, None,
                    self.training, self.momentum, self.eps)

        out.mul_(self.weight[None, :, None, None] * std_att[:, :, None, None])
        out.add_(self.bias[None, :, None, None] * mean_att[:, :, None, None])
        out = out.view(B, C, H, W)

        return out

    def extra_repr(self):
        return '{num_features}, eps={eps}, momentum={momentum}, affine={affine}, ' \
                'track_running_stats={track_running_stats}, ' \
                'batch={batch}'.format(**self.__dict__)


"""
    Объединение файлов Excel путем складывания значений во всех листах с выгрузкой результата в файл RESULT.xls.
    Форматирование в результирующем файле при этом теряется.

    Использование:
        Помещаем файл excel_merge.py в каталог с файлами *.xls, *.xlsx, *.ods и запускаем его
        В результате в текущем каталоге создастся файл RESULT.xls и автоматически откроется.

    Примечания:
        Во всех файлах должны совпадать имена Листов
        По умолчанию обработка ведется начиная со второго столбца (B) второй строки (Можно изменить в переменных
            X_MIN, X_MAX, Y_MIN, Y_MAX)

    Установка:
        Для использования данного скрипта, необходим Python3 и библиотека pyexcel
        Python3 можно скачать по ссылке: https://www.python.org/downloads/ затем установить
        Для установки расширения pyexcel выполнить в командной строке команду:
            pip install pyexcel pyexcel-xls pyexcel-xlsx pyexcel-ods

    Данный скрипт был протестирован на Python 3.6, pyexcel==0.6.2
"""

import os, sys
import pyexcel
import glob

curdir = os.path.abspath(os.path.dirname(__file__)) # Текущий каталог, где лежит этот файл

# === Настройки ===
WORKDIR = curdir # Рабочий каталог, он же текущий по умолчанию

if WORKDIR:
    os.chdir(WORKDIR) # Работа в текущем каталоге

INFILE = None # Начальный файл. Может быть None, тогда начальный файл считается первый по списку # os.path.join(curdir,'out','infile.xlsx')
OUTFILE = 'RESULT.xls' # Имя исходящего результирующего файла XLS (может находиться в другой папке) # os.path.join(curdir,'out','out.xls')

X_MIN = 2 # Номер столбца для начала обработки
X_MAX = 100000 # Максимальный Номер столбца для конца обработки
Y_MIN = 2 # Номер начальной строки
Y_MAX = 100000 # Максимальный Номер конечной строки

ISOPEN_FILE = True  # Открыть результирующий файл после обработки?
ISNOTCLOSE = True  # После обработки файла не закрывать окно скрипта

def parse_float(x):
    if x == '':
        res = 0
    else:
        try:
            res = float(x)
        except:
            res = None
    return res

def merge_all_sheets():
    """ Складывает все значения в файлах Excel в рабочей директории по всем листам и записывает результат в RESULT.xls
          Названия листов во всех файлах должны совпадать, так же как и диапазоны строк и столбцов """
    err_files = set()
    err_all = 0
    files_count = 0
    book_res = None  # Рабочий массив данных
    if INFILE:  # В рабочий массив загружаем данные из начального файла, если он задан
        book_res = pyexcel.get_book_dict(file_name=INFILE)

    files = glob.glob('*.xls*')
    files.extend(glob.glob('*.ods'))
    for fn in files:
        if not fn.startswith('~') and fn != OUTFILE and fn != INFILE:
            print(fn)
            files_count += 1
            if book_res is None:
                book_res = pyexcel.get_book_dict(file_name=fn)
            else:
                book_cur = pyexcel.get_book_dict(file_name=fn)
                for sheet in book_res:
                    #print(sheet)
                    for y, row in enumerate(book_res[sheet]):
                        #print(y, row)
                        for x, cell in enumerate(book_res[sheet][y]):
                            if y>=Y_MIN-1 and y<=Y_MAX-1 and x>=X_MIN-1 and x<=X_MAX-1:  # Ограничиваем обработку только определенным диапазоном
                                try:
                                    cell1 = parse_float(book_res[sheet][y][x])
                                    cell2 = parse_float(book_cur[sheet][y][x])
                                    if cell1 is not None and cell2 is not None:
                                        book_res[sheet][y][x] = cell1 + cell2
                                        #print(x, book_res[sheet][y][x], type(book_res[sheet][y][x]))
                                except:
                                    print('Ошибка! Файл: {} Лист: {} Строка: {} Колонка: {}'.format(fn, sheet, y, x))
                                    err_all += 1
                                    err_files.add(fn)
                    #print(book_res[sheet])
    print('\nОбработка завершена!\n  Всего обработано файлов: {}'.format(files_count))
    if err_all > 0:
        print('  Всего ошибок: {}'.format(err_all))
        print('  Ошибки в файлах: {}'.format(', '.join(err_files)))
    else:
        print('  Ошибок нет')
    pyexcel.save_book_as(bookdict=book_res, dest_file_name=OUTFILE)

    if ISOPEN_FILE:
        import os
        os.startfile(OUTFILE)

def merge_first_sheet():
    """ Объединяет значения в файлах Excel в рабочей директории и записывает результат в RESULT.xls
          Берутся только первые листы
    """
    err_files = set()
    err_all = 0
    files_count = 0
    sheet_res = None  # Рабочий массив данных
    if INFILE:  # В рабочий массив загружаем данные из начального файла, если он задан
        sheet_res = pyexcel.get_book_dict(file_name=INFILE)
        #sheet_name =

    files = glob.glob('*.xls*')
    sheets = []
    for fn in files:
        if not fn.startswith('~') and fn != OUTFILE and fn != INFILE:
            print(fn)
            files_count += 1
            if sheet_res is None:
                #sheet_res = pyexcel.get_sheet(file_name=fn)
                sheet_res = pyexcel.get_array(file_name=fn)
            else:
                sheet_cur = pyexcel.get_array(file_name=fn)
                # Определение названия листа (берется первый)
                sheet_name = list(pyexcel.get_book_dict(file_name=fn).keys())[0]
                #print(sheet_name)
                #print(sheet_res)
                for y, row in enumerate(sheet_res):
                    #print(y, row)
                    for x, cell in enumerate(sheet_res[y]):
                        if y>=Y_MIN-1 and y<=Y_MAX-1 and x>=X_MIN-1 and x<=X_MAX-1:  # Ограничиваем обработку только определенным диапазоном
                            try:
                                cell1 = parse_float(sheet_res[y][x])
                                cell2 = parse_float(sheet_cur[y][x])
                                if cell1 is not None and cell2 is not None:
                                    sheet_res[y][x] = cell1 + cell2
                                    #print(x, sheet_res[y][x], type(sheet_res[y][x]))
                            except:
                                print('Ошибка! Файл: {} Лист: {} Строка: {} Колонка: {}'.format(fn, sheet_name, y, x))
                                err_all += 1
                                err_files.add(fn)
                #print(book_res[sheet])
    print('\nОбработка завершена!\n  Всего обработано файлов: {}'.format(files_count))
    if err_all > 0:
        print('  Всего ошибок: {}'.format(err_all))
        print('  Ошибки в файлах: {}'.format(', '.join(err_files)))
    else:
        print('  Ошибок нет')

    pyexcel.save_as(array=sheet_res, dest_file_name=OUTFILE)

    if ISOPEN_FILE:
        import os
        os.startfile(OUTFILE)

def main():
    print('Версия Python:', sys.version)
    print('Текущая папка:', curdir)  # .decode('utf-8'),
    print('Рабочая папка:', WORKDIR)  # .decode('utf-8'),
    print()
    # Слить все файлы в рабочей директории со всеми листами
    merge_all_sheets()
    # Слить все файлы в рабочей директории только по первым листам
    #merge_first_sheet()

if __name__=='__main__':
    main()
    if ISNOTCLOSE:
        input()


import urllib

from bs4 import BeautifulSoup
import CifFile
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from django.contrib.auth.models import Group

from tardis.tardis_portal.models import ExperimentAuthor

from . import default_settings

class CifHelper(object):

    def __init__(self, cif_url):
        self.cf = CifFile.ReadCif(cif_url)

    def get_cif_file_object(self):
        return self.cf

    def _is_list(self, obj):
        return isinstance(obj, list)

    def as_list(self, obj):
        return obj if self._is_list(obj) else [obj]

class PDBCifHelper(CifHelper):

    def __init__(self, pdb_id):
        super(PDBCifHelper, self).__init__(
            'http://www.pdb.org/pdb/files/' + urllib.quote(pdb_id) + '.cif')
        self.pdb_id = pdb_id

    def __getitem__(self, key):
        return super(PDBCifHelper, self).get_cif_file_object()[
            self.pdb_id][key]

    def get_pdb_id(self):
        return self['_entry.id']

    def get_pdb_title(self):
        return self['_struct.title']

    def get_pdb_url(self):
        return 'http://www.pdb.org/pdb/search/structidSearch.do?structureId=' \
               + urllib.quote(self.get_pdb_id())

    def get_obs_r_value(self):
        return float(self['_refine.ls_R_factor_obs'])

    def get_free_r_value(self):
        return float(self['_refine.ls_R_factor_R_free'])

    def get_resolution(self):
        return float(self['_reflns.d_resolution_high'])

    def get_spacegroup(self):
        return self['_symmetry.space_group_name_H-M']

    def get_unit_cell(self):
        length_a = self['_cell.length_a']
        length_b = self['_cell.length_b']
        length_c = self['_cell.length_c']
        angle_a = self['_cell.angle_alpha']
        angle_b = self['_cell.angle_beta']
        angle_c = self['_cell.angle_gamma']
        return '(a = %s, b = %s, c = %s),(alpha = %s, beta = %s, gamma = %s)' \
               % (length_a, length_b, length_c, angle_a, angle_b, angle_c)

    def get_citations(self):
        ids = self.as_list(self['_citation.id'])
        titles = self.as_list(self['_citation.title'])
        journals = self.as_list(self['_citation.journal_abbrev'])
        volumes = self.as_list(self['_citation.journal_volume'])
        pages_first = self.as_list(self['_citation.page_first'])
        pages_last = self.as_list(self['_citation.page_last'])
        years = self.as_list(self['_citation.year'])
        dois = self.as_list(self['_citation.pdbx_database_id_DOI'])
        author_citation_ids = self.as_list(
            self['_citation_author.citation_id'])
        author_citation_names = self.as_list(self['_citation_author.name'])

        citations = []
        for pub_id, title, journal, volume, page_first, page_last, year, doi \
                in zip(ids, titles, journals, volumes, pages_first, pages_last,
                       years, dois):
            citation = {'_id': pub_id,
                        'title': title,
                        'journal': journal,
                        'volume': volume,
                        'page_first': page_first,
                        'page_last': page_last,
                        'year': year,
                        'doi': doi}

            authors = []
            for auth_pub_id, auth_name in zip(
                    author_citation_ids, author_citation_names):
                if auth_pub_id == pub_id:
                    authors.append(auth_name)
            citation['authors'] = authors
            citations.append(citation)

        return citations

    def get_sequence_info(self):
        try:
            seqs_gen_id = self.as_list(self['_entity_src_gen.entity_id'])
            seqs_gen_org = self.as_list(
                self['_entity_src_gen.pdbx_gene_src_scientific_name'])
            seqs_gen_exp_sys = self.as_list(
                self['_entity_src_gen.pdbx_host_org_scientific_name'])
            seqs_gen = dict(  # R0204:116,12:PDBCifHelper.get_sequence_info: Redefinition of seqs_gen type from list to dict
                zip(seqs_gen_id, zip(seqs_gen_org, seqs_gen_exp_sys)))
        except KeyError:
            seqs_gen = dict()

        try:
            seqs_nat_id = self.as_list(self['_entity_src_nat.entity_id'])
            seqs_nat_org = self.as_list(
                self['_entity_src_nat.pdbx_organism_scientific'])
            seqs_nat = dict(zip(seqs_nat_id, seqs_nat_org))
        except KeyError:
            seqs_nat = dict()

        try:
            seqs_code_id = self.as_list(self['_entity_poly.entity_id'])
            seqs_code = self.as_list(
                self['_entity_poly.pdbx_seq_one_letter_code_can'])
            seqs_code = dict(zip(seqs_code_id, seqs_code))
        except KeyError:
            seqs_code = dict()

        try:
            seqs_name_id = self.as_list(self['_entity_name_com.entity_id'])
            seqs_names = self.as_list(self['_entity_name_com.name'])
            seqs_name = dict(zip(seqs_name_id, seqs_names))
        except KeyError:
            seqs_name = dict()

        sequences = []
        # Only look at the poly entities
        for entity_id in seqs_code.keys():
            seq = {
                'name': '',
                'sequence': seqs_code[entity_id],
                'organism': '',
                'expression_system': ''
            }

            # Protein isolated directly from organism?
            if entity_id in seqs_nat:
                seq['organism'] = seqs_nat[entity_id]
            # Protein isolated using an expression system?
            elif entity_id in seqs_gen:
                seq['organism'], seq['expression_system'] = seqs_gen[entity_id]

            # Has a name?
            if entity_id in seqs_name:
                seq['name'] = seqs_name[entity_id]

            sequences.append(seq)

        return sequences

def check_pdb_status(pdb_id):
    status_page = urllib.urlopen(
        'http://www.rcsb.org/pdb/rest/idStatus?structureId=' +
        urllib.quote(pdb_id.upper())).read()
    soup = BeautifulSoup(status_page)
    if soup.record:
        pdb_status = soup.record['status']
    else:
        pdb_status = 'INVALID'

    return pdb_status

def get_unreleased_pdb_info(pdb_id):
    info_page = urllib.urlopen(
        'http://www.rcsb.org/pdb/rest/getUnreleased?structureId=' +
        urllib.quote(pdb_id.upper())).read()
    soup = BeautifulSoup(info_page)

    info = {}

    info['title'] = soup.record.title.string
    info['authors'] = soup.record.authors.string

    return info

def get_pub_admin_email_addresses():
    return [
        user.email
        for user in Group.objects.get(
            name=getattr(settings, 'PUBLICATION_OWNER_GROUP',
                         default_settings.PUBLICATION_OWNER_GROUP))
            .user_set.all()
        if user.email]

def send_mail_to_authors(publication, subject, message):
    recipients = [author.email for author in
                  ExperimentAuthor.objects.filter(experiment=publication)]
    from_email = getattr(settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
                         default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL)
    msg = EmailMultiAlternatives(
        subject,
        message,
     from_email,
     recipients,
     cc=get_pub_admin_email_addresses())
    msg.send()


import numpy as np
import vtk

major_radius = 1
band_width = 0.8
turns = 1
rad_samples = 90
band_samples = 4

xs = rad_samples+1
ys = band_samples
num_pts_cover = xs * ys

xy_to_id = dict()
for foo in range(num_pts_cover):
    y = foo % ys
    x = foo // ys
    xy_to_id[(x,y)] = foo

equivalence_classes = [ ( (0,y), (xs-1, (-y-1)%ys) ) for y in range(ys)]

for equiv_class in equivalence_classes:
    mm = min(equiv_class)
    for mem in equiv_class:
        xy_to_id[mem] = xy_to_id[mm]

cell_list = []
for foo in range(rad_samples):
    for bar in range(band_samples - 1):
        cell = [xy_to_id[(foo, bar)],
                xy_to_id[(foo, bar+1)],
                xy_to_id[(foo+1, bar+1)]]
        cell_list.append(cell)
        cell2= [xy_to_id[(foo, bar)],
                xy_to_id[(foo+1, bar+1)],
                xy_to_id[(foo+1, bar)]]
        cell_list.append(cell2)
print(cell_list)

num_pts = max(xy_to_id.values()) + 1

geo_pts = dict()
for foo in range(rad_samples):
    for bar in range(band_samples):
        theta = 2*np.pi*foo / rad_samples
        bb = band_width * ((bar/(band_samples-1)) - 0.5)
        x = np.cos(theta) * ( major_radius + bb * np.sin(theta/2 * turns))
        y = np.sin(theta) * ( major_radius + bb * np.sin(theta/2 * turns))
        z = bb * np.cos(theta/2 * turns)
        geo_pts[xy_to_id[(foo,bar)]] = (x,y,z)

if num_pts != len(set(geo_pts)):
    print(num_pts)
    print(geo_pts)
    print(len(set(geo_pts)))
    raise Exception("POINT NUMBER MISMATCH!")

points = vtk.vtkPoints()
triangles = vtk.vtkCellArray()
# verts = vtk.vtkCellArray()

for foo in range(num_pts):
    pt = geo_pts[foo]
    points.InsertNextPoint(pt[0],pt[1],pt[2])
    print(pt[0], pt[1], pt[2])
    # verts.InsertNextCell(1, [foo])

for cell in cell_list:
    # vtkcell = vtk.vtkTriangle()
    # for bar in cell:
    #     vtkcell.InsertNextPoint(bar)
    triangles.InsertNextCell(3, cell)
    # for p in cell:
    #     triangles.InsertNextCellPoint(p)

simplicial_complex = vtk.vtkPolyData()
simplicial_complex.SetPoints(points)
# simplicial_complex.SetVerts(verts)
simplicial_complex.SetPolys(triangles)
simplicial_complex.BuildCells()
simplicial_complex.BuildLinks()

print("cells", simplicial_complex.GetNumberOfCells())
print("points", simplicial_complex.GetNumberOfPoints())

ww = vtk.vtkSTLWriter()
ww.SetFileName("C:\\Users\\sscott\\Pictures\\fat_mobius.stl")
ww.SetInputData(simplicial_complex)
ww.Update()
ww.Write()

# xtris = rad_samples
# ytris = band_samples
# num_pts = xtris*(ytris+1)
# num_tris = 2 * rad_samples * band_samples;
# # vector < array < int, 3 > > cell_list;
#
# parametric_pts = []
# for foo in range(num_pts):
#     pt_col = foo % xtris
#     pt_row = foo // xtris
#     parametric_pts.append([ pt_row/ytris, pt_col/xtris])
#
# for (t,b) in parametric_pts:
#     theta = 2*np.pi * t
#     ll = (b-.5)
#
# cell_list = []
# for foo in range(num_tris):
#     sq = foo // 2;
#     is_down = foo % 2;
#     pt_col = sq % xtris;
#     pt_row = sq // xtris;
#     if ( not is_down):
#         cell = []
#         cell.append( pt_col  + xtris * pt_row );
#         cell.append( (1 + pt_col) % xtris  + xtris * (pt_row) )
#         cell.append( 0 + pt_col + xtris * (pt_row+1) )
#         cell_list.append(cell)
#     else:
#         cell = []
#         cell.append( (1 + pt_col) % xtris + xtris * pt_row );
#         cell.append( (1 + pt_col) % xtris + xtris * (pt_row+1))
#         cell.append( 0 + pt_col  + xtris * (pt_row+1) )
#         cell_list.append(cell)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random

def resample_data(df,frequency):
    df['Date'] = pd.to_datetime(df['Date'])

    df = (df.set_index('Date')
            .resample(frequency).first()
            .reset_index()
           .reindex(columns=df.columns)).dropna()
    return df

def log_return(Data):
    Returns = np.log(Data/Data.shift(1))
    Returns.dropna(inplace=True)
    return Returns

def inverse_log_return(prices, returns):
    returns = np.exp(returns)
    predicted_prices = np.multiply(np.reshape(prices, np.size(prices)), np.reshape(returns, np.size(returns)))
    return predicted_prices

def creating_study_periods(frequencies, frequencies_number_of_samples, frequency_index):
    # Import data
    Data = pd.read_csv('Data_1min.csv', dtype=str)
    Data = resample_data(Data, frequencies[frequency_index])

    # Create datasets
    dates = Data['Date']
    Data.drop('Date', inplace=True, axis=1)

    Data = Data.astype('float64')

    # Log return
    Returns = log_return(Data).to_numpy()

    # Create study peridos
    number_of_study_periods = np.floor(Returns.shape[0]/(frequencies_number_of_samples[frequency_index]/2)).astype(int)-3
    study_periods = np.zeros((2,number_of_study_periods, frequencies_number_of_samples[frequency_index]*2))
    for i in range(number_of_study_periods):
        study_periods[0,i] = Returns[(i*frequencies_number_of_samples[frequency_index]/2).astype(int):\
                                        ((i+4)*frequencies_number_of_samples[frequency_index]/2).astype(int)].flatten()
        study_periods[1,i] = dates.iloc[(i*frequencies_number_of_samples[frequency_index]/2).astype(int):\
                                        ((i+4)*frequencies_number_of_samples[frequency_index]/2).astype(int)].to_numpy().flatten()

    dates = dates[:((number_of_study_periods+3)*frequencies_number_of_samples[frequency_index]/2).astype(int)]
    return number_of_study_periods, study_periods, Data, dates

def visualize_results(mse):
    fig = plt.figure(figsize=(14,8))
    plt.plot(mse)
    plt.title('MSE of models')
    plt.legend(['ARMA', 'LSTM', 'GRU', 'RF', 'Zero return'])
    plt.show()

def visualize_data(Data, Returns):
    Data.plot(subplots=True, legend=False)
    plt.show()
    Returns.plot(subplots=True, legend=False)
    plt.show()
    Returns.hist(bins=100)
    plt.show()
    Returns.boxplot()
    plt.show()
    plt.matshow(Returns.T, interpolation=None, aspect='auto', cmap='Greys')
    plt.show()

def data_split(study_periods):
    train_ratio = 0.5
    valid_ratio = 0.25

    train_size = np.round(np.size(study_periods[0], 1) * train_ratio).astype(int)
    valid_size = np.round(np.size(study_periods[0], 1) * valid_ratio).astype(int)
    test_size = (np.size(study_periods[0], 1) - train_size - valid_size).astype(int)

    return train_size, valid_size, test_size

def append_periods(model_names, frequencies, frequencies_number_of_samples):
    study_periods_predictions = list()
    study_periods_returns = list()
    study_periods_dates = list()
    study_periods_number_of_study_periods = list()
    study_periods_study_periods = list()

    for frequency_index in range(5):
        print(f'Frequency: {frequencies[frequency_index]}')
        number_of_study_periods, study_periods, Data, dates = creating_study_periods(frequencies,\
                                                                                frequencies_number_of_samples,\
                                                                                frequency_index)
        train_size, valid_size, test_size = data_split(study_periods)

        predictions = np.zeros((len(model_names)+1, number_of_study_periods*test_size))

        for model_index in range(len(model_names)):
    #         print(model_names[model_index])
            predictions[model_index] = pd.read_csv('results/'+str(model_names[model_index])+\
                                                    '_predictions_frequency_'+str(frequencies[frequency_index])+\
                                                    '.csv', header=None).dropna(axis='columns').values.flatten()

        predictions[-1] = np.mean(predictions[:-1], axis=0)
        study_periods_predictions.append(predictions)
        study_periods_returns.append(study_periods[0,:,-test_size:].flatten())
        study_periods_dates.append(dates[-predictions.shape[1]:])
        study_periods_number_of_study_periods.append(number_of_study_periods)
        study_periods_study_periods.append(study_periods)
    return  study_periods_predictions, study_periods_returns, study_periods_dates,\
            study_periods_number_of_study_periods, study_periods_study_periods

def np_to_latex_table(data, name, calculate_mean=False, accuracy=2):
    if calculate_mean:
        table = np.zeros((data.shape[0]+1, data.shape[1]+1))
        table[:-1, :-1] = data
        table[-1, :-1] = np.mean(data, axis=0)
        table[:-1, -1] = np.mean(data, axis=1)
        table[-1, -1] = np.mean(np.mean(data))
    else:
        table = data

    if 0.1<np.mean(np.mean(table)):
        fmt = '%1.'+str(accuracy)+'f'
    else:
        fmt = '%2.'+str(accuracy)+'e'

#     print(table)
    np.savetxt(name, table, delimiter=' & ', fmt=fmt, newline=' \\\\\n')