import json
from Tkinter import *
from collections import OrderedDict

from pycparser import c_ast

from config import JsonConfig
from config.JsonConfig import NativeTypeToSimulatorType, NativeTypeToSize, DefaultEnumByteSize, TypeOverrides, \
    Infer2ndByteExceptions, Infer2ndByte, Infer2ndByteSimulatorTypeToNextSmallerType, NativeTypeToSimulatorBitfieldType, \
    InferUpTo4thByte, InferUpTo4thByteExceptions
from parsing.EnumsToJson import EnumsToJson
from parsing.ParseCTypes import CTypesParser, StructDeclVisitor, EnumDeclVisitor

Verbose = True

def printLeaf(fieldPath, fieldMame, fullyQualifiedName, type, bitSize, dimension):
    if bitSize is not None and bitSize is not "":
        bitSize = ":" + bitSize
    else:
        bitSize = ""

    if dimension is not None and dimension is not "":
        dimension = "[" + dimension + "]"
    else:
        dimension = ""

    if Verbose:
        print("%s%s <%s%s>" % (fullyQualifiedName, dimension, type, bitSize))

# def traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue, rootStructName, prefix, callback=printLeaf):
#
#     for structSubDecl in structs[rootStructName]:
#         structSubDeclTypeName = StructDeclVisitor.getStructTypeName(structSubDecl)
#         structSubDeclName = structSubDecl.name
#
#         dimension = None
#         if hasattr(structSubDecl.type, "dim"):
#             dimension = int(structSubDecl.type.dim.value)
#
#         isPointerType = type(structSubDecl.type) == c_ast.PtrDecl
#         if structSubDeclTypeName in nativeTypesToSize.keys() or \
#                         structSubDeclTypeName in enumTypesToValue.keys() or isPointerType == True:  # on enum, native or pointer type
#             bitSize = None
#             if structSubDecl.bitsize != None:
#                 bitSize = structSubDecl.bitsize.value
#
#             callback(prefix, structSubDeclName, prefix + "." + structSubDeclName, structSubDeclTypeName, bitSize,
#                      dimension, isPointerType)
#         else:  # on struct type
#             for dim in range(dimension):
#                 traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
#                                         rootStructName=structSubDeclTypeName, callback=callback,
#                                         prefix=prefix + "." + structSubDeclName + "[" + str(dim) + "]")
#             else:
#                 traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
#                                         rootStructName=structSubDeclTypeName, callback=callback,
#                                         prefix=prefix + "." + structSubDeclName)

def traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue, rootStructName, prefix, callback=printLeaf):
    for structSubDecl in structs[rootStructName]:
        structSubDeclTypeName = StructDeclVisitor.getStructTypeName(structSubDecl)
        structSubDeclName = structSubDecl.name

        dimension = None
        if hasattr(structSubDecl.type, "dim"):
            dimension = int(structSubDecl.type.dim.value)

        isPointerType = isPointerType = type(structSubDecl.type) == c_ast.PtrDecl
        if structSubDeclTypeName in nativeTypesToSize.keys() or \
                        structSubDeclTypeName in enumTypesToValue.keys() or (
            isPointerType == True):  # on enum/native type
            bitSize = None
            if isPointerType:
                bitSize = None
            elif structSubDecl.bitsize != None:
                bitSize = structSubDecl.bitsize.value

            callback(prefix, structSubDeclName, prefix + "." + structSubDeclName, structSubDeclTypeName, bitSize,
                     dimension, isPointerType)
        else:  # on struct type
            if dimension != None:
                for dim in range(dimension):
                    traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
                                            rootStructName=structSubDeclTypeName, callback=callback,
                                            prefix=prefix + "." + structSubDeclName + "[" + str(dim) + "]")
            else:
                traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
                                        rootStructName=structSubDeclTypeName, callback=callback,
                                        prefix=prefix + "." + structSubDeclName)

def appendEnumToSize(enumToSize, typeToSizeMapping):
    for enum in enumToSize.keys():
        typeToSizeMapping[enum] = 2

def appendEnumNames(enumDict, nativeTypeToSimulatorTypeMaping):
    for enum in enumDict.keys():
        nativeTypeToSimulatorTypeMaping[enum] = enum

class FieldDescription:
    def __init__(self, description, fieldPath, fieldName, typeName, arrayIdx=None, isBitField=False, startBit=None,
                 bitsLength=None, isPointerType=False):
        self.startBit = startBit
        self.length = bitsLength
        self.endBit = None
        self.fieldPath = fieldPath
        self.fieldName = fieldName
        self.typeName = typeName
        self.arrayIndex = arrayIdx
        self.isPointerTyp = isPointerType
        self.__isBitField = isBitField

        if self.startBit != None:
            self.startBit = int(self.startBit)

        if self.length != None:
            self.length = int(self.length)
        else:
            self.length = 0

        if self.startBit != None and self.length != None:
            self.endBit = self.startBit + self.length - 1

        self.description = description

    def getArrayIdx(self):
        return self.arrayIndex

    def isArrayField(self):
        return self.arrayIndex != None

    def isPointerType(self):
        return self.isPointerTyp

    def isBitField(self):
        return self.__isBitField

    def getFieldName(self):
        return self.fieldName

    def getFieldPath(self):
        return self.fieldPath

    def getTypeName(self):
        return self.typeName

    def getStartBit(self):
        return self.startBit

    def getEndBit(self):
        return self.endBit

    def getLength(self):
        return self.length

    def getDescription(self):
        return self.description

    def __str__(self):
        return "st[%s]\tlen=%s\tend[%s]\tdesc=%s p=%s f=%s t=<%s>" % (
            self.startBit, self.length, self.endBit, self.description, self.fieldPath, self.fieldName, self.typeName)

class RegisterEntry:
    def __init__(self, name, type, bitSize=None, arraySize=None):
        self.name = name
        self.type = type
        self.fieldDescriptions = {}

        if bitSize != None:
            self.isBitField = True
            self.bitSize = bitSize

        if arraySize != None:
            self.isArray = True
            self.arraySize = arraySize

    def addFieldDescription(self, fieldDescription):
        self.fieldDescriptions[fieldDescription.getStartBit()] = fieldDescription

class LinearStructComposer:
    def __init__(self, typeToSizeMapping):
        self.linearFields = []
        self.typeToSize = typeToSizeMapping
        self.aggregatedLinearFields = OrderedDict()

    def consumeStructField(self, fieldPath, fieldName, fullyQualifiedName, type, bitSize, dimension, isPointerType):

        # on pointer type decl
        if isPointerType == True:
            fieldDescription = FieldDescription("%s*" % fullyQualifiedName, fieldPath, fieldName, type,
                                                arrayIdx=0, isBitField=False, startBit=0, bitsLength=8,
                                                isPointerType=isPointerType)
            self.linearFields.append(fieldDescription)
            return

        isBitField = True
        if bitSize == None:
            isBitField = False
            bitSize = self.typeToSize[type] * 8

        # on normal decl
        if dimension == None:
            startBit = 0
            if bitSize != None:
                if len(self.linearFields) > 0:
                    lastField = self.linearFields[-1]
                    if lastField.isPointerType():
                        startBit = (1 + lastField.getEndBit()) % (8)
                    else:
                        startBit = (1 + lastField.getEndBit()) % (self.typeToSize[lastField.getTypeName()] * 8)
            fieldDescription = FieldDescription(fullyQualifiedName, fieldPath, fieldName, type, arrayIdx=None,
                                                isBitField=isBitField,
                                                startBit=startBit,
                                                bitsLength=bitSize, isPointerType=isPointerType)
            self.linearFields.append(fieldDescription)
        else:
            # on array decl
            for i in range(dimension):
                fieldDescription = FieldDescription("%s[%i]" % (fullyQualifiedName, i), fieldPath, fieldName, type,
                                                    arrayIdx=i, isBitField=isBitField, startBit=0, bitsLength=8,
                                                    isPointerType=isPointerType)
                self.linearFields.append(fieldDescription)

    def __createDescriptionDictIfNotAvailable(self, key):
        if key not in self.aggregatedLinearFields:
            self.aggregatedLinearFields[key] = OrderedDict()

    def aggregateDescriptions(self):
        currentBitFieldId = 0
        for description in reversed(self.linearFields):
            if description.isBitField():
                bitFieldKey = description.getFieldPath() + "<" + str(currentBitFieldId) + ">"
                self.__createDescriptionDictIfNotAvailable(bitFieldKey)
                self.aggregatedLinearFields[bitFieldKey][description.getStartBit()] = description
                if description.getStartBit() == 0:
                    currentBitFieldId += 1
            else:
                bitFieldKey = description.getDescription() + "<" + str(currentBitFieldId) + ">"
                self.__createDescriptionDictIfNotAvailable(bitFieldKey)
                self.aggregatedLinearFields[bitFieldKey][description.getStartBit()] = description
                currentBitFieldId += 1

class LinearStructFieldsToJson:
    jsonSource = None
    doInfer2ndByteFromUint16_t = Infer2ndByte
    doInferUpTo4thByte = InferUpTo4thByte

    def __init__(self):
        pass

    def __compactBitFieldDescriptions(self, fieldDescriptions):
        desc = "("
        isFirst = True
        for fieldDescription in reversed(fieldDescriptions.keys()):
            d = fieldDescriptions[fieldDescription]
            if not isFirst:
                desc += " | "
            isFirst = False

            if d.getStartBit() == d.getEndBit():
                fromTo = str(d.getStartBit())
            else:
                fromTo = str(d.getEndBit()) + ":" + str(d.getStartBit())

            desc += d.getFieldName() + "[" + fromTo + "]"
        return desc + ")"

    def __infer2ndByte(self, startAddressLabel, addressOffet, property, fieldDescription, entries, type,
                       nativeTypeToSize):
        if self.doInfer2ndByteFromUint16_t and \
                        fieldDescription.getTypeName() in nativeTypeToSize and \
                        2 == nativeTypeToSize[fieldDescription.getTypeName()]:
            for exception in Infer2ndByteExceptions:
                if exception["property"] == property and exception["type"] == type:
                    return

            newProperty = property + "[1]"
            if startAddressLabel != "":
                newAddress = startAddressLabel + "+" + str(addressOffet + 1)
            else:
                newAddress = addressOffet + 1

            newType = Infer2ndByteSimulatorTypeToNextSmallerType[type]
            entries[fieldDescription.getFieldPath()].append(
                {"property": newProperty, "type": newType, "address": newAddress})

    def __inferUpTo4ThByte(self, startAddressLabel, addressOffet, property, fieldDescription, entries, type,
                       nativeTypeToSize):
        if self.doInfer2ndByteFromUint16_t and \
                    fieldDescription.getTypeName() in nativeTypeToSize and \
                    4 == nativeTypeToSize[fieldDescription.getTypeName()]:
            for exception in InferUpTo4thByteExceptions:
                if exception["property"] == property and exception["type"] == type:
                    return

            for idx in range(1,4):
                newProperty = property + "["+str(idx)+"]"
                if startAddressLabel != "":
                    newAddress = startAddressLabel + "+" + str(addressOffet + idx)
                else:
                    newAddress = addressOffet + idx

                newType = type + "[" + str(idx) + "]"
                entries[fieldDescription.getFieldPath()].append(
                    {"property": newProperty, "type": newType, "address": newAddress})

    def toJson(self, structFields, nativeTypeToSize={}, nativeTypeToSimulatorType={}, startAddressLabel=""):

        addressOffet = 0
        entries = OrderedDict()
        for field in structFields:

            fieldDescription = structFields[field][structFields[field].keys()[0]]
            if fieldDescription.getFieldPath() not in entries:
                entries[fieldDescription.getFieldPath()] = []

            fieldTypeName = None
            if fieldDescription.isPointerType():
                fieldTypeName = "<pointerType>"
            else:
                fieldTypeName = fieldDescription.getTypeName()

            if fieldDescription.isBitField():
                type = NativeTypeToSimulatorBitfieldType[fieldDescription.getTypeName()]
                property = self.__compactBitFieldDescriptions(structFields[field])
            else:

                if 0 not in structFields[field]:
                    pass

                if fieldDescription.isArrayField():
                    property = structFields[field][0].getFieldName() + "[" + str(fieldDescription.getArrayIdx()) + "]"
                else:
                    try:
                        property = structFields[field][0].getFieldName()
                    except KeyError as ke:
                        print("key %s does not exist" % field)
                        # raise ke
                        pass

                type = nativeTypeToSimulatorType[fieldTypeName]

            self.__infer2ndByte(startAddressLabel, addressOffet, property, fieldDescription, entries,
                                type, nativeTypeToSize)
            self.__inferUpTo4ThByte(startAddressLabel, addressOffet, property, fieldDescription, entries,
                                type, nativeTypeToSize)

            if startAddressLabel != "":
                address = startAddressLabel + "+" + str(addressOffet)
            else:
                address = addressOffet
            addressOffet += nativeTypeToSize[fieldTypeName]

            entries[fieldDescription.getFieldPath()].append({"property": property, "type": type, "address": address})
        self.jsonSource = OrderedDict({"structs": entries})

    def show(self):
        print(json.dumps(self.jsonSource, sort_keys=False, indent=2, separators=(',', ': ')))

    def getDump(self):
        return json.dumps(self.jsonSource, sort_keys=False, indent=2, separators=(',', ': '))

    def getObject(self):
        return self.jsonSource

def parseEnumTypesAndStructs():
    ctp = CTypesParser()
    sdv = StructDeclVisitor()
    structs = ctp.getEntities(sdv)

    if Verbose:
        print("parsed structs")

    if Verbose:
        sdv.show()
    edv = EnumDeclVisitor()
    parsedEnums = ctp.getEntities(edv)

    if Verbose:
        print("\nparsed enums")
        edv.show()
    return structs, parsedEnums

def aggregateFields(structs, enumTypesToByteSize):
    enumToSize = {}
    for enum in enumTypesToByteSize:
        enumToSize[enum] = DefaultEnumByteSize
    typeToSizeMapping = dict(NativeTypeToSize.viewitems() | enumToSize.viewitems())
    composer = LinearStructComposer(typeToSizeMapping)
    traverseStructMembersDF(structs, NativeTypeToSize, enumTypesToByteSize, "Particle", "Particle",
                            callback=composer.consumeStructField)

    if Verbose:
        print("\nlinear ordered struct member list")
        for d in composer.linearFields:
            print ("%s" % d)
    composer.aggregateDescriptions()
    structFieldToDescription = OrderedDict()

    if Verbose:
        print("\naggregated linear ordered struct member list")
    for fieldPath in reversed(composer.aggregatedLinearFields.keys()):
        if Verbose:
            print ("%s" % fieldPath)
        fieldDescriptions = composer.aggregatedLinearFields[fieldPath]
        structFieldToDescription[fieldPath] = OrderedDict()
        for startBitPosition in reversed(fieldDescriptions.keys()):
            field = fieldDescriptions[startBitPosition]
            if Verbose:
                print(" -- %s" % (field))
            structFieldToDescription[fieldPath][field.getStartBit()] = field
    return enumToSize, structFieldToDescription

def generateCStructJson(enumToSize, structFieldToDescription):
    appendEnumToSize(enumToSize, NativeTypeToSize)
    appendEnumNames(enumToSize, NativeTypeToSimulatorType)

    lsfj = LinearStructFieldsToJson()
    lsfj.toJson(structFieldToDescription, nativeTypeToSize=NativeTypeToSize,
                nativeTypeToSimulatorType=NativeTypeToSimulatorType, startAddressLabel="globalStateBase")
    # lsfj.show()
    return lsfj.getObject(), lsfj.getDump()

class Gui:
    def __init__(self, txt):
        self.rootWindow = Tk()
        self.rootWindow.resizable(True, False)
        self.rootWindow.title("content is copied to clipboard")

        self.rootWindow.clipboard_clear()
        self.rootWindow.clipboard_append(txt)

        sb = Scrollbar(self.rootWindow)
        sb.pack(side=RIGHT, fill=Y)
        textField = Text(self.rootWindow, height=30, width=74)
        textField.pack(side=LEFT, fill=Y)
        sb.config(command=textField.yview)
        textField.config(yscrollcommand=sb.set)
        textField.insert(END, "\n" + txt + "\n")

        self.rootWindow.mainloop()

def aggregateToJsonDumpableDescription(structsObject, enumsObject):
    jsonDescription["enums"] = enumsObject["enums"]

    # add sizeof structs
    jsonDescription["sizeofTypes"] = OrderedDict()
    for enumType in enumToSize:
        jsonDescription["sizeofTypes"][enumType] = DefaultEnumByteSize

    # add labels
    jsonDescription["labels"] = JsonConfig.Labels["labels"]

    jsonDescription["structs"] = OrderedDict()
    # append staticly defined register description
    for key in JsonConfig.HardwareRegisters["structs"]:
        jsonDescription["structs"][key] = JsonConfig.HardwareRegisters["structs"][key]

    # append parsed c structs
    for key in structsObject["structs"]:
        jsonDescription["structs"][key] = structsObject["structs"][key]

    if Verbose:
        print("\ntype overridings")
    # override types
    for descriptionSetKey in jsonDescription["structs"]:
        for description in jsonDescription["structs"][descriptionSetKey]:
            # jsonDescription["structs"][descriptionSet][key]
            for override in TypeOverrides:
                substring = override["property"]
                oldType = override["oldType"]
                newType = override["newType"]
                if substring in description["property"] and oldType == description["type"]:
                    if Verbose:
                        print(descriptionSetKey + "." + description["property"] + " " + description[
                            "type"] + " <== " + newType)
                    description["type"] = newType

    return jsonDescription

def simulatorToSize(simulatorType, nativeToSimulator, nativeToSize):
    for nativeType in nativeToSimulator:
        if nativeToSimulator[nativeType] == simulatorType:
            for nativeToSizeKey in nativeToSize:
                if nativeToSizeKey == nativeType:
                    return nativeToSize[nativeToSizeKey]

if __name__ == "__main__":
    structs, parsedEnums = parseEnumTypesAndStructs()
    enumToSize, structFieldToDescription = aggregateFields(structs, parsedEnums)

    jsonDescription = OrderedDict()

    etj = EnumsToJson()
    enumsObject = etj.getObject()

    structsObject, jsonDump = generateCStructJson(enumToSize, structFieldToDescription)
    # print(jsonDump)

    jsonDescription = aggregateToJsonDumpableDescription(structsObject, enumsObject)

    totalSize = 0
    for struct in jsonDescription["structs"]:
        for field in jsonDescription["structs"][struct]:
            totalSize += simulatorToSize("bit", JsonConfig.NativeTypeToSimulatorType, JsonConfig.NativeTypeToSize)
    print("Total struct size: %s Bytes" % totalSize)
    # print(json.dumps(jsonDescription, sort_keys=False, indent=2, separators=(',', ': ')))
    Gui(json.dumps(jsonDescription, sort_keys=False, indent=2, separators=(',', ': ')))


import io
import sys
_INPUT = """\
3
010
"""
sys.stdin = io.StringIO(_INPUT)

N = int(input())
S = input()

for i in range(N):
  if(S[i] == "1"):
    if(i%2 == 0):
      print("Takahashi")
    else:
      print("Aoki")
    exit()


import json
import random
import string
import pkgutil

def str2bool(v):
    if v in ("yes", True, "true", "True", "TRUE", "t", "1"):
        return True
    elif v in ("no", False, "false", "False", "FALSE", "f", "0"):
        return False

def random_string(length=None):
    n = length or 20
    random_str = "".join(
        [random.choice(string.ascii_letters + string.digits) for i in range(n)]
    )
    return random_str

def load_resource(package, resource, as_json=True):
    """
    Open a file, and return the contents as JSON.
    Usage:
    load_resource(__name__, "resources/file.json")
    """
    resource = pkgutil.get_data(package, resource)
    return json.loads(resource) if as_json else resource.decode("utf-8")

def merge_multiple_dicts(*args):
    result = {}
    for d in args:
        result.update(d)
    return result

def filter_resources(resources, filters, attr_pairs):
    """
    Used to filter resources. Usually in get and describe apis.
    """
    result = resources.copy()
    for resource in resources:
        for attrs in attr_pairs:
            values = filters.get(attrs[0]) or None
            if values:
                instance = getattr(resource, attrs[1])
                if (len(attrs) <= 2 and instance not in values) or (
                    len(attrs) == 3 and instance.get(attrs[2]) not in values
                ):
                    result.remove(resource)
                    break
    return result


import web
render = web.template.render('templates/')
urls = (
	'/','index'
)

class index:
	def GET(self):
		name = 'TOM'
		return render.Show_temp(name)

if __name__=='__main__':
	app = web.application(urls,globals())
	app.run()


import cv2
import matplotlib.pyplot as plt
import numpy as np

from DLBio.aug_augmentation_functions import (ConnectedComponentImageToBinaryImage,
                                              IAugmentationFunction)
from DLBio.aug_generator_functions import DecoratorLabelOnly
from DLBio.helpers import cell_labeling

CELL_EDGE_IDX = 2
# function that can be used with a generator

class CCImageToCellAndTransitionLabel(DecoratorLabelOnly):
    def __init__(self, transition_kernel_size, DEBUG=False):
        super(CCImageToCellAndTransitionLabel, self).__init__(
            StandAloneCCImageToCellAndTransitionLabel(
                transition_kernel_size, DEBUG)
        )

class StandAloneCCImageToCellAndTransitionLabel(IAugmentationFunction):
    def __init__(self, transition_kernel_size, DEBUG=False):
        self.get_cell_transitions = ExtractCellTransitionsFromCCImage(
            transition_kernel_size,
            DEBUG
        )
        self.get_binary_image = ConnectedComponentImageToBinaryImage()
        self.__name__ = 'cc_image_to_cell_and_transition_label'

    def __call__(self, cc_image):
        transitions = self.get_cell_transitions(cc_image)
        transitions = reduce_dim_if_necessary(transitions)

        class_image = self.get_binary_image(cc_image)
        class_image = reduce_dim_if_necessary(class_image)

        class_image[transitions == 1] = CELL_EDGE_IDX
        return class_image

def reduce_dim_if_necessary(image):
    if image.ndim == 3:
        return image[..., 0]
    else:
        return image

class ExtractCellTransitionsFromCCImage(IAugmentationFunction):
    def __init__(self, kernel_size, DEBUG=False):
        self.kernel_size = kernel_size
        self.DEBUG = DEBUG

    def __call__(self, cc_image):
        """Extract the pixels of adjacent cells.

        Parameters
        ----------
        cc_image : connected_component image
        0 is background, an integer means pixel belongs to cell i.
        Problems if number of cells > 255!
        kernel_size : int, optional
        border is computed via dilation. The input determines the dilations size
        (the default is config.get('edge_kernel_size')).

        Returns
        -------
        boolean image
        which of the image pixels belong to the transition class.
        """
        if cc_image.ndim == 3:
            cc_image = cc_image[..., 0]

        # keep label values to 0,1,... to avoid values higher than 255
        cc_image = cell_labeling(cc_image)

        if cc_image.dtype != 'uint8':
            cc_image = cc_image.astype('uint8')

        kernel = np.ones((self.kernel_size, self.kernel_size))

        dilated_difference = cv2.dilate(cc_image, kernel).astype(
            'float32') - cc_image.astype('float32')

        # not sure which way of defining a border is better...
        # cell_to_cell_transition = (dilated_difference > 0) * (cc_image > 0)
        cell_to_cell_transition = (dilated_difference != 0) * (cc_image > 0)

        if self.DEBUG:
            _, ax = plt.subplots(1, 3)
            ax[0].imshow(cc_image)
            ax[1].imshow(dilated_difference)
            ax[2].imshow(cell_to_cell_transition)
            plt.show()

        return cell_to_cell_transition

class ExtractEdgesFromCCImage(IAugmentationFunction):

    def __init__(self, kernel_size):
        self.kernel_size = kernel_size

    def __call__(self, cc_image,
                 edge_is_part_of_cell=False
                 ):
        """Extract edges. Either object - background or object - other object.

        Parameters
        ----------
        cc_image : np.array of dtype uint8, int32, int64 or float32
            Zero is background. A specific object is marked with an integer index
            greater than zero.

        kernel_size : int, optional
            edges are extracted via dilation or erosion, the value defines
            the kernel size of the operator (the default is
            config.get('edge_kernel_size')).
        edge_is_part_of_cell : bool, optional
            If true former cell pixels are marked as an edge; else,
            background pixels are used (the default is False).

        Returns
        -------
        binary image: np.array of same size and dtype boolean
            edge pixels are marked as true (or > 0).
        """
        if cc_image.dtype != 'uint8':
            cc_image = cc_image.astype('uint8')

        kernel = np.ones((self.kernel_size, self.kernel_size))
        if edge_is_part_of_cell:
            edges = ((cc_image - cv2.erode(cc_image, kernel))
                     > 0).astype('float32')
        else:
            # edge is part of background
            edges = ((cv2.dilate(cc_image, kernel).astype(
                'float32') - cc_image.astype('float32')) > 0)

        return edges


from __future__ import print_function

from collections import OrderedDict
import importlib
import logging
import re

import voluptuous as vol

from esphome import core, core_config, yaml_util
from esphome.components import substitutions
from esphome.const import CONF_ESPHOME, CONF_PLATFORM, ESP_PLATFORMS
from esphome.core import CORE, EsphomeError
from esphome.helpers import color, indent
from esphome.py_compat import text_type
from esphome.util import safe_print

# pylint: disable=unused-import, wrong-import-order
from typing import List, Optional, Tuple, Union  # noqa
from esphome.core import ConfigType  # noqa
from esphome.yaml_util import is_secret
from esphome.voluptuous_schema import ExtraKeysInvalid

_LOGGER = logging.getLogger(__name__)

_COMPONENT_CACHE = {}

def get_component(domain):
    if domain in _COMPONENT_CACHE:
        return _COMPONENT_CACHE[domain]

    path = 'esphome.components.{}'.format(domain)
    try:
        module = importlib.import_module(path)
    except (ImportError, ValueError) as err:
        _LOGGER.debug(err)
    else:
        _COMPONENT_CACHE[domain] = module
        return module

    _LOGGER.error("Unable to find component %s", domain)
    return None

def get_platform(domain, platform):
    return get_component("{}.{}".format(domain, platform))

def is_platform_component(component):
    return hasattr(component, 'PLATFORM_SCHEMA')

def iter_components(config):
    for domain, conf in config.items():
        if domain == CONF_ESPHOME:
            yield CONF_ESPHOME, core_config, conf
            continue
        component = get_component(domain)
        if getattr(component, 'MULTI_CONF', False):
            for conf_ in conf:
                yield domain, component, conf_
        else:
            yield domain, component, conf
        if is_platform_component(component):
            for p_config in conf:
                p_name = u"{}.{}".format(domain, p_config[CONF_PLATFORM])
                platform = get_component(p_name)
                yield p_name, platform, p_config

ConfigPath = List[Union[str, int]]

def _path_begins_with_(path, other):  # type: (ConfigPath, ConfigPath) -> bool
    if len(path) < len(other):
        return False
    return path[:len(other)] == other

def _path_begins_with(path, other):  # type: (ConfigPath, ConfigPath) -> bool
    ret = _path_begins_with_(path, other)
    return ret

class Config(OrderedDict):
    def __init__(self):
        super(Config, self).__init__()
        self.errors = []  # type: List[Tuple[basestring, ConfigPath]]
        self.domains = []  # type: List[Tuple[ConfigPath, basestring]]

    def add_error(self, message, path):
        # type: (basestring, ConfigPath) -> None
        if not isinstance(message, text_type):
            message = text_type(message)
        self.errors.append((message, path))

    def add_domain(self, path, name):
        # type: (ConfigPath, basestring) -> None
        self.domains.append((path, name))

    def remove_domain(self, path, name):
        self.domains.remove((path, name))

    def lookup_domain(self, path):
        # type: (ConfigPath) -> Optional[basestring]
        best_len = 0
        best_domain = None
        for d_path, domain in self.domains:
            if len(d_path) < best_len:
                continue
            if _path_begins_with(path, d_path):
                best_len = len(d_path)
                best_domain = domain
        return best_domain

    def is_in_error_path(self, path):
        for _, p in self.errors:
            if _path_begins_with(p, path):
                return True
        return False

    def get_error_for_path(self, path):
        for msg, p in self.errors:
            if self.nested_item_path(p) == path:
                return msg
        return None

    def nested_item(self, path):
        data = self
        for item_index in path:
            try:
                data = data[item_index]
            except (KeyError, IndexError, TypeError):
                return {}
        return data

    def nested_item_path(self, path):
        data = self
        part = []
        for item_index in path:
            try:
                data = data[item_index]
            except (KeyError, IndexError, TypeError):
                return part
            part.append(item_index)
        return part

def iter_ids(config, path=None):
    path = path or []
    if isinstance(config, core.ID):
        yield config, path
    elif isinstance(config, core.Lambda):
        for id in config.requires_ids:
            yield id, path
    elif isinstance(config, list):
        for i, item in enumerate(config):
            for result in iter_ids(item, path + [i]):
                yield result
    elif isinstance(config, dict):
        for key, value in config.items():
            for result in iter_ids(value, path + [key]):
                yield result

def do_id_pass(result):  # type: (Config) -> None
    from esphome.cpp_generator import MockObjClass

    declare_ids = []  # type: List[Tuple[core.ID, ConfigPath]]
    searching_ids = []  # type: List[Tuple[core.ID, ConfigPath]]
    for id, path in iter_ids(result):
        if id.is_declaration:
            if id.id is not None and any(v[0].id == id.id for v in declare_ids):
                result.add_error(u"ID {} redefined!".format(id.id), path)
                continue
            declare_ids.append((id, path))
        else:
            searching_ids.append((id, path))
    # Resolve default ids after manual IDs
    for id, _ in declare_ids:
        id.resolve([v[0].id for v in declare_ids])

    # Check searched IDs
    for id, path in searching_ids:
        if id.id is not None:
            # manually declared
            match = next((v[0] for v in declare_ids if v[0].id == id.id), None)
            if match is None:
                # No declared ID with this name
                result.add_error("Couldn't find ID '{}'".format(id.id), path)
                continue
            if not isinstance(match.type, MockObjClass) or not isinstance(id.type, MockObjClass):
                continue
            if not match.type.inherits_from(id.type):
                result.add_error("ID '{}' of type {} doesn't inherit from {}. Please double check "
                                 "your ID is pointing to the correct value"
                                 "".format(id.id, match.type, id.type), path)

        if id.id is None and id.type is not None:
            for v in declare_ids:
                if v[0] is None or not isinstance(v[0].type, MockObjClass):
                    continue
                inherits = v[0].type.inherits_from(id.type)
                if inherits:
                    id.id = v[0].id
                    break
            else:
                result.add_error("Couldn't resolve ID for type '{}'".format(id.type), path)

def validate_config(config):
    result = Config()

    def _comp_error(ex, path):
        # type: (vol.Invalid, List[basestring]) -> None
        if isinstance(ex, vol.MultipleInvalid):
            errors = ex.errors
        else:
            errors = [ex]

        for e in errors:
            path_ = path + e.path
            domain = result.lookup_domain(path_) or ''
            result.add_error(_format_vol_invalid(e, config, path, domain), path_)

    skip_paths = list()  # type: List[ConfigPath]

    # Step 1: Load everything
    result.add_domain([CONF_ESPHOME], CONF_ESPHOME)
    result[CONF_ESPHOME] = config[CONF_ESPHOME]

    for domain, conf in config.items():
        domain = str(domain)
        if domain == CONF_ESPHOME or domain.startswith(u'.'):
            skip_paths.append([domain])
            continue
        result.add_domain([domain], domain)
        result[domain] = conf
        if conf is None:
            result[domain] = conf = {}
        component = get_component(domain)
        if component is None:
            result.add_error(u"Component not found: {}".format(domain), [domain])
            skip_paths.append([domain])
            continue

        if not isinstance(conf, list) and getattr(component, 'MULTI_CONF', False):
            result[domain] = conf = [conf]

        success = True
        dependencies = getattr(component, 'DEPENDENCIES', [])
        for dependency in dependencies:
            if dependency not in config:
                result.add_error(u"Component {} requires component {}".format(domain, dependency),
                                 [domain])
                success = False
        if not success:
            skip_paths.append([domain])
            continue

        success = True
        conflicts_with = getattr(component, 'CONFLICTS_WITH', [])
        for conflict in conflicts_with:
            if conflict in config:
                result.add_error(u"Component {} cannot be used together with component {}"
                                 u"".format(domain, conflict), [domain])
                success = False
        if not success:
            skip_paths.append([domain])
            continue

        esp_platforms = getattr(component, 'ESP_PLATFORMS', ESP_PLATFORMS)
        if CORE.esp_platform not in esp_platforms:
            result.add_error(u"Component {} doesn't support {}.".format(domain, CORE.esp_platform),
                             [domain])
            skip_paths.append([domain])
            continue

        if not hasattr(component, 'PLATFORM_SCHEMA'):
            continue

        result.remove_domain([domain], domain)

        if not isinstance(conf, list) and conf:
            result[domain] = conf = [conf]

        for i, p_config in enumerate(conf):
            if not isinstance(p_config, dict):
                result.add_error(u"Platform schemas must have 'platform:' key", [domain, i])
                skip_paths.append([domain, i])
                continue
            p_name = p_config.get('platform')
            if p_name is None:
                result.add_error(u"No platform specified for {}".format(domain), [domain, i])
                skip_paths.append([domain, i])
                continue
            p_domain = u'{}.{}'.format(domain, p_name)
            result.add_domain([domain, i], p_domain)
            platform = get_platform(domain, p_name)
            if platform is None:
                result.add_error(u"Platform not found: '{}'".format(p_domain), [domain, i])
                skip_paths.append([domain, i])
                continue

            success = True
            dependencies = getattr(platform, 'DEPENDENCIES', [])
            for dependency in dependencies:
                if dependency not in config:
                    result.add_error(u"Platform {} requires component {}"
                                     u"".format(p_domain, dependency), [domain, i])
                    success = False
            if not success:
                skip_paths.append([domain, i])
                continue

            success = True
            conflicts_with = getattr(platform, 'CONFLICTS_WITH', [])
            for conflict in conflicts_with:
                if conflict in config:
                    result.add_error(u"Platform {} cannot be used together with component {}"
                                     u"".format(p_domain, conflict), [domain, i])
                    success = False
            if not success:
                skip_paths.append([domain, i])
                continue

            esp_platforms = getattr(platform, 'ESP_PLATFORMS', ESP_PLATFORMS)
            if CORE.esp_platform not in esp_platforms:
                result.add_error(u"Platform {} doesn't support {}."
                                 u"".format(p_domain, CORE.esp_platform), [domain, i])
                skip_paths.append([domain, i])
                continue

    # Step 2: Validate configuration
    try:
        result[CONF_ESPHOME] = core_config.CONFIG_SCHEMA(result[CONF_ESPHOME])
    except vol.Invalid as ex:
        _comp_error(ex, [CONF_ESPHOME])

    for domain, conf in result.items():
        domain = str(domain)
        if [domain] in skip_paths:
            continue
        component = get_component(domain)

        if hasattr(component, 'CONFIG_SCHEMA'):
            multi_conf = getattr(component, 'MULTI_CONF', False)

            if multi_conf:
                for i, conf_ in enumerate(conf):
                    try:
                        validated = component.CONFIG_SCHEMA(conf_)
                        result[domain][i] = validated
                    except vol.Invalid as ex:
                        _comp_error(ex, [domain, i])
            else:
                try:
                    validated = component.CONFIG_SCHEMA(conf)
                    result[domain] = validated
                except vol.Invalid as ex:
                    _comp_error(ex, [domain])
                    continue

        if not hasattr(component, 'PLATFORM_SCHEMA'):
            continue

        for i, p_config in enumerate(conf):
            if [domain, i] in skip_paths:
                continue
            p_name = p_config['platform']
            platform = get_platform(domain, p_name)

            if hasattr(platform, 'PLATFORM_SCHEMA'):
                try:
                    p_validated = platform.PLATFORM_SCHEMA(p_config)
                except vol.Invalid as ex:
                    _comp_error(ex, [domain, i])
                    continue
                result[domain][i] = p_validated

    if not result.errors:
        # Only parse IDs if no validation error. Otherwise
        # user gets confusing messages
        do_id_pass(result)
    return result

def _nested_getitem(data, path):
    for item_index in path:
        try:
            data = data[item_index]
        except (KeyError, IndexError, TypeError):
            return None
    return data

def humanize_error(config, validation_error):
    offending_item_summary = _nested_getitem(config, validation_error.path)
    if isinstance(offending_item_summary, dict):
        offending_item_summary = None
    validation_error = text_type(validation_error)
    m = re.match(r'^(.*?)\s*(?:for dictionary value )?@ data\[.*$', validation_error)
    if m is not None:
        validation_error = m.group(1)
    validation_error = validation_error.strip()
    if not validation_error.endswith(u'.'):
        validation_error += u'.'
    if offending_item_summary is None or is_secret(offending_item_summary):
        return validation_error

    return u"{} Got '{}'".format(validation_error, offending_item_summary)

def _format_vol_invalid(ex, config, path, domain):
    # type: (vol.Invalid, ConfigType, ConfigPath, basestring) -> unicode
    message = u''
    try:
        paren = ex.path[-2]
    except IndexError:
        paren = domain

    if isinstance(ex, ExtraKeysInvalid):
        if ex.candidates:
            message += u'[{}] is an invalid option for [{}]. Did you mean {}?'.format(
                ex.path[-1], paren, u', '.join(u'[{}]'.format(x) for x in ex.candidates))
        else:
            message += u'[{}] is an invalid option for [{}]. Please check the indentation.'.format(
                ex.path[-1], paren)
    elif u'extra keys not allowed' in ex.error_message:
        message += u'[{}] is an invalid option for [{}].'.format(ex.path[-1], paren)
    elif u'required key not provided' in ex.error_message:
        message += u"'{}' is a required option for [{}].".format(ex.path[-1], paren)
    else:
        message += humanize_error(_nested_getitem(config, path), ex)

    return message

def load_config():
    try:
        config = yaml_util.load_yaml(CORE.config_path)
    except OSError:
        raise EsphomeError(u"Invalid YAML at {}. Please see YAML syntax reference or use an online "
                           u"YAML syntax validator".format(CORE.config_path))
    CORE.raw_config = config
    config = substitutions.do_substitution_pass(config)
    core_config.preload_core_config(config)

    try:
        result = validate_config(config)
    except EsphomeError:
        raise
    except Exception:
        _LOGGER.error(u"Unexpected exception while reading configuration:")
        raise

    return result

def line_info(obj, highlight=True):
    """Display line config source."""
    if not highlight:
        return None
    if hasattr(obj, '__config_file__'):
        return color('cyan', "[source {}:{}]"
                     .format(obj.__config_file__, obj.__line__ or '?'))
    return None

def _print_on_next_line(obj):
    if isinstance(obj, (list, tuple, dict)):
        return True
    if isinstance(obj, str):
        return len(obj) > 80
    if isinstance(obj, core.Lambda):
        return len(obj.value) > 80
    return False

def dump_dict(config, path, at_root=True):
    # type: (Config, ConfigPath, bool) -> Tuple[unicode, bool]
    conf = config.nested_item(path)
    ret = u''
    multiline = False

    if at_root:
        error = config.get_error_for_path(path)
        if error is not None:
            ret += u'\n' + color('bold_red', error) + u'\n'

    if isinstance(conf, (list, tuple)):
        multiline = True
        if not conf:
            ret += u'[]'
            multiline = False

        for i in range(len(conf)):
            path_ = path + [i]
            error = config.get_error_for_path(path_)
            if error is not None:
                ret += u'\n' + color('bold_red', error) + u'\n'

            sep = u'- '
            if config.is_in_error_path(path_):
                sep = color('red', sep)
            msg, _ = dump_dict(config, path_, at_root=False)
            msg = indent(msg)
            inf = line_info(config.nested_item(path_), highlight=config.is_in_error_path(path_))
            if inf is not None:
                msg = inf + u'\n' + msg
            elif msg:
                msg = msg[2:]
            ret += sep + msg + u'\n'
    elif isinstance(conf, dict):
        multiline = True
        if not conf:
            ret += u'{}'
            multiline = False

        for k in conf.keys():
            path_ = path + [k]
            error = config.get_error_for_path(path_)
            if error is not None:
                ret += u'\n' + color('bold_red', error) + u'\n'

            st = u'{}: '.format(k)
            if config.is_in_error_path(path_):
                st = color('red', st)
            msg, m = dump_dict(config, path_, at_root=False)

            inf = line_info(config.nested_item(path_), highlight=config.is_in_error_path(path_))
            if m:
                msg = u'\n' + indent(msg)

            if inf is not None:
                if m:
                    msg = u' ' + inf + msg
                else:
                    msg = msg + u' ' + inf
            ret += st + msg + u'\n'
    elif isinstance(conf, str):
        if is_secret(conf):
            conf = u'!secret {}'.format(is_secret(conf))
        if not conf:
            conf += u"''"

        if len(conf) > 80:
            conf = u'|-\n' + indent(conf)
        error = config.get_error_for_path(path)
        col = 'bold_red' if error else 'white'
        ret += color(col, text_type(conf))
    elif isinstance(conf, core.Lambda):
        if is_secret(conf):
            conf = u'!secret {}'.format(is_secret(conf))

        conf = u'!lambda |-\n' + indent(text_type(conf.value))
        error = config.get_error_for_path(path)
        col = 'bold_red' if error else 'white'
        ret += color(col, conf)
    elif conf is None:
        pass
    else:
        error = config.get_error_for_path(path)
        col = 'bold_red' if error else 'white'
        ret += color(col, text_type(conf))
        multiline = u'\n' in ret

    return ret, multiline

def strip_default_ids(config):
    if isinstance(config, list):
        to_remove = []
        for i, x in enumerate(config):
            x = config[i] = strip_default_ids(x)
            if isinstance(x, core.ID) and not x.is_manual:
                to_remove.append(x)
        for x in to_remove:
            config.remove(x)
    elif isinstance(config, dict):
        to_remove = []
        for k, v in config.items():
            v = config[k] = strip_default_ids(v)
            if isinstance(v, core.ID) and not v.is_manual:
                to_remove.append(k)
        for k in to_remove:
            config.pop(k)
    return config

def read_config(verbose):
    _LOGGER.info("Reading configuration...")
    try:
        res = load_config()
    except EsphomeError as err:
        _LOGGER.error(u"Error while reading config: %s", err)
        return None
    if res.errors:
        if not verbose:
            res = strip_default_ids(res)

        safe_print(color('bold_red', u"Failed config"))
        safe_print('')
        for path, domain in res.domains:
            if not res.is_in_error_path(path):
                continue

            safe_print(color('bold_red', u'{}:'.format(domain)) + u' ' +
                       (line_info(res.nested_item(path)) or u''))
            safe_print(indent(dump_dict(res, path)[0]))
        return None
    return OrderedDict(res)


#!/usr/bin/env python
import os, json, argparse, setlog, parsegenomes, mash, shutil, re
import copyseqsql, makeseqsql, glob, seqsql2fa, subprocess
import multiprocessing as mp
import makehmmsql, getgenematrix, getgenes, concatmsa, ete3helper
import numpy as np
#from scipy.cluster.vq import kmeans2

###
##### Workflow 1: Build query + reference phylogeny from scratch
###

def catTrees(flist,outfile):
    with open(outfile,"w") as ofil:
        for fname in flist:
            if os.path.exists(fname):
                with open(fname,"r") as ifil:
                    for line in ifil:
                        ofil.write(line)
            else:
                log.warning("Tree file not found for MLST: %s"%fname)
    return outfile

def runmafft(finput,output,thread=1,maxit=300,localpair=False,options="",rename="",fast=False):
    if localpair:
        options += " --localpair"
    if rename:
        #Rename fasta headers by ignoring everything after "rename" character
        with open(finput,"r") as infil, open(finput+".renamed","w") as ofil:
            for line in infil:
                if line.startswith(">") and rename in line:
                    ofil.write(line.strip().split(rename)[0]+"\n")
                else:
                    ofil.write(line)
        finput = finput+".renamed"
    if fast:
        cmd = "mafft --quiet --thread %d --treeout %s %s"%(thread,options,finput)
    else:
        cmd = "mafft --quiet --thread %d --localpair --treeout --maxiterate %d %s %s"%(thread,maxit,options,finput)
    cmd = cmd.split()
    with open(os.devnull,"w") as devnull, open(output,"w") as outfil:
        try:
            subprocess.call(cmd, stdout=outfil, stderr=devnull)
            log.info("Finished alignment %s"%output)
        except Exception as e:
            log.error("Failed to run mafft %s (%s)"%(finput,e))

def runtrimal(finput,output,method="automated1"):
    cmd = "trimal -in %s -out %s -%s"%(finput,output,method)
    cmd = cmd.split()
    with open(os.devnull,"w") as devnull:
        try:
            subprocess.call(cmd, stdout=devnull, stderr=devnull)
            log.info("Finished trimming %s"%output)
        except Exception as e:
            log.error("Failed to run trimall %s (%s)"%(finput,e))

def processmlst(indir,aligndir,cpu=1,parallel=True,trim=False,fast=False):
    flist = glob.glob(os.path.join(indir,"*.fna"))
    if not os.path.exists(aligndir):
        os.makedirs(aligndir)

    if cpu > 1 and parallel:
        pool = mp.Pool(cpu)
        try:
            for filename in flist:
                fname = os.path.split(filename)[-1]
                if trim:
                    pool.apply_async(runtrimal, args=(filename, os.path.join(aligndir,fname)))
                else:
                    pool.apply_async(runmafft, (filename, os.path.join(aligndir,fname)),dict(fast=fast))
        finally:
            pool.close()
            pool.join()
    else:
        for filename in flist:
            fname = os.path.split(filename)[-1]
            if trim:
                runtrimal(filename, os.path.join(aligndir,fname))
            else:
                runmafft(filename, os.path.join(aligndir,fname),thread=cpu,fast=fast)

def hmmsearch(outname,hmmdb,infile,mcpu=1,cut="tc"):
    log.info("Searching sequences against hmmdb: %s"%hmmdb)
    cmd=["hmmsearch", "--domtblout", outname, "--noali", "--notextw", hmmdb, infile]
    if mcpu>1:
        cmd[1:1] = ["--cpu", str(mcpu)]
    if cut and cut.lower() in ("ga","tc","nc"):
        cmd[1:1] = ["--cut_%s"%cut.lower()]
    with open(os.devnull,"w") as devnull:
        subprocess.call(cmd, stdout=devnull, stderr=devnull)

def buildseqdb(selorgs,queryseqs,resultdir,sourcedb):
    #Get list of all selected and outgroups
    reflist = list(selorgs.get("selspecies",[]))
    reflist.extend(selorgs.get("seloutgroups",[]))

    #Copy from reference database to query database
    seqsdb = os.path.join(resultdir,"seqs.db")
    copyseqsql.copydb(queryseqs,sourcedb,seqsdb)

def getorgs(resultdir,mashresult,skip="",IGlimit=50,OGlimit=1,minorgs=25):
    #Get user selection if exists
    usersel = os.path.join(resultdir,"userlist.json")
    autosel = os.path.join(resultdir,"autoOrglist.json")
    selection = False
    if os.path.exists(usersel):
        with open(usersel,"r") as fil:
            selection = json.load(fil)
    elif os.path.exists(autosel):
        with open(autosel,"r") as fil:
            selection = json.load(fil)
    elif "skip2" in skip:
        # commonrank = mashresult.get("commonrank",["phyl","",""])
        # if commonrank[0]:
        #     selection = {"selspecies":[x.get("id","") for x in mashresult.get("reforgs",[])[:IGlimit] if str(x.get(commonrank[0]+"id","")) in str(commonrank[1])],
        #              "seloutgroups":[x.get("id","") for x in mashresult.get("outgroups",[])[:OGlimit]]}
        # else:
        qorgs = mashresult.get("queryorgs",[])
        selrefs = mashresult.get("reforgs",[])[:IGlimit-len(qorgs)-OGlimit]
        selection = {"selspecies":[x.get("id","") for x in selrefs]}
        orgrecs = {i:v for i,v in enumerate(qorgs+selrefs)}
        # log.debug("Issue_orgrecs: %s"%orgrecs)
        commonrank = mash.getcommongroup(orgrecs)
        # log.debug("Issue_commonrank: %s"%commonrank)
        OGorgs = mash.getoutgrouporgs(commonrank,mashresult.get("reforgs",[]),glimit=OGlimit)
        # log.debug("Issue_OGorgs: %s"%OGorgs)
        selection["seloutgroups"] = [x.get("id","") for x in OGorgs[:OGlimit]]
        # if not len(selection["selspecies"]) > minorgs:
        #     selection = {"selspecies":[x.get("id","") for x in mashresult.get("reforgs",[])[:minorgs]],
        #              "seloutgroups":[x.get("id","") for x in mashresult.get("outgroups",[])[:OGlimit]]}

        with open(autosel,"w") as fil:
            json.dump(selection,fil)

    return selection

def getmlstselection(resultdir,mlstpriority,maxmlst=100,minmlst=10,skip="",ignoreorgs=None,concat=True):
    if not ignoreorgs:
        ignoreorgs = []
    usersel = os.path.join(resultdir,"usergenes.json")
    autosel = os.path.join(resultdir,"autoMlstlist.json")
    selection = False
    delorgs = set()
    if os.path.exists(usersel):
        with open(usersel,"r") as fil:
            temp = json.load(fil)
            selection = temp["selection"]
            delorgs = temp["delorgs"]
            concat = temp.get("mode","concatenated")
            if concat != "concatenated":
                concat = False
            return selection, list(delorgs), concat

    if os.path.exists(autosel):
        with open(autosel,"r") as fil:
            temp = json.load(fil)
            selection = temp["selection"]
            delorgs = temp["delorgs"]
    else:
        minrecs = [x for x in mlstpriority if not len(x["orgdel"])]
        log.info("MLST genes in all orgs = %s"%len(minrecs))
        if len(minrecs) >= minmlst:
            recs = minrecs[:maxmlst]
        else:
            recs = [x for x in mlstpriority if not any([org in x["orgdel"] for org in ignoreorgs])][:minmlst]
        selection = [x["acc"] for x in recs]
        for x in recs:
            delorgs = delorgs | set(x["orgdel"])
        with open(autosel,"w") as fil:
            json.dump({"selection":selection,"delorgs":list(delorgs)},fil)

    return selection, list(delorgs), concat

def runIQtree(outdir,infasta,partfile="",cpu=1,model="GTR",bs=0,fout="speciestree",titlesep="",outgroup=""):
    #If title seperator exists rename titles in file first
    if titlesep:
        tf = infasta+".renamed"
        with open(tf,"w") as ofil, open(infasta,"r") as ifil:
            for line in ifil:
                lineout = line.strip()
                if line.startswith(">") and titlesep in line:
                    lineout = line.strip().split(titlesep)[0]
                ofil.write(lineout+"\n")
        infasta = tf
    log.info("Building tree: model=%s, bootstrap=%s, file=%s"%(model,bs,fout))
    cmd=["iqtree", "-quiet", "-nt", str(cpu), "-s", infasta, "-m", model,"-pre",os.path.join(outdir,fout)]
    if bs:
        cmd.extend(["-bb",str(bs)])
    if outgroup:
        #get full title of first outgroup to root tree
        with open(infasta,"r") as ifil:
            for line in ifil:
                if line.startswith(">") and outgroup in line:
                    cmd.extend(["-o", "%s"%line[1:].strip()])
                    break
    if partfile:
        cmd.extend(["-spp", partfile])
    with open(os.devnull,"w") as devnull:
        subprocess.call(cmd,stdout=devnull,stderr=devnull)

def runAstral(resultdir,intree,outtree,astjar=""):
    log.info("Started running Astral: %s"%(intree))
    if not astjar or not os.path.realpath(astjar):
        astjar = os.path.join(os.path.dirname(__file__),"astral.5.5.9.jar")
    cmd=["java", "-Xmx3000M", "-jar", str(astjar), "-i", intree, "-o", outtree]
    with open(os.path.join(resultdir,"astral.log"),"w") as astlog:
        subprocess.call(cmd,stdout=astlog,stderr=astlog)

def colphylogeny(resultdir,trimdir,cpu=1,model="MFP",bs=0):
    treedir = os.path.join(resultdir,"trees")
    if not os.path.exists(treedir):
        os.makedirs(treedir)
    flist = glob.glob(os.path.join(trimdir,"*.fna"))
    #build all trees
    if cpu > 1:
        pool = mp.Pool(cpu)
        try:
            for filename in flist:
                pool.apply_async(runIQtree, (treedir, filename), dict(model=model,bs=bs,fout=os.path.split(filename)[-1],titlesep="|"))
        finally:
            pool.close()
            pool.join()
    else:
        for filename in flist:
            runIQtree(treedir, filename, fout=os.path.split(filename)[-1], bs=bs, model=model, titlesep="|")

    #Combine all trees using ASTRAL
    flist = glob.glob(os.path.join(treedir,"*.treefile"))
    alltrees = catTrees(flist,os.path.join(treedir,"alltrees.tree"))
    coltree = os.path.join(treedir,"summaryTree.tree")
    runAstral(resultdir,alltrees,coltree)

def concatphylogeny(resultdir,concatfasta,partfile,cpu=1,model="GTR",bs=0,outgroup=""):
    trimdir = os.path.join(resultdir,"mlst_trimmed")
    log.info("Concatenating all MLST genes")
    concatmsa.concatmsa(os.path.join(trimdir,"*.fna"),concatfasta,partfile,fsplit=" ",checktype=False)
    #Build tree from concatenated fasta
    treedir = os.path.join(resultdir,"trees")
    if not os.path.exists(treedir):
        os.makedirs(treedir)
    runIQtree(treedir,concatfasta,partfile=partfile,cpu=cpu,fout="concatTree.tree",bs=bs,model=model)

def screenmlst(mlstdir,aligndir,cpu=1,mingenes=50):
    """Additional screen for outlier genes with highly discordant parsimony trees"""
    guidetrees = glob.glob(os.path.join(mlstdir,"*.tree"))
    #combine all to tree file
    hkgenes = []
    alltrees = os.path.join(aligndir,"guidetrees.tree")
    with open(alltrees,"w") as gtfil:
        for fname in guidetrees:
            with open(fname,"r") as tfil:
                hk = os.path.split(fname)[-1]
                hk = hk[:hk.find(".")]
                hkgenes.append(hk)
                gtfil.write("".join([x.strip() for x in tfil]).replace(";","")+";\n")
    #Run robinson foulds distance calculation
    cmd = ["iqtree","-nt",str(cpu),"-rf_all",alltrees]
    with open(os.devnull,"w") as devnull:
        subprocess.call(cmd,stdout=devnull,stderr=devnull)

    #Read distance matrix and store median values for each gene
    treedists = []
    with open(alltrees+".rfdist","r") as fil:
        line = fil.next()
        for line in fil:
            temp = np.median([int(x) for x in line.strip().split()[1:] if int(x)>0])
            treedists.append(temp)

    #segment tress into 2 partitions using k-means
    #km,kmgroup = kmeans2(np.array(treedists),2)
    #Ensure group with highest distance is marked as group 1, group 0 is highest group
    #if km[0] > km[1]:
    #    kmgroup = np.ones(len(kmgroup))-kmgroup
    #inds = [i for i,x in enumerate(kmgroup) if x>0]
    #get hkgenes with highest group designation
    # return kmgroup,hkgenes,treedists

    #Assume normal distribution and highlight trees outside 1 standard deviation (~ 15% of distant genes)
    threshold = np.mean(treedists)+np.std(treedists)
    inds = np.array([int(x>=threshold) for x in treedists])

    hkhigh = sorted(np.array(hkgenes)[inds>0], key=lambda x: treedists[hkgenes.index(x)])

    #if lowest group is under min genes remove genes from highest group
    mindiff = mingenes - (len(hkgenes)-len(hkhigh))
    if mindiff > 0:
        hkhigh = hkhigh[mindiff+1:]

    #Mask screened MLST genes
    for hk in hkhigh:
        fname = os.path.join(aligndir,hk+".fna")
        os.rename(fname,fname+".removed")

    #write table of genes and median distances
    fname = os.path.join(aligndir,"removedsumamry.tsv")
    with open(fname,"w") as fil:
        for i,hk in enumerate(hkgenes):
            fil.write("%s\t%s\t%s\n"%(hk,treedists[i],inds[i]))

    return hkhigh

def startwf1(indir,resultdir,checkpoint=False,concat=False,mashmxdist=0.5,cpu=1,
             skip="",refdb="",hmmdb="",rnadb="",maxmlst=100,model="GTR",bs=0,
             kf=False,maxorg=50,filtMLST=True,fast=False,minmlst=10):
    """WORKFLOW 1: Build phylogeny from scratch"""
    if not checkpoint:
        checkpoint = "w1-0"
    queryseqs = os.path.join(resultdir,"queryseqs")

    #Parse all inputs
    if checkpoint == "w1-0":
        log.info("JOB_STATUS::Parsing all genomes...")
        if parsegenomes.parseall(indir,queryseqs):
            checkpoint = "w1-1"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::5/100")
        else:
            log.error("Problem parsing input genomes")
            return False

    #Run MASH distances
    mashresult = False
    if checkpoint == "w1-1":
        log.info("JOB_STATUS::Running MASH ANI estimation against reference sequences...")
        mashresult = mash.getdistances(queryseqs,resultdir,cpu=cpu,maxdist=mashmxdist)
        if mashresult:
            checkpoint = "w1-2"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::10/100")
        else:
            log.error("MASH distance failed")
            return False

    #Get set of organisms to build seq DB
    selorgs = False
    if checkpoint == "w1-2":
        log.info("JOB_STATUS::Loading mash results...")
        if mashresult:
            pass
        elif os.path.exists(os.path.join(resultdir,"reflist.json")):
            with open(os.path.join(resultdir,"reflist.json"),"r") as fil:
                mashresult = json.load(fil)
                log.info("Loading mash results...")
        else:
            log.error("No Mash results to process")
            return False

        selorgs = getorgs(resultdir,mashresult,skip=skip,IGlimit=maxorg,minorgs=25)
        if selorgs:
            checkpoint = "w1-3"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::15/100")
        else:
            checkpoint = "w1-STEP2"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)

    if checkpoint == "w1-STEP2":
        log.info("JOB_STATUS:: Waiting for selected organisms")
        return "waiting"

    #Copy reference sequence database and add query organisms
    orgdb = os.path.join(resultdir,"refquery.db")
    # selorgs = {}
    if not selorgs and os.path.exists(os.path.join(resultdir,"userlist.json")):
        with open(os.path.join(resultdir,"userlist.json"),"r") as fil:
            selorgs = json.load(fil)
    elif not selorgs and os.path.exists(os.path.join(resultdir,"autoOrglist.json")):
        with open(os.path.join(resultdir,"autoOrglist.json"),"r") as fil:
            selorgs = json.load(fil)
    if checkpoint == "w1-3":
        log.info("JOB_STATUS:: Collating selected genomes...")
        #Clear old db if exists
        if os.path.exists(orgdb):
            os.remove(orgdb)

        flist = list(selorgs["selspecies"])
        flist.extend(selorgs["seloutgroups"])
        flist = [x for x in flist if "query" not in x.lower()]
        log.info("refdb: %s"%refdb)
        if copyseqsql.copydb(flist,refdb,orgdb):
            #get file list of query orgs and add to db
            seqlist = glob.glob(os.path.join(queryseqs,"*.fna"))
            allorgs = makeseqsql.runlist(seqlist,orgdb,True,"",True)
            if allorgs:
                checkpoint = "w1-4"
                log.info("JOB_CHECKPOINT::%s"%checkpoint)
        else:
            log.error("Failed at copydb")

    #Run HMM searches on query organism
    if checkpoint == "w1-4":
        if not orgdb:
            log.error("No querydb rerun at checkpoint w1-3")
            return False
        #Write newly added seqences to file
        naseqs = os.path.join(resultdir,"addedseqs.fna")
        aaseqs = os.path.join(resultdir,"addedseqs.faa")
        seqsql2fa.writefasta(orgdb,aaseqs,False,"",True)
        seqsql2fa.writefasta(orgdb,naseqs,True,"",True)

        #Run HMM searches
        log.info("JOB_STATUS:: Searching for MLST genes in query sequences...")
        log.info("JOB_PROGRESS::25/100")
        if not hmmdb:
            hmmdb = os.path.join(os.path.dirname(os.path.realpath(__file__)),"reducedcore.hmm")
        if not rnadb:
            rnadb = os.path.join(os.path.dirname(os.path.realpath(__file__)),"barnap_bact_rRna.hmm")
        hmmsearch(aaseqs+".domhr",hmmdb,aaseqs,mcpu=cpu)
        hmmsearch(naseqs+".domhr",rnadb,naseqs,mcpu=cpu)

        #Add HMMresults to refdb and cleanup unused seqs
        log.info("Adding query HMM results to database")
        status = makehmmsql.run(aaseqs+".domhr",orgdb)
        if not status:
            #If no genes were found report error
            log.error("No MLST genes could be found. Stop processing")
        log.info("Adding query RNA results to database")
        status = makehmmsql.run(naseqs+".domhr",orgdb,rna=True)
        os.remove(aaseqs)
        os.remove(naseqs)
        checkpoint = "w1-5"
        log.info("JOB_CHECKPOINT::%s"%checkpoint)

    #Calculate mlst core genes and export
    mlstdir = os.path.join(resultdir,"mlstgenes")
    mlstpriority = os.path.join(resultdir,"mlstpriority.json")
    genematjson = os.path.join(resultdir,"mlstmatrix.json")
    dndsfile = os.path.join(os.path.dirname(os.path.realpath(__file__)),"dnds.json")
    mlstselection = []
    delorgs = []
    if checkpoint == "w1-5":
        allquery = [os.path.splitext(os.path.split(x)[-1])[0].replace(" ","_") for x in glob.glob(os.path.join(resultdir,"queryseqs","*.fna"))]
        ignoreorgs = list(selorgs.get("seloutgroups",[]))
        ignoreorgs.extend(allquery)

        #getmlstgenes.findsingles(orgdb,maxgenes=500,outdir=mlstdir)
        if os.path.exists(mlstpriority) and os.path.exists(genematjson):
            with open(genematjson,"r") as mfil, open(mlstpriority,"r") as pfil:
                mlstpriority = json.load(pfil)
                temp = json.load(mfil)
                genemat = temp["counts"]
                orgs = temp["orgs"]
                del temp
        else:
            genemat,orgs,mlstpriority = getgenematrix.getmat(orgdb,pct=0.5,pct2=1.0,bh=True,rna=True,savefil=genematjson,prifile=mlstpriority,dndsfile=dndsfile,ignoreorgs=allquery)
        mlstselection, delorgs, concat = getmlstselection(resultdir,mlstpriority,maxmlst,ignoreorgs=ignoreorgs,concat=concat,minmlst=minmlst)
        if "skip3" in skip.lower() or os.path.exists(os.path.join(resultdir,"usergenes.json")):
            #Export selected genes to mlst folder
            log.info("JOB_STATUS:: Writing MLST genes...")
            getgenes.writeallgenes(orgdb,mlstselection,delorgs,outdir=mlstdir,outgroups=selorgs.get("seloutgroups",None),pct=0.5,rename=True)
            checkpoint = "w1-6"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
        else:
            checkpoint = "w1-STEP3"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)

    if checkpoint == "w1-STEP3":
        log.info("JOB_STATUS:: Waiting for selected MLST genes")
        return "waiting"

    ## Align and trim all MLST genes
    aligndir = os.path.join(resultdir,"mlst_aligned")
    trimdir = os.path.join(resultdir,"mlst_trimmed")

    if checkpoint == "w1-6":
        log.info("JOB_STATUS:: Aligning MLST genes")
        log.info("JOB_PROGRESS::30/100")
        #align all
        processmlst(mlstdir,aligndir,cpu=cpu,fast=fast)
        checkpoint = "w1-6b"
        log.info("JOB_CHECKPOINT::%s"%checkpoint)

    if checkpoint == "w1-6b":
        if filtMLST and filtMLST != "False":
            #Extra screen of MLST genes to remove outliers based on starting tree distance
            log.info("JOB_STATUS:: Screening for inconsistent MLST genes")
            log.info("JOB_PROGRESS::55/100")
            excludemlst = screenmlst(mlstdir,aligndir,cpu=cpu)
            log.info("JOB_STATUS:: Excluded MLST genes: %s"%excludemlst)

        #trim all
        log.info("JOB_STATUS:: Trimming alignments")
        log.info("JOB_PROGRESS::65/100")
        processmlst(aligndir,trimdir,cpu=cpu,trim=True)

        checkpoint = "w1-7"
        log.info("JOB_CHECKPOINT::%s"%checkpoint)

    treedir = os.path.join(resultdir,"trees")
    finishedtree = ""
    #Build trees
    if checkpoint == "w1-7":
        log.info("JOB_PROGRESS::75/100")
        if concat and concat != "False":
            log.info("JOB_STATUS:: Running concatenated supermatrix phylogeny")
            concatfasta = os.path.join(resultdir,"concatMLST.fasta")
            partfile = os.path.join(resultdir,"nucpartition.txt")
            concatphylogeny(resultdir, concatfasta, partfile,cpu=cpu,model=model,bs=bs)
            checkpoint = "w1-F"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
        else:
            log.info("JOB_STATUS:: Running coalescent tree phylogeny")
            colphylogeny(resultdir,trimdir,cpu=cpu,model=model,bs=bs)
            checkpoint = "w1-F"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)

    if checkpoint == "w1-F":
        finaltree = os.path.join(resultdir,"final.tree")
        if os.path.exists(os.path.join(treedir,"concatTree.tree.treefile")) and concat:
            finishedtree = os.path.join(treedir,"concatTree.tree.treefile")
        elif os.path.exists(os.path.join(treedir,"summaryTree.tree")):
            finishedtree = os.path.join(treedir,"summaryTree.tree")
        #Copy final tree to root dir
        if finishedtree:
            fmat = 2 if bs or not concat else 5
            log.debug("Saving final tree... %s"%fmat)
            if not ete3helper.rerootTree(finishedtree,finaltree,fmat=fmat):
                shutil.copy(finishedtree,finaltree)
        else:
            log.error("Could not find final tree. Tree building failed")

        #Job finished? do some cleanup
        if not kf:
            if os.path.exists(orgdb):
                os.remove(orgdb)
            shutil.rmtree(os.path.join(resultdir,"queryseqs"))
            shutil.rmtree(os.path.join(resultdir,"mlst_trimmed"))
            shutil.rmtree(os.path.join(resultdir,"mlstgenes"))
            for oldfil in glob.glob(os.path.join(treedir,"*.model")):
                os.remove(oldfil)

def raxmlEPA(outdir,inalgn,reftree):
    #Run raxml EPA
    cmd="raxmlHPC-SSE3 -f v -m GTRGAMMA -p 12345 -w %s -t %s -s %s -n %s"%(outdir,reftree,inalgn,os.path.split(reftree)[-1])
    log.debug("Starting: %s"%cmd)
    try:
        with open(os.devnull,"w") as devnull:
            subprocess.call(cmd.split(),stdout=devnull)
        log.debug("RAxML-EPA: finished %s"%inalgn)
        return True
    except subprocess.CalledProcessError as e:
        log.error("RAxML-EPA: error, could not process %s - %s"%(inalgn,e))
        return False

def addalltrees(inlist,famrefdir,outdir,cpu=1):
    if cpu==1:
        for x in inlist:
            fname = os.path.split(x)[1]
            ftree = os.path.join(famrefdir,fname+".tree")
            raxmlEPA(outdir,x,ftree)
    else:
        pool = mp.Pool(cpu)
        try:
            for x in inlist:
                fname = os.path.split(x)[1]
                ftree = os.path.join(famrefdir,fname+".tree")
                pool.apply_async(raxmlEPA, (outdir,x,ftree))
        finally:
            pool.close()
            pool.join()

def addallalign(inlist,famrefdir,outdir,cpu=1,fast=False):
    if cpu==1:
        for x in inlist:
            fname = os.path.split(x)[1]
            addopt = "--add %s"%x
            finput = os.path.join(famrefdir,fname)
            output = os.path.join(outdir,fname)
            runmafft(finput,output,options=addopt,fast=fast)
    else:
        pool = mp.Pool(cpu)
        try:
            for x in inlist:
                fname = os.path.split(x)[1]
                addopt = "--add %s"%x
                finput = os.path.join(famrefdir,fname)
                output = os.path.join(outdir,fname)
                pool.apply_async(runmafft, (finput, output),dict(options=addopt,fast=fast))
        finally:
            pool.close()
            pool.join()

def getreference(mashresult,refdir):
    reffams = [str(x).lower() for x in os.listdir(refdir)]

    fams = list(set([q.get("familyname","N/A") for q in mashresult["queryorgs"]]))
    qfam = str(fams[0]).lower()
    if len(fams) == 1 and qfam in reffams:
        #get exact case as folder
        family = [fam for fam in os.listdir(refdir) if qfam == fam.lower()]
        return str(fams[0])
    elif len(fams) > 1:
        famlist = [(x,mashresult["queryorgs"][i].get("familyname","N/A")) for i,x in enumerate(mashresult["orglist"])]
        log.error("Multiple families detected in query geneomes: %s" % "\t".join( ["%s = %s |"%x for x in famlist]) )
    else:
        log.error("No matching family detected")

def startwf2(indir,resultdir,refdir="",checkpoint=False,reference="",model="GTR",bs=0,kf=False,maxmlst=100,
             cpu=1,mashmxdist=0.5,rnadb="",hmmdb="",fast=False):
    """WORKFLOW 2: Get all query genomes and identify reference tree to add sequences to"""
    if not checkpoint:
        checkpoint = "w2-0"
    if reference:
        checkpoint = "w2-3"
    queryseqs = os.path.join(resultdir,"queryseqs")

    #Parse all inputs
    if checkpoint == "w2-0":
        log.info("JOB_STATUS::Parsing all genomes...")
        if parsegenomes.parseall(indir,queryseqs):
            checkpoint = "w2-1"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::5/100")
        else:
            log.error("Problem parsing input genomes")
            return False

    #Run MASH distances
    mashresult = False
    if checkpoint == "w2-1":
        log.info("JOB_STATUS::Running MASH ANI estimation against reference sequences...")
        mashresult = mash.getdistances(queryseqs,resultdir,cpu=cpu,maxdist=mashmxdist)
        if mashresult:
            checkpoint = "w2-2"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::10/100")
        else:
            log.error("MASH distance failed")
            return False

    #Detect reference group
    if checkpoint == "w2-2":
        log.info("JOB_STATUS::Loading mash results...")
        if mashresult:
            pass
        elif os.path.exists(os.path.join(resultdir,"reflist.json")):
            with open(os.path.join(resultdir,"reflist.json"),"r") as fil:
                mashresult = json.load(fil)
                log.info("Loading mash results...")
        else:
            log.error("No Mash results to process")
            return False

        reference = getreference(mashresult,refdir)
        if reference:
            checkpoint = "w2-3"
            log.info("JOB_STATUS::Detected reference = %s"%reference)
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.info("JOB_PROGRESS::15/100")
        else:
            checkpoint = "w2-F"
            log.info("JOB_CHECKPOINT::%s"%checkpoint)
            log.error("No matching reference found for all query organisms")

    orgdb = os.path.join(resultdir,"allseqs.db")
    seqlist = glob.glob(os.path.join(queryseqs,"*.fna"))
    #Add to db and run HMM searches. Use model in reference folder, or use global models
    if checkpoint == "w2-3":
        #get file list of query orgs and add to db
        allorgs = makeseqsql.runlist(seqlist,orgdb,True,"",False)

        #Write newly added seqences to file
        naseqs = os.path.join(resultdir,"addedseqs.fna")
        aaseqs = os.path.join(resultdir,"addedseqs.faa")
        seqsql2fa.writefasta(orgdb,aaseqs,False,"",True)
        seqsql2fa.writefasta(orgdb,naseqs,True,"",True)

        #Run HMM searches
        log.info("JOB_STATUS:: Searching for MLST genes in query sequences...")
        log.info("JOB_PROGRESS::25/100")

        if not hmmdb:
            if os.path.exists(os.path.join(refdir,reference,"core.hmm")):
                hmmdb = os.path.join(refdir,reference,"core.hmm")
            else:
                hmmdb = os.path.join(os.path.dirname(os.path.realpath(__file__)),"reducedcore.hmm")
        if not rnadb:
            rnadb = os.path.join(os.path.dirname(os.path.realpath(__file__)),"barnap_bact_rRna.hmm")

        hmmsearch(aaseqs+".domhr",hmmdb,aaseqs,mcpu=cpu)
        hmmsearch(naseqs+".domhr",rnadb,naseqs,mcpu=cpu)

        #Add HMMresults
        log.info("Adding query HMM results to database")
        status = makehmmsql.run(aaseqs+".domhr",orgdb)
        if not status:
            #If no genes were found report error
            log.error("No MLST genes could be found. Stop processing")
        log.info("Adding query RNA results to database")
        status = makehmmsql.run(naseqs+".domhr",orgdb,rna=True)
        os.remove(aaseqs)
        os.remove(naseqs)
        checkpoint = "w2-4"
        log.info("JOB_CHECKPOINT::%s"%checkpoint)

    mlstdir = os.path.realpath(os.path.join(resultdir,"mlstgenes"))
    if not os.path.exists(mlstdir):
        os.makedirs(mlstdir)
    aligndir = os.path.realpath(os.path.join(resultdir,"mlst_aligned"))
    if not os.path.exists(aligndir):
        os.makedirs(aligndir)
    trimdir = os.path.realpath(os.path.join(resultdir,"mlst_trimmed"))
    if not os.path.exists(trimdir):
        os.makedirs(trimdir)

    # extract single copy results from genelist
    if checkpoint == "w2-4":
        genelist = [os.path.splitext(os.path.split(x)[1])[0] for x in glob.glob(os.path.join(refdir,reference,"*.fna"))]
        log.info("JOB_STATUS:: Writing MLST genes...")
        singles = [str(x) for x in getgenematrix.getmat(orgdb,rna=True,bh=True)[2][0] if x in genelist]
        getgenes.writeallgenes(orgdb,singles,[],outdir=mlstdir,outgroups=None,pct=0.5,writeaa=False,rename=True)

        log.info("JOB_STATUS:: Adding MLST genes to alignments...")
        addallalign([os.path.join(mlstdir,x+".fna") for x in singles], os.path.join(refdir,reference), aligndir, cpu=cpu,fast=fast)

        log.info("JOB_STATUS:: Trimming all alignments...")
        processmlst(aligndir,trimdir,cpu=cpu,trim=True)
        checkpoint = "w2-5"
        log.info("JOB_CHECKPOINT::%s"%checkpoint)

    treedir = os.path.realpath(os.path.join(resultdir,"trees"))
    if not os.path.exists(treedir):
        os.makedirs(treedir)

    #Add to prebuilt trees
    if checkpoint == "w2-5":
        log.info("JOB_PROGRESS::55/100")
        log.info("JOB_STATUS:: Adding to reference gene trees...")
        inlist = [os.path.join(trimdir,x) for x in os.listdir(trimdir)]
        addalltrees(inlist,os.path.join(refdir,reference),treedir,cpu=cpu)
        checkpoint = "w2-6"
        log.info("JOB_CHECKPOINT::%s" % checkpoint)

    if checkpoint == "w2-6":
        log.info("JOB_PROGRESS::85/100")
        log.info("JOB_STATUS:: Running coalescent tree phylogeny")
        #Combine all trees using ASTRAL
        flist = glob.glob(os.path.join(treedir,"RAxML_labelledTree.*.tree"))
        #reformat newick trees for astral
        for pbt in flist:
            with open(pbt,"r") as ifil, open(pbt+".newick","w") as ofil:
                x = ifil.next()
                ofil.write(re.sub("\[I\d+?\]|\"|'|QUERY___","",x))
        flist = glob.glob(os.path.join(treedir,"*.newick"))

        alltrees = catTrees(flist,os.path.join(treedir,"alltrees.tree"))
        coltree = os.path.join(treedir,"summaryTree.tree")
        runAstral(resultdir,alltrees,coltree)

        checkpoint = "w2-F"
        log.info("JOB_CHECKPOINT::%s" % checkpoint)

    finishedtree=""
    if checkpoint == "w2-F":
        finaltree = os.path.join(resultdir,"final.tree")
        if os.path.exists(os.path.join(treedir,"summaryTree.tree")):
            finishedtree = os.path.join(treedir,"summaryTree.tree")
        #Copy final tree to root dir
        if finishedtree:
            fmat = 2
            log.debug("Saving final tree... %s"%fmat)
            if not ete3helper.rerootTree(finishedtree,finaltree,fmat=fmat):
                shutil.copy(finishedtree,finaltree)
        else:
            log.error("Could not find final tree. Tree building failed")

def startjob(indir,resultdir,skip="",checkpoint=False,workflow=1,refdb="",cpu=1,concat=False,
             model="GTR",bs=0,kf=False,maxmlst=100,maxorg=50,filtMLST=True,refdir="",fast=False,minmlst=10):
    #Setup working directory
    if not os.path.exists(os.path.join(os.path.realpath(resultdir),"queryseqs")):
        os.makedirs(os.path.join(os.path.realpath(resultdir),"queryseqs")) #query sequence folder

    #Read checkpoint from log if exists or use checkpoint parameter
    if not checkpoint and os.path.exists(os.path.join(resultdir,"automlst.log")):
        with open(os.path.join(resultdir,"automlst.log"),"r") as fil:
            for line in fil:
                x = line.strip().split("::")
                if "JOB_CHECKPOINT" in x[0]:
                    checkpoint = x[1]
                elif "WORKFLOW" in x[0]:
                    workflow = int(x[1])
    #Start log
    global log
    log = setlog.init(os.path.join(resultdir,"automlst.log"),toconsole=True)

    #ensure bootsrtap value is valid and no more than 1000
    try:
        bs = int(bs)
        if bs > 1000:
            bs = 1000
        elif bs < 0:
            bs = 0
    except ValueError:
        log.warning("Invalid bootsrap value found, setting to 0")
        bs = 0

    log.debug("START / RESUME JOB last checkpoint: %s"%checkpoint)
    log.info('JOB_PARAMS::{"resultdir":"%s","skip":"%s","workflow":%s,"concat":"%s","model":"%s"}'%(resultdir,skip,workflow,concat,model))
    if workflow == 1:
        log.info("WORKFLOW::1")
        try:
            return startwf1(indir,resultdir,checkpoint=checkpoint,skip=skip,refdb=refdb,cpu=cpu,concat=concat,
                            model=model,bs=bs,kf=kf,maxmlst=maxmlst,maxorg=maxorg,filtMLST=filtMLST,fast=fast,minmlst=minmlst)
        except Exception as e:
            log.error("Unexpected failure: %s"%e)
            raise
    elif workflow == 2:
        log.info("WORKFLOW::2")
        try:
            return startwf2(indir,resultdir,refdir=refdir,checkpoint=checkpoint,model=model,bs=bs,kf=kf,maxmlst=maxmlst,cpu=cpu,fast=fast)
        except Exception as e:
            log.error("Unexpected failure: %s"%e)
            raise
    else:
        log.error("Improper workflow specified: %s"%workflow)
        return False

# Commandline Execution
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="""Start from input directory of genomes, get all reference distances and build phylogeny""")
    parser.add_argument("indir", help="Directory of input genomes or list of files")
    parser.add_argument("resultdir", help="Directory to store all results")
    parser.add_argument("-skip","--skip", help="Flag to skip manual intervention of organism selection. Ex: 'skip2.skip3' skips organism and mlst selection (default for cmdline execution)",default="skip2.skip3")
    parser.add_argument("-ref","--refdb", help="Reference database of orgs",default="")
    parser.add_argument("-rd","--refdirectory", help="Reference directory of pre-built trees and alignments",default="")
    parser.add_argument("-bs","--bootstrap", help="Bootstrap replicates (default: None)",default="")
    parser.add_argument("-cat","--concat", help="Use concatenated supermatrix to build tree",action="store_true",default=False)
    parser.add_argument("-kf","--keepfiles", help="Do not delete intermediate files after completion",action="store_true",default=False)
    parser.add_argument("-fm","--filtmlst", help="Filter mlst genes by starting tree congruance",action="store_true",default=False)
    parser.add_argument("-fa","--fastalign", help="Align using FFt-2 mode (faster)",action="store_true",default=False)
    parser.add_argument("-m","--model", help="Use specific model for iqtree parameter (default: GTR)",default="GTR")
    parser.add_argument("-ch","--checkpoint", help="Explicitly start at checkpoint",default=False)
    parser.add_argument("-c","--cpu", help="Number of cpu cores to use",type=int, default=1)
    parser.add_argument("-wf","--workflow", help="Workflow to use 1 or 2",choices=(1,2),type=int, default=1)
    parser.add_argument("-mm","--maxmlst", help="Maximum number of MLST genes to use (default:100)",type=int, default=100)
    parser.add_argument("-mn","--minmlst", help="Minimum number of MLST genes to use (default:10)",type=int, default=10)
    parser.add_argument("-mo","--maxorg", help="Maximum number of organisms to use (default:50)",type=int, default=50)
    args = parser.parse_args()
    startjob(args.indir,args.resultdir,args.skip,refdb=args.refdb,cpu=args.cpu,
             concat=args.concat,model=args.model,checkpoint=args.checkpoint,
             bs=args.bootstrap,kf=args.keepfiles,maxmlst=args.maxmlst,
             filtMLST=args.filtmlst,refdir=args.refdirectory,workflow=args.workflow,
             fast=args.fastalign,minmlst=args.minmlst)