#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# here put the import lib

import torch
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
import math
from skimage import morphology
import numpy as np
from torch.autograd import Variable

class VTAM(nn.Module):  # stereo attention block
    def __init__(self, embed_ch):
        super(VTAM, self).__init__()
        self.conv_l = nn.Conv2d(embed_ch, embed_ch, 1, 1, 0, bias=True)
        self.conv_r = nn.Conv2d(embed_ch, embed_ch, 1, 1, 0, bias=True)
        self.rb = ResBlock(embed_ch)
        self.softmax = nn.Softmax(-1)
        self.conv_out = nn.Conv2d(embed_ch * 2 + 1, embed_ch, 1, 1, 0, bias=True)

    def forward(self, x_left, x_right_nbr):  # B * C * H * W
        b, c, h, w = x_left.shape
        buffer_left = self.rb(x_left)
        buffer_right = self.rb(x_right_nbr)
        # print("buffer_right", buffer_right.shape)
        # M_{right_to_left}

        F0 = self.conv_l(buffer_left).contiguous().view(b, -1, h * w).permute(0, 2, 1)  # B, H * W, C
        F1 = self.conv_r(buffer_right).contiguous().view(b, -1, h * w)  # B, C, H * W
        # print(F0.shape)
        # print(F1.shape)

        attention = torch.bmm(F0, F1)  # B, H * W, H * W
        # view相当于reshape：若之前进行了permute，必须先进行contiguous
        # bmm相当于矩阵相乘，仅对后两维进行运算，并仅对三维torch进行运算。

        M_right_to_left = self.softmax(attention)  # B, H * W, H * W

        out_f = x_right_nbr.contiguous().view(b, -1, h * w).permute(0, 2, 1)  # B, H * W, C
        out_f = torch.bmm(M_right_to_left, out_f).contiguous().view(b, h, w, c).permute(0, 3, 1, 2)
        # B * C * H * W

        return out_f,  M_right_to_left

def initialize_weights(net_l, scale=1):
    if not isinstance(net_l, list):
        net_l = [net_l]
    for net in net_l:
        for m in net.modules():
            if isinstance(m, nn.Conv2d):
                init.kaiming_normal_(m.weight, a=0, mode='fan_in')
                m.weight.data *= scale  # for residual block
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                init.kaiming_normal_(m.weight, a=0, mode='fan_in')
                m.weight.data *= scale
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                init.constant_(m.weight, 1)
                init.constant_(m.bias.data, 0.0)

def default_conv(in_channels, out_channels, kernel_size, bias=True):
    return nn.Conv2d(
        in_channels, out_channels, kernel_size,
        padding=(kernel_size // 2), bias=bias)

def make_layer(block, n_layers):
    layers = []
    for _ in range(n_layers):
        layers.append(block())
    return nn.Sequential(*layers)

def conv_extractor(in_channels, out_channels, kernel_size=3, stride=1, dilation=1, bias=True):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, dilation=dilation, padding=((kernel_size-1)//2)*dilation, bias=bias),
        nn.LeakyReLU(negative_slope=0.1, inplace=True)
    )

def upconv_extractor(in_channels, out_channels):
    return nn.Sequential(
        nn.ConvTranspose2d(in_channels, out_channels, kernel_size=4, stride=2, padding=1, bias=True),
        nn.LeakyReLU(negative_slope=0.1, inplace=True)
    )

def default_conv(in_channels, out_channels, kernel_size, bias=True):
    return nn.Conv2d(
        in_channels, out_channels, kernel_size,
        padding=(kernel_size // 2), bias=bias)

def morphologic_process(mask):
    device = mask.device
    b, _, _, _ = mask.shape
    mask = ~mask  # ~：逐位取反
    mask_np = mask.cpu().numpy().astype(bool)
    mask_np = morphology.remove_small_objects(mask_np, 20, 2)  # 形态学处理，去除小于20的区域，2表示8邻接
    mask_np = morphology.remove_small_holes(mask_np, 10, 2)  # 形态学处理，去除小于10的孔，2表示8邻接
    for idx in range(b):
        buffer = np.pad(mask_np[idx, 0, :, :], ((3, 3), (3, 3)), 'constant')
        buffer = morphology.binary_closing(buffer, morphology.disk(3))  # closing 先膨胀再腐蚀
        mask_np[idx, 0, :, :] = buffer[3:-3, 3:-3]
    mask_np = 1 - mask_np
    mask_np = mask_np.astype(float)
    return torch.from_numpy(mask_np).float().to(device)

class ResBlock(nn.Module):
    def __init__(self, embed_ch):
        super(ResBlock, self).__init__()
        self.body = nn.Sequential(
            nn.Conv2d(embed_ch, embed_ch, kernel_size=3, stride=1, padding=1, bias=False),
            nn.LeakyReLU(0.1, inplace=True),
            nn.Conv2d(embed_ch, embed_ch, kernel_size=3, stride=1, padding=1, bias=False),
        )

    def __call__(self, x):
        res = self.body(x)
        return res + x

class ResidualBlock_noBN(nn.Module):
    '''Residual block w/o BN
    ---Conv-ReLU-Conv-+-
     |________________|
    '''

    def __init__(self, nf=64):
        super(ResidualBlock_noBN, self).__init__()
        self.conv1 = nn.Conv2d(nf, nf, 3, 1, 1, bias=True)
        self.conv2 = nn.Conv2d(nf, nf, 3, 1, 1, bias=True)

        # initialization
        initialize_weights([self.conv1, self.conv2], 0.1)

    def forward(self, x):
        identity = x
        out = F.relu(self.conv1(x), inplace=True)
        out = self.conv2(out)
        return identity + out

class Upsampler(nn.Sequential):  # 上采样/放大器
    def __init__(self, conv, scale, n_feat, bn=False, act=False, bias=True):
        # scale:放大倍数
        m = []
        if (scale & (scale - 1)) == 0:  # Is scale = 2^n?
            for _ in range(int(math.log(scale, 2))):  # 2^n,循环n次
                m.append(conv(n_feat, 4 * n_feat, 3, bias))
                m.append(nn.PixelShuffle(2))
                # PixelShuffle,(1,c*4,h,w)----->(1,c,h*2,w*2) 把通道的像素迁移到size上
                if bn: m.append(nn.BatchNorm2d(n_feat))
                if act: m.append(act())
        elif scale == 3:
            m.append(conv(n_feat, 9 * n_feat, 3, bias))
            m.append(nn.PixelShuffle(3))
            if bn: m.append(nn.BatchNorm2d(n_feat))
            if act: m.append(act())
        else:
            raise NotImplementedError

        super(Upsampler, self).__init__(*m)

class PAM(nn.Module):  # stereo attention block
    def __init__(self, embed_ch):
        super(PAM, self).__init__()
        self.conv_l = nn.Conv2d(embed_ch, embed_ch, 1, 1, 0, bias=True)
        self.conv_r = nn.Conv2d(embed_ch, embed_ch, 1, 1, 0, bias=True)
        self.rb = ResBlock(embed_ch)
        self.softmax = nn.Softmax(-1)
        self.conv_out = nn.Conv2d(embed_ch * 2 + 1, embed_ch, 1, 1, 0, bias=True)

    def forward(self, x_left, x_right):  # B * C * H * W
        b, c, h, w = x_left.shape
        buffer_left = self.rb(x_left)
        buffer_right = self.rb(x_right)
        # M_{right_to_left}
        F0 = self.conv_l(buffer_left).permute(0, 2, 3, 1)  # B * H * W * C
        F1 = self.conv_r(buffer_right).permute(0, 2, 1, 3)  # B * H * C * W
        S = torch.bmm(F0.contiguous().view(-1, w, c),
                      F1.contiguous().view(-1, c, w))  # (B*H) * W * W
        # view相当于reshape：若之前进行了permute，必须先进行contiguous
        # bmm相当于矩阵相乘，仅对后两维进行运算，并仅对三维torch进行运算。

        M_right_to_left = self.softmax(S)  # (B*H) * W * W
        # right map transfer to left
        S_T = S.permute(0, 2, 1)  # (B*H) * W * W
        M_left_to_right = self.softmax(S_T)

        # valid mask for transfer information from Feature(right->left)
        V_left_to_right = torch.sum(M_left_to_right.detach(), 1) > 0.1  # (B*H) * 1 * W
        V_left_to_right = V_left_to_right.view(b, 1, h, w)  # B * 1 * H * W
        V_left_to_right = morphologic_process(V_left_to_right)  # 形态学处理

        # V_right_to_left = torch.sum(M_right_to_left.detach(), 1) > 0.1
        # V_right_to_left = V_right_to_left.view(b, 1, h, w)  # B * 1 * H * W
        # V_right_to_left = morphologic_process(V_right_to_left)

        out_f = x_right.permute(0, 2, 3, 1).contiguous().view(-1, w, c)  # (B*H) * W * C
        out_f = torch.bmm(M_right_to_left, out_f).contiguous().view(b, h, w, c).permute(0, 3, 1, 2)
        # B * C * H * W

        return out_f, V_left_to_right, (M_left_to_right, M_right_to_left)

def flow_warp(x, flow, interp_mode='bilinear', padding_mode='zeros'):
    """Warp an image or feature map with optical flow
    Args:
        x (Tensor): size (N, C, H, W)
        flow (Tensor): size (N, H, W, 2), normal value
        interp_mode (str): 'nearest' or 'bilinear'
        padding_mode (str): 'zeros' or 'border' or 'reflection'
    Returns:
        Tensor: warped image or feature map
    """
    assert x.size()[-2:] == flow.size()[1:3]
    B, C, H, W = x.size()
    # mesh grid
    grid_y, grid_x = torch.meshgrid(torch.arange(0, H), torch.arange(0, W))
    grid = torch.stack((grid_x, grid_y), 2).float()  # W(x), H(y), 2
    grid.requires_grad = False
    grid = grid.type_as(x)
    vgrid = grid + flow
    # scale grid to [-1,1]
    vgrid_x = 2.0 * vgrid[:, :, :, 0] / max(W - 1, 1) - 1.0
    vgrid_y = 2.0 * vgrid[:, :, :, 1] / max(H - 1, 1) - 1.0
    vgrid_scaled = torch.stack((vgrid_x, vgrid_y), dim=3)
    output = F.grid_sample(x, vgrid_scaled, mode=interp_mode, padding_mode=padding_mode)
    return output

def flow_warping(x, flo):
    """warp an image/tensor (im2) back to im1, according to the optical flow
     x: [B, C, H, W] (im2)
     flo: [B, 2, H, W] flow   """
    B, C, H, W = x.size()
    # mesh grid
    xx = torch.arange(0, W).view(1, -1).repeat(H, 1)
    yy = torch.arange(0, H).view(-1, 1).repeat(1, W)
    xx = xx.view(1, 1, H, W).repeat(B, 1, 1, 1)
    yy = yy.view(1, 1, H, W).repeat(B, 1, 1, 1)
    grid = torch.cat((xx, yy), 1).float()
    grid.requires_grad = False
    grid = grid.type_as(x)

    vgrid = grid + flo

    # scale grid to [-1,1]
    vgrid[:, 0, :, :] = 2.0 * vgrid[:, 0, :, :].clone() / max(W - 1, 1) - 1.0
    vgrid[:, 1, :, :] = 2.0 * vgrid[:, 1, :, :].clone() / max(H - 1, 1) - 1.0

    vgrid = vgrid.permute(0, 2, 3, 1)
    output = nn.functional.grid_sample(x, vgrid)
        # mask = torch.autograd.Variable(torch.ones(x.size())).cuda()
        # mask = nn.functional.grid_sample(mask, vgrid)

        # if W==128:
        # np.save('mask.npy', mask.cpu().data.numpy())
        # np.save('warp.npy', output.cpu().data.numpy())

        # mask[mask < 0.9999] = 0
        # mask[mask > 0] = 1

    return output

class ResidualUpSample(nn.Module):
    def __init__(self, in_channels, bias=False):
        super(ResidualUpSample, self).__init__()

        self.top = nn.Sequential(nn.Conv2d(in_channels, in_channels,   1, stride=1, padding=0, bias=bias),
                                nn.PReLU(),
                                nn.ConvTranspose2d(in_channels, in_channels, 3, stride=2, padding=1, output_padding=1,bias=bias),
                                nn.PReLU(),
                                nn.Conv2d(in_channels, in_channels, 1, stride=1, padding=0, bias=bias))

        self.bot = nn.Sequential(nn.Upsample(scale_factor=2, mode='bilinear', align_corners=bias),
                                nn.Conv2d(in_channels, in_channels, 1, stride=1, padding=0, bias=bias))

    def forward(self, x):
        top = self.top(x)
        bot = self.bot(x)
        out = top+bot
        return out

class UpSampleFusion(nn.Module):
    def __init__(self, in_channels, scale_factor, stride=2, kernel_size=3):
        super(UpSampleFusion, self).__init__()
        self.scale_factor = int(np.log2(scale_factor))

        modules_body = []
        for i in range(self.scale_factor):
            modules_body.append(ResidualUpSample(in_channels))

        self.body = nn.Sequential(*modules_body)

    def forward(self, x):
        x = self.body(x)
        return x


import random
import argparse

class EndOfProgramException(Exception):
    pass

def format_raw_thue(raw_thue):
    seperator = raw_thue.index("::=")
    instructions = tuple(tuple(rule.split("::=")) for rule in raw_thue[:seperator] if rule != "")
    start_state = "".join(raw_thue[seperator+1:]) # Might be worth looking into other data structures i.e. linked list
    return instructions, start_state

def find_next_rule(rules, current_state, is_random, is_left):
    valid_rules = []
    for i, rule in enumerate(rules):
        if rule[0] in current_state:
            valid_rules.append(i)

    if len(valid_rules) == 0:
        raise EndOfProgramException()

    next_rule = None
    if is_random:
        next_rule = random.choice(valid_rules)
    elif is_left:
        next_rule = min(valid_rules)
    else:
        next_rule = max(valid_rules)

    return rules[next_rule]

def resolve_single_step(rules, curr_state, args):
    next_rule = find_next_rule(rules, curr_state, ~(args.left | args.right), args.left)
    if args.verbose:
        print(f"Using rule {next_rule[0]} => {next_rule[1]}")
    if next_rule[1] == "":
        next_state = curr_state.replace(next_rule[0],"")
    elif next_rule[1][0] == "~":
        print(next_rule[1][1:].replace("\\n","\n"), end= "\n" if args.nl__newline else "")
        next_state = curr_state.replace(next_rule[0],"")
    elif next_rule[1] == ":::":
        user_input = input(">").strip()
        next_state = curr_state.replace(next_rule[0], user_input)
    else:
        next_state = curr_state.replace(next_rule[0], next_rule[1])
    if args.verbose:
        print(f"{curr_state} => {next_state} \n")
    return next_state

def run_thue_program(rules, start_state, args):
    current_state = start_state
    step_counter = 0
    while True:
        if args.Maxiteration is not None:
            if step_counter > args.Maxiteration:
                break
        try:
            current_state = resolve_single_step(rules, current_state, args)
            step_counter += 1
        except EndOfProgramException:
            return current_state

    return f"Max iterations reached! Last program state was {current_state}"

def main():
    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    parser.add_argument("source", help="Location of Thue code")
    parser.add_argument("-v","--verbose", help="shows state after each instruciton", action="store_true")
    group.add_argument("-l","--left", help="executes first matching instruction from the left", action="store_true")
    group.add_argument("-r","--right", help="executes first matching instruction from the right", action="store_true")
    parser.add_argument("-M", "--Maxiteration", type=int, help="maximum number of steps the Thue program can for")
    parser.add_argument("-e", "--endstate", help="show last state before finishing", action="store_true")
    parser.add_argument("-nl" "--newline", help="adds newline after each ~", action="store_true")
    args = parser.parse_args()
    with open(args.source) as f:
        readlines = f.readlines()
        raw_program = [l[:-1] if i != len(readlines)-1 else l for i,l in enumerate(readlines)]

    rules, start_state = format_raw_thue(raw_program)
    end_state = run_thue_program(rules, start_state, args)
    if args.endstate:
        print(end_state)

if __name__ == "__main__":
    main()


import sys
from collections import deque
sys.stdin = open("baek1939.txt")

"""
가능한 최대치를 구하는 문제이므로,
조건을 만족해도 값을 키운다. (upper bound)
"""

def is_possible(weight):

    visited = [False] * (N+1)
    visited[S] = True
    que = deque()
    que.append(S)

    while que:
        node = que.popleft()
        if node == E:
            return True
        for next_node, weight_limit in graph[node]:

            if weight_limit < weight:
                continue
            if visited[next_node]:
                continue
            visited[next_node] = True
            que.append(next_node)
    return False

N, M = map(int, sys.stdin.readline().split())
graph = [[] for _ in range(0, N+1)]

# 같은 노드 사이의 여러 경로가있다면, 최댓값을 저장하자
temp_dict = {}
for i in range(0, M):
    a, b, c = map(int, sys.stdin.readline().split())
    graph[a].append((b, c))
    graph[b].append((a, c))

S, E = map(int, sys.stdin.readline().split())

# upper bound, lower bound 할때 최대치 +1로 시작해야한다! (이부분 주의)
left = 1
right = 1000000001
while left < right:
    mid = (left+right)//2

    if is_possible(mid):
        left = mid+1
    else:
        right = mid

print(left-1)


import cv2
import dlib
import numpy as np
class Landmarks():
  """A class for facial landmarks detection
    Args:
        model_path: expects a path to dlib facial landmarks pretrained predictor

    Attributes:
        detector: initializes and stores the face detector model. This pretrained model is based on HOG and SVM.
        predictor: initializes and stores the landmarks detector model. This pretrained model is based on decision trees ensemble.

    Methods:
        detect_faces: detect faces in original_image
            Returns:
                rectangles: coordinates of facial boundaries

        detect_landmarks: detect 68 facial landmark for each face
            Args:
                rectangles: coordinates of facial boundaries returned from detect_faces method
            Returns:
                rectangles_landmarks: coordinates of 68 facial landmark for each face
                                      indices are as follows: jaw: [0, 16], right eyebrow: [17, 21], left eyebrow: [22, 26], nose: [27, 35], right eye: [36, 41], left eye:[42, 47], mouth: [48, 67].

        apply_rectangles: draw green rectangles around each face
            Args:
                input_image: the image to draw facial boundaries on
                rectangles: coordinates of facial boundaries returned from detect_faces method
            Returns:
                detected_faces_image: modified image after drawing facial boundaries

        apply_landmarks: draw blue dots on each facial landmark
            Args:
                input_image: the image to draw facial landmarks on
                rectangles_coordinates: coordinates of facial landmarks returned from detect_landmarks method
            Returns:
                detected_landmarks_image: modified image after drawing facial landmarks
        image_mask_concat: creates mask and concatenate it with image
            Args:
                image: the image to concatenate facial landmarks mask with
            Returns:
                4 channels array of rgb image concatenated with landmarks mask
                  """

  def __init__(self,model_path):
    self.detector = dlib.get_frontal_face_detector() #instance of face detection model
    self.predictor = dlib.shape_predictor(model_path)  #instance of landmark detection model, pretrained model download link: https://www.kaggle.com/datasets/sergiovirahonda/shape-predictor-68-face-landmarksdat

  #draws a rectangle without overwriting the original image
  def _draw_rectangle(self,image, coordinates1, coordinates2):
    new_image=image.copy()
    thickness=int(0.007*(image.shape[0]*image.shape[1])**0.5)
    new_image=cv2.rectangle(new_image, coordinates1, coordinates2, (0, 255, 0), thickness)   #last two arguments are color of rectangle and thickness
    return new_image

  #draws a circle without overwriting the original image
  def _draw_circle(self,image, x, y):
    new_image=image.copy()
    thickness=int(0.007*(image.shape[0]*image.shape[1])**0.5)
    new_image=cv2.circle(new_image, (x, y), thickness, (0, 0, 255),-1)   #last three arguments are radius,color,and thickness (-1 means filled circles)
    return new_image

  #detects faces and returns rectangles of faces (regions of interests)
  def detect_faces(self,image):
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)  #transform image into gray scale
    rectangles = self.detector(gray_image)   # detect the faces (rectangles)
    return rectangles #rectangles coordinates

  #detects landmarks and returns landmarks of faces
  def detect_landmarks(self,image,rectangles):
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)  #transform image into gray scale
    rectangles_landmarks=[]  #list to append landmarks of each face
    for rectangle in rectangles: #iterate on faces
      landmarks = self.predictor(gray_image, rectangle)   # apply the shape predictor to the face ROI
      landmarks_coordinates=[] #list to append coordinates of each rectangle
      for n in range(landmarks.num_parts):
        x = landmarks.part(n).x
        y = landmarks.part(n).y
        landmarks_coordinates.append((x,y))
      rectangles_landmarks.append(landmarks_coordinates)

    modified_landmarks=[[]]
    for idx,landmark in enumerate(rectangles_landmarks[0]):
      if ((idx<=16) or (idx>=27 and idx<=30)):
        modified_landmarks[0].append(landmark)
    return modified_landmarks #returns list of lists (each list contains landmarks coordinates of 1 face)

  #draw rectangles on input image
  def apply_rectangles(self,input_image,rectangles):
    detected_faces_image=input_image
    for rectangle in rectangles:
      # extract the coordinates of the bounding box
      x1 = rectangle.left()
      y1 = rectangle.top()
      x2 = rectangle.right()
      y2 = rectangle.bottom()
      detected_faces_image=self._draw_rectangle(detected_faces_image, (x1, y1), (x2, y2)) #last two arguments are color of rectangle and thickness
    return detected_faces_image

  #draw landmarks on input image
  def apply_landmarks(self,input_image,rectangles_coordinates):
    detected_landmarks_image=input_image
    for rectangle_coordinates in rectangles_coordinates:
      for x,y in rectangle_coordinates:
        detected_landmarks_image=self._draw_circle(detected_landmarks_image, x, y) #last three arguments are radius,color,and thickness (-1 means filled circles)
    return detected_landmarks_image #landmarks are now drawn on the image

  #Create a landmarks mask with equivalent image dimensions
  def _create_mask(self,image,thickness=1):
    landmarks=self.detect_landmarks(image,self.detect_faces(image))
    modified_landmarks=[[]]
    if len(landmarks) >0:
      for idx,landmark in enumerate(landmarks[0]):
        if ((idx<=16) or (idx>=27 and idx<=30)):
          modified_landmarks[0].append(landmark)
    modified_landmarks_mask=np.zeros((image.shape[0],image.shape[1]))
    for modified_landmark in modified_landmarks:
      for x,y in modified_landmark:
        modified_landmarks_mask=self._draw_circle(modified_landmarks_mask, x, y,thickness)
    return modified_landmarks_mask

  #concatenate landmarks with original image
  def image_mask_concat(self, image, thickness=1):
    mask=self._create_mask(image,thickness)
    return np.dstack((image,mask))

##########################################################
# uncomment this section to test the module individually #
##########################################################
#import matplotlib.pyplot as plt
#image=cv2.imread("dataset_example.jpg")
#model=Landmarks(image)
#rectangles=model.detect_faces()
#landmarks=model.detect_landmarks(rectangles)
#print(rectangles)
#print("landmarks")
#print(landmarks)
#image_with_rectangles=model.apply_rectangles(image,rectangles)
#image_with_both= model.apply_landmarks(image_with_rectangles,landmarks) #draw landmarks on image with rectangles
#plt.imshow(cv2.cvtColor(image_with_both, cv2.COLOR_BGR2RGB));
#plt.show()


import re
import lark
import ifcopenshell.util
import ifcopenshell.util.fm
import ifcopenshell.util.element
import ifcopenshell.util.classification

filter_elements_grammar = lark.Lark("""start: filter_group
    filter_group: facet_list ("+" facet_list)*
    facet_list: facet ("," facet)*

    facet: instance | entity | attribute | type | material | property | classification | location

    instance: not? globalid
    globalid: /[0-3][a-zA-Z0-9_$]{21}/
    entity: not? ifc_class
    attribute: attribute_name comparison value
    type: "type" comparison value
    material: "material" comparison value
    property: pset "." prop comparison value
    classification: "classification" comparison value
    location: "location" comparison value

    pset: quoted_string | unquoted_string | regex_string
    prop: quoted_string | unquoted_string | regex_string

    attribute_name: /[A-Z]\\w+/
    ifc_class: /Ifc\\w+/

    value: special | quoted_string | unquoted_string | regex_string
    unquoted_string: /[^.=\\s]+/
    quoted_string: ESCAPED_STRING
    regex_string: "/" /[^\\/]+/ "/"

    special: null | true | false

    comparison: not? equals
    not: "!"
    equals: "="
    null: "NULL"
    true: "TRUE"
    false: "FALSE"

    // Embed common.lark for packaging
    DIGIT: "0".."9"
    HEXDIGIT: "a".."f"|"A".."F"|DIGIT
    INT: DIGIT+
    SIGNED_INT: ["+"|"-"] INT
    DECIMAL: INT "." INT? | "." INT
    _EXP: ("e"|"E") SIGNED_INT
    FLOAT: INT _EXP | DECIMAL _EXP?
    SIGNED_FLOAT: ["+"|"-"] FLOAT
    NUMBER: FLOAT | INT
    SIGNED_NUMBER: ["+"|"-"] NUMBER
    _STRING_INNER: /.*?/
    _STRING_ESC_INNER: _STRING_INNER /(?<!\\\\)(\\\\\\\\)*?/
    ESCAPED_STRING : "\\"" _STRING_ESC_INNER "\\""
    LCASE_LETTER: "a".."z"
    UCASE_LETTER: "A".."Z"
    LETTER: UCASE_LETTER | LCASE_LETTER
    WORD: LETTER+
    CNAME: ("_"|LETTER) ("_"|LETTER|DIGIT)*
    WS_INLINE: (" "|/\\t/)+
    WS: /[ \\t\\f\\r\\n]/+
    CR : /\\r/
    LF : /\\n/
    NEWLINE: (CR? LF)+

    %ignore WS // Disregard spaces in text
""")

get_element_grammar = lark.Lark("""start: WORD | ESCAPED_STRING | keys_regex | keys_quoted | keys_simple
    keys_regex: "r" ESCAPED_STRING ("." ESCAPED_STRING)*
    keys_quoted: ESCAPED_STRING ("." ESCAPED_STRING)*
    keys_simple: /[^\\W][^.=<>!%*\\]]*/ ("." /[^\\W][^.=<>!%*\\]]*/)*

    // Embed common.lark for packaging
    _STRING_INNER: /.*?/
    _STRING_ESC_INNER: _STRING_INNER /(?<!\\\\)(\\\\\\\\)*?/
    ESCAPED_STRING : "\\"" _STRING_ESC_INNER "\\""
    LCASE_LETTER: "a".."z"
    UCASE_LETTER: "A".."Z"
    LETTER: UCASE_LETTER | LCASE_LETTER
    WORD: LETTER+
    WS: /[ \\t\\f\\r\\n]/+

    %ignore WS // Disregard spaces in text
 """)

def get_element_value(element, query):
    start = get_element_grammar.parse(query)
    filter_query = Selector.parse_filter_query(start.children[0])
    return Selector.get_element_value(element, filter_query["keys"], filter_query["is_regex"])

def filter_elements(ifc_file, query, elements=None):
    transformer = FacetTransformer(ifc_file, elements)
    transformer.transform(filter_elements_grammar.parse(query))
    return transformer.get_results()
    return transformer.elements

class FacetTransformer(lark.Transformer):
    def __init__(self, ifc_file, elements):
        self.file = ifc_file
        self.results = []
        self.elements = elements or set()
        self.container_parents = {}
        self.container_trees = {}

    def get_results(self):
        results = set()
        for r in self.results:
            results |= r
        return results

    def facet_list(self, args):
        if self.elements:
            self.results.append(self.elements)
            self.elements = set()

    def instance(self, args):
        if args[0].data == "globalid":
            self.elements.add(self.file.by_guid(args[0].children[0].value))
        else:
            self.elements.remove(self.file.by_guid(args[1].children[0].value))

    def entity(self, args):
        if args[0].data == "ifc_class":
            self.elements |= set(self.file.by_type(args[0].children[0].value))
        else:
            self.elements -= set(self.file.by_type(args[1].children[0].value))

    def attribute(self, args):
        name, comparison, value = args
        name = name.children[0].value

        def filter_function(element):
            element_value = getattr(element, name, None)
            return self.compare(element_value, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def type(self, args):
        comparison, value = args

        def filter_function(element):
            element_value = getattr(ifcopenshell.util.element.get_type(element), "Name", None)
            return self.compare(element_value, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def material(self, args):
        comparison, value = args

        def filter_function(element):
            materials = ifcopenshell.util.element.get_materials(element)
            result = False if materials else None
            for material in materials:
                if self.compare(material.Name, comparison, value):
                    result = True
                if self.compare(getattr(material, "Category", None), comparison, value):
                    result = True
            if result is not None:
                return result if comparison == "=" else not result
            return self.compare(None, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def property(self, args):
        pset, prop, comparison, value = args

        def filter_function(element):
            if isinstance(pset, str) and isinstance(prop, str):
                element_value = ifcopenshell.util.element.get_pset(element, pset, prop)
                return self.compare(element_value, comparison, value)
            elif isinstance(pset, str) and isinstance(prop, re.Pattern):
                element_props = ifcopenshell.util.element.get_pset(element, pset) or {}
                for element_prop, element_value in element_props.items():
                    if prop.match(element_prop):
                        return self.compare(element_value, comparison, value)
            elif isinstance(pset, re.Pattern):
                element_psets = ifcopenshell.util.element.get_psets(element)
                for element_pset, element_props in element_psets.items():
                    if not pset.match(element_pset):
                        continue
                    if isinstance(prop, str):
                        element_value = element_props.get(prop, None)
                        if element_value is not None:
                            return self.compare(element_value, comparison, value)
                    elif isinstance(prop, re.Pattern):
                        for element_prop, element_value in element_props.items():
                            if prop.match(element_prop):
                                return self.compare(element_value, comparison, value)
            return self.compare(None, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def classification(self, args):
        comparison, value = args

        def filter_function(element):
            references = ifcopenshell.util.classification.get_references(element)
            result = False if references else None
            for reference in references:
                if self.compare(reference.Name, comparison, value):
                    result = True
                if self.compare(
                    getattr(reference, "Identification", getattr(reference, "ItemReference", None)), comparison, value
                ):
                    result = True
            if result is not None:
                return result if comparison == "=" else not result
            return self.compare(None, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def location(self, args):
        comparison, value = args

        def filter_function(element):
            container = ifcopenshell.util.element.get_container(element)
            containers = self.get_container_tree(container)
            result = False if containers else None
            for container in containers:
                if self.compare(container.Name, comparison, value):
                    result = True
            if result is not None:
                return result if comparison == "=" else not result
            return self.compare(None, comparison, value)

        self.elements = set(filter(filter_function, self.elements))

    def get_container_tree(self, container):
        tree = self.container_trees.get(container, None)
        if tree:
            return tree

        tree = []

        while container:
            if container.is_a("IfcProject"):
                break
            tree.append(container)
            container = ifcopenshell.util.element.get_aggregate(container)

        tree_copy = tree.copy()
        while tree_copy:
            self.container_trees[tree_copy.pop(0)] = tree_copy.copy()
        return tree

    def comparison(self, args):
        return "=" if args[0].data == "equals" else "!="

    def pset(self, args):
        return self.value(args)

    def prop(self, args):
        return self.value(args)

    def value(self, args):
        if args[0].data == "unquoted_string":
            return args[0].children[0].value
        elif args[0].data == "quoted_string":
            return args[0].children[0].value[1:-1].replace('\\"', '"')
        elif args[0].data == "regex_string":
            return re.compile(args[0].children[0].value)
        elif args[0].data == "special":
            if args[0].children[0].data == "null":
                return None
            elif args[0].children[0].data == "true":
                return True
            elif args[0].children[0].data == "false":
                return False

    def compare(self, element_value, comparison, value):
        if isinstance(value, str):
            if isinstance(element_value, int):
                value = int(value)
            elif isinstance(element_value, float):
                value = float(value)
            result = element_value == value
        elif isinstance(value, re.Pattern):
            result = bool(value.match(element_value))
        elif value in (None, True, False):
            result = element_value is value
        return result if comparison == "=" else not result

class Selector:
    @classmethod
    def parse(cls, ifc_file, query, elements=None):
        cls.file = ifc_file
        cls.elements = elements
        l = lark.Lark(
            """start: query (lfunction query)*
                    query: selector | group
                    group: "(" query (lfunction query)* ")"
                    selector: (inverse_relationship)? guid_selector | (inverse_relationship)? class_selector
                    guid_selector: "#" /[0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_$]{22}/
                    class_selector: "." WORD filter ?
                    filter: "[" filter_key (comparison filter_value)? "]"
                    filter_key: WORD | ESCAPED_STRING | keys_regex | keys_quoted | keys_simple
                    filter_value: filter_regex | ESCAPED_STRING | SIGNED_FLOAT | SIGNED_INT | BOOLEAN | NULL
                    filter_regex: "r" ESCAPED_STRING
                    keys_regex: "r" ESCAPED_STRING ("." ESCAPED_STRING)*
                    keys_quoted: ESCAPED_STRING ("." ESCAPED_STRING)*
                    keys_simple: /[^\\W][^.=<>!%*\\]]*/ ("." /[^\\W][^.=<>!%*\\]]*/)*
                    lfunction: and | or
                    inverse_relationship: types | decomposed_by | bounded_by | grouped_by
                    types: "*"
                    decomposed_by: "@"
                    bounded_by: "@@"
                    grouped_by: "@@@"
                    and: "&"
                    or: "|"
                    not: "!"
                    comparison: (not)* (oneof | contains | morethanequalto | lessthanequalto | equal | morethan | lessthan)
                    oneof: "%="
                    contains: "*="
                    morethanequalto: ">="
                    lessthanequalto: "<="
                    equal: "="
                    morethan: ">"
                    lessthan: "<"
                    BOOLEAN: "TRUE" | "FALSE" | "true" | "false"| "True" | "False"
                    NULL: "NULL"

                    // Embed common.lark for packaging
                    DIGIT: "0".."9"
                    HEXDIGIT: "a".."f"|"A".."F"|DIGIT
                    INT: DIGIT+
                    SIGNED_INT: ["+"|"-"] INT
                    DECIMAL: INT "." INT? | "." INT
                    _EXP: ("e"|"E") SIGNED_INT
                    FLOAT: INT _EXP | DECIMAL _EXP?
                    SIGNED_FLOAT: ["+"|"-"] FLOAT
                    NUMBER: FLOAT | INT
                    SIGNED_NUMBER: ["+"|"-"] NUMBER
                    _STRING_INNER: /.*?/
                    _STRING_ESC_INNER: _STRING_INNER /(?<!\\\\)(\\\\\\\\)*?/
                    ESCAPED_STRING : "\\"" _STRING_ESC_INNER "\\""
                    LCASE_LETTER: "a".."z"
                    UCASE_LETTER: "A".."Z"
                    LETTER: UCASE_LETTER | LCASE_LETTER
                    WORD: LETTER+
                    CNAME: ("_"|LETTER) ("_"|LETTER|DIGIT)*
                    WS_INLINE: (" "|/\\t/)+
                    WS: /[ \\t\\f\\r\\n]/+
                    CR : /\\r/
                    LF : /\\n/
                    NEWLINE: (CR? LF)+

                    %ignore WS // Disregard spaces in text
                 """
        )

        start = l.parse(query)
        return cls.get_group(start)

    @classmethod
    def get_group(cls, group):
        lfunction = None
        for child in group.children:
            if child.data == "query":
                new_results = cls.get_query(child)
                if not lfunction:
                    results = new_results
                elif lfunction == "or":
                    results.extend(new_results)
                elif lfunction == "and":
                    results = list(set(results).intersection(new_results))
                results = list(set(results))
            elif child.data == "lfunction":
                lfunction = child.children[0].data
        return results

    @classmethod
    def get_query(cls, query):
        for child in query.children:
            if child.data == "selector":
                return cls.get_selector(child)
            elif child.data == "group":
                return cls.get_group(child)

    @classmethod
    def get_selector(cls, selector):
        if len(selector.children) == 1:
            inverse_relationship = None
            class_or_guid_selector = selector.children[0]
        else:
            inverse_relationship = selector.children[0]
            class_or_guid_selector = selector.children[1]

        if class_or_guid_selector.data == "class_selector":
            results = cls.get_class_selector(class_or_guid_selector)
        elif class_or_guid_selector.data == "guid_selector":
            results = cls.get_guid_selector(class_or_guid_selector)

        if not inverse_relationship:
            return results
        return cls.parse_inverse_relationship(results, inverse_relationship.children[0].data)

    @classmethod
    def parse_inverse_relationship(cls, elements, inverse_relationship):
        results = []
        for element in elements:
            if inverse_relationship == "types":
                if hasattr(element, "Types") and element.Types:
                    results.extend(element.Types[0].RelatedObjects)
                elif hasattr(element, "ObjectTypeOf") and element.ObjectTypeOf:
                    results.extend(element.ObjectTypeOf[0].RelatedObjects)
            elif inverse_relationship == "decomposed_by":
                results.extend(ifcopenshell.util.element.get_decomposition(element))
            elif inverse_relationship == "grouped_by":
                results.extend(ifcopenshell.util.element.get_grouped_by(element))
            elif inverse_relationship == "bounded_by" and hasattr(element, "BoundedBy"):
                for relationship in element.BoundedBy:
                    results.append(relationship.RelatedBuildingElement)
        return results

    @classmethod
    def get_class_selector(cls, class_selector):
        if class_selector.children[0] == "COBie":
            elements = ifcopenshell.util.fm.get_cobie_components(cls.file)
        elif class_selector.children[0] == "COBieType":
            elements = ifcopenshell.util.fm.get_cobie_types(cls.file)
        elif class_selector.children[0] == "FMHEM":
            elements = ifcopenshell.util.fm.get_fmhem_types(cls.file)
        else:
            if cls.elements is None:
                elements = cls.file.by_type(class_selector.children[0])
            else:
                elements = [e for e in cls.elements if e.is_a(class_selector.children[0])]
        if len(class_selector.children) > 1 and class_selector.children[1].data == "filter":
            return cls.filter_elements(elements, class_selector.children[1])
        return elements

    @classmethod
    def filter_elements(cls, elements, filter_rule):
        results = []
        filter_query = cls.parse_filter_query(filter_rule.children[0].children[0])
        comparison = value = None
        if len(filter_rule.children) > 1:
            comparison = filter_rule.children[1].children[0].data
            if comparison == "not":
                comparison += filter_rule.children[1].children[1].data
            filter_value = filter_rule.children[2].children[0]
            if isinstance(filter_value, lark.Tree):
                is_regex = True
                token_type = filter_value.data
            else:
                is_regex = False
                token_type = filter_value.type
            if token_type == "filter_regex":
                value = str(filter_value.children[0][1:-1])
            elif token_type == "ESCAPED_STRING":
                value = str(filter_value[1:-1])
            elif token_type == "SIGNED_INT":
                value = int(filter_value)
            elif token_type == "SIGNED_FLOAT":
                value = float(filter_value)
            elif token_type == "BOOLEAN":
                value = filter_value.lower() == "true"
            elif token_type == "NULL":
                value = None
        for element in elements:
            element_value = cls.get_element_value(element, filter_query["keys"], is_regex=filter_query["is_regex"])
            if element_value is None and value is not None and "not" not in comparison:
                continue
            if comparison and cls.filter_element(element, element_value, comparison, value, is_regex=is_regex):
                results.append(element)
            elif not comparison and element_value:
                results.append(element)
        return results

    @classmethod
    def parse_filter_query(cls, filter_query):
        keys = filter_query
        is_regex = False
        if isinstance(keys, str):
            keys = [keys]
        elif keys.data == "keys_regex":
            is_regex = True
            keys = [k[1:-1].replace('\\"', '"') for k in keys.children]
        elif keys.data == "keys_quoted":
            keys = [k[1:-1].replace('\\"', '"') for k in keys.children]
        elif keys.data == "keys_simple":
            keys = keys.children
        return {"keys": keys, "is_regex": is_regex}

    @classmethod
    def get_element_value(cls, element, keys, is_regex=False):
        value = element
        for key in keys:
            key = key.strip()
            if value is None:
                return
            if key == "type":
                value = ifcopenshell.util.element.get_type(value)
            elif key in ("material", "mat"):
                value = ifcopenshell.util.element.get_material(value, should_skip_usage=True)
            elif key in ("item", "i"):
                if value.is_a("IfcMaterialLayerSet"):
                    value = value.MaterialLayers
                elif value.is_a("IfcMaterialProfileSet"):
                    value = value.MaterialProfiles
                elif value.is_a("IfcMaterialConstituentSet"):
                    value = value.MaterialConstituents
            elif key == "container":
                value = ifcopenshell.util.element.get_container(value)
            elif key == "class":
                value = value.is_a()
            elif key == "id":
                value = value.id()
            elif isinstance(value, ifcopenshell.entity_instance):
                if key == "Name" and value.is_a("IfcMaterialLayerSet"):
                    key = "LayerSetName"  # This oddity in the IFC spec is annoying so we account for it.

                attribute = getattr(value, key, None)

                if attribute is not None:
                    value = attribute
                else:
                    # Try to extract pset
                    if is_regex:
                        psets = ifcopenshell.util.element.get_psets(value)
                        matching_psets = []
                        for pset_name, pset in psets.items():
                            if re.match(key, pset_name):
                                matching_psets.append(pset)
                        result = matching_psets or None
                    else:
                        result = ifcopenshell.util.element.get_pset(value, key)

                    value = result
            elif isinstance(value, dict):  # Such as from the result of a prior get_pset
                if is_regex:
                    results = []
                    for prop_name, prop_value in value.items():
                        if re.match(key, prop_name):
                            if isinstance(prop_value, (list, tuple)):
                                results.extend(prop_value)
                            else:
                                results.append(prop_value)
                    value = results
                else:
                    value = value.get(key, None)
            elif isinstance(value, (list, tuple)):  # If we use regex
                if key.isnumeric():
                    try:
                        value = value[int(key)]
                    except IndexError:
                        return
                else:
                    results = []
                    for v in value:
                        subvalue = cls.get_element_value(v, [key], is_regex=is_regex)
                        if isinstance(subvalue, list):
                            results.extend(subvalue)
                        else:
                            results.append(subvalue)
                    value = results
        return value

    @classmethod
    def filter_element(cls, element, element_value, comparison, value, is_regex=False):
        if comparison.startswith("not"):
            return not cls.filter_element(element, element_value, comparison[3:], value, is_regex=is_regex)
        elif comparison == "equal" and isinstance(element_value, list):
            if is_regex:
                for element_v in element_value:
                    if re.match(value, element_v):
                        return True
                return False
            return value in element_value
        elif comparison == "equal":
            if is_regex:
                return bool(re.match(value, element_value))
            return element_value == value
        elif comparison == "contains" and isinstance(element_value, list):
            return bool([ev for ev in element_value if value in str(ev)])
        elif comparison == "contains":
            return value in str(element_value)
        elif comparison == "morethan":
            return element_value > value
        elif comparison == "lessthan":
            return element_value < value
        elif comparison == "morethanequalto":
            return element_value >= value
        elif comparison == "lessthanequalto":
            return element_value <= value
        elif comparison == "oneof":
            return element_value in value.split(",")
        return False

    @classmethod
    def get_guid_selector(cls, guid_selector):
        return [cls.file.by_id(guid_selector.children[0])]


import os
import zipfile
import tempfile
from io import BytesIO
import requests
import shutil
import subprocess
import errno
import yaml
import click
import signal
import multiprocessing
from time import sleep
from uuid import uuid4
from .scheduler import JessScheduler
from .scheduler import LocalScheduler
from .worker import Worker

class GracefulKiller:
    def __init__(self):
        self.kill_now = False
        signal.signal(signal.SIGINT, self.exit_gracefully)
        signal.signal(signal.SIGTERM, self.exit_gracefully)

    def exit_gracefully(self, signum, frame):
        click.echo('Interruption signal received, will exit when the current running job finishes.')
        self.kill_now = True

def work(worker):
    proc_name = multiprocessing.current_process().name
    try:
        rv = worker.run()
        click.echo('Finish task: {} by worker: {}'.format(proc_name, worker.id))
    except:
        # for server mode:
        # if exception happened, worker might not have reported failure to the server
        # need to check the server and report as needed
        pass

class Executor(object):
    def __init__(self, jt_home=None, jt_account=None,
                 ams_server=None, wrs_server=None, jess_server=None,
                 executor_id=None,
                 queue_id=None,
                 workflow_name=None,
                 job_file=None,  # when job_file is provided, it's local mode, no tracking from the server side
                 job_id=None,  # can optionally specify which job to run, not applicable when job_file specified
                 parallel_jobs=1, parallel_workers=1, sleep_interval=5, max_jobs=0, continuous_run=False):

        self._killer = GracefulKiller()

        # TODO: will need to verify jt_account

        # allow user specify executor_id for resuming,
        # TODO: has to check to make sure there is no such executor currently running
        #       need some server side work to support this
        self._id = executor_id if executor_id else str(uuid4())

        self._jt_home = jt_home

        self._parallel_jobs = parallel_jobs
        self._max_jobs = max_jobs
        self._parallel_workers = parallel_workers
        self._sleep_interval = sleep_interval
        self._ran_jobs = 0
        self._continuous_run = continuous_run

        # params for server mode
        if queue_id and job_file is None:
            self._scheduler = JessScheduler(jess_server=jess_server,
                                            wrs_server=wrs_server,
                                            ams_server=ams_server,
                                            jt_account=jt_account,
                                            queue_id=queue_id,
                                            job_id=job_id,  # optionally specify which job to run
                                            executor_id=self.id)

        # local mode if supplied
        elif job_file and queue_id is None:
            self._scheduler = LocalScheduler(job_file=job_file,
                                             workflow_name=workflow_name,
                                             executor_id=self.id)

        else:
            raise Exception('Please specify either queue_id for executing jobs on remote job queue or '
                            'job_file to run local job.')

        self._running_jobs = []
        self._worker_processes = {}

        # init jt_home dir
        self._init_jt_home()

        # init workflow dir
        self._init_workflow_dir()

        # init queue dir
        self._init_queue_dir()

        # init executor dir
        self._init_executor_dir()

        # TODO: check whether previous executor session exists, restore it unless user chose not to (via options)

        click.echo("Executor: %s started." % self.id)

    @property
    def killer(self):
        return self._killer

    @property
    def id(self):
        return self._id

    @property
    def scheduler(self):
        return self._scheduler

    @property
    def jt_home(self):
        return self._jt_home

    @property
    def jt_account(self):
        return self._jt_account

    @property
    def account_id(self):
        return self.scheduler.account_id

    @property
    def node_id(self):
        return self._node_id

    @property
    def node_dir(self):
        return os.path.join(self.jt_home, 'account.%s' % self.account_id, 'node')

    @property
    def workflow_dir(self):
        return os.path.join(self.node_dir,
                            'workflow.%s' % self.scheduler.workflow_id,
                            self.scheduler.workflow_version)

    @property
    def queue_dir(self):
        return os.path.join(self.workflow_dir, 'queue.%s' % self.scheduler.queue_id)

    @property
    def executor_dir(self):
        return os.path.join(self.queue_dir, 'executor.%s' % self.scheduler.executor_id)

    @property
    def sleep_interval(self):
        return self._sleep_interval

    @property
    def parallel_jobs(self):
        return self._parallel_jobs

    @property
    def max_jobs(self):
        return self._max_jobs

    @property
    def parallel_workers(self):
        return self._parallel_workers

    @property
    def running_jobs(self):
        return self._running_jobs

    @property
    def ran_jobs(self):
        return self._ran_jobs

    @property
    def continuous_run(self):
        return self._continuous_run

    @property
    def worker_processes(self):
        return self._worker_processes

    def run(self):
        if self.scheduler.mode == 'local':
            self._run_local()
        else:
            self._run_remote()

    def _run_local(self):
        # TODO: local run is more complicated, let's worry about it later
        click.echo('Run local job not implemented yet.')

    def _run_remote(self):
        # let's register the executor to the queue first
        # TODO: if we are going to support 'resume' an executor, we will need to check JESS for existing executor
        #       and test whether it's resumeable
        while True:
            if self.killer.kill_now:
                click.echo('Received interruption signal, will not pick up new job. Exit after finishing current '
                           'running job (if any) ...')
                break

            if self.max_jobs and self.ran_jobs >= self.max_jobs:
                job_count_text = 'job has' if self.max_jobs == 1 else 'jobs have'
                click.echo('Total number of executed %s reached preset limit of %s, relevant tasks are either '
                           'finished or scheduled, executor will exit after running tasks finish ...' %
                           (job_count_text, self.max_jobs)
                           )

                click.echo("Current running jobs: %s, running tasks: %s" % self._get_run_status())
                break

            if self.scheduler.running_jobs() and len(self.scheduler.running_jobs()) >= self.parallel_jobs:
                # basically we check again to see if running jobs drop to under the parallel limit
                # detail: we need to worry about possible run-away job, job appears to be running but worker died
                #         already, is that possible if executor is still alive? Can executor report the state of a
                #         task ran by a worker whose process exited with error?
                click.echo('Reached limit for parallel running jobs, will start new job after completing a current job.')
                click.echo("Current running jobs: %s, running tasks: %s" % self._get_run_status())

                sleep(self.sleep_interval)
                continue

            worker = Worker(jt_home=self.jt_home, account_id=self.account_id,
                            scheduler=self.scheduler, node_id=self.node_id)

            # get a task from a new job, break if no task returned, which suggests there is no more job
            if not worker.next_task(job_state='queued'):
                if self.continuous_run:
                    click.echo('No job in the queue, will start new job as it arrives.')
                    click.echo("Current running jobs: %s, running tasks: %s" % self._get_run_status())
                    sleep(self.sleep_interval)  # TODO: may want to have a smarter wait intervals
                    continue
                else:
                    click.echo('No job in the queue. Exit after finishing current running job (if any) ...')
                    click.echo("Current running jobs: %s, running tasks: %s" % self._get_run_status())
                    break

            # start the task
            p = multiprocessing.Process(target=work,
                                        name='task:%s job:%s' % (worker.task.get('name'), worker.task.get('job.id')),
                                        args=(worker,)
                                        )

            self._worker_processes[worker.task.get('job.id')] = [p]
            p.start()

            # this is the first task of a new job
            self._ran_jobs += 1
            click.echo('Executor: %s starts no. %s job' % (self.id, self.ran_jobs))

            shutdown = False
            # stay in this loop when there are tasks to be run related to current running jobs
            while self.scheduler.has_next_task():
                if self.killer.kill_now:
                    click.echo(
                        'Received interruption signal, will not pick up new task. Exit when current running task(s) '
                        'finishes...')
                    shutdown = True
                    break

                running_jobs, running_workers = self._get_run_status()

                click.echo('Current running jobs: %s, running tasks: %s' % (running_jobs, running_workers))

                if running_workers < self.parallel_workers:
                    worker = Worker(jt_home=self.jt_home, account_id=self.account_id,
                                    scheduler=self.scheduler, node_id=self.node_id)
                    try:
                        task = worker.next_task(job_state='running')  # get next task in the current running jobs
                    except:
                        pass  # for whatever reason next_task call failed, TODO: add count for failed attempts, fail the job when next_task failed too many times
                    if not task:  # else try to start task for next job if it's appropriate to do so
                        if not (self.max_jobs and self.ran_jobs >= self.max_jobs) and \
                                        not len(self.scheduler.running_jobs()) >= self.parallel_jobs:
                            try:
                                task = worker.next_task(job_state='queued')
                                if task:
                                    self._ran_jobs += 1
                                    click.echo('Executor: %s starts no. %s job' % (self.id, self.ran_jobs))
                            except:
                                pass  # no need to do anything, except maybe counting of failures
                    if task:
                        p = multiprocessing.Process(target=work,
                                                name='task:%s job:%s' % (worker.task.get('name'), worker.task.get('job.id')),
                                                args=(worker,)
                                                )
                        if not self.worker_processes.get(worker.task.get('job.id')):
                            self._worker_processes[worker.task.get('job.id')] = [p]
                        else:
                            self._worker_processes[worker.task.get('job.id')].append(p)
                        p.start()

                sleep(self.sleep_interval)

            if shutdown:
                break

        for j in self.running_jobs:
            if not self.worker_processes: break
            for p in self.worker_processes.get(j):
                if p.is_alive():
                    p.join()

        # call server to mark this executor terminated
        if self.killer.kill_now:
            for j in self.scheduler.running_jobs():
                print('Cancelling job: %s' % j.get('id'))
                self.scheduler.cancel_job(job_id=j.get('id'))

        # report summary about completed jobs and running jobs if any
        click.echo('Executed %s %s.' % (self.ran_jobs, 'job' if self.ran_jobs <= 1 else 'jobs'))

    def _get_run_status(self):
        running_workers = 0
        running_jobs = 0
        for j in self.scheduler.running_jobs():
            click.echo('Running job: %s' % j.get('id'))
            running_jobs += 1
            for p in self.worker_processes.get(j.get('id')):
                if p.is_alive():
                    click.echo('Running task: %s' % p.name)
                    running_workers += 1
                    p.join(timeout=0.1)
        return running_jobs, running_workers

    def _init_jt_home(self):
        # initial it if needed
        node_info_file = os.path.join(self.node_dir,'info.yaml')

        if os.path.isfile(node_info_file):
            with open(node_info_file, 'r') as f:
                node_info = yaml.load(f)
        else:  # not exist
            node_info = {'id': str(uuid4())}  # may need to add other information
            try:
                os.makedirs(self.node_dir)
            except OSError as exc:  # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise
            with open(node_info_file, "w") as f:
                f.write(yaml.dump(node_info, default_flow_style=False))

        self._node_id = node_info.get('id')

    def _init_workflow_dir(self):
        try:
            os.makedirs(self.workflow_dir)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise

        # detect whether workflow has already been installed
        workflow_installation_flag_file = os.path.join(self.workflow_dir, 'workflow.installed')
        if os.path.isfile(workflow_installation_flag_file):
            return

        click.echo('Installing workflow package ...')
        workflow = self.scheduler.get_workflow()

        git_account = workflow.get('git_account')
        git_repo = workflow.get('git_repo')
        git_tag = workflow.get('ver:%s' % self.scheduler.workflow_version).get('git_tag')
        git_path = workflow.get('ver:%s' % self.scheduler.workflow_version).get('git_path')

        # https://github.com/jthub/jtracker-example-workflows/archive/0.2.0.tar.gz
        git_download_url = "https://github.com/%s/%s/archive/%s.zip" % (git_account, git_repo, git_tag)

        tmp_dir = tempfile.mkdtemp()
        request = requests.get(git_download_url)
        zfile = zipfile.ZipFile(BytesIO(request.content))
        zfile.extractall(tmp_dir)

        source_workflow_path = os.path.join(tmp_dir, '%s-%s' % (git_repo, git_tag), git_path, 'workflow')
        source_tool_path = os.path.join(source_workflow_path, 'tools')
        if os.path.isdir(source_tool_path):
            subprocess.check_output(["chmod", "-R", "755", source_tool_path])

        # rm first in case exist
        shutil.rmtree(os.path.join(self.workflow_dir, 'workflow'), ignore_errors=True)

        shutil.move(source_workflow_path, self.workflow_dir)

        # now create the installation flag file
        open(workflow_installation_flag_file, 'a').close()
        click.echo('Workflow package installed')

    def _init_queue_dir(self):
        try:
            os.makedirs(self.queue_dir)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise

    def _init_executor_dir(self):
        try:
            os.makedirs(self.executor_dir)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise