from functools import partial
from inspect import getargspec

from django.template.base import parse_bits, StringOrigin, \
    TemplateEncodingError, Parser, Lexer, TagHelperNode, \
    Template as BaseTemplate
from django.template.base import Library as BaseTemplateLibrary
from django.template.context import (Context)

from django.utils.encoding import force_text
from django.utils.itercompat import is_iterable
from django.utils import six
from django.conf import settings

def compile_string(template_string, origin, output_context=None):
    "Compiles template_string into NodeList ready for rendering"
    if settings.TEMPLATE_DEBUG:
        from django.template.debug import DebugLexer, DebugParser

        lexer_class, parser_class = DebugLexer, DebugParser
    else:
        lexer_class, parser_class = Lexer, Parser
    lexer = lexer_class(template_string, origin)
    parser = parser_class(lexer.tokenize())
    parser.output_context = output_context
    return parser.parse()

def generic_tag_compiler(parser, token, params, varargs, varkw, defaults,
                         name, takes_context, node_class):
    """
    Returns a template.Node subclass.
    """
    bits = token.split_contents()[1:]
    output_context = parser.output_context if parser.output_context is not None else {}
    args, kwargs = parse_bits(parser, bits, params, varargs, varkw,
                              defaults, takes_context, name)
    return node_class(takes_context, output_context, args, kwargs)

class Template(BaseTemplate):
    def __init__(self, template_string, origin=None,
                 name='<Unknown Template>', output_context=None):
        try:
            template_string = force_text(template_string)
        except UnicodeDecodeError:
            raise TemplateEncodingError("Templates can only be constructed "
                                        "from unicode or UTF-8 strings.")
        if settings.TEMPLATE_DEBUG and origin is None:
            origin = StringOrigin(template_string)
        self.nodelist = compile_string(template_string, origin, output_context)
        self.name = name
        self.origin = origin  # for debug toolbar

# The only thing changed compared to django.templates.base.Library.inclusion_tag are the get_template and select_template
class Library(BaseTemplateLibrary):
    def inclusion_tag(self, file_name, context_class=Context,
                      takes_context=False, name=None):
        def dec(func):
            params, varargs, varkw, defaults = getargspec(func)

            class InclusionNode(TagHelperNode):
                def __init__(self, takes_context, output_context, args, kwargs):
                    self.takes_context = takes_context
                    self.args = args
                    self.kwargs = kwargs
                    self.output_context = output_context

                def render(self, context):
                    resolved_args, resolved_kwargs = self.get_resolved_arguments(
                        context)
                    _dict = func(*resolved_args, **resolved_kwargs)

                    if not getattr(self, 'nodelist', False):
                        if isinstance(file_name, Template):
                            t = file_name
                        elif not isinstance(file_name,
                                            six.string_types) and is_iterable(
                                file_name):
                            t = select_template(file_name, self.output_context)
                        else:
                            t = get_template(file_name, self.output_context)
                        self.nodelist = t.nodelist
                    new_context = context_class(_dict, **{
                        'autoescape': context.autoescape,
                        'current_app': context.current_app,
                        'use_l10n': context.use_l10n,
                        'use_tz': context.use_tz,
                    })
                    # Copy across the CSRF token, if present, because
                    # inclusion tags are often used for forms, and we need
                    # instructions for using CSRF protection to be as simple
                    # as possible.
                    csrf_token = context.get('csrf_token', None)
                    if csrf_token is not None:
                        new_context['csrf_token'] = csrf_token
                    return self.nodelist.render(new_context)

            function_name = (name or
                             getattr(func, '_decorated_function',
                                     func).__name__)
            compile_func = partial(generic_tag_compiler,
                                   params=params, varargs=varargs, varkw=varkw,
                                   defaults=defaults, name=function_name,
                                   takes_context=takes_context,
                                   node_class=InclusionNode)
            compile_func.__doc__ = func.__doc__
            self.tag(function_name, compile_func)
            return func

        return dec

from outputcontext.template.loader import get_template, select_template


"""
Provides XML rendering support.
"""
from dict2xml import dict2xml
from lxml import etree
from rest_framework.renderers import BaseRenderer

class XMLRenderer(BaseRenderer):
    """
    Renderer which serializes to XML.
    """

    media_type = "application/xml"
    format = "xml"
    charset = "utf-8"
    item_tag_name = "list-item"

    def render(self, data, accepted_media_type=None, renderer_context=None):
        """
        Renders `data` into serialized XML.
        """
        if data is None:
            return ""

        if isinstance(data, dict):
            return dict2xml(data)
        else:
            return etree.tostring(data, pretty_print=True)


from polygraphy import constants, mod, util
from polygraphy.common import TensorMetadata
from polygraphy.logger import G_LOGGER, LogMode
from polygraphy.tools.script import Script, ensure_safe, inline, safe

np = mod.lazy_import("numpy")

@mod.export()
def cast(val):
    """
    Cast a value from a string to one of:
    [int, float, str, List[int], List[float], List[str]]

    Args:
        val (str): The value to cast.

    Returns:
        object: The casted value.
    """
    val = str(val.strip())

    if val.strip("[]") != val:
        return [cast(elem) for elem in val.strip("[]").split(",")]

    try:
        return int(val)  # This fails for float strings like '0.0'
    except:
        pass

    try:
        return float(val)  # This fails for non-numerical strings like 'isildur'
    except:
        pass
    return val.strip("\"'")

@mod.export()
def run_script(script_func, *args):
    """
    Populates a script using the provided callable, then returns
    the variable indicated by the return value of the callable.

    Args:
        script_func (Callable(Script, *args) -> str):
                A callable that populates a Script and then returns
                the name of an object defined within the script to retrieve.
        args:
                Additional positional argruments to pass to script_func.
                The script_func must accept these by variable name instead
                of taking the values themselves. Values of ``None`` will be
                passed directly instead of by variable name.

    Returns:
        object:
                An object defined within the script, or ``None`` if it is not
                defined by the script.
    """
    script = Script()

    arg_names = []
    for index, arg in enumerate(args):
        if arg is not None:
            arg_name = safe("__arg{:}", index)
            locals()[arg_name.unwrap()] = arg
            arg_names.append(inline(arg_name))
        else:
            arg_names.append(None)

    safe_ret_name = script_func(script, *arg_names)
    exec(str(script), globals(), locals())

    if safe_ret_name is not None:
        ret_name = ensure_safe(safe_ret_name).unwrap()
        if ret_name in locals():
            return locals()[ret_name]
    return None

@mod.export()
def get(args, attr, default=None):
    """
    Gets a command-line argument if it exists, otherwise returns a default value.

    Args:
        args: The command-line arguments.
        attr (str): The name of the command-line argument.
        default (obj): The default value to return if the argument is not found. Defaults to None.
    """
    if hasattr(args, attr):
        return getattr(args, attr)
    return default

@mod.export()
def get_outputs(args, name):
    outputs = get(args, name)
    if outputs is not None and len(outputs) == 2 and outputs == ["mark", "all"]:
        outputs = constants.MARK_ALL
    return outputs

@mod.export()
def get_outputs_for_script(script, outputs):
    if outputs == constants.MARK_ALL:
        script.add_import(["constants"], frm="polygraphy")
        outputs = inline(safe("constants.MARK_ALL"))
    return outputs

def np_types():
    """
    Returns a list of human-readable names of NumPy data types.
    """
    return sorted(set(np.dtype(dtype).name for dtype in np.sctypeDict.values()))

def np_type_from_str(dt_str):
    """
    Converts a string representation of a data type to a NumPy data type.

    Args:
        dt_str (str): The string representation of the data type.

    Returns:
        np.dtype: The NumPy data type.

    Raises:
        KeyError: If the provided string does not correspond to a NumPy data type.
    """
    try:
        return {np.dtype(dtype).name: np.dtype(dtype) for dtype in np.sctypeDict.values()}[dt_str]
    except KeyError:
        G_LOGGER.error(
            f"Could not understand data type: {dt_str}. Did you forget to specify a data type? Please use one of: {np_types()} or `auto`."
        )
        raise

@mod.export()
def parse_dict_with_default(arg_lst, cast_to=None, sep=None, allow_empty_key=None):
    """
    Generate a dictionary from a list of arguments of the form:
    ``<key>:<val>``. If ``<key>`` is empty, the value will be assigned
    to an empty string key in the returned mapping.

    Args:
        arg_lst (List[str]):
                The arguments to map.

        cast_to (Callable):
                A callable to cast types before adding them to the map.
                Defaults to `cast()`.
        sep (str):
                The separator between the key and value strings.
                Defaults to ":".
        allow_empty_key (bool):
                Whether empty keys should be allowed.
                Defaults to True.
    Returns:
        Dict[str, obj]: The mapping.
    """
    sep = util.default(sep, ":")
    cast_to = util.default(cast_to, cast)
    allow_empty_key = util.default(allow_empty_key, True)

    if arg_lst is None:
        return

    arg_map = {}
    for arg in arg_lst:
        key, _, val = arg.rpartition(sep)
        if not key and not allow_empty_key:
            G_LOGGER.critical(
                f"Could not parse argument: {arg}. Expected an argument in the format: `key{sep}value`.\n"
            )
        arg_map[key] = cast_to(val)
    return arg_map

@mod.deprecate(
    remove_in="0.45.0",
    use_instead=": as a separator and write shapes in the form [dim0,...,dimN]",
    name="Using , as a separator",
)
def parse_meta_legacy(meta_args, includes_shape=True, includes_dtype=True):
    """
    Parses a list of tensor metadata arguments of the form "<name>,<shape>,<dtype>"
    `shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled.

    Args:
        meta_args (List[str]): A list of tensor metadata arguments from the command-line.
        includes_shape (bool): Whether the arguments include shape information.
        includes_dtype (bool): Whether the arguments include dtype information.

    Returns:
        TensorMetadata: The parsed tensor metadata.
    """
    SEP = ","
    SHAPE_SEP = "x"
    meta = TensorMetadata()
    for orig_tensor_meta_arg in meta_args:
        tensor_meta_arg = orig_tensor_meta_arg

        def pop_meta(name):
            nonlocal tensor_meta_arg
            tensor_meta_arg, _, val = tensor_meta_arg.rpartition(SEP)
            if not tensor_meta_arg:
                G_LOGGER.critical(
                    f"Could not parse {name} from argument: {orig_tensor_meta_arg}. Is it separated by a comma (,) from the tensor name?"
                )
            if val.lower() == "auto":
                val = None
            return val

        def parse_dtype(dtype):
            if dtype is not None:
                dtype = np_type_from_str(dtype)
            return dtype

        def parse_shape(shape):
            if shape is not None:

                def parse_shape_dim(buf):
                    try:
                        buf = int(buf)
                    except:
                        pass
                    return buf

                parsed_shape = []
                # Allow for quoted strings in shape dimensions
                in_quotes = False
                buf = ""
                for char in shape.lower():
                    if char in ['"', "'"]:
                        in_quotes = not in_quotes
                    elif not in_quotes and char == SHAPE_SEP:
                        parsed_shape.append(parse_shape_dim(buf))
                        buf = ""
                    else:
                        buf += char
                # For the last dimension
                if buf:
                    parsed_shape.append(parse_shape_dim(buf))
                shape = tuple(parsed_shape)
            return shape

        name = None
        dtype = None
        shape = None

        if includes_dtype:
            dtype = parse_dtype(pop_meta("data type"))

        if includes_shape:
            shape = parse_shape(pop_meta("shape"))

        name = tensor_meta_arg
        meta.add(name, dtype, shape)

    new_style = []
    for m_arg in meta_args:
        arg = m_arg
        if includes_shape:
            arg = arg.replace(",", ":[", 1)
            if includes_dtype:
                arg = arg.replace(",", "]:", 1)
            else:
                arg += "]"

        arg = arg.replace(",auto", ":auto")
        arg = arg.replace(",", ":")

        if includes_shape:
            arg = arg.replace("x", ",")

        new_style.append(arg)

    G_LOGGER.warning(
        "The old shape syntax is deprecated and will be removed in Polygraphy 0.45.0\n"
        "See the CHANGELOG entry for v0.32.0 for the motivation behind this deprecation.",
        mode=LogMode.ONCE,
    )
    G_LOGGER.warning(f"Instead of: '{' '.join(meta_args)}', use: '{' '.join(new_style)}'\n")
    return meta

def parse_meta_new_impl(meta_args, includes_shape=True, includes_dtype=True):
    SEP = ":"
    meta = TensorMetadata()
    for meta_arg in meta_args:
        name, shape, dtype = None, None, None

        def pop_meta(func):
            nonlocal meta_arg
            meta_arg, _, val = meta_arg.rpartition(SEP)
            val = cast(val.strip())
            if isinstance(val, str) and val.lower() == "auto":
                return None
            return func(val)

        if includes_dtype:
            dtype = pop_meta(func=np_type_from_str)

        if includes_shape:
            shape = pop_meta(func=lambda s: tuple(e for e in s if e != ""))

        name = meta_arg

        meta.add(name, dtype=dtype, shape=shape)
    return meta

@mod.export()
def parse_meta(meta_args, includes_shape=True, includes_dtype=True):
    """
    Parses a list of tensor metadata arguments of the form "<name>:<shape>:<dtype>"
    `shape` and `dtype` are optional, but `dtype` must always come after `shape` if they are both enabled.

    Args:
        meta_args (List[str]): A list of tensor metadata arguments from the command-line.
        includes_shape (bool): Whether the arguments include shape information.
        includes_dtype (bool): Whether the arguments include dtype information.

    Returns:
        TensorMetadata: The parsed tensor metadata.
    """
    if all((includes_shape and "[" in arg) or (includes_dtype and "," not in arg) for arg in meta_args):
        return parse_meta_new_impl(meta_args, includes_shape, includes_dtype)
    return parse_meta_legacy(meta_args, includes_shape, includes_dtype)

@mod.export()
def parse_num_bytes(num_bytes_arg):
    """
    Parses an argument that indicates a number of bytes. The argument may use scientific notation,
    or contain a `K`, `M`, or `G` suffix (case-insensitive), indicating `KiB`, `MiB`, or `GiB` respectively.
    If the number is fractional, it will be truncated to the nearest integer value.

    If the provided argument is `None`, `None` is returned.

    Args:
        num_bytes_arg (str): The argument indicating the number of bytes.

    Returns:
        int: The number of bytes.
    """
    if num_bytes_arg is None:
        return None

    num_component = num_bytes_arg  # Numerical component of the argument
    multiplier = 1

    suffix_mulitplier = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30}
    for suffix, mult in suffix_mulitplier.items():
        if num_bytes_arg.upper().endswith(suffix):
            num_component = num_bytes_arg.upper().rstrip(suffix)
            multiplier = mult
            break

    try:
        return int(float(num_component) * multiplier)
    except:
        G_LOGGER.critical(
            f"Could not convert {num_bytes_arg} to a number of bytes. "
            "Please use either an integer (e.g. 16000000), scientific notation (e.g. 16e6), "
            "or a number with a valid suffix: K, M, or G (e.g. 16M)."
        )


import logging

import numpy as np
import torch
from torch import nn

from anchor_based import anchor_helper
from anchor_free.Uformer import UTransformerAB
from anchor_based.losses import calc_cls_loss, calc_loc_loss
from evaluate import evaluate
from helpers import data_helper, vsumm_helper, bbox_helper

logger = logging.getLogger()

def xavier_init(module):
    cls_name = module.__class__.__name__
    if 'Linear' in cls_name or 'Conv' in cls_name:
        nn.init.xavier_uniform_(module.weight, gain=np.sqrt(2.0))
        if module.bias is not None:
            nn.init.constant_(module.bias, 0.1)

def train(args, split, save_path):
    model = UTransformerAB(anchor_scales=args.anchor_scales,
                         dim_in=args.num_feature,
                         dim_out=args.num_feature,
                         heads=8,
                         mlp_dim=16,
                         dropout_rate=0.3,
                         attn_dropout_rate=0.3)
    model = model.to(args.device)
    import time
    dummy_input = torch.randn((1, 100, 1024)).cuda()
    print('model　total: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0))
    model.eval()
    t1 = time.time()
    for i in range(10000):
        torch.cuda.empty_cache()
        out=model(dummy_input,dummy_input)#[2, 100, 8,1];[2,400,100];[2,100]
        # out=model(dummy_input)#[2, 100, 8,1];[2,400,100];[2,100]
        # print(out)
    t2 = time.time()
    t = (t2-t1)/10000
    print('average time:',t)

    model.apply(xavier_init)
    print('args.weight_decay:',args.weight_decay)
    parameters = [p for p in model.parameters() if p.requires_grad]
    optimizer = torch.optim.Adam(parameters, lr=args.lr,
                                 weight_decay=args.weight_decay)

    max_val_fscore = -1

    train_set = data_helper.VideoDataset(split['train_keys'])
    train_loader = data_helper.DataLoader(train_set, shuffle=True)

    val_set = data_helper.VideoDataset(split['test_keys'])
    val_loader = data_helper.DataLoader(val_set, shuffle=False)

    for epoch in range(args.max_epoch):
        model.train()
        stats = data_helper.AverageMeter('loss', 'cls_loss', 'loc_loss')

        for _, seq, seqdiff,gtscore, cps, n_frames, nfps, picks, _ in train_loader:
            keyshot_summ = vsumm_helper.get_keyshot_summ(
                gtscore, cps, n_frames, nfps, picks)
            target = vsumm_helper.downsample_summ(keyshot_summ)

            if not target.any():
                continue

            # if target_size<target.size:
            #     target_size = target.size
            # print('target.size:',target_size)
            target_bboxes = bbox_helper.seq2bbox(target)
            target_bboxes = bbox_helper.lr2cw(target_bboxes)
            anchors = anchor_helper.get_anchors(target.size, args.anchor_scales)
            # Get class and location label for positive samples
            cls_label, loc_label = anchor_helper.get_pos_label(
                anchors, target_bboxes, args.pos_iou_thresh)

            # Get negative samples
            num_pos = cls_label.sum()
            cls_label_neg, _ = anchor_helper.get_pos_label(
                anchors, target_bboxes, args.neg_iou_thresh)
            cls_label_neg = anchor_helper.get_neg_label(
                cls_label_neg, int(args.neg_sample_ratio * num_pos))

            # Get incomplete samples
            cls_label_incomplete, _ = anchor_helper.get_pos_label(
                anchors, target_bboxes, args.incomplete_iou_thresh)
            cls_label_incomplete[cls_label_neg != 1] = 1
            cls_label_incomplete = anchor_helper.get_neg_label(
                cls_label_incomplete,
                int(args.incomplete_sample_ratio * num_pos))

            cls_label[cls_label_neg == -1] = -1
            cls_label[cls_label_incomplete == -1] = -1

            cls_label = torch.tensor(cls_label, dtype=torch.float32).to(args.device)
            loc_label = torch.tensor(loc_label, dtype=torch.float32).to(args.device)

            seq = torch.tensor(seq, dtype=torch.float32).unsqueeze(0).to(args.device)
            seqdiff= torch.tensor(seqdiff, dtype=torch.float32).unsqueeze(0).to(args.device)
            # pred_cls, pred_loc = model(seq)

            out = model(seq,seqdiff)

            output_dict = out['d0']
            pred_cls = output_dict['pred_cls']
            pred_loc = output_dict['pred_loc']

            loc_loss = calc_loc_loss(pred_loc, loc_label, cls_label)
            cls_loss = calc_cls_loss(pred_cls, cls_label)

            loss = cls_loss + args.lambda_reg * loc_loss

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            stats.update(loss=loss.item(), cls_loss=cls_loss.item(),
                         loc_loss=loc_loss.item())

        val_fscore, _ = evaluate(model, val_loader, args.nms_thresh, args.device)

        if max_val_fscore < val_fscore:
            max_val_fscore = val_fscore
            torch.save(model.state_dict(), str(save_path))
        lrate=optimizer.state_dict()['param_groups'][0]['lr']

        logger.info(f'Epoch: {epoch}/{args.max_epoch} '
                    f'Loss: {stats.cls_loss:.4f}/{stats.loc_loss:.4f}/{stats.loss:.4f} '
                    f'F-score cur/max: {val_fscore:.4f}/{max_val_fscore:.4f} '
                    f'lr: {lrate:.6f} ')

        # logger.info('Epoch: {epoch}/{args.max_epoch} '
        #             'Loss: {stats.cls_loss:.4f}/{stats.loc_loss:.4f}/{stats.loss:.4f} '
        #             'F-score cur/max: {val_fscore:.4f}/{max_val_fscore:.4f} '
        #             'lr: {lrate:.6f} ')

    return max_val_fscore


import sys
sys.stdin = open('카드2.txt')
from collections import deque

# N = int(input())
N = int(sys.stdin.readline())

queue = deque()
for n in range(1, N+1):
    queue.append(n)

while len(queue) > 1:
    queue.popleft()
    queue.append(queue.popleft())
for x in range(len(queue)):
    print(queue[x])

# 시간 줄이기
# N = int(input())
# square = 2
# while True:
#     if (N == 1 or N == 2):
#         print(N)
#         break
#     square *= 2
#     if (square >= N):
#         print(N - (square // 2) * 2)
#         break


import struct
import threading
from typing import Type
from usbmux import USBMux
from peertalk.PeerTalkThreadBase import TPeerTalkThreadBase

class PeerTalk:

    mux = None
    p_sock = None
    pt_thread = None
    lock = threading.Lock()

    def __init__(self, *args):
        self.port = args[0]
        self.set_blocking = args[1]
        self.set_timeout = args[2]
        print("peerTalk starting")
        self.mux = USBMux()

    def try_connect_abstract(self, class_to_instantiate: Type[TPeerTalkThreadBase]):
        print("Waiting for devices...")
        if not self.mux.devices:
            self.mux.process(1.0)
        if not self.mux.devices:
            print("No device found")

        dev = self.mux.devices[0]
        print("connecting to device %s" % str(dev))
        self.p_sock = self.mux.connect(dev, self.port)  # 2345
        if self.p_sock == -1:
            print("Maybe application on ios is down...")
            return False
        self.p_sock.setblocking(self.set_blocking)
        self.p_sock.settimeout(self.set_timeout)
        self.pt_thread = class_to_instantiate(self.p_sock, self.send)
        self.pt_thread.start()

        return True

    def send(self, r8, t_type, sock):
        with self.lock:
            header_values = (1, t_type, 0, len(r8) + 4)
            frame_structure = struct.Struct("! I I I I")
            packed_data = frame_structure.pack(*header_values)
            sock.send(packed_data)
            message_values = (len(r8), r8)
            fmt_string = "! I {0}s".format(len(r8))
            sm = struct.Struct(fmt_string)
            packed_message = sm.pack(*message_values)
            sock.send(packed_message)


# -*- coding: utf-8 -*-
"""
A multiprocessing approach to generating and saving optical flow and rgb arrays
for each video.
The videos are saved in arrays, with videos of the same length saved together.
If the number of videos multiplied by the length of video exceeds a value which
would proclude the videos from fitting on the GPU, a new array is started
This file also saves an info dict which gives information about the saved arrays
but is not accurate. Need to run generate_dict after to fix this.
"""
from processing_utils import get_flow_rgb, normalise_rgb, normalise_flow, get_len
import numpy as np
import cv2
import skvideo.io
from pathlib import Path
import pickle
import os
from multiprocessing import Pool, Manager
from fix_batches import fix_batches

def work(inputs):
    activities = ['hand-waving','sitting-down', 'walking']
    done_list = inputs['done_list']
    doing_list = inputs['doing_list']
    info_dict = inputs['info_dict']
    doing_lock = inputs['doing_lock']
    lock = inputs['lock']
    frames_lock = inputs['frames_lock']
    max_size = 1024
    image_size = 224
    video_path = inputs['video_path']
    save_path = inputs['save_path']
    flow_path = save_path / 'flow'
    rgb_path = save_path / 'rgb'
    label_path = save_path / 'label'
    info_path = save_path / 'info_file.p'

    for activity in activities:
        activity_path = video_path / activity

        for file in os.listdir(activity_path):

            file_path = activity_path / file

            doing_lock.acquire()
            if (activity + file) in done_list or (activity + file) in doing_list:
                doing_lock.release()
                continue
            doing_list.append(activity + file)
            doing_lock.release()
            # If we haven't processed the file yet, get the flow
            print('Generating flow for: \n' + str(file_path))
            flow, rgb = get_flow_rgb(str(file_path))

            n_frames = flow.shape[0]

            p_name = str(n_frames) + '.p'

            rgb_file_path = rgb_path / p_name
            flow_file_path = flow_path / p_name
            label_file_path = label_path / p_name

            flow = normalise_flow(flow)
            rgb = normalise_rgb(rgb)
            print('Flow generated')
            frames_lock.acquire()
            if not n_frames in [f['n_frames'] for f in info_dict['files']]:
                files = info_dict['files']
                files.append({'n_frames' : n_frames,
                                           'batch_size' : 0,
                                           'save_file' : save_path / 'features' / p_name,
                                           'sample_paths' : {'rgb' : str(rgb_file_path),
                                                             'flow' : str(flow_file_path),
                                                             'label' : str(label_file_path)}})
                info_dict.update({'files' : files})
                lock.acquire()
                frames_lock.release()
                rgb_array = np.zeros([0,n_frames, image_size, image_size, 3])
                flow_array = np.zeros([0,n_frames, image_size, image_size, 2])
                label_array = []

            else:
                lock.acquire()
                frames_lock.release()
                rgb_array = pickle.load(open(rgb_file_path, 'rb'))
                flow_array = pickle.load(open(flow_file_path, 'rb'))
                label_array = pickle.load(open(label_file_path, 'rb'))
            rgb_array = np.append(rgb_array, rgb[None,:,:,:,:], axis = 0)
            flow_array = np.append(flow_array, flow[None,:,:,:,:], axis = 0)
            label_array.append(activity)

            dict_index = [f['n_frames'] for f in info_dict['files']].index(n_frames)
            size = len(label_array)
            info_dict['files'][dict_index].update({'batch_size' : size})
            if size * n_frames >= max_size:
                i = 0
                while(1):
                    new_name = str(n_frames) + '_' + str(i) + '.p'
                    if not os.path.exists(label_path / new_name):
                        break
                    i += 1
                pickle.dump(rgb_array, open(rgb_path / new_name, 'wb'))
                pickle.dump(flow_array, open(flow_path / new_name, 'wb'))
                pickle.dump(label_array, open(label_path / new_name, 'wb'))

                rgb_array = np.zeros([0,n_frames, image_size, image_size, 3])
                flow_array = np.zeros([0,n_frames, image_size, image_size, 2])
                label_array = []
            pickle.dump(rgb_array, open(rgb_file_path, 'wb'))
            pickle.dump(flow_array, open(flow_file_path, 'wb'))
            pickle.dump(label_array, open(label_file_path, 'wb'))
            pickle.dump(dict(info_dict), open(info_path, 'wb'))
            done_list.append(activity + file)
            pickle.dump(list(done_list), open(video_path/'done_list.p','wb'))

            lock.release()
            doing_list.pop(doing_list.index(activity + file))

if __name__ == '__main__':
    n_workers = 8

    base_dir = Path('../test_videos')

    video_path = base_dir / 'videos'
    save_path_rel =  base_dir / 'video_arrays'
    save_path = Path(os.path.abspath(save_path_rel))
    flow_path = save_path / 'flow'
    rgb_path = save_path / 'rgb'
    label_path = save_path / 'label'
    info_path = save_path / 'info_file.p'

    if not os.path.exists(save_path):
        os.mkdir(save_path)
        os.mkdir(flow_path)
        os.mkdir(rgb_path)
        os.mkdir(label_path)

    if not os.path.exists(info_path):
        info_dict = {'saved_size' : 224,
                     'cropped_size' : 224,
                     'files' : []}
    else:
        info_dict = pickle.load(open(info_path,'rb'))

    if not os.path.exists(video_path/'done_list.p'):
        done_list = []
    else:
        done_list = pickle.load(open(video_path/'done_list.p', 'rb'))

    doing_list = []
    m = Manager()
    lock = m.Lock()
    frames_lock = m.Lock()
    doing_lock = m.Lock()
    done_list = m.list(done_list)
    doing_list = m.list(doing_list)
    info_dict = m.dict(info_dict)
    inputs = {'done_list' : done_list,
              'doing_list' : doing_list,
              'info_dict' : info_dict,
              'lock' : lock,
              'frames_lock' : frames_lock,
              'doing_lock' : doing_lock,
              'video_path' : video_path,
              'save_path' : save_path}
    pool = Pool(n_workers)

    pool.map(work, [inputs] * n_workers)

    fix_batches(base_dir)


from .echo_operation import EchoOperation
from echopy.echobase import validation
from ..utils.utils import (connect_echo, disconnect_echo)

async def create_base(node_url, port):
    base = Base(node_url, port)
    await base._init()
    return base

class Base(object):
    def __init__(self, node_url, port):
        self.node_url = "ws://{}:{}".format(node_url, port)
        self.echo_ops = EchoOperation()

    async def _init(self):
        self.echo = await connect_echo(self.node_url)
        self.database_api = self.echo.api.database

    async def _del(self):
        await disconnect_echo(self.echo)

    async def connect_to_echopy_lib(self):
        try:
            await self.echo.connect(url=self.node_url)
        except Exception as _:
            raise Exception("Connection to echopy-lib not established")

    async def get_required_fee(self, operations, asset_id='1.3.0'):
        return await self.echo.database_api.get_required_fees(operations, asset_id)

    async def get_account(self, account_name):
        if validation.is_account_name(account_name):
            return await self.database_api.get_account_by_name(account_name)
        elif validation.is_account_id(account_name):
            return await self.database_api.get_accounts(account_name)

    async def get_account_count(self):
        return await self.database_api.get_account_count()

    def get_public_key(self, account):
        return account["active"]["key_auths"][0][0]


from cached_property import cached_property
from .cigar import (
    cigartuples_to_cigarstring,
    cigar_to_tuple,
    cigartuples_to_named_cigartuples
)
from collections import namedtuple

class BaseTag(object):
    """Generate class template for tags."""

    def __new__(cls, header):
        """Return a Tag class that knows how to format a tag."""
        return type('NamedTagTuple', (namedtuple('tag', 'tid reference_start cigar is_reverse mapq query_alignment_start query_alignment_end'),),
                    {'__str__': lambda self: self.tag_str_template % (self.header['SQ'][self.tid]['SN'],
                                                                      self.reference_start,
                                                                      self.query_alignment_start,
                                                                      self.query_alignment_end,
                                                                      cigartuples_to_cigarstring(self.cigar),
                                                                      'AS' if self.is_reverse else 'S',
                                                                      self.mapq),
                     'reference_name': lambda self: self.header['SQ'][self.tid]['SN'],
                     'header': header,
                     'tag_str_template': "R:%s,POS:%d,QSTART:%d,QEND:%d,CIGAR:%s,S:%s,MQ:%d"})

def make_tag(template, r):
    """Return namedtuple of read attributes."""
    return template(tid=r.tid,
                    reference_start=r.reference_start,
                    cigar=r.cigar,
                    is_reverse=r.is_reverse,
                    mapq=r.mapping_quality,
                    query_alignment_start=r.query_alignment_start,
                    query_alignment_end=r.query_alignment_end)

class Tag(object):
    """Collect tag attributes and conversion."""

    def __init__(self,
                 reference_start,
                 cigar,
                 is_reverse,
                 mapq=None,
                 query_alignment_start=None,
                 query_alignment_end=None,
                 tid=None,
                 reference_name=None,
                 header=None):
        """
        Return new Tag instance from kwds.

        Note that the cigar is always wrt the reference alignment.
        When comparing Tag object by their cigar, one of the cigar needs to be inverted if the
        Tag objects are not in the same orientation.
        """
        self.reference_start = reference_start
        self._cigar = cigar
        self.is_reverse = is_reverse
        self.mapq = mapq
        self.query_alignment_start = query_alignment_start
        self.query_alignment_end = query_alignment_end
        self.tid = tid
        self._reference_name = reference_name
        self.header = header

    @cached_property
    def reference_name(self):
        """Return reference name for this instance."""
        if self._reference_name:
            return self._reference_name
        else:
            try:
                return self.header.references[self.tid]
            except Exception:
                return None

    @cached_property
    def cigar(self):
        """
        Lazily convert cigarstring to tuple if it doesn't exist.

        >>> Tag(reference_start=0, cigar='20M30S', is_reverse='True', mapq=60, query_alignment_start=0, query_alignment_end=20, tid=5).cigar
        [CIGAR(operation=0, length=20), CIGAR(operation=4, length=30)]
        """
        if isinstance(self._cigar, str):
            self._cigar = cigar_to_tuple(self._cigar)
        self._cigar = cigartuples_to_named_cigartuples(self._cigar)
        return self._cigar

    @staticmethod
    def from_read(r, header=None):
        """
        Return Tag instance from pysam.AlignedSegment Instance.

        >>> from tests.helpers import MockAlignedSegment as AlignedSegment
        >>> t = Tag.from_read(AlignedSegment(cigar='20M30S'))
        >>> isinstance(t, Tag)
        True
        >>> t.reference_name is None
        True
        """
        return Tag(tid=r.tid,
                   reference_start=r.reference_start,
                   cigar=r.cigar,
                   is_reverse=r.is_reverse,
                   mapq=r.mapping_quality,
                   query_alignment_start=r.query_alignment_start,
                   query_alignment_end=r.query_alignment_end,
                   header=header)

    @staticmethod
    def from_tag_str(tag_str):
        """
        Return Tag Instance from tag string.

        >>> t = Tag.from_tag_str('R:FBti0019061_rover_Gypsy,POS:7435,QSTART:0,QEND:34,CIGAR:34M91S,S:S,MQ:60')
        >>> isinstance(t, Tag)
        True
        >>> t.cigar == [(0, 34), (4, 91)]
        True
        >>> t = Tag.from_tag_str('R:FBti0019061_rover_Gypsy,POS:7435,QSTART:0,QEND:34,CIGAR:34M91S,S:AS,MQ:60')
        >>> t.is_reverse
        True
        """
        tag_to_attr = {'R': 'reference_name',
                       'POS': 'reference_start',
                       'QSTART': 'query_alignment_start',
                       'QEND': 'query_alignment_end',
                       'CIGAR': 'cigar',
                       'S': 'is_reverse',
                       'MQ': 'mapq'}
        integers = ['reference_start', 'query_alignment_start', 'query_alignment_end', 'mapq']
        tag_d = {tag_to_attr[k]: v for k, v in dict(item.split(':') for item in tag_str.split(',')).items()}
        if tag_d['is_reverse'] == 'S':
            tag_d['is_reverse'] = False
        else:
            tag_d['is_reverse'] = True
        for integer in integers:
            tag_d[integer] = int(tag_d.get(integer, 0))
        return Tag(**tag_d)

    def to_dict(self):
        """
        Serialize self into dictionary.

        >>> t = Tag.from_tag_str('R:FBti0019061_rover_Gypsy,POS:7435,QSTART:0,QEND:34,CIGAR:34M91S,S:S,MQ:60')
        >>> t.to_dict()['is_reverse']
        False
        """
        return {'reference_start': self.reference_start,
                'cigar': self.cigar,
                'mapq': self.mapq,
                'query_alignment_start': self.query_alignment_start,
                'query_alignment_end': self.query_alignment_end,
                'is_reverse': self.is_reverse,
                'tid': self.tid}  # Improve this by passing tid or reference name

    def to_string(self, header=None):
        """
        Serialize to tag string.

        >>> t = Tag.from_tag_str('R:FBti0019061_rover_Gypsy,POS:7435,QSTART:0,QEND:34,CIGAR:34M91S,S:S,MQ:60')
        >>> t.to_string() == 'R:FBti0019061_rover_Gypsy,POS:7435,QSTART:0,QEND:34,CIGAR:34M91S,S:S,MQ:60'
        True

        :param header:
        :return:
        """
        header = header or self.header
        return "R:%s,POS:%d,QSTART:%d,QEND:%d,CIGAR:%s,S:%s,MQ:%d" % (self.reference_name or header['SQ'][self.tid]['SN'],
                                                                      self.reference_start,
                                                                      self.query_alignment_start,
                                                                      self.query_alignment_end,
                                                                      cigartuples_to_cigarstring(self.cigar),
                                                                      'AS' if self.is_reverse else 'S',
                                                                      self.mapq)


#!/usr/bin/env python3
import os
import json
import h5py
import argparse
import pandas as pd
import numpy as np
import tinydb as db
from tinydb.storages import MemoryStorage

import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib.colors import LogNorm

import scipy.stats as stats
import scipy.optimize as opt

from pygama import DataGroup
import pygama.lh5 as lh5
import pygama.analysis.histograms as pgh
import pygama.analysis.peak_fitting as pgf
import cage_utils
plt.style.use('./joule_dissertation.mplstyle')

mpl.use('Agg')

def main():
    user = True
    hit = True
    cal = True

    run=64

    cut1 = 'trapEftp_cal > 1458 and trapEftp_cal < 1462'
    cut2 = 'trapEftp_cal > 59.0 and trapEftp_cal < 60.0'

    # for alpha example
    # cut1 = 'trapEftp_cal > 3900 and trapEftp_cal < 4100 and dcr > 90 and dcr < 100'

    plot_wfs(run, cut2, user=user, hit=hit, cal=cal, norm=True)
    # compare_wfs(run, cut1, cut2, user=user, hit=hit, cal=cal, norm=True)
    # wfs_filtered(run, cut1, user=user,1hit=hit, cal=cal, norm=False)

def plot_wfs(run, cut, user=True, hit=True, cal=True, norm=False):
    df_raw, dg, runtype, rt_min, radius, angle_det, rotary = cage_utils.getDataFrame(run, user=user, hit=hit,
                                                                                cal=cal)
    df = cage_utils.apply_DC_Cuts(run, df_raw)

    #times, super_wf = cage_utils.get_superpulse_taligned(df, dg, cut, nwfs=10, norm=False)
    times, wfs = cage_utils.get_superpulse_taligned(df, dg, cut, nwfs=20, norm=norm)

    print(len(wfs))

    fig, ax = plt.subplots(figsize=(14, 10))
    ax = plt.axes()

    ax.plot(times, wfs)

    plt.title(f'1460 keV waveform')

    plt.xlabel('clock cycles')
    plt.ylabel('normalized ADU')

    plt.savefig('./plots/1460_wf.png', dpi=200)
    plt.savefig('./plots/1460_wf.pdf', dpi=200)

    # plt.xlim(3800, 8000)
    # plt.ylim(3000, 4200)

    # plt.savefig('./plots/60_wfs_tail.png', dpi=200)
    # plt.savefig('./plots/60_wfs_tail.pdf', dpi=200)

    plt.xlim(3500, 4100)
    plt.savefig('./plots/1460_wf_rise.png', dpi=200)
    plt.savefig('./plots/1460_wf_rise.pdf', dpi=200)

def compare_wfs(run, cut1, cut2, user=True, hit=True, cal=True, norm=False):
    df_raw, dg, runtype, rt_min, radius, angle_det, rotary = cage_utils.getDataFrame(run, user=user, hit=hit,
                                                                                cal=cal)
    df = cage_utils.apply_DC_Cuts(run, df_raw)

    #times, super_wf = cage_utils.get_superpulse_taligned(df, dg, cut, nwfs=10, norm=False)
    times, wfs1 = cage_utils.get_wfs(df, dg, cut1, nwfs=1, norm=norm)
    times, wfs2 = cage_utils.get_wfs(df, dg, cut2, nwfs=1, norm=norm)

    fig, ax = plt.subplots(figsize=(14, 10))
    ax = plt.axes()

    ax.plot(times, wfs2[0], c='r', label='60 keV')
    ax.plot(times, wfs1[0], c='g', label='1460 keV')

    plt.title(f'Comparison of 1460 keV to 60 keV waveforms')

    plt.xlabel('clock cycles')
    plt.ylabel('normalized ADU')

    plt.legend()

    plt.savefig('./plots/1460_v_60keV_wfs.png', dpi=200)
    plt.savefig('./plots/1460_v_60keV_wfs.pdf', dpi=200)

#     plt.xlim(3800, 8000)
#     plt.ylim(3000, 4200)

#     plt.savefig('./plots/1460_v_60keV_wfs_tail.png', dpi=200)
#     plt.savefig('./plots/1460_v_60keV_wfs_tail.pdf', dpi=200)

def wfs_filtered(run, cut1, user=True, hit=True, cal=True, norm=False):
    df_raw, dg, runtype, rt_min, radius, angle_det, rotary = cage_utils.getDataFrame(run, user=user, hit=hit,
                                                                                cal=cal)
    df = cage_utils.apply_DC_Cuts(run, df_raw)

    # times, wf = cage_utils.get_wfs(df, dg, cut1, nwfs=1, norm=norm)
    times, wf = cage_utils.get_wfs(df, dg, cut1, nwfs=1, norm=norm, tp_align=0.5, n_pre = 3800, n_post = 4175)

    wf_dcr_pz = cage_utils.double_pole_zero(wf, 21250, 433, 0.045)[0]
    wf_pz = cage_utils.double_pole_zero(wf, 18750, 317, 0.035)[0]
    Etrap = cage_utils.trap_norm(wf_pz, 100, 400)
    dcr_trap = cage_utils.trap_norm(wf_dcr_pz, 750, 2250)
    wf_atrap = cage_utils.asymTrapFilter(wf_pz, 20, 100, 400)
    atrap_max = np.argmax(wf_atrap)
    t0 = cage_utils.time_point_thresh_max(wf_atrap, 10., atrap_max, 3500)
    Eftp = Etrap[t0 + 400]
    dcr = dcr_trap[7900]

    print(Eftp)
    print(t0)

    fig, ax = plt.subplots()
    # fig, ax = plt.subplots(figsize=(14, 10))
    ax = plt.axes()

    ax.plot(times, wf[0], c='b', label='wf_blsub')
    ax.plot(times, wf_pz, c='r', label='wf_pz')
    ax.plot(times, Etrap, c='g', label='wf_trap')
    plt.axhline(Eftp, lw=2, c = 'c', linestyle='dotted', label='trapEftp')
    plt.plot(t0+400, Eftp, '*y', markersize=20)

    plt.title(f'Energy determination')

    plt.xlabel('clock cycles')
    plt.ylabel('ADU')

    plt.legend(fontsize=16)

    plt.savefig('./plots/wf_etrap.png', dpi=200)
    plt.savefig('./plots/wf_etrap.pdf', dpi=200)

    plt.clf()
    plt.close()

    fig, ax = plt.subplots()
    ax = plt.axes()

    fig, ax = plt.subplots()
    # fig, ax = plt.subplots(figsize=(14, 10))
    ax = plt.axes()

    ax.plot(times, wf[0], c='b', label='wf_blsub')
    ax.plot(times, wf_atrap, c='g', label='wf_atrap')
    plt.axvline(t0, lw=2, c = 'r', label='tp_0')

    plt.title(f'tp_0 determination')

    plt.xlabel('clock cycles')
    plt.ylabel('ADU')

    plt.legend()

    plt.savefig('./plots/wf_t0.png', dpi=200)
    plt.savefig('./plots/wf_t0.pdf', dpi=200)

    plt.xlim(3720, 3840)

    plt.savefig('./plots/wf_t0_zoom.png', dpi=200)
    plt.savefig('./plots/wf_t0_zoom.pdf', dpi=200)

    plt.clf()
    plt.close()

    fig, ax = plt.subplots()
    ax = plt.axes()

    ax.plot(times, wf[0], c='b', label='wf_blsub')
    ax.plot(times, wf_dcr_pz, c='r', label='wf_pzDCR')
    ax.plot(times, dcr_trap, c='g', label='dcr_trap')
    plt.axhline(dcr, lw=2, c = 'c', linestyle='dotted', label='dcr')
    plt.plot(7900, dcr, '*y', markersize=20)
    #ax.axvspan(7899, 7901, alpha=0.1, color='g', label='dcr_')

    plt.title(f'DCR determination')
    plt.xlabel('clock cycles')
    plt.ylabel('ADU')

    plt.legend()
    plt.tight_layout()

    plt.savefig('./plots/wf_dcr.png', dpi=200)
    plt.savefig('./plots/wf_dcr.pdf', dpi=200)

    # plt.ylim(-1, 50) # for 1460 keV
    plt.ylim(-1, 120) # for alpha

    plt.savefig('./plots/wf_dcr_bl.png', dpi=200)
    plt.savefig('./plots/wf_dcr_bl.pdf', dpi=200)

    plt.clf()
    plt.close()

if __name__=="__main__":
    main()


#!/usr/bin/env python

import re
import os
import pwd
import time
import glob
import json
import shlex
import base64
import socket
import urllib
import urllib2
import hashlib
import tempfile
import subprocess
from functools import wraps
from contextlib import contextmanager

from cloudify import ctx

REST_VERSION = 'v3.1'
PROCESS_POLLING_INTERVAL = 0.1
CLOUDIFY_SOURCES_PATH = '/opt/cloudify/sources'
MANAGER_RESOURCES_HOME = '/opt/manager/resources'
AGENT_ARCHIVES_PATH = '{0}/packages/agents'.format(MANAGER_RESOURCES_HOME)
MANAGER_RESOURCES_SNAPSHOT_PATHS = [
    os.path.join(MANAGER_RESOURCES_HOME, path)
    for path in (
        'blueprints',
        'deployments',
        'uploaded-blueprints',
        'snapshots',
        'plugins',
    )
]

SSL_CERTS_TARGET_DIR = '/etc/cloudify/ssl'

INTERNAL_SSL_CERT_FILENAME = 'cloudify_internal_cert.pem'
INTERNAL_SSL_KEY_FILENAME = 'cloudify_internal_key.pem'
INTERNAL_SSL_CA_CERT_FILENAME = 'cloudify_internal_ca_cert.pem'
INTERNAL_SSL_CA_KEY_FILENAME = 'cloudify_internal_ca_key.pem'
INTERNAL_PKCS12_FILENAME = 'cloudify_internal.p12'
EXTERNAL_SSL_CERT_FILENAME = 'cloudify_external_cert.pem'
EXTERNAL_SSL_KEY_FILENAME = 'cloudify_external_key.pem'

INTERNAL_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                  INTERNAL_SSL_CERT_FILENAME)
INTERNAL_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                 INTERNAL_SSL_KEY_FILENAME)
INTERNAL_CA_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                     INTERNAL_SSL_CA_CERT_FILENAME)
INTERNAL_CA_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                    INTERNAL_SSL_CA_KEY_FILENAME)
EXTERNAL_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                  EXTERNAL_SSL_CERT_FILENAME)
EXTERNAL_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                 EXTERNAL_SSL_KEY_FILENAME)
CERT_METADATA_FILE_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                       'certificate_metadata')

BASE_LOG_DIR = '/var/log/cloudify'

NGINX_SERVICE_NAME = 'nginx'
DEFAULT_BUFFER_SIZE = 8192
SINGLE_TAR_PREFIX = 'cloudify-manager-resources'

CLOUDIFY_USER = 'cfyuser'
CLOUDIFY_GROUP = 'cfyuser'
CLOUDIFY_HOME_DIR = '/etc/cloudify'
SUDOERS_INCLUDE_DIR = '/etc/sudoers.d'
CLOUDIFY_SUDOERS_FILE = os.path.join(SUDOERS_INCLUDE_DIR, CLOUDIFY_USER)

CLUSTER_DELETE_SCRIPT = '/opt/cloudify/delete_cluster.py'
CFY_EXEC_TEMPDIR_ENVVAR = 'CFY_EXEC_TEMP'

def retry(exception, tries=4, delay=3, backoff=2):
    """Retry calling the decorated function using an exponential backoff.

    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
    """
    def deco_retry(f):
        @wraps(f)
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                except exception as ex:
                    msg = "{0}, Retrying in {1} seconds...".format(ex, mdelay)
                    ctx.logger.warn(msg)
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            return f(*args, **kwargs)
        return f_retry  # true decorator
    return deco_retry

def escape_for_systemd(the_string):
    if '\n' in the_string:
        the_string = the_string.replace('\n', '\\\\n\\\n')
        the_string = "\"" + the_string + "\""
    return the_string

def run(command, retries=0, stdin=b'', ignore_failures=False,
        globx=False, shell=False, env=None):
    if isinstance(command, str) and not shell:
        command = shlex.split(command)
    stderr = subprocess.PIPE
    stdout = subprocess.PIPE
    if globx:
        glob_command = []
        for arg in command:
            glob_command.append(glob.glob(arg))
        command = glob_command
    ctx.logger.debug('Running: {0}'.format(command))
    proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=stdout,
                            stderr=stderr, shell=shell, env=env)
    proc.aggr_stdout, proc.aggr_stderr = proc.communicate(input=stdin)
    if proc.returncode != 0:
        command_str = ' '.join(command)
        if retries:
            ctx.logger.warn('Failed running command: {0}. Retrying. '
                            '({1} left)'.format(command_str, retries))
            proc = run(command, retries - 1)
        elif not ignore_failures:
            msg = 'Failed running command: {0} ({1}).'.format(
                command_str, proc.aggr_stderr)
            raise RuntimeError(msg)
    return proc

def sudo(command, *args, **kwargs):
    if isinstance(command, str):
        command = shlex.split(command)
    if 'env' in kwargs:
        command = ['sudo', '-E'] + command
    else:
        command.insert(0, 'sudo')
    return run(command=command, *args, **kwargs)

def sudo_write_to_file(contents, destination):
    fd, path = tempfile.mkstemp()
    os.close(fd)
    with open(path, 'w') as f:
        f.write(contents)
    return move(path, destination)

def add_entry_to_sudoers(entry, description):
    # Comment out the description and add a N/L after the entry for visibility
    description = '# {0}'.format(description)
    entry = '{0}\n'.format(entry)

    for line in (description, entry):
        # `visudo` handles sudoers file. Setting EDITOR to `tee -a` means that
        # whatever is piped should be appended to the file passed.
        sudo(['/sbin/visudo', '-f', CLOUDIFY_SUDOERS_FILE],
             stdin=line, env={'EDITOR': '/bin/tee -a'})

    valid = sudo(
        ['visudo', '-cf', CLOUDIFY_SUDOERS_FILE],
        ignore_failures=True,
    )
    if valid.returncode != 0:
        ctx.abort_operation(
            "Generated sudoers entry containing \"{entry}\" was "
            "invalid.".format(entry=entry)
        )

def allow_user_to_sudo_command(full_command, description, allow_as='root'):
    entry = '{user}    ALL=({allow_as}) NOPASSWD:{full_command}'.format(
        user=CLOUDIFY_USER,
        allow_as=allow_as,
        full_command=full_command
    )
    add_entry_to_sudoers(entry, description)

def deploy_sudo_command_script(script, description, component=None,
                               allow_as='root'):
    # If passed a component, then script is a relative path, that needs to
    # be downloaded from the scripts folder. Otherwise, it's an absolute path
    if component:
        config_file_temp_destination = os.path.join(tempfile.gettempdir(),
                                                    script)
        ctx.download_resource_and_render(
            os.path.join('components', component, 'scripts', script),
            config_file_temp_destination)
        script = os.path.join('/opt/cloudify/', component, script)
        move(config_file_temp_destination, script)
        chmod('550', script)
        chown('root', CLOUDIFY_GROUP, script)

    ctx.logger.info('Allowing user `{0}` to run `{1}`'
                    .format(CLOUDIFY_USER, script))
    allow_user_to_sudo_command(full_command=script, description=description,
                               allow_as=allow_as)

def deploy_ssl_certificate(private_or_public, destination, group, cert):
    # Root owner, with permissions set below,
    # allow anyone to read a public cert,
    # and allow the owner to read a private cert, but not change it,
    # mitigating risk in the event of the associated service being vulnerable.
    ownership = 'root.{0}'.format(group)
    if private_or_public == 'private':
        private_cert_ok = 'PRIVATE KEY' in cert.split('\n')[0]
        if private_cert_ok:
            permissions = '440'
        else:
            ctx.abort_operation("Private certificate is expected to begin "
                                "with a line containing 'PRIVATE KEY'.")
    elif private_or_public == 'public':
        public_cert_ok = 'BEGIN CERTIFICATE' in cert.split('\n')[0]
        if public_cert_ok:
            permissions = '444'
        else:
            ctx.abort_operation("Public certificate is expected to begin with "
                                "a line containing 'BEGIN CERTIFICATE'.")
    else:
        ctx.abort_operation("Certificates may only be 'private' or 'public', "
                            "not {0}".format(private_or_public))
    ctx.logger.debug(
        "Deploying {0} SSL certificate in {1} for group {2}".format(
            private_or_public, destination, group))
    sudo_write_to_file(cert, destination)
    ctx.logger.debug('Setting permissions ({0}) and ownership ({1}) of '
                     'SSL certificate at {2}'.format(
                         permissions, ownership, destination))
    chmod(permissions, destination)
    sudo('chown {0} {1}'.format(ownership, destination))

def mkdir(dir, use_sudo=True):
    if os.path.isdir(dir):
        return
    ctx.logger.debug('Creating Directory: {0}'.format(dir))
    cmd = ['mkdir', '-p', dir]
    if use_sudo:
        sudo(cmd)
    else:
        run(cmd)

# idempotent move operation
def move(source, destination, rename_only=False):
    if rename_only:
        sudo(['mv', '-T', source, destination])
    else:
        copy(source, destination)
        remove(source)

def copy(source, destination):
    destination_dir = os.path.dirname(destination)
    if not os.path.exists(destination_dir):
        ctx.logger.debug(
            'Path does not exist: {0}. Creating it...'.format(
                destination_dir))
        sudo(['mkdir', '-p', destination_dir])
    sudo(['cp', '-rp', source, destination])

def remove(path, ignore_failure=False):
    ctx.logger.debug('Removing {0}...'.format(path))
    sudo(['rm', '-rf', path], ignore_failures=ignore_failure)

def generate_ca_cert():
    sudo([
        'openssl', 'req',
        '-x509',
        '-nodes',
        '-newkey', 'rsa:2048',
        '-days', '3650',
        '-batch',
        '-out', INTERNAL_CA_CERT_PATH,
        '-keyout', INTERNAL_CA_KEY_PATH
    ])
    # PKCS12 file required for riemann due to JVM
    # While we don't really want the private key in there, not having it
    # causes failures
    # The password is also a bit pointless here since it's in the same place
    # as a readable copy of the certificate and if this path can be written to
    # maliciously then all is lost already.
    pkcs12_path = os.path.join(SSL_CERTS_TARGET_DIR, INTERNAL_PKCS12_FILENAME)
    sudo([
        'openssl', 'pkcs12', '-export',
        '-out', pkcs12_path,
        '-in', INTERNAL_CA_CERT_PATH,
        '-inkey', INTERNAL_CA_KEY_PATH,
        '-password', 'pass:cloudify',
    ])

def _format_ips(ips):
    altnames = set(ips)

    # Ensure we trust localhost
    altnames.add('127.0.0.1')
    altnames.add('localhost')

    subject_altdns = [
        'DNS:{name}'.format(name=name)
        for name in altnames
    ]
    subject_altips = []
    for name in altnames:
        ip_address = False
        try:
            socket.inet_pton(socket.AF_INET, name)
            ip_address = True
        except socket.error:
            # Not IPv4
            pass
        try:
            socket.inet_pton(socket.AF_INET6, name)
            ip_address = True
        except socket.error:
            # Not IPv6
            pass
        if ip_address:
            subject_altips.append('IP:{name}'.format(name=name))

    cert_metadata = ','.join([
        ','.join(subject_altdns),
        ','.join(subject_altips),
    ])
    return cert_metadata

def store_cert_metadata(internal_rest_host, networks=None,
                        filename=CERT_METADATA_FILE_PATH):
    metadata = load_cert_metadata()
    metadata['internal_rest_host'] = internal_rest_host
    if networks is not None:
        metadata['networks'] = networks
    contents = json.dumps(metadata)
    sudo_write_to_file(contents, filename)
    chown(CLOUDIFY_USER, CLOUDIFY_GROUP, filename)

def load_cert_metadata(filename=CERT_METADATA_FILE_PATH):
    try:
        with open(filename) as f:
            return json.load(f)
    except IOError:
        return {}

CSR_CONFIG_TEMPLATE = """
[req]
distinguished_name = req_distinguished_name
req_extensions = server_req_extensions
[ server_req_extensions ]
subjectAltName={metadata}
[ req_distinguished_name ]
commonName = _common_name # ignored, _default is used instead
commonName_default = {cn}
"""

@contextmanager
def _csr_config(cn, metadata):
    """Prepare a config file for creating a ssl CSR.

    :param cn: the subject commonName
    :param metadata: string to use as the subjectAltName, should be formatted
                     like "IP:1.2.3.4,DNS:www.com"
    """
    with tempfile.NamedTemporaryFile(delete=False) as conf_file:
        conf_file.write(CSR_CONFIG_TEMPLATE.format(cn=cn, metadata=metadata))

    try:
        yield conf_file.name
    finally:
        remove(conf_file.name)

def generate_ssl_certificate(ips, cn, cert_path, key_path,
                             sign_cert=INTERNAL_CA_CERT_PATH,
                             sign_key=INTERNAL_CA_KEY_PATH):
    """Generate a public SSL certificate and a private SSL key

    :param ips: the ips (or names) to be used for subjectAltNames
    :type ips: List[str]
    :param cn: the subject commonName for the new certificate
    :type cn: str
    :param cert_path: path to save the new certificate to
    :type cert_path: str
    :param key_path: path to save the key for the new certificate to
    :type key_path: str
    :param sign_cert: path to the signing cert (internal CA by default)
    :type sign_cert: str
    :param sign_key: path to the signing cert's key (internal CA by default)
    :type sign_key: str
    :return: The path to the cert and key files on the manager
    """
    # Remove duplicates from ips
    subject_altnames = _format_ips(ips)
    ctx.logger.info(
        'Generating SSL certificate {0} and key {1} with subjectAltNames: {2}'
        .format(cert_path, key_path, subject_altnames)
    )

    csr_path = '{0}.csr'.format(cert_path)

    with _csr_config(cn, subject_altnames) as conf_path:
        sudo([
            'openssl', 'req',
            '-newkey', 'rsa:2048',
            '-nodes',
            '-batch',
            '-config', conf_path,
            '-out', csr_path,
            '-keyout', key_path,
        ])
        x509_command = [
            'openssl', 'x509',
            '-days', '3650',
            '-req', '-in', csr_path,
            '-extfile', conf_path,
            '-out', cert_path,
            '-extensions', 'server_req_extensions',
        ]
        if sign_cert and sign_key:
            x509_command += [
                '-CA', sign_cert,
                '-CAkey', sign_key,
                '-CAcreateserial'
            ]
        else:
            x509_command += [
                '-signkey', key_path
            ]
        sudo(x509_command)
        remove(csr_path)

    ctx.logger.info('Generated SSL certificate: {0} and key: {1}'.format(
        cert_path, key_path
    ))
    return cert_path, key_path

def generate_internal_ssl_cert(ips, name):
    return generate_ssl_certificate(
        ips,
        name,
        INTERNAL_CERT_PATH,
        INTERNAL_KEY_PATH
    )

def write_to_tempfile(contents):
    fd, file_path = tempfile.mkstemp()
    os.write(fd, contents)
    os.close(fd)
    return file_path

def install_python_package(source, venv='', constraints_file=None):
    cmdline = []
    if venv:
        cmdline.append('{0}/bin/pip'.format(venv))
    else:
        cmdline.append('pip')

    cmdline.extend(['install', source, '--upgrade'])

    log_message = 'Installing {0}'.format(source)

    if venv:
        log_message += ' in virtualenv {0}'.format(venv)
    if constraints_file:
        cmdline.extend(['-c', constraints_file])
        log_message += ' using constraints file {0}'.format(constraints_file)

    ctx.logger.info(log_message)

    sudo(cmdline)

def get_file_content(file_path):
    """
    Using sudo to copy the file to a temp folder, to allow access to the file
    even if it's in a restricted directory (e.g. '/root').
    Then, chmoding the file so the current user can read it.
    :param file_path: the path to the file
    :return: the content of the file
    """
    try:
        temp_dir = tempfile.mkdtemp()
        copy(file_path, temp_dir)
        filename = os.path.basename(file_path)
        new_file_copy = os.path.join(temp_dir, filename)
        chmod('644', new_file_copy)
        with open(new_file_copy) as the_file:
            return the_file.read()
    finally:
        if os.path.exists(temp_dir):
            remove(temp_dir)

def curl_download_with_retries(source, destination):
    """Download file using the curl command.

    :param source: Source URL for the file to download
    :typ source: str
    :param destination:
        Path to the directory where the file should be downloaded
    :type destination: str

    """
    curl_cmd = [
        'curl',
        '--retry', '10',
        '--fail',
        '--silent',
        '--show-error',
        '--location', source,
        '--create-dir',
        '--output', destination,
    ]
    ctx.logger.debug('curling: {0}'.format(' '.join(curl_cmd)))
    run(curl_cmd)

def download_file(url, destination=''):
    if not destination:
        fd, destination = tempfile.mkstemp()
        os.remove(destination)
        os.close(fd)

    if not os.path.isfile(destination):
        ctx.logger.info('Downloading {0} to {1}...'.format(
            url, destination))
        try:
            final_url = urllib.urlopen(url).geturl()
            if final_url != url:
                ctx.logger.debug('Redirected to {0}'.format(final_url))
            f = urllib.URLopener()
            # TODO: try except with @retry
            f.retrieve(final_url, destination)
        except Exception:
            curl_download_with_retries(url, destination)
    else:
        ctx.logger.debug('File {0} already exists...'.format(destination))
    return destination

def get_file_name_from_url(url):
    try:
        return url.split('/')[-1]
    except Exception:
        # in case of irregular url. precaution.
        # note that urlparse is deprecated in Python 3
        from urlparse import urlparse
        disassembled = urlparse(url)
        return os.path.basename(disassembled.path)

def download_cloudify_resource(url, service_name, destination=None):
    """Downloads a resource and saves it as a cloudify resource.

    The resource will be saved under the appropriate service resource path and
    will be used in case of operation execution failure after the resource has
    already been downloaded.
    """
    if destination:
        source_res_path, _ = resource_factory.create(url,
                                                     destination,
                                                     service_name,
                                                     source_resource=True,
                                                     render=False)
        copy(source_res_path, destination)
    else:
        res_name = os.path.basename(url)
        source_res_path, _ = resource_factory.create(url, res_name,
                                                     service_name,
                                                     source_resource=True,
                                                     render=False)
    return source_res_path

def deploy_blueprint_resource(source, destination, service_name,
                              user_resource=False, render=True, load_ctx=True):
    """
    Downloads a resource from the blueprint to a destination. This expands
    `download-resource` as a `sudo mv` is required after having downloaded
    the resource.
    :param source: Resource source.
    :param destination: Resource destination.
    :param service_name: The service name that requires the resource.
    :param user_resource: Set to true for resources that should potentially
    remain identical upon upgrade such as custom security configuration files.
    :param render: Set to false if resource does not require rendering.
    :param load_ctx: Set to false if node props should not be loaded.
    NOTE: This is normally used when using this function from a preconfigure
    script where node properties are not available.
    """
    ctx.logger.info('Deploying blueprint resource {0} to {1}'.format(
        source, destination))
    resource_file, dest = resource_factory.create(source,
                                                  destination,
                                                  service_name,
                                                  user_resource=user_resource,
                                                  render=render,
                                                  load_ctx=load_ctx)
    copy(resource_file, dest)

def copy_notice(service):
    """Deploys a notice file to /opt/SERVICENAME_NOTICE.txt"""
    destn = os.path.join('/opt', '{0}_NOTICE.txt'.format(service))
    source = 'components/{0}/NOTICE.txt'.format(service)
    resource_file, dest = resource_factory.create(source, destn, service,
                                                  render=False)
    copy(resource_file, dest)

def remove_notice(service):
    """Remove the notice file /opt/SERVICENAME_NOTICE.txt"""
    path = os.path.join('/opt', '{0}_NOTICE.txt'.format(service))
    remove(path)

def is_port_open(port, host='localhost'):
    """Try to connect to (host, port), return if the port was listening."""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    return sock.connect_ex((host, port)) == 0

def wait_for_port(port, host='localhost'):
    """Helper function to wait for a port to open before continuing"""
    counter = 1

    ctx.logger.info('Waiting for {0}:{1} to become available...'.format(
        host, port))

    for tries in range(24):
        if not is_port_open(port, host=host):
            ctx.logger.info(
                '{0}:{1} is not available yet, retrying... '
                '({2}/24)'.format(host, port, counter))
            time.sleep(2)
            counter += 1
            continue
        ctx.logger.info('{0}:{1} is open!'.format(host, port))
        return
    ctx.abort_operation('Failed to connect to {0}:{1}...'.format(host, port))

def yum_install(source, service_name):
    """Installs a package using yum.

    yum supports installing from URL, path and the default yum repo
    configured within your image.
    you can specify one of the following:
    [yum install -y] mylocalfile.rpm
    [yum install -y] mypackagename

    If the source is a package name, it will check whether it is already
    installed. If it is, it will do nothing. It not, it will install it.

    If the source is a url to an rpm and the file doesn't already exist
    in a predesignated archives file path (${CLOUDIFY_SOURCES_PATH}/),
    it will download it. It will then use that file to check if the
    package is already installed. If it is, it will do nothing. If not,
    it will install it.

    NOTE: This will currently not take into considerations situations
    in which a file was partially downloaded. If a file is partially
    downloaded, a re-download will not take place and rather an
    installation will be attempted, which will obviously fail since
    the rpm file is incomplete.
    ALSO NOTE: you cannot provide `yum_install` with a space
    separated array of packages as you can with `yum install`. You must
    provide one package per invocation.
    """
    # source is a url
    if source.startswith(('http', 'https', 'ftp')):
        filename = get_file_name_from_url(source)
        source_name, ext = os.path.splitext(filename)
    # source is just the name of the file
    elif source.endswith('.rpm'):
        source_name, ext = os.path.splitext(source)
    # source is the name of a yum-repo based package name
    else:
        source_name, ext = source, ''
    source_path = source_name

    if ext.endswith('.rpm'):
        source_path = download_cloudify_resource(source, service_name)

        rpm_handler = RpmPackageHandler(source_path)
        ctx.logger.info(
            'Checking whether {0} is already installed...'.format(
                source_path))
        if rpm_handler.is_rpm_installed():
            ctx.logger.debug('Package {0} is already installed.'.format(
                source))
            return

        # removes any existing versions of the package that do not match
        # the provided package source version
        rpm_handler.remove_existing_rpm_package()
    else:
        installed = run(['yum', '-q', 'list', 'installed', source_path],
                        ignore_failures=True)
        if installed.returncode == 0:
            ctx.logger.debug('Package {0} is already installed.'.format(
                source))
            return
    ctx.logger.info('yum installing {0}...'.format(source_path))
    sudo(['yum', 'install', '-y', '--disablerepo=*', source_path])

def yum_remove(package, ignore_failures=False):
    ctx.logger.info('yum removing {0}...'.format(package))
    try:
        sudo(['yum', 'remove', '-y', package])
    except BaseException:
        msg = 'Package `{0}` may not been removed successfully!'
        if not ignore_failures:
            ctx.logger.error(msg)
            raise
        ctx.logger.warn(msg)

def get_filepath_from_pkg_name(filename, raise_if_not_found=False):
    local_filepath_list = \
        [fn for fn in glob.glob(os.path.join(CLOUDIFY_SOURCES_PATH, filename))
         if not os.path.basename(fn).startswith(SINGLE_TAR_PREFIX)]
    if not local_filepath_list:
        if raise_if_not_found:
            raise IOError('Local resource not found: {0}'.format(filename))
        ctx.abort_operation("File: {0} does not exist in sources path: {1}".
                            format(filename, CLOUDIFY_SOURCES_PATH))
    if len(local_filepath_list) > 1:
        ctx.abort_operation("More than one file: {0} found in sources path:"
                            " {1}".format(filename, CLOUDIFY_SOURCES_PATH))
    local_filepath = ''.join(local_filepath_list[0])
    ctx.logger.debug("File exists in sources path: {0}".format(local_filepath))

    return local_filepath

class RpmPackageHandler(object):

    def __init__(self, source_path):
        self.source_path = source_path

    def remove_existing_rpm_package(self):
        """Removes any version that satisfies the package name of the given
        source path.
        """
        package_name = self.get_rpm_package_name()
        if self.is_package_installed(package_name):
            ctx.logger.debug(
                'Removing existing package sources for package '
                'with name: {0}'.format(package_name))
            sudo(['rpm', '--noscripts', '-e', package_name])

    @staticmethod
    def is_package_installed(name):
        installed = run(['rpm', '-q', name], ignore_failures=True)
        if installed.returncode == 0:
            return True
        return False

    def is_rpm_installed(self):
        """Returns true if provided rpm is already installed.
        """
        src_query = run(['rpm', '-qp', self.source_path])
        source_name = src_query.aggr_stdout.rstrip('\n\r')

        return self.is_package_installed(source_name)

    def get_rpm_package_name(self):
        """Returns the package name according to the info provided in the
        source file.
        """
        split_index = ' : '
        package_details = {}
        package_details_query = run(['rpm', '-qpi', self.source_path])
        rows = package_details_query.aggr_stdout.split('\n')
        # split raw data according to the ' : ' index
        for row in rows:
            if split_index in row:
                first_columb_index = row.index(split_index)
                key = row[:first_columb_index].strip()
                value = row[first_columb_index + len(split_index):].strip()
                package_details[key] = value
        return package_details['Name']

class SystemD(object):

    def systemctl(self, action, service='', retries=0, ignore_failure=False):
        systemctl_cmd = ['systemctl', action]
        if service:
            systemctl_cmd.append(service)
        return sudo(systemctl_cmd, retries=retries,
                    ignore_failures=ignore_failure)

    def configure(self, service_name, render=True, tmpfiles=False):
        """This configures systemd for a specific service.

        It requires that two files are present for each service one containing
        the environment variables and one contains the systemd config.
        All env files will be named "cloudify-SERVICENAME".
        All systemd config files will be named "cloudify-SERVICENAME.service".

        If `tmpfiles` is True, the directory
        `components/{service_name}/tmpfiles.d` must exist and contain the file
        `cloudify-{service_name}.conf`. This will be deployed to
        `/usr/lib/tmpfiles.d`.
        """
        sid = 'cloudify-{0}'.format(service_name)
        env_dst = "/etc/sysconfig/{0}".format(sid)
        srv_dst = "/usr/lib/systemd/system/{0}.service".format(sid)
        env_src = "components/{0}/config/{1}".format(service_name, sid)
        srv_src = "components/{0}/config/{1}.service".format(service_name, sid)

        ctx.logger.debug('Deploying systemd EnvironmentFile...')
        deploy_blueprint_resource(env_src, env_dst, service_name,
                                  render=render)
        ctx.logger.debug('Deploying systemd .service file...')
        deploy_blueprint_resource(srv_src, srv_dst, service_name,
                                  render=render)
        ctx.logger.debug('Enabling systemd .service...')
        self.systemctl('enable', '{0}.service'.format(sid))

        if tmpfiles:
            tmp_dst = "/usr/lib/tmpfiles.d/{0}.conf".format(sid)
            tmp_src = "components/{0}/config/tmpfiles.d/{1}.conf".format(
                service_name, sid)

            ctx.logger.debug('Deploying tmpfiles.d file...')
            deploy_blueprint_resource(
                tmp_src, tmp_dst, service_name, render=render)
            sudo(['systemd-tmpfiles', '--create'])

        self.systemctl('daemon-reload')

    def remove(self, service_name):
        """Stop and disable the service, and then delete its data
        """
        self.stop(service_name, ignore_failure=True)
        self.disable(service_name, ignore_failure=True)
        remove(self.get_service_file_path(service_name))
        remove(self.get_vars_file_path(service_name))

    @staticmethod
    def get_vars_file_path(service_name):
        """Returns the path to a systemd environment variables file
        for a given service_name. (e.g. /etc/sysconfig/cloudify-rabbitmq)
        """
        sid = 'cloudify-{0}'.format(service_name)
        return '/etc/sysconfig/{0}'.format(sid)

    @staticmethod
    def get_service_file_path(service_name):
        """Returns the path to a systemd service file
        for a given service_name.
        (e.g. /usr/lib/systemd/system/cloudify-rabbitmq.service)
        """
        sid = 'cloudify-{0}'.format(service_name)
        return "/usr/lib/systemd/system/{0}.service".format(sid)

    def enable(self, service_name, retries=0, append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Enabling systemd service {0}...'.format(
            full_service_name))
        self.systemctl('enable', full_service_name, retries)

    def disable(self, service_name, retries=0, append_prefix=True,
                ignore_failure=False):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Disabling systemd service {0}...'.format(
            full_service_name))
        self.systemctl('disable', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def start(self, service_name, retries=0, append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Starting systemd service {0}...'.format(
            full_service_name))
        self.systemctl('start', full_service_name, retries)

    def stop(self, service_name, retries=0, append_prefix=True,
             ignore_failure=False):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Stopping systemd service {0}...'.format(
            full_service_name))
        self.systemctl('stop', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def restart(self,
                service_name,
                retries=0,
                ignore_failure=False,
                append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        self.systemctl('restart', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def is_alive(self, service_name, append_prefix=True):
        service_name = self._get_full_service_name(service_name, append_prefix)
        result = self.systemctl('status', service_name, ignore_failure=True)
        return result.returncode == 0

    def verify_alive(self, service_name, append_prefix=True):
        if self.is_alive(service_name, append_prefix):
            ctx.logger.debug('{0} is running'.format(service_name))
        else:
            ctx.abort_operation('{0} is not running'.format(service_name))

    @staticmethod
    def _get_full_service_name(service_name, append_prefix):
        if append_prefix:
            return 'cloudify-{0}'.format(service_name)
        return service_name

systemd = SystemD()

def replace_in_file(this, with_this, in_here):
    """Replaces all occurences of the regex in all matches
    from a file with a specific value.
    """
    ctx.logger.debug('Replacing {0} with {1} in {2}...'.format(
        this, with_this, in_here))
    with open(in_here) as f:
        content = f.read()
    new_content = re.sub(this, with_this, content)
    fd, temp_file = tempfile.mkstemp()
    os.close(fd)
    with open(temp_file, 'w') as f:
        f.write(new_content)
    move(temp_file, in_here)

def get_selinux_state():
    return subprocess.check_output('getenforce').rstrip('\n\r')

def set_selinux_permissive():
    """This sets SELinux to permissive mode both for the current session
    and systemwide.
    """
    ctx.logger.info('Checking whether SELinux in enforced...')
    if 'Enforcing' == get_selinux_state():
        ctx.logger.debug('SELinux is enforcing, setting permissive state...')
        sudo(['setenforce', 'permissive'])
        replace_in_file(
            'SELINUX=enforcing',
            'SELINUX=permissive',
            '/etc/selinux/config')
    else:
        ctx.logger.debug('SELinux is not enforced.')

def get_rabbitmq_endpoint_ip(endpoint=None):
    """Gets the rabbitmq endpoint IP, using the manager IP if the node
    property is blank.
    """
    if endpoint:
        return endpoint
    return ctx.instance.host_ip

def create_service_user(user, group, home):
    """Creates a user.

    It will not create the home dir for it and assume that it already exists.
    This user will only be created if it didn't already exist.
    """
    ctx.logger.info('Checking whether user {0} exists...'.format(user))
    try:
        pwd.getpwnam(user)
        ctx.logger.debug('User {0} already exists...'.format(user))
    except KeyError:
        ctx.logger.info('Creating group {group} if it does not exist'.format(
            group=group,
        ))
        # --force in groupadd causes it to return true if the group exists.
        # Other behaviour changes don't affect this basic use of the command.
        sudo(['groupadd', '--force', group])

        ctx.logger.info('Creating user {0}, home: {1}...'.format(
            user, home))
        sudo([
            'useradd',
            '--shell', '/sbin/nologin',
            '--home-dir', home, '--no-create-home',
            '--system',
            '--no-user-group',
            '--gid', group,
            user,
        ])

def logrotate(service):
    """Deploys a logrotate config for a service.

    Note that this is not idempotent in the sense that if a logrotate
    file is already copied to /etc/logrotate.d, it will copy it again
    and override it. This is done as such so that if a service deploys
    its own logrotate configuration, we will override it.
    """
    if not os.path.isfile('/etc/cron.hourly/logrotate'):
        ctx.logger.info('Deploying logrotate hourly cron job...')
        move('/etc/cron.daily/logrotate', '/etc/cron.hourly/logrotate')

    ctx.logger.debug('Deploying logrotate config...')
    config_file_source = 'components/{0}/config/logrotate'.format(service)
    logrotated_path = '/etc/logrotate.d'
    config_file_destination = os.path.join(logrotated_path, service)
    if not os.path.isdir(logrotated_path):
        os.mkdir(logrotated_path)
        chown('root', 'root', logrotated_path)
    deploy_blueprint_resource(config_file_source,
                              config_file_destination,
                              service)
    chmod('644', config_file_destination)
    chown('root', 'root', config_file_destination)

def remove_logrotate(service_name):
    ctx.logger.debug('Removing logrotate config...')
    logrotated_path = '/etc/logrotate.d'
    config_file_destination = os.path.join(logrotated_path, service_name)
    remove(config_file_destination)

def chmod(mode, path, recursive=False):
    ctx.logger.debug('chmoding {0}: {1}'.format(path, mode))
    command = ['chmod']
    if recursive:
        command.append('-R')
    command += [mode, path]
    sudo(command)

def chown(user, group, path):
    ctx.logger.debug('chowning {0} by {1}:{2}...'.format(
        path, user, group))
    sudo(['chown', '-R', '{0}:{1}'.format(user, group), path])

def ln(source, target, params=None):
    ctx.logger.debug('Linking {0} to {1} with params {2}'.format(
        source, target, params))
    command = ['ln']
    if params:
        command.append(params)
    command.append(source)
    command.append(target)
    if '*' in source or '*' in target:
        sudo(command, globx=True)
    else:
        sudo(command)

def clean_var_log_dir(service):
    pass

def untar(source,
          destination=None,
          skip_old_files=False,
          unique_tmp_dir=False):
    # TODO: use tarfile instead
    if not destination:
        destination = tempfile.mkdtemp() if unique_tmp_dir else '/tmp'
    ctx.logger.debug('Extracting {0} to {1}...'.format(
        source, destination))
    tar_command = ['tar', '-xzvf', source, '-C', destination, '--strip=1']
    if skip_old_files:
        tar_command.append('--skip-old-files')
    sudo(tar_command)
    return destination

def validate_md5_checksum(resource_path, md5_checksum_file_path):
    ctx.logger.info('Validating md5 checksum for {0}'.format(
        resource_path))
    with open(md5_checksum_file_path) as checksum_file:
        original_md5 = checksum_file.read().rstrip('\n\r').split()[0]

    with open(resource_path) as file_to_check:
        data = file_to_check.read()
        # pipe contents of the file through
        md5_returned = hashlib.md5(data).hexdigest()

    if original_md5 == md5_returned:
        return True
    else:
        ctx.logger.error(
            'md5 checksum validation failed! Original checksum: {0} '
            'Calculated checksum: {1}.'.format(original_md5, md5_returned))
        return False

def write_to_json_file(content, file_path):
    tmp_file = tempfile.NamedTemporaryFile(delete=False)
    mkdir(os.path.dirname(os.path.abspath(file_path)))
    with open(tmp_file.name, 'w') as f:
        f.write(json.dumps(content))
    move(tmp_file.name, file_path)

def repetitive(condition_func,
               timeout=15,
               interval=3,
               timeout_msg='timed out',
               *args,
               **kwargs):

    deadline = time.time() + timeout
    while True:
        if time.time() > deadline:
            ctx.abort_operation(timeout_msg)
        if condition_func(*args, **kwargs):
            return
        time.sleep(interval)

class BlueprintResourceFactory(object):

    BASE_RESOURCES_PATH = '/opt/cloudify'
    RESOURCES_DIR_NAME = 'resources'

    def create(self, source, destination, service_name, user_resource=False,
               source_resource=False, render=True, load_ctx=True):
        """A Factory used to create a local copy of a resource upon deployment.
        This copy allows to later reuse the resource for upgrade/rollback
        purposes.

        :param source: The resource source url
        :param destination: Resource destination path
        :param service_name: used to retrieve node properties to be rendered
        into a resource template
        :param user_resource: Resources that should potentially remain
        identical upon upgrade such as custom security configuration files.
        These file will be reused provided the ctx node property:
        use_existing_on_upgrade set to True.
        :param source_resource: Source resources are source packages that
        should be downloaded with no rendering and only archived.
        :param render: Set to false if resource does not require rendering.
        :param load_ctx: Set to false if properties are not available in the
        context of the script.
        :return: The local resource file path and destination.
        """
        resource_name = os.path.basename(destination)

        # The local path is decided according to whether we are in upgrade
        local_resource_path = self._get_local_file_path(service_name,
                                                        resource_name)

        if not os.path.isfile(local_resource_path):
            mkdir(os.path.dirname(local_resource_path))
            if user_resource:
                self._download_user_resource(source,
                                             local_resource_path,
                                             render=render,
                                             load_ctx=load_ctx)
            elif source_resource:
                self._download_source_resource(source,
                                               local_resource_path)
            elif render:
                self._download_resource_and_render(source,
                                                   local_resource_path,
                                                   load_ctx)
            else:
                self._download_resource(source, local_resource_path)
        return local_resource_path, destination

    @staticmethod
    def local_resource_exists(filename):
        try:
            get_filepath_from_pkg_name(filename, raise_if_not_found=True)
            return True
        except IOError:
            return False

    def _download_user_resource(self, source, dest,
                                render=True, load_ctx=True):
        if not os.path.isfile(dest):
            if render:
                self._download_resource_and_render(source, dest, load_ctx)
            else:
                self._download_resource(source, dest)

    @staticmethod
    def _download_resource(source, dest):
        resource_name = os.path.basename(dest)
        ctx.logger.info('Downloading resource {0} to {1}'.format(
            resource_name, dest))
        tmp_file = ctx.download_resource(source)
        move(tmp_file, dest)

    def _download_resource_and_render(self, source, dest, load_ctx):
        resource_name = os.path.basename(dest)
        ctx.logger.debug('Downloading resource {0} to {1}'.format(
            resource_name, dest))
        if load_ctx:
            params = self._get_node_props()
            tmp_file = ctx.download_resource_and_render(source, '', params)
        else:
            # rendering will be possible only for runtime properties
            tmp_file = ctx.download_resource_and_render(source, '')
        move(tmp_file, dest)

    @staticmethod
    def _download_source_resource(source, local_resource_path):
        is_url = source.startswith(('http://', 'https://', 'ftp://',
                                    'file://'))
        filename = get_file_name_from_url(source) if is_url else source
        is_manager_package = filename.startswith(SINGLE_TAR_PREFIX)
        if is_manager_package or is_url:
            local_filepath = os.path.join(CLOUDIFY_SOURCES_PATH, filename)
        else:
            local_filepath = get_filepath_from_pkg_name(filename)

        if is_url:
            if not os.path.isfile(local_filepath):
                tmp_path = download_file(source)
            elif os.path.isfile(local_filepath) and not is_manager_package:
                remove(local_filepath)
                tmp_path = download_file(source)
            else:
                tmp_path = local_filepath
        # source is just the name of the file, to be retrieved from
        # the manager resources package
        else:
            tmp_path = local_filepath
        ctx.logger.debug('Saving {0} under {1}'.format(
            tmp_path, local_resource_path))
        move(tmp_path, local_resource_path)

    @staticmethod
    def _get_node_props():
        node_props = ctx.node.properties.get_all()
        return {'node': {'properties': node_props}}

    def _is_cloudify_pkg(self, filename):
        """Cloudify packages start with 'cloudify' or include '-agent_'

        and end with one of the suffix '.rpm', '.tar.gz', '.tgz', '.exe'.
        The function who calls this function wish to gel all cloudify
        packages except of single tar package.
        """

        if (filename.startswith('cloudify') or
            filename.find('-agent_') != -1) \
                and not filename.startswith(SINGLE_TAR_PREFIX) \
                and filename.endswith(('.rpm', '.tar.gz', '.tgz', '.exe')):
            return True
        else:
            return False

    def _get_local_file_path(self, service_name, resource_name):
        base_service_res_dir = self.get_resources_dir(service_name)

        if self._is_cloudify_pkg(resource_name):
            local_filepath = get_filepath_from_pkg_name(resource_name)
            dest_file_path = os.path.join(base_service_res_dir,
                                          os.path.basename(local_filepath))
        else:
            dest_file_path = os.path.join(base_service_res_dir, resource_name)

        return dest_file_path

    def get_resources_dir(self, service_name):
        return os.path.join(self.BASE_RESOURCES_PATH,
                            service_name,
                            self.RESOURCES_DIR_NAME)

resource_factory = BlueprintResourceFactory()

def start_service(service_name, append_prefix=True):
    systemd.start(service_name, append_prefix=append_prefix)

def http_request(url,
                 data=None,
                 method='PUT',
                 headers=None,
                 timeout=None,
                 should_fail=False):
    headers = headers or {}
    request = urllib2.Request(url, data=data, headers=headers)
    request.get_method = lambda: method
    try:
        if timeout:
            return urllib2.urlopen(request, timeout=timeout)
        return urllib2.urlopen(request)
    except urllib2.URLError as e:
        if not should_fail:
            ctx.logger.error('Failed to {0} {1} (reason: {2})'.format(
                method, url, e.reason))

def get_auth_headers(username, password):
    return {
        'Authorization': 'Basic ' + base64.b64encode('{0}:{1}'.format(
            username, password)
        ),
        'tenant': 'default_tenant'
    }

@retry((IOError, ValueError))
def check_http_response(url, predicate=None, **request_kwargs):
    req = urllib2.Request(url, **request_kwargs)
    try:
        response = urllib2.urlopen(req)
    except urllib2.HTTPError as e:
        # HTTPError can also be used as a non-200 response. Pass this
        # through to the predicate function, so it can decide if a
        # non-200 response is fine or not.
        response = e

    if predicate is not None and not predicate(response):
        raise ValueError('Failed opening url {0} ({1})'.format(url, response))
    return response

def verify_service_http(service_name, url, *args, **kwargs):
    try:
        return check_http_response(url, *args, **kwargs)
    except (IOError, ValueError) as e:
        ctx.abort_operation('{0} error: {1}: {2}'.format(service_name, url, e))

def remove_component(runtime_props):
    service_name = runtime_props['service_name']
    ctx.logger.info('Uninstalling {0}'.format(service_name))

    systemd.remove(service_name)
    remove_notice(service_name)
    remove_logrotate(service_name)

    packages_to_remove = runtime_props.get('packages_to_remove', [])
    for package in packages_to_remove:
        yum_remove(package, ignore_failures=True)

    files_to_remove = runtime_props.get('files_to_remove', [])
    for f in files_to_remove:
        remove(f)

    user = runtime_props.get('service_user')
    if user:
        sudo(['userdel', '--force', user], ignore_failures=True)

    group = runtime_props.get('service_group')
    if group:
        sudo(['groupdel', group], ignore_failures=True)

def extend_runtime_properties_list(runtime_props, key_name, new_list):
    """Extend a list in the runtime properties
    list.extend doesn't call __setitem__, so we need to do it explicitly
    """
    list_to_extend = runtime_props.get(key_name, [])
    list_to_extend.extend(new_list)
    runtime_props[key_name] = list_to_extend

def set_service_as_cloudify_service(runtime_props):
    """Set the cloudify user and group as the user/group of a service"""

    runtime_props['service_user'] = CLOUDIFY_USER
    runtime_props['service_group'] = CLOUDIFY_GROUP

def delete_cluster_component(component):
    """If the given cluster component exists, teardown it."""
    if os.path.exists(CLUSTER_DELETE_SCRIPT):
        sudo(['/usr/bin/env', 'python', CLUSTER_DELETE_SCRIPT,
              '--component', component])

def get_exec_tempdir():
    return os.environ.get(CFY_EXEC_TEMPDIR_ENVVAR) or tempfile.gettempdir()


import os
import sys
import numpy as np
from PIL import Image

from method1 import restore_background

if __name__ == '__main__':
    in_path = '../data/Ourdata_allRawImage/ISI-letters_dataset/Data/'
    out_path = '../data/Ourdata_allRawImage/ISI-letters_dataset/bg'

    in_files = sorted(os.listdir(in_path))
    for i in range(len(in_files)):
        sys.stdout.write('Progress: [{}/{}]\r'.format(i, len(in_files)))

        in_file_name, ext = os.path.splitext(in_files[i])
        out_file = in_file_name+'_bg'+ext

        im_PIL = Image.open(os.path.join(in_path, in_files[i]))
        if im_PIL.mode != 'RGB':
            im_PIL = im_PIL.convert('RGB')

        im_uint8 = np.asarray(im_PIL)
        im = im_uint8/255.0

        bg = restore_background(im, im_PIL)

        bg_PIL = Image.fromarray(np.squeeze(bg).astype('uint8'))
        bg_PIL.save(os.path.join(out_path, out_file))

    print('')