import os
import numpy as np
import argparse
import cv2
import warnings
import torch
import time
try:
    from torch import npu_batch_nms as NMSOp
    NMS_ON_NPU = True
except:
    from torchvision.ops import batched_nms as NMSOp
    NMS_ON_NPU = False

CLASSES = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
           'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
           'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
           'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
           'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
           'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
           'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
           'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
           'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
           'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
           'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop',
           'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
           'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock',
           'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']

def coco_postprocess(bbox, image_size, net_input_width, net_input_height):
    """
    This function is postprocessing for FasterRCNN output.

    Before calling this function, reshape the raw output of FasterRCNN to
    following form
        numpy.ndarray:
            [x, y, width, height, confidence, probability of 80 classes]
        shape: (100,)
    The postprocessing restore the bounding rectangles of FasterRCNN output
    to origin scale and filter with non-maximum suppression.

    :param bbox: a numpy array of the FasterRCNN output
    :param image_path: a string of image path
    :return: three list for best bound, class and score
    """
    w = image_size[0]
    h = image_size[1]
    scale_w = net_input_width / w
    scale_h = net_input_height / h

    # cal predict box on the image src
    pbox = bbox.copy()
    pbox[:, 0] = (bbox[:, 0]) / scale_w
    pbox[:, 1] = (bbox[:, 1]) / scale_h
    pbox[:, 2] = (bbox[:, 2]) / scale_w
    pbox[:, 3] = (bbox[:, 3]) / scale_h
    return pbox

def np_clip_bbox(bboxes, max_shape):
    x1, y1, x2, y2 = bboxes[:, 0], bboxes[:, 1], bboxes[:, 2], bboxes[:, 3]
    h, w = max_shape
    x1 = x1.clip(min=0, max=w)
    y1 = y1.clip(min=0, max=h)
    x2 = x2.clip(min=0, max=w)
    y2 = y2.clip(min=0, max=h)
    bboxes = np.stack([x1, y1, x2, y2], axis=-1)
    return bboxes

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--bin_data_path",
                        default="./result/dumpOutput_device0")
    parser.add_argument("--test_annotation", default="./coco2017_jpg.info")
    parser.add_argument("--det_results_path", default="./detection-results/")
    parser.add_argument("--net_out_num", default=2, type=int)
    parser.add_argument("--num_pred_box", default=8732, type=int)
    parser.add_argument("--nms_pre", default=-1, type=int)
    parser.add_argument("--net_input_width", default=300, type=int)
    parser.add_argument("--net_input_height", default=300, type=int)
    parser.add_argument("--min_bbox_size", default=0.01, type=float)
    parser.add_argument("--score_threshold", default=0.02, type=float)
    parser.add_argument("--nms", default=True, type=bool)
    parser.add_argument("--iou_threshold", default=0.45, type=float)
    parser.add_argument("--max_per_img", default=200, type=int)
    parser.add_argument("--ifShowDetObj", action="store_true",
                        help="if input the para means True, neither False.")
    parser.add_argument("--start", default=0, type=float)
    parser.add_argument("--end", default=1, type=float)
    parser.add_argument("--device", default=0, type=int)
    parser.add_argument("--clear_cache", action='store_true')
    flags = parser.parse_args()
    # generate dict according to annotation file for query resolution
    # load width and height of input images
    img_size_dict = dict()
    with open(flags.test_annotation)as f:
        for line in f:
            temp = line.split(" ")
            img_file_path = temp[1]
            img_name = temp[1].split("/")[-1].split(".")[0]
            img_width = int(temp[2])
            img_height = int(temp[3])
            img_size_dict[img_name] = (img_width, img_height, img_file_path)

    # read bin file for generate predict result
    bin_path = flags.bin_data_path
    det_results_path = flags.det_results_path
    os.makedirs(det_results_path, exist_ok=True)
    total_img = set([name[:name.rfind('_')]
                     for name in os.listdir(bin_path) if "bin" in name])
    total_img = sorted(total_img)
    num_img = len(total_img)
    start = int(flags.start * num_img)
    end = int(flags.end * num_img)
    task_len = end - start + 1

    finished = 0
    time_start = time.time()
    for img_id in range(start, end):
        # for img_id, bin_file in enumerate(sorted(total_img)):
        bin_file = total_img[img_id]
        path_base = os.path.join(bin_path, bin_file)
        det_results_file = os.path.join(det_results_path, bin_file + ".txt")
        if os.path.exists(det_results_file) and not flags.clear_cache:
            continue

        # load all detected output tensor
        bbox_file = path_base + "_" + str(0) + ".bin"
        score_file = path_base + "_" + str(1) + ".bin"
        assert os.path.exists(
            bbox_file), '[ERROR] file `{}` not exist'.format(bbox_file)
        assert os.path.exists(
            score_file), '[ERROR] file `{}` not exist'.format(score_file)
        bboxes = np.fromfile(bbox_file, dtype="float32").reshape(
            flags.num_pred_box, 4)
        scores = np.fromfile(score_file, dtype="float32").reshape(
            flags.num_pred_box, 80)

        bboxes = torch.from_numpy(bboxes)
        scores = torch.from_numpy(scores)
        try:
            bboxes = bboxes.npu(flags.device)
            scores = scores.npu(flags.device)
        except:
            warnings.warn('npu is not available, running on cpu')

        max_scores, _ = scores.max(-1)
        keep_inds = (max_scores > flags.score_threshold).nonzero(
            as_tuple=False).view(-1)
        bboxes = bboxes[keep_inds, :]
        scores = scores[keep_inds, :]

        if flags.nms_pre > 0 and flags.nms_pre < bboxes.shape[0]:
            max_scores, _ = scores.max(-1)
            _, topk_inds = max_scores.topk(flags.nms_pre)
            bboxes = bboxes[topk_inds, :]
            scores = scores[topk_inds, :]

        # clip bbox border
        bboxes[:, 0::2].clamp_(min=0, max=flags.net_input_width - 1)
        bboxes[:, 1::2].clamp_(min=0, max=flags.net_input_height - 1)

        # remove small bbox
        bboxes_width_height = bboxes[:, 2:] - bboxes[:, :2]
        valid_bboxes = bboxes_width_height > flags.min_bbox_size
        keep_inds = (valid_bboxes[:, 0] & valid_bboxes[:, 1]
                     ).nonzero(as_tuple=False).view(-1)
        bboxes = bboxes[keep_inds, :]
        scores = scores[keep_inds, :]

        # rescale bbox to original image size
        original_img_info = img_size_dict[bin_file]
        rescale_factor = torch.tensor([
            original_img_info[0] / flags.net_input_width,
            original_img_info[1] / flags.net_input_height] * 2,
            dtype=bboxes.dtype, device=bboxes.device)
        bboxes *= rescale_factor

        if flags.nms:
            if NMS_ON_NPU:
                # repeat bbox for each class
                # (N, 4) -> (B, N, 80, 4), where B = 1 is the batchsize
                bboxes = bboxes[None, :, None, :].repeat(1, 1, 80, 1)
                # (N, 80) -> (B, N, 80), where B = 1 is the batchsize
                scores = scores[None, :, :]

                # bbox batched nms
                bboxes, scores, labels, num_total_bboxes = \
                    NMSOp(
                        bboxes.half(), scores.half(),
                        score_threshold=flags.score_threshold,
                        iou_threshold=flags.iou_threshold,
                        max_size_per_class=flags.max_per_img,
                        max_total_size=flags.max_per_img)
                bboxes = bboxes[0, :num_total_bboxes, :]
                scores = scores[0, :num_total_bboxes]
                class_idxs = labels[0, :num_total_bboxes]
            else:
                # repeat bbox and class idx for each class
                bboxes = bboxes[:, None, :].repeat(
                    1, 80, 1)  # (N, 4) -> (N, 80, 4)
                class_idxs = torch.arange(80, dtype=torch.long, device=bboxes.device
                                          )[None, :].repeat(bboxes.shape[0], 1)  # (80) -> (N, 80)

                # reshape bbox for torch nms
                bboxes = bboxes.view(-1, 4)
                scores = scores.view(-1)
                class_idxs = class_idxs.view(-1)

                # bbox batched nms
                keep_inds = NMSOp(bboxes, scores, class_idxs,
                                  flags.iou_threshold)
                bboxes = bboxes[keep_inds]
                scores = scores[keep_inds]
                class_idxs = class_idxs[keep_inds]
        else:
            # repeat bbox and class idx for each class
            bboxes = bboxes[:, None, :].repeat(
                1, 80, 1)  # (N, 4) -> (N, 80, 4)
            class_idxs = torch.arange(80, dtype=torch.long, device=bboxes.device
                                      )[None, :].repeat(bboxes.shape[0], 1)  # (80) -> (N, 80)

            # reshape bbox for torch nms
            bboxes = bboxes.view(-1, 4)
            scores = scores.view(-1)
            class_idxs = class_idxs.view(-1)

        # keep topk max_per_img bbox
        if flags.max_per_img > 0 and flags.max_per_img < bboxes.shape[0]:
            _, topk_inds = scores.topk(flags.max_per_img)
            bboxes = bboxes[topk_inds, :]
            scores = scores[topk_inds]
            class_idxs = class_idxs[topk_inds]

        # move to cpu if running on npu
        if bboxes.device != 'cpu':
            bboxes = bboxes.cpu()
            scores = scores.cpu()
            class_idxs = class_idxs.cpu()

        # convert to numpy.ndarray
        bboxes = bboxes.numpy()
        scores = scores.numpy()
        class_idxs = class_idxs.numpy()

        # make det result file
        if flags.ifShowDetObj == True:
            imgCur = cv2.imread(original_img_info[2])

        det_results_str = ''
        for idx in range(bboxes.shape[0]):
            x1, y1, x2, y2 = bboxes[idx, :]
            predscore = scores[idx]
            class_ind = class_idxs[idx]

            class_name = CLASSES[int(class_ind)]
            det_results_str += "{} {} {} {} {} {}\n".format(
                class_name, predscore, x1, y1, x2, y2)
            if flags.ifShowDetObj == True:
                imgCur = cv2.rectangle(imgCur, (int(x1), int(
                    y1)), (int(x2), int(y2)), (0, 255, 0), 1)
                imgCur = cv2.putText(imgCur, class_name + '|' + str(predscore),
                                     (int(x1), int(y1)), cv2.FONT_HERSHEY_SIMPLEX,
                                     0.5, (0, 0, 255), 1)

        if flags.ifShowDetObj == True:
            cv2.imwrite(os.path.join(det_results_path, bin_file +
                                     '.jpg'), imgCur, [int(cv2.IMWRITE_JPEG_QUALITY), 70])

        with open(det_results_file, "w") as detf:
            detf.write(det_results_str)

        finished += 1
        speed = finished / (time.time() - time_start)
        print('processed {:5d}/{:<5d} images, speed: {:.2f}FPS'.format(
            finished, task_len, speed), end='\r')


import time, pytest, inspect
import utils
from utils import run_brave, create_config_file
from PIL import Image

'''
Does the mixer correctly position inputs, so that we can do picture-in-picture or a video-wall?
'''

def test_mixer_positioning(run_brave, create_config_file):
    subtest_start_brave_with_multiple_positioned_inputs(run_brave, create_config_file)
    subtest_move_input3_right()
    subtest_cut_with_position_and_size()
    subtest_cut_by_setting_source_property()

def subtest_start_brave_with_multiple_positioned_inputs(run_brave, create_config_file):
    config = {
        'inputs': [
            {'type': 'test_video', 'pattern': 3}, # white
            {'type': 'test_video', 'pattern': 4}, # red
            {'type': 'test_video', 'pattern': 5}, # green
            {'type': 'test_video', 'pattern': 6}, # blue
        ],
        'mixers': [
            {'width': 160, 'height': 90, 'pattern': 2, 'sources': [
                { 'uid': 'input1', 'zorder': 1 },
                { 'uid': 'input2', 'zorder': 2, 'width': 80 },
                { 'uid': 'input3', 'zorder': 3, 'width': 80, 'height': 45 },
                { 'uid': 'input4', 'zorder': 4, 'xpos': 80, 'ypos': 45 }
            ]}
        ],
        'outputs': [{'type': 'image', 'width': 160, 'height': 90}]
    }
    config_file = create_config_file(config)
    run_brave(config_file.name)
    utils.check_brave_is_running()
    time.sleep(4)
    utils.assert_everything_in_playing_state()
    utils.assert_mixers([{
        'id': 1,
        'sources': [
            {'uid': 'input1', 'in_mix': True, 'zorder': 1},
            {'uid': 'input2', 'in_mix': True, 'zorder': 2, 'width': 80},
            {'uid': 'input3', 'in_mix': True, 'zorder': 3, 'width': 80, 'height': 45},
            {'uid': 'input4', 'in_mix': True, 'zorder': 4, 'xpos': 80, 'ypos': 45}
        ]
    }])

    im = utils.get_raw_image_for_output(1)
    utils.assert_image_color_at_dimension(im, (10,10), (0, 255, 0))
    utils.assert_image_color_at_dimension(im, (150,10), (255, 255, 255))
    utils.assert_image_color_at_dimension(im, (10, 80), (255, 0, 0))
    utils.assert_image_color_at_dimension(im, (150,80), (0, 0, 255))

def subtest_move_input3_right():
    utils.overlay_source('input3', 1, details={'xpos': 80})
    utils.assert_mixers([{
        'id': 1,
        'sources': [
            {'uid': 'input1', 'in_mix': True, 'zorder': 1},
            {'uid': 'input2', 'in_mix': True, 'zorder': 2, 'width': 80},
            {'uid': 'input3', 'in_mix': True, 'zorder': 3, 'xpos': 80, 'width': 80, 'height': 45},
            {'uid': 'input4', 'in_mix': True, 'zorder': 4, 'xpos': 80, 'ypos': 45}
        ]
    }])
    time.sleep(3)
    # Top-right now green, top-left falls back to red
    im = utils.get_raw_image_for_output(1)
    utils.assert_image_color_at_dimension(im, (10,10), (255, 0, 0))
    utils.assert_image_color_at_dimension(im, (150,10), (0, 255, 0))
    utils.assert_image_color_at_dimension(im, (10, 80), (255, 0, 0))
    utils.assert_image_color_at_dimension(im, (150,80), (0, 0, 255))

def subtest_cut_with_position_and_size():
    # 'Cut' rather than 'overlay' will remove all other sources
    # 'width' and 'height' are bigger than the mixer
    # xpos not being set, so should retain the value from earlier.
    utils.cut_to_source('input4', 1, details={'ypos': 0, 'width': 300, 'height': 300})
    time.sleep(0.5)
    utils.assert_mixers([{
        'id': 1,
        'sources': [
            {'uid': 'input1', 'in_mix': False, 'zorder': 1},
            {'uid': 'input2', 'in_mix': False, 'zorder': 2, 'width': 80},
            {'uid': 'input3', 'in_mix': False, 'zorder': 3, 'xpos': 80, 'width': 80, 'height': 45},
            {'uid': 'input4', 'in_mix': True, 'zorder': 4, 'xpos': 80, 'ypos': 0, 'width': 300, 'height': 300}
        ]
    }])

    time.sleep(3)
    # Right-hand-side blue, left-hand-side black
    im = utils.get_raw_image_for_output(1)
    utils.assert_image_color_at_dimension(im, (10,10), (0, 0, 0))
    utils.assert_image_color_at_dimension(im, (150,10), (0, 0, 255))
    utils.assert_image_color_at_dimension(im, (10, 80), (0, 0, 0))
    utils.assert_image_color_at_dimension(im, (150,80), (0, 0, 255))

def subtest_cut_by_setting_source_property():
    utils.update_mixer(1, {'sources': [
        {'uid': 'input1'},
        {'uid': 'input4', 'xpos': 0, 'height': 45}
    ]})
    time.sleep(0.5)
    utils.assert_mixers([{
        'id': 1,
        'sources': [
            {'uid': 'input1', 'in_mix': True, 'zorder': 1},
            {'uid': 'input2', 'in_mix': False, 'zorder': 2, 'width': 80},
            {'uid': 'input3', 'in_mix': False, 'zorder': 3, 'xpos': 80, 'width': 80, 'height': 45},
            {'uid': 'input4', 'in_mix': True, 'zorder': 4, 'xpos': 0, 'ypos': 0, 'width': 300, 'height': 45}
        ]
    }])

    time.sleep(3)
    # Top blue, bottom white
    im = utils.get_raw_image_for_output(1)
    utils.assert_image_color_at_dimension(im, (10,10), (0, 0, 255))
    utils.assert_image_color_at_dimension(im, (150,10), (0, 0, 255))
    utils.assert_image_color_at_dimension(im, (10, 80), (255, 255, 255))
    utils.assert_image_color_at_dimension(im, (150,80), (255, 255, 255))


import os
import sys
import time

from PIL import Image, ImageDraw, ImageFont

from cuars import faces
import adargb

io = adargb.BoardIO()
display = adargb.Display(135, 240)

table = faces.Table(240, 135, os.listdir(os.getcwd()))
font = ImageFont.truetype("fonts/BebasNeue.otf", 22)
image = Image.new("RGB", (240, 135))
draw = ImageDraw.Draw(image)

def main():
    print("Interface should appear on the RGB display now")
    while True:
        # Display image.
        for badge in table.badges:
            rect = (badge.x, badge.y,
                    badge.x + badge.width, badge.y + badge.height)
            bgcolor, color = badge.bgcolor, badge.color
            name = badge.name
            draw.rectangle(rect, outline=bgcolor, fill=bgcolor)
            draw.text(rect, name, font=font, fill=color)
        display.show(image, 270)
        # Accept input
        if not io.B.value and not io.A.value:  # both buttons pressed
            print("restart")
        elif io.B.value and not io.A.value:  # button 1 pressed
            print("select")
        elif io.A.value and not io.B.value:  # button 2 pressed
            print("start")
        # Wait for refresh
        time.sleep(0.1)

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        print("Keyboard Interrupt (Control-C)...")
    io.backlight.value = not io.backlight.value
    sys.exit()


import os
import sys
import datetime
import xarray as xr
import pismx.open

# processed runs
PROC_RUNS = ['alpcyc4.2km.grip.0820', 'alpcyc4.2km.grip.1040.pp',
             'alpcyc4.2km.epica.0970', 'alpcyc4.2km.epica.1220.pp',
             'alpcyc4.2km.md012444.0800', 'alpcyc4.2km.md012444.1060.pp',
             'alpcyc4.1km.epica.1220.pp']

# global attributes
GLOB_ATTRS = dict(
    title='Alpine ice sheet deglaciation aggregated variables',
    author='Julien Seguinot',
    institution='ETH Zürich, Switzerland',
    command='{user}@{host} {time}: {cmdl}\n'.format(
        user=os.getlogin(), host=os.uname()[1], cmdl=' '.join(sys.argv),
        time=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')),
    comment="""Aggregated dataset contents:
* Spatial variables aggregated in time corresponding to the last model output
before final deglaciation.
""")

def postprocess_extra(run_path):
    """Postprocess extra dataset for one run."""

    # output file and subtitle
    _, res, rec, *other = os.path.basename(run_path).split('.')
    out_file = 'processed/alpdeg.{}.{}.{}.agg.nc'.format(
        res, rec[:4], 'pp' if 'pp' in other else 'cp')

    # load output data (in the future combine='by_coords' will be the default)
    print("postprocessing " + out_file + "...")
    boot = pismx.open.dataset(
        '~/pism/input/boot/'+'alps.srtm.hus12.gou11simi.{}.nc'.format(res))
    ex = pismx.open.mfdataset(run_path+'/ex.???????.nc')

    # init postprocessed dataset with global attributes
    pp = xr.Dataset(attrs=ex.attrs, coords=dict(lon=ex.lon, lat=ex.lat))
    pp.attrs.update(
        subtitle='{} {} simulation {} precipitation reductions'.format(
            res, rec.upper(), 'with' if 'pp' in other else 'without'),
        **GLOB_ATTRS)
    pp.attrs.update(history=pp.command+pp.history)

    # compute grid size
    dt = ex.age[0] - ex.age[1]
    dt = dt*1e3  # convert ka to a

    # compute last ice cover transgressive variables
    idx = (ex.thk >= 1)[::-1].argmax(axis=0).compute()
    pp['deglacage'] = (ex.age[-idx].where(idx > 0)).assign_attrs(
        long_name='last ice cover age', units=ex.age.units)

    # compute last basal velocity transgressive variables
    # compute index first (xarray indexing with dask array issue #2511)
    idx = ((ex.thk >= 1)*(ex.velbase_mag >= 1))[::-1].argmax(axis=0).compute()
    pp['lastslipa'] = ex.age[-idx].where(idx > 0).assign_attrs(
        long_name='last basal velocity age', units=ex.age.units)
    pp['lastslipu'] = ex.uvelbase[-idx].where(idx > 0).assign_attrs(
        long_name='last basal velocity x-component', units=ex.uvelbase.units)
    pp['lastslipv'] = ex.vvelbase[-idx].where(idx > 0).assign_attrs(
        long_name='last basal velocity y-component', units=ex.vvelbase.units)
    pp.drop('time')

    # export to netcdf
    pp.to_netcdf(out_file, mode='w', encoding={var: dict(
        zlib=True, shuffle=True, complevel=1) for var in pp.variables})

    # close datasets
    boot.close()
    ex.close()
    pp.close()

def main():
    """Main program called during execution."""

    # create directory if missing
    if not os.path.exists('processed'):
        os.makedirs('processed')

    # activate dask client http://localhost:8787/status
    # from dask.distributed import Client
    # print(Client().scheduler_info()['services'])

    # postprocess selected runs
    for run in PROC_RUNS:
        postprocess_extra(os.environ['HOME'] + '/pism/output/e9d2d1f/' + run)

if __name__ == '__main__':
    main()


import torch
import torch.nn.functional as F
from mmcv.cnn import ConvModule, Scale
from torch import nn

from torch.nn import init#
import functools#
from torch.autograd import Variable#
import numpy as np#

from mmseg.core import add_prefix
from ..builder import HEADS
from ..utils import SelfAttentionBlock as _SelfAttentionBlock
from .decode_head import BaseDecodeHead
from mmseg.ops import resize

class PAM(_SelfAttentionBlock):
    """Position Attention Module (PAM)
    Args:
        in_channels (int): Input channels of key//query feature.
        channels (int): Output channels of key//query transform.
    """

    def __init__(self, in_channels, channels):
        super(PAM, self).__init__(
            key_in_channels=in_channels,
            query_in_channels=in_channels,
            channels=channels,
            out_channels=in_channels,
            share_key_query=False,
            query_downsample=None,
            key_downsample=None,
            key_query_num_convs=1,
            key_query_norm=False,
            value_out_num_convs=1,
            value_out_norm=False,
            matmul_norm=False,
            with_out=False,
            conv_cfg=None,
            norm_cfg=None,
            act_cfg=None)

        self.gamma = Scale(0)## This layer scales the input by a learnable factor. It multiplies a learnable scale parameter of shape (1,) with input of any shape. 就是里面的α

    def forward(self, x):
        """Forward function."""
        out = super(PAM, self).forward(x, x)

        out = self.gamma(out) + x
        return out

class ResidualBlock(nn.Module):
    def __init__(self, input_channels, output_channels, stride=1):
        super(ResidualBlock, self).__init__()
        self.input_channels = input_channels
        self.output_channels = output_channels
        self.stride = stride
        self.conv1 = nn.Conv2d(input_channels, output_channels//4, 1, 1, bias = False)
        self.bn1 = nn.BatchNorm2d(output_channels//4)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(output_channels//4, output_channels//4, 3, stride, padding = 1, bias = False)
        self.bn2 = nn.BatchNorm2d(output_channels//4)
        self.relu = nn.ReLU(inplace=True)
        self.conv3 = nn.Conv2d(output_channels//4, output_channels, 1, 1, bias = False)
        self.bn3 = nn.BatchNorm2d(output_channels)
        self.relu = nn.ReLU(inplace=True)
        self.conv4 = nn.Conv2d(input_channels, output_channels , 1, stride, bias = False)

    def forward(self, x):
        residual = x
        out_c1 = self.conv1(x)
        out_b1 = self.bn1(out_c1)
        out_r1 = self.relu(out_b1)
        out_c2 = self.conv2(out_r1)
        out_b2 = self.bn2(out_c2)
        out_r2 = self.relu(out_b2)
        out_c3 = self.conv3(out_r2)
        out_b3 = self.bn3(out_c3)
        out_r3 = self.relu(out_b3)
        if (self.input_channels != self.output_channels) or (self.stride !=1 ):
            residual = self.conv4(out_b1)
        out = out_r3+residual
        return out

class AttentionModule(nn.Module):##in和out改成一样的，size变成具体的数
    def __init__(self, in_channels, height, width):
        super(AttentionModule, self).__init__()

        self.first_residual_blocks = ResidualBlock(in_channels, in_channels)

        self.mpool1 = nn.AdaptiveMaxPool2d((height//2,width//2))

        self.softmax1_blocks = ResidualBlock(in_channels, in_channels)

        self.skip1_connection_residual_block = ResidualBlock(in_channels, in_channels)

        self.mpool2 = nn.AdaptiveMaxPool2d((height//4,width//4))

        self.softmax2_blocks = ResidualBlock(in_channels, in_channels)

        self.skip2_connection_residual_block = ResidualBlock(in_channels, in_channels)

        self.mpool3 = nn.AdaptiveMaxPool2d((height//8,width//8))

        self.softmax3_blocks = nn.Sequential(
            ResidualBlock(in_channels, in_channels),
            ResidualBlock(in_channels, in_channels)
        )

        self.interpolation3 = nn.UpsamplingBilinear2d(size=(height//4,width//4))

        self.softmax4_blocks = ResidualBlock(in_channels, in_channels)

        self.interpolation2 = nn.UpsamplingBilinear2d(size=(height//2,width//2))

        self.softmax5_blocks = ResidualBlock(in_channels, in_channels)

        self.interpolation1 = nn.UpsamplingBilinear2d(size=(height,width))

        '''self.softmax6_blocks = nn.Sequential(
            nn.BatchNorm2d(in_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(in_channels, in_channels , kernel_size = 1, stride = 1, bias = False),
            nn.BatchNorm2d(in_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(in_channels, in_channels , kernel_size = 1, stride = 1, bias = False),
            nn.Sigmoid()
        )'''

        self.last_blocks = ResidualBlock(in_channels, in_channels)

    def forward(self, x):
        x = self.first_residual_blocks(x)
        #print(x.shape)
        out_mpool1 =  self.mpool1(x)
        #print('out_mpool1',out_mpool1.shape)
        out_softmax1 = self.softmax1_blocks(out_mpool1)
        #print('out_softmax1',out_softmax1.shape)
        out_skip1_connection = self.skip1_connection_residual_block(out_softmax1)#
        #print('out_skip1_connection',out_skip1_connection.shape)
        out_mpool2 = self.mpool2(out_softmax1)
        #print('out_mpool2',out_mpool2.shape)
        out_softmax2 = self.softmax2_blocks(out_mpool2)
        #print('out_softmax2',out_softmax2.shape)
        out_skip2_connection = self.skip2_connection_residual_block(out_softmax2)#
        #print('out_skip2_connection',out_skip2_connection.shape)
        out_mpool3 = self.mpool3(out_softmax2)
        #print('out_mpool3.shape',out_mpool3.shape)
        out_softmax3 = self.softmax3_blocks(out_mpool3)
        #print('out_softmax3.shape',out_softmax3.shape)
        out_interp3 = self.interpolation3(out_softmax3)
        #print('out_skip2_connection.shape',out_skip2_connection.shape)
        #print('out_interp3.shape',out_interp3.shape)
        out = out_interp3 + out_skip2_connection#
        out_softmax4 = self.softmax4_blocks(out)
        out_interp2 = self.interpolation2(out_softmax4)
        out1 = out_interp2 + out_skip1_connection#
        out_softmax5 = self.softmax5_blocks(out1)
        out_interp1 = self.interpolation1(out_softmax5)
        out2 = self.last_blocks(out_interp1)
        #out_softmax6 = self.softmax6_blocks(out_interp1)###attention的输出
        return out2

class CAM(nn.Module):
    """Channel Attention Module (CAM)"""

    def __init__(self, in_channels, conv_cfg, norm_cfg, act_cfg):##
        super(CAM, self).__init__()
        self.gamma = Scale(0)##This layer scales the input by a learnable factor. It multiplies a learnable scale parameter of shape (1,) with input of any shape. 就是里面的β
        self.in_channels = in_channels
        self.conv_cfg = conv_cfg
        self.norm_cfg = norm_cfg
        self.act_cfg = act_cfg
        self.input_redu_conv = ConvModule(##
            self.in_channels,
            self.in_channels,
            1,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)

        self.attention_module = AttentionModule(self.in_channels, 56, 56)

    def forward(self, x):
        """Forward function."""
        batch_size, channels, height, width = x.size()#C*H*W###?
        y = self.input_redu_conv(x)###

        z = self.attention_module(x)###
        a = resize(z, size=(height, width))###
        proj_query = y.view(batch_size, channels, -1) ###batch_size1 *C*(hw)=batch_size1*C*N
        proj_key = a.view(batch_size, channels, -1).permute(0, 2, 1)##batch_size1*(hw)*C
        energy = torch.bmm(proj_query, proj_key)#3维矩阵乘法，如size（2，2，5）的cc和size（2，5，2）dd，torch.bmm(cc,dd)后size=（2，2，2）。这里batch_size*C*C
        energy_new = torch.max(
            energy, -1, keepdim=True)[0].expand_as(energy) - energy#torch.max()[0]， 只返回最大值的每个数,troch.max()[1]， 只返回最大值的每个索引
        attention = F.softmax(energy_new, dim=-1)
        proj_value = x.view(batch_size, channels, -1)

        out = torch.bmm(attention, proj_value)
        out1 = out.view(batch_size, channels, height, width)

        out2 = self.gamma(out1) + x
        return out2

@HEADS.register_module()
class REDAHead(BaseDecodeHead):
    """Dual Attention Network for Scene Segmentation.
    This head is the implementation of `DANet
    <https:////arxiv.org//abs//1809.02983>`_.
    Args:
        pam_channels (int): The channels of Position Attention Module(PAM).
    """

    def __init__(self, pam_channels, **kwargs):###
        super(REDAHead, self).__init__(**kwargs)
        self.pam_channels = pam_channels
        self.pam_in_conv = ConvModule(
            self.in_channels,
            self.channels,
            3,
            padding=1,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)
        self.pam = PAM(self.channels, pam_channels)
        self.pam_out_conv = ConvModule(
            self.channels,
            self.channels,
            3,
            padding=1,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)
        self.pam_conv_seg = nn.Conv2d(
            self.channels, self.num_classes, kernel_size=1)

        self.cam_in_conv = ConvModule(
            self.in_channels,
            self.channels,
            3,
            padding=1,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)
        self.cam = CAM(
            self.channels,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)##
        self.cam_out_conv = ConvModule(
            self.channels,
            self.channels,
            3,
            padding=1,
            conv_cfg=self.conv_cfg,
            norm_cfg=self.norm_cfg,
            act_cfg=self.act_cfg)
        self.cam_conv_seg = nn.Conv2d(
            self.channels, self.num_classes, kernel_size=1)

    def pam_cls_seg(self, feat):
        """PAM feature classification."""
        if self.dropout is not None:
            feat = self.dropout(feat)
        output = self.pam_conv_seg(feat)
        return output

    def cam_cls_seg(self, feat):
        """CAM feature classification."""
        if self.dropout is not None:
            feat = self.dropout(feat)
        output = self.cam_conv_seg(feat)
        return output

    def forward(self, inputs):
        """Forward function."""
        x = self._transform_inputs(inputs)
        pam_feat = self.pam_in_conv(x)
        pam_feat = self.pam(pam_feat)
        pam_feat = self.pam_out_conv(pam_feat)
        pam_out = self.pam_cls_seg(pam_feat)

        cam_feat = self.cam_in_conv(x)
        cam_feat = self.cam(cam_feat)
        cam_feat = self.cam_out_conv(cam_feat)
        cam_out = self.cam_cls_seg(cam_feat)

        feat_sum = pam_feat + cam_feat
        pam_cam_out = self.cls_seg(feat_sum)

        return pam_cam_out, pam_out, cam_out

    def forward_test(self, inputs, img_metas, test_cfg):
        """Forward function for testing, only ``pam_cam`` is used."""
        return self.forward(inputs)[0]

    def losses(self, seg_logit, seg_label):
        """Compute ``pam_cam``, ``pam``, ``cam`` loss."""
        pam_cam_seg_logit, pam_seg_logit, cam_seg_logit = seg_logit
        loss = dict()
        loss.update(
            add_prefix(
                super(REDAHead, self).losses(pam_cam_seg_logit, seg_label),
                'pam_cam'))
        loss.update(
            add_prefix(
                super(REDAHead, self).losses(pam_seg_logit, seg_label), 'pam'))
        loss.update(
            add_prefix(
                super(REDAHead, self).losses(cam_seg_logit, seg_label), 'cam'))
        return loss


# -*- coding: utf-8 -*-
# This program is developed with python 3.x
# The following two types of input are avaliable:
#     1. Directory path of VPRN .txt files
#     2. the List of VPRN .txt files

#from PyQt5 import QtCore, QtGui, QtWidgets

import datetime
import re
import os

# workbook
from  openpyxl.workbook  import  Workbook
# ExcelWriter
from  openpyxl.writer.excel  import  ExcelWriter
# convert number to column alpha-belta
# from  openpyxl.cell  import  get_column_letter

#class extractVPRNCfg(QtCore.QThread):
class extractVPRNCfg():
    """ class extractVPRNCfg """

    # VPRN Patterns
    ptnVPRN             = re.compile(r'\s{8}vprn\s*(\S*)')
    ptnVPRNDescription  = re.compile(r'\s{12}description\s*"(.*)"')
    ptnVPRNAutonomous   = re.compile(r'\s{12}autonomous-system\s*(\S*)')
    ptnVPRNRouteDist    = re.compile(r'\s{12}route-distinguisher\s*(\S*)')
    ptnInterface        = re.compile(r'\s{12}interface\s*"(.*)"')
    ptnIFDescription    = re.compile(r'\s{16}description\s*"(.*)"')
    ptnIFAddress        = re.compile(r'\s{16}address\s*(\S*)')
    ptnIFSecondary      = re.compile(r'\s{16}secondary\s*(\S*)')
    ptnIFSap            = re.compile(r'\s{16}sap\s*(\S*)')
    ptnIFIngress        = re.compile(r'\s{20}ingress')
    ptnIFEgress         = re.compile(r'\s{20}egress')
    ptnIFQos            = re.compile(r'\s{24}qos\s*(\S*)')
    ptnIFIngressEnd     = re.compile(r'\s{20}exit')
    ptnIFEgressEnd      = re.compile(r'\s{20}exit')
    ptnInterfaceEnd     = re.compile(r'\s{12}exit')
    ptnVPRNEnd          = re.compile(r'\s{8}exit')
    #ptnVPRNInstance      = re.compile(r'vpn-instance\s*(\S*)')
    #ptnIPAddress        = re.compile(r'ip address\s*(\S.*)')

    vprnPtnDict = { 'description' : ptnVPRNDescription, \
                     'route-distinguisher' : ptnVPRNRouteDist, \
                     'autonomous-system' : ptnVPRNAutonomous, \
                     'interface' : ptnInterface }
    ifPtnDict = { 'description' : ptnIFDescription, \
                    'address' : ptnIFAddress, \
                    'secondary' : ptnIFSecondary, \
                    'sap' : ptnIFSap, \
                    'qos' : ptnIFQos, \
                    'ingress' : ptnIFIngress, \
                    'egress' : ptnIFEgress }

    # signal
    # sigProcessFiles = QtCore.pyqtSignal(str)
    # sigRecord = QtCore.pyqtSignal(str)

#    sigRecordClear = QtCore.pyqtSignal()
#    sigLineProcessed = QtCore.pyqtSignal(int)
#    sigTotalLineNum = QtCore.pyqtSignal(int)
#    sigStartTimer = QtCore.pyqtSignal()
#    sigStopTimer = QtCore.pyqtSignal()

    def __init__(self):
        super(extractVPRNCfg, self).__init__()
        self.path = None

    def loadPara(self, path, outFile = None):
        self.path = path
        if outFile is None:
            self.outFile = self.genOutFilename()
        else:
            self.outFile = outFile
        print(self.outFile)
        self.workbook = Workbook()
        # self.excelWriter = ExcelWriter(self.workbook)
        # self.worksheet = self.workbook.create_sheet(title="VPRN")
        self.worksheet = self.workbook.active
        self.worksheet.title = 'VPRN'
        self.worksheet.append(['filename', 'vprn', 'description', 'autonomous-system', 'route-distinguisher', \
                'interface:name', 'interface:description', 'interface:description header', 'interface:address', \
                'interface:secondary address', 'interface:sap', 'interface:ingress qos', 'interface:egress qos'])

    def genOutFilename(self):
        t = datetime.datetime.now()
        s = datetime.datetime.strftime(t, '%Y%m%d_%H%M%S')
        filename = 'VPRN_' + s + '.xlsx'
        if type(self.path) is str: # input is a directory string
            path = self.path
        elif type(self.path) is list: # input is a file list
            path = os.path.split(self.path[0])[0]
        return os.path.join(path, filename)

    def run(self):
        if self.path is None:
            return
        if type(self.path) is str: # input is a directory string
            self.directoryExtract(self.path)
        elif type(self.path) is list: # input is a file list
            for fn in self.path:
                self.fileExtract(fn)
            #if os.path.exists(self.path):
        # save the output file
        self.saveOutFile()

    def directoryExtract(self, dn):
        if os.path.isdir(dn):
            for parent, dirnames, filenames in os.walk(dn):
                for filename in filenames:
                    fn = os.path.join(parent, filename)
                    self.fileExtract(fn)

    def fileExtract(self, fn):
        # check if file is *.txt
        if os.path.splitext(fn)[-1] != '.txt':
            return

        fnBase = os.path.split(os.path.splitext(fn)[0])[-1]
        print(fn)
        print(fnBase)
        # start analyse
        isVPRNStart = False
        isInterfaceStart = False # a VPRN may contain several Interfaces.
        item = {}
        with open(fn, 'r') as fp:
            for line in fp:
                #line = line.strip()
                if isVPRNStart:
                    # reach the end of the instance
                    if self.ptnVPRNEnd.match(line):
                        isVPRNStart = False
                    else:
                        if isInterfaceStart:
                            if self.ptnInterfaceEnd.match(line):
                                self.saveItem(item)
                                isInterfaceStart = False
                            else:
                                # match interface param
                                for title, ptn in self.ifPtnDict.items():
                                        result = ptn.match(line)
                                        if result:
                                            if title == 'secondary':
                                                if item['interface'].get(title):
                                                    pass
                                                else:
                                                    item['interface'][title] = []
                                                item['interface'][title].append(result.group(1))
                                            else:
                                                if title == 'ingress' or title == 'egress':
                                                    xgress = title
                                                    break
                                                elif title == 'qos':
                                                    title = xgress
                                                # debug start
                                                if item['interface'].get(title):
                                                    print(title, 'exists')
                                                # debug end
                                                item['interface'][title] = result.group(1)
                                            break
                        else:
                            #match interface internal parameters
                            for title, ptn in self.vprnPtnDict.items():
                                result = ptn.match(line)
                                if result:
                                    if title == 'interface':
                                        item[title] = {'name': result.group(1)}
                                        isInterfaceStart = True
                                    else:
                                        item[title] = result.group(1)
                                    break

                else: # interface not started, find the start
                    result = self.ptnVPRN.match(line)
                    if result:
                        item = {}
                        item['vprn'] = result.group(1)
                        item['filename'] = fnBase
                        isVPRNStart = True

    def saveOutFile(self):
        self.workbook.save(self.outFile)

    def saveItem(self, item):
        if self.checkAndFormat(item):
            self.writeItemToExcel(item)
        else:
            pass

    def checkAndFormat(self, item): # TODO
        # interface name must contains '.'
        if len(item.get('interface', '')) <= 1:
            return False
        # format interface description
        interfaceDescription = item['interface'].get('description')
        if interfaceDescription:
            try:
                result = re.match(r'\w\d{7}', interfaceDescription.split()[0])
                if result:
                    item['interface']['description header'] = result.group(0)
            except Exception as inst:
                print(inst)
                pass
        # format secondary ip address
        ipList = item['interface'].get('secondary')
        if ipList is not None:
            item['interface']['secondary'] = ' '.join(ipList)
        return True

    def writeItemToExcel(self, item):
        self.worksheet.append( [\
                                item.get('filename'), \
                                item.get('vprn'), \
                                item.get('description', ''), \
                                item.get('autonomous-system', ''), \
                                item.get('route-distinguisher', ''), \
                                item['interface'].get('name', ''), \
                                item['interface'].get('description', ''), \
                                item['interface'].get('description header', ''), \
                                item['interface'].get('address', ''), \
                                item['interface'].get('secondary', ''), \
                                item['interface'].get('sap', ''), \
                                item['interface'].get('ingress', ''), \
                                item['interface'].get('egress', '')]
                                )
        #, 'ip-address sub')
        pass

    def fRecord(self, s):
        #self.sigRecord.emit(s)
        print(s)

    def fNowPastTimeStr(self):
        delta = datetime.datetime.now() - self.tAnalyseStartTime
        return '%d.%d seconds' % (delta.seconds, delta.microseconds)

    def fGetFileLines(self, fn):
        i = -1
        with open(fn) as f:
            for i, l in enumerate(f):
                pass
        return i + 1

    def fSummary(self):
        s = []
        s.append( 'Summary:' )
        self.fRecord('\r\n'.join(s))

    def fStartTimer(self):
        self.sigStartTimer.emit()

    def fTimerEvent(self):
        self.sigLineProcessed.emit(self.lineProcess)

    def fStopTimer(self):
        self.sigStopTimer.emit()
        self.sigLineProcessed.emit(self.lineProcess)

if __name__ == "__main__":
    fn = input('Input file:')
    app = extractVPRNCfg()
    #app.sigRecord.connect(print)
    if os.path.isdir(fn):
        inParam = fn
    else:
        inParam = [fn]
    app.loadPara(inParam)
    app.run()
    input('')


import torch
from load import load_data
from dl import Bat_Net
from lr import LR
import numpy as np

def test():
    train_features, train_labels, test_features, test_labels = load_data()

    # 使用单一决策树建模
    from sklearn.tree import DecisionTreeClassifier
    tree = DecisionTreeClassifier(criterion='entropy', random_state=1, max_depth=1)
    from sklearn.metrics import accuracy_score
    tree = tree.fit(train_features, train_labels)
    y_train_pred = tree.predict(train_features)
    y_test_pred = tree.predict(test_features)
    tree_train = accuracy_score(train_labels, y_train_pred)
    tree_test = accuracy_score(test_labels, y_test_pred)
    print('Decision tree train/test accuracies %.3f/%.3f' % (tree_train, tree_test))

    if torch.cuda.is_available():
        test_features = test_features.cuda()

    lr = torch.load('./best_lr.pkl')
    y_pred = lr(test_features)
    mask_lr = y_pred.ge(0.5).float().squeeze()  # 以0.5为阈值进行分类

    dl = torch.load('./best_dl.pkl')
    y_pred = dl(test_features)
    mask_dl = y_pred.ge(0.5).float().squeeze()  # 以0.5为阈值进行分类
    mask = []
    for i in range(len(mask_dl)):
        if 0.3 * y_test_pred[i] + 0.3 * mask_dl[i] + 0.4 * mask_lr[i] >= 0.5:
            mask.append(1)
        else:
            mask.append(0)
    mask = np.array(mask)
    correct = 0
    for i in range(len(mask)):
        if mask[i] == test_labels[i]:
            correct += 1
    # correct = ((mask == test_labels).sum())  # 计算正确预测的样本个数
    acc = correct / test_labels.size(0)  # 计算分类准确率
    print(acc)

test()


__license__ = "MIT"
__author__ = "Augusto Cunha, Axelle Pochet"
__email__ = "acunha@tecgraf.puc-rio.br, axelle@tecgraf.puc-rio.br"
__credits__ = ["Augusto Cunha", "Axelle Pochet", "Helio Lopes", "Marcelo Gattass"]

from tensorflow.keras import backend as K
# K.set_image_dim_ordering('tf')
# K.set_image_data_format('channels_last')

import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
K.set_session(tf.Session(config=config))

import gc
import cv2, os, numpy, sys
import pandas as pd
import multiprocessing
import time

from tensorflow.keras.models import model_from_json, Sequential
from joblib import Parallel, delayed

numpy.random.seed(1337)

# Reset Keras Session
def reset_keras(base_model, model):
    sess = K.get_session()
    K.clear_session()
    sess.close()
    sess = K.get_session()

    try:
        del base_model
        del model
    except:
        pass

    print(gc.collect()) # if it's done something you should see a number being outputted

    # use the same config as you used to create the session
    config = tf.ConfigProto()
    config.gpu_options.allow_growth=True
    #config.gpu_options.visible_device_list = "0"
    K.set_session(tf.Session(config=config))

def processPatches(data, patch_size, pixel_step, resize, nb_channels):
    # get data
    if isinstance(data, pd.DataFrame):
        section_mat = data.values
    else:
        section_mat = data
    half_patch = int(patch_size/2)

    # get image info
    nb_rows = data.shape[0]
    nb_cols = data.shape[1]
    #print(nb_rows)
    #print(nb_cols)

    count_patches = 0
    patch_name_list = []
    patch_list = []
    for i in range (half_patch, nb_rows - half_patch, pixel_step):
        for j in range (half_patch, nb_cols - half_patch, pixel_step):
            # create patch
            start_row = i - half_patch
            start_col = j - half_patch
            patch =  numpy.zeros((patch_size,patch_size)) # 1 empty patch
            for x in range(patch_size):
                for y in range(patch_size):
                    patch[x][y] = section_mat[start_row + x][start_col + y]
            # resize, clip
            patch = cv2.resize(patch, dsize=(resize, resize), interpolation=cv2.INTER_CUBIC)
            patch = numpy.clip(patch, -1., 1.)
            # append to global list
            patch_list.append(patch)
            patch_name = 'patch_p_' + str(i) + '_' + str(j) + '.csv'
            patch_name_list.append(patch_name)
            # count
            count_patches +=1

    return patch_list, patch_name_list

def classify(input_dir, patch_size, resize_size, pixel_step, jsonModelFilePath, weightsFilePath, modelName):
    start_time = time.time()

    # set params
    resize = resize_size
    imageChannels = 1

    # create output directory
    directory = "output/classification/"
    if not os.path.exists(directory):
        os.makedirs(directory)

    # read all files in 1 step
    df_list = []
    df_names = []
    files = os.listdir(input_dir)
    for i in range(0, len(files)):
        filename = files[i]
        section_name = filename.split('_')[0]
        df = pd.read_csv(input_dir + filename, delimiter=' ', header = None)
        df_list.append(df)
        df_names.append(modelName + "_" + section_name)

    # load model
    jsonModelFile = open(jsonModelFilePath, 'r' )
    base_model = jsonModelFile.read()
    jsonModelFile.close()
    model = model_from_json(base_model)
    model.load_weights(weightsFilePath)
    model.compile( loss='binary_crossentropy', optimizer='sgd', metrics=[ 'accuracy' ] )

    # prepare save prediction for all sections
    nb_sections = len(df_list)

    # create patches in parrallel
    s = 0
    nb_section = len(df_list)
    section_step = 4
    while 1 :
        s_init = s

        if(s == nb_section):
            break

        if(s+section_step > len(df_list)):
            df_sub_list = df_list[s:len(df_list)]
            s = nb_section
        else:
            df_sub_list = df_list[s:s+section_step]
            s = s+section_step

        print("Creating patches...")
        num_cores = multiprocessing.cpu_count()
        results = Parallel(n_jobs=num_cores, verbose = 100)(delayed(processPatches)(i,  patch_size, pixel_step, resize, imageChannels) for i in df_sub_list)
        current_sections_patch_lists, current_sections_patch_name_lists = zip(*results)

        # classify and save
        current_sections_prediction_lists = []
        for i in range(0, len(current_sections_patch_lists)):
            print("Classifying section " + str(s_init + i + 1) + "/" + str(nb_section))
            patch_list = current_sections_patch_lists[i]
            patches = numpy.array( patch_list )
            patches = patches.reshape( patches.shape[0], resize, resize, imageChannels)
            patches = patches.astype( 'float32' )
            # classify
            classesPredictionList = []
            classesPredictionList = model.predict_classes(patches)
            current_sections_prediction_lists.append(classesPredictionList)

        print("Writing classification files...")
        for i in range(0, len(current_sections_patch_lists)):
            print("Section " + df_names[s_init + i])
            predictionsFile = open(directory + 'classification_' + df_names[s_init + i] + '.txt', 'w')
            for j in range(0, len(current_sections_prediction_lists[i])):
                patch_name = current_sections_patch_name_lists[i][j]
                prediction = current_sections_prediction_lists[i][j]
                predictionsFile.write( patch_name + " " + str(prediction) + "\n" )
            predictionsFile.close()

    reset_keras(base_model,model)
    print("--- %s seconds ---" % (time.time() - start_time))

def classifySVM(input_dir, patch_size, resize_size, pixel_step, jsonModelFilePath, weightsFilePath, modelName, svmModelPath):
    start_time = time.time()

    # set params
    resize = resize_size # todo = read from json model
    imageChannels = 1

    # create output directory
    directory = "output/classification/"
    if not os.path.exists(directory):
        os.makedirs(directory)

    # read all files in 1 step
    df_list = []
    df_names = []
    files = os.listdir(input_dir)
    for i in range(0, len(files)):
        filename = files[i]
        section_name = filename.split('_')[0]
        df = pd.read_csv(input_dir + filename, delimiter=' ', header = None)
        df_list.append(df)
        df_names.append(modelName + "_" + section_name)

    # load CNN
    jsonModelFile = open(jsonModelFilePath, 'r' )
    jsonModel = jsonModelFile.read()
    jsonModelFile.close()
    base_model = model_from_json(jsonModel)
    base_model.load_weights(weightsFilePath)
    #------ delete last layers -------
    model = Sequential(base_model.layers[:-7])

    #------ Load SVM
    from sklearn.externals import joblib
    clf = joblib.load(svmModelPath)

    # prepare save prediction for all sections
    nb_sections = len(df_list)

    # create patches in parrallel
    s = 0
    nb_section = len(df_list)
    section_step = 4
    while 1 :

        s_init = s

        if(s == nb_section):
            break

        if(s+section_step > len(df_list)):
            df_sub_list = df_list[s:len(df_list)]
            s = nb_section
        else:
            df_sub_list = df_list[s:s+section_step]
            s = s+section_step

        print("Creating patches...")
        num_cores = multiprocessing.cpu_count()
        results = Parallel(n_jobs=num_cores, verbose = 100)(delayed(processPatches)(i,  patch_size, pixel_step, resize, imageChannels) for i in df_sub_list)
        current_sections_patch_lists, current_sections_patch_name_lists = zip(*results) # returns tuples :/

        # classify and save
        current_sections_prediction_lists = []
        for i in range(0, len(current_sections_patch_lists)):
            print("Classifying section " + str(s_init + i + 1) + "/" + str(nb_section))
            patch_list = current_sections_patch_lists[i]
            patches = numpy.array( patch_list )
            patches = patches.reshape( patches.shape[0], resize, resize, imageChannels)
            patches = patches.astype( 'float32' )
            # classify
            classesPredictionList = []
            features = model.predict(patches)
            classesPredictionList = clf.predict(features)
            current_sections_prediction_lists.append(classesPredictionList)

        print("Writing classification files...")
        for i in range(0, len(current_sections_patch_lists)):
            print("Section " + df_names[s_init + i])
            predictionsFile = open(directory + 'classification_' + df_names[s_init + i] + '.txt', 'w')
            for j in range(0, len(current_sections_prediction_lists[i])):
                patch_name = current_sections_patch_name_lists[i][j]
                prediction = current_sections_prediction_lists[i][j]
                predictionsFile.write( patch_name + " " + str(prediction) + "\n" )
            predictionsFile.close()

    reset_keras(base_model,model)
    print("--- %s seconds ---" % (time.time() - start_time))


import torch
import torch.nn as nn
import numpy as np

"""
This script implements a kernel logistic regression model, a radial basis function network model
and a two-layer feed forward network.
"""

class Kernel_Layer(nn.Module):

    def __init__(self, sigma, hidden_dim=None):
        """
        Set hyper-parameters.
        Args:
            sigma: the sigma for Gaussian kernel (radial basis function)
            hidden_dim: the number of "kernel units", default is None, then the number of "kernel units"
                                       will be set to be the number of training samples
        """
        super(Kernel_Layer, self).__init__()
        self.sigma = sigma
        self.hidden_dim = hidden_dim

    def reset_parameters(self, X):
        """
        Set prototypes (stored training samples or "representatives" of training samples) of
        the kernel layer.
        """
        if self.hidden_dim is not None:
            X = self._k_means(X)
        self.prototypes = nn.Parameter(torch.tensor(X).float(), requires_grad=False)

    def _k_means(self, X):
        """
        K-means clustering

        Args:
            X: A Numpy array of shape [n_samples, n_features].

        Returns:
            centroids: A Numpy array of shape [self.hidden_dim, n_features].
        """
        ### YOUR CODE HERE
        inputs = torch.as_tensor(X)
        rand_points = torch.randint(0, X.shape[0], (self.hidden_dim,), dtype=torch.long)
        # randomly initialize centroids
        centroids = inputs[rand_points, :]
        # previous assignment
        prev = torch.randint(0, self.hidden_dim, (X.shape[0],), dtype=torch.long)
        iter = 0
        while iter < 1000:
            iter += 1
            # distance between each pair of vector
            cdist = torch.cdist(inputs, centroids, p=2)
            # cluster assignment
            new = cdist.argmin(dim=1)
            # stop if nothing changed
            if new.eq(prev).all() and iter > 1:
                break
            else:
                prev = new
            # new centroids
            for k in range(self.hidden_dim):
                centroids[k, :] = torch.mean(inputs[new.eq(k), :], dim=0)
        if iter == 1000:
            print("iter limit reached")
        ### END YOUR CODE
        return centroids

    def forward(self, x):
        """
        Compute Gaussian kernel (radial basis function) of the input sample batch
        and self.prototypes (stored training samples or "representatives" of training samples).

        Args:
            x: A torch tensor of shape [batch_size, n_features]

        Returns:
            A torch tensor of shape [batch_size, num_of_prototypes]
        """
        assert x.shape[1] == self.prototypes.shape[1]
        ### YOUR CODE HERE
        # Basically you need to follow the equation of radial basis function
        # in the section 5 of note at http://people.tamu.edu/~sji/classes/nnkernel.pdf
        r = torch.empty(x.shape[0], self.prototypes.shape[0])
        dimension = (x.shape[0], self.prototypes.shape[0], x.shape[1])
        xi = x.unsqueeze(1).expand(dimension)
        xj = self.prototypes.unsqueeze(0).expand(dimension)
        #print(xi.shape)
        norms = ((xi - xj)**2).sum(-1)
        r = torch.exp(-1 * norms / (2 * (self.sigma ** 2)))
        return r

        ### END YOUR CODE

class Kernel_LR(nn.Module):

    def __init__(self, sigma, hidden_dim):
        """
        Define network structure.

        Args:
            sigma: used in the kernel layer.
            hidden_dim: the number of prototypes in the kernel layer,
                                       in this model, hidden_dim has to be equal to the
                                       number of training samples.
        """
        super(Kernel_LR, self).__init__()
        self.hidden_dim = hidden_dim
        ### YOUR CODE HERE
        # Use pytorch nn.Sequential object to build a network composed of a
        # kernel layer (Kernel_Layer object) and a linear layer (nn.Linear object)

        # Remember that kernel logistic regression model uses all training samples
        # in kernel layer, so set 'hidden_dim' argument to be None when creating
        # a Kernel_Layer object.

        self.net = nn.Sequential(
            Kernel_Layer(sigma),
            nn.Linear(in_features=hidden_dim,out_features=1,bias=False)
        )

        # How should we set the "bias" argument of nn.Linear?

        ### END YOUR CODE

    def forward(self, x):
        """
        Forward pass.

        Args:
            x: A torch tensor of shape [batch_size, n_features]

        Returns:
            A torch tensor of shape [batch_size, 1]
        """
        return self.net(x)

    def reset_parameters(self, X):
        """
        Initialize the weights of the linear layer and the prototypes of the kernel layer.

        Args:
            X: A Numpy array of shape [n_samples, n_features], training data matrix.
        """
        assert X.shape[0] == self.hidden_dim
        for layer in self.net:
            if hasattr(layer, 'reset_parameters'):
                if isinstance(layer, Kernel_Layer):
                    layer.reset_parameters(X)
                else:
                    layer.reset_parameters()

class RBF(nn.Module):

    def __init__(self, sigma, hidden_dim):
        """
        Define network structure.

        Args:
            sigma: used in the kernel layer.
            hidden_dim: the number of prototypes in the kernel layer,
                                       in this model, hidden_dim is a user-specified hyper-parameter.
        """
        super(RBF, self).__init__()
        ### YOUR CODE HERE
        # Use pytorch nn.Sequential object to build a network composed of a
        # kernel layer (Kernel_Layer object) and a linear layer (nn.Linear object)
        # How should we set the "bias" argument of nn.Linear?
        self.net = nn.Sequential(
            Kernel_Layer(sigma, hidden_dim),
            nn.Linear(in_features=hidden_dim, out_features=1, bias=False)
        )
        ### END CODE HERE

    def forward(self, x):
        """
        Forward pass.

        Args:
            x: A torch tensor of shape [batch_size, n_features]

        Returns:
            A torch tensor of shape [batch_size, 1]
        """
        return self.net(x)

    def reset_parameters(self, X):
        """
        Initialize the weights of the linear layer and the prototypes of the kernel layer.

        Args:
            X: A Numpy array of shape [n_samples, n_features], training data matrix.
        """
        for layer in self.net:
            if hasattr(layer, 'reset_parameters'):
                if isinstance(layer, Kernel_Layer):
                    layer.reset_parameters(X)
                else:
                    layer.reset_parameters()

class FFN(nn.Module):

    def __init__(self, input_dim, hidden_dim):
        """
        Define network structure.

        Args:
            input_dim: number of features of each input.
            hidden_dim: the number of hidden units in the hidden layer, a user-specified hyper-parameter.
        """
        super(FFN, self).__init__()
        ### YOUR CODE HERE
        # Use pytorch nn.Sequential object to build a network composed of
        # two linear layers (nn.Linear object)
        self.net = nn.Sequential(
            nn.Linear(in_features=input_dim, out_features=hidden_dim, bias=False),
            nn.Linear(in_features=hidden_dim, out_features=1, bias=False)
        )
        ### END CODE HERE

    def forward(self, x):
        """
        Forward pass.

        Args:
            x: A torch tensor of shape [batch_size, n_features]

        Returns:
            A torch tensor of shape [batch_size, 1]
        """
        return self.net(x)

    def reset_parameters(self):
        """
        Initialize the weights of the linear layers.
        """
        for layer in self.net:
            if hasattr(layer, 'reset_parameters'):
                layer.reset_parameters()


from typing import Any, Dict, Type, TypeVar, Union

import attr

from ..types import UNSET, Unset

T = TypeVar("T", bound="CodatPublicApiModelsClientsBankingSettingsModelsBankIntegration")

@attr.s(auto_attribs=True)
class CodatPublicApiModelsClientsBankingSettingsModelsBankIntegration:
    """
    Attributes:
        name (Union[Unset, None, str]):
        integration_guid (Union[Unset, None, str]):
        selected (Union[Unset, bool]):
    """

    name: Union[Unset, None, str] = UNSET
    integration_guid: Union[Unset, None, str] = UNSET
    selected: Union[Unset, bool] = UNSET

    def to_dict(self) -> Dict[str, Any]:
        name = self.name
        integration_guid = self.integration_guid
        selected = self.selected

        field_dict: Dict[str, Any] = {}
        field_dict.update({})
        if name is not UNSET:
            field_dict["name"] = name
        if integration_guid is not UNSET:
            field_dict["integrationGuid"] = integration_guid
        if selected is not UNSET:
            field_dict["selected"] = selected

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        name = d.pop("name", UNSET)

        integration_guid = d.pop("integrationGuid", UNSET)

        selected = d.pop("selected", UNSET)

        codat_public_api_models_clients_banking_settings_models_bank_integration = cls(
            name=name,
            integration_guid=integration_guid,
            selected=selected,
        )

        return codat_public_api_models_clients_banking_settings_models_bank_integration


"""
The first frame should be of the static background i.e. without any objects.
"""

import cv2
import pandas
from datetime import datetime

first_frame = None
status_list = [None, None]
timestamp = []
df = pandas.DataFrame(columns=['Start', 'End'])
video = cv2.VideoCapture(0)

while True:
    check, frame = video.read()
    status = 0  # no object in frame
    gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # See docs: https://docs.opencv.org/3.1.0/d4/d13/tutorial_py_filtering.html
    # This is done to reduce noise and increase accuracy
    gray_frame = cv2.GaussianBlur(gray_frame, (21, 21), 0)

    if first_frame is None:
        first_frame = gray_frame
        continue

    # Difference between the pixel values of two frames
    delta_frame = cv2.absdiff(first_frame, gray_frame)
    print(delta_frame)

    # pixels with difference greater than 30 will be colored white(255)
    # it returns a tuple with the frame object as the second element
    thresh_frame = cv2.threshold(delta_frame, 30, 255, cv2.THRESH_BINARY)[1]

    # this is done smooth out the threshold
    # removes the black holes from the threshold frame
    thresh_frame = cv2.dilate(thresh_frame, None, iterations=2)

    # draw contours
    (_, contours, _) = cv2.findContours(thresh_frame.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    for contour in contours:
        if cv2.contourArea(contour) < 10000:  # to detect bigger objects, give big pixel number
            continue
        status = 1  # object detected
        (x, y, w, h) = cv2.boundingRect(contour)
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 3)

    cv2.imshow('object detection', gray_frame)
    cv2.imshow('delta frame', delta_frame)
    cv2.imshow('thresh frame', thresh_frame)
    cv2.imshow('color frame', frame)

    key = cv2.waitKey(1) & 0xFF
    if key == ord('q'):
        if status == 1:
            timestamp.append(datetime.now())
        break

    status_list.append(status)

    if status_list[-2] == 0 and status_list[-1] == 1:
        timestamp.append(datetime.now())
    elif status_list[-2] == 1 and status_list[-1] == 0:
        timestamp.append(datetime.now())

print('first:')
print(first_frame)

for i in range(0, len(timestamp), 2):
    df = df.append({'Start': timestamp[i], 'End': timestamp[i+1]}, ignore_index=True)
df.to_csv('timestamp.csv')

video.release()
cv2.destroyAllWindows()


from swagger_server.models import AgeGroup, Image, LanguageSpec, Phone, UserId
from swagger_server.models.age_group import AgeGroupEnum
from swagger_server.models.language_spec import LanguageSpecEnum
from swagger_server.repository.user import User

import random
from typing import List, Optional, Tuple

u1 = User(id=1, age_group=AgeGroupEnum.G_18, language=LanguageSpecEnum.RU_RU, image="", name="1", phone="")  # our user

u2 = User(id=2, age_group=AgeGroupEnum.G_18_25, language=LanguageSpecEnum.RU_RU, image="", name="2", phone="")
u3 = User(id=3, age_group=AgeGroupEnum.G_26_35, language=LanguageSpecEnum.EN_US, image="", name="3", phone="")
u4 = User(id=4, age_group=AgeGroupEnum.G_36_45, language=LanguageSpecEnum.FR_FR, image="", name="4", phone="")

partners = [u2, u3, u4]

Score = int

def age_to_scale(age: AgeGroupEnum) -> Score:
    scale = {
        AgeGroupEnum.G_18: 1,
        AgeGroupEnum.G_18_25: 2,
        AgeGroupEnum.G_26_35: 3,
        AgeGroupEnum.G_36_45: 4,
        AgeGroupEnum.G_45: 5,
    }
    return scale[age]

def age_score(lhs: AgeGroupEnum, rhs: AgeGroupEnum) -> Score:
    """
    :return: Score in range 0..4
    """
    lhs_scale = age_to_scale(lhs)
    rhs_scale = age_to_scale(rhs)
    return abs(lhs_scale - rhs_scale)

def language_score(lhs: LanguageSpecEnum, rhs: LanguageSpecEnum) -> Score:
    """
    :return: Score in range 0..3
    """
    if lhs == rhs:
        return 0
    else:
        return 3

BEST_SCORE: Score = age_score(AgeGroupEnum.G_18, AgeGroupEnum.G_18) \
                    + language_score(LanguageSpecEnum.RU_RU, LanguageSpecEnum.RU_RU)

WORST_SCORE: Score = age_score(AgeGroupEnum.G_18, AgeGroupEnum.G_45) \
                     + language_score(LanguageSpecEnum.RU_RU, LanguageSpecEnum.EN_US)

def match_users(user: User, online: List[User]) -> Optional[Tuple[User, Score]]:
    """
    Match users based on heuristic score. The less the score -- the better.

    :param user: current user
    :param online: other users
    :return:
    """
    online = [u for u in online if u.id != user.id]  # sanity check

    if len(online) == 0:
        return None

    scores = [0] * len(online)
    for i in range(len(online)):
        scores[i] += age_score(user.age_group, online[i].age_group)
        scores[i] += language_score(user.language, online[i].language)

    best_score = min(scores)
    bestest = [idx for idx, score in enumerate(scores) if score == best_score]

    assert len(bestest) != 0, "should never happen"
    lucky_idx = random.choice(bestest)

    lucky_user = online[lucky_idx]
    return lucky_user, best_score

print(match_users(u1, partners))


'''borf.borf: for running borf'''

import argparse
import os
import sys
import re
from Bio import SeqIO
import pandas as pd
from .get_orfs import get_orfs, write_orf_fasta, write_orf_data, write_orf_cds, batch_iterator

def main():

    parser = argparse.ArgumentParser(description='Get orf predicitions from a nucleotide fasta file')

    parser.add_argument('Fasta', metavar='fasta_file', type=str, help='fasta file to predict ORFs')
    parser.add_argument('-o', '--output_path', type=str, help='path to write output files. [OUTPUT_PATH].pep and [OUTPUT_PATH].txt (default: input .fa file name)')
    parser.add_argument('-s', '--strand', action='store_true', help='Predict orfs for both strands')
    parser.add_argument('-a', '--all_orfs', action='store_true', help='Return all ORFs for each sequence longer than the cutoff')
    parser.add_argument('-l', '--orf_length', type=int, default=100, help='Minimum ORF length (AA). (default: %(default)d)')
    parser.add_argument('-u', '--upstream_incomplete_length', type=int, default=50, help='Minimum length (AA) of uninterupted sequence upstream of ORF to be included for incomplete_5prime transcripts (default: %(default)d)')
    parser.add_argument('-c', '--genetic_code', type=int, default=1, help='Genetic code (int: 1-14) to use for translation (default: %(default)d). See https://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi for list')
    parser.add_argument('-b', '--batch_size', type=int, default=10000, help='Number of fasta records to read in in each batch')
    parser.add_argument('-f', '--force_overwrite', action='store_true', help='Force overwriting of output files?')

    args = parser.parse_args()

    input_file = args.Fasta

    if args.output_path is None:
        output_path = os.path.splitext(input_file)[0]
    else:
        output_path = args.output_path

    output_path_pep = output_path + '.pep'
    output_path_txt = output_path + '.txt'
    output_path_cds = output_path + '.cds'

    # check if files exist already
    if os.path.isfile(output_path_pep) or os.path.isfile(output_path_txt) or os.path.isfile(output_path_cds):

        if os.path.isfile(output_path_pep) and os.path.isfile(output_path_txt) and os.path.isfile(output_path_cds):
             print(output_path_pep + ", " + output_path_txt + " and " + output_path_cds  + " already exist")
        elif os.path.isfile(output_path_pep) and os.path.isfile(output_path_txt):
             print(output_path_pep + " and " + output_path_txt + " already exist")
        elif os.path.isfile(output_path_pep) and os.path.isfile(output_path_cds):
             print(output_path_pep + " and " + output_path_cds + " already exist")
        elif os.path.isfile(output_path_cds) and os.path.isfile(output_path_txt):
             print(output_path_txt + " and " + output_path_cds + " already exist")
        elif os.path.isfile(output_path_pep):
            print(output_path_pep + " already exists")
        elif os.path.isfile(output_path_cds):
            print(output_path_cds + " already exists")
        else:
            print(output_path_txt + " already exists")

        if not args.force_overwrite:
            overwrite = input("Do you want to overwrite these files? ([Y]/n): ").lower().strip()[:1]
            if not (overwrite == "y" or overwrite == ""):
                sys.exit(1)
            else:
                # remove old files so you don't append new data to old files
                if os.path.isfile(output_path_pep):
                    os.remove(output_path_pep)
                if os.path.isfile(output_path_txt):
                    os.remove(output_path_txt)
                if os.path.isfile(output_path_cds):
                    os.remove(output_path_cds)
        else:
            print('Overwriting files')
            if os.path.isfile(output_path_pep):
                os.remove(output_path_pep)
            if os.path.isfile(output_path_txt):
                os.remove(output_path_txt)
            if os.path.isfile(output_path_cds):
                os.remove(output_path_cds)

    # number of sequences
    n_seqs = 0
    for record in SeqIO.parse(input_file, 'fasta'):
        n_seqs += 1

    batch_size = args.batch_size

    record_iter = SeqIO.parse(open(input_file), 'fasta')

    strand_warning = False
    for i, batch in enumerate(batch_iterator(record_iter, batch_size)):
        all_sequences = []
        for record in batch:
            all_sequences.append(record.upper())

        if i == 0:
            # check strandedness

            orf_data = get_orfs(all_sequences, both_strands=True,
                                min_orf_length=args.orf_length, all_orfs=True,
                                min_upstream_length=args.upstream_incomplete_length,
                                genetic_code=args.genetic_code)

            orf_data_strand_bias = orf_data.sort_values(by='orf_length', ascending = False)
            orf_data_strand_bias = orf_data_strand_bias.drop_duplicates('id', keep='first')

            if len(orf_data_strand_bias) >= 10:

                pos_bias = (orf_data_strand_bias['strand'][orf_data_strand_bias['orf_class'] == "complete"] == "+").sum()
                neg_bias = (orf_data_strand_bias['strand'][orf_data_strand_bias['orf_class'] == "complete"] == "-").sum()
                positive_strand_bias = pos_bias / (pos_bias+neg_bias)

                if positive_strand_bias > 0.7 and args.strand == True:
                    #data is likely from a stranded assembly.
                    print("Are you sure your input .fasta file isn't stranded?")
                    print(str(positive_strand_bias*100)+ "% of transcripts have the longest ORF on the + strand")
                    strand_warning = True

                if positive_strand_bias <= 0.7 and args.strand == False:
                    print("Are you sure your input .fasta file is stranded?")
                    print(str(positive_strand_bias*100)+ "% of transcripts have the longest ORF on the + strand")
                    strand_warning = True

            if args.strand == False:
                orf_data = orf_data[orf_data['strand'] == '+']
            if args.all_orfs == False:
                idx = orf_data.groupby(['id'])['orf_length'].transform(max) == orf_data['orf_length']
                orf_data = orf_data[idx]
                orf_data['isoform_number'] = 1
                orf_data['fasta_id'] = [re.sub("[.]orf[0-9]*",".orf1", x) for x in orf_data['fasta_id']]

        else:
            orf_data = get_orfs(all_sequences, both_strands=args.strand,
                                min_orf_length=args.orf_length, all_orfs=args.all_orfs,
                                min_upstream_length=args.upstream_incomplete_length,
                                genetic_code=args.genetic_code)

        # extract nt seqs at CDS
        nucleotide_seq = []
        nucleotide_id = []
        for seq_string in all_sequences:
            nucleotide_seq.append(str(seq_string.seq))
            nucleotide_id.append(str(seq_string.id))
        seq_df = pd.DataFrame(list(zip(nucleotide_id, nucleotide_seq)), columns=['id', 'nt_seq'])

        # merge orfs with all_sequences
        orf_data = pd.merge(seq_df, orf_data,  on='id', how='right')
        orf_data['cds_seq'] = orf_data.apply(lambda x: x['nt_seq'][(x['start_site_nt']-1):x['stop_site_nt']], axis=1)

        write_orf_data(orf_data, output_path_txt)
        write_orf_fasta(orf_data, output_path_pep)
        write_orf_cds(orf_data, output_path_cds)

        start_seq_n = (i*batch_size) + 1
        end_seq_n = min(start_seq_n + (batch_size - 1), n_seqs)
        print("Processed sequences " + str(start_seq_n) + " to " + str(end_seq_n) + " of " + str(n_seqs))

    print("Done with borf.")
    print("Results in " + output_path_pep + " and " + output_path_txt + " and " + output_path_cds)

    if strand_warning == True:
        print("This data caused a warning based on strandedness. Please check the top of the log for details and rerun with appropriate flags if neccessary.")