text:
import argparse
import os
import json
import numpy as np

def parse_args():
    parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
    parser.add_argument(
        "--expName",
        dest="expName",
        default="",
        help="name of Exp",
        type=str,
    )
    parser.add_argument(
        "--expCode",
        dest="expCode",
        default="",
        help="name of Exp",
        type=str,
    )
    parser.add_argument(
        "--expVersion",
        dest="expVersion",
        default="",
        help="name of Exp",
        type=str,
    )
    parser.add_argument(
        "--node_nums",
        help="num of used nodes",
        type=int,
        required=True
    )
    parser.add_argument(
        "--gpus_per_node",
        help="num of gpus per node is used for multi nodes or gpus used if use single node",
        type=int,
        required=True
    )
    args, unparsed = parser.parse_known_args()
    extra_args = " ".join(unparsed)
    return args, extra_args

def main():
    args, extra_args = parse_args()
    init_on_aml()
    root_path = "/blob/workstation/mmdetection"
    #<========================Change Here==========================================
    output_dir = "{}/{}/{}/{}/output/".format(root_path, args.expName, args.expCode, args.expVersion)
    tmp_outdir = "./output/"
    #<========================Change Here==========================================
    config_file = "./configs/{}/{}.py".format(args.expName, args.expCode) #DARDet/exp1.py
    copy_data_from_blob(config_file)
    from mmcv import Config
    cfg = Config.fromfile(config_file)
    if args.node_nums > 1:
        assert args.node_nums == 1, "for now, only support single node"
    else:
        cmd = "local=$(pwd) \n export PYTHONPATH=${local}/mmdet \n"
        cmd += "export MKL_THREADING_LAYER=GNU \n"
        cmd += "mkdir -p {} \n".format(tmp_outdir)
        cmd += "cp {}/* {} \n".format(output_dir, tmp_outdir)
        cmd += "python -m torch.distributed.launch --nproc_per_node={} tools/train.py {}  --launcher pytorch --work-dir {} --no-validate".format(args.gpus_per_node,config_file,tmp_outdir)
        print(cmd)
        os.system(cmd)
    # As it is too slow when writing logs into Azure Blob in realtime, we first log it in the dorcker image and copy it into Azure Blob finally.
    cmd = "mkdir -p {} \n".format(output_dir)
    cmd += "cp {}/* {} \n".format(tmp_outdir, output_dir)
    print(cmd)
    os.system(cmd)

    # # Make result file for MLT2017 RPN_ONLY test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "python /detectron/tools/demo_af_rpn.py --config-file {} --im_or_folder /detectron/datasets/icdar2015_mlt_test/JPEGImages/ --output /origin_results/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/rpn/model_final.pth MODEL.RPN_ONLY True DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip /res_txt.zip -@ \n"
    # #<========================Change Here===========================================
    # cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/rpn/res_txt_50.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # # Change threshold
    # cmd += "sudo mkdir /result \n"
    # cmd += "sudo cp /blob/workstation/scripts/change_threshold.py /change_threshold.py \n"
    # cmd += "cd /result \n"
    # for th in range(500, 975, 25):
    #     th = th / 1000.0
    #     cmd += "python /change_threshold.py /origin_results/txt {} \n".format(th)
    #     cmd += "sudo zip -r /test_scratch_{:.3f}.zip ./ \n".format(th)
    #     cmd += "sudo cp /test_scratch_{:.3f}.zip {}/{}/{}/{}/rpn/test_scratch_{:.3f}.zip \n".format(th, root_path, args.expName, args.expCode, args.expVersion, th)
    # print(cmd)
    # os.system(cmd)

    # # Make result file for MLT2017 FRCN test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "python /detectron/tools/demo_af_rpn.py --config-file {} --im_or_folder /detectron/datasets/icdar2015_mlt_test/JPEGImages/ --output /origin_results/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip /res_txt.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/frcn/res_txt_50.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # # Change threshold
    # cmd += "sudo mkdir /result \n"
    # cmd += "sudo cp /blob/workstation/scripts/change_threshold.py /change_threshold.py \n"
    # cmd += "cd /result \n"
    # for th in range(500, 975, 25):
    #     th = th / 1000.0
    #     cmd += "python /change_threshold.py /origin_results/txt {} \n".format(th)
    #     cmd += "sudo zip -r /test_scratch_{:.3f}.zip ./ \n".format(th)
    #     cmd += "sudo cp /test_scratch_{:.3f}.zip {}/{}/{}/{}/frcn/test_scratch_{:.3f}.zip \n".format(th, root_path, args.expName, args.expCode, args.expVersion, th)
    # print(cmd)
    # os.system(cmd)

    # Make result file for MSRA_POD FRCN test
    cmd = "sudo mkdir /origin_results \n"
    cmd += "sudo chmod -R 777 /origin_results \n"
    cmd += "local=$(pwd) \n export PYTHONPATH=${local}/mmdet \n"
    cmd += "rm -r /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "mkdir -p /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "ln -s /mmdetection/datasets/POD_RevB_combined/raw_images_horizontal /mmdetection/tools/msra_pod_measurement_tool/test_dataset/images \n"
    cmd += "ln -s /mmdetection/datasets/POD_RevB_combined/xml /mmdetection/tools/msra_pod_measurement_tool/test_dataset/xml \n"
    cmd += "python /mmdetection/demo/inference_demo_pod.py --config {} --im_or_folder /mmdetection/datasets/POD_RevB_combined/raw_images_horizontal/ --output /origin_results/ --num_loader 64 --checkpoint {}/{}/{}/{}/output/latest.pth \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/txt \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/output/res_txt_th0.5_horizontal.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/image \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/output/res_ims_th0.5_horizontal.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /detectron \n"
    # cmd += "python /detectron/tools/demo_pod.py --config-file {} --im_or_folder /blob/data/kownlege_lake_testset/test_images/ --output /origin_results2/ --confidence-threshold 0.1 --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    #<========================Change Here============================================
    # cmd += "cd /origin_results2/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims_kl.zip -@ \n"
    # cmd += "sudo cp /res_ims_kl.zip {}/{}/{}/{}/frcn/res_ims_kl_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    print(cmd)
    os.system(cmd)

    # Make result file for Rotated_360_MSRA_POD FRCN test
    cmd = "sudo mkdir /origin_results \n"
    cmd += "sudo chmod -R 777 /origin_results \n"
    cmd += "local=$(pwd) \n export PYTHONPATH=${local}/mmdet \n"
    cmd += "rm -r /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "rm -rf /mmdetection/tools/msra_pod_measurement_tool/annots.pkl \n"
    cmd += "mkdir -p /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "ln -s /mmdetection/datasets/rotated_360_POD_RevB_combined/raw_images_horizontal /mmdetection/tools/msra_pod_measurement_tool/test_dataset/images \n"
    cmd += "ln -s /mmdetection/datasets/rotated_360_POD_RevB_combined/xml /mmdetection/tools/msra_pod_measurement_tool/test_dataset/xml \n"
    cmd += "python /mmdetection/demo/inference_demo_pod.py --config {} --im_or_folder /mmdetection/datasets/rotated_360_POD_RevB_combined/raw_images_horizontal/ --output /origin_results/ --num_loader 64 --checkpoint {}/{}/{}/{}/output/latest.pth \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/txt \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/output/res_txt_th0.5_360.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/image \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/output/res_ims_th0.5_360.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /detectron \n"
    # cmd += "python /detectron/tools/demo_pod.py --config-file {} --im_or_folder /blob/data/kownlege_lake_testset/test_images/ --output /origin_results2/ --confidence-threshold 0.1 --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # #<========================Change Here============================================
    # cmd += "cd /origin_results2/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims_kl.zip -@ \n"
    # cmd += "sudo cp /res_ims_kl.zip {}/{}/{}/{}/frcn/res_ims_kl_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    print(cmd)
    os.system(cmd)

    # Make result file for Rotated_45_MSRA_POD FRCN test
    cmd = "sudo mkdir /origin_results \n"
    cmd += "sudo chmod -R 777 /origin_results \n"
    cmd += "local=$(pwd) \n export PYTHONPATH=${local}/mmdet \n"
    cmd += "rm -r /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "rm -rf /mmdetection/tools/msra_pod_measurement_tool/annots.pkl \n"
    cmd += "mkdir -p /mmdetection/tools/msra_pod_measurement_tool/test_dataset \n"
    cmd += "ln -s /mmdetection/datasets/rotated_45_POD_RevB_combined/raw_images_horizontal /mmdetection/tools/msra_pod_measurement_tool/test_dataset/images \n"
    cmd += "ln -s /mmdetection/datasets/rotated_45_POD_RevB_combined/xml /mmdetection/tools/msra_pod_measurement_tool/test_dataset/xml \n"
    cmd += "python /mmdetection/demo/inference_demo_pod.py --config {} --im_or_folder /mmdetection/datasets/rotated_45_POD_RevB_combined/raw_images_horizontal/ --output /origin_results/ --num_loader 64 --checkpoint {}/{}/{}/{}/output/latest.pth \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/txt \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/output/res_txt_th0.5_45.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    cmd += "cd /origin_results/image \n"
    cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    #<========================Change Here============================================
    cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/output/res_ims_th0.5_45.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /detectron \n"
    # cmd += "python /detectron/tools/demo_pod.py --config-file {} --im_or_folder /blob/data/kownlege_lake_testset/test_images/ --output /origin_results2/ --confidence-threshold 0.1 --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # #<========================Change Here============================================
    # cmd += "cd /origin_results2/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims_kl.zip -@ \n"
    # cmd += "sudo cp /res_ims_kl.zip {}/{}/{}/{}/frcn/res_ims_kl_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    print(cmd)
    os.system(cmd)

    # # Make result file for cTDaR2019_TRACKA FRCN test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/trackA \n"
    # cmd += "ln -s /detectron/datasets/cTDaR2019_TRACKA/testing_data/raw_xml /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/trackA \n"
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/cTDaR2019_TRACKA/testing_data/converted_jpg_ims/ --output /origin_results/ --confidence-threshold 0.5 --no_demo --do_eval --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/frcn/res_txt_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # print(cmd)
    # os.system(cmd)

    # # Make result file for publaynet_val FRCN test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/publaynet_val/trackA \n"
    # cmd += "ln -s /detectron/datasets/publaynet_val/val_gt_xml_for_table_only /detectron/tools/ctdar_measurement_tool/annotations/publaynet_val/trackA \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/publaynet_val.json \n"
    # cmd += "ln -s /detectron/datasets/publaynet_val/val.json /detectron/tools/coco_eval_tool/publaynet_val.json \n"
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/publaynet_val/val/ --output /origin_results/ --confidence-threshold 0.5 --no_demo --do_eval --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/frcn/res_txt_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # print(cmd)
    # os.system(cmd)

    # # Make result file for IIIT-AR-13K_val/test FRCN test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/IIIT-AR-13K_evaluation_tool/input/val/ground-truth \n"
    # cmd += "ln -s /blob/data/IIIT-AR-13K_val/gt_txt_for_eval_tool /detectron/tools/IIIT-AR-13K_evaluation_tool/input/val/ground-truth \n"
    # cmd += "rm -r /detectron/tools/IIIT-AR-13K_evaluation_tool/input/test/ground-truth \n"
    # cmd += "ln -s /blob/data/IIIT-AR-13K_test/gt_txt_for_eval_tool /detectron/tools/IIIT-AR-13K_evaluation_tool/input/test/ground-truth \n"
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/IIIT-AR-13K_val/validation_images --output /origin_results/ --confidence-threshold 0.5 --do_eval --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_txt.zip {}/{}/{}/{}/frcn/val_res_txt_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/frcn/val_res_ims.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /detectron \n"
    # cmd += "sudo mkdir /origin_results2 \n"
    # cmd += "sudo chmod -R 777 /origin_results2 \n"
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/IIIT-AR-13K_test/test_images --output /origin_results2/ --confidence-threshold 0.5 --do_eval --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results2/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.txt' | sudo zip -q /res_txt2.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_txt2.zip {}/{}/{}/{}/frcn/test_res_txt_th0.5.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results2/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims2.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_ims2.zip {}/{}/{}/{}/frcn/test_res_ims.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # print(cmd)
    # os.system(cmd)

    # # Make result file for TableBank val/test FRCN test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_word_val.json \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_word_test.json \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_latex_val.json \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_latex_test.json \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_word_latex_val.json \n"
    # cmd += "rm -r /detectron/tools/coco_eval_tool/tablebank_word_latex_test.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_word_val.json /detectron/tools/coco_eval_tool/tablebank_word_val.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_word_test.json /detectron/tools/coco_eval_tool/tablebank_word_test.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_latex_val.json /detectron/tools/coco_eval_tool/tablebank_latex_val.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_latex_test.json /detectron/tools/coco_eval_tool/tablebank_latex_test.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_word_latex_val.json /detectron/tools/coco_eval_tool/tablebank_word_latex_val.json \n"
    # cmd += "ln -s /blob/data/TableBank/annotations/tablebank_word_latex_test.json /detectron/tools/coco_eval_tool/tablebank_word_latex_test.json \n"
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/TableBank/images/ --name_list /blob/data/TableBank/namelists/tablebank_word_val.txt --output /origin_results/tablebank_word_val/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/TableBank/images/ --name_list /blob/data/TableBank/namelists/tablebank_word_test.txt --output /origin_results/tablebank_word_test/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/TableBank/images/ --name_list /blob/data/TableBank/namelists/tablebank_latex_val.txt --output /origin_results/tablebank_latex_val/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "python /detectron/tools/demo_table_detection.py --config-file {} --im_or_folder /detectron/datasets/TableBank/images/ --name_list /blob/data/TableBank/namelists/tablebank_latex_test.txt --output /origin_results/tablebank_latex_test/ --confidence-threshold 0.5 --no_demo --num_loader 64 --opts MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.RPN_ONLY False DEBUG False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/ \n"
    # cmd += "sudo zip -q -r /results.zip ./ \n"
    # cmd += "sudo cp /results.zip {}/{}/{}/{}/frcn/results.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /detectron/tools/coco_eval_tool/ \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_word_val /origin_results/tablebank_word_val/txt/ \n"
    # cmd += "echo tablebank_word_val \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_latex_val /origin_results/tablebank_latex_val/txt/ \n"
    # cmd += "echo tablebank_latex_val \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_word_latex_val /origin_results/tablebank_word_val/txt/ /origin_results/tablebank_latex_val/txt/ \n"
    # cmd += "echo tablebank_word_latex_val \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_word_test /origin_results/tablebank_word_test/txt/ \n"
    # cmd += "echo tablebank_word_test \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_latex_test /origin_results/tablebank_latex_test/txt/ \n"
    # cmd += "echo tablebank_latex_test \n"
    # cmd += "python ./evaluate_tablebank.py -tablebank_word_latex_test /origin_results/tablebank_word_test/txt/ /origin_results/tablebank_latex_test/txt/ \n"
    # cmd += "echo tablebank_word_latex_test \n"
    # print(cmd)
    # os.system(cmd)

    # # Make result file for MSRA_TSR test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/MSRA_TSR/trackB1 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/MSRA_TSR/cropped_textbox_xml_for_eval_tool \n"
    # cmd += "ln -s /blob/data/MSRA_TSR_test/cropped_gt_xml_for_eval_tool /detectron/tools/ctdar_measurement_tool/annotations/MSRA_TSR/trackB1 \n"
    # cmd += "ln -s /blob/data/MSRA_TSR_test/cropped_textbox_xml_for_eval_tool /detectron/tools/ctdar_measurement_tool/annotations/MSRA_TSR/cropped_textbox_xml_for_eval_tool \n"
    # cmd += "python /detectron/tools/demo_TSR.py --config-file {} --im_or_folder /blob/data/MSRA_TSR_test/cropped_table_image/ --output /origin_results/ --no_demo --do_eval --num_loader 64 --opts MODEL.DEVICE cuda MODEL.WEIGHTS {}/{}/{}/{}/rpn/model_final.pth MODEL.MERGE_HEAD_ON False DEBUG False MODEL.SPLIT_HEAD.USE_CURVE_FITTING.ENABLED False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "python /detectron/tools/demo_TSR.py --config-file {} --im_or_folder /blob/data/MSRA_TSR_test/cropped_table_image/ --output /origin_results/ --do_eval --num_loader 64 --opts MODEL.DEVICE cuda MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.MERGE_HEAD_ON True DEBUG False MODEL.SPLIT_HEAD.USE_CURVE_FITTING.ENABLED False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.xml' | sudo zip -q /res_xml.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_xml.zip {}/{}/{}/{}/frcn/res_xml.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/frcn/res_ims.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # print(cmd)
    # os.system(cmd)

    # # Make result file for cTDaR2019_TSR test
    # cmd = "sudo mkdir /origin_results \n"
    # cmd += "sudo chmod -R 777 /origin_results \n"
    # cmd += "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/trackB1 \n"
    # cmd += "rm -r /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/cropped_textbox_xml_for_eval_tool \n"
    # cmd += "ln -s /detectron/datasets/cTDaR2019_TSR_test/cropped_gt_xml_for_eval_tool /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/trackB1 \n"
    # cmd += "ln -s /detectron/datasets/cTDaR2019_TSR_test/cropped_textbox_xml_for_eval_tool /detectron/tools/ctdar_measurement_tool/annotations/cTDaR2019/cropped_textbox_xml_for_eval_tool \n"
    # cmd += "python /detectron/tools/demo_TSR.py --config-file {} --im_or_folder /detectron/datasets/cTDaR2019_TSR_test/cropped_table_image/ --output /origin_results/ --no_demo --do_eval --num_loader 64 --opts MODEL.DEVICE cuda MODEL.WEIGHTS {}/{}/{}/{}/rpn/model_final.pth MODEL.MERGE_HEAD_ON False DEBUG False MODEL.SPLIT_HEAD.USE_CURVE_FITTING.ENABLED False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "python /detectron/tools/demo_TSR.py --config-file {} --im_or_folder /detectron/datasets/cTDaR2019_TSR_test/cropped_table_image/ --output /origin_results/ --do_eval --num_loader 64 --opts MODEL.DEVICE cuda MODEL.WEIGHTS {}/{}/{}/{}/frcn/model_final.pth MODEL.MERGE_HEAD_ON True DEBUG False MODEL.SPLIT_HEAD.USE_CURVE_FITTING.ENABLED False \n".format(config_file, root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/txt \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.xml' | sudo zip -q /res_xml.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_xml.zip {}/{}/{}/{}/frcn/res_xml.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # cmd += "cd /origin_results/image \n"
    # cmd += "sudo find ./ -mindepth 1 -maxdepth 1 -name '*.jpg' | sudo zip -q /res_ims.zip -@ \n"
    # #<========================Change Here============================================
    # cmd += "sudo cp /res_ims.zip {}/{}/{}/{}/frcn/res_ims.zip \n".format(root_path, args.expName, args.expCode, args.expVersion)
    # print(cmd)
    # os.system(cmd)

def init_on_aml():
    cmd = "bash chixma_init_on_aml.sh"
    print(cmd)
    os.system(cmd)

def copy_data_from_blob(config_file):
    cmd = "local=$(pwd) \n export PYTHONPATH=${local}/detectron2 \n"
    cmd += "echo $PYTHONPATH \n"
    cmd += "python copy_datasets_from_blob.py --config-file {}".format(config_file)
    print(cmd)
    os.system(cmd)

if __name__ == '__main__':
    main()

max_stars_repo_name: JarvisUSTC/DARDet

max_issues_repo_name: JarvisUSTC/DARDet

max_forks_repo_name: JarvisUSTC/DARDet


text:
#try:
import numpy as np
import pandas as pd
import networkx as nx
import seaborn as sns
from matplotlib import pyplot as plt

def plot_map(G, layout=None):
    """
    Draw map using networkX's draw functionality

    Parameters
    ----------
    G : networkx Graph
        A networkx Graph to plot.
    node_labels : dict
        Dictionary mapping node numbers to node labels.
    layout : str
        Networkx graph layout (options: 'circular_layout','random_layout','spring_layout','spectral_layout').
        If None is provided, 'shell_layout' is used.
        usage: 'layout=nx.spring_layout(G)'

    Returns
    -------
    matplotlib figure
        A visualisation of graph G

    """

    # set figure size
    fig = plt.figure(figsize=(26,13))

    # edge width by weight
    #weights = np.array( [G.get_edge_data(u,v)[0]['weight'] for u,v in G.edges()] )
    weights = list( nx.get_edge_attributes(G, name='weight').values() )
    #if not pos:
    #    pos=layout
    if layout=='circular':
        pos = nx.layout.circular_layout(G)
    elif layout=='spectral':
        pos = nx.layout.spectral_layout(G)
    elif layout=='spring':
        pos = nx.layout.spring_layout(G)
    elif layout=='spiral':
        pos = nx.layout.spiral_layout(G)
    else:
        pos = nx.layout.shell_layout(G)

    node_colors = list( nx.get_node_attributes(G, name='node_color').values() )
    node_labels = list( nx.get_node_attributes(G, name='node_label').values() )

    # scale nodes by metric:pagerank
    #node_sizes = list( nx.pagerank(G).values() )

    # draw networkx edges, nodes and labels
    nc = nx.draw_networkx_nodes(G
                                , pos
                                #, nodelist=G.nodes()
                                , node_color=node_colors
                                , label=node_labels
                                , node_size= 1000
                                , alpha=0.7
                                #, cmap=plt.cm.rainbow
                               )
    ec = nx.draw_networkx_edges(G
                                , pos
                                , alpha=0.8
                                , width=weights
                                , arrowstyle='->'
                                , arrowsize=20
                                #, arrows=True
                                #, connectionstyle='arc3,rad=0.2'
                                )
    la = nx.draw_networkx_labels(G
                                , pos
                                , labels=nx.get_node_attributes(G, name='node_label')
                                )

    plt.axis('off')
    #plt.legend(loc='upper right')
    #plt.show()
    #colorbar()

    return fig

def plot_ego_map(G, n, direction=None, layout=None):
    """
    Helper function to draw the ego network of node n.

    Parameters
    ----------
    G : networkx graph
        Networkx graph containing node n
    n : int or str
        Node for which ego graph is drawn
    direction : str, optional
        Specifies direction of ego network. Options: 'incoming' or 'undirected'.
        If None, 'outgoing' ego-network is drawn
    layout : str, optional
        Specifies networkX graph visualisation layout.
        If None, 'shell_layout' is used.

    Returns
    -------
    matplotlib figure
        Plot of the ego graph of node n.
    """

    D = G.copy()

    if direction == 'incoming':
        ego = nx.ego_graph(D.reverse(), n)
    elif direction == 'undirected':
        ego = nx.ego_graph(D, n, undirected=True)
    else:
        ego = nx.ego_graph(D, n, undirected=False)

    # set node color of node n to lightgrey
    attr = {n : {'node_color':'lightgrey'}}

    nx.set_node_attributes(ego, attr)

    # plot ego graph
    fig = plot_map( ego, layout)

    return fig

def plot_graph_deltas(deltas=[]):
    """
    Helper function to plot histograms of graph deltas

    Parameters
    ----------
    deltas : dict
        Dictionary of grap deltas.

    TODO: example of dictionary

    Returns
    -------
    matplotlib figure
        Plot of histograms of graph deltas.
    """

    # plot with various axes scales
    plt.figure()

    # delta_n_nodes
    ax1 = plt.subplot(231)
    ax1.hist(deltas['delta_n_nodes'])
    plt.title('delta_n_nodes')
    plt.grid(True)

    # delta_n_nodes
    ax2 = plt.subplot(232)
    ax2.hist(deltas['delta_n_edges'])
    plt.title('delta_n_edges')
    plt.grid(True)

    # delta_density
    ax3 = plt.subplot(233)
    ax3.hist(deltas['delta_density'])
    plt.title('delta_density')
    plt.grid(True)

    return

def quadrant_scatter(df
                    , x='Out_degree_wg'
                    , y='In_degree_wg'
                    , z='Pagerank'
                    , skip = None
                    , prefix = 'Rank of'
                    , divide_canvas = True
                    , annotate = True
                    ):

    """
    Draw a scatter plot of graph nodes along node metrics.

    Parameters
    ----------
    df : pandas DataFrame
        Dataframe where index corresponds to node label and column are metrics.
    x : str
        Variable (column) to plot on x-axis. (Default: 'Out_degree_wg')
    y : str
        Variable (column) to plot on y-axis. (Default: 'In_degree_wg')
    z : str
        Variable (column) to plot on z-axes. (Default: 'Pagerank')
    skip : list, optional
        List of node labels that will not be included in scatter plot. (Default: None)
    prefix : str, optional
        Prefix to axis labels. Node labels are used as axis labels. (Default: 'Rank of')
    divide_canvas : bool
        Split canvas in two shaded regions. (Default: True)
    annotate : bool
        Annotate scatter points with node labels. (Default: True)

    Returns
    -------
    matplotlib figure
        Scatter plot ("bubble plot") of graph nodes along desired node metrics.

    """

    def _compute_scatter_size(min_new=1, max_new=50):
        return ((max_new - min_new) / ( max(Z) - min(Z) )) * ( Z - max(Z) ) + max_new

    # remove skip nodes
    if skip is not None:
        df = df.drop( skip )

    # convert prefix to 'str'
    if prefix is not None:
        prefix = str(prefix) + ' '

    X = df[x].values
    Y = df[y].values
    Z = df[z].values

    # compute ranks
    X = df[x].rank(method='first').values
    Y = df[y].rank(method='first').values
    Z = df[z].rank(method='first').values

    scatter_size = _compute_scatter_size(min_new=100, max_new=2000)

    fig,ax = plt.subplots(figsize=(20,20))

    # axis labels
    ax.set_xlabel(prefix + x, fontsize=25)
    ax.set_ylabel(prefix + y, fontsize=25)
    ax.tick_params(axis="both", labelsize=25)
    # axis lims
    axmax = max(max(X),max(Y))
    ax.set_xlim([0, axmax+2])
    ax.set_ylim([0, axmax+2])

    if divide_canvas:

        ax.plot( [0,1]
                ,[0,1]
                , '--'
                , color='black'
                , alpha=0.8
                , transform=ax.transAxes
                , linewidth=5
                )
        ax.fill_between( [0, axmax]
                        , 0
                        , [0, axmax]
                        , facecolor='peru'
                        , alpha=0.3
                        , transform=ax.transAxes
                        )
        ax.fill_between( [0, axmax]
                        , axmax
                        , [0, axmax]
                        , facecolor='moccasin'
                        , alpha=0.3
                        , transform=ax.transAxes
                        )
    # Draw scatter
    sc = ax.scatter( X
                    , Y
                    , s=scatter_size
                    , c=Z
                    , edgecolors='black'
                    , linewidths=1
                    , alpha=1.
                    , cmap='coolwarm'
                    )
     # colour bar
    c_bar = fig.colorbar(sc, pad=0.15)
    c_bar.set_label(prefix + z, rotation=270, size=25, labelpad=+30)
    c_bar.ax.tick_params(labelsize=20)

    if annotate:
        for i, txt in enumerate(df.index.values):
            ax.annotate(txt,(X[i],Y[i]), ha='center', va='baseline', size=18, name='Courier') #weight='bold'

    fig.show()

    return fig

def plot_graph_perturbations(M, degN_noise, deg1_noise):
    """
    Helper function to draw perturbations to adjacency matrix during synthesis

    Parameters
    ----------
    M : numpy.matrix
        Matrixx where each element represents a perturbation to the corresponding edge in the original adjacency matrix
    degN_noise : list
        List of noise added to adjancy matrix.
    deg1_noise : list
        List of noise added to degree-1 edges.

    Returns
    -------
    matplotlib figure
        Figure showing perturbations to 1) adjacency matrix, 2) degree N>1 nodes and 3) degree N==1 nodes
    """

    fig = plt.figure(figsize=(15,5))

    # Total pertubations to adjaceny matrix
    m_pert = pd.DataFrame(M)
    ax1 = plt.subplot(131)
    ax1 = sns.heatmap(m_pert, mask = (m_pert==0.), linewidth=.5)
    plt.title('Non-zero perturbations to graph adjacency matrix')

    # Noise smearing on degree N>1 edges
    ax2 = plt.subplot(132)
    ax2.hist(degN_noise)
    plt.title('Degree N>1 noise smearing')

    # Noise smearing on degree N=1 edges
    ax3 = plt.subplot(133)
    ax3.hist(deg1_noise)
    plt.title('Degree N=1 noise smearing')

    fig.show()

    return

def plot_comparative_importance(G,
                                var='Eigenvector_centrality',
                                col = ('orange','skyblue'),
                                compare_rank=False,
                                excl=None
                               ):

    """
    Returns a lollipop plot of the difference between graph G_a and graph G_b in a user provided node property.

    Parameters
    ----------
    G : tuple of CoMap objects
        Tuple of two CoMap graphs to compare (e.g. (G_a, G_b))
    var : str
        Node property (Default: 'Eigenvector_centrality')
    col : tuple of str
        Colors for G_a and G_b (default: ('orange','skyblue'))
    compare_rank : bool, optional
        Compare relative difference in rank of 'var' rather than difference in value (default: False)
    excl : list, optional
        List of strings referring to nodes to exclude from plot.

    Returns
    -------
    matplotlib figure
        A lollipop graph showing the relative difference in 'var' between graphs in tuple G.

    """

    node_prpty = var

    # get map properties
    df_A = G[0].map_properties( sort_by=[ node_prpty ] ).drop(excl)
    df_B = G[1].map_properties( sort_by=[ node_prpty ] ).drop(excl)

    # compute rank differences
    if(compare_rank):
        var = var + '_rank'
        df_A[ var ] = df_A[ node_prpty ].rank()
        df_B[ var ] = df_B[ node_prpty ].rank()

    # compute normalised difference
    df_diff = (df_A - df_B)# / df_B

    # dropna to only get nodes present in both graphs
    df_diff.dropna(inplace=True)

    #  sort by var
    df_diff.sort_values(by=var,inplace=True)

    #Create a color if the group is "B"
    color_scheme = np.where( df_diff[var]>=0, col[0], col[1])

    # Compute range
    plt_range=range(1, len(df_diff.index)+1)

    fig,ax = plt.subplots(figsize=(1/5*len(df_diff),1/3*len(df_diff)))

    # plot
    plt.axvline(x=0,color='grey',linestyle='--')
    plt.hlines(y=plt_range, xmin=0, xmax=df_diff[var], color=color_scheme)
    plt.scatter(df_diff[var], plt_range, color=color_scheme, alpha=1)

    # decorations
    plt.yticks(plt_range, df_diff.index)
    plt.xlabel('$\Delta$ '+var, fontsize=18)
    plt.ylabel('Node', fontsize=18)
    ax.yaxis.grid(True, linestyle='--')

    x = .3 * np.where( abs(df_diff[var].min()) > df_diff[var].max(), df_diff[var].max(), abs(df_diff[var].min()) )
    y = 1.1*len(df_diff)

    bbox_props_a = dict(boxstyle="rarrow,pad=0.7", fc=col[0], ec="grey", lw=1)
    ax.text(x,  y, "Viktigere i \n"+G[0].name, ha="left",  va="center", bbox=bbox_props_a, rotation=0, size=11, fontsize=12 )

    bbox_props_b = dict(boxstyle="larrow,pad=0.7", fc=col[1], ec="grey", lw=1)
    ax.text(-x, y, "Viktigere i \n"+G[1].name, ha="right", va="center", bbox=bbox_props_b, rotation=0, size=11, fontsize=12 )

    xlim = np.where( df_diff[var].max()>=abs(x), 1.2*df_diff[var].max(), 1.2*abs(x))
    ylim = 1.1*y
    plt.xlim(-xlim,xlim)
    plt.ylim(0,ylim)

    fig.show()

    return fig

ext: py

max_stars_repo_name: navikt/CoMap

max_issues_repo_name: navikt/CoMap

max_forks_repo_name: navikt/CoMap

reason: import numpy,import networkx


text:
from multiprocessing import Process, Queue
import numpy as np
import time, math, os, argparse
from gpuinfo import GPUInfo

class_list = ["Birds", "Empty", "Fox", "Humans", "Rodents"]
NUM_IMAGE = 10
INF_DIR = "/racelab/SantaCruzIsland_Labeled_5Class/Birds"
MODEL_DIR = '/racelab/checkpoints/resnet50_model.h5'
WIDTH = 1920
HEIGHT = 1080

class Scheduler:
    def __init__(self, gpu_num):
        self._queue = Queue()
        self._gpu_num = gpu_num
        self.__init_workers()

    def __init_workers(self):
        self._workers = list()
        for gpuid in range (self._gpu_num):
            self._workers.append(Worker(gpuid, self._queue))

    def start(self, image_list):

        for img in image_list:
            self._queue.put(img)

        # Add a None to indicate the end of queue
        self._queue.put(None)

        for worker in self._workers:
            worker.start()

        for worker in self._workers:
            worker.join()
        print ("All image are done inferencing...")

class Worker(Process):
    def __init__(self, gpuid, queue):
        Process.__init__(self, name="ModelProcessor")
        self._gpuid = gpuid
        self._queue = queue

    def run(self):
        #set enviornment
        os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
        os.environ["CUDA_VISIBLE_DEVICES"] = str(self._gpuid)

        from tensorflow.keras.applications.resnet50 import preprocess_input
        from tensorflow.keras.preprocessing import image
        from tensorflow.keras.models import load_model
        trained_model = load_model(MODEL_DIR)

        while True:
            img_path = self._queue.get()
            if img_path == None:
                self._queue.put(None)
                break
            img = image.load_img(path=img_path, target_size=(1920, 1080))
            x = image.img_to_array(img)
            x = np.expand_dims(x, axis=0)
            x = preprocess_input(x)
            y_prob = trained_model.predict(x)
            index = y_prob.argmax()
            print ("GPU {0} image : {1}, index : {2}".format(self._gpuid, img_path, index))

        print("GPU {} has done inferencing...".format(self._gpuid))

def run_sequential(image_list):
    from tensorflow.keras.applications.resnet50 import preprocess_input
    from tensorflow.keras.preprocessing import image
    from tensorflow.keras.models import load_model

    trained_model = load_model('/racelab/checkpoints/resnet50_model.h5')
    for img_path in image_list:
        img = image.load_img(path=img_path, target_size=(1920, 1080))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)
        y_prob = trained_model.predict(x)
        index = y_prob.argmax()
        # print ("image : {0}, index : {1}".format(img_path, index))

def handler(event, context):

    if isinstance(event['data'], dict) and "num_image" in event['data']:
        global NUM_IMAGE
        NUM_IMAGE = int(event['data']['num_image'])

    # Get GPU counts
    NUM_GPU = 0
    available_devices = GPUInfo.check_empty()
    if available_devices != None:
        NUM_GPU = len(available_devices)
    print ("Current GPU num is {0}".format(NUM_GPU))

    counter = 0
    image_list = list()
    for img in os.listdir(INF_DIR):
        image_list.append(os.path.join(INF_DIR, img))
        counter += 1
        if counter == NUM_IMAGE:
            break

    start = time.time()

    if NUM_GPU == 0:
        run_sequential(image_list)
    else:
        # initialize Scheduler
        scheduler = Scheduler(NUM_GPU)
        # start multiprocessing
        scheduler.start(image_list)

    end = time.time()
    print ("Time with model loading {0} for {1} images.".format(end - start, NUM_IMAGE))
    return ("Time with model loading {0} for {1} images.".format(end - start, NUM_IMAGE))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("num_image")
    args = parser.parse_args()
    handler({"data" : {"num_image" : args.num_image}}, {})

max_stars_repo_name: Heronalps/STOIC

max_issues_repo_name: Heronalps/STOIC

max_forks_repo_name: Heronalps/STOIC

reason: import numpy


text:
import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import os
import uuid
import shutil
from functools import lru_cache, partial
from pyhdx.support import grouper, autowrap
from pyhdx.plot import plot_peptides
from tqdm.auto import tqdm

try:
    import pylatex as pyl

except ModuleNotFoundError:
    pyl = None
import tempfile

geometry_options = {
    "lmargin": "1in",
    "rmargin": "1.5in"
    }

class Report(object):
    """

    .pdf output document
    """

    def __init__(self, output, name=None, doc=None, add_date=True):
        if not pyl:
            raise ModuleNotFoundError('pylatex module not installed')
        name = name or output.series.state
        self.output = output
        self.doc = doc or self._init_doc(name, add_date=add_date)
        self._temp_dir = self.make_temp_dir()

    def make_temp_dir(self):
        _tmp_path = os.path.abspath(os.path.join(tempfile.gettempdir(), str(id(self))))

        if not os.path.exists(_tmp_path):
            os.makedirs(_tmp_path)
        return _tmp_path

    def _init_doc(self, name, add_date=True):
        doc = pyl.Document(name, geometry_options=geometry_options)
        doc.packages.append(pyl.Package('hyperref'))
        doc.preamble.append(pyl.Command('title', f'Supplementary Figures for {name}'))
        if add_date:
            doc.preamble.append(pyl.Command('date', pyl.NoEscape(r'\today')))
        else:
            doc.preamble.append(pyl.Command('date', pyl.NoEscape(r'')))
        doc.append(pyl.NoEscape(r'\maketitle'))
        doc.append(pyl.NewPage())

        return doc

    def _save_fig(self, fig, *args, extension='pdf', **kwargs):
        filename = '{}.{}'.format(str(uuid.uuid4()), extension.strip('.'))
        filepath = os.path.join(self._temp_dir, filename)
        fig.savefig(filepath, *args, **kwargs)
        return filepath

    def test_mpl(self):
        fig = plt.figure()
        plt.plot([2,3,42,1])

        file_path = self._save_fig(fig)

        with self.doc.create(pyl.Figure(position='htbp')) as plot:
            plot.add_image(pyl.NoEscape(file_path), width=pyl.NoEscape(r'1\textwidth'))
            plot.add_caption('I am a caption.')

    def add_coverage_figures(self, layout=(6, 2), close=True, **kwargs):
        funcs = [partial(self.output._make_coverage_graph, i, **kwargs) for i in range(len(self.output.series))]
        self.make_subfigure(funcs, layout=layout, close=close)

    def add_peptide_figures(self, layout=(5, 4), close=True, **kwargs):
        funcs = [partial(self.output._make_peptide_graph, i, **kwargs) for i in range(len(self.output.series.coverage))]
        self.make_subfigure(funcs, layout=layout, close=close)

    def make_subfigure(self, fig_funcs, layout=(5, 4), close=True):
        #todo figure out how to iterate properly
        n = np.product(layout)
        chunks = grouper(n, fig_funcs)
        w = str(1/layout[1])
        pbar = tqdm(total=len(fig_funcs))
        for chunk in chunks:
            with self.doc.create(pyl.Figure(position='ht')) as tex_fig:
                for i, fig_func in enumerate(chunk):
                    if fig_func is None:
                        continue
                    with self.doc.create(pyl.SubFigure(position='b', width=pyl.NoEscape(w + r'\linewidth'))) as subfig:
                        fig = fig_func()
                        file_path = self._save_fig(fig, bbox_inches='tight') # todo access these kwargs
                        if close:
                            plt.close(fig)
                        subfig.add_image(file_path, width=pyl.NoEscape(r'\linewidth'))
                    if i % layout[1] == layout[1] - 1:
                        self.doc.append('\n')
                    pbar.update(1)

            self.doc.append(pyl.NewPage())

    def test_subfigure(self):
        fig = plt.figure()
        plt.plot([2,3,42,1])

        file_path = self._save_fig(fig)

        with self.doc.create(pyl.Figure(position='h!')) as kittens:
            w = str(0.25)
            for i in range(8):
                with self.doc.create(pyl.SubFigure(
                        position='b',
                        width=pyl.NoEscape(w + r'\linewidth'))) as left_kitten:

                    left_kitten.add_image(file_path,
                                          width=pyl.NoEscape(r'\linewidth'))
                    left_kitten.add_caption(f'Kitten on the {i}')
                if i % 4 == 3:
                    self.doc.append('\n')
            kittens.add_caption("Two kittens")

    def rm_temp_dir(self):
        """Remove the temporary directory specified in ``_tmp_path``."""

        if os.path.exists(self._temp_dir):
            shutil.rmtree(self._temp_dir)

    def generate_pdf(self, file_path=None):
        self.doc.generate_pdf(file_path, compiler='pdflatex')

class Output(object):

    def __init__(self, series, fit_results, **settings):
        #todo series is now an attribute in fit_results
        self.series = series
        self.fit_results = fit_results # dictionary with name: KineticsFitresult
        self.settings = settings #dictionary with additional settings

    @property
    def fit_timepoints(self):
        timepoints = self.series.timepoints
        x_axis_type = self.settings.get('fit_time_axis', 'Log')
        if x_axis_type == 'Linear':
            time = np.linspace(0, timepoints.max(), num=250)
        elif x_axis_type == 'Log':
            time = np.logspace(-2, np.log10(timepoints.max()), num=250)
        return time

    @lru_cache(8)
    def call_fitresult(self, fit_name, time=None):
        """

        Parameters
        ----------
        fit_name : :obj:`str`
            Name of the fit result to use from `fit_results` dictionary
        time : array_like
            Optional array of timepoints

        Returns
        -------
        uptake : :obj:`str`
            Numpy array with fitted uptake values for each peptide per row
        """
        fit_timepoints = time or self.fit_timepoints

        # Moved to fit results
        # fit_result = self.fit_results[fit_name]
        # d_list = []
        # if fit_result.model_type == 'Single':
        #     for time in fit_timepoints:
        #         p = fit_result.get_p(time)
        #         p = np.nan_to_num(p)
        #         d = self.series.coverage.X.dot(p)
        #         d_list.append(d)
        # elif fit_result.model_type == 'Global':
        #     for time in fit_timepoints:
        #         d = fit_result.get_d(time)
        #         d_list.append(d)
        #
        # uptake = np.vstack(d_list).T

        uptake = self.fit_results[fit_name](fit_timepoints)

        return uptake

    def add_peptide_fits(self, ax_scale='log', fit_names=None):
        pass

    def peptide_graph_generator(self, **kwargs):
        for i in range(len(self.series.coverage)):
            yield from self._make_peptide_graph(i, **kwargs)

    def _make_peptide_graph(self, index, figsize=(4,4), ax_scale='log', **fig_kwargs):
        """yield single peptide grpahs"""

        fig, ax = plt.subplots(figsize=figsize)
        if ax_scale == 'log':
            ax.set_xscale('log')

        for k in self.fit_results.keys():
            ax.plot(self.fit_timepoints, self.call_fitresult(k)[index], label=k)

        ax.scatter(self.series.timepoints, self.series.uptake_corrected.T[index], color='k')
        t_unit = fig_kwargs.get('time_unit', '')
        t_unit = f'({t_unit})' if t_unit else t_unit
        ax.set_xlabel(f'Time' + t_unit)
        ax.set_ylabel('Corrected D-uptake')
        start, end = self.series.coverage.data['_start'][index], self.series.coverage.data['_end'][index]
        ax.set_title(f'peptide_{start}_{end}')
        ax.legend()
        plt.tight_layout()

        return fig

    def _make_coverage_graph(self, index, figsize=(14, 4), cbar=True, **fig_kwargs):
        peptides = self.series[index]

        cmap = fig_kwargs.get('cmap', 'jet')
        if cbar:
            fig, (ax_main, ax_cbar) = plt.subplots(1, 2, figsize=figsize, gridspec_kw={'width_ratios': [40, 1], 'wspace': 0.025})

            norm = mpl.colors.Normalize(vmin=0, vmax=100)
            cmap = mpl.cm.get_cmap(cmap)
            cb1 = mpl.colorbar.ColorbarBase(ax_cbar, cmap=mpl.cm.get_cmap(cmap),
                                            norm=norm,
                                            orientation='vertical', ticks=[0, 100])
            cb1.set_label('Uptake %', x=-1, rotation=270)
            # cbar_ax.xaxis.set_ticks_position('top')
            cb1.set_ticks([0, 100])

        else:
            fig, ax_main = plt.subplots(figsize=figsize)

        wrap = autowrap(peptides)
        plot_peptides(peptides, wrap, ax_main, **fig_kwargs)
        ax_main.set_xlabel('Residue number')
        t_unit = fig_kwargs.get('time_unit', '')
        fig.suptitle(f'Deuterium uptake at t={peptides.exposure} ' + t_unit)
        plt.tight_layout()

        return fig

max_stars_repo_name: sebaztiano/PyHDX

max_issues_repo_name: sebaztiano/PyHDX

max_forks_repo_name: sebaztiano/PyHDX


text:
import json
import time

from scapy.all import PcapReader
from scapy.layers.inet import IP, TCP, UDP, ICMP, Ether
import scapy.all as sc
sc.load_layer("tls")
import sys
import os
import struct
import copy
from collections import defaultdict
import hashlib
import hmac
import socket

from string import digits
from datetime import datetime

import numpy as np

import fiat_module.utils as utils

# TCP flags
FIN = 0x01
SYN = 0x02
RST = 0x04
PSH = 0x08
ACK = 0x10
URG = 0x20
ECE = 0x40
CWR = 0x80
# MEANINGFUL_FLAG = 0xff
MEANINGFUL_FLAG = FIN | SYN | RST

def get_tls_version(pkt):
    ret = 0
    if TCP in pkt:
        try:
            if TLS in pkt:
                ret = pkt[TLS].version
            elif TLS in TLS(bytes(pkt[TCP].payload)):
                ret = TLS(bytes(pkt[TCP].payload)).version
        except Exception as e:
            print('proto error!', e, pkt)
    elif UDP in pkt:
        try:
            if TLS in pkt:
                ret = pkt[TLS].version
            elif TLS in TLS(bytes(pkt[UDP].payload)):
                ret = TLS(bytes(pkt[UDP].payload)).version
        except Exception as e:
            print('proto error!', e, pkt)
    return ret

NUM_SHORT_FEATURE = 12
def extract_feature_short(pkts, device_ip):
    # features:
    # dst_IP, dst_domain,
    # tcp/udp, flags, src_port, dst_port,
    # tls,
    # pkt size, inter-arrival time
    features = []

    pkt_sizes = []
    int_times = []
    last_ts = 0
    for pkt in pkts:
        pkt_sizes.append(len(pkt))
        if pkt[IP].src == device_ip:
            features.append(1)
            features.extend([int(dst) for dst in pkt[IP].dst.split('.')])
            # features.append(utils.LOCAL_DNS[pkt[IP].dst])
            if TCP in pkt:
                features.append(0)
                features.append(int(pkt[TCP].flags) & MEANINGFUL_FLAG)
                features.append(pkt[TCP].sport)
                features.append(pkt[TCP].dport)
                features.append(get_tls_version(pkt))
                # if TLS in pkt:
                #     features.append(pkt[TLS].version)
                # elif TLS in TLS(pkt[TCP].payload):
                #     features.append(TLS(pkt[TCP].payload).version)
                # else:
                #     features.append(0)
                features.append(len(pkt))
            elif UDP in pkt:
                features.append(1)
                features.append(0)
                features.append(pkt[UDP].sport)
                features.append(pkt[UDP].dport)
                features.append(get_tls_version(pkt))
                # if TLS in pkt:
                #     features.append(pkt[TLS].version)
                # elif TLS in TLS(pkt[UDP].payload):
                #     features.append(TLS(pkt[UDP].payload).version)
                # else:
                #     features.append(0)
                features.append(len(pkt))
            else:
                features.append([-1] * 6)
        else:
            features.append(0)
            features.extend([int(src) for src in pkt[IP].src.split('.')])
            # features.append(utils.LOCAL_DNS[pkt[IP].src])
            if TCP in pkt:
                features.append(0)
                features.append(int(pkt[TCP].flags) & MEANINGFUL_FLAG)
                features.append(pkt[TCP].dport)
                features.append(pkt[TCP].sport)
                features.append(get_tls_version(pkt))
                # if TLS in pkt:
                #     features.append(pkt[TLS].version)
                # elif TLS in TLS(bytes(pkt[TCP].payload)):
                #     features.append(TLS(bytes(pkt[TCP].payload)).version)
                # else:
                #     features.append(0)
                features.append(len(pkt))
            elif UDP in pkt:
                features.append(1)
                features.append(0)
                features.append(pkt[UDP].dport)
                features.append(pkt[UDP].sport)
                features.append(get_tls_version(pkt))
                # if TLS in pkt:
                #     features.append(pkt[TLS].version)
                # elif TLS in TLS(bytes(pkt[UDP].payload)):
                #     features.append(TLS(bytes(pkt[UDP].payload)).version)
                # else:
                #     features.append(0)
                features.append(len(pkt))
            else:
                features.append([-1] * 6)

        if last_ts == 0:
            features.append(0)
        else:
            features.append(float(pkt.time) - last_ts)
            int_times.append(float(pkt.time) - last_ts)
        last_ts = float(pkt.time)

    if len(pkts) < utils.SHORT_PKT_THRES:
        features.extend([-1] * NUM_SHORT_FEATURE * int(utils.SHORT_PKT_THRES - len(pkts)))
        # features.extend(
        #     [-1] * int(
        #         (utils.SHORT_PKT_THRES - len(pkts)) * (len(features) / len(pkts))
        #     ))

    features.append(len(pkt_sizes))
    features.append(np.average(pkt_sizes))
    features.append(np.std(pkt_sizes))

    features.append(sum(int_times))
    features.append(np.average(int_times))
    features.append(np.std(int_times))

    return features

NUM_LONG_FEATURE = 11 # 12
def extract_feature_long(pkts, device_ip):
    # features:
    # first 3 distinct dst ip,
    # tcp/udp, flags, src_port, dst_port,
    # tls,
    # pkt size, inter-arrival time
    features = []
    distinct_ip = []
    last_ts = 0
    inter_arrival_time = []
    pkt_sizes = []
    upload, download = [], []
    for pkt in pkts:
        if last_ts == 0:
            last_ts = float(pkt.time)
        else:
            inter_arrival_time.append(float(pkt.time) - last_ts)
            last_ts = float(pkt.time)
        pkt_sizes.append(len(pkt))

        # first 3 distinct ip
        if pkt[IP].src == device_ip:
            upload.append(len(pkt))
            if (pkt[IP].dst not in distinct_ip) and len(distinct_ip) < 3:
                distinct_ip.append(pkt[IP].dst)
                features.extend(pkt[IP].dst.split('.'))
                if TCP in pkt:
                    features.append(0)
                    features.append(int(pkt[TCP].flags))
                    features.append(pkt[TCP].sport)
                    features.append(pkt[TCP].dport)
                    features.append(get_tls_version(pkt))
                    # features.append(pkt[TLS].version if TLS in pkt else 0)
                    features.append(len(pkt))
                elif UDP in pkt:
                    features.append(1)
                    features.append(0)
                    features.append(pkt[UDP].sport)
                    features.append(pkt[UDP].dport)
                    features.append(get_tls_version(pkt))
                    # features.append(0)
                    features.append(len(pkt))
                else:
                    features.append([-1] * 6)
        else:
            download.append(len(pkt))
            if (pkt[IP].src not in distinct_ip) and len(distinct_ip) < 3:
                distinct_ip.append(pkt[IP].src)
                features.extend(pkt[IP].src.split('.'))
                if TCP in pkt:
                    features.append(0)
                    features.append(int(pkt[TCP].flags))
                    features.append(pkt[TCP].sport)
                    features.append(pkt[TCP].dport)
                    features.append(get_tls_version(pkt))
                    # features.append(pkt[TLS].version if TLS in pkt else 0)
                    features.append(len(pkt))
                elif UDP in pkt:
                    features.append(1)
                    features.append(0)
                    features.append(pkt[UDP].sport)
                    features.append(pkt[UDP].dport)
                    features.append(get_tls_version(pkt))
                    # features.append(0)
                    features.append(len(pkt))
                else:
                    features.append([-1] * 6)

    if len(features) < 3*NUM_LONG_FEATURE:
        features.extend([-1] * (3*NUM_LONG_FEATURE - len(features)))

    # statistics
    features.append(len(pkts))
    features.append(np.average(pkt_sizes))
    features.append(np.std(pkt_sizes))
    features.append(min(pkt_sizes))
    features.append(np.percentile(pkt_sizes, 25))
    features.append(np.percentile(pkt_sizes, 50))
    features.append(np.percentile(pkt_sizes, 75))
    features.append(max(pkt_sizes))

    features.append(sum(inter_arrival_time))
    features.append(np.average(inter_arrival_time))
    features.append(np.std(inter_arrival_time))
    features.append(min(inter_arrival_time))
    features.append(np.percentile(inter_arrival_time, 25))
    features.append(np.percentile(inter_arrival_time, 50))
    features.append(np.percentile(inter_arrival_time, 75))
    features.append(max(inter_arrival_time))

    features.append(float(sum(download)) / (sum(upload)+sum(download)))

    return features


text:
import numpy as np

# TODO c++ should be faster
# TODO batch get

class RangeTree:
    def __init__(self, values):
        capacity = len(values)
        self.size = 1
        while self.size < capacity:
            self.size *= 2
        self.values = np.zeros(2 * self.size)

        self.values[self.size:self.size + capacity] = values
        for pos in range(self.size - 1, -1, -1):
            self.values[pos] = self.values[2 * pos] + self.values[2 * pos + 1]

        self.capacity = capacity
        self.constant_add = 0

    def update(self, pos, x):
        pos += self.size
        self.values[pos] = x - self.constant_add
        pos //= 2
        while (pos):
            self.values[pos] = self.values[2 * pos] + self.values[2 * pos + 1]
            pos //= 2

    # adds constant to all values in the tree
    def add_value(self, value):
        self.constant_add += value

    def get_sum(self):
        return self.values[1] + self.constant_add * self.capacity # ignores elements in [capacity, size)

    def get(self, x):
        pos = 1
        cnt = self.size
        while pos < self.size:
            pos *= 2
            cnt /= 2
            if x > self.values[pos] + self.constant_add * cnt:
                x -= self.values[pos] + self.constant_add * cnt
                pos += 1
        return pos - self.size, self.values[pos] + self.constant_add

max_stars_repo_name: mahkons/prioritized-image-replay

max_issues_repo_name: mahkons/prioritized-image-replay

max_forks_repo_name: mahkons/prioritized-image-replay


text:
"""Average profiles from LAMMPS."""
import argparse
from math import ceil
import pathlib
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.cm import get_cmap
from matplotlib.gridspec import GridSpec

plt.style.use('seaborn-talk')

def read_lammps_profile(filename):
    """Read the profiles from chunked LAMMPS output."""
    with open(filename, 'r') as infile:
        cols = 0  # number of columns
        lines_to_read = 0  # number of lines in a set
        data = []
        keys = []
        for lines in infile:
            if lines.find('# Chunk ') != -1:
                # read shape of data
                linesplit = lines.strip().split()
                cols = len(linesplit) - 1
                keys = [key.lower() for key in linesplit[1:]]
                continue
            if cols > 0:
                linesplit = lines.strip().split()
                if lines_to_read > 0:
                    if len(linesplit) == cols:
                        data.append([float(i) for i in linesplit])
                        lines_to_read -= 1
                else:
                    if data:
                        yield keys, step, data
                        data = []
                    step, lines_to_read = [int(i) for i in linesplit[:2]]
        if data:
            yield keys, step, data

def update_variance(xdata, length, mean, var_m2):
    """Update estimate of mean and variance with new observations."""
    length = length + 1.0
    delta = xdata - mean
    mean = mean + delta / length
    var_m2 = var_m2 + delta * (xdata - mean)
    if length < 2:
        variance = float('inf')
    else:
        variance = var_m2 / (length - 1.0)
    return length, mean, var_m2, variance

def update_the_variance(key, average_vars, column):
    """Update the variance for the given key."""
    if key not in average_vars['average']:
        average_vars['average'][key] = np.zeros_like(column)
        average_vars['m2'][key] = np.zeros_like(column)
        average_vars['n'][key] = 0.0
        average_vars['var'][key] = np.full_like(column, float('inf'))
    new_n, new_mean, new_m2, new_var = update_variance(
        column,
        average_vars['n'][key],
        average_vars['average'][key],
        average_vars['m2'][key],
    )
    average_vars['n'][key] = new_n
    average_vars['average'][key] = new_mean
    average_vars['m2'][key] = new_m2
    average_vars['var'][key] = new_var

def average_profiles(infile):
    """Read the given input file and return averaged profiles."""
    raw_data_matrix = []
    raw_data = {}
    # Variables for averaging:
    average_vars = {
        'average': {},
        'n': {},
        'm2': {},
        'var': {},
    }
    for keys, _, data in read_lammps_profile(infile):
        raw_data_matrix.append(data)
        new_data = np.array(data)
        for i, key in enumerate(keys):
            column = new_data[:, i]
            update_the_variance(key, average_vars, column)
            if key not in raw_data:
                raw_data[key] = []
            raw_data[key].append(column)
    return (raw_data,
            raw_data_matrix,
            average_vars['average'],
            average_vars['var'])

def plot_all_items(data, error):
    """Plot all items in a dict."""
    fig = plt.figure()
    if len(data) < 3:
        ncol = 1
    else:
        ncol = 2
    nrow = ceil(len(data) / ncol)
    grid = GridSpec(nrow, ncol)
    for i, (key, val) in enumerate(data.items()):
        row, col = divmod(i, ncol)
        axi = fig.add_subplot(grid[row, col])
        xpos = np.arange(len(val))
        std = np.sqrt(error[key])
        axi.fill_between(xpos, val - std, val + std, alpha=0.5)
        axi.plot(xpos, val, lw=2)
        axi.set_xlabel('Bin no.')
        axi.set_ylabel(key)
    fig.tight_layout()

def plot_xy_data(xdata, ydata, yerror=None, xlabel='x', ylabel='y'):
    """Plot xy data."""
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    if yerror is not None:
        ax1.fill_between(xdata, ydata - yerror, ydata + yerror, alpha=0.5)
        ax1.plot(xdata, ydata - yerror, ls=':', alpha=0.5, color='#262626')
        ax1.plot(xdata, ydata + yerror, ls=':', alpha=0.5, color='#262626')
    line, = ax1.plot(xdata, ydata, lw=2)
    ax1.scatter(xdata, ydata, marker='o', alpha=0.5, color=line.get_color())
    ax1.set_xlabel(xlabel)
    ax1.set_ylabel(ylabel)
    fig.tight_layout()

def plot_all_sets(raw_data, key, color_map_name='viridis'):
    """Plot all sets for a given variable."""
    data = raw_data[key]
    cmap = get_cmap(name=color_map_name)
    colors = cmap(np.linspace(0, 1, len(data)))
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    xpos = None
    for i, datai in enumerate(data):
        if xpos is None:
            xpos = np.arange(len(datai))
        ax1.plot(xpos, datai, color=colors[i], alpha=0.5)
    ax1.set_xlabel('Bin no.')
    ax1.set_ylabel(key)
    fig.tight_layout()

def write_output(filename, keys, data):
    """Store the data in a new file."""
    header = ' '.join(['#'] + keys)
    cols = len(keys)
    rows = len(data[keys[0]])
    data_matrix = np.zeros((rows, cols))
    for i, key in enumerate(keys):
        data_matrix[:, i] = data[key]
    print('Writing file "{}"'.format(filename))
    np.savetxt(filename, data_matrix, header=header)

def write_output_error(filename, keys, data, error):
    """Store the data in a new file."""
    header = ['#']
    for key in keys:
        header.append(key)
        header.append('std_{}'.format(key))
    cols = len(keys) * 2
    rows = len(data[keys[0]])
    data_matrix = np.zeros((rows, cols))
    for i, key in enumerate(keys):
        data_matrix[:, 2 * i] = data[key]
        data_matrix[:, 2 * i + 1] = np.sqrt(error[key])
    print('Writing file "{}"'.format(filename))
    np.savetxt(filename, data_matrix, header=' '.join(header))

def main(infile, make_plot, split=False):
    """Read the input file and average the profiles within it."""
    print('Reading file "{}"'.format(infile))
    raw_data, raw_matrix, average_data, var_data = average_profiles(infile)
    print('Data sets: {}'.format(len(raw_matrix)))
    print('Variables in sets:')
    for i in raw_data:
        print('- "{}"'.format(i))
    if make_plot:
        print('Plotting all averaged profiles.')
        plot_all_items(average_data, var_data)
        for ykey in raw_data:
            if ykey in ('coord1', 'chunk'):
                continue
            plot_all_sets(raw_data, ykey)
            xkey = 'coord1'
            plot_xy_data(
                average_data[xkey],
                average_data[ykey],
                yerror=np.sqrt(var_data[ykey]),
                xlabel=xkey,
                ylabel=ykey,
            )
    write_output(
        'averaged-{}.txt'.format(pathlib.Path(infile).stem),
        [i for i in average_data],
        average_data
    )
    write_output_error(
        'averaged-error-{}.txt'.format(pathlib.Path(infile).stem),
        [i for i in average_data],
        average_data,
        var_data,
    )
    if split:
        for i in average_data:
            write_output_error(
                'averaged-{}-{}.txt'.format(
                    i, pathlib.Path(infile).stem
                ),
                [i],
                average_data,
                var_data,
            )

def create_parser():
    """Create a parser."""
    parser = argparse.ArgumentParser(description='Average profile(s) from LAMMPS')
    parser.add_argument(
        '-f',
        '--file',
        help='File to average',
        required=True,
        default='x-temp.txt'
    )
    parser.add_argument(
        '-s',
        '--split',
        help='Write a new file for each variable',
        required=False,
        action='store_true'
    )
    parser.add_argument(
        '-p',
        '--plot',
        help='Write a new file for each variable',
        required=False,
        action='store_true'
    )
    return parser

if __name__ == '__main__':
    ARGS = create_parser().parse_args()
    main(ARGS.file, ARGS.plot, split=ARGS.split)
    if ARGS.plot:
        plt.show()


text:
import tornado.ioloop
import tornado.web

from astropy.time import Time
from astropy.coordinates import solar_system_ephemeris, EarthLocation
from astropy.coordinates import get_body_barycentric, get_body, get_moon
from astropy.timeseries import TimeSeries
from astropy import units as u

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
from math import sin, cos, radians
from random import randint

planets = ['sun',
 'mercury',
 'venus',
 'earth',
 'moon',
 'mars',
 'jupiter',
 'saturn',
 'uranus',
 'neptune']
 # 'pluto'

planets1 = ['sun',
            'mercury',
            'venus',
            'earth',
            'mars']

sizes = [100, 15, 37, 40, 22]

clrs = ['#%06X' % randint(0, 0xFFFFFF) for i in range(len(planets1))]

def get_coordinates(t):
    with solar_system_ephemeris.set('builtin'):
        coords = [get_body(i, t) for i in planets1]
        x = [i.heliocentrictrueecliptic.lon.deg for i in coords]
        y = [i.heliocentrictrueecliptic.lat.deg for i in coords]
        z = [i.heliocentrictrueecliptic.distance.value for i in coords]

        dx = [i.heliocentrictrueecliptic.distance.value * sin(radians(i.heliocentrictrueecliptic.lon.deg)) * cos(radians(i.heliocentrictrueecliptic.lat.deg)) for i in coords]
        dy = [i.heliocentrictrueecliptic.distance.value * sin(radians(i.heliocentrictrueecliptic.lon.deg)) * sin(radians(i.heliocentrictrueecliptic.lat.deg)) for i in coords]
        dz = [i.heliocentrictrueecliptic.distance.value * cos(radians(i.heliocentrictrueecliptic.lon.deg)) for i in coords]
    return (dx,dy,dz)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        t = Time.now()

        fig = plt.figure()
        ax = plt.axes(projection='3d')

        dx,dy,dz = get_coordinates(t)
        ax.scatter(dx, dy, dz, marker='o', sizes = sizes, c = clrs)

        for i, txt in enumerate(planets1):
            ax.text(dx[i], dy[i], dz[i], txt)

        for i in range(10):
            print(t)
            t -= 10
            x,y,z = get_coordinates(t)
            ss = [s * (0.5 - 0.02 * i) for s in sizes]
            ax.scatter(x, y, z, marker='o', sizes = ss, c = clrs)

        print(t)
        plt.show()

        self.write("Hello, world")

def make_app():
    return tornado.web.Application([
        (r"/", MainHandler),
    ])

if __name__ == "__main__":
    app = make_app()
    app.listen(8888)
    tornado.ioloop.IOLoop.current().start()

max_stars_repo_name: Cadovvl/astrorouting

max_issues_repo_name: Cadovvl/astrorouting

max_forks_repo_name: Cadovvl/astrorouting

reason: from astropy


text:
import numpy as np
import pyqtgraph as pg

from .plot import Step2Plot
from .get import Get

class CustomAxis(pg.AxisItem):
    def tickStrings(self, values, scale, spacing):
        values[values == 0] = np.NaN  # remove 0 before division
        return ['{:.4f}'.format(1. / i) for i in values]

class Step2GuiHandler(object):
    col_width = [70, 50, 50, 50, 50]

    def __init__(self, parent=None):
        self.parent = parent

    def update_widgets(self):
        o_step2_plot = Step2Plot(parent=self.parent)
        o_step2_plot.prepare_data()
        o_step2_plot.display_image()
        o_step2_plot.init_roi_table()
        self.check_run_normalization_button()

    def check_add_remove_roi_buttons(self):
        nbr_row = self.parent.ui.normalization_tableWidget.rowCount()
        if nbr_row == 0:
            _status_remove = False
        else:
            _status_remove = True

        self.parent.ui.normalization_remove_roi_button.setEnabled(_status_remove)

    def check_run_normalization_button(self):
        o_get = Get(parent=self.parent)
        _status = o_get.status_of_run_normalization_button()
        self.parent.ui.normalization_button.setEnabled(_status)
        self.update_instructions()

    def enable_xaxis_button(self, tof_flag=True):
        list_ui = [self.parent.step2_ui['xaxis_file_index'],
                   self.parent.step2_ui['xaxis_lambda'],
                   self.parent.step2_ui['xaxis_tof']]

        if tof_flag:
            for _ui in list_ui:
                _ui.setEnabled(True)
        else:
            list_ui[1].setEnabled(False)
            list_ui[2].setEnabled(False)
            list_ui[0].setChecked(True)

    def update_instructions(self):
        o_get = Get(parent=self.parent)
        _status = o_get.status_of_run_normalization_button()
        if not _status:
            instructions = 'Activate at least one Background ROI ' \
                           'to enable the Normalization switch!'
        else:
            instructions = 'A Background ROI will improve the normalization!'
        self.parent.ui.normalization_instructions.setHtml(instructions)

max_stars_repo_name: ornlneutronimaging/iBeatles

max_issues_repo_name: ornlneutronimaging/iBeatles

max_forks_repo_name: ornlneutronimaging/iBeatles


text:
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

# name = 'Com_Sensores'
name = 'Sem_Sensores'

def saveToFile(nameFile, y):
    np.save(nameFile+name+'array.npy', y)    # .npy extension is added if not given

def loadFromFile(nameFile):
    d = np.load(nameFile+name+'array.npy')
    return d

def graphAcc(y, x):
    # x = np.arange(100)
    fig = plt.figure()
    ax = plt.subplot(111)
    ax.plot(x, y, label='$'+name)

    plt.title('Pacotes Recebidos x Delay Entre 100 Pacotes de 30 bytes')
    plt.xlabel('Delay(us)')
    plt.ylabel('Pacotes Recebidos(%)')
    # fig.savefig('img/pacotesBLE'+name+'.png')

def graphPyTime(y, x):
    # x = np.arange(100)
    fig = plt.figure()
    ax = plt.subplot(111)
    ax.plot(x, y, label='$'+name)

    plt.title('Tempo Decorrido de Captura x Delay Entre 100 Pacotes de 30 bytes')
    plt.xlabel('Delay(us)')
    plt.ylabel('Tempo da Captura de Python(s)')
    # fig.savefig('img/pyCaptTime'+name+'.png')
    plt.show()

if __name__ == "__main__":
    acerto = np.zeros((100,1))
    tempDec = np.zeros((100,1))

    leitAtual = 0
    while(leitAtual < 100):
        totalSum = leitAtual
        lastnum = 66

        print("Leitura[",leitAtual,"]")

        acc = (totalSum)/100
        # print("Acerto: ", acc)
        acerto[leitAtual] = acc
        # acerto = np.insert(acerto, leitAtual, acc, axis=1)
        # print(acerto)
        elapsed_time = 53
        tempDec[leitAtual] = elapsed_time

        # tempDec = np.insert(tempDec, leitAtual, elapsed_time, axis=1)
        # print("Tempo decorrido: ", tempDec)
        print("\n")
        leitAtual+=1

    acerto = np.reshape(acerto, (100, ))

    tempDec = np.reshape(tempDec, (100, ))
    print(acerto)
    print("Tempo decorrido: ", tempDec)

    delay = np.arange(200000, 200,-1998)
    print('Delay ', delay)

    saveToFile('delay', delay)
    saveToFile('acerto', acerto)
    saveToFile('TempoDec', tempDec)
    print('Opening Files')

    delay = loadFromFile('delay')
    acerto = loadFromFile('acerto')
    tempDec = loadFromFile('TempoDec')

    graphAcc(acerto, delay)
    graphPyTime(tempDec, delay)

max_stars_repo_name: abdullah-zaiter/Sign-Language-By-Glove

max_issues_repo_name: abdullah-zaiter/Sign-Language-By-Glove

max_forks_repo_name: abdullah-zaiter/Sign-Language-By-Glove


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Generates figures depicting pipeline distributions + parameter impact
"""

from pathlib import Path

import matplotlib.colors as mcolors
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.utils.extmath import randomized_svd

from vibecheck.plotting import savefig

plt.rcParams['svg.fonttype'] = 'none'
plt.rcParams['font.sans-serif'] = ['Myriad Pro']
plt.rcParams['font.size'] = 20.0

DATA_DIR = Path('./data/derivatives').resolve()
FIG_DIR = Path('./figures').resolve()
GENENORM_ORDER = ['None', 'srs', 'zscore']
NORMSTRU_ORDER = [False, True]
DIST_PARAMS = (
    ('dist_spearmanr', 'gene_norm', GENENORM_ORDER, dict(
        xticks=(-0.6, -0.4, -0.2, 0), yticks=(0, 2.5, 5),
        xticklabels=(-0.6, None, None, 0),
        yticklabels=(0, None, 5),
        xlim=(-0.6, 0), ylim=[None, 5.0],
        xlabel='correlation (r)', ylabel=''
    )),
    ('silhouette', 'gene_norm', GENENORM_ORDER, dict(
        xticks=(-0.9, -0.7, -0.5, -0.3, -0.1), yticks=(0, 3.5, 7),
        xticklabels=(-0.9, None, None, None, -0.1),
        yticklabels=(0, None, 7),
        xlim=(-0.9, -0.1), ylim=[None, 7.0],
        xlabel='silhouette score', ylabel=''
    )),
    ('t1t2_spearmanr', 'gene_norm', GENENORM_ORDER, dict(
        xticks=(-0.2, 0.1, 0.4, 0.7, 1.0), yticks=(0, 4, 8, 12, 16),
        xticklabels=(-0.2, None, None, None, 1.0),
        yticklabels=(0, None, None, None, 16),
        xlim=(-0.2, 1.0), ylim=[None, 16],
        xlabel='correlation (rho)', ylabel=''
    ))
)
PARAMETERS = dict(
    ibf_threshold='ibf threshold',
    probe_selection='probe selection',
    donor_probes='donor probe selection',
    lr_mirror='mirroring samples',
    missing='missing data',
    tolerance='sample-to-region distance',
    sample_norm='sample normalization',
    gene_norm='gene normalization',
    norm_matched='normalize matched',
    norm_structures='normalize structures',
    region_agg='sample-to-region method',
    agg_metric='sample-to-region metric',
    corrected_mni='updated mni',
    reannotated='updated annotations',
    atlas_name='atlas type'
)
PALETTE = (np.row_stack([(35, 95, 192),
                         (205, 78, 78),
                         (123, 30, 153)]) / 255).tolist()

def make_hexplot(x, y, parameter, fname=None, vmin=None, vmax=None, cmaps=None,
                 order=None, **kwargs):
    if vmin is not None or vmax is not None:
        norm = mcolors.LogNorm(vmin=vmin, vmax=vmax)
    else:
        norm = None
    fig, ax = plt.subplots(1, 1)
    if order is None:
        order = parameter.astype('category').cat.categories
    if cmaps is None:
        cmaps = ['Blues', 'Reds', 'Purples']
    cbar = None
    for n, code in enumerate(order):
        mask = np.asarray(parameter == code)
        opts = {'norm': norm} if norm is not None else {'bins': 'log'}
        opts.update(kwargs)
        coll = ax.hexbin(x[mask], y[mask], cmap=cmaps[n],
                         alpha=1.0, mincnt=1, linewidth=0.2, **opts)
        if cbar is None:
            cbar = fig.colorbar(coll, shrink=0.83, label='density')
            cbar.ax.collections[0].set_cmap('gray_r')
            if norm is None:
                norm = mcolors.LogNorm(vmin=cbar.vmin, vmax=cbar.vmax)
    ax.set(xticks=(-0.4, 0, 0.4, 0.8),
           xticklabels=(-0.4, None, None, 0.8),
           xlim=(-0.4, 0.9), xlabel='pc1 scores',
           yticks=(-0.3, 0, 0.3, 0.6),
           yticklabels=(-0.3, None, None, 0.6),
           ylim=(-0.4, 0.7), ylabel='pc2 scores')
    sns.despine(ax=ax, trim=True)
    fig.tight_layout()
    if fname is not None:
        if not fname.endswith('.svg'):
            fname += '.svg'
        savefig(fig, FIG_DIR / fname, dpi=1000)
    else:
        return fig

def make_displots(pipelines):
    fig, axes = plt.subplots(1, 3, figsize=(15, 4))

    rgb = np.array([[154, 27, 91]]) / 255
    ylim = ((0, 10), (0, 5), (0, 20))
    for n, params in enumerate(DIST_PARAMS):
        est, kwargs = params[0], params[-1].copy()
        del kwargs['yticks'], kwargs['yticklabels'], kwargs['ylim']
        ax = sns.kdeplot(x=est, data=pipelines, color=rgb, fill=True,
                         ax=axes[n])
        ax.set(**kwargs, ylim=ylim[n], yticks=np.linspace(*ylim[n], 6))
        sns.despine(ax=ax, offset=5, trim=True)

    savefig(fig, FIG_DIR / 'full_distributions.svg')

def make_colorbar_plot(cmap):
    fig, ax = plt.subplots(1, 1, figsize=(9, 1.5))
    a = np.array([[0, 1]])
    img = ax.imshow(a, cmap=cmap)
    ax.set_visible(False)
    cax = fig.add_axes([0.1, 0.2, 0.8, 0.6])
    cbar = fig.colorbar(img, orientation="horizontal", cax=cax)
    cbar.set_ticks([])
    savefig(fig, FIG_DIR / f'colorbar_{cmap}.svg')

def make_impact_plot(ranks):
    n_colors = len(ranks)
    cmap = sns.color_palette('rocket_r', n_colors=n_colors)
    fig, ax = plt.subplots(1, 1)
    ax = sns.heatmap(ranks, square=True, cmap=cmap,
                     yticklabels=[PARAMETERS.get(p) for p in ranks.index],
                     xticklabels=['CGE', 'GCE', 'RGE'], ax=ax)
    ax.set(xlabel='', ylabel='')
    ax.tick_params(width=0)
    colorbar = ax.collections[0].colorbar
    r = colorbar.vmax - colorbar.vmin
    colorbar.set_ticks([
        colorbar.vmin + 0.5 * r / (n_colors) + r * i / (n_colors)
        for i in range(0, n_colors, 7)
    ])
    colorbar.set_ticklabels((1, 8, 15))
    colorbar.set_label('impact rank')
    colorbar.ax.tick_params(width=0)
    colorbar.ax.invert_yaxis()
    savefig(fig, FIG_DIR / 'impact.svg')

def pca(data):
    data = np.asarray(data)
    data -= data.mean(axis=0, keepdims=True)
    u, s, v = randomized_svd(data, 2, random_state=1234)
    v = v.T
    scores = data @ v
    return scores

if __name__ == "__main__":
    pipelines = pd.read_csv(DATA_DIR / 'pipelines.csv.gz') \
                  .replace({np.nan: "None"})
    impact = pd.read_csv(DATA_DIR / 'impact.csv', index_col=0)
    twilight = np.asarray(sns.color_palette('twilight', n_colors=9))

    # make distributional plots showing impact of pipelines
    make_displots(pipelines)

    # now split distributions based on selected parameter
    fig, axes = plt.subplots(1, 3, figsize=(15, 4))
    for n, (key, hue, hue_order, params) in enumerate(DIST_PARAMS):
        ax = sns.kdeplot(x=key, hue=hue, data=pipelines, fill=True,
                         palette=dict(zip(hue_order, twilight[[3, 1, 7]])),
                         hue_order=hue_order, ax=axes[n], legend=False)
        ax.set(**params)
        sns.despine(ax=ax, trim=True, offset=5)
    savefig(fig, FIG_DIR / 'distplot_genenorm.svg')

    # make plot showing relative parameter importance
    order = impact.rank().mean(axis=1).sort_values(ascending=False).index
    ranks = impact.loc[order].rank(ascending=False)
    make_impact_plot(ranks)

    # make plot of PCs of pipelines
    scores = pca(pipelines[['dist_spearmanr', 'silhouette', 't1t2_spearmanr']])
    gn = [sns.light_palette(f, as_cmap=True) for f in twilight[[3, 1, 7]]]
    make_hexplot(*scores.T, pipelines['gene_norm'], 'hex_genenorm',
                 vmin=1, vmax=5000, rasterized=True, cmaps=gn,
                 order=GENENORM_ORDER)
    ns = [sns.light_palette(f, as_cmap=True) for f in twilight[[2, 6]]]
    make_hexplot(*scores.T, pipelines['norm_structures'], 'hex_normstruct',
                 vmin=1, vmax=5000, rasterized=True, cmaps=ns,
                 order=NORMSTRU_ORDER)

    make_colorbar_plot('BuPu')


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 24 15:22:02 2019

@author: liuxingyu
"""

import os
import numpy as np
import pandas as pd
import subprocess
import glob
import mne
from mne_bids import BIDSPath, write_raw_bids

#%% bash script to reorganize files from date-wise to sub-wise

# =============================================================================
# raw_root=/nfs/e5/studyforrest/forrest_movie_meg/raw_data
# raw_subwise=/nfs/e5/studyforrest/forrest_movie_meg/raw_data_subwise
#
# # reorganize meg data
# for i in $(ls ${raw_root}/2019* -d) ; do cd $i; for j in $(ls S* -d); do mkdir ${raw_subwise}/sub-${j: 1: 2}-raw ; ln -s $i/$j ${raw_subwise}/sub-${j: 1: 2}-raw/ ; done ; done
# # reorganize mri data
# for i in $(ls ${raw_root}/2019* -d) ; do cd ${i}/MRI; for j in $(ls *S* -d); do mkdir ${raw_subwise}/sub-${j: 0-2: 2}-raw ; ln -s $i/MRI/$j ${raw_subwise}/sub-${j: 0-2: 2}-raw/MRI ; done ; done
# # reorganize emptyroom meg
# mkdir ${raw_subwise}/sub-emptyroom
# for i in $(ls ${raw_root}/2019* -d) ; do cd $i; for j in $(ls *Noise* -d); do ln -s $i/$j ${raw_subwise}/sub-emptyroom/ ; done ; done
# =============================================================================

#%% convert raw data to bids-format
raw_subwise = '/nfs/e5/studyforrest/forrest_movie_meg/raw_data_subwise'
bids_dir = '/nfs/e5/studyforrest/forrest_movie_meg'

#%% data of real participant
sublist = ['{0:0>2d}'.format(i) for i in np.arange(1, 12)]
for subid in sublist:
    sub_raw_dir = os.path.join(raw_subwise, 'sub-{0}-raw'.format(subid))
    os.chdir(sub_raw_dir)

    # MRI
    if len(glob.glob(os.path.join(sub_raw_dir, '*t1*.nii.gz'))) == 0:
        # convert to dicom
        cmd_dcm2nii = 'dcm2niix -o {0} -b y {0}/MRI'.format(sub_raw_dir)
        subprocess.call(cmd_dcm2nii, shell=True)
        subprocess.call('rm {0}/*localizer*'.format(sub_raw_dir), shell=True)

    # rename T1w
    t1 = glob.glob(os.path.join(sub_raw_dir, '*t1*.nii.gz'))
    t1_bids_name = 'sub-{0}_ses-movie_T1w.nii.gz'.format(subid)
    subprocess.call('mv {0} {1}/{2}'.format(t1[0], sub_raw_dir, t1_bids_name), shell=True)
    subprocess.call('mv {0} {1}/{2}'.format(t1[0].replace('nii.gz', 'json'), sub_raw_dir,
                                            t1_bids_name.replace('nii.gz', 'json')), shell=True)
    # deface for t1w
    anat_dir = os.path.join(bids_dir, 'sub-{0}'.format(subid), 'ses-movie', 'anat')
    if os.path.exists(anat_dir) is False: os.makedirs(anat_dir)
    subprocess.call('cp {0}/{1} {2}/{1}'.format(sub_raw_dir,t1_bids_name.replace('nii.gz', 'json'), anat_dir), shell=True)
    subprocess.call('pydeface {0}/{1} --outfile {2}/{1}'.format(sub_raw_dir, t1_bids_name, anat_dir), shell=True)

    print('sub-{0}: MRI convertion done'.format(subid))

    # MEG
    meg_files = glob.glob('S*.ds')
    meg_files.sort()

    for file in meg_files:
        runid = file[-5:-3]
        raw = mne.io.read_raw_ctf(file)
        raw.info['line_freq'] = 50

        # event data
        raw.set_annotations(None)
        events = mne.find_events(raw, stim_channel='UPPT001', min_duration=2/raw.info['sfreq'])

        # repair annotations
        events_id = {'watch_{0}_{1}s'.format((i-1)*10, i*10) : int(i) for i in np.unique(events[:,-1]) if i != 255}
        events_id['beginning'] = 255
        annot_new = mne.annotations_from_events(events, raw.info['sfreq'], event_desc={v: k for k, v in events_id.items()},
                                                first_samp=raw.first_samp)
        # raw.set_annotations(annot_new)

        # convert data to bids format
        bids_path = BIDSPath(subject=subid, session='movie', task='movie', run=runid,
                             datatype='meg', root=bids_dir)
        write_raw_bids(raw, bids_path=bids_path, overwrite=True,
                       events_data=events, event_id=events_id)

    print('sub-{0}: MEG convertion done'.format(subid))

#%% data of empty room
filelist = os.listdir(os.path.join(raw_subwise, 'sub-emptyroom'))
seslist = [f.split('_')[-2] for f in filelist]

for ses in seslist:
    meg_file = glob.glob(os.path.join(raw_subwise, 'sub-emptyroom','*{0}*.ds'.format(ses)))[0]

    raw = mne.io.read_raw_ctf(meg_file)
    raw.info['line_freq'] = 50
    bids_path = BIDSPath(subject='emptyroom', session=ses, task='noise', datatype='meg', root=bids_dir)

    write_raw_bids(raw, bids_path=bids_path, overwrite=True)


text:
# Copyright 2020 The Flax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for flax.examples.imagenet.train."""

from absl.testing import absltest

from flax import nn
import train

import jax
from jax import random
import jax.numpy as jnp

import numpy as onp
import tensorflow as tf

# Parse absl flags test_srcdir and test_tmpdir.
jax.config.parse_flags_with_absl()
# Require JAX omnistaging mode.
jax.config.enable_omnistaging()

class TrainTest(absltest.TestCase):

  def setUp(self):
    super().setUp()
    # Make sure tf does not allocate gpu memory.
    tf.config.experimental.set_visible_devices([], 'GPU')

  def test_create_model(self):
    variables = train.initialized(random.PRNGKey(0), 224)
    x = random.normal(random.PRNGKey(1), (8, 224, 224, 3))
    y = train.model(train=False).apply(variables, x)
    self.assertEqual(y.shape, (8, 1000))

if __name__ == '__main__':
  absltest.main()

reason: import numpy,import jax,from jax