text:
'''
Author: Alex Wong <alexw@cs.ucla.edu>

If you use this code, please cite the following paper:

A. Wong, and S. Soatto. Unsupervised Depth Completion with Calibrated Backprojection Layers.
https://arxiv.org/pdf/2108.10531.pdf

@inproceedings{wong2021unsupervised,
  title={Unsupervised Depth Completion with Calibrated Backprojection Layers},
  author={Wong, Alex and Soatto, Stefano},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={12747--12756},
  year={2021}
}
'''
import os, sys, glob, argparse
import multiprocessing as mp
import numpy as np
import cv2
sys.path.insert(0, 'src')
import data_utils

VOID_ROOT_DIRPATH       = os.path.join('data', 'void_release')
VOID_DATA_150_DIRPATH   = os.path.join(VOID_ROOT_DIRPATH, 'void_150')
VOID_DATA_500_DIRPATH   = os.path.join(VOID_ROOT_DIRPATH, 'void_500')
VOID_DATA_1500_DIRPATH  = os.path.join(VOID_ROOT_DIRPATH, 'void_1500')

VOID_OUTPUT_DIRPATH     = os.path.join('data', 'void_kbnet')

VOID_TRAIN_IMAGE_FILENAME         = 'train_image.txt'
VOID_TRAIN_SPARSE_DEPTH_FILENAME  = 'train_sparse_depth.txt'
VOID_TRAIN_VALIDITY_MAP_FILENAME  = 'train_validity_map.txt'
VOID_TRAIN_GROUND_TRUTH_FILENAME  = 'train_ground_truth.txt'
VOID_TRAIN_INTRINSICS_FILENAME    = 'train_intrinsics.txt'
VOID_TEST_IMAGE_FILENAME          = 'test_image.txt'
VOID_TEST_SPARSE_DEPTH_FILENAME   = 'test_sparse_depth.txt'
VOID_TEST_VALIDITY_MAP_FILENAME   = 'test_validity_map.txt'
VOID_TEST_GROUND_TRUTH_FILENAME   = 'test_ground_truth.txt'
VOID_TEST_INTRINSICS_FILENAME     = 'test_intrinsics.txt'

TRAIN_REFS_DIRPATH      = os.path.join('training', 'void')
TEST_REFS_DIRPATH       = os.path.join('testing', 'void')

# VOID training set 150 density
VOID_TRAIN_IMAGE_150_FILEPATH           = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_image_150.txt')
VOID_TRAIN_SPARSE_DEPTH_150_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_sparse_depth_150.txt')
VOID_TRAIN_VALIDITY_MAP_150_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_validity_map_150.txt')
VOID_TRAIN_GROUND_TRUTH_150_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_ground_truth_150.txt')
VOID_TRAIN_INTRINSICS_150_FILEPATH      = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_intrinsics_150.txt')
# VOID training set 500 density
VOID_TRAIN_IMAGE_500_FILEPATH           = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_image_500.txt')
VOID_TRAIN_SPARSE_DEPTH_500_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_sparse_depth_500.txt')
VOID_TRAIN_VALIDITY_MAP_500_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_validity_map_500.txt')
VOID_TRAIN_GROUND_TRUTH_500_FILEPATH    = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_ground_truth_500.txt')
VOID_TRAIN_INTRINSICS_500_FILEPATH      = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_intrinsics_500.txt')
# VOID training set 1500 density
VOID_TRAIN_IMAGE_1500_FILEPATH          = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_image_1500.txt')
VOID_TRAIN_SPARSE_DEPTH_1500_FILEPATH   = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_sparse_depth_1500.txt')
VOID_TRAIN_VALIDITY_MAP_1500_FILEPATH   = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_validity_map_1500.txt')
VOID_TRAIN_GROUND_TRUTH_1500_FILEPATH   = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_ground_truth_1500.txt')
VOID_TRAIN_INTRINSICS_1500_FILEPATH     = os.path.join(TRAIN_REFS_DIRPATH, 'void_train_intrinsics_1500.txt')
# VOID testing set 150 density
VOID_TEST_IMAGE_150_FILEPATH            = os.path.join(TEST_REFS_DIRPATH, 'void_test_image_150.txt')
VOID_TEST_SPARSE_DEPTH_150_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_sparse_depth_150.txt')
VOID_TEST_VALIDITY_MAP_150_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_validity_map_150.txt')
VOID_TEST_GROUND_TRUTH_150_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_ground_truth_150.txt')
VOID_TEST_INTRINSICS_150_FILEPATH       = os.path.join(TEST_REFS_DIRPATH, 'void_test_intrinsics_150.txt')
# VOID testing set 500 density
VOID_TEST_IMAGE_500_FILEPATH            = os.path.join(TEST_REFS_DIRPATH, 'void_test_image_500.txt')
VOID_TEST_SPARSE_DEPTH_500_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_sparse_depth_500.txt')
VOID_TEST_VALIDITY_MAP_500_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_validity_map_500.txt')
VOID_TEST_GROUND_TRUTH_500_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_test_ground_truth_500.txt')
VOID_TEST_INTRINSICS_500_FILEPATH       = os.path.join(TEST_REFS_DIRPATH, 'void_test_intrinsics_500.txt')
# VOID testing set 1500 density
VOID_TEST_IMAGE_1500_FILEPATH           = os.path.join(TEST_REFS_DIRPATH, 'void_test_image_1500.txt')
VOID_TEST_SPARSE_DEPTH_1500_FILEPATH    = os.path.join(TEST_REFS_DIRPATH, 'void_test_sparse_depth_1500.txt')
VOID_TEST_VALIDITY_MAP_1500_FILEPATH    = os.path.join(TEST_REFS_DIRPATH, 'void_test_validity_map_1500.txt')
VOID_TEST_GROUND_TRUTH_1500_FILEPATH    = os.path.join(TEST_REFS_DIRPATH, 'void_test_ground_truth_1500.txt')
VOID_TEST_INTRINSICS_1500_FILEPATH      = os.path.join(TEST_REFS_DIRPATH, 'void_test_intrinsics_1500.txt')
# VOID unused testing set 150 density
VOID_UNUSED_IMAGE_150_FILEPATH          = os.path.join(TEST_REFS_DIRPATH, 'void_unused_image_150.txt')
VOID_UNUSED_SPARSE_DEPTH_150_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_sparse_depth_150.txt')
VOID_UNUSED_VALIDITY_MAP_150_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_validity_map_150.txt')
VOID_UNUSED_GROUND_TRUTH_150_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_ground_truth_150.txt')
VOID_UNUSED_INTRINSICS_150_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_unused_intrinsics_150.txt')
# VOID unused testing set 500 density
VOID_UNUSED_IMAGE_500_FILEPATH          = os.path.join(TEST_REFS_DIRPATH, 'void_unused_image_500.txt')
VOID_UNUSED_SPARSE_DEPTH_500_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_sparse_depth_500.txt')
VOID_UNUSED_VALIDITY_MAP_500_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_validity_map_500.txt')
VOID_UNUSED_GROUND_TRUTH_500_FILEPATH   = os.path.join(TEST_REFS_DIRPATH, 'void_unused_ground_truth_500.txt')
VOID_UNUSED_INTRINSICS_500_FILEPATH     = os.path.join(TEST_REFS_DIRPATH, 'void_unused_intrinsics_500.txt')
# VOID unused testing set 1500 density
VOID_UNUSED_IMAGE_1500_FILEPATH         = os.path.join(TEST_REFS_DIRPATH, 'void_unused_image_1500.txt')
VOID_UNUSED_SPARSE_DEPTH_1500_FILEPATH  = os.path.join(TEST_REFS_DIRPATH, 'void_unused_sparse_depth_1500.txt')
VOID_UNUSED_VALIDITY_MAP_1500_FILEPATH  = os.path.join(TEST_REFS_DIRPATH, 'void_unused_validity_map_1500.txt')
VOID_UNUSED_GROUND_TRUTH_1500_FILEPATH  = os.path.join(TEST_REFS_DIRPATH, 'void_unused_ground_truth_1500.txt')
VOID_UNUSED_INTRINSICS_1500_FILEPATH    = os.path.join(TEST_REFS_DIRPATH, 'void_unused_intrinsics_1500.txt')

def process_frame(inputs):
    '''
    Processes a single frame

    Arg(s):
        inputs : tuple
            image path at time t=0,
            image path at time t=1,
            image path at time t=-1,
            sparse depth path at time t=0,
            validity map path at time t=0,
            ground truth path at time t=0,
            boolean flag if set then create paths only
    Returns:
        str : image reference directory path
        str : output concatenated image path at time t=0
        str : output sparse depth path at time t=0
        str : output validity map path at time t=0
        str : output ground truth path at time t=0
    '''

    image_path1, \
        image_path0, \
        image_path2, \
        sparse_depth_path, \
        validity_map_path, \
        ground_truth_path, \
        paths_only = inputs

    if not paths_only:
        # Create image composite of triplets
        image1 = cv2.imread(image_path1)
        image0 = cv2.imread(image_path0)
        image2 = cv2.imread(image_path2)
        imagec = np.concatenate([image1, image0, image2], axis=1)

        # Get validity map
        sparse_depth, validity_map = data_utils.load_depth_with_validity_map(sparse_depth_path)

    image_refpath = os.path.join(*image_path0.split(os.sep)[2:])

    # Set output paths
    image_outpath = os.path.join(VOID_OUTPUT_DIRPATH, image_refpath)
    sparse_depth_outpath = sparse_depth_path
    validity_map_outpath = validity_map_path
    ground_truth_outpath = ground_truth_path

    # Verify that all filenames match
    image_out_dirpath, image_filename = os.path.split(image_outpath)
    sparse_depth_filename = os.path.basename(sparse_depth_outpath)
    validity_map_filename = os.path.basename(validity_map_outpath)
    ground_truth_filename = os.path.basename(ground_truth_outpath)

    assert image_filename == sparse_depth_filename
    assert image_filename == validity_map_filename
    assert image_filename == ground_truth_filename

    if not paths_only:
        cv2.imwrite(image_outpath, imagec)

    return (image_refpath,
            image_outpath,
            sparse_depth_outpath,
            validity_map_outpath,
            ground_truth_outpath)

parser = argparse.ArgumentParser()

parser.add_argument('--paths_only', action='store_true')

args = parser.parse_args()

data_dirpaths = [
    VOID_DATA_150_DIRPATH,
    VOID_DATA_500_DIRPATH,
    VOID_DATA_1500_DIRPATH
]

train_output_filepaths = [
    [
        VOID_TRAIN_IMAGE_150_FILEPATH,
        VOID_TRAIN_SPARSE_DEPTH_150_FILEPATH,
        VOID_TRAIN_VALIDITY_MAP_150_FILEPATH,
        VOID_TRAIN_GROUND_TRUTH_150_FILEPATH,
        VOID_TRAIN_INTRINSICS_150_FILEPATH
    ],
    [
        VOID_TRAIN_IMAGE_500_FILEPATH,
        VOID_TRAIN_SPARSE_DEPTH_500_FILEPATH,
        VOID_TRAIN_VALIDITY_MAP_500_FILEPATH,
        VOID_TRAIN_GROUND_TRUTH_500_FILEPATH,
        VOID_TRAIN_INTRINSICS_500_FILEPATH
    ],
    [
        VOID_TRAIN_IMAGE_1500_FILEPATH,
        VOID_TRAIN_SPARSE_DEPTH_1500_FILEPATH,
        VOID_TRAIN_VALIDITY_MAP_1500_FILEPATH,
        VOID_TRAIN_GROUND_TRUTH_1500_FILEPATH,
        VOID_TRAIN_INTRINSICS_1500_FILEPATH
    ]
]
test_output_filepaths = [
    [
        VOID_TEST_IMAGE_150_FILEPATH,
        VOID_TEST_SPARSE_DEPTH_150_FILEPATH,
        VOID_TEST_VALIDITY_MAP_150_FILEPATH,
        VOID_TEST_GROUND_TRUTH_150_FILEPATH,
        VOID_TEST_INTRINSICS_150_FILEPATH
    ],
    [
        VOID_TEST_IMAGE_500_FILEPATH,
        VOID_TEST_SPARSE_DEPTH_500_FILEPATH,
        VOID_TEST_VALIDITY_MAP_500_FILEPATH,
        VOID_TEST_GROUND_TRUTH_500_FILEPATH,
        VOID_TEST_INTRINSICS_500_FILEPATH
    ],
    [
        VOID_TEST_IMAGE_1500_FILEPATH,
        VOID_TEST_SPARSE_DEPTH_1500_FILEPATH,
        VOID_TEST_VALIDITY_MAP_1500_FILEPATH,
        VOID_TEST_GROUND_TRUTH_1500_FILEPATH,
        VOID_TEST_INTRINSICS_1500_FILEPATH
    ]
]
unused_output_filepaths = [
    [
        VOID_UNUSED_IMAGE_150_FILEPATH,
        VOID_UNUSED_SPARSE_DEPTH_150_FILEPATH,
        VOID_UNUSED_VALIDITY_MAP_150_FILEPATH,
        VOID_UNUSED_GROUND_TRUTH_150_FILEPATH,
        VOID_UNUSED_INTRINSICS_150_FILEPATH
    ],
    [
        VOID_UNUSED_IMAGE_500_FILEPATH,
        VOID_UNUSED_SPARSE_DEPTH_500_FILEPATH,
        VOID_UNUSED_VALIDITY_MAP_500_FILEPATH,
        VOID_UNUSED_GROUND_TRUTH_500_FILEPATH,
        VOID_UNUSED_INTRINSICS_500_FILEPATH
    ],
    [
        VOID_UNUSED_IMAGE_1500_FILEPATH,
        VOID_UNUSED_SPARSE_DEPTH_1500_FILEPATH,
        VOID_UNUSED_VALIDITY_MAP_1500_FILEPATH,
        VOID_UNUSED_GROUND_TRUTH_1500_FILEPATH,
        VOID_UNUSED_INTRINSICS_1500_FILEPATH
    ]
]

for dirpath in [TRAIN_REFS_DIRPATH, TEST_REFS_DIRPATH]:
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

data_filepaths = \
    zip(data_dirpaths, train_output_filepaths, test_output_filepaths, unused_output_filepaths)

for data_dirpath, train_filepaths, test_filepaths, unused_filepaths in data_filepaths:
    # Training set
    train_image_filepath = os.path.join(data_dirpath, VOID_TRAIN_IMAGE_FILENAME)
    train_sparse_depth_filepath = os.path.join(data_dirpath, VOID_TRAIN_SPARSE_DEPTH_FILENAME)
    train_validity_map_filepath = os.path.join(data_dirpath, VOID_TRAIN_VALIDITY_MAP_FILENAME)
    train_ground_truth_filepath = os.path.join(data_dirpath, VOID_TRAIN_GROUND_TRUTH_FILENAME)
    train_intrinsics_filepath = os.path.join(data_dirpath, VOID_TRAIN_INTRINSICS_FILENAME)

    # Read training paths
    train_image_paths = data_utils.read_paths(train_image_filepath)
    train_sparse_depth_paths = data_utils.read_paths(train_sparse_depth_filepath)
    train_validity_map_paths = data_utils.read_paths(train_validity_map_filepath)
    train_ground_truth_paths = data_utils.read_paths(train_ground_truth_filepath)
    train_intrinsics_paths = data_utils.read_paths(train_intrinsics_filepath)

    assert len(train_image_paths) == len(train_sparse_depth_paths)
    assert len(train_image_paths) == len(train_validity_map_paths)
    assert len(train_image_paths) == len(train_ground_truth_paths)
    assert len(train_image_paths) == len(train_intrinsics_paths)

    # Testing set
    test_image_filepath = os.path.join(data_dirpath, VOID_TEST_IMAGE_FILENAME)
    test_sparse_depth_filepath = os.path.join(data_dirpath, VOID_TEST_SPARSE_DEPTH_FILENAME)
    test_validity_map_filepath = os.path.join(data_dirpath, VOID_TEST_VALIDITY_MAP_FILENAME)
    test_ground_truth_filepath = os.path.join(data_dirpath, VOID_TEST_GROUND_TRUTH_FILENAME)
    test_intrinsics_filepath = os.path.join(data_dirpath, VOID_TEST_INTRINSICS_FILENAME)

    # Read testing paths
    test_image_paths = data_utils.read_paths(test_image_filepath)
    test_sparse_depth_paths = data_utils.read_paths(test_sparse_depth_filepath)
    test_validity_map_paths = data_utils.read_paths(test_validity_map_filepath)
    test_ground_truth_paths = data_utils.read_paths(test_ground_truth_filepath)
    test_intrinsics_paths = data_utils.read_paths(test_intrinsics_filepath)

    assert len(test_image_paths) == len(test_sparse_depth_paths)
    assert len(test_image_paths) == len(test_validity_map_paths)
    assert len(test_image_paths) == len(test_ground_truth_paths)
    assert len(test_image_paths) == len(test_intrinsics_paths)

    # Get test set directories
    test_seq_dirpaths = set(
        [test_image_paths[idx].split(os.sep)[-3] for idx in range(len(test_image_paths))])

    # Initialize placeholders for training output paths
    train_image_outpaths = []
    train_sparse_depth_outpaths = []
    train_validity_map_outpaths = []
    train_ground_truth_outpaths = []
    train_intrinsics_outpaths = []

    # Initialize placeholders for testing output paths
    test_image_outpaths = []
    test_sparse_depth_outpaths = []
    test_validity_map_outpaths = []
    test_ground_truth_outpaths = []
    test_intrinsics_outpaths = []

    # Initialize placeholders for unused testing output paths
    unused_image_outpaths = []
    unused_sparse_depth_outpaths = []
    unused_validity_map_outpaths = []
    unused_ground_truth_outpaths = []
    unused_intrinsics_outpaths = []

    # For each dataset density, grab the sequences
    seq_dirpaths = glob.glob(os.path.join(data_dirpath, 'data', '*'))
    n_sample = 0

    for seq_dirpath in seq_dirpaths:
        # For each sequence, grab the images, sparse depths and valid maps
        image_paths = \
            sorted(glob.glob(os.path.join(seq_dirpath, 'image', '*.png')))
        sparse_depth_paths = \
            sorted(glob.glob(os.path.join(seq_dirpath, 'sparse_depth', '*.png')))
        validity_map_paths = \
            sorted(glob.glob(os.path.join(seq_dirpath, 'validity_map', '*.png')))
        ground_truth_paths = \
            sorted(glob.glob(os.path.join(seq_dirpath, 'ground_truth', '*.png')))
        intrinsics_path = os.path.join(seq_dirpath, 'K.txt')

        assert len(image_paths) == len(sparse_depth_paths)
        assert len(image_paths) == len(validity_map_paths)

        # Load intrinsics
        kin = np.loadtxt(intrinsics_path)

        intrinsics_refpath = \
            os.path.join(*intrinsics_path.split(os.sep)[2:])
        intrinsics_outpath = \
            os.path.join(VOID_OUTPUT_DIRPATH, intrinsics_refpath[:-3]+'npy')
        image_out_dirpath = \
            os.path.join(os.path.dirname(intrinsics_outpath), 'image')

        if not os.path.exists(image_out_dirpath):
            os.makedirs(image_out_dirpath)

        # Save intrinsics
        np.save(intrinsics_outpath, kin)

        if seq_dirpath.split(os.sep)[-1] in test_seq_dirpaths:
            start_idx = 0
            offset_idx = 0
        else:
            # Skip first stationary 30 frames (1 second) and skip every 10
            start_idx = 30
            offset_idx = 10

        pool_input = []
        for idx in range(start_idx, len(image_paths)-offset_idx-start_idx):
            pool_input.append((
                image_paths[idx-offset_idx],
                image_paths[idx],
                image_paths[idx+offset_idx],
                sparse_depth_paths[idx],
                validity_map_paths[idx],
                ground_truth_paths[idx],
                args.paths_only))

        with mp.Pool() as pool:
            pool_results = pool.map(process_frame, pool_input)

            for result in pool_results:
                image_refpath, \
                    image_outpath, \
                    sparse_depth_outpath, \
                    validity_map_outpath, \
                    ground_truth_outpath = result

                # Split into training, testing and unused testing sets
                if image_refpath in train_image_paths:
                    train_image_outpaths.append(image_outpath)
                    train_sparse_depth_outpaths.append(sparse_depth_outpath)
                    train_validity_map_outpaths.append(validity_map_outpath)
                    train_ground_truth_outpaths.append(ground_truth_outpath)
                    train_intrinsics_outpaths.append(intrinsics_outpath)
                elif image_refpath in test_image_paths:
                    test_image_outpaths.append(image_outpath)
                    test_sparse_depth_outpaths.append(sparse_depth_outpath)
                    test_validity_map_outpaths.append(validity_map_outpath)
                    test_ground_truth_outpaths.append(ground_truth_outpath)
                    test_intrinsics_outpaths.append(intrinsics_outpath)
                else:
                    unused_image_outpaths.append(image_outpath)
                    unused_sparse_depth_outpaths.append(sparse_depth_outpath)
                    unused_validity_map_outpaths.append(validity_map_outpath)
                    unused_ground_truth_outpaths.append(ground_truth_outpath)
                    unused_intrinsics_outpaths.append(intrinsics_outpath)

        n_sample = n_sample + len(pool_input)

        print('Completed processing {} examples for sequence={}'.format(
            len(pool_input), seq_dirpath))

    print('Completed processing {} examples for density={}'.format(n_sample, data_dirpath))

    void_train_image_filepath, \
        void_train_sparse_depth_filepath, \
        void_train_validity_map_filepath, \
        void_train_ground_truth_filepath, \
        void_train_intrinsics_filepath = train_filepaths

    print('Storing {} training image file paths into: {}'.format(
        len(train_image_outpaths), void_train_image_filepath))
    data_utils.write_paths(
        void_train_image_filepath, train_image_outpaths)

    print('Storing {} training sparse depth file paths into: {}'.format(
        len(train_sparse_depth_outpaths), void_train_sparse_depth_filepath))
    data_utils.write_paths(
        void_train_sparse_depth_filepath, train_sparse_depth_outpaths)

    print('Storing {} training validity map file paths into: {}'.format(
        len(train_validity_map_outpaths), void_train_validity_map_filepath))
    data_utils.write_paths(
        void_train_validity_map_filepath, train_validity_map_outpaths)

    print('Storing {} training groundtruth depth file paths into: {}'.format(
        len(train_ground_truth_outpaths), void_train_ground_truth_filepath))
    data_utils.write_paths(
        void_train_ground_truth_filepath, train_ground_truth_outpaths)

    print('Storing {} training camera intrinsics file paths into: {}'.format(
        len(train_intrinsics_outpaths), void_train_intrinsics_filepath))
    data_utils.write_paths(
        void_train_intrinsics_filepath, train_intrinsics_outpaths)

    void_test_image_filepath, \
        void_test_sparse_depth_filepath, \
        void_test_validity_map_filepath, \
        void_test_ground_truth_filepath, \
        void_test_intrinsics_filepath = test_filepaths

    print('Storing {} testing image file paths into: {}'.format(
        len(test_image_outpaths), void_test_image_filepath))
    data_utils.write_paths(
        void_test_image_filepath, test_image_outpaths)

    print('Storing {} testing sparse depth file paths into: {}'.format(
        len(test_sparse_depth_outpaths), void_test_sparse_depth_filepath))
    data_utils.write_paths(
        void_test_sparse_depth_filepath, test_sparse_depth_outpaths)

    print('Storing {} testing validity map file paths into: {}'.format(
        len(test_validity_map_outpaths), void_test_validity_map_filepath))
    data_utils.write_paths(
        void_test_validity_map_filepath, test_validity_map_outpaths)

    print('Storing {} testing groundtruth depth file paths into: {}'.format(
        len(test_ground_truth_outpaths), void_test_ground_truth_filepath))
    data_utils.write_paths(
        void_test_ground_truth_filepath, test_ground_truth_outpaths)

    print('Storing {} testing camera intrinsics file paths into: {}'.format(
        len(test_intrinsics_outpaths), void_test_intrinsics_filepath))
    data_utils.write_paths(
        void_test_intrinsics_filepath, test_intrinsics_outpaths)

    void_unused_image_filepath, \
        void_unused_sparse_depth_filepath, \
        void_unused_validity_map_filepath, \
        void_unused_ground_truth_filepath, \
        void_unused_intrinsics_filepath = unused_filepaths

    print('Storing {} unused testing image file paths into: {}'.format(
        len(unused_image_outpaths), void_unused_image_filepath))
    data_utils.write_paths(
        void_unused_image_filepath, unused_image_outpaths)

    print('Storing {} unused testing sparse depth file paths into: {}'.format(
        len(unused_sparse_depth_outpaths), void_unused_sparse_depth_filepath))
    data_utils.write_paths(
        void_unused_sparse_depth_filepath, unused_sparse_depth_outpaths)

    print('Storing {} unused testing validity map file paths into: {}'.format(
        len(unused_validity_map_outpaths), void_unused_validity_map_filepath))
    data_utils.write_paths(
        void_unused_validity_map_filepath, unused_validity_map_outpaths)

    print('Storing {} unused testing groundtruth depth file paths into: {}'.format(
        len(unused_ground_truth_outpaths), void_unused_ground_truth_filepath))
    data_utils.write_paths(
        void_unused_ground_truth_filepath, unused_ground_truth_outpaths)

    print('Storing {} unused testing camera intrinsics file paths into: {}'.format(
        len(unused_intrinsics_outpaths), void_unused_intrinsics_filepath))
    data_utils.write_paths(
        void_unused_intrinsics_filepath, unused_intrinsics_outpaths)


text:
# encoding: utf-8
import os
import random
import subprocess
import sys

import numpy as np
import torch

from utils.serialization import Logger
from pprint import pprint

def _random_seed(seed):
    torch.manual_seed(seed)  # cpu
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)  # gpu
    np.random.seed(seed)  # numpy
    random.seed(seed)  # random and transforms
    torch.backends.cudnn.deterministic = True  # cudnn

def prepare_running(opt):
    print(torch.cuda.is_available())
    os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(i) for i in eval(opt.gpus)])
    sys.stdout = Logger(os.path.join('./exps', opt.exp, 'log_train.txt'))
    print('current commit hash: {}'.format(subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip()))
    pprint(opt)
    _random_seed(opt.seed)
    torch.backends.cudnn.benchmark = True
    torch.autograd.set_detect_anomaly(True)

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import argparse
import random

import numpy as np
import os, shutil,re
import glob

parser = argparse.ArgumentParser(description='Simple image registration by translation')
parser.add_argument('--root', '-R', help='copy from')
parser.add_argument('--output', '-o', default="data/cbct_2p8", help='copy to')
parser.add_argument('--slices_per_patient', '-s', default=8, type=int, help='')
parser.add_argument('--trainA', '-a', default=[5,8], type=int, nargs="*", help='patient ID')
parser.add_argument('--trainB', '-b', default=[5,8], type=int, nargs="*", help='patient ID')
parser.add_argument('--testA', '-ta', default=[1,2], type=int, nargs="*", help='patient ID')
parser.add_argument('--testB', '-tb', default=[1,2], type=int, nargs="*", help='patient ID')
args = parser.parse_args()

targets = {
           "trainA": ["dicom_CBCT_p{:02}_1".format(i) for i in args.trainA],
           "trainB": ["dicom_PlanCT_p{:02}_1_rigid".format(i) for i in args.trainB],
           "testA": ["dicom_CBCT_p{:02}_1".format(i) for i in args.testA],
           "testB": ["dicom_PlanCT_p{:02}_1_rigid".format(i) for i in args.testB]
           }

num = lambda val : int(re.sub("\\D", "", val+"0"))
for dn in ["trainA","trainB","testA","testB"]:
    os.makedirs(os.path.join(args.output,dn), exist_ok=True)
    cdir = os.path.join(args.root,dn)
    for root, dirs, files in os.walk(cdir):
#        print(root,dirs,files)
        for dirname in dirs:
            if dirname in targets[dn]:
                ipath = os.path.join(root,dirname)
                fns = [f for f in os.listdir(ipath) if (".dcm" in f) ]
                fns.sort(key=num)
                s = len(fns)//args.slices_per_patient
                fns = fns[::s]
                opath = os.path.join(os.path.join(args.output,dn),dirname)
                os.makedirs(opath, exist_ok=True)
                for f in fns:
                    shutil.copy(os.path.join(ipath,f), opath)

max_stars_repo_name: shizuo-kaji/DicomTools

max_issues_repo_name: shizuo-kaji/DicomTools

max_forks_repo_name: shizuo-kaji/DicomTools


text:
import numpy as np

from collections import namedtuple
from collections import defaultdict
from scipy.spatial import distance

Mine = namedtuple("Mine", ["name", "x", "y", "blast_radius"])

def get_dist(x1, y1, x2, y2):
    return distance.euclidean([x1, y1], [x2, y2])

def longest_chain(mines):
    # create a m * n array based on number of mines
    n = len(mines)
    M = np.zeros((n, n))

    for mine in mines:
        for otherMine in mines:
            if mine == otherMine:
                continue
            else:
                dist = get_dist(mine.x, mine.y, otherMine.x, otherMine.y)
                if dist <= mine.blast_radius:
                    conn[mine.name].add(otherMine.name)

    # for k, v in conn.iteritems():

    return conn

# Diagram: https://i.imgur.com/xEZZQKP.png
mines = [
    Mine("A", 7, 13, 3),
    Mine("B", 6.5, 17, 5),
    Mine("C", 12, 10, 4.5),
    Mine("D", 14.5, 7, 3.5),
    Mine("E", 17, 9, 2),
    Mine("F", 7, 11, 2.5),
    Mine("G", 8.5, 11.5, 3),
]

print(longest_chain(mines))
# print(longest_chain(mines) == ('C', 6))

reason: import numpy,from scipy


text:
#!/usr/bin/env python

import os
import sys
import glob
import shutil

try:
    from setuptools import setup, Extension
    from setuptools.command.build_ext import build_ext
except ImportError:
    from distutils.core import setup, Extension
    from distutils.command.build_ext import build_ext

def invoke_f2py(files, flags=[], wd=None):
    from numpy.f2py import main

    olddir = os.path.abspath(os.curdir)
    oldargv = list(sys.argv)
    try:
        if wd is not None:
            os.chdir(wd)
        sys.argv = ['f2py']
        sys.argv.extend(files)
        sys.argv.extend(flags)

        main()
    finally:
        sys.argv = oldargv
        os.chdir(olddir)

class build_fsps(build_ext):

    def run(self):
        # Generate the Fortran signature/interface.
        files = ['fsps.f90']
        flags = " -m _fsps -h fsps.pyf --overwrite-signature".split()
        print("Running f2py on {0} with flags {1}".format(files, flags))
        invoke_f2py(['fsps.f90'], flags, wd='fsps')

        # Find the FSPS source files.
        fsps_dir = os.path.join(os.environ["SPS_HOME"], "src")
        fns = [f for f in glob.glob(os.path.join(fsps_dir, "*.o"))
               if os.path.basename(f) not in ["autosps.o", "simple.o",
                                              "lesssimple.o"]]

        # Check to make sure that all of the required modules exist.
        flag = len(fns)
        flag *= os.path.exists(os.path.join(fsps_dir, "sps_utils.mod"))
        flag *= os.path.exists(os.path.join(fsps_dir, "sps_vars.mod"))
        if not flag:
            raise RuntimeError("You need to run make in $SPS_HOME/src first")

        # Add the interface source files to the file list.
        fns += ["fsps.f90", "fsps.pyf"]

        # Compile the library.
        flags = '-c -I{0} --f90flags=-cpp --f90flags=-fPIC'.format(fsps_dir)
        flags = flags.split()
        print("Running f2py on {0} with flags {1}".format(fns, flags))
        invoke_f2py(fns, flags, wd='fsps')

        # Move the compiled library to the correct directory.
        infn = os.path.abspath(self.get_ext_filename("fsps._fsps"))
        outfn = os.path.abspath(self.get_ext_fullpath("fsps._fsps"))
        if infn != outfn:
            try:
                os.makedirs(os.path.dirname(outfn))
            except os.error:
                pass
            print("Copying {0} to {1}".format(infn, outfn))
            shutil.copyfile(infn, outfn)

if "publish" in sys.argv[-1]:
    os.system("python setup.py sdist upload")
    sys.exit()

# Hackishly inject a constant into builtins to enable importing of the
# package before the library is built.
if sys.version_info[0] < 3:
    import __builtin__ as builtins
else:
    import builtins
builtins.__FSPS_SETUP__ = True
from fsps import __version__  # NOQA

# This is a fake extension that is used to trick distutils into building our
# real library using the `build_fsps` function above even when `install` is
# called.
ext = Extension("fsps._fsps", sources=["fsps/fsps.f90"])

# The final setup command. Note: we override the `build_ext` command with our
# custom version from above.
setup(
    name="fsps",
    url="https://github.com/dfm/python-fsps",
    version=__version__,
    author="Dan Foreman-Mackey",
    author_email="danfm@nyu.edu",
    description="Python bindings for Charlie Conroy's FSPS.",
    long_description=open("README.rst").read(),
    packages=["fsps"],
    package_data={
        "": ["README.rst", "LICENSE.rst", "AUTHORS.rst"],
        "fsps": ["_fsps.so"],
    },
    include_package_data=True,
    ext_modules=[ext],
    scripts=glob.glob("scripts/*.py"),
    cmdclass={
        "build_ext": build_fsps,
    },
    classifiers=[
        # "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
    ],
)

max_stars_repo_name: gbrammer/python-fsps

max_issues_repo_name: gbrammer/python-fsps

max_forks_repo_name: gbrammer/python-fsps

reason: from numpy


text:
from asap3 import *
import cPickle as pickle
from numpy import *
import sys, os, time
from asap3.testtools import *
from asap3.mpi import world
from ase.lattice.cubic import *

debug = 0
if debug == 1:
    DebugOutput("moveatomsLJ%d.log", nomaster=True)
elif debug == 2:
    time.sleep(world.rank)
    print "PID:", os.getpid()
    time.sleep(20)

print_version(1)
#set_verbose(1)

ismaster = world.rank == 0
isparallel = world.size != 1
if world.size == 1:
    cpulayout = None
elif world.size == 2:
    cpulayout = [2,1,1]
elif world.size == 3:
    cpulayout = [1,3,1]
elif world.size == 4:
    cpulayout = [2,1,2]

picklefile = "moveatomsLJ.pickle"

if isparallel:
    print "RUNNING PARALLEL VERSION OF TEST SCRIPT"
else:
    print "RUNNING SERIAL VERSION OF TEST SCRIPT"

if ismaster:
    atoms = FaceCenteredCubic(size=(10,10,10), symbol="Cu",
                              latticeconstant=1.09*2.7*1.41,
                              pbc=True)
else:
    atoms = None

#set_verbose(1)

if isparallel:
    atoms = MakeParallelAtoms(atoms, cpulayout, distribute=False)
    atoms.ghosts["ID"] = zeros(0,int)
    print "Distributing"
    atoms.distribute()
    nTotalAtoms = atoms.get_number_of_atoms()
else:
    nTotalAtoms = len(atoms)
    atoms.arrays["ID"] = arange(nTotalAtoms)

print "Setting potential"
atoms.set_calculator(LennardJones([29], [0.15], [2.7], 2.7*1.9, True))

try:
    expected = pickle.load(open(picklefile))
except IOError:
    expected = None
    newexpected = []

def report():
    for i in range(world.size):
        if i == world.rank:
            print "Data on processor", i
            for key in atoms.arrays.keys():
                print "  ", key, atoms.arrays[key].shape
            r = atoms.get_positions()
            if len(r):
                print "Limits to the positions:"
                print ("[%.4f, %.4f]  [%.4f, %.4f]  [%.4f, %.4f]" %
                       (min(r[:,0]), max(r[:,0]), min(r[:,1]), max(r[:,1]),
                        min(r[:,2]), max(r[:,2])))
            if world.size > 1:
                print "Ghost data on processor", i
                for key in atoms.ghosts.keys():
                    print "  ", key, atoms.ghosts[key].shape
                r = atoms.ghosts['positions']
                if len(r):
                    print "Limits to the ghost positions:"
                    print ("[%.4f, %.4f]  [%.4f, %.4f]  [%.4f, %.4f]" %
                           (min(r[:,0]), max(r[:,0]), min(r[:,1]), max(r[:,1]),
                            min(r[:,2]), max(r[:,2])))
        world.barrier()

for i in range(10):
    print "Step number", i
    if i:
        dz = 0.1 * sin(atoms.arrays["ID"] / 10.0)
        r = atoms.get_positions()
        r[:,2] += dz
        atoms.set_positions(r)
    e = atoms.get_potential_energy() / nTotalAtoms
    print "\n%8f" % (e,)
    if expected:
        ReportTest("Step "+str(i), e, expected[i], 1e-9)
    else:
        newexpected.append(e)
    #report()
    #print atoms.get_potential_energies()

if expected is None and ismaster:
    print "Saving energies"
    out = open(picklefile, "w")
    pickle.dump(newexpected, out)
    out.close()

ReportTest.Summary()


text:
#!/usr/bin/python
"""
Make a triangle plot for a set of parameters.
"""
import numpy as np
import pylab as P
from rfwrapper import rf
import matplotlib.patches
import matplotlib.cm
import matplotlib.ticker
from units import *
from mpi4py import MPI

import os
import euclid

print "Obsolete."
exit()

USE_DETF_PLANCK_PRIOR = True # If False, use Euclid prior instead
MARGINALISE_OVER_W0WA = True # Whether to fix or marginalise over (w0, wa)

cosmo = rf.experiments.cosmo

names = ['EuclidRef', 'cexptL', 'iexptM', 'iexptM'] #, 'exptS']
labels = ['DETF IV', 'Facility', 'Mature', 'Planck'] #, 'Snapshot']

# TESTING
names = ['cexptL', 'cexptL', 'cexptL']
labels = ['PPlanck', 'Planck', 'Facility']
#names = ['cexptL', 'iexptM', 'exptS']
#labels = ['Facility', 'Mature', 'Snapshot']
colours = [ ['#CC0000', '#F09B9B'],
            ['#1619A1', '#B1C9FD'],
            ['#FFB928', '#FFEA28'],
            ['#5B9C0A', '#BAE484'] ]

scale_idx = 2 #1 # Index of experiment to use as reference for setting the x,y scales
nsigma = 4. #4.2 # No. of sigma (of reference experiment 1D marginal) to plot out to

# Set-up triangle plot
Nparam = 6 # No. of parameters
fig = P.figure()
axes = [[fig.add_subplot(Nparam, Nparam, (j+1) + i*Nparam) for i in range(j, Nparam)] for j in range(Nparam)]

# Fixed width and height for each subplot
w = 1.0 / (Nparam+1.)
h = 1.0 / (Nparam+1.)
l0 = 0.1
b0 = 0.1

# Prepare to save 1D marginals
params_1d = []; params_lbls = []

# Loop though rf.experiments.
_k = range(len(names))[::-1] # Reverse order of rf.experiments.
for k in _k:
    root = "output/" + names[k]

    print "-"*50
    print names[k]
    print "-"*50

    # Load cosmo fns.
    dat = np.atleast_2d( np.genfromtxt(root+"-cosmofns-zc.dat") ).T
    zc, Hc, dAc, Dc, fc = dat
    zs, Hs, dAs, Ds, fs = np.genfromtxt(root+"-cosmofns-smooth.dat").T
    kc = np.genfromtxt(root+"-fisher-kc.dat").T

    # Load Fisher matrices as fn. of z
    Nbins = zc.size
    F_list = [np.genfromtxt(root+"-fisher-full-%d.dat" % i) for i in range(Nbins)]

    # EOS FISHER MATRIX
    # Actually, (aperp, apar) are (D_A, H)
    pnames = ['A', 'b_HI', 'Tb', 'sigma_NL', 'sigma8', 'n_s', 'f', 'aperp', 'apar',
             'omegak', 'omegaDE', 'w0', 'wa', 'h', 'gamma'] #, 'Mnu']
    #if "Euclid" not in names[k]: pnames.append('Mnu')
    pnames += ["pk%d" % i for i in range(kc.size)]

    zfns = [1,]
    excl = [2,  6,7,8,   14,] #15] # 4:sigma8
    #if "Euclid" not in names[k]: excl.append(15)
    excl  += [i for i in range(len(pnames)) if "pk" in pnames[i]]

    F, lbls = rf.combined_fisher_matrix( F_list,
                                                expand=zfns, names=pnames,
                                                exclude=excl )

    # Apply Planck prior
    if USE_DETF_PLANCK_PRIOR:
        # DETF Planck prior
        print "*** Using DETF Planck prior ***"
        l2 = ['n_s', 'w0', 'wa', 'omega_b', 'omegak', 'omegaDE', 'h']
        F_detf = euclid.detf_to_rf("DETF_PLANCK_FISHER.txt", cosmo)
        Fpl, lbls = rf.add_fisher_matrices(F, F_detf, lbls, l2, expand=True)
    else:
        # Euclid Planck prior
        print "*** Using Euclid (Mukherjee) Planck prior ***"
        l2 = ['n_s', 'w0', 'wa', 'omega_b', 'omegak', 'omegaDE', 'h']
        Fe = euclid.planck_prior_full
        F_eucl = euclid.euclid_to_rf(Fe, cosmo)
        Fpl, lbls = rf.add_fisher_matrices(F, F_eucl, lbls, l2, expand=True)

    # FIXME: Use Planck prior alone
    if labels[k] == 'PPlanck':
        lbls = ['n_s', 'w0', 'wa', 'omega_b', 'omegak', 'omegaDE', 'h', 'sigma8']
        F2 = euclid.detf_to_rf("DETF_PLANCK_FISHER.txt", cosmo)
        Fpl = np.eye(F2.shape[0]+1) * 1e2 #sigma8
        Fpl[:F2.shape[0],:F2.shape[0]] = F2
    elif labels[k] == 'Planck':
        pass
    else:
        # Revert Fisher matrix to prev. values (without Planck prior)
        Fpl[:-1,:-1] = F
        tmp = Fpl[-1,-1]
        Fpl[-1,:] = 0.
        Fpl[:,-1] = 0.
        Fpl[-1,-1] = 1e2

    # Remove unwanted params
    fixed_params = ['w0', 'wa']
    Fpl, lbls = rf.combined_fisher_matrix( [Fpl,], expand=[],
                     names=lbls, exclude=[lbls.index(p) for p in fixed_params] )

    # Invert matrices
    cov_pl = np.linalg.inv(Fpl)

    # Store 1D marginals
    params_1d.append(np.sqrt(np.diag(cov_pl)))
    params_lbls.append(lbls)

    # Set which parameters are going into the triangle plot
    params = ['h', 'omega_b', 'omegak', 'omegaDE', 'n_s', 'sigma8'][::-1]
    label = ['h', 'omega_b', 'omegak', 'omegaDE', 'n_s', 'sigma8'][::-1]
    fid = [ cosmo['h'], cosmo['omega_b_0'], 0., cosmo['omega_lambda_0'], cosmo['ns'], cosmo['sigma_8'] ][::-1]

    # Loop through rows, columns, repositioning plots
    # i is column, j is row
    for j in range(Nparam):
        for i in range(Nparam-j):
            ax = axes[j][i]

            # Hide tick labels for subplots that aren't on the main x,y axes
            if j != 0:
                for tick in ax.xaxis.get_major_ticks():
                    tick.label1.set_visible(False)
            if i != 0:
                for tick in ax.yaxis.get_major_ticks():
                    tick.label1.set_visible(False)

            # Fiducial values
            ii = Nparam - i - 1
            x = fid[ii] #rf.experiments.cosmo['w0']
            y = fid[j] #rf.experiments.cosmo['wa']
            p1 = lbls.index(params[ii])
            p2 = lbls.index(params[j])

            ax.tick_params(axis='both', which='major', labelsize=12)
            ax.tick_params(axis='both', which='minor', labelsize=12)

            # Plot ellipse *or* 1D
            if p1 != p2:
                # Plot contours
                AAA = 1.
                if labels[k] == 'PPlanck': AAA = 0.5
                ww, hh, ang, alpha = rf.ellipse_for_fisher_params(
                                                      p1, p2, None, Finv=cov_pl)
                ellipses = [matplotlib.patches.Ellipse(xy=(x, y), width=alpha[kk]*ww,
                            height=alpha[kk]*hh, angle=ang, fc=colours[k][kk],
                            ec=colours[k][0], lw=1.5, alpha=0.5*AAA) for kk in [1,0]]
                for e in ellipses: ax.add_patch(e)

                # Centroid and axis scale
                if k == scale_idx:
                    sig1 = np.sqrt(cov_pl[p1,p1])
                    sig2 = np.sqrt(cov_pl[p2,p2])
                    ax.plot(x, y, 'kx')
                    ax.set_xlim((x-nsigma*sig1, x+nsigma*sig1))
                    ax.set_ylim((y-nsigma*sig2, y+nsigma*sig2))
            else:
                sig = np.sqrt(cov_pl[p1,p1])
                xx = np.linspace(x-20.*sig, x+20.*sig, 4000)
                yy = 1./np.sqrt(2.*np.pi*sig**2.) * np.exp(-0.5 * ((xx-x)/sig)**2.)
                yy /= np.max(yy)
                ax.plot(xx, yy, ls='solid', color=colours[k][0], lw=1.5)

                # Match x scale, and hide y ticks
                if k == scale_idx:
                    ax.set_xlim((x-nsigma*sig, x+nsigma*sig))
                ax.tick_params(axis='y', which='both', left='off', right='off')
                ax.set_ylim((0., 1.08))

            # Set position of subplot
            pos = ax.get_position().get_points()
            ax.set_position([l0+w*i, b0+h*j, w, h])

            if j == 0:
                ax.set_xlabel(label[ii], fontdict={'fontsize':'20'}, labelpad=20.)
            #if i == Nparam-j-1: ax.set_title(label[ii], fontdict={'fontsize':'20'})
            if i == 0:
                ax.set_ylabel(label[j], fontdict={'fontsize':'20'}, labelpad=20.)
                ax.get_yaxis().set_label_coords(-0.3,0.5)

params = ['h', 'omega_b', 'omegaDE', 'n_s', 'sigma8']
print [names[kk] for kk in _k]
for p in params:
    idxs = [params_lbls[i].index(p) for i in range(3)]
    print "%9s: %5.5f %5.5f %5.5f" % (p, params_1d[0][idxs[0]], params_1d[1][idxs[1]], params_1d[2][idxs[2]])

# Set size and save
P.gcf().set_size_inches(16.5,10.5)
P.show()

max_stars_repo_name: sjforeman/RadioFisher

max_issues_repo_name: sjforeman/RadioFisher

max_forks_repo_name: sjforeman/RadioFisher


text:
# Load the library with the iris dataset
from sklearn.datasets import load_iris

# Load scikit's random forest classifier library
from sklearn.ensemble import RandomForestClassifier

# Using Skicit-learn to split data into training and testing sets
from sklearn.model_selection import train_test_split

from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, mean_squared_error

from sklearn.ensemble import RandomForestRegressor

#Import scikit-learn metrics module for accuracy calculation
from sklearn import metrics

import matplotlib.pyplot as plt

from statistics import mean, stdev

import seaborn as sns

from sklearn.model_selection import StratifiedKFold
# Load pandas
import pandas as pd

# Load numpy
import numpy as np

from numpy import array
from sklearn.model_selection import KFold

from sklearn.model_selection import cross_val_score

def average(nums, default=float('nan')):
    return sum(nums) / float(len(nums)) if nums else default

def read_csv(csv_file, nrows=None):
    df = pd.read_csv(csv_file, nrows=nrows)
    print("File = {}".format(csv_file))
    print("Shape = {:,} rows, {:,} columns".format(df.shape[0], df.shape[1]))
    print("Memory usage = {:.2f}GB".format(df.memory_usage().sum() / 1024**3))
    return df

data_dir = r'''D:\CLoud\Academic\Research\___\Analysis (Photoshop)\4.2 Analysis Visualization - pyLDAvis (Using 750 symmetrical data)\_Working\topic_distribution_mallet_30_V1_train.csv'''

df = read_csv(data_dir)

# Set random seed
np.random.seed(0)

labelIndex =  df.columns.get_loc("Label")

onlyTopic = labelIndex-5
X = df.iloc[:, 1:labelIndex].values
y = df.iloc[:, labelIndex].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0, random_state=0) # 70% training and 30% test

#X_train = X
#X_test = X_t
#y_train = y
#y_test = y_t

#clf = RandomForestClassifier(
#    n_estimators=490,
#    criterion='gini',
#    max_depth=100,
#    min_samples_split=2,
#    min_samples_leaf=1,
#    min_weight_fraction_leaf=0.0,
#    max_features=15,
#    max_leaf_nodes=None,
#    min_impurity_decrease=0.0,
#    min_impurity_split=None,
#    bootstrap=True,
#    oob_score=False,
#    n_jobs=-1,
#    random_state=1,
#    verbose=0,
#    warm_start=False,
#    class_weight='balanced'
#)

clf = RandomForestClassifier(
        n_estimators=360,
        criterion='gini',
        max_depth=100,
        min_samples_split=13,
        min_samples_leaf=2,
        min_weight_fraction_leaf=0.0,
        max_features=15,
        max_leaf_nodes=None,
        min_impurity_decrease=0.0,
        min_impurity_split=None,
        bootstrap=True,
        oob_score=False,
        n_jobs=-1,
        random_state=0,
        verbose=0,
        warm_start=False,
        class_weight='balanced'
        )

featureImpValues = []

kf = StratifiedKFold(n_splits=10, random_state= 0, shuffle = True)
count = 1
# test data is not needed for fitting
accuracyList = []
stdList = []
for train, testInd in kf.split(X_train, y_train):

    xxtr = X_train[train, :]
    yytr = y_train[train]

    xxts = X_train[testInd, :]
    yyts = y_train[testInd]
    clf.fit(X_train[train, :],y_train[train])
    y_pred=clf.predict(X_train[testInd, :])
    confMat = confusion_matrix(y_train[testInd], y_pred)
    modelAccuracy = metrics.accuracy_score(y_train[testInd], y_pred)

    accuracyList.append(modelAccuracy)
    print("Accuracy:",modelAccuracy)

    # sort the feature index by importance score in descending order
#    feature_imp = pd.Series(clf.feature_importances_,df.columns.values[1:labelIndex ]).sort_values(ascending=False)
    feature_imp = (pd.Series(clf.feature_importances_,df.columns.values[1:labelIndex ]).tolist())
    #feature_labels = feature_imp.index
    featureImpValues.append(feature_imp)

#    plt.figure()
#    plt.bar(feature_labels, clf.feature_importances_[label])
#    plt.xticks(feature_labels, rotation='vertical')
#    plt.ylabel('Importance')
#    plt.xlabel('Features')
#    plt.title('Fold {}'.format(count))
#    count = count + 1
#plt.show()
#

feature_imp = []
feature_imp = [average(feature) for feature in zip(*featureImpValues)]
#commnet out the following if you want to normalize x axis within the range [0-1]
#feature_imp = [average(feature)/max(feature_imp)*100 for feature in zip(*featureImpValues)]
feature_sum = sum(feature_imp)
feature_labels=df.columns.values[1:labelIndex ]

features= pd.Series(feature_imp,feature_labels).sort_values(ascending=False)

print("Mean Accuracy:",mean(accuracyList))
print("Standard Deviation", stdev(accuracyList))

print(features)
print(feature_sum)

# Creating a bar plot
sns.barplot(x=features, y=features.index)
# Add labels to your graph
plt.xlabel('Relative Feature Importance Score', fontsize=18)
#plt.ylabel('Features',fontsize=18)
plt.title("Visualizing Important Features",fontsize=28)
plt.legend()
plt.show()

max_stars_repo_name: ShahedSabab/tutVis

max_issues_repo_name: ShahedSabab/tutVis

max_forks_repo_name: ShahedSabab/tutVis

reason: import numpy,from numpy


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

import os

import keras.backend as K
from keras.models import Model, Sequential
from keras.layers import Conv2D, MaxPool2D, Flatten, Dense, Input, Subtract, Lambda
from keras.optimizers import Adam, SGD
from keras.regularizers import l2
import keras.backend as K
from sklearn.model_selection import train_test_split
import tensorflow as tf

import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import numpy as np
import cv2
import itertools
import time

# In[2]:

labels = {
  "surprise": 0,
  "angry": 1,
  "happy": 2,
  "fear": 3,
  "disgust": 4,
  "sad": 5,
  "neutral": 6
}

# In[3]:

def abc(str1):
    for i in range (len(str1)-5,0,-1):
        try:
            z = int(str1[i])
        except:
            return str1[:i+1]

# In[17]:

def getImage(im,k):
    im = plt.imread('../Datasets/Facial-Expression-Recognization-using-JAFFE-master/jaffe/AllFiles/'+im.numpy()[k].decode('utf8'))
    im = im.astype('float32')
    im = im/255
    im=cv2.resize(im,(64,64))
    img = im.reshape((-1,64 * 64))
    return img

# In[5]:

data_path = '../Datasets/Facial-Expression-Recognization-using-JAFFE-master/id'
data_dir_list = os.listdir(data_path)

n_class = 7
img_data_list=[]
x = []
y = []
for dataset in data_dir_list:
    img_list=os.listdir(data_path+'/'+ dataset)
    print ('Loaded the images of dataset-'+'{}\n'.format(dataset))
    combinations = list(itertools.combinations(range(len(img_list)), 2))
    for i in range (len(combinations)):
        imgName1 = img_list[combinations[i][0]]
        imgName2 = img_list[combinations[i][1]]
#         label1 = labels[abc(imgName1)]
#         label2 = labels[abc(imgName2)]
        label1 = tf.one_hot(labels[abc(imgName1)], n_class)
        label2 = tf.one_hot(labels[abc(imgName2)], n_class)
#         img1 = plt.imread(data_path + '/'+ dataset + '/'+ imgName1)
#         img2 = plt.imread(data_path + '/'+ dataset + '/'+ imgName2)
#         img1 = cv2.resize(img1,(128,128))
#         img2 = cv2.resize(img2,(128,128))
#         img1 = img1.astype('float32')
#         img1 = img1/255
#         img2 = img2.astype('float32')
#         img2 = img2/255
#         img1 = img1.reshape((-1, 128 * 128))
#         img2 = img2.reshape((-1, 128 * 128))
        x.append((imgName1,imgName2))
        y.append((label1,label2))

# img_data = img_data.reshape((-1, 128 * 128))

# In[6]:

len(x)

# In[7]:

#x = x[:-14]

# In[8]:

#y = y[:-14]

# In[9]:

# Building model
n_class = 7
input_dim = 4096
z_dim = 256

# Encoder
def make_encoder_model():
    inputs = tf.keras.Input(shape=(input_dim,), name='Original_input')
    x = tf.keras.layers.Dense(2048, activation='relu')(inputs)
    x = tf.keras.layers.Dense(1024, activation='relu')(x)
    x = tf.keras.layers.Dense(512, activation='relu')(x)
    latent = tf.keras.layers.Dense(z_dim, activation='linear', name='Latent_variables')(x)
    observed = tf.keras.layers.Dense(n_class, activation='softmax', name='Observed_variables')(x)

    model = tf.keras.Model(inputs=inputs, outputs=[latent, observed], name='Encoder')
    return model

encoder = make_encoder_model()

# # In[ ]:

encoder.summary()

# In[10]:

# # Decoder
def make_decoder_model():
    inputted_latent = tf.keras.Input(shape=(z_dim,), name='Latent_variables')
    inputted_observed = tf.keras.Input(shape=(n_class,), name='Observed_variables')

    x = tf.keras.layers.concatenate([inputted_latent, inputted_observed], axis=-1)
    x = tf.keras.layers.Dense(512, activation='relu')(x)
    x = tf.keras.layers.Dense(1024, activation='relu')(x)
    x = tf.keras.layers.Dense(2048, activation='relu')(x)
    reconstruction = tf.keras.layers.Dense(input_dim, activation='linear', name='Reconstruction')(x)
    model = tf.keras.Model(inputs=[inputted_latent, inputted_observed], outputs=reconstruction, name='Decoder')
    return model

decoder = make_decoder_model()

# In[11]:

# Multipliers
alpha = 1000.0
beta = 100.0
gamma = 1000.0
zeta = 100

# Loss functions
# Reconstruction cost
mse_loss_fn = tf.keras.losses.MeanSquaredError()

# Supervised cost
cat_loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=False)

# In[35]:

# Unsupervised cross-covariance cost
def xcov_loss_fn(latent, observed,batch_size):
    latent_centered = latent - tf.reduce_mean(latent, axis=0, keepdims=True)
    observed_centered = observed - tf.reduce_mean(observed, axis=0, keepdims=True)
    xcov_loss = 0.5 * tf.reduce_sum(
        tf.square(tf.matmul(latent_centered, observed_centered, transpose_a=True)))

    return xcov_loss

# In[12]:

optimizer = tf.keras.optimizers.Adam(lr=0.0001)

# In[14]:

batch_size = 8
n_epochs = 100

#x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.15, random_state=2)
train_dataset = tf.data.Dataset.from_tensor_slices((x, y))
train_dataset = train_dataset.batch(batch_size)
train_dataset = train_dataset.shuffle(buffer_size=1024)

#np.save('x_test.npy',x_test)
#np.save('y_test.npy',y_test)
# In[24]:

# Training step
# @tf.function
def train_on_batch(batch_x, batch_y):
    with tf.GradientTape() as tape:
        # Inference
        batch_x_1 = np.zeros((batch_size,64*64))
        batch_x_2 = np.zeros((batch_size,64*64))

        batch_y_1 = np.zeros((batch_size,7))
        batch_y_2 = np.zeros((batch_size,7))

        for i in range (batch_size):
            batch_x_1[i] = getImage(batch_x[i],0)
            batch_x_2[i] = getImage(batch_x[i],1)

            batch_y_1[i] = batch_y[i].numpy()[0]
            batch_y_2[i] = batch_y[i].numpy()[1]

        batch_latent_1, batch_observed_1 = encoder(batch_x_1)
        batch_latent_2, batch_observed_2 = encoder(batch_x_2)
        batch_reconstruction_1 = decoder([batch_latent_1, batch_observed_1])
        batch_reconstruction_2 = decoder([batch_latent_2, batch_observed_2])

#         plt.figure()
#         print(batch_x_1)
#         print(batch_observed_1)
        # Loss functions
        recon_loss_1 = alpha * mse_loss_fn(batch_x_1, batch_reconstruction_1)
        recon_loss_2 = alpha * mse_loss_fn(batch_x_2, batch_reconstruction_2)
#         print(batch_reconstruction_1.shape)
#         plt.imshow(batch_latent_1)
#     plt.imshow(batch_latent_1)

        cat_loss_1 = beta * cat_loss_fn(batch_y_1, batch_observed_1)
        cat_loss_2 = beta * cat_loss_fn(batch_y_2, batch_observed_2)

        xcov_loss_1 = gamma * xcov_loss_fn(batch_latent_1, batch_observed_1, tf.cast(tf.shape(batch_x_1)[0], tf.float32))
        xcov_loss_2 = gamma * xcov_loss_fn(batch_latent_2, batch_observed_2, tf.cast(tf.shape(batch_x_2)[0], tf.float32))

        similarity_loss = zeta*mse_loss_fn(batch_latent_1,batch_latent_2)

        # Final loss function
        ae_loss = recon_loss_1 + recon_loss_2 + cat_loss_1 +cat_loss_2 + xcov_loss_1 + xcov_loss_2 + similarity_loss

    gradients = tape.gradient(ae_loss, encoder.trainable_variables + decoder.trainable_variables)
    optimizer.apply_gradients(zip(gradients, encoder.trainable_variables + decoder.trainable_variables))
    recon_loss = (recon_loss_1 + recon_loss_2)/2
    cat_loss = (cat_loss_1 + cat_loss_2)/2
    xcov_loss = (xcov_loss_1 + xcov_loss_2)/2
    return recon_loss, cat_loss, xcov_loss, similarity_loss

# In[25]:

for epoch in range(n_epochs):
    start = time.time()

    # Functions to calculate epoch's mean performance
    epoch_recon_loss_avg = tf.metrics.Mean()
    epoch_cat_loss_avg = tf.metrics.Mean()
    epoch_xcov_loss_avg = tf.metrics.Mean()
    epoch_sim_loss_avg = tf.metrics.Mean()

    for batch, (batch_x, batch_y) in enumerate(train_dataset):
        if(len(batch_x)<8):break
        recon_loss, cat_loss, xcov_loss, similarity_loss = train_on_batch(batch_x, batch_y)
        epoch_recon_loss_avg(recon_loss)
        epoch_cat_loss_avg(cat_loss)
        epoch_xcov_loss_avg(xcov_loss)
        epoch_sim_loss_avg(similarity_loss)

    epoch_time = time.time() - start
    print('{:3d}: {:.2f}s ETA: {:.2f}s  Reconstruction cost: {:.4f}  Supervised cost: {:.4f}  XCov cost: {:.4f}  Similarity cost: {:.4f}'
          .format(epoch + 1, epoch_time,
                  epoch_time * (n_epochs - epoch),
                  epoch_recon_loss_avg.result(),
                  epoch_cat_loss_avg.result(),
                epoch_xcov_loss_avg.result(),epoch_sim_loss_avg.result()))

encoder.save('Ronaks encoder.h5')
decoder.save('Ronaks decoder.h5')


text:
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import textwrap
import os

# -- Project information -----------------------------------------------------

project = 'MegEngine Documents'
copyright = '2020, Megvii'
author = 'Megvii'

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.mathjax',
    'sphinx.ext.viewcode',
    'sphinx.ext.autosummary',
    'sphinx.ext.doctest',
    'sphinxcontrib.jupyter',
    'sphinx_autodoc_typehints',
    'sphinx.ext.intersphinx',
    'nbsphinx'
]

api_url = "/api/latest"

if 'API_DOC_URL' in os.environ:
    api_url = os.environ['API_DOC_URL']

intersphinx_mapping = {'python': (api_url,'../build_api/html/objects.inv')}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
locale_dirs = ['locale/']

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
gettext_compact = False
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'pydata_sphinx_theme'
html_title = '文档-天元MegEngine'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# --  Options for jupyter output ---------------------------------------------------

jupyter_kernels = {
    'python3': {
        'kernelspec': {
            'display_name': 'Python',
            'language': 'python3',
            'name': 'python3'
        },
        'file_extension': '.py'
    },
}

# -- Options for skipping specific docs --------------------------------------

skip_blacklist = frozenset(
    [
        '__weakref__', '__module__', '__doc__', '__abstractmethods__',
        '__hyperparam_spec__', '__hyperparam_trans_dict__', '__param_init_spec__'
    ]
)
skip_whitelist = frozenset(['', ''])

def handle_skip(app, what, name, obj, skip, options):
    if name.startswith('_abc_') or name in skip_blacklist:
        return True
    if name.startswith('__testcase'):
        return False
    if (name.startswith('__') and name.endswith('__') and
            getattr(obj, '__doc__', None)):
        return False
    return skip

# -- Options for doctest -----------------------------------------------------

doctest_global_setup = '''
import numpy as np
np.random.seed(0)
import megengine as mge
np.set_printoptions(precision=4)
'''

def setup(app):
    app.connect("autodoc-skip-member", handle_skip)

max_stars_repo_name: MegEngine/doc

max_issues_repo_name: MegEngine/Doc

max_forks_repo_name: MegEngine/Doc


text:
# Standard Library
from itertools import product
from math import sqrt
from statistics import mean

# Third Party
import numpy as np

# Local
from ..utilities import modularity_matrix, modularity

##############
# MATH HELPERS
##############

def inverse_euclidean_dist(A):
    p1 = np.sum(A ** 2, axis=1)[:, np.newaxis]
    p2 = -2 * np.dot(A, A)
    p3 = np.sum(A.T ** 2, axis=1)
    E = 1 / np.sqrt(p1 + p2 + p3)

    return E

def cosine_sim(A):
    d = A @ A.T
    norm = (A * A).sum(0, keepdims=True) ** 0.5
    C = d / norm / norm.T

    return C

##############
# ALGO HELPERS
##############

# TODO: Implement Chebyshev and Manhattan distances
def node_similarity_matrix(adj_matrix, metric):
    if metric == "cosine":
        N = cosine_sim(adj_matrix)
    elif metric == "euclidean":
        N = inverse_euclidean_dist(adj_matrix)

    np.fill_diagonal(N, 0.0)
    return N

def find_best_merge(C):
    merge_indices = np.unravel_index(C.argmax(), C.shape)
    return min(merge_indices), max(merge_indices)

def merge_communities(communities, C, N, linkage):
    # Merge the two most similar communities
    c_i, c_j = find_best_merge(C)
    communities[c_i] |= communities[c_j]
    communities.pop(c_j)

    # Update the community similarity matrix, C
    C = np.delete(C, c_j, axis=0)
    C = np.delete(C, c_j, axis=1)

    for c_j in range(len(C)):
        if c_j == c_i:
            continue

        sims = []
        for u, v in product(communities[c_i], communities[c_j]):
            sims.append(N[u, v])

        if linkage == "single":
            similarity = min(sims)
        elif linkage == "complete":
            similarity = max(sims)
        elif linkage == "mean":
            similarity = mean(sims)
        # TODO: Add centroid-linkage

        C[c_i, c_j] = similarity
        C[c_j, c_i] = similarity

    return communities, C

######
# MAIN
######

def hierarchical_clustering(adj_matrix : np.ndarray, metric : str = "cosine",
                            linkage : str = "single", n : int = None) -> list:
    metric, linkage = metric.lower(), linkage.lower()

    communities = [{node} for node in range(len(adj_matrix))]
    M = modularity_matrix(adj_matrix)
    N = node_similarity_matrix(adj_matrix, metric)
    C = np.copy(N) # Community similarity matrix

    best_Q = -0.5
    while True:
        Q = 0.0
        if not n:
            Q = modularity(M, communities)
            if Q <= best_Q:
                break
        elif n and len(communities) == n:
            break

        communities, C = merge_communities(communities, C, N, linkage)
        best_Q = Q

    return communities

max_stars_repo_name: shobrook/communities

max_issues_repo_name: zdqf/communities

max_forks_repo_name: zdqf/communities


text:
from aeropy.geometry.parametric import CoordinateSystem
from aeropy.structural.stable_solution import (properties, boundary_conditions)
from aeropy.structural.shell import shell
from aeropy.xfoil_module import output_reader

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
from optimization_tools.DOE import DOE
import pickle

def input_function(x):
    return np.array([0,0] + list(x))

np.set_printoptions(precision=5)

# Results from Abaqus
abaqus_data = pickle.load(open('neutral_line.p', 'rb'))

# Beam properties
bp = properties()
bc = boundary_conditions(concentrated_load=np.array([[0.0, 0.0, -1.0], ]))
EB_solution = bc.concentrated_load[0][2]/(6*bp.young*bp.inertia) * \
    np.array([0, 0, 3, -1])

curve_parent = CoordinateSystem.polynomial(D=[0, 0, 0, 0], chord = 1, color = 'b')
curve_child = CoordinateSystem.polynomial(D=EB_solution, chord = 1, color  ='0.5')

beam = shell(curve_parent, curve_child, bp, bc)
chord_bounds = [[0., 2],]
beam.g_p.bounds = chord_bounds
beam.g_c.bounds = chord_bounds
beam.theta1 = np.linspace(0, beam.g_p.arclength()[0], 20)
beam.update_parent()
beam.update_child()
eulerBernoulle = beam.g_c.r(beam.g_c.x1_grid)

# Find stable solution
bounds = np.array(((-0.02,0.02),
                  (-0.02,0.02)))

print(EB_solution[2:4])
beam.minimum_potential(x0=[0,0], input_function = lambda x: [0,0] + list(x),
                       bounds = bounds)
# coefficients, results, arc_length = beam.stepped_loading(x0=[0,0],
#                                              input_function = lambda x: [0,0] + list(x),
#                                              bounds = bounds)
# print('arc_length')
# print(arc_length)
# plt.figure()
# plt.scatter(-1*np.array(results[:,0]), -1*np.array(results[:,1]))
# plt.plot([0, 1], [0, 0.00564])
# plt.show()
#
# plt.figure()
# plt.plot(arc_length[:,0], arc_length[:,1])
# plt.show()

# beam.g_c.a = input_function(x)
# beam.mesh.mesh_child()
# beam.calculate_position()
# beam.strain()
# beam.stress(loading_condition='plane_stress')
#
# print('x', x)
# print('f', beam.opt_f)
# print('Residual', beam.residual(x, input_type = 'Geometry',
#                                 loading_condition = 'plane_stress',
#                                 input_function = input_function))
print('Strain energy for min: ', beam.U)
[x,y,z] = eulerBernoulle.T
beam.g_p.plot(label='Parent')
plt.plot(x,z, 'k', lw = 3, label='Euler-Bernoulle', linestyle = '-', zorder=0)
beam.g_c.plot(label='Child', linestyle = '--')

plt.scatter(abaqus_data['coord'][0:401:40,0], abaqus_data['coord'][0:401:40,1], c='g', label='FEA', edgecolors='k', zorder = 10)
plt.legend()
plt.show()

# Plot beam results
plt.figure()
u = beam.u()
u1 = beam.u(diff='x1')
u2 = beam.u(diff='x2')
plt.plot(beam.r_p[0], beam.r_p[1], label='parent')
plt.scatter(beam.r_p[0], beam.r_p[1], label='parent')
plt.plot(beam.r_c[0], beam.r_c[1], label='child')
plt.scatter(beam.r_c[0], beam.r_c[1], label='child')
plt.plot(eulerBernoulle[0], eulerBernoulle[1], label='Euler-Bernoulle')
plt.scatter(eulerBernoulle[0], eulerBernoulle[1], label='Euler-Bernoulle')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['coord'][:,1], label='Abaqus')
plt.title('Position')
plt.grid()
plt.legend()

plt.figure()
plt.plot(beam.opt_f)
plt.ylabel('Energy (J)')
plt.xlabel('Iteration')

plt.figure()
plt.plot(beam.mesh.x_p, beam.epsilon[0][0], label=r'$\epsilon_{11}$')
plt.plot(beam.mesh.x_p, beam.epsilon[0][1], label=r'$\epsilon_{12}$')
plt.plot(beam.mesh.x_p, beam.epsilon[1][1], label=r'$\epsilon_{22}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['epsilon'][0][0],
         label=r'Abaqus $\epsilon_{11}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['epsilon'][0][1],
         label=r'Abaqus $\epsilon_{12}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['epsilon'][1][1],
         label=r'Abaqus $\epsilon_{22}$')
plt.title('Strain')
plt.legend()

plt.figure()
plt.plot(beam.mesh.x_p, beam.sigma[0][0], label=r'$\sigma_{11}$')
plt.plot(beam.mesh.x_p, beam.sigma[0][1], label=r'$\sigma_{12}$')
plt.plot(beam.mesh.x_p, beam.sigma[1][1], label=r'$\sigma_{22}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['sigma'][0][0],
         label=r'Abaqus $\sigma_{11}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['sigma'][0][0],
         label=r'Abaqus $\sigma_{12}$')
plt.plot(abaqus_data['coord'][:,0], abaqus_data['sigma'][0][0],
         label=r'Abaqus $\sigma_{22}$')
plt.legend()
plt.title('Stress')
plt.show()


text:
import warnings

import numpy as np
import pandas as pd

class SkdownscaleGroupGeneratorBase:
    pass

def MONTH_GROUPER(x):
    return x.month

def DAY_GROUPER(x):
    return x.day

class PaddedDOYGrouper(SkdownscaleGroupGeneratorBase):
    def __init__(self, df, offset=15):
        self.n = 1
        self.df = df
        self.max = 366
        # check for leap days
        # if leap days present, flag for day groups count
        if len(self.df[((self.df.index.month == 2) & (self.df.index.day == 29))]) > 0:
            self.leap = 'leap'
        else:
            self.leap = 'noleap'
        # split up data by leap and non leap years
        # necessary because pandas dayofyear
        self.df_leap = self.df[self.df.index.is_leap_year]
        self.df_noleap = self.df[~self.df.index.is_leap_year]
        self.offset = offset
        self.days_of_nonleap_year = np.arange(self.n, self.max)
        self.days_of_leap_year = np.arange(self.n, self.max + 1)
        self.days_of_nonleap_year_wrapped = np.pad(
            self.days_of_nonleap_year, self.offset, mode='wrap'
        )
        self.days_of_leap_year_wrapped = np.pad(self.days_of_leap_year, self.offset, mode='wrap')

    def __iter__(self):
        self.n = 1
        return self

    def __next__(self):
        # n as day of year
        if self.n > self.max:
            raise StopIteration

        i = self.n - 1
        total_days = (2 * self.offset) + 1

        # create day groups with +/- offset # of days
        first_set_leap = self.days_of_leap_year_wrapped[i : i + self.offset]
        first_set_noleap = self.days_of_nonleap_year_wrapped[i : i + self.offset]

        sec_set_leap = self.days_of_leap_year_wrapped[self.n + self.offset : i + total_days]
        sec_set_noleap = self.days_of_nonleap_year_wrapped[self.n + self.offset : i + total_days]

        all_days_leap = np.concatenate((first_set_leap, np.array([self.n]), sec_set_leap), axis=0)
        all_days_noleap = np.concatenate(
            (first_set_noleap, np.array([self.n]), sec_set_noleap), axis=0
        )

        # check that day groups contain the correct number of days
        if len(set(all_days_leap)) != total_days and self.leap == 'noleap':
            warnings.warn('leap days not included, day groups in leap years missing leap days')

        if len(set(all_days_noleap)) != total_days and self.n != 366:
            raise ValueError('no leap day groups do not contain the correct set of days')

        result = pd.concat(
            [
                self.df_leap[self.df_leap.index.dayofyear.isin(all_days_leap)],
                self.df_noleap[self.df_noleap.index.dayofyear.isin(all_days_noleap)],
            ]
        )

        self.n += 1

        return self.n - 1, result

    def mean(self):
        arr_means = np.full((self.max, 1), np.inf)
        for key, group in self:
            arr_means[key - 1] = group.mean().values[0]
        result = pd.DataFrame(arr_means, index=self.days_of_leap_year)
        return result

max_stars_repo_name: norlandrhagen/scikit-downscale

max_issues_repo_name: norlandrhagen/scikit-downscale

max_forks_repo_name: norlandrhagen/scikit-downscale


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths

import os
import cv2
import time
import xlwt
import numpy as np
import math

from opts import opts
from detectors.detector_factory import detector_factory
workbook = xlwt.Workbook(encoding = "utf-8")
booksheet = workbook.add_sheet('Sheet1',cell_overwrite_ok=True)
image_ext = ['jpg', 'jpeg', 'png', 'webp']
video_ext = ['mp4', 'mov', 'avi', 'mkv']
time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge']

def get_angle1(x1,y1,x2,y2,x3,y3):
    a=math.sqrt((x2-x3)**2+(y2-y3)**2)
    b=math.sqrt((x1-x3)**2+(y1-y3)**2)
    c=math.sqrt((x2-x1)**2+(y2-y1)**2)
    if c*b==0:
      cosA=1
    else:
      cosA=(a**2-c**2-b**2)/(-2*c*b)
    if cosA < -1.0:
      cosA=-1.0
    elif cosA>1.0:
      cosA=1.0
    A=math.acos(cosA)
    deg=math.degrees(A)
    return deg

def duquwenjian(glo,points, img_id='default'):
    points = np.array(points, dtype=np.int32).reshape(17, 2)
    A8=get_angle1(points[8,0],points[8,1],points[6,0],points[6,1],points[10,0],points[10,1])
    A7=get_angle1(points[7,0],points[7,1],points[5,0],points[5,1],points[9,0],points[9,1])
    A6=get_angle1(points[6,0],points[6,1],points[8,0],points[8,1],points[12,0],points[12,1])
    A5=get_angle1(points[5,0],points[5,1],points[7,0],points[7,1],points[11,0],points[11,1])
    A12=get_angle1(points[12,0],points[12,1],points[14,0],points[14,1],points[11,0],points[11,1])
    A11=get_angle1(points[11,0],points[11,1],points[13,0],points[13,1],points[12,0],points[12,1])
    A14=get_angle1(points[14,0],points[14,1],points[16,0],points[16,1],points[12,0],points[12,1])
    A13=get_angle1(points[13,0],points[13,1],points[15,0],points[15,1],points[11,0],points[11,1])
    xcenter=(points[11,0]+points[12,0])/2
    ycenter=(points[11,1]+points[12,1])/2
    distance={}
    field=0
    for j in range(17):
      distance[j]=math.sqrt((points[j, 0]-xcenter)**2+(points[j, 1]-ycenter)**2)
      field = field +distance[j]
    print('field',field)
    print(points[7])
    #booksheet.write(glo,1,float(A8))
    #booksheet.write(glo,2,float(A7))
    #booksheet.write(glo,3,float(A6))
    #booksheet.write(glo,4,float(A5))
    #booksheet.write(glo,5,float(A12))
    #booksheet.write(glo,6,float(A11))
    #booksheet.write(glo,7,float(A14))
    #booksheet.write(glo,8,float(A13))
    #booksheet.write(glo,9,float(field))

    booksheet.write(glo,1,float(points[5,0]))
    booksheet.write(glo,2,float(points[5,1]))
    booksheet.write(glo,3,float(points[6,0]))
    booksheet.write(glo,4,float(points[6,1]))
    booksheet.write(glo,5,float(points[7,0]))
    booksheet.write(glo,6,float(points[7,1]))
    booksheet.write(glo,7,float(points[8,0]))
    booksheet.write(glo,8,float(points[8,1]))
    booksheet.write(glo,9,float(points[9,0]))
    booksheet.write(glo,10,float(points[9,1]))
    booksheet.write(glo,11,float(points[10,0]))
    booksheet.write(glo,12,float(points[10,1]))
    booksheet.write(glo,13,float(points[11,0]))
    booksheet.write(glo,14,float(points[11,1]))
    booksheet.write(glo,15,float(points[12,0]))
    booksheet.write(glo,16,float(points[12,1]))
    booksheet.write(glo,17,float(points[13,0]))
    booksheet.write(glo,18,float(points[13,1]))
    booksheet.write(glo,19,float(points[14,0]))
    booksheet.write(glo,20,float(points[14,1]))
    booksheet.write(glo,21,float(points[15,0]))
    booksheet.write(glo,22,float(points[15,1]))
    booksheet.write(glo,23,float(points[16,0]))
    booksheet.write(glo,24,float(points[16,1]))

def demo(opt):
  os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
  opt.debug = max(opt.debug, 1)
  Detector = detector_factory[opt.task]
  detector = Detector(opt)

  #if True:
  if opt.demo == 'webcam' or \
    opt.demo[opt.demo.rfind('.') + 1:].lower() in video_ext:
    cam = cv2.VideoCapture(0 if opt.demo == 'webcam' else opt.demo)
    #cam = cv2.VideoCapture("../images/webwxgetvideo")
    detector.pause = False
    t=0
    size = (int(cam.get(cv2.CAP_PROP_FRAME_WIDTH)),int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    codec = cv2.VideoWriter_fourcc(*'DIVX')
    output = cv2.VideoWriter('differ.avi',codec,25.0,size)
    while cam.isOpened():
        ret, img = cam.read()

        #print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%",img.shape)
        #cv2.imshow('input', img)
        s=time.time()
        ret = detector.run(img)
        #print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^",img.shape)
        results = ret['results']
        #print(results[1])
        for bbox in results[1]:
            if bbox[4] > opt.vis_thresh:
                duquwenjian(t,bbox[5:39],img_id='multi_pose')
        e=time.time()
        fps=1/(e-s)
        print("FPS",fps)
        #print("FPS",1/ret['tot'])
        time_str = ''
        for stat in time_stats:
          time_str = time_str + '{} {:.3f}s |'.format(stat, ret[stat])
        print(time_str)
        if t in [0,1,39,87,88,177,178,180,181,182,184,195,196,200,201,202,203,204,205,206,207,208,212,217,222,223,229,230,239,240,241,257,258,259,285,286,343,346,347,348,349,350,351,352,353,358,359,360,366,371,379,386,387,388]:
            output.write(img)
        t=t+1
        #print(t)
        if cv2.waitKey(10) & 0XFF == ord('q'):
            print("ed")
            workbook.save("dance22.xls")
            break  # esc to quit
    cam.release()
    cv2.destroyAllWindows()
    workbook.save("test.xls")

    #stream.release()

  else:
    if os.path.isdir(opt.demo):
      image_names = []
      ls = os.listdir(opt.demo)
      for file_name in sorted(ls):
          ext = file_name[file_name.rfind('.') + 1:].lower()
          if ext in image_ext:
              image_names.append(os.path.join(opt.demo, file_name))
    else:
      image_names = [opt.demo]

    for (image_name) in image_names:
      ret = detector.run(image_name)
      print(ret)
      time_str = ''
      for stat in time_stats:
        time_str = time_str + '{} {:.3f}s |'.format(stat, ret[stat])
      print(time_str)
if __name__ == '__main__':
  opt = opts().init()
  demo(opt)


text:
#!/usr/bin/env python
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.widgets import Slider
from myplot.basemap import Basemap
from myplot.quiver import Quiver
import misc
import matplotlib
matplotlib.quiver.Quiver = Quiver # for error ellipses

def quiver_args(position,disp_array,cov_array=None,mask=None):
  N = len(position)
  if mask is None:
    mask = np.zeros(N,dtype=bool)

  x = position[:,0]
  y = position[:,1]
  u = disp_array[:,0]
  u[mask] = 0.0
  v = disp_array[:,1]
  v[mask] = 0.0

  if cov_array is not None:
    var_u = cov_array[:,0]
    var_v = cov_array[:,1]
    #cov_uv = cov_array[:,0,1]
    cov_uv = 0*var_u
    var_u[mask] = 1e-8
    var_v[mask] = 1e-8
    cov_uv[mask] = 1e-8
    sigma_u = np.sqrt(var_u)
    sigma_v = np.sqrt(var_v)
    rho = cov_uv/(sigma_u*sigma_v)
    return (x,y,u,v,(sigma_u,sigma_v,rho))

  else:
    return (x,y,u,v)

def create_default_basemap(lat_lst,lon_lst):
  '''
  creates a basemap that bounds lat_lst and lon_lst
  '''
  lon_buff = (max(lon_lst) - min(lon_lst))/10.0
  lat_buff = (max(lat_lst) - min(lat_lst))/10.0
  if lon_buff < 0.5:
    lon_buff = 0.5

  if lat_buff < 0.5:
    lat_buff = 0.5

  llcrnrlon = min(lon_lst) - lon_buff
  llcrnrlat = min(lat_lst) - lat_buff
  urcrnrlon = max(lon_lst) + lon_buff
  urcrnrlat = max(lat_lst) + lat_buff
  lon_0 = (llcrnrlon + urcrnrlon)/2.0
  lat_0 = (llcrnrlat + urcrnrlat)/2.0
  return Basemap(projection='tmerc',
                 lon_0 = lon_0,
                 lat_0 = lat_0,
                 llcrnrlon = llcrnrlon,
                 llcrnrlat = llcrnrlat,
                 urcrnrlon = urcrnrlon,
                 urcrnrlat = urcrnrlat,
                 resolution = 'h')

def view(data_list,
         name_list=None,
         draw_map=True,
         quiver_scale=0.00001,
         scale_length=1.0):
  # data list is a list of dictionary-lie objects with keys: mean,
  # covariance, mask, position, time
  mask_list = []
  mean_list = []
  cov_list = []
  if name_list is None:
    name_list = ['displacement %s' % i for i in range(len(data_list))]

  for data in data_list:
    mask_list += [data['mask']]
    mean_list += [data['mean']]
    cov_list += [data['covariance']]

  times = data_list[0]['time'][:]
  lon = data_list[0]['position'][:,0]
  lat = data_list[0]['position'][:,1]
  station_names = data_list[0]['name'][:]

  _view(mean_list,
        cov_list,
        times,
        station_names,
        lon,
        lat,
        mask_list,
        disp_type=name_list,
        draw_map=draw_map,
        quiver_scale=quiver_scale,
        scale_length=scale_length)

def _view(displacement_list,
          covariance_list,
          times,
          station_names,
          lon,
          lat,
          mask,
          disp_type=None,
          colors=None,
          draw_map=False,
          scale_length=1.0,
          quiver_scale=0.0001,
          map_resolution=200,
          artists=None):

  #times -= 2010.0
  N = len(displacement_list)
  Nx = len(lon)
  if disp_type is None:
    disp_type = ['']*N

  if colors is None:
    colors = ['k','b','r','g','m']

  if artists is None:
    artists = []

  # setup background of main figure
  sub_fig = plt.figure('Time Series',figsize=(9.0,6.6))
  main_fig = plt.figure('Map View',figsize=(10,11.78))

  slider_ax = main_fig.add_axes([0.08,0.88,0.76,0.04])
  sub_ax1 = sub_fig.add_subplot(311)
  sub_ax2 = sub_fig.add_subplot(312)
  sub_ax3 = sub_fig.add_subplot(313)
  main_ax = main_fig.add_axes([0.08,0.08,0.76,0.76])

  time_slider = Slider(slider_ax,'time',
                       min(times),max(times),
                       valinit=min(times),
                       color='black')
  time = min(times)
  time_idx = np.argmin(abs(times - time))

  if draw_map is True:
    basemap = create_default_basemap(lat,lon)
    position = basemap(lon,lat)
    position = np.array(position).transpose()
    main_ax.patch.set_facecolor([0.0,0.0,1.0,0.2])
    basemap.drawtopography(ax=main_ax,vmin=-6000,vmax=4000,
                           alpha=1.0,resolution=map_resolution,zorder=0)
    basemap.drawcoastlines(ax=main_ax,linewidth=1.5,zorder=1)
    basemap.drawcountries(ax=main_ax,linewidth=1.5,zorder=1)
    basemap.drawstates(ax=main_ax,linewidth=1,zorder=1)
    basemap.drawrivers(ax=main_ax,linewidth=1,zorder=1)
    basemap.drawmeridians(np.arange(np.floor(basemap.llcrnrlon),
                        np.ceil(basemap.urcrnrlon),1.0),
                        labels=[0,0,0,1],dashes=[2,2],
                        ax=main_ax,zorder=1)
    basemap.drawparallels(np.arange(np.floor(basemap.llcrnrlat),
                        np.ceil(basemap.urcrnrlat),1.0),
                        labels=[1,0,0,0],dashes=[2,2],
                        ax=main_ax,zorder=1)
    basemap.drawmapscale(units='km',
                       lat=basemap.latmin+(basemap.latmax-basemap.latmin)/10.0,
                       lon=basemap.lonmax-(basemap.lonmax-basemap.lonmin)/5.0,
                       fontsize=16,
                       lon0=(basemap.lonmin+basemap.lonmax)/2.0,
                       lat0=(basemap.latmin+basemap.latmax)/2.0,
                       barstyle='fancy',ax=main_ax,
                       length=100,zorder=10)

  else:
    position = np.array([lon,lat]).transpose()
    main_ax.set_aspect('equal')

  station_point_lst = []
  station_point_label_lst = []
  for sid in range(Nx):
    loni = lon[sid]
    lati = lat[sid]
    x,y = position[sid,:]
    #x,y = basemap(loni,lati)
    station_point = main_ax.plot(x,y,'ko',markersize=3,picker=8,zorder=2)
    station_point_label_lst += [station_point[0].get_label()]
    station_point_lst += station_point

  station_point_label_lst = np.array(station_point_label_lst,dtype=str)

  Q_lst = []
  for idx in range(N):
    if covariance_list[idx] is not None:
      args = quiver_args(position,
                         displacement_list[idx][time_idx,:,:],
                         covariance_list[idx][time_idx,:,:],
                         mask[idx][time_idx,:])

      Q_lst += [main_ax.quiver(args[0],args[1],args[2],args[3],sigma=args[4],
                               scale_units='xy',
                               angles='xy',
                               width=0.004,
                               scale=quiver_scale,
                               color=colors[idx],
                               ellipse_edgecolors=colors[idx],
                               zorder=3)]
    else:
      args = quiver_args(position,
                         displacement_list[idx][time_idx,:,:])

      Q_lst += [main_ax.quiver(args[0],args[1],args[2],args[3],
                               scale_units='xy',
                               angles='xy',
                               width=0.004,
                               scale=quiver_scale,
                               color=colors[idx],
                               ellipse_edgecolors=colors[idx],
                               zorder=3)]

  u_scale = np.array([scale_length])
  v_scale = np.array([0.0])
  z_scale = np.array([0.0])
  xlim = main_ax.get_xlim()
  ylim = main_ax.get_ylim()
  xo = xlim[0]
  yo = ylim[0]
  xwidth = xlim[1] - xlim[0]
  ywidth = ylim[1] - ylim[0]
  x_scale = np.array([xo + xwidth/10.0])
  y_scale = np.array([yo + ywidth/10.0])
  x_text = xo + xwidth/10.0
  y_text = yo + ywidth/15.0
  dy_text = ywidth/20.0

  for i in range(N):
    main_ax.text(x_text,y_text+i*dy_text,disp_type[i],fontsize=16)
    main_ax.quiver(x_scale,y_scale+i*dy_text,u_scale,v_scale,
                  scale_units='xy',
                  angles='xy',
                  width=0.004,
                  scale=quiver_scale,
                  color=colors[i])

  main_ax.text(x_text,y_text+N*dy_text,
               '%s meter displacement' % np.round(scale_length,3),
               fontsize=16)

  def _slider_update(t):
    time_idx = np.argmin(abs(t - times))
    for idx in range(N):
      if covariance_list[idx] is not None:
        args = quiver_args(position,
                           displacement_list[idx][time_idx,:,:],
                           covariance_list[idx][time_idx,:,:],
                           mask[idx][time_idx,:])

        Q_lst[idx].set_UVC(args[2],args[3],sigma=args[4])

      else:
        args = quiver_args(position,
                           displacement_list[idx][time_idx])

        Q_lst[idx].set_UVC(args[2],args[3])

    main_fig.canvas.draw()
    return

  def _onpick(event):
    idx, = np.nonzero(str(event.artist.get_label()) == station_point_label_lst)
    station_label = station_names[idx[0]]
    sub_ax1.cla()
    sub_ax2.cla()
    sub_ax3.cla()
    for i in range(N):
      midx = mask[i][:,idx]
      disp = displacement_list[i][:,idx,:]
      cov = covariance_list[i][:,idx,:]
      if i == 0:
        sub_ax1.errorbar(times[~midx],
                         disp[~midx,0],
                         np.sqrt(cov[~midx,0]),
                         color=colors[i],capsize=0,fmt='.')
        sub_ax2.errorbar(times[~midx],
                         disp[~midx,1],
                         np.sqrt(cov[~midx,1]),
                         color=colors[i],capsize=0,fmt='.')
        sub_ax3.errorbar(times[~midx],
                         disp[~midx,2],
                         np.sqrt(cov[~midx,2]),
                         color=colors[i],capsize=0,fmt='.')
      else:
        sub_ax1.plot(times[~midx],
                     disp[~midx,0],
                     color=colors[i])
        sub_ax2.plot(times[~midx],
                     disp[~midx,1],
                     color=colors[i])
        sub_ax3.plot(times[~midx],
                     disp[~midx,2],
                     color=colors[i])

    sub_ax1.set_title(station_label,fontsize=16)
    sub_ax1.set_ylabel('easting',fontsize=16)
    sub_ax2.set_ylabel('northing',fontsize=16)
    sub_ax3.set_ylabel('vertical',fontsize=16)
    sub_fig.canvas.draw()
    event.artist.set_markersize(10)
    main_fig.canvas.draw()
    event.artist.set_markersize(3.0)
    return

  time_slider.on_changed(_slider_update)
  main_fig.canvas.mpl_connect('pick_event',_onpick)

  for a in artists:
    ax.add_artist(a)

  plt.show()

max_stars_repo_name: treverhines/PSGI

max_issues_repo_name: treverhines/PSGI

max_forks_repo_name: treverhines/PSGI


text:
import pandas as pd
import numpy as np
from collections import Counter

data = pd.read_csv('out/negex_all.txt', sep="\t", header=None)
print(data.shape)

data.columns = ['PAT_DEID','NOTE_DEID','NOTE_DATE','ENCOUNTER_DATE','NOTE_CODE','TEXT_SNIPPET','lower_text','STATUS']
df = data.groupby(['PAT_DEID','NOTE_DEID','NOTE_DATE','ENCOUNTER_DATE','NOTE_CODE'])['STATUS'].apply(','.join).reset_index()
df_text = data.groupby(['PAT_DEID','NOTE_DEID','NOTE_DATE','ENCOUNTER_DATE','NOTE_CODE'])['TEXT_SNIPPET'].apply(' ##### '.join).reset_index()
df_text_required = df_text[['NOTE_DEID','TEXT_SNIPPET']]
df_fin = pd.merge(df, df_text_required, on='NOTE_DEID', how='inner')
df1 = df_fin.copy()

def check(l):
    # l1 = l['STATUS'].tolist()
    # l2 = str(l1).split(',')
    l2 = l['STATUS'].split(',')
    c = Counter(l2)
    affirmed = c['affirmed']
    negated = c['negated']
    if (affirmed > negated or affirmed == negated):
        return "Affirmed"
    else:
        return "Negated"

def majority_rule(var1,var2):
    df[var2] = df.apply(check, axis = 1)
    return df

df1 = majority_rule('STATUS','STATUS_FINAL')
print(df1.shape)
df2 = pd.merge(df1, df_text_required, on='NOTE_DEID', how='inner')

df2.to_pickle("out/annotated_note_all.pkl")

max_stars_repo_path: src/data processing - clinical notes and structured data/step5_note_level_tagging.py

max_stars_repo_name: arjun-parthi/SSRI-Project

max_issues_repo_name: arjun-parthi/SSRI-Project

max_forks_repo_name: arjun-parthi/SSRI-Project


text:
import numpy as np
from PIL import ImageGrab
# from draw_lines import draw_lines
import cv2
import time

# explicite what are the lines
def draw_lines(img, lines):
    try:
        for line in lines:
            coords = line[0]
            cv2.line(img, (coords[0],coords[1]), (coords[2],coords[3]), [255,0,0], 2)
    except:

        pass
# region of interest
def roi(img, vertices):
    # blank mask:
    mask = np.zeros_like(img)

    # defining the number of  color channel to fill the mask with depending on the input image
    if len(img.shape) > 2:
        channel_count = img.shape[3]  # 3 channels
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255

    # fill the mask
    cv2.fillPoly(mask, vertices, ignore_mask_color)

    # now only show the area that is the mask
    masked = cv2.bitwise_and(img, mask)
    return masked

# processed the grabbed image into Gray > Edge > Smooth > Mask (region of interest) > lines drawing
# from the edge detector
def hough_lines(original_image):
    min_line_len = 100 # 300
    max_line_gap = 100 # 100
    processed_img = cv2.cvtColor(original_image, cv2.COLOR_BGR2GRAY)  #  GRAY HSV hue, saturation, value
    processed_img = cv2.Canny(processed_img, threshold1=210, threshold2=500)
    processed_img = cv2.GaussianBlur(processed_img, (1,1), 0)
    vertices = np.array([[0,635],[0,317], [200,270], [600,270], [800,317], [800,635]], np.int32)
    roi_img = roi(processed_img, [vertices])

    # rho=2 and theta= np.pi/180 are the distance and angular resolution of the grid in Hough space.
    # threshold = 180 is minimum number of intersections in a grid for candidate line to go to output
    lines = cv2.HoughLinesP(roi_img, 2, np.pi/180, 180, np.array([]), minLineLength= min_line_len, maxLineGap=max_line_gap)
    line_processed_img = np.zeros((roi_img.shape[0], roi_img.shape[1], 3), dtype=np.uint8)

    # draw_lines(roi_img, lines)
    draw_lines(line_processed_img, lines)
    return line_processed_img

# Merge of image processed + image with lines
def weighted_img(processed_img, original_image, α=0.8, β=1., λ=0.):

    return cv2.addWeighted(original_image, α, processed_img, β, λ)

# Grabbing the image from screen upper left (position 0,40) with resolution of 635x800
# then processed the previous functions until resize the inital input to 89x120 while avoiding loosing min. info
def grab2np():
    last_time = time.time()
    while True:

        # 800x635 windowed mode
        screen = np.array(ImageGrab.grab(bbox=(0, 40, 800, 635)))

        last_time = time.time()
        new_screen = hough_lines(screen)
        new_screen = weighted_img(new_screen, screen, α=0.8, β=1., λ=0.)
        new_screen = cv2.resize(new_screen, None,fx=0.15,fy=0.15,interpolation=cv2.INTER_AREA)

        # cv2.imshow('window', new_screen)
        cv2.imshow('window',cv2.cvtColor(new_screen, cv2.COLOR_BGR2RGB))

        if cv2.waitKey(25) & 0xFF == ord('q'):
            cv2.destroyAllWindows()
            break
        return new_screen
# grab2np()