text:
import numpy as np
from scipy.ndimage import map_coordinates

def rectify_orders(image_data, trace, half_window=10, debug=False, nullify_mapped_values=True):
    """
    :param image_data: xwavecal.images.Image.data
    :param trace: Trace object
    :param half_window: half of the size of the extraction window about the center of the traces. E.g. 10 means 21 pixel
                        window will be extracted.
    :param debug: boolean for returning an internal copy of the image which is used in extract.
    :param nullify_mapped_values: bool.
                                  True if the flux from a pixel can only be used in one order's spectrum.
                                  False if a pixel's flux can be double (or triple etc..) counted.
    :return rectified_orders: Dictionary
    Dictionary where the keys are the trace id's from trace.get_id(),
    where rectified_2d_spectrum[trace_id]['val'] is a 2d float array (flux for the trace_id order).
    If half extraction window was 10, then rectified_2d_spectrum[trace_id]['val']
    is 21 rows by 4096 columns (for a 4096 pixel wide image). One would sum this 2d
    spectrum along-columns to get a box extracted spectrum.
    """
    rectified_orders = {}
    num_orders = trace.num_traces_found()
    image_copy = np.copy(image_data)
    x_coordinates, y_coordinates = np.meshgrid(np.arange(image_data.shape[1]), np.arange(image_data.shape[0]))
    image_coordinates = {'x': x_coordinates, 'y': y_coordinates}
    for index in np.arange(num_orders):
        rectified_orders[trace.get_id(index)], image_copy = rectify_order(image_copy, image_coordinates,
                                                                          trace.get_centers(index),
                                                                          half_window=half_window,
                                                                          nullify_mapped_values=nullify_mapped_values)
    if debug:
        return rectified_orders, image_copy
    else:
        return rectified_orders

def rectify_order(image_data, image_coordinates, single_order_centers, half_window=10, nullify_mapped_values=True):
    """
    :param image_data: xwavecal.images.Image.data
    :param image_coordinates: dictionary with x and y coordinates for each pixel.
    :param single_order_centers: the y centers of the center of the rectification window
    :param half_window: the half width of the window about which one is rectifying the trace/order
    :param nullify_mapped_values: True if the flux from a pixel can only be used in one order's spectrum.
                                  False if a pixel's flux can be double (or triple etc..) counted.
    :return: An array of 2*half_window + 1 rows by width(image_data) about the order. The center of the order
             is right at the index of half_window, e.g. rectified_order[half_window] gives the flux down the
             center of the order.
    """
    spectrum_shape = (2*half_window + 1, image_data.shape[1])
    rectified_order = {'val': np.zeros(spectrum_shape),
                       'y': np.zeros(spectrum_shape),
                       'x': np.zeros(spectrum_shape)}
    x_coords = np.arange(image_data.shape[1])
    for offset, row in zip(np.arange(-half_window, half_window + 1), np.arange(spectrum_shape[0])):
        mapped_y_values = map_coordinates(image_coordinates['y'], [single_order_centers + offset, x_coords],
                                          order=0, mode='constant', cval=0, prefilter=False)
        rectified_order['val'][row] = image_data[(mapped_y_values, x_coords)]
        rectified_order['y'][row] = mapped_y_values
        rectified_order['x'][row] = x_coords
        if nullify_mapped_values:
            image_data[(mapped_y_values, x_coords)] = 0
    return rectified_order, image_data

max_stars_repo_name: gmbrandt/echelle

max_issues_repo_name: gmbrandt/echelle

max_forks_repo_name: gmbrandt/echelle


text:
import pandas as pd
import numpy as np
import tensorflow as tf
from datetime import datetime
from typing import List, Tuple

from data_mining import ColorizedLogger

logger = ColorizedLogger('BuildModel', 'cyan')

class BuildModel:
    __slots__ = ('window_size',)

    window_size: int

    def __init__(self, window_size: int):
        self.window_size = window_size

    def trim_dataset(self, df: pd.DataFrame) -> pd.DataFrame:
        # Calculate the number of weeks extra days
        num_days = df.shape[0]
        num_day_groups, extra_days = divmod(num_days, self.window_size)
        logger.info(f"Number of days in the dataset: {num_days}")
        logger.info(
            f"Number of {self.window_size}-day groups: {num_day_groups} + {extra_days} extra days")
        # Drop Last extra_days rows from the datasets
        logger.info(f"Dropping the last {extra_days} rows from the covid dataset")
        covid_df_clean_9 = df.drop(df.tail(extra_days).index)
        df = df.drop(df.tail(extra_days).index)
        logger.info(f"Dropped {num_days - covid_df_clean_9.shape[0]} rows")

        return df

    def split_dataset(self, df: pd.DataFrame, train_perc: float = 0.7) \
            -> Tuple[np.ndarray, np.ndarray, pd.DataFrame, pd.DataFrame]:
        """ Split a univariate dataset into train/test sets """

        df = df.reset_index(drop=True)
        # Calculate the train and test lengths
        total_days = len(df)
        train_days = (int(total_days * train_perc) // self.window_size) * self.window_size

        # Keep the date column and the drop
        date_train_col, date_test_col = df.date.iloc[:train_days], df.date.iloc[train_days:]
        df = df.drop(columns='date')

        # Get the values from the DF
        data = df.values

        # split into standard weeks
        train, test = data[:train_days], data[train_days:]
        # restructure into windows of weekly data
        train = np.array(np.split(train, len(train) / self.window_size))
        test = np.array(np.split(test, len(test) / self.window_size))

        logger.info(f"Train shape: {train.shape}")
        logger.info(f"Test shape: {test.shape}")

        return train, test, date_train_col, date_test_col

    @staticmethod
    def rejoin_dataset(predictions: np.ndarray, test: np.ndarray, after_pivot_columns: List[str],
                       date_column: pd.Series) -> Tuple[pd.DataFrame, pd.DataFrame]:

        # Flatten Numpy Arrays
        predictions_flat = predictions.reshape((predictions.shape[0] * predictions.shape[1],
                                                predictions.shape[2]))
        test_flat = test.reshape((test.shape[0] * test.shape[1], test.shape[2]))

        # Prepare for unpivoting
        after_pivot_columns_witout_date = [c for c in after_pivot_columns if c != 'date']
        date_test_col = date_column.reset_index(drop=True)

        # Convert the Numpy Arrays to Dataframes
        test_expanded_df = pd.DataFrame(test_flat, columns=after_pivot_columns_witout_date)
        predictions_expanded_df = pd.DataFrame(predictions_flat,
                                               columns=after_pivot_columns_witout_date)

        # Reinsert the date column
        test_expanded_df.insert(0, 'date', date_test_col)
        predictions_expanded_df.insert(0, 'date', date_test_col)

        return predictions_expanded_df, test_expanded_df

    def build_model(self, train: np.ndarray, loss: tf.keras.losses.Loss,
                    activation='linear', neurons_per_layer: List[int] = None) \
            -> Tuple[tf.keras.Model, np.ndarray, np.ndarray]:
        """ Train the model """

        if neurons_per_layer is None:
            neurons_per_layer = [200, 200, 100]

        # prepare data
        num_features = train.shape[2]
        train_x, train_y = self.to_supervised(train, self.window_size, self.window_size)
        # define parameters
        n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]

        # define model
        model = tf.keras.Sequential()
        model.add(tf.keras.layers.LSTM(neurons_per_layer[0], activation='relu',
                                       input_shape=(self.window_size, num_features)))
        model.add(tf.keras.layers.RepeatVector(n_outputs))
        model.add(tf.keras.layers.LSTM(neurons_per_layer[1], activation='relu',
                                       return_sequences=True))
        model.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(neurons_per_layer[2],
                                                                        activation='relu')))
        model.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(num_features,
                                                                        activation=activation)))
        model.compile(loss=loss, optimizer='adam')
        tf.keras.utils.plot_model(
            model, to_file='img/model.png', show_shapes=False, show_dtype=False,
            show_layer_names=True, rankdir='TB', expand_nested=False, dpi=96
        )

        return model, train_x, train_y

    @classmethod
    def train_model(cls, model: tf.keras.Model, train_x: np.ndarray, train_y: np.ndarray, epochs: int,
                    batch_size: int, verbose: int) -> tf.keras.Model:

        # Reshape output into [samples, timesteps, features]
        train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))

        # Fit the Model
        log_dir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
        tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
        model.fit(train_x, train_y, epochs=epochs,
                  batch_size=batch_size, verbose=verbose, callbacks=[tensorboard_callback])
        return model

    @staticmethod
    def to_supervised(train: np.ndarray, n_input: int, n_out: int) \
            -> Tuple[np.ndarray, np.ndarray]:
        """ Convert history into inputs and outputs """

        # flatten data
        data = train.reshape((train.shape[0] * train.shape[1], train.shape[2]))
        x, y = list(), list()
        in_start = 0
        # step over the entire history one time step at a time
        for _ in range(len(data)):
            # define the end of the input sequence
            in_end = in_start + n_input
            out_end = in_end + n_out
            # ensure we have enough data for this instance
            if out_end <= len(data):
                x.append(data[in_start:in_end, :])
                y.append(data[in_end:out_end, 0])
            # move along one time step
            in_start += 1

        train_x, train_y = np.array(x), np.array(y)

        return train_x, train_y

    def generate_predictions_one_by_one(self, train: np.ndarray, test: np.ndarray,
                                        model: tf.keras.Model) -> np.ndarray:

        # history is a list of weekly data
        history = [x for x in train]
        # walk-forward validation over each week

        predictions = list()
        num_groups = test.shape[0]
        num_days_per_group = test.shape[1]
        num_features = test.shape[2]
        test_flat = test.reshape((test.shape[0] * test.shape[1], test.shape[2]))
        test2 = []
        for ind, _ in enumerate(test_flat[:]):
            test_group = []
            for test_days in test_flat[ind:ind + num_days_per_group]:
                test_group.append(test_days)
            if len(test_group) < num_days_per_group:
                test_group += [np.empty(num_features) for _ in
                               range(num_days_per_group - len(test_group))]
            test2.append(np.array(test_group))
        test2 = np.array(test2)

        for i in range(len(test2)):
            # predict the week
            yhat_sequence = self.forecast(model, history, self.window_size)
            # store the predictions
            predictions.append(yhat_sequence[0])
            extra_seq_diff = i - ((num_groups - 1) * num_days_per_group + 1)
            if extra_seq_diff > 0:
                test2[i, -extra_seq_diff:] = yhat_sequence[-extra_seq_diff:]
            # get real observation and add to history for predicting the next week
            history.append(test2[i, :])
        predictions = np.array(predictions)
        predictions = predictions.reshape((num_groups, predictions.shape[0] // num_groups,
                                           predictions.shape[1]))
        return predictions

    def generate_predictions_whole_group(self, train: np.ndarray, test: np.ndarray,
                                         model: tf.keras.Model) -> np.ndarray:

        # history is a list of weekly data
        history = [x for x in train]
        # walk-forward validation over each week
        predictions = list()
        for i in range(len(test)):
            # predict the week
            yhat_sequence = self.forecast(model, history, self.window_size)
            # store the predictions
            predictions.append(yhat_sequence)
            # get real observation and add to history for predicting the next week
            history.append(test[i, :])
        predictions = np.array(predictions)
        return predictions

    @staticmethod
    def forecast(model: tf.keras.Model, history: List[np.ndarray], n_input: int) \
            -> np.ndarray:
        """ Make a forecast """

        # flatten data
        data = np.array(history)
        data = data.reshape((data.shape[0] * data.shape[1], data.shape[2]))
        # retrieve last observations for input data
        input_x = data[-n_input:, :]
        # reshape into [1, n_input, n]
        input_x = input_x.reshape((1, input_x.shape[0], input_x.shape[1]))
        # forecast the next week
        yhat = model.predict(input_x, verbose=0)
        # we only want the vector forecast
        yhat = yhat[0]
        return yhat

    @classmethod
    def evaluate_predictions(cls, actual_df: pd.DataFrame, predicted_df: pd.DataFrame) \
            -> pd.DataFrame:
        """ Evaluate the predictions """

        scores_columns = list(actual_df.columns)[:2] + ['rmse']
        scores_df = pd.DataFrame(columns=scores_columns)
        actual = actual_df.values
        predicted = predicted_df.values
        # calculate an RMSE score for each day
        for row in range(actual.shape[0]):
            # RMSE
            curr_actual = np.array(actual[row, 2])
            curr_predicted = np.array(predicted[row, 2])
            rmse = cls.rmse(curr_actual, curr_predicted)
            # Save Result
            scores_df = scores_df.append({scores_columns[0]: actual[row, 0],
                                          scores_columns[1]: actual[row, 1],
                                          scores_columns[2]: rmse},
                                         ignore_index=True)

        return scores_df

    @staticmethod
    def rmse(actual, prediction):
        return np.sqrt(((prediction - actual) ** 2).mean())

    @staticmethod
    def summarize_scores(scores_df: pd.DataFrame) \
            -> [float, pd.Series, pd.Series]:
        """ Summarize scores """

        scores_columns = scores_df.columns
        # Total average scores
        total_avg = scores_df[scores_columns[2]].mean()
        # Per Date average scores
        per_date_avg = scores_df.groupby(scores_df.date)[scores_columns[2]].mean()
        # Per Country average scores
        per_country_avg = scores_df.groupby(scores_df.iso_code)[scores_columns[2]].mean()

        return total_avg, per_date_avg, per_country_avg

ext: py

reason: import numpy


text:
import os
import pandas as pd
import numpy as np

"""
    通过代码测试：
    input_file = "data\\0.input\\enron\\ia-enron-email-dynamic.edges"
    dataframe = pd.read_csv(input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])
    dataframe['timestamp']=pd.to_datetime(dataframe['timestamp'],unit='ms').dt.strftime('%Y-%m-%d %H:%M:%s')
    print(dataframe['timestamp'].unique())
    最终确定粒度为小时的时候比较合适，会落在以下这几个区间内
    ['1970-01-01 00' '1970-01-14 22' '1970-01-15 01' '1970-01-14 11'
     '1970-01-14 23' '1970-01-14 16' '1970-01-14 17' '1970-01-15 02'
     '1970-01-15 04' '1970-01-15 05' '1970-01-14 15' '1970-01-14 18'
     '1970-01-15 00' '1970-01-14 20' '1970-01-14 14' '1970-01-15 03'
     '1970-01-14 21' '1970-01-14 19' '1970-01-14 13' '1970-01-14 10'
     '1970-01-14 09' '1970-01-14 12' '1970-01-15 06']
"""

def check_and_make_path(to_make):
    if to_make == '':
        return
    if not os.path.exists(to_make):
        os.makedirs(to_make)

class FBTransformer:
    input_file: str
    format_output_path: str
    use_good_data: bool
    good_data_format: str
    good_data_list: list

    def __init__(self, input_file, output_base_path):
        self.input_file = input_file
        self.format_output_path = os.path.join(output_base_path, "1.format")
        self.node_list_output_path = os.path.join(output_base_path, "nodes_set")

        # 创建路径
        check_and_make_path(self.format_output_path)

    def transform(self, trans_type=None, use_good_data=False):
        print("transforming FB...")

        self.use_good_data = use_good_data

        if trans_type is None:
            trans_type = ['day', 'month', 'year']

        if 'day' in trans_type:
            self.handle_by_day()

        if 'month' in trans_type:
            self.handle_by_month()

        if 'year' in trans_type:
            self.handle_by_year()

        print("transforming FB complete\n")

    def handle_by_day(self):
        # 按天处理
        dataframe = pd.read_csv(self.input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])
        dataframe['timestamp'] = pd.to_datetime(dataframe['timestamp'], unit='s').dt.strftime('%Y-%m-%d')
        candidate = ['1970-01-01', '2006-09-05', '2006-09-06', '2006-09-07', '2006-09-08',
                     '2006-09-09', '2006-09-10', '2006-09-11', '2006-09-12', '2006-09-13',
                     '2006-09-14', '2006-09-15', '2006-09-16', '2006-09-17', '2006-09-18',
                     '2006-09-19', '2006-09-20', '2006-09-21', '2006-09-22', '2006-09-23',
                     '2006-09-24', '2006-09-25', '2006-09-26', '2006-09-27', '2006-09-28',
                     '2006-09-29', '2006-09-30', '2006-10-01', '2006-10-02', '2006-10-03',
                     '2006-10-04', '2006-10-05', '2006-10-06', '2006-10-07', '2006-10-08',
                     '2006-10-09', '2006-10-10', '2006-10-11', '2006-10-12', '2006-10-13',
                     '2006-10-14', '2006-10-15', '2006-10-16', '2006-10-17', '2006-10-18',
                     '2006-10-19', '2006-10-20', '2006-10-21', '2006-10-22', '2006-10-23',
                     '2006-10-24', '2006-10-25', '2006-10-26', '2006-10-27', '2006-10-28',
                     '2006-10-29', '2006-10-30', '2006-10-31', '2006-11-01', '2006-11-02',
                     '2006-11-03', '2006-11-04', '2006-11-05', '2006-11-06', '2006-11-07',
                     '2006-11-08', '2006-11-09', '2006-11-10', '2006-11-11', '2006-11-12',
                     '2006-11-13', '2006-11-14', '2006-11-15', '2006-11-16', '2006-11-17',
                     '2006-11-18', '2006-11-19', '2006-11-20', '2006-11-21', '2006-11-22',
                     '2006-11-23', '2006-11-24', '2006-11-25', '2006-11-26', '2006-11-27',
                     '2006-11-28', '2006-11-29', '2006-11-30', '2006-12-01', '2006-12-02',
                     '2006-12-03', '2006-12-04', '2006-12-05', '2006-12-06', '2006-12-07',
                     '2006-12-08', '2006-12-09', '2006-12-10', '2006-12-11', '2006-12-12',
                     '2006-12-13', '2006-12-14', '2006-12-15', '2006-12-16', '2006-12-17',
                     '2006-12-18', '2006-12-19', '2006-12-20', '2006-12-21', '2006-12-22',
                     '2006-12-23', '2006-12-24', '2006-12-25', '2006-12-26', '2006-12-27',
                     '2006-12-28', '2006-12-29', '2006-12-30', '2006-12-31', '2007-01-01',
                     '2007-01-02', '2007-01-03', '2007-01-04', '2007-01-05', '2007-01-06',
                     '2007-01-07', '2007-01-08', '2007-01-09', '2007-01-10', '2007-01-11',
                     '2007-01-12', '2007-01-13', '2007-01-14', '2007-01-15', '2007-01-16',
                     '2007-01-17', '2007-01-18', '2007-01-19', '2007-01-20', '2007-01-21',
                     '2007-01-22', '2007-01-23', '2007-01-24', '2007-01-25', '2007-01-26',
                     '2007-01-27', '2007-01-28', '2007-01-29', '2007-01-30', '2007-01-31',
                     '2007-02-01', '2007-02-02', '2007-02-03', '2007-02-04', '2007-02-05',
                     '2007-02-06', '2007-02-07', '2007-02-08', '2007-02-09', '2007-02-10',
                     '2007-02-11', '2007-02-12', '2007-02-13', '2007-02-14', '2007-02-15',
                     '2007-02-16', '2007-02-17', '2007-02-18', '2007-02-19', '2007-02-20',
                     '2007-02-21', '2007-02-22', '2007-02-23', '2007-02-24', '2007-02-25',
                     '2007-02-26', '2007-02-27', '2007-02-28', '2007-03-01', '2007-03-02',
                     '2007-03-03', '2007-03-04', '2007-03-05', '2007-03-06', '2007-03-07',
                     '2007-03-08', '2007-03-09', '2007-03-10', '2007-03-11', '2007-03-12',
                     '2007-03-13', '2007-03-14', '2007-03-15', '2007-03-16', '2007-03-17',
                     '2007-03-18', '2007-03-19', '2007-03-20', '2007-03-21', '2007-03-22',
                     '2007-03-23', '2007-03-24', '2007-03-25', '2007-03-26', '2007-03-27',
                     '2007-03-28', '2007-03-29', '2007-03-30', '2007-03-31', '2007-04-01',
                     '2007-04-02', '2007-04-03', '2007-04-04', '2007-04-05', '2007-04-06',
                     '2007-04-07', '2007-04-08', '2007-04-09', '2007-04-10', '2007-04-11',
                     '2007-04-12', '2007-04-13', '2007-04-14', '2007-04-15', '2007-04-16',
                     '2007-04-17', '2007-04-18', '2007-04-19', '2007-04-20', '2007-04-21',
                     '2007-04-22', '2007-04-23', '2007-04-24', '2007-04-25', '2007-04-26',
                     '2007-04-27', '2007-04-28', '2007-04-29', '2007-04-30', '2007-05-01',
                     '2007-05-02', '2007-05-03', '2007-05-04', '2007-05-05', '2007-05-06',
                     '2007-05-07', '2007-05-08', '2007-05-09', '2007-05-10', '2007-05-11',
                     '2007-05-12', '2007-05-13', '2007-05-14', '2007-05-15', '2007-05-16',
                     '2007-05-17', '2007-05-18', '2007-05-19', '2007-05-20', '2007-05-21',
                     '2007-05-22', '2007-05-23', '2007-05-24', '2007-05-25', '2007-05-26',
                     '2007-05-27', '2007-05-28', '2007-05-29', '2007-05-30', '2007-05-31',
                     '2007-06-01', '2007-06-02', '2007-06-03', '2007-06-04', '2007-06-05',
                     '2007-06-06', '2007-06-07', '2007-06-08', '2007-06-09', '2007-06-10',
                     '2007-06-11', '2007-06-12', '2007-06-13', '2007-06-14', '2007-06-15',
                     '2007-06-16', '2007-06-17', '2007-06-18', '2007-06-19', '2007-06-20',
                     '2007-06-21', '2007-06-22', '2007-06-23', '2007-06-24', '2007-06-25',
                     '2007-06-26', '2007-06-27', '2007-06-28', '2007-06-29', '2007-06-30',
                     '2007-07-01', '2007-07-02', '2007-07-03', '2007-07-04', '2007-07-05',
                     '2007-07-06', '2007-07-07', '2007-07-08', '2007-07-09', '2007-07-10',
                     '2007-07-11', '2007-07-12', '2007-07-13', '2007-07-14', '2007-07-15',
                     '2007-07-16', '2007-07-17', '2007-07-18', '2007-07-19', '2007-07-20',
                     '2007-07-21', '2007-07-22', '2007-07-23', '2007-07-24', '2007-07-25',
                     '2007-07-26', '2007-07-27', '2007-07-28', '2007-07-29', '2007-07-30',
                     '2007-07-31', '2007-08-01', '2007-08-02', '2007-08-03', '2007-08-04',
                     '2007-08-05', '2007-08-06', '2007-08-07', '2007-08-08', '2007-08-09',
                     '2007-08-10', '2007-08-11', '2007-08-12', '2007-08-13', '2007-08-14',
                     '2007-08-15', '2007-08-16', '2007-08-17', '2007-08-18', '2007-08-19',
                     '2007-08-20', '2007-08-21', '2007-08-22', '2007-08-23', '2007-08-24',
                     '2007-08-25', '2007-08-26', '2007-08-27', '2007-08-28', '2007-08-29',
                     '2007-08-30', '2007-08-31', '2007-09-01', '2007-09-02', '2007-09-03',
                     '2007-09-04', '2007-09-05', '2007-09-06', '2007-09-07', '2007-09-08',
                     '2007-09-09', '2007-09-10', '2007-09-11', '2007-09-12', '2007-09-13',
                     '2007-09-14', '2007-09-15', '2007-09-16', '2007-09-17', '2007-09-18',
                     '2007-09-19', '2007-09-20', '2007-09-21', '2007-09-22', '2007-09-23',
                     '2007-09-24', '2007-09-25', '2007-09-26', '2007-09-27', '2007-09-28',
                     '2007-09-29', '2007-09-30', '2007-10-01', '2007-10-02', '2007-10-03',
                     '2007-10-04', '2007-10-05', '2007-10-06', '2007-10-07', '2007-10-08',
                     '2007-10-09', '2007-10-10', '2007-10-11', '2007-10-12', '2007-10-13',
                     '2007-10-14', '2007-10-15', '2007-10-16', '2007-10-17', '2007-10-18',
                     '2007-10-19', '2007-10-20', '2007-10-21', '2007-10-22', '2007-10-23',
                     '2007-10-24', '2007-10-25', '2007-10-26', '2007-10-27', '2007-10-28',
                     '2007-10-29', '2007-10-30', '2007-10-31', '2007-11-01', '2007-11-02',
                     '2007-11-03', '2007-11-04', '2007-11-05', '2007-11-06', '2007-11-07',
                     '2007-11-08', '2007-11-09', '2007-11-10', '2007-11-11', '2007-11-12',
                     '2007-11-13', '2007-11-14', '2007-11-15', '2007-11-16', '2007-11-17',
                     '2007-11-18', '2007-11-19', '2007-11-20', '2007-11-21', '2007-11-22',
                     '2007-11-23', '2007-11-24', '2007-11-25', '2007-11-26', '2007-11-27',
                     '2007-11-28', '2007-11-29', '2007-11-30', '2007-12-01', '2007-12-02',
                     '2007-12-03', '2007-12-04', '2007-12-05', '2007-12-06', '2007-12-07',
                     '2007-12-08', '2007-12-09', '2007-12-10', '2007-12-11', '2007-12-12',
                     '2007-12-13', '2007-12-14', '2007-12-15', '2007-12-16', '2007-12-17',
                     '2007-12-18', '2007-12-19', '2007-12-20', '2007-12-21', '2007-12-22',
                     '2007-12-23', '2007-12-24', '2007-12-25', '2007-12-26', '2007-12-27',
                     '2007-12-28', '2007-12-29', '2007-12-30', '2007-12-31', '2008-01-01',
                     '2008-01-02', '2008-01-03', '2008-01-04', '2008-01-05', '2008-01-06',
                     '2008-01-07', '2008-01-08', '2008-01-09', '2008-01-10', '2008-01-11',
                     '2008-01-12', '2008-01-13', '2008-01-14', '2008-01-15', '2008-01-16',
                     '2008-01-17', '2008-01-18', '2008-01-19', '2008-01-20', '2008-01-21',
                     '2008-01-22', '2008-01-23', '2008-01-24', '2008-01-25', '2008-01-26',
                     '2008-01-27', '2008-01-28', '2008-01-29', '2008-01-30', '2008-01-31',
                     '2008-02-01', '2008-02-02', '2008-02-03', '2008-02-04', '2008-02-05',
                     '2008-02-06', '2008-02-07', '2008-02-08', '2008-02-09', '2008-02-10',
                     '2008-02-11', '2008-02-12', '2008-02-13', '2008-02-14', '2008-02-15',
                     '2008-02-16', '2008-02-17', '2008-02-18', '2008-02-19', '2008-02-20',
                     '2008-02-21', '2008-02-22', '2008-02-23', '2008-02-24', '2008-02-25',
                     '2008-02-26', '2008-02-27', '2008-02-28', '2008-02-29', '2008-03-01',
                     '2008-03-02', '2008-03-03', '2008-03-04', '2008-03-05', '2008-03-06',
                     '2008-03-07', '2008-03-08', '2008-03-09', '2008-03-10', '2008-03-11',
                     '2008-03-12', '2008-03-13', '2008-03-14', '2008-03-15', '2008-03-16',
                     '2008-03-17', '2008-03-18', '2008-03-19', '2008-03-20', '2008-03-21',
                     '2008-03-22', '2008-03-23', '2008-03-24', '2008-03-25', '2008-03-26',
                     '2008-03-27', '2008-03-28', '2008-03-29', '2008-03-30', '2008-03-31',
                     '2008-04-01', '2008-04-02', '2008-04-03', '2008-04-04', '2008-04-05',
                     '2008-04-06', '2008-04-07', '2008-04-08', '2008-04-09', '2008-04-10',
                     '2008-04-11', '2008-04-12', '2008-04-13', '2008-04-14', '2008-04-15',
                     '2008-04-16', '2008-04-17', '2008-04-18', '2008-04-19', '2008-04-20',
                     '2008-04-21', '2008-04-22', '2008-04-23', '2008-04-24', '2008-04-25',
                     '2008-04-26', '2008-04-27', '2008-04-28', '2008-04-29', '2008-04-30',
                     '2008-05-01', '2008-05-02', '2008-05-03', '2008-05-04', '2008-05-05',
                     '2008-05-06', '2008-05-07', '2008-05-08', '2008-05-09', '2008-05-10',
                     '2008-05-11', '2008-05-12', '2008-05-13', '2008-05-14', '2008-05-15',
                     '2008-05-16', '2008-05-17', '2008-05-18', '2008-05-19', '2008-05-20',
                     '2008-05-21', '2008-05-22', '2008-05-23', '2008-05-24', '2008-05-25',
                     '2008-05-26', '2008-05-27', '2008-05-28', '2008-05-29', '2008-05-30',
                     '2008-05-31', '2008-06-01', '2008-06-02', '2008-06-03', '2008-06-04',
                     '2008-06-05', '2008-06-06', '2008-06-07', '2008-06-08', '2008-06-09',
                     '2008-06-10', '2008-06-11', '2008-06-12', '2008-06-13', '2008-06-14',
                     '2008-06-15', '2008-06-16', '2008-06-17', '2008-06-18', '2008-06-19',
                     '2008-06-20', '2008-06-21', '2008-06-22', '2008-06-23', '2008-06-24',
                     '2008-06-25', '2008-06-26', '2008-06-27', '2008-06-28', '2008-06-29',
                     '2008-06-30', '2008-07-01', '2008-07-02', '2008-07-03', '2008-07-04',
                     '2008-07-05', '2008-07-06', '2008-07-07', '2008-07-08', '2008-07-09',
                     '2008-07-10', '2008-07-11', '2008-07-12', '2008-07-13', '2008-07-14',
                     '2008-07-15', '2008-07-16', '2008-07-17', '2008-07-18', '2008-07-19',
                     '2008-07-20', '2008-07-21', '2008-07-22', '2008-07-23', '2008-07-24',
                     '2008-07-25', '2008-07-26', '2008-07-27', '2008-07-28', '2008-07-29',
                     '2008-07-30', '2008-07-31', '2008-08-01', '2008-08-02', '2008-08-03',
                     '2008-08-04', '2008-08-05', '2008-08-06', '2008-08-07', '2008-08-08',
                     '2008-08-09', '2008-08-10', '2008-08-11', '2008-08-12', '2008-08-13',
                     '2008-08-14', '2008-08-15', '2008-08-16', '2008-08-17', '2008-08-18',
                     '2008-08-19', '2008-08-20', '2008-08-21', '2008-08-22', '2008-08-23',
                     '2008-08-24', '2008-08-25', '2008-08-26', '2008-08-27', '2008-08-28',
                     '2008-08-29', '2008-08-30', '2008-08-31', '2008-09-01', '2008-09-02',
                     '2008-09-03', '2008-09-04', '2008-09-05', '2008-09-06', '2008-09-07',
                     '2008-09-08', '2008-09-09', '2008-09-10', '2008-09-11', '2008-09-12',
                     '2008-09-13', '2008-09-14', '2008-09-15', '2008-09-16', '2008-09-17',
                     '2008-09-18', '2008-09-19', '2008-09-20', '2008-09-21', '2008-09-22',
                     '2008-09-23', '2008-09-24', '2008-09-25', '2008-09-26', '2008-09-27',
                     '2008-09-28', '2008-09-29', '2008-09-30', '2008-10-01', '2008-10-02',
                     '2008-10-03', '2008-10-04', '2008-10-05', '2008-10-06', '2008-10-07',
                     '2008-10-08', '2008-10-09', '2008-10-10', '2008-10-11', '2008-10-12',
                     '2008-10-13', '2008-10-14', '2008-10-15', '2008-10-16', '2008-10-17',
                     '2008-10-18', '2008-10-19', '2008-10-20', '2008-10-21', '2008-10-22',
                     '2008-10-23', '2008-10-24', '2008-10-25', '2008-10-26', '2008-10-27',
                     '2008-10-28', '2008-10-29', '2008-10-30', '2008-10-31', '2008-11-01',
                     '2008-11-02', '2008-11-03', '2008-11-04', '2008-11-05', '2008-11-06',
                     '2008-11-07', '2008-11-08', '2008-11-09', '2008-11-10', '2008-11-11',
                     '2008-11-12', '2008-11-13', '2008-11-14', '2008-11-15', '2008-11-16',
                     '2008-11-17', '2008-11-18', '2008-11-19', '2008-11-20', '2008-11-21',
                     '2008-11-22', '2008-11-23', '2008-11-24', '2008-11-25', '2008-11-26',
                     '2008-11-27', '2008-11-28', '2008-11-29', '2008-11-30', '2008-12-01',
                     '2008-12-02', '2008-12-03', '2008-12-04', '2008-12-05', '2008-12-06',
                     '2008-12-07', '2008-12-08', '2008-12-09', '2008-12-10', '2008-12-11',
                     '2008-12-12', '2008-12-13', '2008-12-14', '2008-12-15', '2008-12-16',
                     '2008-12-17', '2008-12-18', '2008-12-19', '2008-12-20', '2008-12-21',
                     '2008-12-22', '2008-12-23', '2008-12-24', '2008-12-25', '2008-12-26',
                     '2008-12-27', '2008-12-28', '2008-12-29', '2008-12-30', '2008-12-31',
                     '2009-01-01', '2009-01-02', '2009-01-03', '2009-01-04', '2009-01-05',
                     '2009-01-06', '2009-01-07', '2009-01-08', '2009-01-09', '2009-01-10',
                     '2009-01-11', '2009-01-12', '2009-01-13', '2009-01-14', '2009-01-15',
                     '2009-01-16', '2009-01-17', '2009-01-19', '2009-01-20', '2009-01-21']
        good_data = ['2008-01-01', '2008-01-02', '2008-01-03', '2008-01-04', '2008-01-05', '2008-01-06', '2008-01-07']
        # 处理时间戳为0的(就变成了这里的1970-01-01)
        if self.use_good_data:
            self.good_data_format = '%Y-%m-%d'
            self.good_data_list = good_data
        for day in (good_data if self.use_good_data else candidate):
            tem = dataframe[['from_id', 'to_id']][dataframe['timestamp'] == day]
            tem['from_id'] = tem['from_id'].map(lambda x: "U" + str(x))  # user
            tem['to_id'] = tem['to_id'].map(lambda x: "U" + str(x))  # user
            # 统计权重
            tem = tem.groupby(['from_id', 'to_id']).size().reset_index().rename(columns={0: 'weight'})
            tem.to_csv(os.path.join(self.format_output_path, str(day) + ".csv"), sep='\t', header=1, index=0)

    def handle_by_month(self):
        # 按月处理
        dataframe = pd.read_csv(self.input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])
        dataframe['timestamp'] = pd.to_datetime(dataframe['timestamp'], unit='s').dt.strftime('%Y-%m')
        candidate = ['1970-01', '2006-09', '2006-10', '2006-11', '2006-12', '2007-01', '2007-02', '2007-03', '2007-04',
                     '2007-05', '2007-06', '2007-07', '2007-08', '2007-09', '2007-10', '2007-11', '2007-12', '2008-01',
                     '2008-02', '2008-03', '2008-04', '2008-05', '2008-06', '2008-07', '2008-08', '2008-09', '2008-10',
                     '2008-11', '2008-12', '2009-01']
        good_data = ['2006-10', '2006-11', '2006-12', '2007-01', '2007-02', '2007-03', '2007-04', '2007-05', '2007-06',
                     '2007-07', '2007-08', '2007-09', '2007-10', '2007-11', '2007-12', '2008-01', '2008-02', '2008-03',
                     '2008-04', '2008-05', '2008-06', '2008-07', '2008-08', '2008-09', '2008-10', '2008-11', '2008-12']
        # good_data = ['2006-09', '2006-10', '2006-11', '2006-12', '2007-01', '2007-02', '2007-03', '2007-04', '2007-05',
        #              '2007-06', '2007-07', '2007-08', '2007-09', '2007-10', '2007-11', '2007-12', '2008-01', '2008-02',
        #              '2008-03', '2008-04', ]
        # good_data = ['2007-01', '2007-02', '2007-03', '2007-04', '2007-05', '2007-06', '2007-07']
        # 处理时间戳为0的(就变成了这里的1970-01-01)
        if self.use_good_data:
            self.good_data_format = '%Y-%m'
            self.good_data_list = good_data
        for month in (good_data if self.use_good_data else candidate):
            tem = dataframe[['from_id', 'to_id']][dataframe['timestamp'] == month]
            tem['from_id'] = tem['from_id'].map(lambda x: "U" + str(x))  # user
            tem['to_id'] = tem['to_id'].map(lambda x: "U" + str(x))  # user
            # 统计权重
            tem = tem.groupby(['from_id', 'to_id']).size().reset_index().rename(columns={0: 'weight'})
            tem.to_csv(os.path.join(self.format_output_path, str(month) + ".csv"), sep='\t', header=1, index=0)

    def handle_by_year(self):
        # 按年处理
        dataframe = pd.read_csv(self.input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])
        dataframe['timestamp'] = pd.to_datetime(dataframe['timestamp'], unit='s').dt.strftime('%Y')
        candidate = ['1970', '2006', '2007', '2008', '2009']
        # 处理时间戳为0的(就变成了这里的1970-01-01)
        for year in candidate:
            tem = dataframe[['from_id', 'to_id']][dataframe['timestamp'] == year]
            tem['from_id'] = tem['from_id'].map(lambda x: "U" + str(x))  # user
            tem['to_id'] = tem['to_id'].map(lambda x: "U" + str(x))  # user
            # 统计权重
            tem = tem.groupby(['from_id', 'to_id']).size().reset_index().rename(columns={0: 'weight'})
            tem.to_csv(os.path.join(self.format_output_path, str(year) + ".csv"), sep='\t', header=1, index=0)

    def test_granularity(self, time_format='%Y-%m-%d %H:%M:%s'):
        dataframe = pd.read_csv(self.input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])
        print("top 10 rows:\n")
        print(dataframe[0:10])
        print("shape:")
        print(dataframe.shape)
        dataframe['timestamp'] = pd.to_datetime(dataframe['timestamp'], unit='s').dt.strftime(time_format)
        print(np.sort(dataframe['timestamp'].unique()))

    def get_full_node_set(self):
        print("get full node set")

        nodes_set_path = self.node_list_output_path
        check_and_make_path(nodes_set_path)

        dataframe = pd.read_csv(self.input_file, sep=" ", names=['from_id', 'to_id', 'type', 'timestamp'])

        if self.use_good_data:
            dataframe['timestamp'] = pd.to_datetime(dataframe['timestamp'], unit='s').dt.strftime(self.good_data_format)
            dataframe = dataframe[np.isin(dataframe['timestamp'], self.good_data_list, invert=False)]

        # nodes_set = np.sort(pd.concat([dataframe['from_id'], dataframe['to_id']], axis=0, ignore_index=True).unique())
        nodes_set = pd.concat([dataframe['from_id'], dataframe['to_id']], axis=0, ignore_index=True).unique()
        pd_data = pd.DataFrame(nodes_set, columns=['node'])
        pd_data['node'] = pd_data['node'].map(lambda x: "U" + str(x))
        pd_data = pd_data.sort_values(by='node')

        pd_data.to_csv(os.path.join(nodes_set_path, 'nodes.csv'), index=0, header=0)

        print("got it\n")

def transform(trans_type=None, use_good_data=False):
    if trans_type is None:
        trans_type = ['day']
    fb = FBTransformer(input_file="/data/facebook/0.input/fb-wosn-friends.edges", output_base_path="/data/facebook/")
    fb.transform(trans_type, use_good_data=use_good_data)
    fb.get_full_node_set()

if __name__ == '__main__':
    transform(trans_type=['month'], use_good_data=True)

max_stars_repo_name: beyondxz/CTGCN

max_issues_repo_name: dherath/CTGCN

max_forks_repo_name: dherath/CTGCN


text:
have_PIL = True
try:
    import PIL.Image
    import pydicom
except ImportError:
    have_PIL = False

have_numpy = True
try:
    import numpy as np
except ImportError:
    have_numpy = False

from PyQt5.QtGui import QPixmap,QImage
from PIL.ImageQt import ImageQt

def get_LUT_value(data, window, level):
    """Apply the RGB Look-Up Table for the given data and window/level value."""
    if not have_numpy:
        raise ImportError("Numpy is not available. See http://numpy.scipy.org/"
                            " to download and install")

    return np.piecewise(data,
                        [data <= (level - 0.5 - (window - 1) / 2),
                        data > (level - 0.5 + (window - 1) / 2)],
                        [0, 255, lambda data: ((data - (level - 0.5)) \
                        / (window - 1) + 0.5) * (255 - 0)])

def get_PIL_image(dataset):
    """Get Image object from Python Imaging Library(PIL)"""
    if not have_PIL:
        raise ImportError("Python Imaging Library is not available. "
                        "See http://www.pythonware.com/products/pil/ "
                        "to download and install")

    if ('PixelData' not in dataset):
        raise TypeError("Cannot show image -- DICOM dataset does not have "
                        "pixel data")
        # can only apply LUT if these window info exists
    if ('WindowWidth' not in dataset) or ('WindowCenter' not in dataset):
        bits = dataset.BitsAllocated
        samples = dataset.SamplesPerPixel
        if bits == 8 and samples == 1:
            mode = "L"
        elif bits == 8 and samples == 3:
            mode = "RGB"
        elif bits == 16:
            # not sure about this -- PIL source says is 'experimental'
            # and no documentation. Also, should bytes swap depending
            # on endian of file and system??
            mode = "I;16"
        else:
            raise TypeError("Don't know PIL mode for %d BitsAllocated "
                                "and %d SamplesPerPixel" % (bits, samples))

            # PIL size = (width, height)
        size = (dataset.Columns, dataset.Rows)

            # Recommended to specify all details
            # by http://www.pythonware.com/library/pil/handbook/image.htm
        im = PIL.Image.frombuffer(mode, size, dataset.PixelData,
                                    "raw", mode, 0, 1)

    else:
        image = get_LUT_value(dataset.pixel_array, dataset.WindowWidth,
                                dataset.WindowCenter)
            # Convert mode to L since LUT has only 256 values:
            #   http://www.pythonware.com/library/pil/handbook/image.htm
        im = PIL.Image.fromarray(image).convert('L')

    return im

    # Bubble Sort algorithm
def sortSliceLocation(datasets):
    n = len(datasets)
    for i in range(n):
        for j in range(0,n-i-1):
            loc1 = float(pydicom.dcmread(datasets[j]).SliceLocation)
            loc2 = float(pydicom.dcmread(datasets[j+1]).SliceLocation)
            if loc1 > loc2:
                datasets[j],datasets[j+1] = datasets[j+1],datasets[j] # Swap if the element found is greater

    return datasets

max_stars_repo_name: sivaroot/ProjectCPE

max_issues_repo_name: sivaroot/ProjectCPE

max_forks_repo_name: sivaroot/ProjectCPE


text:
import numpy as np;

from .reward import Reward;

class DeltaReward(Reward):
    def _reset(self):
        self.prev_mean_fitness = 0;

    def __call__(self,population,  fitness):
        mean_fitness = np.mean(fitness);
        reward = mean_fitness - self.prev_mean_fitness;
        self.prev_mean_fitness = mean_fitness;
        return reward;

max_stars_repo_name: realtwister/LearnedEvolution

max_issues_repo_name: realtwister/LearnedEvolution

max_forks_repo_name: realtwister/LearnedEvolution


text:
from ctypes import *
import os
import cv2
import time
import darknet_jphack
import argparse
import math
import numpy as np
import datetime
import json
import subprocess

#学習されたクラス名を取得
def classname_management(altNames,data_path):
    if altNames is None:
        try:
            with open(data_path) as metaFH:
                metaContents = metaFH.read()
                import re
                match = re.search("names *= *(.*)$", metaContents, re.IGNORECASE | re.MULTILINE)
                if match:
                    result = match.group(1)
                else:
                    result = None
                try:
                    if os.path.exists(result):
                        with open(result) as namesFH:
                            namesList = namesFH.read().strip().split("\n")
                            altNames = [x.strip() for x in namesList]
                except TypeError:
                    pass
        except Exception:
            pass
    return altNames

#クラスごとの枠を色を決める
def color_management(num_classes,class_colors,altNames):
    class_colors_own = []
    for i in range(0, num_classes):
        hue = 255*i/num_classes
        col = np.zeros((1,1,3)).astype("uint8")
        col[0][0][0] = hue
        col[0][0][1] = 128
        col[0][0][2] = 255
        cvcol = cv2.cvtColor(col, cv2.COLOR_HSV2BGR)
        col = (int(cvcol[0][0][0]), int(cvcol[0][0][1]), int(cvcol[0][0][2]))
        class_colors_own.append(col)
        class_colors[altNames[i]] = class_colors_own[i]

#YOLOによる物体検出
def image_detection(frame, network, class_names, class_colors, thresh,save_result_path,width,height):
    #物体検出のための画像の前処理
    darknet_image = darknet_jphack.make_image(width, height, 3)
    image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    image_resized = cv2.resize(image_rgb, (width, height),
                               interpolation=cv2.INTER_LINEAR)

    darknet_jphack.copy_image_from_bytes(darknet_image, image_resized.tobytes())
    #YOLOによる物体検出
    detections = darknet_jphack.detect_image(network, class_names, darknet_image, thresh=thresh)
    #検出結果枠作成
    image = darknet_jphack.draw_boxes(detections, image_resized, class_colors)
    result_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    #検出結果画像保存
    pred_image = cv2.imwrite(save_result_path,result_image)
    darknet_jphack.print_detections(detections)
    return result_image, detections

#カメラからの映像を取得して物体検出した結果をリアルタイムで表示
def start_process(cap,save_img_dir,save_result_dir,width,height,network,class_names,class_colors,threshold):
    start_time = time.time()
    while True:
        #state.jsonを読み込みモードでオープン
        with open('state.json', 'r') as fr:
            state_json_file = json.load(fr)

        #メインコンピュータから起動を求められている場合
        if state_json_file["state"] == "1":
            person_count = 0
            #カメラからの映像取得
            ret, frame = cap.read()
            #画像のパス生成
            dt_now = datetime.datetime.now()
            date = dt_now.strftime('%Y%m%d_%H%M%S%f')
            save_result_path = str(save_result_dir + date + ".jpg")
            save_img_path = str(save_img_dir + date + ".jpg")
            #カメラからの映像を画像として保存
            cv2.imwrite(save_img_path,frame)

            #YOLOによる物体検出
            image, detections = image_detection(frame, network, class_names, class_colors, threshold,save_result_path,width,height)

            #何も検出しなかったとき
            if len(detections) == 0:
                monitor = str(0)

            #何か検出したとき
            else:
                row = np.shape(detections)[0]
                for r in range(row):
                    detection_result = detections[r][0]
                    #人を検出した場合
                    if detection_result == "person":
                        person_count += 1
                    #人以外を検出した場合
                    else:
                        pass
            if person_count >= 1:
                monitor = str(1)
            else:
                monitor = str(0)

            #経過時間
            elapsed_time = time.time() - start_time

            #前回の更新から1秒経過した場合
            if elapsed_time > 1:
                jphack_dict = {"person": monitor}
                #jphack.jsonを書き込みモードでオープン
                with open('jphack.json', 'w') as fw:
                    #ベッドに人がいるかどうかの結果を書き込む
                    json.dump(jphack_dict, fw, ensure_ascii=False)

                #時間をリセット
                start_time = time.time()

        #メインコンピュータから停止を求められている場合
        elif state_json_file["state"] == "0":
            print("finish YOLO")
            break

    #パソコンのパスワード
    passward = b"MIKHA\n"
    #Jetson Nano 再起動
    subprocess.run(("sudo","-S","reboot"),input = passward , check = True)

#メインコンピュータからの命令によってAI Camera起動
def ready_yolo():
    #カメラ起動
    cap = cv2.VideoCapture(0)
    fps = cap.get(cv2.CAP_PROP_FPS)

    #YOLOの物体検出に必要なファイルのパス
    weights_path = "yolov4-tiny.weights"
    cfg_path = "cfg/yolov4-tiny.cfg"
    data_path = "cfg/coco.data"
    threshold = 0.25

    #CNNやデータのロード
    network, class_names, class_colors = darknet_jphack.load_network(
        cfg_path,
        data_path,
        weights_path,
        batch_size=1
    )

    #検出結果の画像サイズ
    width = darknet_jphack.network_width(network)
    height = darknet_jphack.network_height(network)

    #データセットの情報取得
    global metaMain, netMain, altNames
    altNames = None
    altNames = classname_management(altNames,data_path)
    num_classes = len(altNames)
    #クラスごとの枠の色決定
    color_management(num_classes,class_colors,altNames)

    #画像を保存するディレクトリ
    jphack_dir = "jphack"
    ori_img_dir = "jphack/video"
    result_img_dir = "jphack/result"
    #ディレクトリがない場合にディレクトリ作成
    if not os.path.isdir(img_dir):
        os.makedirs(img_dir)
    if not os.path.isdir(ori_img_dir):
        os.makedirs(ori_img_dir)
    if not os.path.isdir(result_img_dir):
        os.makedirs(result_img_dir)

    #撮影日時のフォルダ作成
    dt_now = datetime.datetime.now()
    date = dt_now.strftime('%Y%m%d_%H%M%S')
    save_img_dir = ori_img_dir + "/" + date
    save_result_dir = result_img_dir + "/" + date
    os.makedirs(save_img_dir, exist_ok=True)
    os.makedirs(save_result_dir, exist_ok=True)
    #画像のディレクトリ作成
    save_img_dir = save_img_dir + "/"
    save_result_dir = save_result_dir + "/"

    print("--------------------------------------------------------------------------------")

    #カメラからの映像に対してリアルタイムで検出し、1秒おきに検出結果を書き込む
    start_process(cap,save_img_dir,save_result_dir,width,height,network,class_names,class_colors,threshold)


text:
"""
 Copyright (c) 2018-2019 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import unittest

import numpy as np

from extensions.middle.FusePermutesSequence import FusePermutesSequence
from extensions.middle.NormalizeFullyConnected import NormalizeFullyConnected
from mo.middle.passes.eliminate_test import build_graph
from mo.middle.passes.fusing.fuse_linear_ops_test import compare_graphs

# The dictionary with nodes attributes used to build various graphs. A key is the name of the node and the value is the
# dictionary with node attributes.
nodes_attributes = {
    'placeholder_1': {'name': 'placeholder_1', 'value': None, 'shape': None, 'type': 'Placeholder', 'kind': 'op',
                      'op': 'Placeholder'},
    'placeholder_1_data': {'name': 'placeholder_1_data', 'value': None, 'shape': None, 'kind': 'data',
                           'data_type': None},
    'reshape_1': {'type': 'Reshape', 'value': None, 'kind': 'op', 'op': 'Reshape'},
    'reshape_1_data': {'value': None, 'shape': None, 'kind': 'data'},

    'fc': {'type': 'FullyConnected', 'value': None, 'kind': 'op', 'op': 'MatMul'},
    'fc_data': {'value': None, 'shape': None, 'kind': 'data'},
    'fc_weights': {'value': None, 'shape': None, 'kind': 'data'},

    'reshape_2': {'type': 'Reshape', 'value': None, 'kind': 'op', 'op': 'Reshape'},
    'reshape_2_data': {'value': None, 'shape': None, 'kind': 'data'},
}

class NormalizeFullyConnectedTest(unittest.TestCase):
    def test_1(self):
        graph = build_graph(nodes_attributes,
                            [('placeholder_1', 'placeholder_1_data'),
                             ('placeholder_1_data', 'fc'),
                             ('fc_weights', 'fc'),
                             ('fc', 'fc_data'),
                             ],
                            {'placeholder_1_data': {'shape': np.array([1, 16, 512])},
                             'fc': {'out-size': 101},
                             'fc_weights': {'shape': np.array([512,101]), 'value': np.ones([512, 101]), 'input_channel_dim': 1},
                             'fc_data': {'shape': np.array([1, 16, 101])},
                             }, nodes_with_edges_only=True)

        graph_ref = build_graph(nodes_attributes,
                            [('placeholder_1', 'placeholder_1_data'),
                             ('placeholder_1_data', 'reshape_1'),
                             ('reshape_1', 'reshape_1_data'),
                             ('reshape_1_data', 'fc'),
                             ('fc_weights', 'fc'),
                             ('fc', 'fc_data'),
                             ('fc_data', 'reshape_2'),
                             ('reshape_2', 'reshape_2_data'),
                             ],
                            {'placeholder_1_data': {'shape': np.array([1, 16, 512])},
                             'reshape_1_data': {'shape': np.array([16, 512])},
                             'reshape_2_data': {'shape': np.array([1, 16, 101])},
                             'fc_weights': {'shape': np.array([512,101]), 'value': np.ones([512, 101])},
                             'fc': {'out-size': 101},
                             'fc_data': {'shape': np.array([16, 101])},
                             }, nodes_with_edges_only=True)

        pattern = NormalizeFullyConnected()
        pattern.find_and_replace_pattern(graph)

        (flag, resp) = compare_graphs(graph, graph_ref, 'placeholder_1_data', 'placeholder_1_data', check_op_attrs=True)
        self.assertTrue(flag, resp)

    def test_2(self):
        graph = build_graph(nodes_attributes,
                            [('placeholder_1', 'placeholder_1_data'),
                             ('placeholder_1_data', 'fc'),
                             ('fc_weights', 'fc'),
                             ('fc', 'fc_data'),
                             ],
                            {'placeholder_1_data': {'shape': np.array([2, 32, 16, 512])},
                             'fc': {'out-size': 101},
                             'fc_weights': {'shape': np.array([512,101]), 'value': np.ones([512, 101]), 'input_channel_dim': 1},
                             'fc_data': {'shape': np.array([2, 32, 16, 101])},
                             }, nodes_with_edges_only=True)

        graph_ref = build_graph(nodes_attributes,
                            [('placeholder_1', 'placeholder_1_data'),
                             ('placeholder_1_data', 'reshape_1'),
                             ('reshape_1', 'reshape_1_data'),
                             ('reshape_1_data', 'fc'),
                             ('fc_weights', 'fc'),
                             ('fc', 'fc_data'),
                             ('fc_data', 'reshape_2'),
                             ('reshape_2', 'reshape_2_data'),
                             ],
                            {'placeholder_1_data': {'shape': np.array([2, 32, 16, 512])},
                             'reshape_1_data': {'shape': np.array([2 * 32 * 16, 512])},
                             'reshape_2_data': {'shape': np.array([2, 32, 16, 101])},
                             'fc_weights': {'shape': np.array([512,101]), 'value': np.ones([512, 101])},
                             'fc': {'out-size': 101},
                             'fc_data': {'shape': np.array([2 * 32 * 16, 101])},
                             }, nodes_with_edges_only=True)

        pattern = NormalizeFullyConnected()
        pattern.find_and_replace_pattern(graph)

        (flag, resp) = compare_graphs(graph, graph_ref, 'placeholder_1_data', 'placeholder_1_data', check_op_attrs=True)
        self.assertTrue(flag, resp)

max_stars_repo_name: shinh/dldt


text:
import numpy as np
import torch.nn as nn
import torch
import torch.nn.functional as F

class DecoderRNN(nn.Module):
    def __init__(self, embedding, embedding_size, hidden_size, rnn_layers, vocab_size):
        super(DecoderRNN, self).__init__()
        self.embedding = embedding
        self.embedding_size = embedding_size
        self.hidden_size = hidden_size
        self.rnn_layers = rnn_layers
        self.vocab_size = vocab_size
        self.gru = nn.GRU(self.embedding_size, self.hidden_size, self.rnn_layers, batch_first=True)

    def forward(self, x, hidden):
        decoder_output, decoder_hidden = self.gru(x, hidden)
        return decoder_output, decoder_hidden

max_stars_repo_name: MoxinC/DST-SC

max_issues_repo_name: MoxinC/DST-SC

max_forks_repo_name: MoxinC/DST-SC


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Dec  2 09:29:13 2019

@author: SimonKern
"""
import numpy as np
from tqdm import tqdm
from plotting import specgram_multitaper
import ospath
from SleepData import SleepRecord
import matplotlib.pyplot as plt

# alle histogramme plotten um zu sehen wie die noise mässig verteilt sind
datafolder = '../../'
files = ospath.list_files(datafolder, subfolders=True, exts='edf')
for file in tqdm(files):
    sr = SleepRecord(file, channel='ECG I')
    data = sr.raw
    mesh = specgram_multitaper(data, sr.sfreq, show_plot=False)
    plt.hist(mesh.ravel(), bins=np.arange(0,1,0.01), alpha=0.1)
    plt.show()
    plt.pause(0.1)

plt.savefig('distribution_all.png')
# plot has been saved at distribution_all.png
#%%
plt.close('all')
datafolder = '../../NT1'
files = ospath.list_files(datafolder, subfolders=True, exts='edf')
for file in tqdm(files):
    sr = SleepRecord(file)
    data = sr.raw
    mesh = specgram_multitaper(data, sr.sfreq, show_plot=False)
    plt.hist(mesh.ravel(), bins=np.arange(0,1,0.01), alpha=0.1)
    plt.show()
    plt.pause(0.1)

plt.savefig('distribution_NT1.png')

#%%
plt.close('all')
datafolder = '../../control'
files = ospath.list_files(datafolder, subfolders=True, exts='edf')
for file in tqdm(files):
    sr = SleepRecord(file)
    data = sr.raw
    mesh = specgram_multitaper(data, sr.sfreq, show_plot=False)
    plt.hist(mesh.ravel(), bins=np.arange(0,1,0.01), alpha=0.1)
    plt.show()
    plt.pause(0.1)

plt.savefig('distribution_control.png')


text:
#!/usr/bin/python
"""
GridFlag

Use XArray, Dask, and Numpy to load CASA Measurement Set (MS) data and
create binned UV data.

Todo:
[ ] Add options for choosing stokes parameters or amplitude/complex components
"""

import numpy as np

import dask
import dask.array as da
import numba as nb

from . import groupby_apply, groupby_partition, annulus_stats

import logging

# Add colours for warnings and errors
logging.addLevelName(logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING))
logging.addLevelName(logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)-20s %(levelname)-8s %(message)s',
    handlers=[
        logging.FileHandler("gridflag.log"),
        logging.StreamHandler()
    ]
)

logger = logging.getLogger()

def process_stokes_options(ds_ind, stokes='I', use_existing_flags=True):
    """
    Determine columns to use for flagging algorithm from DATA and FLAG tables.

    Inputs:
    ds_ind              The input dask dataframe
    stokes              Which Stokes to flag on, str
    use_existing_flags  Take into account existing flags in the MS, bool

    Returns:
    vals                Visibilities, dask dataframe
    flags               Flags, dask dataframe
    """

    # Determine which polarization state to grid and flag
    if stokes=='I':
        vals = ds_ind.DATA[:,0].data + ds_ind.DATA[:,-1].data
    elif stokes=='Q':
        vals = ds_ind.DATA[:,0].data - ds_ind.DATA[:,-1].data
    elif stokes=='U':
        vals = ds_ind.DATA[:,1].data + ds_ind.DATA[:,2].data
    elif stokes=='V':
        vals = ds_ind.DATA[:,1].data - ds_ind.DATA[:,2].data
    else:
        raise ValueError(f"compute_ipflag_grid: the stokes argument, '{stokes}', \
            is not currently implemented, please select another value.")

    # Take only the real part for Gaussian stats
    vals = vals.real

    if use_existing_flags:
        if stokes=='I':
            flags = ds_ind.FLAG.data[:,0] | ds_ind.FLAG.data[:,-1]
        elif stokes=='Q':
            flags = ds_ind.FLAG.data[:,0] | ds_ind.FLAG.data[:,-1]
        elif stokes=='U':
            flags = ds_ind.FLAG.data[:,1] | ds_ind.FLAG.data[:,2]
        elif stokes=='V':
            flags = ds_ind.FLAG.data[:,1] | ds_ind.FLAG.data[:,2]
    else:
        flags = None

    return vals, flags

def compute_ipflag_grid(ds_ind, uvbins, sigma=3.0, partition_level=4, stokes='I', use_existing_flags=True, client=None):
    """
    Map functions to a concurrent dask functions to an Xarray dataset with pre-
    computed grid indicies.

    Parameters
    ----------
    ds_ind : xarray.dataset
        An xarray datset imported from a measurement set. It must contain
        coordinates U_bins and V_bins, and relevant data and position variables.
    uvbins : list
        List of two arrays, each containing the bounds for the discrete U and V
        bins in the ds_ind dataset.
    sigma : float
        Depth of flagging - corresponds to the significance of a particular
        observation to be labeled RFI.
    partition_level : int
        Internal parameter used to split the dataset in to congruent chunks in
        the uv-bin space. Used for concurrency.
    stokes : string
        Reduce the data via one or more combinations of columns - 'I', 'Q', 'U',
        'V' for corresponding stokes parameters. For amplitude of each columns
        use 'A'.
    use_existing_flags : bool
        Either remove rows that are flagged in the input MS file before flagging
        (True) or ignore and overwrite existing flags (False).
    client : dask.distributed.client
        Pass dask distributed client or 'None' for local computation.

    Returns
    -------
    flag_list: numpy.array
        A list of flag indicies to be applied to the original measurement set.
    median_grid_unflagged : numpy.array
        A two-dimensional array representing a uv-grid. Each cell in the grid is
        the median of all values, after zero removal and flagging, in that bin.
    median_grid : numpy.array
        The same as the above but after flagged visibilities are removed.
    """

    # Load data from dask-ms dastaset
    ubins = ds_ind.U_bins.data
    vbins = ds_ind.V_bins.data

    flags, vals = process_stokes_options(ds_ind, stokes, True)

    # The array 'p' is used to map flags back to the original file order
    p = da.arange(len(ds_ind.newrow), dtype=np.int64, chunks=ubins.chunks)

    chunks = list(ubins.chunks[0])

    logger.debug(f"Original chunk sizes: ", chunks)

    # Execute partition function which does a partial sort on U and V bins
    split_points, ubins_part, vbins_part, vals_part, flags_part, p_part = dask_partition_sort(
        ubins, vbins, vals, flags, p, chunks, partition_level, 0, client=client)

    print(f"Dataset split into {len(ubins_part.chunks[0])} uv-plane partitions.")
    print(f"Flagging: {ubins_part.chunks[0]} rows.")

    # Convert back to delayed one final time
    dd_bins = da.stack([ubins_part, vbins_part, p_part]).T
    dd_bins = dd_bins.rechunk((ubins_part.chunks[0], 3))

    dd_bins = dd_bins.to_delayed()
    dd_vals = vals_part.to_delayed()

    dd_flags = flags_part.to_delayed()

    vdim = vbins_part.ndim

    if use_existing_flags:
        if vdim > 1:
            group_bins_sort = [dask.delayed(groupby_partition.sort_bins_multi)
            (
                part[0][0],
                part[1][0],
                part[2]
            ) for part in zip(dd_bins, dd_vals, dd_flags)]
        else:
            group_bins_sort = [dask.delayed(groupby_partition.sort_bins)
            (
                part[0][0],
                part[1],
                part[2]
            ) for part in zip(dd_bins, dd_vals, dd_flags)]

    else:
        if vdim > 1:
            group_bins_sort = [dask.delayed(groupby_partition.sort_bins_multi)
            (
                part[0][0],
                part[1][0]
            ) for part in zip(dd_bins, dd_vals)]
        else:
            group_bins_sort = [dask.delayed(groupby_partition.sort_bins)
            (
                part[0][0],
                part[1]
            ) for part in zip(dd_bins, dd_vals)]

    group_chunks = [dask.delayed(groupby_partition.create_bin_groups_sort)
                            (c[0], c[1], c[2]) for c in group_bins_sort]
    function_chunks = [dask.delayed(groupby_partition.apply_grid_median)
                            (c[1], c[2]) for c in group_chunks]
    median_grid = dask.delayed(groupby_partition.combine_function_partitions)(
                                function_chunks)

    if client:
        median_grid_unflagged = client.compute(median_grid)
    else:
        median_grid_unflagged = median_grid.compute()

    # Autoamatically compute annulus widths (naive)
    annulus_width = dask.delayed(annulus_stats.compute_annulus_bins)(
        median_grid_unflagged,
        uvbins,
        10
    )

    print(f"Initiate flagging with sigma = {sigma}.")

    annuli_data = dask.delayed(annulus_stats.process_annuli)(
        median_grid_unflagged,
        annulus_width,
        uvbins[0],
        uvbins[1],
        sigma=sigma
    )

    # Process only one column for multi-dim data until we finish this function
    if vdim > 1:
        flag_results = [dask.delayed(annulus_stats.flag_one_annulus)(
            c[0],
            c[1][:,0],
            c[2],
            c[3],
            annuli_data[0],
            annuli_data[1],
            sigma=sigma) for c in group_chunks]
    else:
        flag_results = [dask.delayed(annulus_stats.flag_one_annulus)(
            c[0],
            c[1],
            c[2],
            c[3],
            annuli_data[0],
            annuli_data[1],
            sigma=sigma) for c in group_chunks]

    results = dask.delayed(groupby_partition.combine_annulus_results)(
        [fr[0] for fr in flag_results],
        [fr[1] for fr in flag_results],
        [fr[2] for fr in flag_results],
        [fr[3] for fr in flag_results])

    print("Compute median grid on the partitions.")

    if client:
        ms_flag_list, da_flag_list, median_grid_flagged, count_grid = \
            client.compute(results).result()
        median_grid_unflagged = median_grid_unflagged.result()
    else:
        ms_flag_list, da_flag_list, median_grid_flagged, count_grid = \
            results.compute()

    return ms_flag_list, median_grid_unflagged, median_grid_flagged

@nb.njit(nogil=True)
def strip_zero_values(a, b, v, p):

    null_flags = p[np.where(v==0)]
    a = a[np.where(v!=0)]
    newlen, oldlen = len(a), len(b)
    print("Removed ", oldlen - newlen, " zero value rows of ", oldlen,
        " rows (", round(100*(oldlen-newlen)/oldlen, 2), "%)." )
    b = b[np.where(v!=0)]
    v = v[np.where(v!=0)]
    p = p[np.where(v!=0)]

    return newlen, a, b, v, p, null_flags

@nb.njit(nogil=True)
def median_of_medians(data, sublist_length=11):
    # Compute median of medians on each chunk
    data = np.array([np.median(k) for k in [data[j:(j + sublist_length)] for j in range(0,len(data),sublist_length)]])
    return data

def apply_median_func(data, depth):
    for i in np.arange(depth):
        data = median_of_medians(data, 11)
    return data

def da_median(a):
    m = np.median(a)
#     m = da.mean(a)
    return np.array(m)[None,None] # add dummy dimensions

def combine_median(meds):
    umed_list = np.concatenate(meds)
#     print(f"umeds:", len(umed_list))
    return np.median(umed_list)

combine_median = dask.delayed(combine_median)

partition_permutation = dask.delayed(groupby_partition.partition_permutation)
partition_permutation_multi = dask.delayed(groupby_partition.partition_permutation_multi)

da_median = dask.delayed(da_median)
apply_median_func = dask.delayed(apply_median_func)

# Latest Version
def dask_partition_sort(a, b, v, f, p, chunks, binary_chunks, partition_level, client=None):

    split_points = np.array([])

    # Persist to compute DAG up to this point on workers. This improves the performance of
    # in-place sorting six-fold.
    if client:
        a = client.persist(a)
        b = client.persist(b)
        v = client.persist(v)
        f = client.persist(f)
        p = client.persist(p)
    else:
        a = a.persist()
        b = b.persist()
        v = v.persist()
        f = f.persist()
        p = p.persist()

    a_min, a_max = da.min(a), da.max(a)

    v = da.squeeze(v)

    if v.ndim > 1:
        ncols = v.shape[1]
    else:
        ncols = 1

    # print("Columns: ", ncols, v.ndim, v.shape, v.dtype, nb.typeof(v[:10].compute()))

    a = a.to_delayed()
    b = b.to_delayed()
    v = v.to_delayed()
    f = f.to_delayed()
    p = p.to_delayed()

    # Compute the median-of-medians heuristic (not the proper definition of MoM but we only need an approximate pivot)
    #    umed = [da_median(a_)[0] for a_ in a]
    resid = 50
    sublist_length=11
    min_nrow = np.min(chunks)

    med_depth = np.int32(np.floor((np.log(min_nrow/resid)/np.log(sublist_length))))

    umed = [apply_median_func(a_, med_depth) for a_ in a]

#     umeds = [len(u.compute()) for u in umed]
    if client:
        pivot = combine_median(umed)
        pivot = client.compute(pivot).result()
    else:
        pivot = combine_median(umed).compute()

    if pivot == a_max:
        pivot-=0.5
    if pivot == a_min:
        pivot+=0.5

    if ncols > 1:
        results = [partition_permutation_multi(a_, b_, v_[0], f_, p_, pivot) for a_, b_, v_, f_, p_ in zip(a, b, v, f, p)]
    else:
        results = [partition_permutation(a_, b_, v_, f_, p_, pivot) for a_, b_, v_, f_, p_ in zip(a, b, v, f, p)]

    print(f"Partition Level {partition_level}, med_depth: {med_depth}, pivot: {pivot}")

    if client:
        results = client.persist(results)
        sp0 = [r[0].compute() for r in results]
        a = [r[1] for r in results]
        b = [r[2] for r in results]
        v = [r[3] for r in results]
        f = [r[4] for r in results]
        p = [r[5] for r in results]

    else:
        results = dask.persist(results)
        sp0 = [r[0].compute() for r in results[0]]
        a = [r[1] for r in results[0]]
        b = [r[2] for r in results[0]]
        v = [r[3] for r in results[0]]
        f = [r[4] for r in results[0]]
        p = [r[5] for r in results[0]]

    partition_level+=1

    if partition_level < binary_chunks:

        # Split each chunk of data in to two partitions as computed above then recurse on each partition.
        a1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_),), dtype=np.int32)    for x_, s_, c_ in zip(a, sp0, chunks)])
        b1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_),), dtype=np.int32)    for x_, s_, c_ in zip(b, sp0, chunks)])
        f1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_),), dtype=np.int64)    for x_, s_, c_ in zip(f, sp0, chunks)])
        p1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_),), dtype=np.int64)    for x_, s_, c_ in zip(p, sp0, chunks)])

        a2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_),), dtype=np.int32) for x_, s_, c_ in zip(a, sp0, chunks)])
        b2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_),), dtype=np.int32) for x_, s_, c_ in zip(b, sp0, chunks)])
        f2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_),), dtype=np.int64) for x_, s_, c_ in zip(f, sp0, chunks)])
        p2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_),), dtype=np.int64) for x_, s_, c_ in zip(p, sp0, chunks)])

        # Requires a different format for array shape for 1d and 2d arrays
        if ncols > 1:
            v1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_,ncols)), dtype=np.float32)  for x_, s_, c_ in zip(v, sp0, chunks)])
            v2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_,ncols)), dtype=np.float32) for x_, s_, c_ in zip(v, sp0, chunks)])
        else:
            v1 = da.concatenate([da.from_delayed(x_[0:s_], ((s_),), dtype=np.float32)  for x_, s_, c_ in zip(v, sp0, chunks)])
            v2 = da.concatenate([da.from_delayed(x_[s_:c_], ((c_-s_),), dtype=np.float32) for x_, s_, c_ in zip(v, sp0, chunks)])

        # Compute chunk size here as delayed objects don't have metadata
        chunks1 = list(a1.chunks[0])
        chunks2 = list(a2.chunks[0])

        # Do recursion step on each partition
        sp1, b1_, a1_, v1_, f1_, p1_ = dask_partition_sort(b1, a1, v1, f1, p1, chunks1, binary_chunks, partition_level, client=client)
        sp2, b2_, a2_, v2_, f2_, p2_ = dask_partition_sort(b2, a2, v2, f2, p2, chunks2, binary_chunks, partition_level, client=client)

        # Combine the partially sorted partitions into the original array shape
        a = da.concatenate([a1_, a2_])
        b = da.concatenate([b1_, b2_])
        v = da.concatenate([v1_, v2_])
        f = da.concatenate([f1_, f2_])
        p = da.concatenate([p1_, p2_])

        split_points = np.concatenate((sp1, sp2))

    else:
        # Break out of the recursion and combine the partitions into a partially sorted array.
        a1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),), dtype=np.int32) for x_, sp_, ch_ in zip(a, sp0, chunks)])
        a1 = a1.rechunk(len(a1))
        a2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),), dtype=np.int32) for x_, sp_, ch_ in zip(a, sp0, chunks)])
        a2 = a2.rechunk(len(a2))

        if (len(a1) == 0) or (len(a2) == 0):
            raise Exception(f"Partition level {binary_chunks} resulted in one or more zero-length partitions. Please choose a smaller value for 'partition_level' or increase the number of UV bins.")

        a = da.concatenate([a1, a2])

        b1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),), dtype=np.int32) for x_, sp_, ch_ in zip(b, sp0, chunks)])
        b1 = b1.rechunk(len(b1))
        b2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),), dtype=np.int32) for x_, sp_, ch_ in zip(b, sp0, chunks)])
        b2 = b2.rechunk(len(b2))

        b = da.concatenate([b1, b2])

        f1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),), dtype=np.int64) for x_, sp_, ch_ in zip(f, sp0, chunks)])
        f1 = f1.rechunk(len(f1))
        f2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),), dtype=np.int64) for x_, sp_, ch_ in zip(f, sp0, chunks)])
        f2 = f2.rechunk(len(f2))

        f = da.concatenate([f1, f2])

        p1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),), dtype=np.int64) for x_, sp_, ch_ in zip(p, sp0, chunks)])
        p1 = p1.rechunk(len(p1))
        p2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),), dtype=np.int64) for x_, sp_, ch_ in zip(p, sp0, chunks)])
        p2 = p2.rechunk(len(p2))

        p = da.concatenate([p1, p2])

        if ncols > 1:
            v1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),ncols), dtype=np.float32) for x_, sp_, ch_ in zip(v, sp0, chunks)])
            v1 = v1.rechunk(len(v1))
            v2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),ncols), dtype=np.float32) for x_, sp_, ch_ in zip(v, sp0, chunks)])
            v2 = v2.rechunk(len(v2))
        else:
            v1 = da.concatenate([da.from_delayed(x_[:sp_], ((sp_),), dtype=np.float32) for x_, sp_, ch_ in zip(v, sp0, chunks)])
            v1 = v1.rechunk(len(v1))
            v2 = da.concatenate([da.from_delayed(x_[sp_:ch_], ((ch_ - sp_),), dtype=np.float32) for x_, sp_, ch_ in zip(v, sp0, chunks)])
            v2 = v2.rechunk(len(v2))

        v = da.concatenate([v1, v2])

        # The split points are indicies for the array position of the partitions, use later to define dask chunk with unique (u,v) bin ranges
        # for concurrent statistical calculations.
        split_points = np.concatenate((a1.chunks, a2.chunks), axis=None)

    return split_points, a, b, v, f, p

def compute_median_grid(
    ds_ind,
    uvbins,
    partition_level=4,
    stokes='I',
    client=None
):

    print("Load in-memory data for sort.")
    ubins = ds_ind.U_bins.data
    vbins = ds_ind.V_bins.data

    vals, flags = process_stokes_options(ds_ind, stokes, True)

    #Comute chunks
    chunks = list(ubins.chunks[0])

    # The array 'p' is used to map flags back to the original file order
    p = da.arange(len(ds_ind.newrow), dtype=np.int64, chunks=ubins.chunks)

    print("Compute parallel partitions and do partial sort.")
    split_points,
    ubins_part,
    vbins_part,
    vals_part,
    flags_part,
    p_part = dask_partition_sort(ubins,
                                 vbins,
                                 vals,
                                 flags,
                                 p,
                                 chunks,
                                 partition_level,
                                 0,
                                 client=client
                                 )

    print("Preparing dask delayed...")
    dd_bins = da.stack([ubins_part, vbins_part, p_part]).T
    dd_bins = dd_bins.rechunk((ubins_part.chunks[0], 3))

    dd_bins = dd_bins.to_delayed()
    dd_vals = vals_part.to_delayed()

    dd_flags = flags_part.to_delayed()

    vdim = vbins_part.ndim

    del ubins, vbins, vals, p

    print("Compute UV map and median grid.")

    # group_bins_sort = [dask.delayed(groupby_partition.sort_bins)(part[0][0], part[1]) for part in zip(dd_bins, dd_vals)]
    if vdim > 1:
        group_bins_sort = [dask.delayed(groupby_partition.sort_bins_multi)(
                            part[0][0],
                            part[1][0],
                            part[2]) for part in zip(dd_bins, dd_vals, dd_flags)]
    else:
        group_bins_sort = [dask.delayed(groupby_partition.sort_bins)(
                            part[0][0],
                            part[1],
                            part[2]
                            ) for part in zip(dd_bins, dd_vals, dd_flags)]

    group_chunks = [dask.delayed(groupby_partition.create_bin_groups_sort)(c[0], c[1], c[2]) for c in group_bins_sort]
    function_chunks = [dask.delayed(groupby_partition.apply_grid_median)(c[1], c[2]) for c in group_chunks]
    median_grid = dask.delayed(groupby_partition.combine_function_partitions)(function_chunks)

    if client:
        median_grid = client.compute(median_grid).result()
    else:
        median_grid = median_grid.compute()

    return median_grid

def check_exising_flags(ds_ind, stokes='I', client=None):
    """ Check the existing flags in the input Measurement Set."""

    # Determine which polarization state to grid and flag
    if stokes=='I':
        flags = ds_ind.FLAG.data[:,0] | ds_ind.FLAG.data[:,-1]
    elif stokes=='Q':
        flags = ds_ind.FLAG.data[:,0] | ds_ind.FLAG.data[:,-1]
    elif stokes=='U':
        flags = ds_ind.FLAG.data[:,1] | ds_ind.FLAG.data[:,2]
    elif stokes=='V':
        flags = ds_ind.FLAG.data[:,1] | ds_ind.FLAG.data[:,2]
    elif stokes=='A':
        flags = da.sum(ds_ind.FLAG.data, axis=1, dtype=np.bool)
    else:
        raise ValueError(f"check_existing_flags: the stokes argument, \
        '{stokes}', is not currently implemented, please select another value.")

    flag_loc = da.where(flags==True)
    if not client is None:
        flag_loc = client.compute(flag_loc)
    else:
        flag_loc = flag_loc[0].compute()

    nflags = len(flag_loc)
    nrows = len(flags)

    print( f"Rows alrady flagged: {(100*nflags/nrows):.1f}% ({nflags}/{nrows}),\
 in file \"{ds_ind.attrs['Measurement Set']}\"." )

def map_amplitude_grid(
    ds_ind,
    data_columns,
    stokes='I',
    chunk_size:int=10**6,
    return_index:bool=False
):
    """
    Map functions to a concurrent dask functions to an Xarray dataset with
    pre-computed grid indicies.

    Parameters
    ----------
    ds_ind : xarray.dataset
        An xarray datset imported from a measurement set. It must contain
        coordinates U_bins and V_bins, and relevant data and position
        variables.
    data_columns : list
        Components for Stokes terms to be used to compute amplitude. Depends
        on dataset.
    chunk_size : int
        The chunk size for computing split-apply functions in dask, Default is
        '10**6'.
    return_index : bool
        Determines the return data type. If true, it returns a 2d grid of lists
        of values and indicies for each bin.

    Returns
    -------
    value_groups: numpy.array
        A two-dimensional array representing a uv-grid. Each cell in the grid
        contains a list of values from the dataset.
    uvbins: xarray.Dataset
        A two-dimensional array representing a uv-grid. Each cell in the grid
        contains a list of indicies to be used to map subsequent computations
        back to the original dataset.
    """

    # Get dask arrays of UV-bins and visibilities from XArray dataset
    dd_ubins = ds_ind.U_bins.data
    dd_vbins = ds_ind.V_bins.data
    dd_flgs = (ds_ind.FLAG[:,data_columns[0]].data |
               ds_ind.FLAG[:,data_columns[1]].data)

    if stokes=='I':
        dd_vals = (np.absolute(ds_ind.DATA[:,data_columns[0]].data +
                               ds_ind.DATA[:,data_columns[1]].data))
    elif stokes=='Q':
        dd_vals = (np.absolute(ds_ind.DATA[:,data_columns[0]].data -
                               ds_ind.DATA[:,data_columns[1]].data))

    # Combine U and V bins into one dask array
    dd_bins = da.stack([dd_ubins, dd_vbins]).T

    # Apply unifrom chunks to both dask arrays
    dd_bins = dd_bins.rechunk([chunk_size, 2])
    dd_vals = dd_vals.rechunk([chunk_size, 1])
    dd_flgs = dd_flgs.rechunk([chunk_size, 1])

    # Convert to delayed data structures
    bin_partitions = dd_bins.to_delayed()
    val_partitions = dd_vals.to_delayed()
    flg_partitions = dd_flgs.to_delayed()

    # Compute indicies for each bin in the grid for each chunk
    group_chunks = [dask.delayed(groupby_apply.group_bin_flagval_wrap)(
                            part[0][0],
                            part[1],
                            part[2],
                            init_index=(chunk_size*kth)
                            ) for kth, part in enumerate(zip(bin_partitions,
                                                             val_partitions,
                                                             flg_partitions)
                                                        )
                    ]
    groups = dask.delayed(groupby_apply.combine_group_flagval)(group_chunks)

#     group_chunks = [dask.delayed(groupby_apply.group_bin_idx_val_wrap)(part[0][0], part[1]) for part in zip(bin_partitions, val_partitions)]
#     groups = dask.delayed(groupby_apply.combine_group_idx_val)(group_chunks)

    if return_index:
        # Compute the grid from above without doing the apply step
        groups = groups.compute()
        index_groups, value_groups, flag_groups = groups[0], groups[1], groups[2]
        return index_groups, value_groups, flag_groups

    else:
        # Apply the function to the grid without explicitly computing the indicies
        median_grid = dask.delayed(groupby_apply.apply_to_groups)(value_groups_, np.median)
        std_grid = dask.delayed(groupby_apply.apply_to_groups)(value_groups_, np.std)
#         median_grid = median_grid_.compute()
#         std_grid = std_grid_.compute()
        return median_grid, std_grid

def bin_rms(a):
    mu = np.median(a)
    rms = np.sqrt(np.sum((a-mu)**2))
    return rms

def bin_rms_grid(ds_ind, flag_list):
    """ Create a two dimensional UV-grid with the RMS value for each bin, after
    removing flags in a provided list of flags.

    Parameters
    ----------
    ds_ind : xarray.dataset
        Dataset for input MS file.
    flag_list : array of ints
        Indicies for flagged rows in the dataset.

    Returns
    -------
    rms_grid : array of shape (ubins, vbins)
        A two-dimensional array wiht the RMS value for the values in each  UV bin.

    """

    flags = np.zeros((len(ds_ind.newrow)), dtype=bool)
    flags[flag_list] = True

    # Use Calculated Flags Column
    ubins = ds_ind.U_bins.data
    vbins = ds_ind.V_bins.data
    vals = (da.absolute(ds_ind.DATA[:,0].data + ds_ind.DATA[:,-1].data))

    print("Processing RMS Grid with ", np.sum(1*flags), len(flag_list), "flags.")

    p = da.arange(len(ds_ind.newrow), dtype=np.int64, chunks=ubins.chunks)

    dd_bins = da.stack([ubins, vbins, p]).T

    dd_bins = dd_bins.rechunk((ubins.chunks[0], 3))

    bins = dd_bins.compute()
    vals = vals.compute()

    bins_sort, vals_sort, null_flags = groupby_partition.sort_bins(bins, vals, flags)

    print(len(vals_sort))

    uv_ind, values, grid_row_map, null_flags = groupby_partition.create_bin_groups_sort(bins_sort,
                                                                                vals_sort,
                                                                                null_flags)

    print(len(values))

    rms_grid = groupby_partition.apply_grid_function(values, grid_row_map, bin_rms)

    return rms_grid

max_stars_repo_name: idia-astro/ipflag

max_issues_repo_name: idia-astro/ipflag

max_forks_repo_name: idia-astro/ipflag

reason: import numpy,import numba


text:
import torch
import torch.nn as nn
from config import config
from utils.TTA_back import *
import os
import numpy as np
import cv2

def get_avg(outs):
    num = len(outs)
    out = np.zeros_like(outs[0])

    for i in outs:
        out+=i
    return out/num

def get_rev_out(imgs,model,output,h,w):

    # print('get inv img')
    img_inv = imgs['img_inv'][0].cuda()
    img_out_inv = model(img_inv)
    img_out_inv = nn.functional.interpolate(input=img_out_inv,size=(h,w),mode='bilinear',align_corners=True)
    img_out_inv = inv_back([img_out_inv])
    output.append(img_out_inv[0])
    return output

def get_rotate_out(imgs,model,output,h,w):

    # print('get rotation img')
    r_outs = []
    r_imgs = imgs['rotation']
    for img in r_imgs:
        r_out = model(img.cuda())
        r_out = nn.functional.interpolate(input=r_out,size=(384,384),mode='bilinear',align_corners=True)
        r_outs.append(r_out)

    r_outs = rotation_back(r_outs)#numpy
    r_output = get_avg(r_outs)
    r_output = np.transpose(r_output,(2,0,1))
    r_output = torch.from_numpy(r_output).unsqueeze(0).cuda()
    r_output = nn.functional.interpolate(input=r_output,size=(h,w),mode='bilinear',align_corners=True)
    output.append(r_output)

    # print('get rotation_inv img')
    r_outs_inv = []
    r_imgs_inv = imgs['rotation_inv']
    for img in r_imgs_inv:

        r_out_inv = model(img.cuda())
        r_out_inv = nn.functional.interpolate(input=r_out_inv,size=(384,384),mode='bilinear',align_corners=True)
        r_outs_inv.append(r_out_inv)

    r_outs_inv = inv_back(r_outs_inv)
    r_outs_inv = rotation_back(r_outs_inv)#numpy
    r_output_inv = get_avg(r_outs_inv)
    r_output_inv = np.transpose(r_output_inv,(2,0,1))
    r_output_inv = torch.from_numpy(r_output_inv).unsqueeze(0).cuda()
    r_output_inv = nn.functional.interpolate(input=r_output_inv,size=(h,w),mode='bilinear',align_corners=True)
    output.append(r_output_inv)
    return output

def get_shift_out(imgs,model,output,h,w):
    if config.TEST.SHIFT==True:
        s_outs = []
        s_imgs = imgs['shift']
        # print('get shift img')
        for i,img in enumerate(s_imgs):

            s_out = model(img.cuda())
            s_out = nn.functional.interpolate(input=s_out,size=(384,384),mode='bilinear',align_corners=True)
            s_outs.append(s_out)

        s_outs = shift_back(s_outs)
        s_output = get_avg(s_outs)
        s_output = np.transpose(s_output,(2,0,1))
        s_output = torch.from_numpy(s_output).unsqueeze(0).cuda()
        s_output = nn.functional.interpolate(input=s_output,size=(h,w),mode='bilinear',align_corners=True)
        output.append(s_output)

        # print('get shift_inv img')
        s_outs_inv = []
        s_imgs_inv = imgs['shift_inv']
        for i,img in enumerate(s_imgs_inv):

            s_out_inv = model(img.cuda())
            s_out_inv = nn.functional.interpolate(input=s_out_inv,size=(384,384),mode='bilinear',align_corners=True)
            s_outs_inv.append(s_out_inv)
        s_outs_inv = inv_back(s_outs_inv)
        s_outs_inv = shift_back(s_outs_inv)
        s_output_inv = get_avg(s_outs_inv)
        s_output_inv = np.transpose(s_output_inv,(2,0,1))
        s_output_inv = torch.from_numpy(s_output_inv).unsqueeze(0).cuda()
        s_output_inv = nn.functional.interpolate(input=s_output_inv,size=(h,w),mode='bilinear',align_corners=True)
        output.append(s_output_inv)
    return output

max_stars_repo_name: Anikily/CDinkNet

max_issues_repo_name: Anikily/CDinkNet

max_forks_repo_name: Anikily/CDinkNet


text:
import sys, os
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../../')

import numpy as np
import chainer
import chainer.functions as F
import chainer.links as L
from tqdm import tqdm
from utils import plot_result
from utils import NNfuncs

class Model(chainer.Chain):
    def __init__(self, n_in, n_units1, n_units2, n_out):
        super(Model, self).__init__(
            l1=L.Linear(n_in, n_units1),
            l2=L.Linear(n_units1, n_units2),
            l3=L.Linear(n_units2, n_out),
        )

    def __call__(self, x, t):
        y = self.l3(F.relu(self.l2(F.relu(self.l1(x)))))
        self.loss = self.jsd(t, y)
        #self.loss = self.listwise_cost(t, y)
        return self.loss

    def predict(self, x):
        h1 = F.relu(self.l1(x))
        h2 = F.relu(self.l2(h1))
        h = F.relu(self.l3(h2))
        return h.data

    def kld(self, vec_true, vec_compare):
        # Kullback–Leibler divergence
        nd = vec_true.data * vec_compare.data > 0
        nd_var = chainer.Variable(nd)
        include_nan = vec_true * F.log(vec_true / vec_compare)
        z = chainer.Variable(np.zeros((len(nd), 1), dtype=np.float32))
        return F.sum(F.where(nd_var, include_nan, z))

    def jsd(self, vec_true, vec_compare):
        # Jensen–Shannon divergence
        vec_mean = 0.5 * (vec_true + vec_compare)
        return 0.5 * self.kld(vec_true, vec_mean) + 0.5 * self.kld(vec_compare, vec_mean)

    def topkprob(self, vec, k=5):
        vec_sort = np.sort(vec)[-1::-1]
        topk = vec_sort[:k]
        ary = np.arange(k)
        return np.prod([np.exp(topk[i]) / np.sum(np.exp(topk[i:])) for i in ary])

    def listwise_cost(self, list_ans, list_pred):
        return - np.sum(self.topkprob(list_ans) * np.log(self.topkprob(list_pred)))

class ListNet(NNfuncs.NN):
    def __init__(self, resumemodelName=None):
        self.resumemodelName = resumemodelName
        self.train_loss, self.test_loss = [], []
        self.train_acc, self.test_acc = [], []
        if resumemodelName is not None:
            print("load resume model!")
            self.loadModel(resumemodelName)

    def ndcg(self, y_true, y_score, k=20):
        y_true = y_true.ravel()
        y_score = y_score.ravel()
        y_true_sorted = sorted(y_true, reverse=True)
        ideal_dcg = 0
        for i in range(k):
            ideal_dcg += (2 ** y_true_sorted[i] - 1.) / np.log2(i + 2)
        dcg = 0
        argsort_indices = np.argsort(y_score)[::-1]
        for i in range(k):
            dcg += (2 ** y_true[argsort_indices[i]] - 1.) / np.log2(i + 2)
        ndcg = dcg / ideal_dcg
        return ndcg

    def trainModel(self, x_train, y_train, x_test, y_test, n_epoch, batchsize):
        print("Start training and validation loop......")
        N = len(x_train)
        N_test = len(x_test)
        for epoch in range(1, n_epoch + 1):
            print('epoch', epoch)
            # training
            perm = np.random.permutation(N)
            sum_loss = 0
            for i in tqdm(range(0, N, batchsize)):
                x = chainer.Variable(np.asarray(x_train[perm[i:i + batchsize]]))
                t = chainer.Variable(np.asarray(y_train[perm[i:i + batchsize]]))
                self.optimizer.update(self.model, x, t)
                sum_loss += float(self.model.loss.data) * len(t.data)

            print('train mean loss={}'.format(sum_loss / N))
            self.train_loss.append(sum_loss / N)

            perm = np.random.permutation(N_test)
            sum_loss = 0
            for j in tqdm(range(0, N_test, batchsize)):
                x = chainer.Variable(np.asarray(x_test[perm[j:j + batchsize]]), volatile='off')
                t = chainer.Variable(np.asarray(y_test[perm[j:j + batchsize]]), volatile='off')
                loss = self.model(x, t)
                sum_loss += float(loss.data) * len(t.data)
            print('test  mean loss={}'.format(sum_loss / N_test))
            self.test_loss.append(sum_loss / N_test)

            train_score = self.model.predict(chainer.Variable(x_train))
            test_score = self.model.predict(chainer.Variable(x_test))
            train_ndcg = self.ndcg(y_train, train_score)
            test_ndcg = self.ndcg(y_test, test_score)
            self.train_acc.append(train_ndcg)
            self.test_acc.append(test_ndcg)
            print("epoch: {0}".format(epoch + 1))
            print("NDCG@20 | train: {0}, test: {1}".format(train_ndcg, test_ndcg))

    def fit(self, fit_X, fit_y, batchsize=20, n_epoch=20, n_units1=512, n_units2=128, tv_ratio=0.95, optimizerAlgorithm="Adam"):
        train_X, train_y, validate_X, validate_y = self.splitData(fit_X, fit_y, tv_ratio)
        print("The number of data, train:", len(train_X), "validate:", len(validate_X))
        if self.resumemodelName is None:
            self.initializeModel(Model, train_X, n_units1, n_units2, optimizerAlgorithm)
        self.trainModel(train_X, train_y, validate_X, validate_y, n_epoch, batchsize)

    def test(self, fit_X, fit_y, batchsize=10, n_epoch=1, tv_ratio=0.95, optimizerAlgorithm="Adam"):
        train_X, train_y, validate_X, validate_y = self.splitData(fit_X, fit_y, tv_ratio)
        print("The number of data, train:", len(train_X), "validate:", len(validate_X))
        self.trainModel(train_X, train_y, validate_X, validate_y, n_epoch, batchsize)


text:
import os
import numpy as np
import SimpleITK as sitk
import glob
from scipy.ndimage import interpolation
import h5py

def itensity_normalize_one_volume(volume):
    """
    normalize the itensity of an nd volume based on the mean and std of nonzeor region
    inputs:
        volume: the input nd volume
    outputs:
        out: the normalized nd volume
    """

    pixels = volume[volume > 0]
    mean = pixels.mean()
    std  = pixels.std()
    out = (volume - mean)/std
    out_random = np.random.normal(0, 1, size = volume.shape)
    out[volume == 0] = out_random[volume == 0]
    return out

class MedicalImageDeal(object):
    def __init__(self, img, percent=1):
        self.img = img
        self.percent = percent

    @property
    def valid_img(self):
        from skimage import exposure
        cdf = exposure.cumulative_distribution(self.img)
        watershed = cdf[1][cdf[0] >= self.percent][0]
        return np.clip(self.img, self.img.min(), watershed)

def mask2bbox(image, mask, pert=[0, 0, 0]):
    boundary = np.array(np.where(mask > 0)).transpose(1, 0)
    bbox_index = []
    for i in range(len(mask.shape)):
        index = (max(0, boundary[:, i].min() - pert[i]),
                 min(mask.shape[i], boundary[:, i].max() + pert[i]))
        bbox_index.append(index)
    print(image.shape, mask.shape)
    print(bbox_index)
    cropped_image = image[bbox_index[0][0]:bbox_index[0][1], bbox_index[1][0]:bbox_index[1][1],
                          bbox_index[2][0]:bbox_index[2][1]]
    cropped_mask = mask[bbox_index[0][0]:bbox_index[0][1], bbox_index[1][0]:bbox_index[1][1],
                        bbox_index[2][0]:bbox_index[2][1]]

    if cropped_image.shape == cropped_mask.shape:
        return cropped_image, cropped_mask
    else:
        print("Error")

mask_path = glob.glob(
    "../data/brats18/*/*_flair.nii.gz")
for case in mask_path:
    msk_itk = sitk.ReadImage(case)
    origin = msk_itk.GetOrigin()
    spacing = msk_itk.GetSpacing()
    direction = msk_itk.GetDirection()
    mask = sitk.GetArrayFromImage(msk_itk)

    img_path = case.replace("flair", "seg")
    if os.path.exists(img_path):
        img_itk = sitk.ReadImage(img_path)
        image = sitk.GetArrayFromImage(img_itk)
        cropped_mask, cropped_image = mask2bbox(image, mask)
        cropped_image = MedicalImageDeal(cropped_image, percent=0.999).valid_img
        cropped_mask[cropped_mask > 0] = 1
        print(cropped_image.shape)
        cropped_image = cropped_image.astype(np.float32)
        item = case.split("/")[-1].split(".")[0].replace("_flair", "")
        name = "../data/processed_brats18/" + item
        cropped_img_itk = sitk.GetImageFromArray(cropped_image)
        cropped_lab_itk = sitk.GetImageFromArray(cropped_mask)
        cropped_img_itk.SetOrigin(origin)
        cropped_img_itk.SetSpacing(spacing)
        cropped_img_itk.SetDirection(direction)
        cropped_lab_itk.SetOrigin(origin)
        cropped_lab_itk.SetSpacing(spacing)
        cropped_lab_itk.SetDirection(direction)
        sitk.WriteImage(cropped_img_itk, name + "_img.nii.gz")
        sitk.WriteImage(cropped_lab_itk, name + "_lab.nii.gz")

reason: import numpy,from scipy