text:
"""
Trains a model, saving checkpoints and tensorboard summaries along the way.
"""
from datetime import datetime
import json
import shutil
import os

import numpy as np
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

from eval_helper import EvalHelper
from resnet_model import ResNetModel, make_data_augmentation_fn

# load configuration: first load the base config, and then update using the
# job_parameters, if any
with open('config.json', 'r') as base_config_file:
    config = json.load(base_config_file)
if os.path.exists('job_parameters.json'):
    with open('job_parameters.json', 'r') as job_parameters_file:
        job_parameters = json.load(job_parameters_file)
    # make sure we didn't e.g. make some typo
    for k in job_parameters.keys():
        if k not in config.keys():
            print("{} config not in base config file!".format(k))
        # assert k in config.keys()
    config.update(job_parameters)

# Setting up training parameters
tf.set_random_seed(config['random_seed'])
np.random.seed(config['random_seed'])

max_num_training_steps = config['max_num_training_steps']
num_output_steps = config['num_output_steps']
num_summary_steps = config['num_summary_steps']
num_checkpoint_steps = config['num_checkpoint_steps']

batch_size = config['training_batch_size']

# Setting up the data and the model
clean_train_images = np.load(config["clean_dataset_dir"] + "/train_images.npy").astype(np.float32)
clean_train_labels = np.load(config["clean_dataset_dir"] + "/train_labels.npy").astype(np.int64)
num_train_examples = len(clean_train_images)

clean_test_images = np.load(config["clean_dataset_dir"] + "/test_images.npy").astype(np.float32)
clean_test_labels = np.load(config["clean_dataset_dir"] + "/test_labels.npy").astype(np.int64)
num_test_examples = len(clean_test_images)

# We assume inputs are as follows
#   - train_{images,labels}.npy -- the x% poisoned dataset
#   - test_{images,labels}.npy -- trigger applied to all test images
#   - poisoned_train_indices.npy -- which indices were poisoned
#   - train_no_trigger_{images,labels}.npy -- the x% poisoned dataset, but without any triggers applied
poisoned_train_images = np.load(config["already_poisoned_dataset_dir"] + "/train_images.npy").astype(np.float32)
poisoned_train_labels = np.load(config["already_poisoned_dataset_dir"] + "/train_labels.npy").astype(np.int64)
poisoned_test_images = np.load(config["already_poisoned_dataset_dir"] + "/test_images.npy").astype(np.float32)
poisoned_test_labels = np.load(config["already_poisoned_dataset_dir"] + "/test_labels.npy").astype(np.int64)

poisoned_train_indices = np.load(config["already_poisoned_dataset_dir"] + "/poisoned_train_indices.npy")
if len(poisoned_train_indices) > 0:
    poisoned_only_train_images = poisoned_train_images[poisoned_train_indices]
    poisoned_only_train_labels = poisoned_train_labels[poisoned_train_indices]
    poisoned_no_trigger_train_images = np.load(config["already_poisoned_dataset_dir"] + "/train_no_trigger_images.npy").astype(np.float32)
    # These are identical to the training labels
    poisoned_no_trigger_train_labels = np.load(config["already_poisoned_dataset_dir"] + "/train_labels.npy").astype(np.int64)
    poisoned_no_trigger_train_images = poisoned_no_trigger_train_images[poisoned_train_indices]
    poisoned_no_trigger_train_labels = poisoned_no_trigger_train_labels[poisoned_train_indices]

def prepare_dataset(images, labels):
    images_placeholder = tf.placeholder(tf.float32, images.shape)
    labels_placeholder = tf.placeholder(tf.int64, labels.shape)
    dataset = tf.data.Dataset.from_tensor_slices((images_placeholder, labels_placeholder))
    dataset = dataset.shuffle(buffer_size=10000, seed=config['random_seed']).repeat()

    if config['augment_dataset']:
        dataset = dataset.map(
            make_data_augmentation_fn(
                standardization=config['augment_standardization'],
                flip=config['augment_flip'],
                padding=config['augment_padding'],
                is_training=True))

    dataset = dataset.batch(batch_size)
    iterator = dataset.make_initializable_iterator()
    return (images_placeholder, labels_placeholder), dataset, iterator

clean_placeholder, clean_train_dataset_batched, clean_training_iterator = prepare_dataset(clean_train_images, clean_train_labels)
poisoned_placeholder, _, poisoned_training_iterator = prepare_dataset(poisoned_train_images, poisoned_train_labels)
if len(poisoned_train_indices) > 0:
    poisoned_only_placeholder, _, poisoned_only_training_iterator = prepare_dataset(poisoned_only_train_images, poisoned_only_train_labels)
    poisoned_no_trigger_placeholder, _, poisoned_no_trigger_training_iterator = prepare_dataset(poisoned_no_trigger_train_images, poisoned_no_trigger_train_labels)

iterator_handle = tf.placeholder(tf.string, shape=[])
input_iterator = tf.data.Iterator.from_string_handle(iterator_handle,
                                                             clean_train_dataset_batched.output_types,
                                                             clean_train_dataset_batched.output_shapes)
x_input, y_input = input_iterator.get_next()

global_step = tf.train.get_or_create_global_step()

# Choose model and set up optimizer
model = ResNetModel(x_input, y_input, random_seed=config['random_seed'])

weight_decay = 0.0002
boundaries = config['learning_rate_boundaries']
values = config['learning_rates']
learning_rate = tf.train.piecewise_constant(
    tf.cast(global_step, tf.int32),
    boundaries,
    values)
momentum = 0.9
total_loss = model.xent + weight_decay * model.weight_decay_loss
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
opt = tf.train.MomentumOptimizer(
    learning_rate,
    momentum,
)
with tf.control_dependencies(update_ops):
    train_step = opt.minimize(total_loss, global_step=global_step)

# Setting up the Tensorboard and checkpoint outputs
model_dir = config['model_dir']
if not os.path.exists(model_dir):
    os.makedirs(model_dir)

saver = tf.train.Saver(max_to_keep=3)
merged_summaries = tf.summary.merge([
    tf.summary.scalar('accuracy poison train', model.accuracy),
    tf.summary.scalar('xent poison train', model.xent / batch_size),
    tf.summary.image('images poison train', model.x_image),
    tf.summary.histogram('conf in y_input', model.confidence_in_correct),
    tf.summary.histogram('conf in y_pred', model.confidence_in_prediction),
])
clean_histogram = tf.summary.histogram('conf in clean', model.confidence_in_correct)
poison_only_merged_summaries = tf.summary.merge([
    tf.summary.scalar('accuracy poison only train', model.accuracy),
    tf.summary.scalar('xent poison only train', model.xent / batch_size), # NB shouldn't divide like this
    tf.summary.image('images poison only train', model.x_image),
    tf.summary.histogram('conf in poisoned only', model.confidence_in_correct),
])
poison_no_trigger_merged_summaries = tf.summary.merge([
    tf.summary.scalar('accuracy poison train (no trigger)', model.accuracy),
    tf.summary.scalar('xent poison train (no trigger)', model.xent / batch_size), # NB shouldn't divide like this
    tf.summary.image('images poison train (no trigger)', model.x_image),
    tf.summary.histogram('conf in poisoned (no trigger)', model.confidence_in_correct),
])

shutil.copy('config.json', model_dir)

with tf.Session() as sess:
    # Initialize the summary writer, global variables, and our time counter.
    summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
    sess.run(tf.global_variables_initializer())

    sess.run(clean_training_iterator.initializer,
             feed_dict={clean_placeholder[0]: clean_train_images,
                        clean_placeholder[1]: clean_train_labels})
    sess.run(poisoned_training_iterator.initializer,
             feed_dict={poisoned_placeholder[0]: poisoned_train_images,
                        poisoned_placeholder[1]: poisoned_train_labels})
    if len(poisoned_train_indices) > 0:
        sess.run(poisoned_only_training_iterator.initializer,
                 feed_dict={poisoned_only_placeholder[0]: poisoned_only_train_images,
                            poisoned_only_placeholder[1]: poisoned_only_train_labels})
        sess.run(poisoned_no_trigger_training_iterator.initializer,
                 feed_dict={poisoned_no_trigger_placeholder[0]: poisoned_no_trigger_train_images,
                            poisoned_no_trigger_placeholder[1]: poisoned_no_trigger_train_labels})

    clean_training_handle = sess.run(clean_training_iterator.string_handle())
    poisoned_training_handle = sess.run(poisoned_training_iterator.string_handle())
    if len(poisoned_train_indices) > 0:
        poisoned_only_training_handle = sess.run(poisoned_only_training_iterator.string_handle())
        poisoned_no_trigger_training_handle = sess.run(poisoned_no_trigger_training_iterator.string_handle())

    evalHelper = EvalHelper(
        sess,
        {
            "clean_train": (clean_train_images, clean_train_labels),
            "poisoned_train": (poisoned_train_images, poisoned_train_labels),
            "poisoned_only_train": (poisoned_only_train_images, poisoned_only_train_labels),
            "poisoned_no_trigger_train": (poisoned_no_trigger_train_images, poisoned_no_trigger_train_labels),
            "clean_test": (clean_test_images, clean_test_labels),
            "poisoned_test": (poisoned_test_images, poisoned_test_labels),
        },
        iterator_handle
    )

    # Main training loop
    for ii in range(max_num_training_steps):
        clean_dict = {iterator_handle: clean_training_handle,
                      model.is_training: True}
        poison_dict = {iterator_handle: poisoned_training_handle,
                       model.is_training: True}

        # Output to stdout
        if ii % num_output_steps == 0:
            clean_acc = sess.run(model.accuracy, feed_dict=clean_dict)
            poison_acc = sess.run(model.accuracy, feed_dict=poison_dict)
            print('Step {}:    ({})'.format(ii, datetime.now()))
            print('    training clean accuracy {:.4}%'.format(clean_acc * 100))
            print('    training poison accuracy {:.4}%'.format(poison_acc * 100))

        # Tensorboard summaries
        if ii % num_summary_steps == 0:
            summary = sess.run(merged_summaries, feed_dict=poison_dict)
            summary_writer.add_summary(summary, global_step.eval(sess))
            summary_clean = sess.run(clean_histogram, feed_dict=clean_dict)
            summary_writer.add_summary(summary_clean, global_step.eval(sess))
            if len(poisoned_train_indices) > 0:
                poison_only_dict = {iterator_handle: poisoned_only_training_handle,
                                    model.is_training: True}
                poison_no_trigger_dict = {iterator_handle: poisoned_no_trigger_training_handle,
                                          model.is_training: True}
                summary_poison_only = sess.run(poison_only_merged_summaries, feed_dict=poison_only_dict)
                summary_writer.add_summary(summary_poison_only, global_step.eval(sess))
                summary_poison_no_trigger = sess.run(poison_no_trigger_merged_summaries, feed_dict=poison_no_trigger_dict)
                summary_writer.add_summary(summary_poison_no_trigger, global_step.eval(sess))

        # Write a checkpoint
        if ii % num_checkpoint_steps == 0:
            saver.save(sess,
                       os.path.join(model_dir, 'checkpoint'),
                       global_step=global_step)

        # Run an eval
        if (config['num_eval_steps'] > 0
                and ii % config['num_eval_steps'] == 0):
            print('Starting eval ...', flush=True)
            evalHelper.evaluate_session(model, sess)

        # Actual training step
        sess.run(train_step, feed_dict=poison_dict)


text:
# imports
import json
import time
import pickle
import scipy.misc
import skimage.io
import cv2
import caffe

import numpy as np
import os.path as osp

from random import shuffle
from PIL import Image
import random, copy
from voc import load_pascal_annotation
import multiprocessing

class ImageSegDataLayer(caffe.Layer):

    """
    This is a simple syncronous datalayer for training a Detection model on
    PASCAL.
    """

    def setup(self, bottom, top):

        self.top_names = ['data', 'label_1s', 'label_2s', 'label_3s', 'label']

        # === Read input parameters ===
        # params is a python dictionary with layer parameters.
        params = eval(self.param_str)

        SimpleTransformer.check_params(params)
        # store input as class variables
        self.batch_size = params['batch_size']
        self.input_shape = params['crop_size']

        # Create a batch loader to load the images.
        self.batch_loader = BatchLoader(params)

        # === reshape tops ===
        # since we use a fixed input image size, we can shape the data layer
        # once. Else, we'd have to do it in the reshape call.
        top[0].reshape(
            self.batch_size, 3, self.input_shape[0], self.input_shape[1])
        # Note the 20 channels (because PASCAL has 20 classes.)
        top[1].reshape(
            self.batch_size, 1, self.input_shape[0], self.input_shape[1])

        top[2].reshape(
            self.batch_size, 1, self.input_shape[0], self.input_shape[1])

        top[3].reshape(
            self.batch_size, 1, self.input_shape[0], self.input_shape[1])

        top[4].reshape(
            self.batch_size, 1, self.input_shape[0], self.input_shape[1])

        print_info("ImageSegDataLayer", params)

    def forward(self, bottom, top):
        """
        Load data.
        """
        for itt in range(self.batch_size):
            # Use the batch loader to load the next image.
            im, label_1s, label_2s, label_3s, label = self.batch_loader.perpare_next_data()

            # Add directly to the caffe data layer
            top[0].data[itt, ...] = im
            top[1].data[itt, ...] = label_1s
            top[2].data[itt, ...] = label_2s
            top[3].data[itt, ...] = label_3s
            top[4].data[itt, ...] = label

    def reshape(self, bottom, top):
        """
        There is no need to reshape the data, since the input is of fixed size
        (rows and columns)
        """
        pass

    def backward(self, top, propagate_down, bottom):
        """
        These layers does not back propagate
        """
        pass

class BatchLoader(object):

    """
    This class abstracts away the loading of images.
    Images can either be loaded singly, or in a batch. The latter is used for
    the asyncronous data layer to preload batches while other processing is
    performed.
    """

    def __init__(self, params):
        self.batch_size = params['batch_size']
        self.root_folder = params['root_folder']
        self.source = params['source']
        self.voc_dir = params['voc_dir']
        if 'is_only_human' not in params:
            params['is_only_human'] = False
        self.is_only_human = params['is_only_human']
        # get list of image indexes.
        self.indexlist = [line.strip().split() for line in open(self.source)]
        self._cur = 0  # current image
        # this class does some simple data-manipulations
        self.transformer = SimpleTransformer(params)
        self.q = multiprocessing.Queue(maxsize=self.batch_size*2)
        #self.start_batch()

        print "BatchLoader initialized with {} images".format(
            len(self.indexlist))

    def load_next_image(self):
        """
        Load the next image in a batch.
        """
        return self.q.get()

    def perpare_next_data(self):
        # Did we finish an epoch?
        if self._cur == len(self.indexlist):
            self._cur = 0
            shuffle(self.indexlist)

        # Load an image
        index = self.indexlist[self._cur]  # Get the image index
        image_file_path, label_file_path = index
        # image = Image.open(osp.join(self.root_folder, image_file_path))
        # label = Image.open(osp.join(self.root_folder, label_file_path))

        image = cv2.imread(self.root_folder+image_file_path, cv2.IMREAD_COLOR)
        label = cv2.imread(self.root_folder+label_file_path, cv2.IMREAD_GRAYSCALE)
        img_id = osp.splitext(osp.basename(label_file_path))[0]
        annotation = load_pascal_annotation(img_id, self.voc_dir, self.is_only_human)

        self._cur += 1
        return self.transformer.preprocess(image, label, annotation)

    def start_batch(self):
        thread = multiprocessing.Process(target=self.data_generator_task)
        thread.daemon = True
        thread.start()

    def data_generator_task(self):
        while True:
            output = self.perpare_next_data()
            self.q.put(output)

class SimpleTransformer:

    """
    SimpleTransformer is a simple class for preprocessing and deprocessing
    images for caffe.
    """

    def __init__(self, params):
        SimpleTransformer.check_params(params)
        self.mean = params['mean']
        self.is_mirror = params['mirror']
        self.crop_h, self.crop_w = params['crop_size']
        self.scale = params['scale']
        self.phase = params['phase']
        self.ignore_label = params['ignore_label']

    def set_mean(self, mean):
        """
        Set the mean to subtract for centering the data.
        """
        self.mean = mean

    def set_scale(self, scale):
        """
        Set the data scaling.
        """
        self.scale = scale

    # def generate_scale_range(self, boxes):
    #     if not len(boxes):
    #         return 0.5, 15

    #     def fitness(item):
    #         x1, y1, x2, y2 = item
    #         sq = (x2 - x1) * (y2 - y1)
    #         return sq
    #     boxes = sorted(boxes, key=fitness, reverse=True)
    #     x1, y1, x2, y2 = boxes[0]
    #     max_sp = (x2 - x1) * (y2 - y1)
    #     x1, y1, x2, y2 = boxes[-1]
    #     min_sq = (x2 - x1) * (y2 - y1)
    #     if min_sq > 50176:
    #         base, ran = 0.5, 7
    #     elif max_sp < 12544:
    #         base, ran = 0.8, 12
    #     else:
    #         base, ran = 0.5, 15

    #     return base, ran

    def generate_scale_label(self, image, label, annotation):
        boxes = annotation['boxes']
        gt_classes = annotation['gt_classes']
        annos = zip(boxes, gt_classes)
        #base, ran = self.generate_scale_range(boxes)
        f_scale = 0.5 + random.randint(0, 10) / 10.0
        image = cv2.resize(image, None, fx=f_scale, fy=f_scale, interpolation = cv2.INTER_LINEAR)
        label = cv2.resize(label, None, fx=f_scale, fy=f_scale, interpolation = cv2.INTER_NEAREST)

        #label_1s, label_2s, label_3s = copy.copy(label), copy.copy(label), copy.copy(label)
        # label_1s, label_2s, label_3s = np.full(label.shape, self.ignore_label, dtype=np.uint8), np.full(label.shape, self.ignore_label, dtype=np.uint8), np.full(label.shape, self.ignore_label, dtype=np.uint8)
        label_1s, label_2s, label_3s = np.zeros_like(label), np.zeros_like(label), np.zeros_like(label)
        def fitness(annos_item):
            x1, y1, x2, y2 = annos_item[0]
            sq = (x2 - x1) * (y2 - y1)
            return sq
        annos = sorted(annos, key=fitness, reverse=True)

        for box, cls_id in annos:
            box = np.array(box) * f_scale
            x1, y1, x2, y2 = box
            sq = (x2 - x1) * (y2 - y1)
            s1 = label_1s[y1:y2, x1:x2]
            s2 = label_2s[y1:y2, x1:x2]
            s3 = label_3s[y1:y2, x1:x2]
            s0 = label[y1:y2, x1:x2]

            if sq < 12544:
                s1[:] = s0[:]
                s2[:] = 255
                s3[:] = 255
            elif sq >= 12544 and sq < 40000:
                s1[:] = s0[:]
                s2[:] = s0[:]
                s3[:] = 255
            elif sq >= 40000 and sq < 120000:
                s1[:] = 255
                s2[:] = s0[:]
                s3[:] = 255
            elif sq >= 120000 and sq < 160000:
                s1[:] = 255
                s2[:] = s0[:]
                s3[:] = s0[:]
            elif sq >= 160000:
                s1[:] = 255
                s2[:] = 255
                s3[:] = s0[:]
        # cv2.imshow('image  ', image)
        # cv2.imshow('label 1', label_1s)
        # cv2.imshow('label 2', label_2s)
        # cv2.imshow('label 3', label_3s)
        # cv2.waitKey()
        return image, label_1s, label_2s, label_3s, label

    def preprocess(self, image, label, annotation):
        """
        preprocess() emulate the pre-processing occuring in the vgg16 caffe
        prototxt.
        """
        # image = cv2.convertTo(image, cv2.CV_64F)
        image, label_1s, label_2s, label_3s, label = self.generate_scale_label(image, label, annotation)
        image = np.asarray(image, np.float32)
        image -= self.mean
        image *= self.scale

        img_h, img_w = label_1s.shape
        pad_h = max(self.crop_h - img_h, 0)
        pad_w = max(self.crop_w - img_w, 0)
        if pad_h > 0 or pad_w > 0:
            img_pad = cv2.copyMakeBorder(image, 0, pad_h, 0,
                pad_w, cv2.BORDER_CONSTANT,
                value=(0.0, 0.0, 0.0))
            label_1s_pad = cv2.copyMakeBorder(label_1s, 0, pad_h, 0,
                pad_w, cv2.BORDER_CONSTANT,
                value=(self.ignore_label,))
            label_2s_pad = cv2.copyMakeBorder(label_2s, 0, pad_h, 0,
                pad_w, cv2.BORDER_CONSTANT,
                value=(self.ignore_label,))
            label_3s_pad = cv2.copyMakeBorder(label_3s, 0, pad_h, 0,
                pad_w, cv2.BORDER_CONSTANT,
                value=(self.ignore_label,))
            label_pad = cv2.copyMakeBorder(label, 0, pad_h, 0,
                pad_w, cv2.BORDER_CONSTANT,
                value=(self.ignore_label,))
        else:
            img_pad, label_1s_pad, label_2s_pad, label_3s_pad, label_pad = image, label_1s, label_2s, label_3s, label

        img_h, img_w = label_1s_pad.shape
        if self.phase == 'Train':
            h_off = random.randint(0, img_h - self.crop_h)
            w_off = random.randint(0, img_w - self.crop_w)
        else:
            h_off = (img_h - self.crop_h) / 2
            w_off = (img_w - self.crop_w) / 2

        # roi = cv2.Rect(w_off, h_off, self.crop_w, self.crop_h);

        image = np.asarray(img_pad[h_off : h_off+self.crop_h, w_off : w_off+self.crop_w].copy(), np.float32)
        label_1s = np.asarray(label_1s_pad[h_off : h_off+self.crop_h, w_off : w_off+self.crop_w].copy(), np.float32)
        label_2s = np.asarray(label_2s_pad[h_off : h_off+self.crop_h, w_off : w_off+self.crop_w].copy(), np.float32)
        label_3s = np.asarray(label_3s_pad[h_off : h_off+self.crop_h, w_off : w_off+self.crop_w].copy(), np.float32)
        label    = np.asarray(label_pad[h_off : h_off+self.crop_h, w_off : w_off+self.crop_w].copy(), np.float32)

        #image = image[:, :, ::-1]  # change to BGR
        image = image.transpose((2, 0, 1))

        if self.is_mirror:
            flip = np.random.choice(2) * 2 - 1
            image = image[:, :, ::flip]
            label_1s = label_1s[:, ::flip]
            label_2s = label_2s[:, ::flip]
            label_3s = label_3s[:, ::flip]
            label    = label[:, ::flip]

        return image, label_1s, label_2s, label_3s, label

    @classmethod
    def check_params(cls, params):
        if 'crop_size' not in params:
            params['crop_size'] = (505, 505)
        if 'mean' not in params:
            params['mean'] = [128, 128, 128]
        if 'scale' not in params:
            params['scale'] = 1.0
        if 'mirror' not in params:
            params['mirror'] = False
        if 'phase' not in params:
            params['phase'] = 'Train'
        if 'ignore_label' not in params:
            params['ignore_label'] = 255

def print_info(name, params):
    """
    Ouput some info regarding the class
    """
    print "{} initialized for split: {}, with bs: {}, im_shape: {}.".format(
        name,
        params['source'],
        params['batch_size'],
        params['crop_size'])

if __name__ == '__main__':
    params = {'batch_size': 2,
              'mean': (104.008, 116.669, 122.675),
              'root_folder': 'D:/v-zihuan/segmentation_with_scale/experiment/voc_part/data/',
              'source': 'D:/v-zihuan/segmentation_with_scale/experiment/voc_part/list/train_3s.txt',
              'mirror': True,
              'crop_size': (505, 505)}
    t = SimpleTransformer(params)

    image = Image.open(r'D:/v-zihuan/segmentation_with_scale/experiment/voc_part/data/images/2008_000003.jpg')
    label = Image.open(r'D:/v-zihuan/segmentation_with_scale/experiment/voc_part/data/part_mask_scale_3/2008_000003.png')
    t.preprocess(image, label)

ext: py

max_stars_repo_name: speedinghzl/Scale-Adaptive-Network

max_issues_repo_name: speedinghzl/Scale-Adaptive-Network

max_forks_repo_name: speedinghzl/Scale-Adaptive-Network

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
# gaussian process, t process

import numpy as np                                                # type: ignore
import warnings
from functools import partial
from math import pi, exp, log, sqrt
from numpy import ndarray
from scipy.linalg import cho_solve, cholesky                      # type: ignore
from scipy.stats import norm                                      # type: ignore
from typing import Any, Callable, Optional, Sequence, Tuple, Type, Union
from ..base import BayesianSupervisedModel, Thetas, Density
from ..infer import Dirac, Gaussian, Student, LogDensity, \
    GradientDescentOptimizer, MarkovChainMonteCarloSampler
from ..util import audit_X_Z, audit_X_y, audit_X_y_update, is_array
from .kernels import Kernel, RBFKernel, WhiteKernel

class GaussianProcessRegressor(BayesianSupervisedModel):

    """
    Gaussian process regressor
    """

    inferences = {'exact'}
    solvers = {'l-bfgs-b'}
    aquisitions = {'pi', 'ei', 'lcb'}

    def __init__(self, kernel: Kernel = 1. * RBFKernel(1.) + 1. * WhiteKernel(),
                 inference: str = 'exact', solver: str = 'l-bfgs-b'):

        super().__init__()

        if isinstance(kernel, Kernel):
            self._kernel = kernel
        else:
            raise TypeError('kernel must be a Kernel object.')

        if inference in self.inferences:
            self._inference = inference
        else:
            raise ValueError('inference must be in {}.'.format(self.inferences))

        if solver in self.solvers:
            self._optimizer = \
                GradientDescentOptimizer(solver=solver,
                                         bounds=kernel.thetas.bounds,
                                         x0=kernel.thetas.values)
        else:
            raise ValueError('solver must be in {}.'.format(self.solvers))

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return 'GaussianProcessRegressor(kernel={})'.format(self.kernel)

    @property
    def kernel(self):
        return self._kernel

    @property
    def thetas(self):
        return self.kernel.thetas

    @property
    def hyperparameters(self):
        return self.kernel.hyperparameters

    @property
    def inference(self):
        return self._inference

    @property
    def optimizer(self):
        return self._optimizer

    def fitted(self) -> bool:
        return hasattr(self, 'dual_weights')

    def _set(self, kparams: ndarray):
        self.kernel._set(kparams)

    def _obj(self, X: ndarray, y: ndarray) -> Callable:
        Y = y.reshape(-1, 1)
        n = X.shape[0]
        I = np.identity(n)
        kfun = self.kernel._obj(X)
        # log marginal likelihood and its gradient w.r.t. log params
        # ref. GPML algorithm 2.1, equation 5.9
        def fun(kparams: ndarray, grad: bool = True) \
            -> Union[Tuple[float, ndarray], float]:
            assert is_array(kparams, 1, np.number)
            K, dK = kfun(kparams)
            K[np.diag_indices_from(K)] += 1e-8  # jitter
            L = cholesky(K, lower=True)
            # dual weights of training points in kernel space
            invK_y = cho_solve((L, True), Y)
            # log marginal likelihood
            log_mll = -.5 * n * log(2. * pi)
            log_mll -= .5 * (Y.T @ invK_y).item()
            log_mll -= np.log(np.diag(L)).sum()
            if grad:
                # gradient w.r.t. kernel log parameters
                invK = cho_solve((L, True), I)
                S = invK_y @ invK_y.T - invK
                grad = np.einsum('ij,ijk->k', S, dK) * .5
                return log_mll, np.array(grad)
            else:
                return log_mll
        return fun

    def _acq(self, acquisition: str) -> Callable:
        if not self.fitted():
            raise AttributeError('surrogate model is not fitted yet.')
        # probability of improvement
        def fun_pi(x: ndarray, xi=.01) -> float:
            assert is_array(x, 1, np.number)
            assert isinstance(xi, float) and xi > 0.
            p = self.posterior_predictive(x[np.newaxis, :])
            mu, sigma = p.mu.item(), sqrt(p.cov.item())
            y_min = self.y.min()
            return norm.cdf(y_min, mu + xi, sigma)
        # expected improvement
        def fun_ei(x: ndarray) -> float:
            assert is_array(x, 1, np.number)
            p = self.posterior_predictive(x[np.newaxis, :])
            mu, sigma = p.mu.item(), sqrt(p.cov.item())
            y_min = self.y.min()
            return (y_min - mu) * norm.cdf(y_min, mu, sigma) + \
                   sigma * norm.pdf(y_min, mu, sigma)
        # lower confidence bound
        def fun_lcb(x: ndarray, beta: float = 1.) -> float:
            assert is_array(x, 1, np.number)
            assert isinstance(beta, float) and beta > 0.
            p = self.posterior_predictive(x[np.newaxis, :])
            mu, sigma = p.mu.item(), sqrt(p.cov.item())
            return mu - beta * sigma
        # entropy search
        def fun_es():
            raise NotImplementedError
        # knowledge gradient
        def fun_kg():
            raise NotImplementedError
        # dispatch
        if acquisition == 'pi':
            return fun_pi
        elif acquisition == 'ei':
            return fun_ei
        elif acquisition == 'lcb':
            return fun_lcb
        else:
            raise ValueError('acquisition must be in {}.'\
                             .format(self.aquisitions))

    def config(self, x0: Optional[ndarray] = None,
                n_restarts: Optional[int] = None):
        if x0 is not None:
            self.optimizer.X0 = x0
        if n_restarts is not None:
            self.optimizer.n_restarts = n_restarts

    def hyper_prior(self, n_samples: int = 0):
        super().hyper_prior(n_samples)
        raise NotImplementedError
        if n_samples <= 0:
            return
        else:
            return

    def hyper_posterior(self, n_samples: int = 0, **kwargs):
        super().hyper_posterior(n_samples)
        log_mll = partial(self._obj(self.X, self.y), grad=False)
        k = len(self.thetas)
        hyper_posterior = LogDensity(log_mll, k)
        if n_samples <= 0:
            return hyper_posterior
        else:
            sampler = MarkovChainMonteCarloSampler(hyper_posterior,
                          Gaussian(np.zeros(k), np.eye(k)),
                          self.thetas.values, n_samples, **kwargs)
            return sampler.sample()
        # only support flat hyperprior θ for now

    def fit(self, X: ndarray, y: ndarray):
        """ empirical bayes, a.k.a. type II maximum likelihood """
        super().fit(X, y)
        result = self.optimizer.maximize(self._obj(self.X, self.y), True)
        success, log_mll, kparams = (result[k] for k in ['success', 'f', 'x'])
        if not success:
            warnings.warn( 'optimzation fails. try changing x0 or bounds, '
                           'or increasing number of restarts.' )
        self._set(kparams)
        self.log_mll = log_mll
        # precompute for prediction
        K = self.kernel(self.X, self.X)
        K[np.diag_indices_from(K)] += 1e-8  # jitter
        self.L = cholesky(K, lower=True)
        self.dual_weights = cho_solve((self.L, True), self.y)
        return self

    def predict(self, X: ndarray) -> ndarray:
        super().predict(X)
        Kzx = self.kernel(X, self.X)
        mu = np.einsum('ij,j->i', Kzx, self.dual_weights)
        return mu

    def prior_predictive(self, X: ndarray, n_samples: int = 0) \
        -> Union[Gaussian, ndarray]:
        super().prior_predictive(X, n_samples)
        mu = np.zeros(len(X))
        cov = self.kernel(X, X)
        prior = Gaussian(mu, cov)
        if n_samples <= 0:
            return prior
        else:
            return prior.sample(n_samples)
        # only support Dirac θ for now

    def posterior_predictive(self, X: ndarray, n_samples: int = 0) \
        -> Union[Gaussian, ndarray]:
        super().posterior_predictive(X, n_samples)
        Kzx = self.kernel(X, self.X)
        Kzz = self.kernel(X, X)
        mu = np.einsum('ij,j->i', Kzx, self.dual_weights)
        cov = Kzz - Kzx @ cho_solve((self.L, True), Kzx.T)
        # if np.any(cov < 0.):
        #     warnings.warn('posterior covariance has negative elements. '
        #                   'possibly numerical issues - correcting to 0.')
        # cov[cov < 0.] = 0.
        posterior = Gaussian(mu, cov, allow_singular=True)
        if n_samples <= 0:
            return posterior
        else:
            return posterior.sample(n_samples)
        # only support Dirac θ for now

class StudentProcessRegressor(BayesianSupervisedModel):

    """
    Student process regressor
    """

    def __init__(self, *args, **kwargs):
        raise NotImplementedError

max_stars_repo_name: zack-x-zhang/GPie

max_issues_repo_name: zack-x-zhang/GPie

max_forks_repo_name: zack-x-zhang/GPie

reason: import numpy,from numpy,from scipy


text:
#!/usr/bin/python
import time
import numpy as np
import pandas as pd
import argparse
from math import exp
from math import sqrt
from datetime import datetime
from os import listdir
import sys

# BOKEH
from bokeh import events
from bokeh.io import output_file, show
from bokeh.models import CustomJS, HoverTool, ColumnDataSource, Slider, CheckboxGroup, RadioGroup, Button, MultiSelect
from bokeh.plotting import figure
from bokeh.transform import linear_cmap
from bokeh.transform import log_cmap
from bokeh.util.hex import axial_to_cartesian
from bokeh.util.hex import cartesian_to_axial
from bokeh.layouts import column, row
from bokeh.palettes import Viridis256, Greys256

def import_table(file):
    '''
    This function imports the pkl files from the tables forlder, and returns a pandas dataframe.
    '''
    table = pd.read_pickle(file)
    return table

def create_array(table):
    '''
    This function recieves a dataframe with logP and Mass values for every ChEBI identifier.
    It returns two numpy arrays: for mass and logP.
    '''
    # create lists
    x = [float(logP) for logP in table.logP]
    y = [float(mass) for mass in table.Mass]

    return np.asarray(x), np.asarray(y)

def hexbin(df, x, y, size, aspect_scale, orientation):
    '''
    This function recieves x and y coordinate arrays and converts these into q and r hexagon coordinates by calling Bokeh's "cartesian_to_axial" function.
    The q and r coordinates are added to the dataframe, and this dataframe is returend.
    '''
    q, r = cartesian_to_axial(x, y, size, orientation=orientation, aspect_scale=aspect_scale)

    df.loc[:'q'] = q
    df.loc[:'r'] = r

    return df

def add_tooltip_columns(df, table):
    '''
    For every hexagon, a tooltip will be created that will be shown when the user hovers with the mouse over the hexagon.
    The tooltip will show the 3 most frequent ChEBI identifiers and additional information.

    This function recieves:
    - a "df" dataframe (with hexagonal coordinates) that will be the source for the multiplot
    - the "table" dataframe with information that is needed for the tooltip such as name, mass, logP for every ChEBI identifier

    In this function, the tooltip information will be added to the original dataframe in additional columns.
    These columns will be used by JavaScript code to display in the tooltip.

    '''
    table = table.drop(['Class', 'logP', 'Mass'], axis=1)
    table = table.reset_index()

    # Define tooltip size
    TOOLTIP_COUNT = 3
    columns = table.columns
    tooltip_columns = {column+str(i):[] for i in range(1, TOOLTIP_COUNT+1) for column in columns}

    # Extract ChEBI identifiers from dataframe
    chebi_ids = [ids if isinstance(ids, list) else [] for ids in df.ChEBI]
    list_for_df = []

    # Use chebi identifiers to look up information in the "table" dataframe
    for ids in chebi_ids:
        rows = table[table.ChEBI.isin(ids)]

        # Sort and select most frequent ChEBI identifiers
        rows = rows.sort_values(by='Count', ascending=False)
        values_nested = rows[0:TOOLTIP_COUNT-1].values.tolist()

        # Unnest information from the table,
        values_unnested = [item for sublist in values_nested for item in sublist]
        while len(values_unnested) < (TOOLTIP_COUNT*len(columns)):
            values_unnested.append("-")

        list_for_df.append(values_unnested)

    df_tooltip = pd.DataFrame(list_for_df, columns = tooltip_columns)

    df = df.join(df_tooltip, how='left')

    return df

def get_blur(x,y,sigma_x,sigma_y):
    '''
    This function recieves x, y values and sigma x, sigma y values and returns the calculated blur value.
    See https://en.wikipedia.org/wiki/Multivariate_normal_distribution
    '''
    return exp(-0.5*(x*x/sigma_x/sigma_x + y*y/sigma_y/sigma_y))

def get_rows(q, r, counts, tfidf, kernel, blur_max, step_size):
    '''
    For every hexagon (=row in the dataframe), counts will be distributed to surrounding hexagons in a gaussian manner (blur).
    This function recieves information for one hexagon so that it can distribute its counts to other hexagons.
    To distribute the counts to other hexagons/rows, we use the "kernel".
    We create new rows for other hexagons, which means we might create many rows with the same hexagonal coorindates.
    Newly created rows will be returned
    '''
    # initiate original row
    rows = [[q, r, counts, tfidf] + [counts for i in np.arange(0, blur_max+step_size, step_size)] + [tfidf for i in np.arange(0, blur_max+step_size, step_size)]]

    # use kernel to distribute counts to other rows
    for coords_new, blur_factors in kernel.items():
        q_new = q + coords_new[0]
        r_new = r + coords_new[1]

        # create new row, use kernels coordinates and calculated blur values
        new_row = [q_new, r_new, 0, 0] + list(map(lambda n: n * counts, blur_factors)) + list(map(lambda n: n*tfidf, blur_factors))
        rows.append(new_row)
    return rows

def construct_kernel(blur_max, step_size):
    '''
    This function recieves the maximum blur and step size to construct the kernel.
    The kernel is a dictionary that uses the coordinates as keys, and the blur values as values.
    The blur values depend on sd_x, so the kernel will return a list of blur values from 0 to blur_max.
    Blur values are calculated by "get_blur()"".
    The kernel will be used by "get_rows()" to distribute counts to surrounding hexagons.
    '''
    coordinates_to_distance = {(-5,2):(7.5, sqrt(3)/2),(-5,3):(7.5, sqrt(3)/2),
    (-4,1):(6, sqrt(3)),(-4,2):(6, 0),(-4,3):(6,sqrt(3)),
    (-3,0):(4.5, 3*sqrt(3)/2),(-3,1):(4.5, sqrt(3)/2),(-3,2):(4.5, sqrt(3)/2),(-3,3):(4.5, 3*sqrt(3)/2),
    (-2,-1):(3, 2*sqrt(3)),(-2,0):(3, sqrt(3)),(-2,1):(3, 0),(-2,2):(3, sqrt(3)),(-2,3):(3, 2*sqrt(3)),
    (-1,-1):(1.5, 3*sqrt(3)/2),(-1,0):(1.5, sqrt(3)/2),(-1,1):(1.5, sqrt(3)/2),(-1,2):(1.5, 3*sqrt(3)/2),
    (0,-2):(0, 2*sqrt(3)),(0,-1):(0, sqrt(3)),(0,1):(0, sqrt(3)),(0,2):(0, 2*sqrt(3)),
    (1,-2):(1.5, 3*sqrt(3)/2),(1,-1):(1.5, sqrt(3)/2),(1,0):(1.5, sqrt(3)/2),(1,1):(1.5, 3*sqrt(3)/2),(2,-3):(3, 2*sqrt(3)),
    (2,-2):(3, sqrt(3)),(2,-1):(3, 0),(2,0):(3, sqrt(3)),(2,1):(3, 2*sqrt(3)),
    (3,-3):(4.5, 3*sqrt(3)/2),(3,-2):(4.5, sqrt(3)/2),(3,-1):(4.5, sqrt(3)/2),(3,0):(4.5, 3*sqrt(3)/2),
    (4,-3):(6, sqrt(3)),(4,-2):(6, 0),(4,-1):(6, sqrt(3)),
    (5,-3):(7.5, sqrt(3)/2),(5,-2):(7.5, sqrt(3)/2)}

    kernel = {}
    for key, distance in coordinates_to_distance.items():
        kernel[key] = [0]
        for sd_x in np.arange(step_size, blur_max+step_size, step_size):
            kernel[key].append(get_blur(distance[0], distance[1], sd_x, sd_x/2))

    return kernel

def add_gaussian_blur(df, blur_max, step_size):
    '''
    Function:
    This function adds gaussian blur to the plot by going through all hexagons and applying the kernel to the neighbouring hexagons.
    To speed up the process, the pandas dataframe is put in a python dictionary, so to quickly find the neighbouring hexagon coordinates using the coordinates as keys.
    After bluring, the dictionary is put in a pandas dataframe again, and this dataframe is returned.

    Columns:
    sd_x values for calculating blur values are used to name a column with counts that result from blurring with that specific sd_x.
    This makes selecting correct sd_x column easy with the slider code, because the slider returns values that represent sd_x values.
    ColumnDataSource does not accept intergers as column names, so sd_x column names are changed to string.
    sd_x columns contain lists, in a list the first value is the normal counts, the second value is the tfidf count.
    '''

    kernel = construct_kernel(blur_max, step_size)

    columns = ['q', 'r', 'Count', 'TFIDF'] + [str(sd_x) for sd_x in np.arange(0, (blur_max+step_size), step_size)] + ['%s_tfidf' % str(sd_x) for sd_x in np.arange(0, (blur_max+step_size), step_size)]

    df_blur = pd.concat([pd.DataFrame(get_rows(q, r, counts, tfidf, kernel, blur_max, step_size),
        columns=columns) for q, r, counts, tfidf in zip(df.q, df.r, df.Count, df.TFIDF)], ignore_index=True)

    df_blur = df_blur.groupby(['q', 'r'], as_index=False).agg(sum)

    df_joined = df_blur.merge(df.loc[:,['q', 'r', 'ChEBI']], on=['q', 'r'], how='outer')

    return df_joined

def check_for_ids(id_list, class_id):
    '''
    This functions checks if the Class identifier is in the "id_list".
    This ChEBI ID-specific list contains other ChEBI identifers of chemicals that are of higher level hierarchical class.
    '''
    # check = any(class_id == id for id in id_list)
    check = any(str(class_id) == str(id) for id in id_list)
    return check

def create_class_source(table, size, ratio, orientation, class_id):
    '''
    This function finds all chemicals belonging to class "class_id" as defined by the ChEBI ontology.
    It returns a dataframe filtered for these chemicals.
    '''
    if class_id == None:
        # if no class id is given, then class source should be empty
        df = pd.DataFrame().reindex_like(table)

    else:
        table_class_only = table[[check_for_ids(id_list, class_id) for id_list in table["Class"]]]

        # create array
        x, y = create_array(table_class_only)
        q, r = cartesian_to_axial(x, y, size, orientation=orientation, aspect_scale=ratio)
        df = table_class_only.reset_index()
        df.loc[:,"q"] = q
        df.loc[:,"r"] = r

        df = df.groupby(['q', 'r']).agg({'Count': 'sum', 'TFIDF': 'sum', 'ChEBI': list}).reset_index()
        df = df.drop(columns='ChEBI')
    # Create plot source
    source = ColumnDataSource(df)

    return source

def create_data_source(table, term, size, ratio, orientation, BLUR_MAX, BLUR_STEP_SIZE):
    '''
    This function recieves a query-specific "table" dataframe, and constructs a source for the hexagonal plot.
    The plot source will be created depending on plot-specific values (size, ratio, orientation).
    Additionally, a gaussian blur will be applied for different values of sd(x) (BLUR_MAX, BLUR_STEP_SIZE)
    '''
    # create array with mass and logP values
    x, y = create_array(table)

    # create hexagonal coordinates from mass and logP values
    q, r = cartesian_to_axial(x, y, size, orientation=orientation, aspect_scale=ratio)

    # create dataframe with hexagonal coordinates as rows (row = hexagon)
    df = table.reset_index()
    df.loc[:,"q"] = q
    df.loc[:,"r"] = r

    # sum rows with identical coordinates together, add blur, and add tooltip information
    df = df.groupby(['q', 'r']).agg({'Count': 'sum', 'TFIDF': 'sum', 'ChEBI': list}).reset_index()
    df = add_gaussian_blur(df, BLUR_MAX, BLUR_STEP_SIZE)
    df = add_tooltip_columns(df, table)
    df = df.drop(columns='ChEBI')

    # plot title and source
    title = 'Hexbin plot for '+str(len(x))+' annotated chemicals with query '+str(term)
    source = ColumnDataSource(df)
    return source, title

def return_JS_code(widget):
    '''
    This function recieves a string that indicates the widget.
    It returns a JavaScript callback code corresponding to the widget.
    '''
    if widget == 'multi_select_class':
        code = """
            var source_data = source.data;
            var source_class_data = source_class.data;
            var term_to_source = term_to_source;
            var term = cb_obj.value[0];
            var f = slider1.value
            var sd_x = slider2.value;
            var p = p;
            var mapper = mapper;
            var class_hex = class_hex;
            var checkbox_class = checkbox_class
            var term_to_class = term_to_class;

            if (sd_x % 1 == 0){var sd_x = sd_x.toFixed(1)}
            var sd_x = String(sd_x);

            // check for tfidf

            if (checkbox.active.length == 1) {
            sd_x = sd_x.concat('_tfidf')
            }

            // select new_data
            var new_data = term_to_source[term]['source'].data
            var new_class = term_to_class[term]['source'].data

            // replace current data
            for (var key in new_data) {
                source_data[key] = [];
                for (i=0;i<new_data[key].length;i++) {
                    source_data[key].push(new_data[key][i]);
                }
            }

            // class

            class_hex.visible = false

            for (var key in new_class){
                source_class_data[key] = [];
                for (i=0;i<new_class[key].length;i++) {
                    source_class_data[key].push(new_class[key][i])
                }
            }

            class_hex.source = source_class_data
            checkbox_class.active = []

            // new title
            var title = term_to_source[term]['title']
            p.title.text = title

            // apply blur and saturation

            for (var i = 0; i < source_data[sd_x].length; i++) {
                source_data['Count'][i] = Math.pow(source_data[sd_x][i], 1/f)
                }

            // maximum value for fill_color
            mapper.transform.high = Math.max.apply(Math, source_data['Count'])

            // apply changes
            source_class.change.emit();
            source.change.emit();
        """
    if widget == 'multi_select':
        code = """
            var source_data = source.data;
            var term_to_source = term_to_source;
            var term = cb_obj.value[0];
            var f = slider1.value
            var sd_x = slider2.value;
            var p = p;
            var mapper = mapper;

            if (sd_x % 1 == 0){var sd_x = sd_x.toFixed(1)}
            var sd_x = String(sd_x);

            // check for tfidf

            if (checkbox.active.length == 1) {
            sd_x = sd_x.concat('_tfidf')
            }

            // select new_data
            var new_data = term_to_source[term]['source'].data

            // replace current data
            for (var key in new_data) {
                source_data[key] = [];
                for (i=0;i<new_data[key].length;i++) {
                    source_data[key].push(new_data[key][i]);
                }
            }

            // new title
            var title = term_to_source[term]['title']
            p.title.text = title

            // apply blur and saturation

            for (var i = 0; i < source_data[sd_x].length; i++) {
                source_data['Count'][i] = Math.pow(source_data[sd_x][i], 1/f)
                }

            // maximum value for fill_color
            mapper.transform.high = Math.max.apply(Math, source_data['Count'])

            // apply changes
            source.change.emit();
        """
    elif widget == 'slider2':
        code = """
            var source_data = source.data;
            var f = slider1.value
            var mapper = mapper;
            var checkbox = checkbox;
            var sd_x = cb_obj.value;

            if (sd_x % 1 == 0){var sd_x = sd_x.toFixed(1)}
            var sd_x = String(sd_x);

            // check for tfidf

            if (checkbox.active.length == 1) {
            sd_x = sd_x.concat('_tfidf')
            }

            // apply blur and saturation

            for (var i = 0; i < source_data[sd_x].length; i++) {
                source_data['Count'][i] = Math.pow(source_data[sd_x][i], 1/f)
                }

            // maximum value for fill_color
            mapper.transform.high = Math.max.apply(Math, source_data['Count'])

            // apply changes
            source.change.emit();
        """
    elif widget == 'tooltips':
        code = """

              <col width="100">
              <col width="80">
              <col width="80">
              <col width="305">

                Total Counts
                @Count
                @TFIDF
                ($x, $y)

                Chebi ID
                Count
                TFIDF
                Name

                @ChEBI1
                @Count1
                @TFIDF1
                @Names1

                @ChEBI2
                @Count2
                @TFIDF2
                @Names2

                @ChEBI3
                @Count3
                @TFIDF3
                @Names3

            """

    elif widget == 'slider1':
        code = """
            var mapper = mapper
            var source_data = source.data;
            var f = cb_obj.value;
            var checkbox = checkbox;
            var sd_x = slider2.value;
            if (sd_x % 1 == 0){var sd_x = sd_x.toFixed(1)}
            var sd_x = String(sd_x);

            // check for tfidf

            if (checkbox.active.length == 1) {
            sd_x = sd_x.concat('_tfidf')
            }

            // apply scaling

            for (var i = 0; i < source_data[sd_x].length; i++) {
                source_data['Count'][i] = Math.pow(source_data[sd_x][i], 1/f)
                }

            // maximum value for fill color
            mapper.transform.high = Math.max.apply(Math, source_data['Count'])

            // apply changes
            source.change.emit();
            """

    elif widget == 'rbg':
        code = """
            var active = cb_obj.active;
            var p = p;
            var Viridis256 = Viridis256;
            var Greys256 = Greys256;
            // var class_hex = class_hex;
            // var term_to_class = term_to_class;
            var term = multi_select.value[0]

            if (active == 1){
            mapper.transform.palette = Greys256
            p.background_fill_color = '#000000'
            // if (term_to_class[term]['show_class']){class_hex.glyph.fill_color = 'red'}
            }

            if (active == 0){
            mapper.transform.palette = Viridis256
            p.background_fill_color = '#440154'
            // if(term_to_class[term]['show_class']){class_hex.glyph.fill_color = 'pink'}
            }

            """

    elif widget == 'checkbox':
        code = """
            var source_data = source.data;
            var active = cb_obj.active
            var f = slider1.value
            var mapper = mapper
            var sd_x = slider2.value;

            if (sd_x % 1 == 0){var sd_x = sd_x.toFixed(1)}
            var sd_x = String(sd_x);

            // check for tfidf

            if (active.length == 1) {
            sd_x = sd_x.concat('_tfidf')
            }

            // apply scaling

            for (var i = 0; i < source_data[sd_x].length; i++) {
                source_data['Count'][i] = Math.pow(source_data[sd_x][i], 1/f)
                }

            // maximum value for fill color
            mapper.transform.high = Math.max.apply(Math, source_data['Count'])

            // apply changes
            source.change.emit();
            """

    elif widget == 'hover':
        code = """
            var tooltips = document.getElementsByClassName("bk-tooltip");
            for (var i = 0, len = tooltips.length; i < len; i ++) {
                tooltips[i].style.top = ""; // unset what bokeh.js sets
                tooltips[i].style.left = "";
                tooltips[i].style.bottom = "150px";
                tooltips[i].style.left = "575px";
                tooltips[i].style.width = "500px";
            }
            """

    elif widget == 'button':
        code = """
            var term_to_metadata = term_to_metadata
            var term = multi_select.value[0]
            var metadata = term_to_metadata[term]
            var wnd = window.open("about:blank", "", "_blank");
            wnd.document.write(metadata)
            """
    elif widget == 'stats':
        code = """
            var term_to_stats = term_to_stats
            var term = multi_select.value[0]
            var stats_description = term_to_stats[term]
            var wnd = window.open("about:blank", "", "_blank");
            wnd.document.write(stats_description)
            """

    elif widget == 'class':
        code = """
            var term_to_class = term_to_class;
            var multi_select = multi_select;
            var class_hex = class_hex;
            var active = cb_obj.active;

            if (active.length == 1) {
                class_hex.visible = true
            } else {
                class_hex.visible = false
            }

            """
    return code

def create_stats_description(table):
    '''
    This function recieves the "table" variable and retrieves summary statistcs (mean, std).
    Statistics are returned in html style.
    '''
    total_count = table.Count.sum()
    mean_logP = np.repeat(table.logP, table.Count).values.astype(float).mean()
    mean_mass = np.repeat(table.Mass, table.Count).values.astype(float).mean()
    stdev_logP = np.repeat(table.logP, table.Count).values.astype(float).std()
    stdev_mass = np.repeat(table.Mass, table.Count).values.astype(float).std()
    title = 'Statistics of mass and logP values'
    stats_listed = [title, 'Total amount of chemicals: %d' % total_count , 'LogP mean: %f' % mean_logP, 'LogP standard deviations: %f' % stdev_logP,
    'Mass mean: %f' % mean_mass, 'Mass standard deviation: %f' % stdev_mass]
    stats_description = return_html(stats_listed)
    return stats_description

def return_html(metadata):
    '''
    This function recieves the metadata. The metadata is put in the html string and this string returned.
    '''

    html_content = """

    <TITLE>%s</TITLE>

    %s

%s

%s

%s

%s

%s

    """ % (metadata[0], metadata[0], metadata[1], metadata[2], metadata[3], metadata[4], metadata[5])
    return html_content

def plot(tables, output_filename, xmin, xmax, ymin, ymax, class_id):
    '''
    This is the plot function that uses Bokeh functions and widgets to make an interactive hexagon plot.

    This function recieves:
    - tables: dictionary with tables used to create arrays of repeated x, y coordinates (depending on the counts) for the hexagon plot.
    - output_filename: filename of .html output in the plots folder

    The coordinate arrays are used to create a pandas dataframe with Bokeh functions. This dataframe contains the q, r coordinates and counts used to plot the
    hexagons. To this dataframe, extra information is added (e.g. most common chemicals), which is displayed in the hover tooltip.

    Gaussian blur is added to copies of this dataframe and given as input to the Bokeh slider widget.
    Other widgets are added as well, for saturation, normalization etc. Bokeh allows to customize these widges with javascript code.

    The hexagon plot is saved as a .html file and also shown in the browser.
    '''

    file_name = 'plots/'+str(output_filename)+'.html'
    output_file(file_name)

    # Blur and saturation values
    BLUR_MAX = 4
    BLUR_STEP_SIZE = 0.25
    SATURATION_MAX = 5
    SATURATION_STEP_SIZE = 0.25

    # Hexagon plot properties
    SIZE_HEXAGONS = 10
    orientation = 'flattop' #bokeh alows 2 different hexagon orientations which also influences hexagon size calculations, but we currently have only calculated blur distances for this orientation
    ratio = ((ymax-ymin) / (xmax-xmin) )
    size = SIZE_HEXAGONS / ratio
    hexagon_height = sqrt(3) * size
    hexagon_height = hexagon_height*ratio

    # make figure
    p = figure(x_range = [xmin, xmax],y_range=[ymin-(hexagon_height/2), ymax],
               tools="wheel_zoom,reset,save", background_fill_color= '#440154')

    p.grid.visible = False
    p.xaxis.axis_label = "log(P)"
    p.yaxis.axis_label = "mass in Da"
    p.xaxis.axis_label_text_font_style = 'normal'
    p.yaxis.axis_label_text_font_style = 'normal'

    # source for widgets
    term_to_source = dict()
    term_to_class = dict()
    term_to_metadata = dict()
    term_to_stats = dict()

    options = []
    # Loop for plot sources
    for term in tables.keys():
        print('sourcing %s' % term)

        # add term to the options for the multiplot
        options.append((term, term))

        # get table
        table = tables[term]['table']

        # plot sources
        source, title = create_data_source(table, term, size, ratio, orientation, BLUR_MAX, BLUR_STEP_SIZE)
        source_class = create_class_source(table, size, ratio, orientation, class_id)
        stats_description = create_stats_description(table)
        term_to_source[term] = {'source': source, 'title': title}
        term_to_class[term] =  {'source': source_class, 'show_class': True}
        term_to_stats[term] = stats_description

        # metadata
        metadata = return_html(tables[term]['metadata'])
        term_to_metadata[term] = metadata

    # make default souce for plot, this is the first source shown in the plot, and also works like a container. Old data is thrown out and new data is thrown in.
    default_term = list(tables.keys())[0] # pick the first one
    table = tables[default_term]['table']
    source, title = create_data_source(table, default_term, size, ratio, orientation, BLUR_MAX, BLUR_STEP_SIZE)
    p.title.text = title
    metadata = tables[default_term]['metadata']
    metadata = return_html(metadata)

    # color mapper
    mapper = linear_cmap('Count', 'Viridis256', 0, max(source.data['Count']))
    hex = p.hex_tile(q="q", r="r", size=size, line_color=None, source=source, aspect_scale=ratio, orientation=orientation,
           fill_color=mapper)

    if class_id:
        source_class = create_class_source(table, size, ratio, orientation, class_id)
        class_hex = p.hex_tile(q='q', r="r", size=size, line_color=None, source=source_class, aspect_scale=ratio,orientation=orientation,
            fill_color='#ff007f')
        class_hex.visible = False

    # HOVER
    TOOLTIPS = return_JS_code('tooltips')
    code_callback_hover = return_JS_code('hover')
    callback_hover = CustomJS(code=code_callback_hover)
    hover = HoverTool(renderers=[hex], tooltips=TOOLTIPS, callback=callback_hover, show_arrow=False)
    p.add_tools(hover)

    # Widgets
    slider1 = Slider(start=1, end=SATURATION_MAX, value=1, step=SATURATION_STEP_SIZE, title="Saturation", width=100)
    slider2 = Slider(start=0, end=BLUR_MAX, value=0, step=BLUR_STEP_SIZE, title="Blur", width=100)
    multi_select = MultiSelect(title=output_filename, value=[default_term], options=options, width=100, height=300)
    checkbox = CheckboxGroup(labels=["TFIDF"], active=[])
    radio_button_group = RadioGroup(labels=["Viridis256", "Greys256"], active=0)
    button = Button(label="Metadata",button_type="default", width=100)
    stats = Button(label="Statistics",button_type="default", width=100)
    if class_id:
        label = "Show "+str(class_id)
        checkbox_class = CheckboxGroup(labels=["Show "+str(class_id)], active=[])

    # Javacode
    code_callback_slider1 = return_JS_code('slider1')
    code_callback_slider2 = return_JS_code('slider2')
    code_callback_ms = return_JS_code('multi_select')
    code_callback_checkbox = return_JS_code('checkbox')
    code_callback_rbg = return_JS_code('rbg')
    code_callback_button = return_JS_code('button')
    code_callback_stats = return_JS_code('stats')
    if class_id:
        code_callback_ms = return_JS_code('multi_select_class')
        code_callback_class = return_JS_code('class')

    # Callbacks
    callback_slider1 = CustomJS(args={'source': source, 'mapper': mapper, 'slider2': slider2, 'checkbox': checkbox}, code=code_callback_slider1)
    callback_slider2 = CustomJS(args={'source': source, 'mapper': mapper, 'slider1': slider1, 'checkbox': checkbox}, code=code_callback_slider2)
    callback_ms = CustomJS(args={'source': source, 'term_to_source': term_to_source, 'slider1': slider1, 'slider2': slider2, 'checkbox': checkbox, 'p': p, 'mapper': mapper}, code=code_callback_ms)
    callback_checkbox = CustomJS(args={'source': source, 'slider2': slider2, 'mapper': mapper, 'slider1': slider1}, code=code_callback_checkbox)
    callback_radio_button_group = CustomJS(args={'p': p, 'multi_select': multi_select, 'mapper': mapper, 'term_to_class': term_to_class, 'Viridis256': Viridis256, 'Greys256': Greys256}, code=code_callback_rbg)
    callback_button = CustomJS(args={'term_to_metadata': term_to_metadata, 'multi_select': multi_select},code=code_callback_button)
    callback_stats = CustomJS(args={'term_to_stats': term_to_stats, 'multi_select': multi_select},code=code_callback_stats)
    if class_id:
        callback_ms = CustomJS(args={'source': source, 'term_to_source': term_to_source, 'slider1': slider1, 'slider2': slider2,
        'checkbox': checkbox, 'p': p, 'mapper': mapper, 'source_class': source_class, 'term_to_class': term_to_class, 'class_hex': class_hex,
        'checkbox_class': checkbox_class}, code=code_callback_ms)
        callback_class = CustomJS(args={'multi_select': multi_select, 'term_to_class': term_to_class, 'class_hex': class_hex}, code=code_callback_class)

    # On change
    slider1.js_on_change('value', callback_slider1)
    slider2.js_on_change('value', callback_slider2)
    multi_select.js_on_change("value", callback_ms)
    checkbox.js_on_change('active', callback_checkbox)
    radio_button_group.js_on_change('active', callback_radio_button_group)
    button.js_on_event(events.ButtonClick, callback_button)
    stats.js_on_event(events.ButtonClick, callback_stats)
    if class_id:
        checkbox_class.js_on_change('active', callback_class)

    # Layout
    if class_id:
        layout = row(multi_select, p, column(slider1, slider2, checkbox, checkbox_class, radio_button_group, button, stats))
    else:
        layout = row(multi_select, p, column(slider1, slider2, checkbox, radio_button_group, button, stats))
    show(layout)

def get_tables(files):
    '''
    This function recieves a list of files, imports these files and their corresponding metadata, and saves them in a dictionary.
    '''
    tables = dict()
    for file in files:
        table = import_table(file)
        term = file.split('/')[1].split('_')[0]
        metadata_file = 'metadata/'+str(term)+'.txt'
        metadata_lines = open(metadata_file, 'r')
        metadata = metadata_lines.readlines()

        tables[term] = {'table': table, 'metadata': metadata}

    return tables

def get_files(folder):
    '''
    This function recieves the input folder and returns a list of file paths in this folder.
    '''
    files = ['%s/%s' % (folder, file) for file in listdir(folder) if '.pkl' in file]
    return files

def parser():
    parser = argparse.ArgumentParser(description='This script makes a table of the query IDs, their names and their properties')
    parser.add_argument('-i', required=True, metavar='input_folder', dest='input_folder', help='[i] to select input folder')
    parser.add_argument('-o', required=True, metavar='output_filename', dest='output_filename', help='[o] to name .html output filename (a \'plots\' folder is required!)')
    parser.add_argument('-xmin', required=False, metavar='xmin', dest='xmin', help='[xmin] to select x axis minimum (logP), default is -5')
    parser.add_argument('-xmax', required=False, metavar='xmax', dest='xmax', help='[xmax] to select x axis maximum (logP), default is 10')
    parser.add_argument('-ymax', required=False, metavar='ymax', dest='ymax', help='[ymax] to select y axix maximum (mass in Da), default is 1600')
    parser.add_argument('-class', required=False, metavar='class_id', dest='class_id', help='[c] to select a class to be shown in the plot with a click on the class button')
    arguments = parser.parse_args()
    return arguments

def main():
    # startTime = datetime.now()

    args = parser()
    folder = args.input_folder
    output_filename = args.output_filename
    xmin = args.xmin
    xmax = args.xmax
    ymax = args.ymax

    # default settings
    if not xmin:
        xmin = -5
    else:
        xmin = float(args.xmin)
    if not xmax:
        xmax = 10
    else:
        xmax = float(args.xmax)
    if not ymax:
        ymax = 1600
    else:
        ymax = float(args.ymax)
    ymin = 0 # cannot be negative

    files = get_files(folder)
    tables = get_tables(files)

    plot(tables, output_filename, xmin, xmax, ymin, ymax, args.class_id)

    # print(datetime.now() - startTime)

if __name__ == '__main__':
    main()

max_stars_repo_name: ReinV/SCOPE

max_issues_repo_name: ReinV/SCOPE

max_forks_repo_name: ReinV/SCOPE

reason: import numpy


text:
# yolo检测的接口函数
# 详细信息请参考 https://blog.csdn.net/qq_20241587/article/details/113349874?utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7EBlogCommendFromBaidu%7Edefault-6.control&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7EBlogCommendFromBaidu%7Edefault-6.control

import numpy as np
import cv2
import torch
from numpy import random
#import evaluator
from models.experimental import attempt_load
from utils.general import check_img_size, non_max_suppression, scale_coords, \
    set_logging
from utils.torch_utils import select_device, time_synchronized

def letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True):
    # Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232
    shape = img.shape[:2]  # current shape [height, width]
    if isinstance(new_shape, int):
        new_shape = (new_shape, new_shape)

    # Scale ratio (new / old)
    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
    if not scaleup:  # only scale down, do not scale up (for better test mAP)
        r = min(r, 1.0)

    # Compute padding
    ratio = r, r  # width, height ratios
    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding
    if auto:  # minimum rectangle
        dw, dh = np.mod(dw, 32), np.mod(dh, 32)  # wh padding
    elif scaleFill:  # stretch
        dw, dh = 0.0, 0.0
        new_unpad = (new_shape[1], new_shape[0])
        ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]  # width, height ratios

    dw /= 2  # divide padding into 2 sides
    dh /= 2

    if shape[::-1] != new_unpad:  # resize
        img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)
    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
    img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border
    return img, ratio, (dw, dh)

weights = r'weights/best.pt'
opt_device = ''  # device = 'cpu' or '0' or '0,1,2,3'
imgsz = 640
opt_conf_thres = 0.6
opt_iou_thres = 0.45

# Initialize
set_logging()
device = select_device(opt_device)
half = device.type != 'cpu'  # half precision only supported on CUDA

# Load model
model = attempt_load(weights, map_location=device)  # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max())  # check img_size
if half:
    model.half()  # to FP16

# Get names and colors
names = model.module.names if hasattr(model, 'module') else model.names
colors = [[random.randint(0, 255) for _ in range(3)] for _ in names]

def predict(im0s):
    # Run inference
    img = torch.zeros((1, 3, imgsz, imgsz), device=device)  # init img
    _ = model(img.half() if half else img) if device.type != 'cpu' else None  # run once

    # Set Dataloader & Run inference
    img = letterbox(im0s, new_shape=imgsz)[0]
    # Convert
    img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416
    img = np.ascontiguousarray(img)

    img = torch.from_numpy(img).to(device)
    img = img.half() if half else img.float()  # uint8 to fp16/32
    img /= 255.0  # 0 - 255 to 0.0 - 1.0
    if img.ndimension() == 3:
        img = img.unsqueeze(0)

    # Inference
    # pred = model(img, augment=opt.augment)[0]
    pred = model(img)[0]

    # Apply NMS
    pred = non_max_suppression(pred, opt_conf_thres, opt_iou_thres)

    # Process detections
    ret = []
    for i, det in enumerate(pred):  # detections per image
        if len(det):
            # Rescale boxes from img_size to im0 size
            det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0s.shape).round()
            # Write results
            for *xyxy, conf, cls in reversed(det):
                label = f'{names[int(cls)]}'
                prob = round(float(conf) * 100, 2)  # round 2
                ret_i = [label, prob, xyxy]
                ret.append(ret_i)
    # 返回信息
    # label 标签信息 'face' 'smoke' 'drink' 'phone'
    # prob 为对应的置信度
    # xyxy 为对应的位置信息（外框）
    return ret

reason: import numpy,from numpy


text:
import copy
import numpy as np
from scipy.sparse import csgraph
import datetime
import os.path
import argparse
import yaml

from sklearn.decomposition import TruncatedSVD
from sklearn import cluster
from sklearn.decomposition import PCA
from conf import sim_files_folder, save_address
from util_functions import *
from Articles import ArticleManager
from Users.Users import UserManager
from Users.CoUsers import CoUserManager
from RewardManager import RewardManager
from DatasetRewardManager import DatasetRewardManager
from YahooRewardManager import YahooRewardManager
from DiffList.DiffManager import DiffManager
from conf import *
from LastFM_util_functions import *
from YahooExp_util_functions import *

from lib.LinUCB import LinUCBAlgorithm, Uniform_LinUCBAlgorithm, Hybrid_LinUCBAlgorithm
from lib.hLinUCB import HLinUCBAlgorithm
from lib.factorUCB import FactorUCBAlgorithm
from lib.CoLin import CoLinUCBAlgorithm
from lib.GOBLin import GOBLinAlgorithm
from lib.CLUB import *
from lib.PTS import PTSAlgorithm
from lib.UCBPMF import UCBPMFAlgorithm
from lib.FairUCB import FairUCBAlgorithm
from lib.ThompsonSampling import ThompsonSamplingAlgorithm
from lib.LinPHE import LinPHEAlgorithm
from lib.NeuralUCB import NeuralUCBAlgorithm
from lib.NeuralPHE import NeuralPHEAlgorithm

def pca_articles(articles, order):
    X = []
    for i, article in enumerate(articles):
        X.append(article.featureVector)
    pca = PCA()
    X_new = pca.fit_transform(X)
    # X_new = np.asarray(X)
    # print('pca variance in each dim:', pca.explained_variance_ratio_)

    # print X_new
    # default is descending order, where the latend features use least informative dimensions.
    if order == "random":
        np.random.shuffle(X_new.T)
    elif order == "ascend":
        X_new = np.fliplr(X_new)
    elif order == "origin":
        X_new = X
    for i, article in enumerate(articles):
        articles[i].featureVector = X_new[i]
    return

def generate_algorithms(alg_dict, W, system_params):
    gen = alg_dict["general"] if "general" in alg_dict and alg_dict["general"] else {}
    algorithms = {}
    diffLists = DiffManager()
    for i in alg_dict["specific"]:
        print("")
        print(str(i))
        try:
            tmpDict = globals()["create" + i + "Dict"](
                alg_dict["specific"][i] if alg_dict["specific"][i] else {},
                gen,
                W,
                system_params,
            )
        except KeyError:
            tmpDict = createBaseAlgDict(
                alg_dict["specific"][i] if alg_dict["specific"][i] else {},
                gen,
                W,
                system_params,
            )
        try:
            algorithms[i] = globals()[i + "Algorithm"](tmpDict)
        except KeyError:
            raise NotImplementedError(i + " not currently implemented")
        diffLists.add_algorithm(i, algorithms[i].getEstimateSettings())
    # print algorithms
    return algorithms, diffLists

def addDatasetParams(rewardManagerDict):
    rewardManagerDict["dataset"] = gen["dataset"]
    if gen["dataset"] == "LastFM":
        rewardManagerDict["relationFileName"] = LastFM_relationFileName
        rewardManagerDict["address"] = LastFM_address
        rewardManagerDict["save_address"] = LastFM_save_address
        rewardManagerDict["FeatureVectorsFileName"] = LastFM_FeatureVectorsFileName
        rewardManagerDict["itemNum"] = 19000
    elif gen["dataset"] == "Delicious":
        rewardManagerDict["relationFileName"] = Delicious_relationFileName
        rewardManagerDict["address"] = Delicious_address
        rewardManagerDict["save_address"] = Delicious_save_address
        rewardManagerDict["FeatureVectorsFileName"] = Delicious_FeatureVectorsFileName
        rewardManagerDict["itemNum"] = 190000
    elif gen["dataset"] == "Yahoo":
        rewardManagerDict["address"] = Yahoo_address
        rewardManagerDict["save_address"] = Yahoo_save_address
        rewardManagerDict["itemNum"] = 200000

def createW(gen):
    OriginaluserNum = 2100
    nClusters = 100
    Gepsilon = 0.3
    # won't work when there is a clusterfile procided. args.diagnol doesn't exist

    if "clusterfile" in gen:
        label = read_cluster_label(gen["clusterfile"])
        rewardManagerDict["label"] = label
        userNum = nClusters = int(args.clusterfile.name.split(".")[-1])  # Get cluster number.
        W = initializeW_label(
            nClusters,
            rewardManagerDict["relationFileName"],
            label,
            args.diagnol,
            args.showheatmap,
        )  # Generate user relation matrix
        GW = initializeGW_label(Gepsilon, nClusters, relationFileName, label, args.diagnol)
    else:
        normalizedNewW, newW, label = initializeW_clustering(
            OriginaluserNum, rewardManagerDict["relationFileName"], nClusters
        )
        rewardManagerDict["label"] = label
        GW = initializeGW_clustering(Gepsilon, rewardManagerDict["relationFileName"], newW)
        W = normalizedNewW
    return W, GW, nClusters

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="")
    parser.add_argument(
        "--alg",
        dest="alg",
        help="Select a specific algorithm, could be CoLin, hLinUCB, factorUCB, LinUCB, etc.",
    )
    parser.add_argument("--contextdim", type=int, help="Set dimension of context features.")
    parser.add_argument("--hiddendim", type=int, help="Set dimension of hidden features.")
    parser.add_argument("--config", dest="config", help="yaml config file")
    parser.add_argument(
        "--dataset",
        required=False,
        choices=["LastFM", "Delicious"],
        help="Select dataset to run. No Selection resuts in simulated rewards",
    )
    parser.add_argument(
        "--clusterfile",
        dest="clusterfile",
        help="input an clustering label file",
        metavar="FILE",
        type=lambda x: is_valid_file(parser, x),
    )

    args = parser.parse_args()
    cfg = {}

    with open(args.config, "r") as ymlfile:
        cfg = yaml.load(ymlfile)
    gen = cfg["general"] if "general" in cfg else {}
    simulating = True if not ("dataset" in gen) or gen["dataset"] == "None" else False
    user = cfg["user"] if "user" in cfg else {}
    article = cfg["article"] if "article" in cfg else {}
    reco = cfg["reward"] if "reward" in cfg else {}

    rewardManagerDict = {}

    if args.contextdim:
        context_dimension = args.contextdim
    else:
        context_dimension = gen["context_dimension"] if "context_dimension" in gen else 20
    rewardManagerDict["context_dimension"] = context_dimension
    if args.hiddendim:
        latent_dimension = args.hiddendim
    else:
        latent_dimension = gen["hidden_dimension"] if "hidden_dimension" in gen else 0
    rewardManagerDict["latent_dimension"] = latent_dimension

    rewardManagerDict["training_iterations"] = gen["training_iterations"] if "training_iterations" in gen else 0
    rewardManagerDict["testing_iterations"] = gen["testing_iterations"] if "testing_iterations" in gen else 100
    rewardManagerDict["plot"] = gen["plot"] if "plot" in gen else True

    rewardManagerDict["NoiseScale"] = 0.01

    rewardManagerDict["epsilon"] = 0  # initialize W

    ArticleGroups = article["groups"] if "groups" in article else 5
    if "dataset" in gen:
        if gen["dataset"] == "LastFM":
            print("LastFM")
            n_users = 2100
            n_articles = 19000
        elif gen["dataset"] == "Delicious":
            print("Delicious")
            n_users = 2100
            n_articles = 190000

    UserGroups = user["groups"] if "groups" in user else 5

    rewardManagerDict["poolArticleSize"] = gen["pool_article_size"] if "pool_article_size" in gen else 10
    rewardManagerDict["batchSize"] = gen["batch_size"] if "batch_size" in gen else 1

    # Matrix parameters
    # matrixNoise = 0.01
    # rewardManagerDict["matrixNoise"] = lambda: np.random.normal(scale=matrixNoise)
    # rewardManagerDict["sparseLevel"] = n_users
    # if smaller or equal to 0 or larger or equal to usernum, matrix is fully connected

    # Parameters for GOBLin
    rewardManagerDict["Gepsilon"] = 1

    use_coUsers = ("collaborative" in user) and user["collaborative"]
    reward_type = reco["type"] if "type" in reco else "linear"

    UM = UserManager(context_dimension + latent_dimension, user, argv={"l2_limit": 1})
    UM.CoTheta()

    # rewardManagerDict["W"] = UM.getW()
    # rewardManagerDict["users"] = UM.getUsers()
    rewardManagerDict["k"] = reco["k"] if "k" in reco else 1

    addDatasetParams(rewardManagerDict)
    W, GW, nClusters = createW(gen)
    experiment = DatasetRewardManager(arg_dict=rewardManagerDict)

    system_params = {
        "context_dim": context_dimension,
        "latent_dim": latent_dimension,
        "n_users": n_users,
        "n_clusters": nClusters,
        "n_articles": n_articles,
    }
    algorithms, diffLists = generate_algorithms(cfg["alg"], W, system_params)

    experiment.runAlgorithms(algorithms, diffLists)

max_stars_repo_name: yilingjia/BanditLib

max_issues_repo_name: yilingjia/BanditLib

max_forks_repo_name: yilingjia/BanditLib

reason: import numpy,from scipy


text:
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.utils import np_utils
import numpy as np
import keras

classes = ["car", "bycycle", "motorcycle", "pedestrian"]
num_class = len(classes)
image_size = 50
num_testdata = 50

# メインの関数を定義する
def main():
    X_train, X_test, y_train, y_test = np.load("./vehicle.npy", allow_pickle=True)
    X_train = X_train.astype("float") / 256
    X_test = X_test.astype("float") / 256
    y_train = np_utils.to_categorical(y_train, num_class)
    y_test = np_utils.to_categorical(y_test, num_class)

    model = model_train(X_train, y_train)
    model_eval(model, X_test, y_test)

def model_train(X, y):
    model = Sequential()
    model.add(Conv2D(32, (3,3), padding='same', input_shape=X.shape[1:]))
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3,3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3,3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(4))
    model.add(Activation('softmax'))

    opt = keras.optimizers.RMSprop(lr=0.0001, decay=1e-6)
    model.compile(
        loss='categorical_crossentropy',
        optimizer=opt,
        metrics=['accuracy']
    )

    model.fit(X, y, batch_size=32, epochs=100)

    # モデルの保存
    model.save('./vehicle_cnn.h5')

    return model

def model_eval(model, X, y):
    scores = model.evaluate(X, y, verbose=1)
    print('Test Loss: ', scores[0])
    print('Test Accuracy: ', scores[1])

if __name__ == '__main__':
    main()


text:
import os
import numpy as np
import tensorflow as tf

bert_model_name = 'uncased_L-12_H-768_A-12'
bert_ckpt_dir = os.path.join('input/model/', bert_model_name)
bert_ckpt_file = os.path.join(bert_ckpt_dir, 'bert_model.ckpt')
bert_config_file = os.path.join(bert_ckpt_dir, 'bert_config.json')

checkpoint_filepath = 'output/weights/checkpoint/cp.ckpt'


text:
from torch.utils.data import Dataset
from torchvision.datasets.utils import download_url

from PIL import Image
import torch
import numpy as np
import random
import decord
from decord import VideoReader
import json
import os
from data.utils import pre_caption

decord.bridge.set_bridge("torch")

class ImageNorm(object):
    """Apply Normalization to Image Pixels on GPU
    """
    def __init__(self, mean, std):
        self.mean = torch.tensor(mean).view(1, 3, 1, 1)
        self.std = torch.tensor(std).view(1, 3, 1, 1)

    def __call__(self, img):

        if torch.max(img) > 1 and self.mean.max() <= 1:
            img.div_(255.)
        return img.sub_(self.mean).div_(self.std)

def load_jsonl(filename):
    with open(filename, "r") as f:
        return [json.loads(l.strip("\n")) for l in f.readlines()]

class VideoDataset(Dataset):

    def __init__(self, video_root, ann_root, num_frm=4, frm_sampling_strategy="rand", max_img_size=384, video_fmt='.mp4'):
        '''
        image_root (string): Root directory of video
        ann_root (string): directory to store the annotation file
        '''
        url = 'https://storage.googleapis.com/sfr-vision-language-research/datasets/msrvtt_test.jsonl'
        filename = 'msrvtt_test.jsonl'

        download_url(url,ann_root)
        self.annotation = load_jsonl(os.path.join(ann_root,filename))

        self.num_frm = num_frm
        self.frm_sampling_strategy = frm_sampling_strategy
        self.max_img_size = max_img_size
        self.video_root = video_root
        self.video_fmt = video_fmt
        self.img_norm = ImageNorm(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711))

        self.text = [pre_caption(ann['caption'],40) for ann in self.annotation]
        self.txt2video = [i for i in range(len(self.annotation))]
        self.video2txt = self.txt2video

    def __len__(self):
        return len(self.annotation)

    def __getitem__(self, index):

        ann = self.annotation[index]

        video_path = os.path.join(self.video_root, ann['clip_name'] + self.video_fmt)

        vid_frm_array = self._load_video_from_path_decord(video_path, height=self.max_img_size, width=self.max_img_size)

        video = self.img_norm(vid_frm_array.float())

        return video, ann['clip_name']

    def _load_video_from_path_decord(self, video_path, height=None, width=None, start_time=None, end_time=None, fps=-1):
        try:
            if not height or not width:
                vr = VideoReader(video_path)
            else:
                vr = VideoReader(video_path, width=width, height=height)

            vlen = len(vr)

            if start_time or end_time:
                assert fps > 0, 'must provide video fps if specifying start and end time.'

                start_idx = min(int(start_time * fps), vlen)
                end_idx = min(int(end_time * fps), vlen)
            else:
                start_idx, end_idx = 0, vlen

            if self.frm_sampling_strategy == 'uniform':
                frame_indices = np.arange(start_idx, end_idx, vlen / self.num_frm, dtype=int)
            elif self.frm_sampling_strategy == 'rand':
                frame_indices = sorted(random.sample(range(vlen), self.num_frm))
            elif self.frm_sampling_strategy == 'headtail':
                frame_indices_head = sorted(random.sample(range(vlen // 2), self.num_frm // 2))
                frame_indices_tail = sorted(random.sample(range(vlen // 2, vlen), self.num_frm // 2))
                frame_indices = frame_indices_head + frame_indices_tail
            else:
                raise NotImplementedError('Invalid sampling strategy {} '.format(self.frm_sampling_strategy))

            raw_sample_frms = vr.get_batch(frame_indices)
        except Exception as e:
            return None

        raw_sample_frms = raw_sample_frms.permute(0, 3, 1, 2)

        return raw_sample_frms

max_stars_repo_name: MikeWangWZHL/BLIP

max_issues_repo_name: MikeWangWZHL/BLIP

max_forks_repo_name: MikeWangWZHL/BLIP