text:
import numpy as np
import pandas as pd
from scipy import stats
import random
import sys

def drawbinom(I, prob):
    # Look at scipy.stats.binom...option binom.rvs
    """Random draw from binomial distribution

    Utility function:
    Draw from a binomial distribution:
    Only return if the draw is different than 0
    """
    Ns = 0
    while Ns == 0:
        A = np.random.RandomState()
        Ns = A.binomial(I, prob)
    return Ns

def create_distribution(xk, pk, name="TLD"):
    return stats.rv_discrete(name=name, values=(xk, pk))

def draw_from_distrib(vals, pdf, size=1):
    """Random Draw from given distribution
    """
    vals = np.array(vals)
    distrib = stats.rv_discrete(values=(range(len(vals)), pdf))
    return vals[distrib.rvs(size=size)]

def AdjustTTHistory(time, temp):
    """Calculate adjusted thermal history

    Useful when one needs to calculate thermal history
    in a borehole when some of the sample reaches
    surface temperature
    """

    def zerointersect(pt1, pt2):
        x1, y1 = pt1
        x2, y2 = pt2

        xmax = max(x1, x2)
        xmin = min(x1, x2)

        x = np.array([[x1, 1.0], [x2, 1.0]])
        y = np.array([y1, y2])
        A, B = np.linalg.solve(x, y)
        X, Y = -B/A, 0.0
        if(X > xmin and X < xmax):
            return X, Y
        else:
            return None, None

    if(len(time) != len(temp)):
            return "Error"

    TT = [[time[i], temp[i]] for i in range(0, len(time))]

    newTT = []
    for i in range(0, len(TT)-1):
        pt1 = TT[i]
        pt2 = TT[i+1]
        X, Y = zerointersect(pt1, pt2)
        if(X is not None):
            newTT.append([X, Y])

    TT.extend(newTT)
    newTT = []
    for elem in TT:
        if(elem[1] >= 0.0):
            newTT.append(elem)

    newTT.sort()
    newTT.reverse()
    time = [elem[0] for elem in newTT]
    temp = [elem[1] for elem in newTT]
    return time, temp

def read_mtx_file(filename):
    """ MTX (Madtrax) file parser """

    # Check that the extension is actually mtx
    if filename.split(".")[-1] != "mtx":
        raise ValueError("Specify and mtx file")

    lines = open(filename, "r").read().splitlines()
    lines = (line.strip() for line in lines)

    data = {}

    # First Line is the name
    (data["name"]) = next(lines).split(".")[0]

    # Skip Second line (not sure what that is)
    next(lines)

    # Third line contains count numbers and zeta information
    # nconstraints is the number of boxes defined to constraint
    # thermal history

    line2 = next(lines).split()
    nconstraints, ntl, ncounts = (int(val) for val in line2[:3])
    (data["zeta"],
     data["rhod"],
     data["nd"]) = (float(val) for val in line2[3:])

    # Skip the constraints
    for i in range(nconstraints):
        next(lines)

    # After the constraints we find:
    # - The Age and associated error
    # - The Mean track length and associated error
    # - The standard deviation and associated error

    (data["FTage"],
     data["FTageE"]) = (float(val) for val in next(lines).split())
    (data["MTL"],
     data["MTLE"])   = (float(val) for val in next(lines).split())
    (data["STDEV"],
     data["STDEVE"]) = (float(val) for val in next(lines).split())

    # After we find the counts Ns and Ni
    data["Ns"] = []
    data["Ni"] = []

    for row in range(ncounts):
        Ns, Ni = (int(val) for val in next(lines).split())
        data["Ns"].append(Ns)
        data["Ni"].append(Ni)

    # Finally the track lengths
    data["TL"] = []
    for row in range(ntl):
        data["TL"].append(float(next(lines)))

    return data

def h5store(filename, df, **kwargs):
    store = pd.HDFStore(filename)
    store.put('mydata', df)
    store.get_storer('mydata').attrs.metadata = kwargs
    store.close()

def h5load(store):
    data = store['mydata']
    metadata = store.get_storer('mydata').attrs.metadata
    return data, metadata

max_stars_repo_name: underworldcode/pyFTracks

max_issues_repo_name: rbeucher/pyFTracks


text:
''' Batched NDH navigation environment '''

import json
import os
import numpy as np
import math
import random
import networkx as nx
from collections import defaultdict
import copy

import MatterSim

from r2r.data_utils import load_nav_graphs
from r2r.data_utils import new_simulator
from r2r.data_utils import angle_feature, get_all_point_angle_feature
from r2r.env import R2RBatch

class NDHNavBatch(R2RBatch):
    def __init__(self, feat_db, instr_data, connectivity_dir,
        batch_size=64, angle_feat_size=4, seed=0, name=None, sel_data_idxs=None,
        use_player_path=False):
        super().__init__(feat_db, instr_data, connectivity_dir, batch_size=batch_size,
            angle_feat_size=angle_feat_size, seed=seed, name=name, sel_data_idxs=sel_data_idxs)
        self.use_player_path = use_player_path

    def _get_gt_trajs(self, data):
        return {x['instr_id']: (x['scan'], x['end_panos']) for x in data if 'end_panos' in x}

    def _next_minibatch(self, batch_size=None, **kwargs):
        super()._next_minibatch(batch_size=batch_size, **kwargs)
        batch = copy.deepcopy(self.batch)
        for item in batch:
            scan = item['scan']
            if 'end_panos' in item:
                use_player_path = self.use_player_path and np.random.rand() > 0.5
                if use_player_path:
                    item['path'] = item['nav_steps'][item['nav_idx']:]
                else:
                    end_pano = np.random.choice(item['end_panos'])
                    item['path'] = self.shortest_paths[scan][item['start_pano']][end_pano]
            else:
                item['path'] = [item['start_pano']]
            item['heading'] = item['start_heading']
        self.batch = batch

    def _get_obs(self, t=None, shortest_teacher=False):
        obs = []
        for i, (feature, state) in enumerate(self.env.getStates()):
            item = self.batch[i]
            base_view_id = state.viewIndex

            if feature is None:
                feature = np.zeros((36, 2048))

            # Full features
            candidate = self.make_candidate(feature, state.scanId, state.location.viewpointId, state.viewIndex)
            # [visual_feature, angle_feature] for views
            feature = np.concatenate((feature, self.angle_feature[base_view_id]), -1)

            scan = state.scanId
            viewpoint = state.location.viewpointId

            obs.append({
                'instr_id' : item['instr_id'],
                'scan' : scan,
                'viewpoint' : viewpoint,
                'viewIndex' : state.viewIndex,
                'heading' : state.heading,
                'elevation' : state.elevation,
                'feature' : feature,
                'candidate': candidate,
                'navigableLocations' : state.navigableLocations,
                'instr_encoding': item['instr_encoding'],
                'teacher' : self._teacher_path_action(state, item['path'], t=t, shortest_teacher=shortest_teacher),
                'gt_path' : item['path'],
            })

            # A3C reward. There are multiple gt end viewpoints on REVERIE.
            if 'end_panos' in item:
                min_dist = np.inf
                for end_pano in item['end_panos']:
                    min_dist = min(min_dist, self.shortest_distances[scan][viewpoint][end_pano])
            else:
                min_dist = 0
            obs[-1]['distance'] = min_dist
        return obs

    ############### Nav Evaluation ###############
    def _eval_item(self, scan, path, end_panos):
        scores = {}

        start_pano = path[0]
        end_panos = set(end_panos)
        shortest_distances = self.shortest_distances[scan]

        scores['trajectory_steps'] = len(path) - 1
        scores['trajectory_lengths'] = np.sum([shortest_distances[a][b] for a, b in zip(path[:-1], path[1:])])
        gt_lengths = np.min([shortest_distances[start_pano][end_pano] for end_pano in end_panos])

        # navigation: success is to arrive to a viewpoint in end_panos
        scores['success'] = float(path[-1] in end_panos)
        scores['oracle_success'] = float(any(x in end_panos for x in path))
        scores['spl'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01)
        scores['gp'] = gt_lengths - np.min([shortest_distances[path[-1]][end_pano] for end_pano in end_panos])

        return scores

    def eval_metrics(self, preds):
        ''' Evaluate each agent trajectory based on how close it got to the goal location
        the path contains [view_id, angle, vofv]'''
        print('eval %d predictions' % (len(preds)))

        metrics = defaultdict(list)
        for item in preds:
            instr_id = item['instr_id']
            traj = [x[0] for x in item['trajectory']]
            scan, end_panos = self.gt_trajs[instr_id]
            traj_scores = self._eval_item(scan, traj, end_panos)
            for k, v in traj_scores.items():
                metrics[k].append(v)
            metrics['instr_id'].append(instr_id)

        avg_metrics = {
            'steps': np.mean(metrics['trajectory_steps']),
            'lengths': np.mean(metrics['trajectory_lengths']),
            'sr': np.mean(metrics['success']) * 100,
            'oracle_sr': np.mean(metrics['oracle_success']) * 100,
            'spl': np.mean(metrics['spl']) * 100,
            'gp': np.mean(metrics['gp']),
        }
        return avg_metrics, metrics

ext: py

max_stars_repo_name: cshizhe/VLN-HAMT

max_issues_repo_name: biubiuisacat/VLN-HAMT

max_forks_repo_name: biubiuisacat/VLN-HAMT

reason: import numpy,import networkx


text:
import cv2
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon

from mmdet.core import BitmapMasks
from mmdet.core.bbox.transforms import mask2polybox

def show_anno(img, gt_bboxes, gt_masks, gt_keypoints, kp_skeleton='coco', save_img=False):
    if img.dtype == np.float32 or img.dtype == np.float64:
        img = img.astype(np.uint16) + 1

    if kp_skeleton != 'coco':
        raise NotImplementedError
    else:
        skeleton = np.array([[15, 13], [13, 11], [16, 14], [14, 12],
                             [11, 12], [5, 11], [6, 12], [5, 6], [5, 7],
                             [6, 8], [7, 9], [8, 10], [1, 2], [0, 1],
                             [0, 2], [1, 3], [2, 4], [3, 5], [4, 6]])
    gt_mask_polys = []
    masks = gt_masks.masks if isinstance(gt_masks, BitmapMasks) else gt_masks
    if gt_bboxes is None:
        gt_bboxes = []
        for mask in masks:
            polys, bbox = mask2polybox(mask)
            gt_bboxes.append(bbox)
            gt_mask_polys.append(polys)
    else:
        for mask in masks:
            polys, _ = mask2polybox(mask)
            gt_mask_polys.append(polys)

    num_instances = len(gt_bboxes)
    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    plt.imshow(img)
    ax = plt.gca()
    ax.set_autoscale_on(False)
    polygons = []
    color = []
    plt.text(10, 30, f'#obj={num_instances}', fontsize=14, color='r')

    for i in range(num_instances):
        # > bbox drawing
        text_pos = {}
        if np.size(gt_bboxes) > 0 and type(gt_bboxes[i]) in (np.ndarray, list):
            c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0]
            bbox = gt_bboxes[i]
            [x1, y1, x2, y2] = bbox
            poly = [[x1, y1], [x1, y2],
                    [x2, y2], [x2, y1]]
            np_poly = np.array(poly).reshape((4, 2))

            polygons.append(Polygon(np_poly))
            color.append(c)

            text_pos['pos'] = [x1, y1]

        # > mask drawing
        if np.size(gt_mask_polys) > 0 and type(gt_mask_polys[i]) in (np.ndarray, list):
            c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0]
            # one instance is likely to have multiple polygons
            for poly in gt_mask_polys[i]:
                polygons.append(Polygon(poly))
                color.append(c)

        # > keypoint drawing
        if np.size(gt_keypoints) > 0 and type(gt_keypoints[i]) in (np.ndarray, list):
            c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0]
            # turn skeleton into zero-based index
            sks = skeleton
            kps = gt_keypoints[i]
            x = kps[:, 0]
            y = kps[:, 1]
            v = kps[:, 2]
            if kp_skeleton == 'cmu':  # map back to coco type
                map_v = {2: 0, 0: 1, 1: 2, 3: 0}
                v = np.vectorize(map_v.get)(v)
            text_pos['num_kps'] = sum(v > 0)
            for sk in sks:
                if np.all(v[sk] > 0):
                    plt.plot(x[sk], y[sk], linewidth=2, color=c)
            # > `coco -> cmu`: not labeled={0:2}, labeled={1:0, 2:1}
            plt.plot(x[v > 0], y[v > 0], 'o', markersize=7, markerfacecolor=c, markeredgecolor='k',
                     markeredgewidth=1)

        if bool(text_pos):
            pos = text_pos['pos']
            num_kps = int(text_pos['num_kps'])
            plt.text(pos[0], pos[1] + 30, f'[{i}]k={num_kps}', fontsize=12, color='b')

    p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4)
    ax.add_collection(p)
    p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2)
    ax.add_collection(p)
    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    if save_img:
        plt.imsave('ann_resize_anno.jpg', cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    plt.show()

max_stars_repo_name: sokunmin/mmdetection

max_issues_repo_name: sokunmin/mmdetection

max_forks_repo_name: sokunmin/mmdetection

reason: import numpy


text:
import json
import time

import confluent_kafka
from streamz import Stream
from tornado import gen
from tornado.ioloop import IOLoop
import numpy as np

from minas.map_minas_support import *

kafkaConfig = {'bootstrap.servers': 'localhost:9092', 'group.id': 'streamz-minas'}
topic = 'test'

np.random.seed(300)
classes = list(map(mkClass, range(1000)))
clusters = sampleClusters(classes)
inputStream = loopExamplesIter(classes)
examples = list(zip(range(200), inputStream))

print('confluent_kafka.Producer()')
kprod = confluent_kafka.Producer(kafkaConfig)
print(kprod.list_topics().topics)
init = time.time()
counter = 0
for i, example in examples:
    # value = repr(example)[:-1] + ', item=' + repr(example.item) + ')'
    example.timestamp = time.time_ns()
    value = json.dumps({'example': example.__getstate__()})
    kprod.produce(topic=topic, value=value, key=str(i))
    counter += 1
elapsed = time.time() - init
print(f'kafka produce {elapsed} seconds, consumed {counter} items, {int(counter / elapsed)} i/s')

max_stars_repo_name: luis-puhl/minas-py

max_issues_repo_name: luis-puhl/minas-py

max_forks_repo_name: luis-puhl/minas-py


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import numpy as np
import tensorflow as tf
import argparse
import os
import glob
import cv2

from tensorpack import *
from tensorpack.tfutils.symbolic_functions import *
from tensorpack.tfutils.summary import *
from fabri import Fabri
from tensorflow.contrib.layers import variance_scaling_initializer

BATCH_SIZE = 48

def top_k_smooth_entropy(logits, labels, k):
    # https://arxiv.org/pdf/1512.00486.pdf
    k = k - 1
    inf = 1e4
    p_y = tf.reduce_sum(logits * tf.one_hot(labels, 100), 1)

    values, indices = tf.nn.top_k(logits - inf * tf.one_hot(labels, 100), k)

    mask_top_k = tf.reduce_sum(tf.one_hot(indices, 100), 1)

    exp_p_i_minus_p_y = tf.exp(logits - tf.expand_dims(p_y, 1) - inf * mask_top_k)

    exp_p_i_minus_p_y_smooth = exp_p_i_minus_p_y * (1 - mask_top_k)

    return tf.log(tf.reduce_sum(exp_p_i_minus_p_y_smooth, 1))

class Model(ModelDesc):

    def __init__(self, n):
        super(Model, self).__init__()
        self.n = n

    def _get_inputs(self):
        return [InputDesc(tf.float32, [None, 224, 224, 10, 3], 'input'),
                InputDesc(tf.int32, [None], 'label')]

    def _build_graph(self, inputs):
        image, label = inputs
        image = image / 255.0 - tf.constant([0.45834960097,0.44674252445,0.41352266842], dtype=tf.float32)
        assert tf.test.is_gpu_available()
        # image = tf.transpose(image, [0, 3, 1, 2])

        net = tf.layers.Conv3D(64, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv1')
        net = tf.layers.max_pooling3d(net, (1, 2, 2), (1, 2, 2))

        net = tf.layers.Conv3D(128, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv2')
        net = tf.layers.max_pooling3d(net, (2, 2, 2), (2, 2, 2))

        net = tf.layers.Conv3D(256, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv3')
        net = tf.layers.max_pooling3d(net, (2, 2, 2), (2, 2, 2))

        net = tf.layers.Conv3D(512, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv4')
        net = tf.layers.max_pooling3d(net, (2, 2, 2), (2, 2, 2))

        net = tf.layers.Conv3D(512, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv5a')
        net = tf.layers.Conv3D(512, (3, 3, 3), (1, 1, 1), 'same', activation=tf.nn.relu, use_bias=True,
                kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32),
                kernel_regularizer=0.0001, name='conv5b')
        net = tf.layers.max_pooling3d(net, (2, 2, 2), (2, 2, 2))

        net = FullyConnected('fc6', net, out_dim=4096, nl=tf.nn.relu)
        net = FullyConnected('fc7', net, out_dim=4096, nl=tf.nn.relu)

        logits = FullyConnected('linear', net, out_dim=14, nl=tf.identity)

        prob = tf.nn.softmax(logits, name='output')

        cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
        # cost = top_k_smooth_entropy(logits=logits, labels=label, k=5)
        cost = tf.reduce_mean(cost, name='cross_entropy_loss')

        wrong = prediction_incorrect(logits, label)
        # monitor training error
        add_moving_summary(tf.reduce_mean(wrong, name='train_error'))

        # weight decay on all W of fc layers
        wd_w = tf.train.exponential_decay(0.0002, get_global_step_var(),
                                          480000, 0.2, True)
        wd_cost = tf.multiply(wd_w, regularize_cost('.*/W', tf.nn.l2_loss), name='wd_cost')
        add_moving_summary(cost, wd_cost)

        add_param_summary(('.*/W', ['histogram']))   # monitor W
        self.cost = tf.add_n([cost, wd_cost], name='cost')

    def _get_optimizer(self):
        lr = get_scalar_var('learning_rate', 0.01, summary=True)
        opt = tf.train.MomentumOptimizer(lr, 0.9)
        return opt

def get_data(train_or_test):
    isTrain = train_or_test == 'train'
    ds = Fabri(train_or_test)
    pp_mean = ds.get_per_pixel_mean()
    if isTrain:
        augmentors = [
            imgaug.CenterPaste((256, 256)),
            imgaug.RandomCrop((224, 224)),
            imgaug.Flip(horiz=True),
            imgaug.Flip(vert=True),
	        imgaug.Brightness(20),
            imgaug.Contrast((0.6,1.4)),
            imgaug.MapImage(lambda x: x - pp_mean),
        ]
    else:
        augmentors = [
	    imgaug.CenterPaste((256, 256)),
            imgaug.RandomCrop((224, 224)),
           # imgaug.Flip(horiz=True),
           # imgaug.Flip(vert=True),
           # imgaug.Brightness(20),
           # imgaug.Contrast((0.6,1.4)),
            imgaug.MapImage(lambda x: x - pp_mean)
        ]
    ds = AugmentImageComponent(ds, augmentors)
    ds = BatchData(ds, BATCH_SIZE, remainder=not isTrain)
    if isTrain:
        ds = PrefetchData(ds, 3, 2)
    return ds

def get_config():
    log_dir = 'train_log/places-single-fisrt%s-second%s-max%s' % (str(args.drop_1), str(args.drop_2), str(args.max_epoch))
    logger.set_logger_dir(log_dir, action='n')

    # prepare dataset
    dataset_train = get_data('train')
    steps_per_epoch = dataset_train.size()
    dataset_test = get_data('val')

    return TrainConfig(
        dataflow=dataset_train,
        callbacks=[
            ModelSaver(),
            InferenceRunner(dataset_test,
                [ScalarStats('cost'), ClassificationError()]),
            ScheduledHyperParamSetter('learning_rate',
                                      [(1, 1e-2), (20, 1e-3), (40, 1e-4), (60, 1e-5), (95, 1e-6), (105, 1e-7)]),
        ],
        model=Model(n=args.n),
        steps_per_epoch=steps_per_epoch,
        max_epoch=args.max_epoch,
    )

def augment(img):

    def crop(img):
	i = np.random.randint(0, 16)
	j = np.random.randint(0, 16)
	img = img[i:i+160, j:j+160]
	return img

    def brightness(img, delta):
        old_dtype = img.dtype
        img = img.astype('float32')
        img += np.random.uniform(-delta, delta)
        img = np.clip(img, 0, 255)
        return img.astype(old_dtype)

    def contrast(img, factor_range):
        old_dtype = img.dtype
        img = img.astype('float32')
        mean = np.mean(img, axis=(0, 1), keepdims=True)
        r = np.random.uniform(factor_range[0], factor_range[1])
        img = (img - mean) * r + mean
        img = np.clip(img, 0, 255)
        return img.astype(old_dtype)

    def flip(img):
        if np.random.uniform(0, 1.0) > 0.5:
            img = cv2.flip(img, 0)
        if np.random.uniform(0, 1.0) > 0.5:
            img = cv2.flip(img, 1)
        return img

    img = crop(img)
    img = brightness(img, 20)
    img = contrast(img, (0.6, 1.4))
    img = flip(img)
    return img

def run(model_path, test_file, output_file=None):
    pred_config = PredictConfig(
        model=Model(n=args.n),
        session_init=tfutils.get_model_loader(model_path),
        input_names=['input'],
        output_names=['output'])
    predictor = OfflinePredictor(pred_config)
    output_f = open(output_file, 'w')
    for fname in glob.glob(os.path.join(test_file, '*')):
        im = cv2.imread(fname)
	im = cv2.resize(im, (176, 176))
	ims = [augment(im.copy().astype('float32')) for i in xrange(30)]
        output = np.squeeze(predictor([ims]))
	output = output.reshape((30, 100))
	output = np.squeeze(np.sum(output, axis=0))
        output = np.argsort(output)[-5:]
        fname = fname.split('data/')[-1]
        output_f.write('%s %d %d %d %d %d\n'%(fname, output[-1], output[-2], output[-3], output[-4], output[-5]))
        print('%s %d %d %d %d %d'%(fname, output[-1], output[-2], output[-3], output[-4], output[-5]))
    output_f.close()
    # im = cv2.imread(image_path)
    # assert im is not None
    # im = cv2.resize(im, (im.shape[1] // 16 * 16, im.shape[0] // 16 * 16))
    # outputs = predictor([[im.astype('float32')]])
    # if output is None:
    #     for k in range(6):
    #         pred = outputs[k][0]
    #         cv2.imwrite("out{}.png".format(
    #             '-fused' if k == 5 else str(k + 1)), pred * 255)
    # else:
    #     pred = outputs[5][0]
    #     cv2.imwrite(output, pred * 255)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', help='comma separated list of GPU(s) to use.') # nargs='*' in multi mode
    parser.add_argument('--load', help='load model')
    parser.add_argument('--drop_1',default=150, help='Epoch to drop learning rate to 0.01.') # nargs='*' in multi mode
    parser.add_argument('--drop_2',default=225,help='Epoch to drop learning rate to 0.001')
    parser.add_argument('--n',default=18, help='Number of units')
    parser.add_argument('--max_epoch',default=300,help='max epoch')
    parser.add_argument('--run', help='run model on images')
    parser.add_argument('--test_file', default='./data/test')
    parser.add_argument('--output_file', default='./output.txt')
    args = parser.parse_args()

    if args.gpu:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu

    config = get_config()
    if args.load:
        config.session_init = SaverRestore(args.load)
    if args.gpu:
        config.nr_tower = len(args.gpu.split(','))
    if args.run:
        run(args.load, args.test_file, args.output_file)
    else:
       SyncMultiGPUTrainer(config).train()


text:
import math
import random

import numpy as np
import pytest

from rlmolecule.alphazero.alphazero import AlphaZero
from rlmolecule.mcts.mcts import MCTS
from rlmolecule.molecule.builder.builder import MoleculeBuilder
from rlmolecule.tree_search.reward import LinearBoundedRewardFactory
from tests.qed_optimization_problem import QEDWithMoleculePolicy, QEDWithRandomPolicy

@pytest.fixture
def problem(request, engine):
    name = request.param
    builder = MoleculeBuilder(max_atoms=4,
                              min_atoms=1,
                              try_embedding=False,
                              sa_score_threshold=None,
                              stereoisomers=False,
                              canonicalize_tautomers=True
                              )

    if name == 'random':
        return QEDWithRandomPolicy(reward_class=LinearBoundedRewardFactory(), builder=builder, engine=engine)
    if name == 'MoleculePolicy':
        return QEDWithMoleculePolicy(reward_class=LinearBoundedRewardFactory(),
                                     builder=builder,
                                     engine=engine,
                                     features=8,
                                     num_heads=2,
                                     num_messages=1)

@pytest.fixture
def solver(request):
    name = request.param
    if name == 'MCTS':
        return MCTS
    if name == 'AlphaZero':
        return AlphaZero
    raise ValueError('Unknown problem type.')

def setup_game(solver, problem):
    game = solver(problem)
    root = game._get_root()
    return game, root

@pytest.mark.parametrize('solver,problem', [("MCTS", "random")], indirect=True)
def test_mcts_ucb_score(solver, problem):
    game, root = setup_game(solver, problem)
    game._evaluate([root])
    child = root.children[0]

    root.update(1.0)
    assert game._ucb_score(root, child) == math.inf
    assert root.visit_count == 1
    assert root.value == pytest.approx(1.)

    root.update(1.0)
    assert game._ucb_score(root, child) == math.inf
    assert root.visit_count == 2
    assert root.value == pytest.approx(1.)

    child.update(0.0)
    child.update(1.0)

    assert game._ucb_score(root, child) == pytest.approx(0.5 + game.ucb_constant * math.sqrt(math.log(2) / 2))

@pytest.mark.parametrize('solver,problem', [("AlphaZero", "random")], indirect=True)
def test_az_ucb_score(solver, problem):
    game, root = setup_game(solver, problem)
    game._evaluate([root])
    child = root.children[0]

    child.value_sum = 2.
    child.visit_count = 3

    root.value_sum = 3.
    root.visit_count = 4

    pb_c = np.log((root.visit_count + game._pb_c_base + 1) / game._pb_c_base) + game._pb_c_init
    pb_c *= np.sqrt(root.visit_count) / (child.visit_count + 1)

    assert game._ucb_score(root, child) == pytest.approx(pb_c * root.child_priors[child] + child.value)

@pytest.mark.parametrize('solver,problem', [("MCTS", "random"), ("AlphaZero", "random"),
                                            ("AlphaZero", "MoleculePolicy")],
                         indirect=True)
class TestMCTSwithMoleculeState:
    def test_reward(self, solver, problem):
        game, root = setup_game(solver, problem)
        game._expand(root)
        assert problem.reward_wrapper(root.children[-1]).raw_reward > 0.01

    def test_get_successors(self, solver, problem):
        game, root = setup_game(solver, problem)
        game._expand(root)
        children = root.children
        assert len(children) == 9
        assert children[-1].state._forced_terminal
        assert not children[0].state._forced_terminal

    def test_update(self, solver, problem):
        game, root = setup_game(solver, problem)

        root.update(2.)
        assert root.visit_count == 1
        assert root.value == pytest.approx(2.)

        root.update(4.)
        assert root.visit_count == 2
        assert root.value == pytest.approx(3.)

    def test_children(self, solver, problem):
        game, root = setup_game(solver, problem)
        game._expand(root)

        children = root.children
        assert root.children is not None
        assert children[-1].state.forced_terminal
        assert not children[0].state.forced_terminal

        children[0].update(4.)

        assert root.children[0].value == 4.
        assert root.children[0].visit_count == 1

    def test_evaluate(self, solver, problem):
        game, root = setup_game(solver, problem)
        reward = game._evaluate([root])

        if solver == MCTS:
            assert reward.raw_reward > 0.

    def test_mcts_sample(self, solver, problem):
        random.seed(42)

        game, root = setup_game(solver, problem)
        game.sample(root, 10)

        assert root.visit_count == 10
        assert root.value > 0.1

        game.sample(root, 100000, timeout=0.1)
        assert root.visit_count <= 100000

    def test_run_mcts(self, solver, problem):

        random.seed(42)
        game, root = setup_game(solver, problem)

        history, reward = game.run(num_mcts_samples=5)
        assert len(history) > 1
        assert np.isfinite(reward.raw_reward)

        try:
            assert reward.raw_reward == problem.reward_wrapper(history[-1]).raw_reward
        except AttributeError:  # Handle alphazero's history object
            assert reward.raw_reward == problem.reward_wrapper(history[-1][0]).raw_reward

max_stars_repo_name: harrysorensennrel/rlmolecule

max_issues_repo_name: harrysorensennrel/rlmolecule

max_forks_repo_name: harrysorensennrel/rlmolecule


text:
import os
import time
import cdt
import argparse
import pandas as pd
import numpy as np
import networkx as nx

import sys
sys.path.append(".")
from gran_dag.plot import plot_adjacency
from gran_dag.utils.save import dump
from gran_dag.utils.metrics import edge_errors
from gran_dag.data import DataManagerFile
from cam import CAM_with_score

def main(opt, metrics_callback=None, plotting_callback=None):
    time0 = time.time()
    opt.model = "cam"

    # load data
    train_data = DataManagerFile(opt.data_path, opt.i_dataset, opt.train_samples, opt.test_samples, train=True,
                                 normalize=opt.normalize_data, random_seed=opt.random_seed)
    test_data = DataManagerFile(opt.data_path, opt.i_dataset, opt.train_samples, opt.test_samples, train=False,
                                normalize=opt.normalize_data, mean=train_data.mean, std=train_data.std,
                                random_seed=opt.random_seed)
    gt_dag = train_data.adjacency.detach().cpu().numpy()
    train_data_pd = pd.DataFrame(train_data.dataset.detach().cpu().numpy())

    # apply CAM
    if test_data.dataset is not None:
        test_data_pd = pd.DataFrame(test_data.dataset.detach().cpu().numpy())
        obj = CAM_with_score(opt.score, opt.cutoff, opt.variable_sel, opt.sel_method,
                             opt.pruning, opt.prune_method)
        dag, train_score, val_score = obj.get_score(train_data_pd, test_data_pd)
    else:
        obj = cdt.causality.graph.CAM(opt.score, opt.cutoff, opt.variable_sel, opt.sel_method, opt.pruning,
                                      opt.prune_method)
        dag = obj.predict(train_data_pd)
        train_score, val_score = None, None
    dag = nx.to_numpy_matrix(dag)

    # Compute SHD and SID metrics
    sid = float(cdt.metrics.SID(target=gt_dag, pred=dag))
    shd = float(cdt.metrics.SHD(target=gt_dag, pred=dag, double_for_anticausal=False))
    shd_cpdag = float(cdt.metrics.SHD_CPDAG(target=gt_dag, pred=dag))
    fn, fp, rev = edge_errors(dag, gt_dag)
    timing = time.time() - time0

    #save
    if not os.path.exists(opt.exp_path):
        os.makedirs(opt.exp_path)

    if metrics_callback is not None:
        metrics_callback(stage="cam", step=0,
                         metrics={"train_score": train_score, "val_score": val_score, "sid": sid, "shd": shd,
                                  "shd_cpdag": shd_cpdag, "fn": fn, "fp": fp, "rev": rev}, throttle=False)

    dump(opt, opt.exp_path, 'opt')
    dump(timing, opt.exp_path, 'timing', True)
    dump(train_score, opt.exp_path, 'train_score', True)
    dump(val_score, opt.exp_path, 'test_score', True)
    dump(sid, opt.exp_path, 'sid', True)
    dump(shd, opt.exp_path, 'shd', True)
    np.save(os.path.join(opt.exp_path, "DAG"), dag)

    plot_adjacency(gt_dag, dag, opt.exp_path)

def _print_metrics(stage, step, metrics, throttle=None):
    for k, v in metrics.items():
        print("    %s:" % k, v)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument('--data-path', type=str, default=None,
                        help='Path to data files')
    parser.add_argument('--i-dataset', type=str, default=None,
                        help='dataset index')
    parser.add_argument('--exp-path', type=str, default='exp',
                        help='Path to experiments')
    # parser.add_argument('--score', type=str, default='nonlinear',
    #                     help='Score used to fit the gaussian processes')

    # Variable selection (PNS)
    parser.add_argument('--variable-sel', action="store_true",
                        help='Perform a variable selection step')
    # parser.add_argument('--sel-method', type=str, default='gamboost',
    #                     help='Method used for variable selection')
    parser.add_argument('--cutoff', type=float, default=0.001,
                        help='Threshold value for vaiable selection')

    # Pruning
    parser.add_argument('--pruning', action="store_true", default=True,
                        help='Perform an initial pruning step')
    # parser.add_argument('--prune-method', type=str, default="gam",
    #                     help='after to-dag or pruning, retrain model from scratch before reporting nll-val')
    parser.add_argument('--train-samples', type=int, default=0.8,
                        help='Number of samples used for training (default is 80% of the total size)')
    parser.add_argument('--test-samples', type=int, default=None,
                        help='Number of samples used for testing (default is whatever is not used for training)')
    parser.add_argument('--normalize-data', action="store_true",
                        help='(x - mu) / std')
    parser.add_argument('--random-seed', type=int, default=42,
                        help="Random seed for pytorch and numpy")

    opt = parser.parse_args()
    opt.score = 'nonlinear'
    opt.sel_method = 'gamboost'
    opt.prune_method = 'gam'

    main(opt, metrics_callback=_print_metrics)

max_stars_repo_name: mnchsmn/GraN-DAG

max_issues_repo_name: mnchsmn/GraN-DAG

max_forks_repo_name: mnchsmn/GraN-DAG


text:
# coding: utf-8
#
# This code is part of lattpy.
#
# Copyright (c) 2021, Dylan Jones
#
# This code is licensed under the MIT License. The copyright notice in the
# LICENSE file in the root directory and this permission notice shall
# be included in all copies or substantial portions of the Software.

"""Spatial algorithms and data structures."""

import math
import numpy as np
import itertools
import matplotlib.pyplot as plt
from scipy.spatial import cKDTree, Voronoi
from typing import Iterable, Sequence, Optional, Union
from .utils import ArrayLike, min_dtype, chain
from .plotting import draw_points, draw_vectors, draw_lines, draw_surfaces

__all__ = [
    "distance", "interweave", "vindices", "vrange", "cell_size", "cell_volume",
    "compute_vectors", "compute_neighbors", "KDTree", "VoronoiTree", "WignerSeitzCell",
    "rx", "ry", "rz", "rotate2d", "rotate3d", "build_periodic_translation_vector"
]

def distance(r1: ArrayLike, r2: ArrayLike, decimals: Optional[int] = None) -> float:
    """ Calculates the euclidian distance bewteen two points.

    Parameters
    ----------
    r1: array_like
        First input point.
    r2: array_like
        Second input point of matching size.
    decimals: int, optional
        Optional decimals to round distance to.

    Returns
    -------
    distance: float
    """
    dist = math.sqrt(np.sum(np.square(r1 - r2)))
    if decimals is not None:
        dist = round(dist, decimals)
    return dist

def interweave(arrays: Sequence[np.ndarray]) -> np.ndarray:
    """ Interweaves multiple arrays along the first axis

    Example
    -------
    >>> arr1 = np.array([[1, 1], [3, 3], [5, 5]])
    >>> arr2 = np.array([[2, 2], [4, 4], [6, 6]])
    >>> interweave([arr1, arr2])
    array([[1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6]])

    Parameters
    ----------
    arrays: (M) Sequence of (N, ...) array_like
        The input arrays to interwave. The shape of all arrays must match.

    Returns
    -------
    interweaved: (M*N, ....) np.ndarray
    """
    shape = list(arrays[0].shape)
    shape[0] = sum(x.shape[0] for x in arrays)
    result = np.empty(shape, dtype=arrays[0].dtype)
    n = len(arrays)
    for i, arr in enumerate(arrays):
        result[i::n] = arr
    return result

def vindices(limits: Iterable[Sequence[int]], sort_axis: Optional[int] = 0,
             dtype: Optional[Union[int, str, np.dtype]] = None) -> np.ndarray:
    """ Return an array representing the indices of a d-dimensional grid.

    Parameters
    ----------
    limits: (D, 2) array_like
        The limits of the indices for each axis.
    sort_axis: int, optional
        Optional axis that is used to sort indices.
    dtype: int or str or np.dtype, optional
        Optional data-type for storing the lattice indices. By default the given limits
        are checked to determine the smallest possible data-type.

    Returns
    -------
    vectors: (N, D) np.ndarray
    """
    if dtype is None:
        dtype = min_dtype(limits, signed=True)
    limits = np.asarray(limits)
    dim = limits.shape[0]

    # Create meshgrid reshape grid to array of indices

    # version 1:
    # axis = np.meshgrid(*(np.arange(*lim, dtype=dtype) for lim in limits))
    # nvecs = np.asarray([np.asarray(a).flatten("F") for a in axis]).T

    # version 2:
    # slices = [slice(lim[0], lim[1], 1) for lim in limits]
    # nvecs = np.mgrid[slices].astype(dtype).reshape(dim, -1).T

    # version 3:
    size = limits[:, 1] - limits[:, 0]
    nvecs = np.indices(size, dtype=dtype).reshape(dim, -1).T + limits[:, 0]

    # Optionally sort indices along given axis
    if sort_axis is not None:
        nvecs = nvecs[np.lexsort(nvecs.T[[sort_axis]])]

    return nvecs

def vrange(start=None, *args,
           dtype: Optional[Union[int, str, np.dtype]] = None,
           sort_axis: Optional[int] = 0, **kwargs) -> np.ndarray:
    """ Return evenly spaced vectors within a given interval.

    Parameters
    ----------
    start: array_like, optional
        The starting value of the interval. The interval includes this value.
        The default start value is 0.
    stop: array_like
        The end value of the interval.
    step: array_like, optional
        Spacing between values. If `start` and `stop` are sequences and the `step`
        is a scalar the given step size is used for all dimensions of the vectors.
        The default step size is 1.
    sort_axis: int, optional
        Optional axis that is used to sort indices.
    dtype: dtype, optional
        The type of the output array.  If `dtype` is not given, infer the data
        type from the other input arguments.

    Returns
    -------
    vectors: (N, D) np.ndarray
    """
    # parse arguments
    if len(args) == 0:
        stop = start
        start = np.zeros_like(stop)
        step = kwargs.get("step", 1.0)
    elif len(args) == 1:
        stop = args[0]
        step = kwargs.get("step", 1.0)
    else:
        stop, step = args

    start = np.atleast_1d(start)
    stop = np.atleast_1d(stop)
    if step is None:
        step = np.ones_like(start)
    elif not hasattr(step, "__len__"):
        step = np.ones_like(start) * step

    # Create grid and reshape to array of vectors
    slices = [slice(i, f, s) for i, f, s in zip(start, stop, step)]
    array = np.mgrid[slices].reshape(len(slices), -1).T
    # Optionally sort array along given axis
    if sort_axis is not None:
        array = array[np.lexsort(array.T[[sort_axis]])]

    return array if dtype is None else array.astype(dtype)

def cell_size(vectors: ArrayLike) -> np.ndarray:
    """ Computes the shape of the box spawned by the given vectors.

    Parameters
    ----------
    vectors: array_like
        The basis vectors defining the cell.

    Returns
    -------
    size: np.ndarray
    """
    max_values = np.max(vectors, axis=0)
    min_values = np.min(vectors, axis=0)
    min_values[min_values > 0] = 0
    return max_values - min_values

def cell_volume(vectors: ArrayLike) -> float:
    r""" Computes the volume of the unit cell defined by the primitive vectors.

    The volume of the unit-cell in two and three dimensions is defined by
    .. math::
        V_{2d} = \abs{a_1 \cross a_2}, \quad V_{3d} = a_1 \cdot \abs{a_2 \cross a_3}

    For higher dimensions the volume is computed using the determinant:
    .. math::
        V_{d} = \sqrt{\det{A A^T}}
    where .math:`A` is the array of vectors.

    Parameters
    ----------
    vectors: array_like
        The basis vectors defining the cell.

    Returns
    -------
    vol: float
    """
    dim = len(vectors)
    if dim == 1:
        v = float(vectors)
    elif dim == 2:
        v = np.cross(vectors[0], vectors[1])
    elif dim == 3:
        cross = np.cross(vectors[1], vectors[2])
        v = np.dot(vectors[0], cross)
    else:
        v = np.sqrt(np.linalg.det(np.dot(vectors.T, vectors)))
    return abs(v)

def build_periodic_translation_vector(indices, axs):
    limits = np.array([np.min(indices, axis=0), np.max(indices, axis=0)])
    nvec = np.zeros(indices.shape[1] - 1, dtype=np.int)
    for ax in np.atleast_1d(axs):
        nvec[ax] = np.floor(limits[1][ax]) + 1
    return nvec

def compute_vectors(a: float, b: Optional[float] = None, c: Optional[float] = None,
                    alpha: Optional[float] = None, beta: Optional[float] = None,
                    gamma: Optional[float] = None,
                    decimals: Optional[int] = 0) -> np.ndarray:
    """ Computes lattice vectors by the lengths and angles. """
    if b is None and c is None:
        vectors = [a]
    elif c is None:
        alpha = np.deg2rad(alpha)
        ax = a
        bx = b * np.cos(alpha)
        by = b * np.sin(alpha)
        vectors = np.array([
            [ax, 0],
            [bx, by]
        ])
    else:
        alpha = np.deg2rad(alpha)
        beta = np.deg2rad(beta)
        gamma = np.deg2rad(gamma)
        ax = a
        bx = b * np.cos(gamma)
        by = b * np.sin(gamma)
        cx = c * np.cos(beta)
        cy = (abs(c) * abs(b) * np.cos(alpha) - bx * cx) / by
        cz = np.sqrt(c ** 2 - cx ** 2 - cy ** 2)
        vectors = np.array([
            [ax, 0, 0],
            [bx, by, 0],
            [cx, cy, cz]
        ])
    if decimals:
        vectors = np.round(vectors, decimals=decimals)
    return vectors

# noinspection PyUnresolvedReferences
class KDTree(cKDTree):
    """Simple wrapper of scipy's cKTree with global query settings."""

    def __init__(self, points, k=1, max_dist=np.inf, eps=0., p=2):
        super().__init__(points)
        self.max_dist = max_dist
        self.k = k
        self.p = p
        self.eps = eps

    def query_ball_point(self, x, r):
        return super().query_ball_point(x, r, self.p, self.eps)

    def query_ball_tree(self, other, r):
        return super().query_ball_tree(other, r, self.p, self.eps)

    def query_pairs(self, r):
        return super().query_pairs(r, self.p, self.eps)

    def query(self, x=None, num_jobs=1, decimals=None, include_zero=False, compact=True):
        x = self.data if x is None else x
        distances, neighbors = super().query(x, self.k, self.eps, self.p, self.max_dist, num_jobs)

        # Remove zero-distance neighbors and convert dtype
        if not include_zero and np.all(distances[:, 0] == 0):
            distances = distances[:, 1:]
            neighbors = neighbors[:, 1:]
        neighbors = neighbors.astype(min_dtype(self.n, signed=False))

        # Remove neighbors with distance larger than max_dist
        if self.max_dist < np.inf:
            invalid = distances > self.max_dist
            neighbors[invalid] = self.n
            distances[invalid] = np.inf

        # Remove all invalid columns
        if compact:
            mask = np.any(distances != np.inf, axis=0)
            neighbors = neighbors[:, mask]
            distances = distances[:, mask]

        # Round distances
        if decimals is not None:
            distances = np.round(distances, decimals=decimals)

        return neighbors, distances

def compute_neighbors(positions, k=20, max_dist=np.inf, num_jobs=1, decimals=None, eps=0.,
                      include_zero=False, compact=True, x=None):
    # Build tree and query neighbors
    x = positions if x is None else x
    tree = KDTree(positions, k=k, max_dist=max_dist, eps=eps)
    distances, neighbors = tree.query(x, num_jobs, decimals, include_zero, compact)
    return neighbors, distances

class VoronoiTree:

    def __init__(self, points):
        points = np.asarray(points)
        dim = points.shape[1]
        edges = list()
        if dim == 1:
            vertices = points / 2
            idx = np.where((vertices == np.zeros(vertices.shape[1])).all(axis=1))[0]
            vertices = np.delete(vertices, idx)
            vertices = np.atleast_2d(vertices).T
        else:
            vor = Voronoi(points)
            # Save only finite vertices
            vertices = vor.vertices  # noqa
            for pointidx, simplex in zip(vor.ridge_points, vor.ridge_vertices):  # noqa
                simplex = np.asarray(simplex)
                if np.all(simplex >= 0):
                    edges.append(simplex)

        self.dim = dim
        self.points = points
        self.edges = edges
        self.vertices = vertices
        self.tree = cKDTree(points)  # noqa
        self.origin = self.query(np.zeros(dim))

    def query(self, x, k=1, eps=0):
        return self.tree.query(x, k, eps)  # noqa

    def draw(self, ax=None, color="C0", size=3, lw=1, alpha=0.15, point_color="k", point_size=3,
             draw_data=True, points=True, draw=True, fill=True):

        if ax is None:
            fig = plt.figure()
            ax = fig.add_subplot(111, projection="3d" if self.dim == 3 else None)

        if draw_data:
            draw_points(ax, self.points, size=point_size, color=point_color)
            if self.dim > 1:
                draw_vectors(ax, self.points, lw=0.5, color=point_color)

        if points:
            draw_points(ax, self.vertices, size=size, color=color)

        if self.dim == 2 and draw:
            segments = np.array([self.vertices[i] for i in self.edges])
            draw_lines(ax, segments, color=color, lw=lw)
        elif self.dim == 3:
            if draw:
                segments = np.array([self.vertices[np.append(i, i[0])] for i in self.edges])
                draw_lines(ax, segments, color=color, lw=lw)
            if fill:
                surfaces = np.array([self.vertices[i] for i in self.edges])
                draw_surfaces(ax, surfaces, color=color, alpha=alpha)

        if self.dim == 3:
            ax.set_aspect("equal")
        else:
            ax.set_aspect("equal", "box")

        return ax

    def __repr__(self):
        return f"{self.__class__.__name__}(vertices: {len(self.vertices)})"

    def __str__(self):
        return f"vertices:\n{self.vertices}\n" \
               f"egdes:\n{self.edges}"

class WignerSeitzCell(VoronoiTree):

    def __init__(self, points):
        super().__init__(points)
        self._root = self.query(np.zeros(self.dim))[1]

    @property
    def limits(self):
        return np.array([np.min(self.vertices, axis=0), np.max(self.vertices, axis=0)]).T

    @property
    def size(self):
        return self.limits[1] - self.limits[0]

    def check(self, points):
        cells = np.asarray(self.query(points)[1])
        return cells == self._root

    def arange(self, steps, offset=0.):
        limits = self.limits * (1 + offset)
        steps = [steps] * self.dim if not hasattr(steps, "__len__") else steps
        return [np.arange(*lims, step=step) for lims, step in zip(limits, steps)]

    def linspace(self, nums, offset=0.):
        limits = self.limits * (1 + offset)
        nums = [nums] * self.dim if not hasattr(nums, "__len__") else nums
        return [np.linspace(*lims, num=num) for lims, num in zip(limits, nums)]

    def meshgrid(self, nums=None, steps=None, offset=0., check=True):
        if nums is not None:
            grid = np.array(np.meshgrid(*self.linspace(nums, offset)))
        elif steps is not None:
            grid = np.array(np.meshgrid(*self.arange(steps, offset)))
        else:
            raise ValueError("Either the number of points or the step size muste be specified")

        if check:
            lengths = grid.shape[1:]
            dims = range(len(lengths))
            for item in itertools.product(*[range(n) for n in lengths]):
                point = np.array([grid[d][item] for d in dims])
                if not self.check(point):
                    for d in dims:
                        grid[d][item] = np.nan
        return grid

    def symmetry_points(self):
        origin = np.zeros((1,))
        corners = self.vertices.copy()
        face_centers = None
        if self.dim == 1:
            return origin, corners, None, None
        elif self.dim == 2:
            edge_centers = np.zeros((len(self.edges), 2))
            for i, simplex in enumerate(self.edges):
                p1, p2 = self.vertices[simplex]
                edge_centers[i] = p1 + (p2 - p1) / 2
        elif self.dim == 3:
            edge_centers = list()
            face_centers = list()
            for i, simplex in enumerate(self.edges):
                edges = self.vertices[simplex]
                # compute face centers
                face_centers.append(np.mean(edges, axis=0))
                # compute edge centers
                for p1, p2 in chain(edges, cycle=True):
                    edge_centers.append(p1 + (p2 - p1) / 2)
            edge_centers = np.asarray(edge_centers)
            face_centers = np.asarray(face_centers)
        else:
            raise NotImplementedError()
        return origin, corners, edge_centers, face_centers

def rx(theta: float) -> np.ndarray:
    """X-Rotation matrix."""
    sin, cos = np.sin(theta), np.cos(theta)
    return np.array([[1, 0, 0], [0, cos, -sin], [0, sin, cos]])

def ry(theta: float) -> np.ndarray:
    """Y-Rotation matrix."""
    sin, cos = np.sin(theta), np.cos(theta)
    return np.array([[cos, 0, sin], [0, 1, 0], [-sin, 0, +cos]])

def rz(theta: float) -> np.ndarray:
    """Z-Rotation matrix."""
    sin, cos = np.sin(theta), np.cos(theta)
    return np.array([[cos, -sin, 0], [sin, cos, 0], [0, 0, 1]])

def rot(thetax: float = 0., thetay: float = 0., thetaz: float = 0.) -> np.ndarray:
    """General rotation matrix"""
    r = np.eye(3)
    if thetaz:
        r = np.dot(r, rz(thetaz))
    if thetay:
        r = np.dot(r, ry(thetay))
    if thetax:
        r = np.dot(r, rz(thetax))
    return r

def rotate2d(a, theta):
    """Applies the z-rotation matrix to a 2D point"""
    return np.dot(a, rz(theta)[:2, :2])

def rotate3d(a, thetax=0., thetay=0., thetaz=0.):
    """Applies the general rotation matrix to a 3D point"""
    return np.dot(a, rot(thetax, thetay, thetaz))

max_stars_repo_name: dylanljones/lattpy

max_issues_repo_name: dylanljones/lattpy

max_forks_repo_name: dylanljones/lattpy

reason: import numpy,from scipy


text:
from numba import njit

@njit
def find_divisible_subsets(residues: list[int], n: int) -> list[int]:
    subsets = [0] * n
    subsets[0] = 1

    for i in range(1, len(residues)):
        subsets = [(value + subsets[(j - residues[i]) % len(subsets)]) % 10**16
                   for (j, value) in enumerate(subsets)]

    subsets[0] -= 1

    return subsets

def main() -> None:
    m = 250_250
    n = 250

    residues = [pow(i, i, n) for i in range(0, m + 1)]
    subsets = find_divisible_subsets(residues, n)

    print(subsets[0])

if __name__ == '__main__':
    main()

max_stars_repo_name: GustasG/project-euler

max_issues_repo_name: GustasG/project-euler

max_forks_repo_name: GustasG/project-euler

reason: from numba


text:
from ScopeFoundry.data_browser import HyperSpectralBaseView
import h5py
import numpy as np
from .scalebars import SEMScaleBar
import pyqtgraph as pg

class HyperSpecCLH5View(HyperSpectralBaseView):

    name = 'hyperspec_cl_h5'

    def is_file_supported(self, fname):
        return "_hyperspec_cl.h5" in fname

    def setup(self):
        self.settings.New('show_description', dtype=bool, initial=False)
        self.settings.show_description.add_listener(self.update_display)
        HyperSpectralBaseView.setup(self)

    def load_data(self, fname):
        self.dat = h5py.File(fname)
        self.M = self.dat['measurement/hyperspec_cl']
        self.spec_map = self.M['spec_map'][0, 0, :, :, :]
        (self.nx, ny, nw) = self.spec_map.shape
        remcon = self.dat['hardware/sem_remcon/settings'].attrs
        self.spec_x_array = np.array(self.M['wls'])
        self.mag = remcon['magnification'] / \
            (self.M['settings'].attrs['h_span']/20)
        self.hyperspec_data = self.spec_map
        self.display_image = self.spec_map.sum(axis=-1)
        desc = self.M['settings'].attrs['description']
        self.desc_txt = pg.TextItem(text=desc)

    def update_display(self):
        if self.display_image is not None and hasattr(self, 'dat'):
            # pyqtgraph axes are x,y, but data is stored in (frame, y,x, time),
            # so we need to transpose
            self.imview.getImageItem().setImage(self.display_image.T)

            if hasattr(self, 'scalebar'):
                self.imview.getView().removeItem(self.scalebar)

            self.scalebar = SEMScaleBar(mag=self.mag, num_px=self.nx)
            self.scalebar.setParentItem(self.imview.getView())
            self.scalebar.anchor((1, 1), (1, 1), offset=(-20, -20))

            if hasattr(self, 'desc_txt'):
                if self.settings['show_description']:
                    self.imview.getView().addItem(self.desc_txt)
                else:
                    self.imview.getView().removeItem(self.desc_txt)

            self.on_change_rect_roi()
            self.on_update_circ_roi()

    def reset(self):
        if hasattr(self, 'dat'):
            self.dat.close()

        if hasattr(self, 'scalebar'):
            self.imview.getView().removeItem(self.scalebar)
            del self.scalebar

        if hasattr(self, 'desc_txt'):
            self.imview.getView().removeItem(self.desc_txt)
            del self.desc_txt

        HyperSpectralBaseView.reset(self)

    def scan_specific_setup(self):
        self.spec_plot.setLabel('left', 'Intensity', units='counts')
        self.spec_plot.setLabel('bottom', 'Wavelength', units='nm')

max_stars_repo_name: ScopeFoundry/FoundryDataBrowser

max_issues_repo_name: ScopeFoundry/FoundryDataBrowser

max_forks_repo_name: ScopeFoundry/FoundryDataBrowser


text:
#!/usr/bin/env python
import sys, os
scriptdir = os.path.split(__file__)[0]
sys.path.insert(0, os.path.join(scriptdir, '..'))

import numpy
from PIL import Image
import ospray

W = 1024
H = 768

cam_pos = (0.0, 0.0, 0.0)
cam_up = (0.0, 1.0, 0.0)
cam_view = (0.1, 0.0, 1.0)

vertex = numpy.array([
   [-1.0, -1.0, 3.0],
   [-1.0, 1.0, 3.0],
   [1.0, -1.0, 3.0],
   [0.1, 0.1, 0.3]
], dtype=numpy.float32)

color = numpy.array([
    [0.9, 0.5, 0.5, 1.0],
    [0.8, 0.8, 0.8, 1.0],
    [0.8, 0.8, 0.8, 1.0],
    [0.5, 0.9, 0.5, 1.0]
], dtype=numpy.float32)

index = numpy.array([
    [0, 1, 2], [1, 2, 3]
], dtype=numpy.uint32)

ospray.load_module('denoiser')

ospray.init(sys.argv)

camera = ospray.Camera('perspective')
camera.set_param('aspect', W/H)
camera.set_param('position', cam_pos)
camera.set_param('direction', cam_view)
camera.set_param('up', cam_up)
camera.commit()

mesh = ospray.Geometry('mesh')
mesh.set_param('vertex.position', ospray.copied_data_constructor_vec(vertex))
mesh.set_param('vertex.color', ospray.copied_data_constructor_vec(color))
mesh.set_param('index', ospray.copied_data_constructor_vec(index))
mesh.commit()

gmodel = ospray.GeometricModel(mesh)
gmodel.commit()

group = ospray.Group()
group.set_param('geometry', [gmodel])
group.commit()

instance = ospray.Instance(group)
instance.commit()

material = ospray.Material('pathtracer', 'obj')
material.commit()

gmodel.set_param('material', material)
gmodel.commit()

world = ospray.World()
world.set_param('instance', [instance])

light = ospray.Light('ambient')
light.set_param('color', (1.0, 1, 1))
light.set_param('intensity', 1.0)
light.commit()

world.set_param('light', [light])
world.commit()

renderer = ospray.Renderer('pathtracer')
renderer.set_param('backgroundColor', (1.0, 1.0, 1.0, 1.0))
renderer.commit()

denoise = ospray.ImageOperation('denoiser')
denoise.commit()

format = ospray.OSP_FB_RGBA32F
# Denoiser needs albedo and normal passes for better performance
channels = int(ospray.OSP_FB_COLOR) | int(ospray.OSP_FB_ACCUM) | int(ospray.OSP_FB_ALBEDO) | int(ospray.OSP_FB_NORMAL)

framebuffer = ospray.FrameBuffer(W, H, format, channels)
framebuffer.set_param('imageOperation', [denoise])
framebuffer.commit()

framebuffer.clear()

# Render only a single sample, should be nicely noisy
future = framebuffer.render_frame(renderer, camera, world)
future.wait()

colors = framebuffer.get(ospray.OSP_FB_COLOR, (W,H), format)
print(colors.shape, colors.dtype)

colors = (colors * 255).astype('uint8')

img = Image.frombuffer('RGBA', (W,H), colors, 'raw', 'RGBA', 0, 1)
img = img.transpose(Image.FLIP_TOP_BOTTOM)
img.save('with-denoise.png')

# Render again without denoising

framebuffer = ospray.FrameBuffer(W, H, format, channels)
framebuffer.commit()

framebuffer.clear()

future = framebuffer.render_frame(renderer, camera, world)
future.wait()

colors = framebuffer.get(ospray.OSP_FB_COLOR, (W,H), format)
print(colors.shape, colors.dtype)

colors = (colors * 255).astype('uint8')

img = Image.frombuffer('RGBA', (W,H), colors, 'raw', 'RGBA', 0, 1)
img = img.transpose(Image.FLIP_TOP_BOTTOM)
img.save('without-denoise.png')

max_stars_repo_name: surfsara-visualization/ospray-python

max_issues_repo_name: surfsara-visualization/ospy

max_forks_repo_name: surfsara-visualization/ospray-python


text:
import os

import torchvision.utils
from PIL import Image

import torch
from torch.utils.data import Dataset, DataLoader
from torch import nn
from torchvision import transforms

import numpy as np
import matplotlib.pyplot as plt

class CustomImageDataset(Dataset):
    def read_data_set(self):

        all_img_files = []
        all_labels = []

        class_names = os.walk(self.data_set_path).__next__()[1]

        for index, class_name in enumerate(class_names):
            label = index
            img_dir = os.path.join(self.data_set_path, class_name)
            img_files = os.walk(img_dir).__next__()[2]

            for img_file in img_files:
                img_file = os.path.join(img_dir, img_file)
                img = Image.open(img_file)
                if img is not None:
                    all_img_files.append(img_file)
                    all_labels.append(label)

        return all_img_files, all_labels, len(all_img_files), len(class_names)

    def __init__(self, data_set_path, transforms=None):
        self.data_set_path = data_set_path
        self.image_files_path, self.labels, self.length, self.num_classes = self.read_data_set()
        self.transforms = transforms

    def __getitem__(self, index):
        image = Image.open(self.image_files_path[index])
        image = image.convert("RGB")

        if self.transforms is not None:
            image = self.transforms(image)

        return {'image': image, 'label': self.labels[index]}

    def __len__(self):
        return self.length

class CustomConvNet(nn.Module):
    def __init__(self, num_classes):
        super(CustomConvNet, self).__init__()

        self.layer1 = self.conv_module(3, 16)
        self.layer2 = self.conv_module(16, 32)
        self.layer3 = self.conv_module(32, 64)
        self.layer4 = self.conv_module(64, 128)
        self.layer5 = self.conv_module(128, 256)
        self.gap = self.global_avg_pool(256, num_classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = self.layer5(out)
        out = self.gap(out)
        out = out.view(-1, num_classes)

        return out

    def conv_module(self, in_num, out_num):
        return nn.Sequential(
            nn.Conv2d(in_num, out_num, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(out_num),
            nn.LeakyReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))

    def global_avg_pool(self, in_num, out_num):
        return nn.Sequential(
            nn.Conv2d(in_num, out_num, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(out_num),
            nn.LeakyReLU(),
            nn.AdaptiveAvgPool2d((1, 1)))

hyper_param_epoch = 20
hyper_param_batch = 8
hyper_param_learning_rate = 0.001

transforms_train = transforms.Compose([transforms.Resize((128, 128)),
                                       transforms.RandomRotation(10.),
                                       transforms.ToTensor()])

transforms_test = transforms.Compose([transforms.Resize((128, 128)),
                                      transforms.ToTensor()])

train_data_set = CustomImageDataset(data_set_path="./data/train", transforms=transforms_train)
train_loader = DataLoader(train_data_set, batch_size=hyper_param_batch, shuffle=True)

test_data_set = CustomImageDataset(data_set_path="./data/test", transforms=transforms_test)
test_loader = DataLoader(test_data_set, batch_size=hyper_param_batch, shuffle=True)

if not (train_data_set.num_classes == test_data_set.num_classes):
    print("error: Numbers of class in training set and test set are not equal")
    exit()

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

num_classes = train_data_set.num_classes
custom_model = CustomConvNet(num_classes=num_classes).to(device)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(custom_model.parameters(), lr=hyper_param_learning_rate)
#
# for e in range(hyper_param_epoch):
#     for i_batch, item in enumerate(train_loader):
#         images = item['image'].to(device)
#         labels = item['label'].to(device)
#
#         # Forward pass
#         outputs = custom_model(images)
#         loss = criterion(outputs, labels)
#
#         # Backward and optimize
#         optimizer.zero_grad()
#         loss.backward()
#         optimizer.step()
#
#         if (i_batch + 1) % hyper_param_batch == 0:
#             print('Epoch [{}/{}], Loss: {:.4f}'
#                   .format(e + 1, hyper_param_epoch, loss.item()))
#
#             # model save
#             PATH = './weights/'
#             torch.save(custom_model, PATH + 'model_' + str(e+1) + '.pth')
#
# # Test the model
# custom_model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
# with torch.no_grad():
#     correct = 0
#     total = 0
#     for item in test_loader:
#         images = item['image'].to(device)
#         labels = item['label'].to(device)
#         outputs = custom_model(images)
#         _, predicted = torch.max(outputs.data, 1)
#         total += len(labels)
#         correct += (predicted == labels).sum().item()
#
#     print('Test Accuracy of the model on the {} test images: {} %'.format(total, 100 * correct / total))

classes = ("kagami", "kuroko")

def imshow(img):
    img = img / 2 + 0.5
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

def main() :
    custom_model = torch.load('./weights/model_19.pth')
    custom_model.eval()  # layer들의 동작을 inference(eval) mode로 바꿔준다.
    with torch.no_grad():   # autograd engine을 꺼버림(자동으로 gradient를 트래킹하지 않음) -> inference시 메모리 사용량을 줄여 연산 속도를 높히기 위함
        for item in test_loader:
            image = item['image'].to(device)
            labels = item['label'].to(device)
            outputs = custom_model(image)
            _, predicted = torch.max(outputs.data, 1)

            #imshow(torchvision.utils.make_grid(image))
            print('predicted: ' + str(predicted[0]))
            print('labels: ' + str(labels))

if __name__ == "__main__" :
    main()

max_stars_repo_name: dongjundev/Image-Classification

max_issues_repo_name: dongjundev/Image-Classification

max_forks_repo_name: dongjundev/Image-Classification


text:
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import numpy as np

celsius_q    = np.array([-40, -10, 0, 8, 15, 22, 38], dtype=float)
fahrenheit_a = np.array([-40, 14, 32, 46, 59, 72, 100], dtype=float)

def loss_graph(history):
	import matplotlib.pyplot as plt
	plt.xlabel('Epoch')
	plt.ylabel('Loss')
	plt.plot(history)

if __name__ == '__main__':
	layer_0 = tf.keras.layers.Dense(units=4, input_shape=[1])
	layer_1 = tf.keras.layers.Dense(units=4)
	layer_2 = tf.keras.layers.Dense(units=1)
	model = tf.keras.Sequential([layer_0, layer_1, layer_2])
	model.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(0.1))
	history = model.fit(celsius_q, fahrenheit_a, epochs=500, verbose=False)
	print(model.predict([15.0]))

	#loss_graph(history.history['loss'])

max_stars_repo_name: rodchenk/machine-learning

max_issues_repo_name: rodchenk/machine-learning

max_forks_repo_name: rodchenk/machine-learning


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 11 11:07:29 2020

@author: kianweelee
"""

# Importing the required packages
import os.path
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import PySimpleGUI as sg
from sklearn.impute import SimpleImputer
import numpy as np
from scipy import stats
import statsmodels.api as sm
from statsmodels.formula.api import ols
from scipy.stats import linregress
from scipy.stats import chi2_contingency
import itertools
from sklearn.preprocessing import LabelEncoder
from Report import model
from Graph import plot

def main():
    # Create a path to access csv file and also define a path to deposit EDA plots and report
        ## Required to create a GUI using pySimpleGUI

    layout = [
    [sg.Frame(layout=[
    [sg.Radio('CSV', "file_format", default=True, size=(10,1)), sg.Radio('XLS', "file_format")]], title='File Format',title_color='red', relief=sg.RELIEF_SUNKEN)],
    [sg.Text('Please input the folder path')],
    [sg.Text('File path:', size=(18, 1)), sg.Input(), sg.FileBrowse()],
    [sg.Text('Export plots to:', size=(18, 1)), sg.Input(), sg.FolderBrowse()],
    [sg.Text('Export report to:', size=(18, 1)), sg.Input(), sg.FolderBrowse()],
    [sg.Text('Export cleaned csv to:', size=(18, 1)), sg.Input(), sg.FolderBrowse()],
    [sg.Submit(), sg.Cancel()]]

    window = sg.Window('Edator', layout)

    event, values = window.read()
    csv_option, xls_option, file_path, plot_path, report_path, clean_csv_path = values[0], values[1], values[2], values[3], values[4], values[5]
    print(values[0])
    print(values[1])
    window.close()

    # Creating a txt file in the report_path
    filename = os.path.join(report_path, "report" + ".txt")

    # Assigning csv file to a variable call 'data'
    if csv_option:
    	data = pd.read_csv(file_path)
    else:
    	data = pd.read_excel(file_path, index_col = None)

    # Create a function to separate out numerical and categorical data
        ## Using this function to ensure that all non-numerical in a numerical column
        ## and non-categorical in a categorical column is annotated
    def cat_variable(df):
        return list(df.select_dtypes(include = ['category', 'object']))

    def num_variable(df):
        return list(df.select_dtypes(exclude = ['category', 'object']))

    categorical_variable = cat_variable(data)
    numerical_variable = num_variable(data)

    # Assigning variable filename to report and enable writing mode
    report = open(filename, "w")

    # Execute overview function in model module
    data = model.overview(data, numerical_variable, report)

    # Create a function to decide whether to drop all NA values or replace them
    ## Drop it if NAN count < 5 %
    nan_prop = (data.isna().mean().round(2)*100) # Show % of NaN values per column

    def drop_na():
        return [i for i, v in nan_prop.items() if v < 5 and v > 0]

    cols_to_drop = drop_na()

    data = data.dropna(subset = cols_to_drop)

    ## Using Imputer to fill NaN values
    ## Counting the proportion of NaN

    def fill_na():
        return [i for i, v in nan_prop.items() if v > 5]

    cols_to_fill = fill_na()

    cat_var_tofill = []
    num_var_tofill = []

    for var in cols_to_fill:
        if var in categorical_variable:
            cat_var_tofill.append(var)
        else:
            num_var_tofill.append(var)

    imp_cat = SimpleImputer(missing_values = np.nan, strategy='most_frequent')
    try:
        data[cat_var_tofill] = imp_cat.fit_transform(data[cat_var_tofill])
    except ValueError:
        pass

    imp_num = SimpleImputer(missing_values = np.nan, strategy='median')
    try:
        data[num_var_tofill] = imp_num.fit_transform(data[num_var_tofill])
    except ValueError:
        pass

    # Create a function to process outlier data
    def outlier():
        z = np.abs(stats.zscore(data[numerical_variable]))
        z_data = data[(z < 3).all(axis=1)] # Remove any outliers with Z-score > 3 or < -3
        return z_data

    data = outlier()

    # Create a function to compute correlation
    ## Pearson and Spearsman correlation for numerical-numerical data
    ## One-Way ANOVA for numerical-categorical data
    ## Chi-Square test for categorical-categorical data

    ## Creating possible combinations among a list of numerical variables
    num_var_combination = list(itertools.combinations(numerical_variable, 2))

    ## Creating possible combinations among a list of categorical variables
    cat_var_combination = list(itertools.combinations(categorical_variable, 2))

    ## Creating possible combinations among a list of numerical and categorical variuable
    catnum_combination = list(itertools.product(numerical_variable, categorical_variable))

    ## Running the report now
    model.run(num_var_combination,catnum_combination,cat_var_combination,report,data)

    # Create an output file that shows cleaned data
    data2 = data.copy()
    data2.to_csv(r'{}/cleaned_csv.csv'.format(clean_csv_path), index = False)

    # Running plot.py from Graph package
    plot.run(data, categorical_variable,numerical_variable,plot_path)

# Running program
if __name__ == "__main__":
    main()

max_stars_repo_name: kianweelee/Edator

max_issues_repo_name: kianweelee/Edator

max_forks_repo_name: kianweelee/Edator

reason: import numpy,from scipy,import statsmodels,from statsmodels


text:
"""Model class template

This module provides a template for users to implement custom models.
You can specify '--model template' to use this model.
The class name should be consistent with both the filename and its model option.
The filename should be <model>_dataset.py
The class name should be <Model>Dataset.py
It implements a simple image-to-image translation baseline based on regression loss.
Given input-output pairs (data_A, data_B), it learns a network netG that can minimize the following L1 loss:
    min_<netG> ||netG(data_A) - data_B||_1
You need to implement the following functions:
    <modify_commandline_options>:　Add model-specific options and rewrite default values for existing options.
    <__init__>: Initialize this model class.
    <set_input>: Unpack input data and perform data pre-processing.
    <forward>: Run forward pass. This will be called by both <optimize_parameters> and <test>.
    <optimize_parameters>: Update network weights; it will be called in every training iteration.
"""
import torch
import numpy as np
from .base_model import BaseModel
from . import networks
import torch.nn as nn
from util.util import prepare_z_y, one_hot, visualize_imgs
from torch.distributions import Categorical
from collections import OrderedDict

# from TTUR import fid
# from util.inception import get_inception_score
# from inception_pytorch import inception_utils

import copy
import math

class CDEGANModel(BaseModel):
    @staticmethod
    def modify_commandline_options(parser, is_train=True):
        #parser.set_defaults(dataset_mode='aligned')  # You can rewrite default values for this model. For example, this model usually uses aligned dataset as its dataset
        if is_train:
            parser.add_argument('--g_loss_mode', nargs='*', default=['nsgan','lsgan','vanilla'], help='lsgan | nsgan | vanilla | wgan | hinge | rsgan')
            parser.add_argument('--d_loss_mode', nargs='*', default=['vanilla'], help='lsgan | nsgan | vanilla | wgan | hinge | rsgan')
            parser.add_argument('--which_D', type=str, default='S', help='Standard(S) | Relativistic_average (Ra)')

            parser.add_argument('--lambda_f', type=float, default=0.1, help='the hyperparameter that balance Fq and Fd')
            parser.add_argument('--g_candi_num', type=int, default=1, help='# of survived generator candidatures in each evolutinary iteration.')
            parser.add_argument('--d_candi_num', type=int, default=2, help='# of survived discriminator candidatures in each evolutinary iteration.')
            parser.add_argument('--eval_size', type=int, default=64, help='batch size during each evaluation.')

            parser.add_argument('--lambda_weight', type=float, default=1.0, help='batch size during each evaluation.')
            parser.add_argument('--mean_type', type=str, default='arithmetic', help='arithmetic | geometric | harmonic')
            parser.add_argument('--weight_type', type=str, default='normal', help='normal | log')

            parser.add_argument('--D_different', default=True, help='if different discriminators sample are different for D update')
            parser.add_argument('--G_different', default=True, help='if different discriminators sample are different for G update')
        return parser

    def __init__(self, opt):
        BaseModel.__init__(self, opt)
        self.opt = opt
        self.loss_names = ['D_real', 'D_fake', 'D_gp', 'D', 'G']
        self.visual_names = ['real_visual', 'gen_visual']

        if self.isTrain:  # only defined during training time
            self.model_names = ['G', 'D']
        else:
            self.model_names = ['G']

        # define networks

        self.print_mutaion = []
        if self.isTrain:  # only defined during training time
            # define G mutations
            self.G_mutations = []
            for g_loss in opt.g_loss_mode:
                self.G_mutations.append(networks.GANLoss(g_loss, 'G', opt.which_D).to(self.device))
            # define D mutations
            self.D_mutations = []
            for d_loss in opt.d_loss_mode:
                self.D_mutations.append(networks.GANLoss(d_loss, 'D', opt.which_D).to(self.device))

            # vanilla discriminator criterion for calculating fitness score
            self.vanilla_Dcriterion = networks.GANLoss('vanilla', 'D', opt.which_D).to(self.device)

        self.netG = networks.define_G(opt.z_dim, opt.output_nc, opt.ngf, opt.netG,
                                        opt.g_norm, opt.cgan, opt.cat_num, not opt.no_dropout, opt.init_type, opt.init_gain, self.gpu_ids)
        self.optimizer_G = torch.optim.Adam(self.netG.parameters(), lr=opt.lr_g, betas=(opt.beta1, opt.beta2))
        self.optimizers.append(self.optimizer_G)
        self.G_candis = []
        self.optG_candis = []
        for i in range(opt.g_candi_num):
            self.G_candis.append(copy.deepcopy(self.netG.state_dict()))
            self.optG_candis.append(copy.deepcopy(self.optimizer_G.state_dict()))

        self.netD = networks.define_D(opt.input_nc, opt.ndf, opt.netD, opt.d_norm, opt.cgan, opt.cat_num, opt.init_type, opt.init_gain, self.gpu_ids)
        self.optimizer_D = torch.optim.Adam(self.netD.parameters(), lr=opt.lr_d, betas=(opt.beta1, opt.beta2))
        self.optimizers.append(self.optimizer_D)
        self.D_candis = []
        self.optD_candis = []
        for i in range(opt.d_candi_num):
            netD = networks.define_D(opt.input_nc, opt.ndf, opt.netD, opt.d_norm, opt.cgan, opt.cat_num, opt.init_type, opt.init_gain, self.gpu_ids)
            optimD = torch.optim.Adam(netD.parameters(), lr=opt.lr_d, betas=(opt.beta1, opt.beta2))
            self.D_candis.append(netD)
            self.optD_candis.append(optimD)

        # visulize settings
        self.N =int(np.trunc(np.sqrt(min(opt.batch_size, 64))))
        if self.opt.z_type == 'Gaussian':
            self.z_fixed = torch.randn(self.N*self.N, opt.z_dim, 1, 1, device=self.device)
        elif self.opt.z_type == 'Uniform':
            self.z_fixed = torch.rand(self.N*self.N, opt.z_dim, 1, 1, device=self.device)*2. - 1.

        # the # of image for each evluation
        # TODO batch size details
        self.eval_size = max(math.ceil((opt.batch_size * opt.D_iters) / opt.d_candi_num), opt.eval_size)

    def set_input(self, input):
        self.input_imgs = input['image'].to(self.device)
        # self.real_imgs just for visualizing
        self.real_imgs = self.input_imgs[:self.opt.batch_size]

    # TODO
    def optimize_parameters(self, iters):
        self.g_useless_real_score = self.netD(self.input_imgs[:self.opt.batch_size])
        for i in range(self.opt.D_iters + 1):
            if i == 0:
                self.G_Fitness, self.evalimgs, self.g_sel_mut = self.Evo_G()
                self.print_mutaion.append(self.g_sel_mut)
            else:
                real_imgs = self.input_imgs[(i-1)*self.opt.batch_size*self.opt.d_candi_num: i*self.opt.batch_size*self.opt.d_candi_num]
                fake_imgs = self.evalimgs[(i-1)*self.opt.batch_size*self.opt.d_candi_num: i*self.opt.batch_size*self.opt.d_candi_num]
                self.D_Fitness, self.d_sel_mut = self.Evo_D(fake_imgs, real_imgs)
        if iters %  self.opt.print_freq == 0 and iters > 0:
            print('iters:', iters, 'g_sel_mul:', self.g_sel_mut, 'd_sel_mul:', self.d_sel_mut)

    def Evo_D(self, fake_imgs, real_imgs):
        eval_real_imgs = self.input_imgs[-self.opt.eval_size:]
        eval_fake_imgs = self.evalimgs[-self.opt.eval_size:]

        F_list = -100*np.ones(self.opt.d_candi_num)
        D_F = []
        D_list = []
        optD_list = []
        selected_mutation = []
        count = 0
        for i in range(self.opt.d_candi_num):
            if self.opt.D_different:
                train_fake_imgs = fake_imgs[i*self.opt.batch_size:(i+1)*self.opt.batch_size]
                train_real_imgs = real_imgs[i*self.opt.batch_size:(i+1)*self.opt.batch_size]
            else:
                train_fake_imgs = real_imgs[:self.opt.batch_size]
                train_real_imgs = real_imgs[:self.opt.batch_size]

            for j, criterionD in enumerate(self.D_mutations):
                # backward and update
                self.netD.load_state_dict(self.D_candis[i].state_dict())
                self.optimizer_D.load_state_dict(self.optD_candis[i].state_dict())
                self.set_requires_grad(self.netD, True)
                self.optimizer_D.zero_grad()
                self.loss_D, self.loss_D_fake, self.loss_D_real, self.loss_D_gp = self.backward_D(
                    self.netD, train_fake_imgs, train_real_imgs, criterionD)
                self.optimizer_D.step()

                # Evaluation
                Fd_D = self.D_diverse_score(self.netD, eval_fake_imgs, eval_real_imgs)
                # selection
                if count < self.opt.d_candi_num:
                    F_list[count] = Fd_D
                    D_F.append(Fd_D)
                    D_list.append(copy.deepcopy(self.netD))
                    optD_list.append(copy.deepcopy(self.optimizer_D))
                    selected_mutation.append(self.opt.d_loss_mode[j])
                else:
                    fit_com = Fd_D - F_list
                    if min(fit_com) < 0:
                        ids_replace = np.where(fit_com==max(fit_com))[0][0]
                        F_list[ids_replace] = Fd_D
                        D_F[ids_replace] = Fd_D
                        D_list[ids_replace] = copy.deepcopy(self.netD)
                        optD_list[ids_replace] = copy.deepcopy(self.optimizer_D)
                        selected_mutation[ids_replace] = self.opt.d_loss_mode[j]
                count +=1
        self.D_candis = copy.deepcopy(D_list)
        self.optD_candis = copy.deepcopy(optD_list)
        return np.array(D_F), selected_mutation

    def D_diverse_score(self, netD, fake_imgs, real_imgs):
        self.set_requires_grad(netD, True)
        fake_score = netD(fake_imgs)
        real_score = netD(real_imgs)

        # Diversity fitness score
        fake_loss, real_loss = self.vanilla_Dcriterion(fake_score, real_score)
        eval_loss_D = fake_loss + real_loss
        gradients = torch.autograd.grad(outputs=eval_loss_D, inputs=netD.parameters(),
                                        grad_outputs=torch.ones(eval_loss_D.size()).to(self.device),
                                        create_graph=True, retain_graph=True, only_inputs=True)
        with torch.no_grad():
            for i, grad in enumerate(gradients):
                grad = grad.view(-1)
                allgrad = grad if i == 0 else torch.cat([allgrad,grad])
        Fd = -torch.log(torch.norm(allgrad)).data.cpu().numpy()
        return Fd

    def backward_D(self, netD, fake_imgs, real_imgs, criterionD):
        fake_score = netD(fake_imgs)
        real_score = netD(real_imgs)
        loss_D_fake, loss_D_real = criterionD(fake_score, real_score)

        # if self.opt.use_gp is True:
        if 1 == 1:
            loss_D_gp = networks.cal_gradient_penalty(netD, real_imgs, fake_imgs, self.device, type='mixed', constant=1.0, lambda_gp=10.0)[0]
        else:
            loss_D_gp = 0.
        loss_D = loss_D_fake + loss_D_real + loss_D_gp
        loss_D.backward()
        return loss_D, loss_D_fake, loss_D_real, loss_D_gp

    def Evo_G(self):
        eval_real_imgs = self.input_imgs[-self.eval_size:,:,:,:]
        F_list = np.zeros(self.opt.g_candi_num)
        Fit_list = []
        G_list = []
        optG_list = []
        evalimg_list = []
        selected_mutation = []
        count = 0
        # variation-evluation-selection
        for i in range(self.opt.g_candi_num):
            for j, criterionG in enumerate(self.G_mutations):
                # Variation
                self.netG.load_state_dict(self.G_candis[i])
                self.optimizer_G.load_state_dict(self.optG_candis[i])
                self.optimizer_G.zero_grad()
                self.loss_G = self.boosted_backward_G(criterionG)
                self.optimizer_G.step()
                # Evaluation
                with torch.no_grad():
                    eval_fake_imgs = self.forward(batch_size=self.eval_size+self.opt.D_iters*self.opt.batch_size*self.opt.d_candi_num).detach()
                Fq, Fd = self.fitness_score(eval_fake_imgs[:self.eval_size], eval_real_imgs)
                F = Fq + self.opt.lambda_f * Fd
                # Selection
                if count < self.opt.g_candi_num:
                    F_list[count] = F
                    Fit_list.append([Fq, Fd, F])
                    G_list.append(copy.deepcopy(self.netG.state_dict()))
                    optG_list.append(copy.deepcopy(self.optimizer_G.state_dict()))
                    evalimg_list.append(eval_fake_imgs)
                    selected_mutation.append(self.opt.g_loss_mode[j])
                else:
                    fit_com = F - F_list
                    if max(fit_com) > 0:
                        ids_replace = np.where(fit_com==max(fit_com))[0][0]
                        F_list[ids_replace] = F
                        Fit_list[ids_replace] = [Fq, Fd, F]
                        G_list[ids_replace] = copy.deepcopy(self.netG.state_dict())
                        optG_list[ids_replace] = copy.deepcopy(self.optimizer_G.state_dict())
                        evalimg_list[ids_replace] = eval_fake_imgs
                        selected_mutation[ids_replace] = self.opt.g_loss_mode[j]
                count += 1
        self.G_candis = copy.deepcopy(G_list)
        self.optG_candis = copy.deepcopy(optG_list)

        # shuffle
        evalimg_list = torch.cat(evalimg_list, dim=0)
        shuffle_ids = torch.randperm(evalimg_list.size()[0])
        evalimg_list = evalimg_list[shuffle_ids]
        return np.array(Fit_list), evalimg_list, selected_mutation

    def boosted_backward_G(self, criterionG):
        gen_imgs = self.forward(batch_size=self.opt.d_candi_num*self.opt.batch_size)
        g_losses = []
        for i in range(self.opt.d_candi_num):
            netD = self.D_candis[i]
            self.set_requires_grad(netD, False)

            if self.opt.G_different:
                fake_logits = netD(gen_imgs[i*self.opt.batch_size:(i+1)*self.opt.batch_size])
            else:
                fake_logits = netD(gen_imgs[:self.opt.batch_size])

            loss_G_fake, loss_G_real = criterionG(fake_logits, self.g_useless_real_score)
            loss_G = loss_G_fake + loss_G_real
            g_losses.append(loss_G)
        loss_G = self.mix_prediction(g_losses)
        loss_G.backward()
        return loss_G

    def mix_prediction(self, losses):
        lam = self.opt.lambda_weight
        assert self.opt.mean_type in ['arithmetic','geometric','harmonic', 'sum']
        assert self.opt.weight_type in ['normal','log']
        if lam == 0.:
            weights = torch.ones_like(losses)
        else:
            if self.opt.weight_type == 'log':
                weights = [torch.pow(losses[i], lam).detach() for i in range(len(losses))]
            else:
                weights = [torch.exp(lam * losses[i]).detach() for i in range(len(losses))]
        weights_normed = [weights[i]/sum(weights) for i in range(len(weights))]
        loss = 0
        if self.opt.mean_type == 'arithmetic':
            for i in range(len(losses)):
                loss += weights_normed[i]*losses[i]
        elif self.opt.mean_type == 'sum':
            for i in range(len(losses)):
                loss += losses[i]
        else:
            raise NotImplementedError('other mean_type in mix_prediction not implemented')
        return loss

    def forward(self, batch_size = None):
        bs = self.opt.batch_size if batch_size is None else batch_size
        if self.opt.z_type == 'Gaussian':
            z = torch.randn(bs, self.opt.z_dim, 1, 1, device=self.device)
        elif self.opt.z_type == 'Uniform':
            z = torch.rand(bs, self.opt.z_dim, 1, 1, device=self.device)*2. - 1.
        else:
            raise NotImplementedError('z type not implemented')
        # Fake images
        gen_imgs = self.netG(z)
        return gen_imgs

    def fitness_score(self, eval_fake_imgs, eval_real_imgs):
        eval_fake = []
        # eval_real = []
        Fd = 0
        for i in range(self.opt.d_candi_num):
            netD = self.D_candis[i]
            self.set_requires_grad(netD, True)
            eval_fake_score = netD(eval_fake_imgs)
            eval_real_score = netD(eval_real_imgs)
            eval_fake.extend(eval_fake_score)
            Fd += self.calculate_gradient(netD, eval_fake_score, eval_real_score)
        eval_fake = torch.tensor(eval_fake)
        Fq = nn.Sigmoid()(eval_fake).data.mean().cpu().numpy()
        return Fq, Fd

    def calculate_gradient(self, netD, fake_score, real_score):
        eval_D_fake, eval_D_real = self.vanilla_Dcriterion(fake_score, real_score)
        eval_D = eval_D_fake + eval_D_real
        gradients = torch.autograd.grad(outputs=eval_D, inputs=netD.parameters(),
                                        grad_outputs=torch.ones(eval_D.size()).to(self.device),
                                        create_graph=True, retain_graph=True, only_inputs=True)
        with torch.no_grad():
            for i, grad in enumerate(gradients):
                grad = grad.view(-1)
                allgrad = grad if i == 0 else torch.cat([allgrad,grad])
        Fd = -torch.log(torch.norm(allgrad)).data.cpu().numpy()
        return Fd

max_stars_repo_name: shiming-chen/CDE-GAN

max_issues_repo_name: shiming-chen/CDE-GAN

max_forks_repo_name: shiming-chen/CDE-GAN


text:
import wx
from wx import xrc
import motmot.wxvideo.wxvideo as wxvideo
import pkg_resources # part of setuptools
import numpy as nx
import movies
import imagesk
RSRC_FILE = pkg_resources.resource_filename( __name__, "test.xrc" )

class TestDrawApp(wx.App):
    def OnInit( self ):

        rsrc = xrc.XmlResource(RSRC_FILE)
        self.frame = rsrc.LoadFrame(None,"FRAME")
        self.frame.Show()
        self.img_panel = xrc.XRCCTRL(self.frame,"PANEL")
        box = wx.BoxSizer( wx.VERTICAL )
        self.img_panel.SetSizer( box )
        self.img_wind = wxvideo.DynamicImageCanvas( self.img_panel, -1 )
        self.img_wind.set_resize(True)
        box.Add( self.img_wind, 1, wx.EXPAND )
        self.img_panel.SetAutoLayout( True )
        self.img_panel.Layout()

        wx.EVT_LEFT_DOWN(self.img_wind,self.MouseClick)
        #self.img_wind.Bind(wx.EVT_LEFT_DOWN,self.MouseClick)

        self.filename = '/home/kristin/FLIES/data/walking_arena/movie20071009_155327.sbfmf'
        self.movie = movies.Movie(self.filename,True)
        imd,stamp = self.movie.get_frame( 1 )
        print 'imd = ' + str(imd)
        print 'minv = ' + str(nx.min(imd))
        im8 = imagesk.double2mono8(imd)
        print 'im8 = ' + str(im8)
        print 'minv = ' + str(nx.min(im8))
        print 'im8.shape = ' + str(im8.shape)

        #im8 = nx.zeros((1024,1024),dtype=nx.uint8)

        point = [[500,500]]
        line = [[250,250,750,750]]
        pointcolors = [(1,0,0)]
        linecolors = [(1,1,0)]

        self.img_wind.update_image_and_drawings('camera',im8,format='MONO8',
                                                linesegs=line,points=point,
                                                point_colors=pointcolors,
                                                lineseg_colors=linecolors)

        return True

    def MouseClick(self,evt):

        print 'mouse clicked'

def main():
    app = TestDrawApp( 0 )
    app.MainLoop()

if __name__ == '__main__':
    main()

max_stars_repo_name: johnabender/ctrax-tmp

max_issues_repo_name: johnabender/ctrax-tmp

max_forks_repo_name: johnabender/ctrax-tmp


text:
# fmt: off
import os
import numpy as np
import argparse
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

from rlo import experiment_result
from rlo import plotting
from rlo import utils

def plot_rewrite_sequences(outdir, events, linear_scale=False, plot_all=True):

    #make a new directory for the rewrite sequence plots
    if not os.path.isdir(outdir):
        os.mkdir(outdir)
    event = [e for e in events if 'eval_expr' in e and e['event']=='rollout_end' and "generation" in e]
    events_by_generation = utils.group_by(event, lambda r: r["generation"])

    # for later: might want to limit the number of generations, ok for now
    for generation, events in events_by_generation.items():
        events_by_expr = utils.group_by(events, lambda r: r["eval_expr"])
        n_expressions = len(events_by_expr)
        assert n_expressions > 0

        # for each generation create a separate plot containing all expressions
        fig, axs = plt.subplots(n_expressions, figsize=[15, 6 * (n_expressions)])
        if n_expressions == 1:
            axs = [axs]
        fig.tight_layout(rect=(0.05, 0, 1, 1))

        for ax, (expr, ev) in zip(axs, events_by_expr.items()):
            by_rep = utils.group_by(ev, lambda r: r["repetition"])
            best_ep_costs = [
                [cost for (_action, cost) in utils.single_elem(by_rep[k])["best_ep"][1]]
                for k in sorted(by_rep.keys())
            ]

            if not plot_all:
                best_ep_costs = np.array(plotting.apply_padding_by_last(best_ep_costs))
                num_steps = np.arange(best_ep_costs.shape[1])
                col = plotting.random_color_from_string(expr)
                plotting.plot_with_confidence(ax, num_steps, best_ep_costs, col, label=expr, probs=[20, 50, 80], axis=0)
            else:
                # plot rewrite sequences for all repetitions without padding
                for i, best_ep in enumerate(best_ep_costs):
                    col = plotting.random_color_from_string(i)
                    ax.plot(np.arange(len(best_ep)), best_ep, color=col, label=expr + " at rep " + str(i))

            if linear_scale:
                ax.set_yscale("linear")
            else:
                ax.set_yscale("log")
            ax.set_ylabel("cost", fontsize=8)
            ax.set_xlabel("steps", fontsize=8)
            ax.legend(*ax.get_legend_handles_labels(), loc="upper right")

        plt.savefig(outdir + "/cost_per_steps_generation_" + str(generation) + ".png")
        plt.title("Cost at each rewrite step")
        plt.close()

def plot_rewrite_sequences_from_config(config, events, linear_scale=False, plot_all=True):
    plot_rewrite_sequences(
        plotting.format_figure_filename(config, "rewrite_sequences"),
        events, linear_scale, plot_all)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("run_id", type=str, help="(1) a run ID (e.g., 2019_01_06_13_15_48_13172), (2) path to a config.json file, or (3) path to a directory containing /rep/events.json files")
    parser.add_argument("--linear_scale", default=False, action="store_true")
    parser.add_argument("--plot_summary", dest="plot_all", action="store_false")
    parser.set_defaults(plot_all=True)
    args = parser.parse_args()

    config, logs = experiment_result.load_config_events(args.run_id)
    if config is not None:
        plot_rewrite_sequences_from_config(config, logs, args.linear_scale, args.plot_all)
    else:
        plot_rewrite_sequences(os.path.join(args.run_id, "rewrite_sequences"),
                               logs,
                               args.linear_scale,
                               args.plot_all)
if __name__ == "__main__": main()

max_stars_repo_name: tomjaguarpaw/knossos-ksc

max_issues_repo_name: tomjaguarpaw/knossos-ksc

max_forks_repo_name: tomjaguarpaw/knossos-ksc


text:
from sorting import merge_sort, insertion_sort, quick_sort
import numpy as np
from math import log2
"""
coeffs = np.array([[  7.02924770e-06, -6.41693631e-07, 1.11454411e-08],
             [  2.77095379e-05, -4.98456258e-06, 1.04908416e-07],
             [  1.52831053e-05, -1.95691895e-06, 2.02251640e-08]])
intercepts = np.array([1.07302477871e-05, -5.74258092972e-05, -5.38219028179e-06])
"""

coeffs = np.array([[  7.95957933e-08  , 3.13214568e-07 ,  7.84754425e-09],
 [  1.17647476e-05,  -2.22052775e-06   ,8.49854700e-08],
 [  1.25715975e-05  ,-1.62537198e-06  , 2.26052259e-08]])
intercepts = np.array([  4.55142710e-05 , -2.21855518e-05 , -1.30556964e-05])

lhyra_hook = lambda t: lhyra_sort(t)
def lhyra_sort(data):
    features = np.array([len(data), len(data)*log2(len(data)+1), len(data)**2]) if len(data) > 0 else [0,0,0]
    choice = np.argmin(coeffs @ features + intercepts)
    if choice == 0: return merge_sort(data, lhyra_hook, {})
    elif choice == 1: return insertion_sort(data, lhyra_hook, {})
    else: return quick_sort(data, lhyra_hook, {})
"""
coeffs = np.array([[  1.41278615e-05 , -1.31231425e-06 ,  6.86773299e-09,  -1.50966275e-03],
[ -2.58740136e-05 ,  3.65124088e-06 ,  2.50896971e-08 , -1.79948173e-03] ,
[  1.41002391e-05 , -1.94680931e-06  , 2.07435650e-08 , -1.09403240e-03]])
intercepts = np.array([3.76358559105e-05, 0.00032508733845, 8.1364347528e-05])

lhyra_hook = lambda t: lhyra_sort(t)
def lhyra_sort(data):
    features = [len(data), len(data)*log2(len(data)+1), len(data)**2, data[0] == 1] if len(data) > 0 else [0,0,0,1]
    choice = np.argmin(coeffs @ features + intercepts)
    if choice == 0: return merge_sort(data, lhyra_hook, {})
    elif choice == 1: return insertion_sort(data, lhyra_hook, {})
    else: return quick_sort(data, lhyra_hook, {})
"""

if __name__ == '__main__':
    from random import random
    l = [random() for i in range(10000)]
    lhyra_sort(l)

max_stars_repo_name: joshuagruenstein/lhyra

max_issues_repo_name: joshuagruenstein/lhyra

max_forks_repo_name: joshuagruenstein/lhyra