text:
from flask import Flask, send_file, jsonify, make_response
from flask_cors import CORS
from flask_socketio import SocketIO
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.optim as optim
import argparse
import os
import signal
import json
import glob
from numpy import random
from pathlib import Path
from optimizer_utils import train_model, train_model_from_scratch, continue_training_model, Logger, calculate_and_print_accuracy, calculate_and_print_accuracy_and_return_predictions
from random_cnn_generator import RandomConvNet, random_conv_net_from_template, generate_ablations_from_template
from queue import Queue
from threading import Thread
import numpy as np
import pandas as pd

# Need to calculate distances for new models to send to frontend
MODEL_DISTANCES_PATH = os.path.join('20190330_quickdraw_model_projection_data.json')
from scipy.spatial.distance import hamming
import neural_network
import nn_comparators
from general_utils import get_dok_mat_with_set_coords
model_distances_data = []
with open(MODEL_DISTANCES_PATH) as f:
    model_distances_data = json.load(f)
hamming_distances = np.loadtxt(os.path.join('hamming_distances_quickdraw_20190330.csv'), delimiter=',')
hamming_projected_coords = np.loadtxt(os.path.join('hamming_projected_coords_quickdraw_20190330.csv'), delimiter=',')
HAMMING_DISTANCES_SQUARED = np.square(hamming_distances)
HAMMING_PROJECTIONS_PINV = np.linalg.pinv(hamming_projected_coords)
otmann_distances = np.loadtxt(os.path.join('otmann_distances_quickdraw_20190330.csv'), delimiter=',')
otmann_projected_coords = np.loadtxt(os.path.join('otmann_projected_coords_quickdraw_20190330.csv'), delimiter=',')
OTMANN_DISTANCES_SQUARED = np.square(otmann_distances)
OTMANN_PROJECTIONS_PINV = np.linalg.pinv(otmann_projected_coords)

new_models_info = []
model_layer_dict = {}
# print(model_distances_data[0])
for model in model_distances_data:
    model_layer_dict[model['id']] = model['layers']

cnn_layer_labels = ['ip', 'op', 'dense',
                    'dropout', 'softmax',
                    'conv1', 'conv3', 'conv5',
                    'tanh', 'relu', 'sigmoid',
                    'pool2', 'pool3', 'pool5']

non_assignment_penalty = 1
# cnn_layer_labels, label_mismatch_penalty = \
    # nn_comparators.get_cnn_layer_label_mismatch_penalties(non_assignment_penalty)

label_mismatch_penalty = []
# ip penalties
label_mismatch_penalty.append([0] + [np.inf]*13)
# op penalties
label_mismatch_penalty.append([np.inf, 0] + [np.inf]*12)
# dense penalties
label_mismatch_penalty.append([np.inf, np.inf, 0] + [np.inf]*11)
# dropout penalties
label_mismatch_penalty.append([np.inf]*3 + [0] + [np.inf]*10)
# softmax penalties
label_mismatch_penalty.append([np.inf]*4 + [0] + [np.inf]*9)
# conv1 penalties
label_mismatch_penalty.append([np.inf]*5 + [0, 0.2, 0.3] + [np.inf]*6)
# conv3 penalties
label_mismatch_penalty.append([np.inf]*5 + [0.2, 0, 0.2] + [np.inf]*6)
# conv5 penalties
label_mismatch_penalty.append([np.inf]*5 + [0.3, 0.2, 0] + [np.inf]*6)
# tanh penalties
label_mismatch_penalty.append([np.inf]*8 + [0, 0.25, 0.1] + [np.inf]*3)
# relu penalties
label_mismatch_penalty.append([np.inf]*8 + [0.25, 0, 0.25] + [np.inf]*3)
# sigmoid penalties
label_mismatch_penalty.append([np.inf]*8 + [0.1, 0.25, 0] + [np.inf]*3)
# pool2 penalties
label_mismatch_penalty.append([np.inf]*11 + [0, 0.2, 0.3])
# pool3 penalties
label_mismatch_penalty.append([np.inf]*11 + [0.2, 0, 0.2])
# pool5 penalties
label_mismatch_penalty.append([np.inf]*11 + [0.3, 0.2, 0])

label_mismatch_penalty = np.array(label_mismatch_penalty)

tp_comp = nn_comparators.OTMANNDistanceComputer(cnn_layer_labels,
    label_mismatch_penalty, non_assignment_penalty,
    nn_comparators.CNN_STRUCTURAL_PENALTY_GROUPS,
    nn_comparators.PATH_LENGTH_TYPES,
    dflt_mislabel_coeffs=1.0, dflt_struct_coeffs=1.0)

# print("model_distances_data[0] is ", model_distances_data[0])

MODEL_DIR = 'models_quickdraw_final'
DATA_DIR = 'data'
LOG_FILE = 'model_quickdraw_logs.json'
# TRUE_LABELS = pd.read_csv('cifar_test_labels.csv')['true_class']
from quickdraw_dataset import QuickdrawDataset
testset = QuickdrawDataset(mode='test')
TRUE_LABELS = []

it = 0
# for (_, label) in testset:
for i in range(40000):
    (_, label) = testset[i]
    TRUE_LABELS.append(label)
    it = it + 1

TRUE_LABELS = pd.Series(TRUE_LABELS)

IMAGE_DIR = os.path.join('static')
NUM_EPOCHS_TRAIN = 10
# app = Flask(__name__, static_folder=IMAGE_DIR)
app = Flask(__name__)
CORS(app)
app.config['SECRET_KEY'] = '196_BOSTON_AVENUE'
socketio = SocketIO(app)

parser = argparse.ArgumentParser()

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

transform = None
trainset = QuickdrawDataset(transform=transform)
print("got trainset")
trainloader = torch.utils.data.DataLoader(trainset, batch_size=256,
                                        shuffle=True, num_workers=2)
print("got trainloader")

testset = QuickdrawDataset(mode='test', transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=256,
                                        shuffle=False, num_workers=2)
classes = (
    'face',
    'moustache',
    'pear',
    'umbrella',
    'pineapple',
    'mouth',
    'nose',
    'wine bottle',
    'apple',
    'octopus'
)

# trainset = torchvision.datasets.CIFAR10(root=DATA_DIR, train=True,
#                                         download=True, transform=transform)
# trainloader = torch.utils.data.DataLoader(trainset, batch_size=32,
#                                           shuffle=True, num_workers=2)

# testset = torchvision.datasets.CIFAR10(root=DATA_DIR, train=False,
#                                        download=True, transform=transform)
# testloader = torch.utils.data.DataLoader(testset, batch_size=32,
                                         # shuffle=False, num_workers=2)

# classes = ('plane', 'car', 'bird', 'cat',
#            'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
logger = Logger(LOG_FILE, load_log=True, socket=socketio)

def worker():
    while True:
        item = training_queue.get()
        net = item['net']
        epochs = item['epochs']
        train_model_for_frontend(net, epochs)
        # train_model_for_frontend(net, 3)
        # Maybe we should store the models and their results after they are completed?
        # training_queue.task_done()

training_queue = Queue()
current_model_id = ''
training_thread = Thread(target=worker)
training_thread.daemon = True
training_thread.start()

# The training all happens in a separate thread, which implements a training queue

def add_model_to_training_queue(net, num_epochs=NUM_EPOCHS_TRAIN):
    training_queue.put({'net': net, 'epochs': num_epochs})

def generate_confusion_matrix_bins(predlabels):
    # need to roll our own because sklearn doesn't produce bins
    predictions = np.array(predlabels)[:,1]
    # Using https://stackoverflow.com/questions/21153865/obtain-indices-corresponding-to-instances-for-each-type-of-error-in-a-confusion
    return [[[int(x) for x in np.where(np.array(np.logical_and(TRUE_LABELS==r, predictions==c)))[0].tolist()] for c in range(10)] for r in range(10)]

def calculate_class_accuracies(confusion_matrix):
    total_true = [0 for _ in range(10)]
    total_predicted = [0 for _ in range(10)]
    total_correct = [0 for _ in range(10)]
    accuracies = [0.0 for _ in range(10)]

    for i in range(10):
        for j in range(10):
            count = len(confusion_matrix[i][j])
            if (i == j):
                total_correct[i] = count

            total_true[i] = total_true[i] + count
            total_predicted[j] = total_predicted[j] + count

    for i in range(10):
        accuracies[i] = 1.0 * total_correct[i] / total_true[i]
    return accuracies

def get_incremental_mds_embedding(d, projection_type='hamming'):
    # According to implementation at http://www.math.sjsu.edu/~gchen/Math285F15/Math%20285%20-%20Project%20WFSalinas_DLi.pdf
    # d is the vector of distances from our new point to all original points
    # projection_type is either hamming or otmannn

    # Assign our variables
    if projection_type is 'hamming':
        D = HAMMING_DISTANCES_SQUARED
        pseudoY = HAMMING_PROJECTIONS_PINV
    else:
        D = OTMANN_DISTANCES_SQUARED
        pseudoY = OTMANN_PROJECTIONS_PINV

    # First, we get d^2
    d2 = np.square(np.array(d))

    # Then, we calculate b
    (n, _) = D.shape
    a = ((1.0 / n) * d2 - d2).reshape(-1, 1)
    b = (1.0 / n) * np.dot(D, np.ones((n, 1)))
    c = (1.0 / (n**2)) * np.dot(np.dot(np.ones((n, n)), D), np.ones((n, 1)))
    b = 0.5 * a + b - c

    # The approximate solution is y_x = (Y^T Y) ^(-1) Y^t * b
    #  a.k.a pseudoinverse(Y) * b
    return np.dot(pseudoY, b)

def get_hamming_distances(predictions):
    predictions = np.array(predictions)[:,1]
    return [hamming(predictions, x['predictions']) for x in model_distances_data]

def get_otmann_distances(layers):
    new_nasbot_model = convert_model_object_to_nasbot_model({'layers': layers})
    # print("calculating otmann_distances...")

    # Then, we calculate the distances to every other model
    nasbot_models = [m['nasbot_model'] for m in model_distances_data]
    # print("comparing nasbot model ", new_nasbot_model)
    # print("other nasbot model", nasbot_models[0])
    otmann_distances = tp_comp.evaluate([new_nasbot_model], nasbot_models)
    return otmann_distances

# Then, we convert our models into these neural network classes
def convert_layer_to_nasbot_layer(layer):
    if layer['type'] == 'Activation':
        return layer['activation']
    elif layer['type'] == 'AveragePooling2D' or layer['type'] == 'MaxPool':
        return 'pool' + str(layer['pool_size'])
    elif layer['type'] == 'Conv2D':
        return 'conv' + str(layer['kernel_size'])
    elif layer['type'] == 'Dense':
        return 'dense'
    elif layer['type'] == 'Dropout':
        return 'dropout'
    else:
        print("got unmatched layer type ", layer)

def convert_layer_to_nasbot_filters(layer):
    if layer['type'] == 'Conv2D':
        return layer['filters']
    elif layer['type'] == 'Dense':
        return layer['units']
    else:
        return None

def convert_layer_to_nasbot_stride(layer):
    if layer['type'] == 'Conv2D':
        return 1
    elif layer['type'] == 'AveragePooling2D' or layer['type'] == 'MaxPool':
        return 2
    else:
        return None

def convert_model_object_to_nasbot_model(model_object):
    # for now, our connectivity matrix is just 1s on the up off diagonal, because
    # we have strictly feed forward layers
    layers = model_object['layers']
    num_layers = len(layers) + 3 # for input, softmax, and output

    connection_coords = []
    for j in range(1, num_layers):
        connection_coords.append((j-1, j))

    connectivity_matrix = get_dok_mat_with_set_coords(num_layers, connection_coords)
    # print("layers are ", layers)
    # print("connectivity_matrix is ", connectivity_matrix.keys())
    # print("cnn_layer_labels is ", cnn_layer_labels)
    # print("['ip'] + [convert_layer_to_nasbot_layer(l) for l in layers] + ['softmax', 'op'] is ", ['ip'] + [convert_layer_to_nasbot_layer(l) for l in layers] + ['softmax', 'op'])
    conv = neural_network.ConvNeuralNetwork(
        ['ip'] + [convert_layer_to_nasbot_layer(l) for l in layers] + ['softmax', 'op'],
        connectivity_matrix,
        [None] + [convert_layer_to_nasbot_filters(l) for l in layers] + [None, None],
        [None] + [convert_layer_to_nasbot_stride(l) for l in layers] + [None, None],
        cnn_layer_labels
        )

    return conv
    # return None

for (i, model_object) in enumerate(model_distances_data):
    model_distances_data[i]['nasbot_model'] = convert_model_object_to_nasbot_model(model_object)

def delete_model_from_training_queue(model_id):
    global training_queue
    global training_thread
    global current_model_id

    if current_model_id == model_id:
        print("THIS ISN'T SUPPORTED AND DOES NOTHING")
        return False
        os.kill(signal.CTRL_C_EVENT, 0)
        training_thread = Thread(target=worker)
        training_thread.daemon = True
        training_thread.start()
    else:
        new_training_queue = Queue()
        while not training_queue.empty():
            m = training_queue.get()
            net = m['net']
            epochs = m['epochs']
            net_metadata = net.metadata()
            if net_metadata['id'] != model_id:
                new_training_queue.put({'net': net, 'epochs': epochs})
        training_queue = new_training_queue

    return True

def reorder_model_from_training_queue(model_ids_order):
    global training_queue
    global training_thread
    global current_model_id

    new_training_queue = Queue()
    models_dict = {}
    while not training_queue.empty():
        m = training_queue.get()
        net = m['net']
        epochs = m['epochs']
        net_metadata = net.metadata()
        models_dict[net_metadata['id']] = {'net': net, 'epochs': epochs}
    # Then, reconstruct queue
    for model_id in model_ids_order:
        new_training_queue.put(models_dict[model_id])
        # if net_metadata['id'] != model_id:
        #     new_training_queue.put(net)
    training_queue = new_training_queue

    return True

    # training_thread = Thread(target=worker)
    # training_thread.daemon = True
    # training_thread.start()

def generate_ablations_from_model(model_id, ablation_instructions = None, num_epochs=NUM_EPOCHS_TRAIN):
    # We look for the model's layers in our list of models
    if model_id not in model_layer_dict:
        # Bad request
        return False
    else:
        layers = model_layer_dict[model_id]
        net_list = generate_ablations_from_template(784, 1, 10, layers, ablation_instructions, parent_id=model_id)
        for net_info in net_list:
            net = net_info['net']
            changes = net_info['changes']
            change_indices = net_info['changeIndices']
            if torch.__version__ >= '0.3':
                net = net.to(device)
            net_metadata = net.metadata()
            # logger = Logger(LOG_FILE, load_log=True, socket=socketio)
            if (logger.num_parameters(net) < 3000000):
                model_layer_dict[net_metadata['id']] = net_metadata['_layers']

                socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'parentId': model_id, 'parentType': 'Ablation', 'changes': changes, 'changeIndices': change_indices, 'metadata': net_metadata, 'parameters': logger.num_parameters(net), 'trainingStatus': 'not_started'});

                # At this point, we should add this model to a queue, rather than train it explicitly in this thread.
                add_model_to_training_queue(net, num_epochs)

        return True

def generate_variations_from_model(model_id, number_models, variation_instructions = None, number_changes=1, num_epochs=NUM_EPOCHS_TRAIN):
    # We look for the model's layers in our list of models
    if model_id not in model_layer_dict:
        # Bad request
        return False
    else:
        layers = model_layer_dict[model_id]
        # Nothing to do with ablation instructions yet
        for i in range(number_models):
            net, changes = random_conv_net_from_template(784, 1, 10, layers, variation_instructions, number_changes=number_changes, parent_id=model_id)
            if torch.__version__ >= '0.3':
                net = net.to(device)
            net_metadata = net.metadata()
            # logger = Logger(LOG_FILE, load_log=True, socket=socketio)
            if (logger.num_parameters(net) < 3000000):
                model_layer_dict[net_metadata['id']] = net_metadata['_layers']

                socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'parentId': model_id, 'parentType': 'Variation', 'changes': changes, 'metadata': net_metadata, 'parameters': logger.num_parameters(net), 'trainingStatus': 'not_started'});

                # At this point, we should add this model to a queue, rather than train it explicitly in this thread.
                add_model_to_training_queue(net, num_epochs)

        return True

def generate_handcrafted_from_model(model_id, handcrafted_instructions, changes, num_epochs=NUM_EPOCHS_TRAIN):
    # We look for the model's layers in our list of models
    if model_id not in model_layer_dict:
        # Bad request
        return False
    else:
        # net = exact_conv_net_from_template(1024, 3, 10, handcrafted_instructions, parent_id=model_id)
        net = RandomConvNet(784, 1, 10, template=handcrafted_instructions, skip_template_softmax=True)
        if torch.__version__ >= '0.3':
            net = net.to(device)
        net_metadata = net.metadata()
        # logger = Logger(LOG_FILE, load_log=True, socket=socketio)
        # if (logger.num_parameters(net) < 10000000):
        model_layer_dict[net_metadata['id']] = net_metadata['_layers']

        socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'parentId': model_id, 'parentType': 'Handcrafted', 'changes': changes, 'metadata': net_metadata, 'parameters': logger.num_parameters(net), 'trainingStatus': 'not_started'});

        # At this point, we should add this model to a queue, rather than train it explicitly in this thread.
        add_model_to_training_queue(net, num_epochs)

    return True

def train_model_for_frontend(net, epochs=NUM_EPOCHS_TRAIN):
    global current_model_id

    # logger = Logger(args.logfile, load_log=True)
    if torch.__version__ >= '0.3':
        net = net.to(device)
    net_metadata = net.metadata()

    current_model_id = net_metadata['id']

    # This doesn't seem to work with pytorch 0.2
    logger.log_model(net, net_metadata)

    # print("should be changing to training:")
    socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'metadata': net_metadata, 'parameters': logger.num_parameters(net), 'trainingStatus': 'in_progress'});
    # if device == 'cuda':
    #     net = torch.nn.DataParallel(net)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(net.parameters(), lr=0.001)
    # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2)
    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.75)
    # optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
    # optimizer = optim.Adam(net.parameters())

    # with open(args.model_dir + net_metadata['id'] + '_metadata.json', 'w') as outfile:
    with open(os.path.join(MODEL_DIR, net_metadata['id'] + '_metadata.json'), 'w') as outfile:
        json.dump(net_metadata, outfile)

    print("device is ", device)
    train_model(net, net_metadata, criterion, optimizer, trainloader, testloader, model_dir=MODEL_DIR, num_epochs=epochs, verbose=False, device=device, logger=logger, early_stopping=True, scheduler=scheduler)

    val_acc, _, predictions = calculate_and_print_accuracy_and_return_predictions(net, net_metadata['id'], MODEL_DIR, testloader, verbose=False, device=device)
    hamming_distances = get_hamming_distances(predictions)
    otmann_distances = get_otmann_distances(net_metadata['_layers'])
    hamming_embedding = get_incremental_mds_embedding(hamming_distances, projection_type='hamming')
    otmann_embedding = get_incremental_mds_embedding(otmann_distances, projection_type='otmann')
    print('hamming_embedding', hamming_embedding)
    print('otmann_embedding', otmann_embedding)
    confusion_matrix = generate_confusion_matrix_bins(predictions)
    class_accuracies = calculate_class_accuracies(confusion_matrix)
    socketio.emit('updateModelStatus', {'id': net_metadata['id'], 'hamming_embedding': hamming_embedding.tolist(), 'otmann_embedding': otmann_embedding.tolist(), 'class_accuracies': class_accuracies, 'confusion_matrix': confusion_matrix, 'trainingStatus': 'completed'})
    # print("GOT DISTANCES.  hamming_distances: ", hamming_distances)
    # print("otmann_distances: ", otmann_distances)

    print("saving model to ", MODEL_DIR + net_metadata['id'] + '.model')
    torch.save(net, os.path.join(MODEL_DIR, net_metadata['id'] + '.model'))
    logger.save_log()
    # socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'metadata': net_metadata, 'trainingStatus': 'completed' });

@app.route("/start_model_training/<string:modelId>")
def start_model_training(modelId, device='cpu', num_epochs=2):
    filepath = MODEL_DIR + modelId
    # logger = Logger(args.logfile, load_log=True)
    # logger = Logger(LOG_FILE, load_log=True)
    val = train_model_from_scratch(filepath, trainloader, testloader, model_dir=MODEL_DIR, num_epochs=num_epochs, verbose=True, device=device, logger=logger)
    logger.save_log()
    return str(val)

@app.route("/continue_model_training/<string:modelId>")
def continue_model_training(modelId, device='cpu', num_epochs=2):
    filepath = MODEL_DIR + modelId
    # logger = Logger(args.logfile, load_log=True)
    # logger = Logger(LOG_FILE, load_log=True)
    val = continue_training_model(filepath, trainloader, testloader, model_dir=MODEL_DIR, num_epochs=num_epochs, verbose=True, device=device, logger=logger)
    logger.save_log()
    return str(val)

@app.route("/get_image/<string:imageId>")
def return_image(imageId):
    filepath = os.path.join(IMAGE_DIR, imageId)
    if os.path.exists(filepath):
        return app.send_static_file(imageId)
    else:
        return "INVALID_PATH"

from PIL import Image
import io
import base64

@app.route("/get_random_class_image/<string:image_class>")
def return_random_class_image(image_class):
    # First, calculate image index
    image_class_index = classes.index(image_class)
    print("image class index is ", image_class_index)
    # options = range(image_class_index * NUM_PER_CLASS, (image_class_index + 1) * NUM_PER_CLASS)
    # The way the indexing works for quickdraw, we need to actually subtract out the offsets.
    options = range(image_class_index * int((0.2) * 20000), (image_class_index + 1) * int((0.2) * 20000))
    pointers = random.multinomial(n=1, pvals=[(1.0 / len(options))] * len(options), size=None)
    image_index = options[np.nonzero(pointers)[0][0]]
    # print("sampling uniformly, got", sample)
    print("image_index is ", image_index)
    print("testset length is ", len(testset))
    (img, label) = testset[image_index]
    # print("before reshape, img shape is ", img.shape)
    assert(image_class_index == int(label))
    img = img.reshape((28,28))
    print("img is ", img.shape)
    print("type(img) is ", type(img))
    print(img)
    image = Image.fromarray(img*255)
    img_io = io.BytesIO()
    if image.mode != 'RGB':
        image = image.convert('RGB')
    image.save(img_io, format='PNG')
# pil_img = Image.fromarray(img)
# buff = BytesIO()
# pil_img.save(buff, format="JPEG")
# new_image_string = base64.b64encode(buff.getvalue()).decode("utf-8")
    # img_str = base64.b64encode(img_io.getvalue()).decode('utf-8')
    # print("img_str class name is ", img_str.__class__.__name__)
    print("image_index class name is ", image_index.__class__.__name__)
            # byte_io = io.BytesIO()
            # byte_io.write(file.read())
    img_io.seek(0)
    response = make_response(send_file(img_io,mimetype='image/png'))
    response.headers['Content-Transfer-Encoding']='base64'
    response.headers['Mast-Image-Id'] = image_index
    response.headers['Access-Control-Expose-Headers'] = 'Mast-Image-Id'
    return response

@app.route("/get_image_by_id/<string:imageId>")
def return_image_by_id(imageId):
    filepath = os.path.join(IMAGE_DIR, imageId)
    class_glob_path = Path(os.path.join(IMAGE_DIR))
    class_images = list(class_glob_path.glob(imageId + '_*.png'))
    filepath = class_images[0]
    if filepath and os.path.exists(filepath):
        return app.send_static_file(filepath.name)
    else:
        return "INVALID_PATH"

@app.after_request
def add_header(r):
    """
    Add headers to both force latest IE rendering engine or Chrome Frame,
    and also to cache the rendered page for 10 minutes.
    """
    r.headers["Access-Control-Allow-Origin"] = "*"
    r.headers['Access-Control-Allow-Headers'] = 'Origin, X-Requested-With, Content-Type, Accept';
    r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"
    r.headers["Pragma"] = "no-cache"
    r.headers["Expires"] = "0"
    return r

# Socket IO stuff
training_models = []
@socketio.on('get_models')
def handle_get_models(message):
    print('returning models' + training_models)

@socketio.on('train_random_model')
def handle_train_random_model(message):
    net = RandomConvNet(1024, 3, 10) # for CIFAR sizes
    if torch.__version__ >= '0.3':
        net = net.to(device)
    net_metadata = net.metadata()
    # logger = Logger(LOG_FILE, load_log=True, socket=socketio)
    if (logger.num_parameters(net) < 10000000):
        model_layer_dict[net_metadata['id']] = net_metadata['_layers']

        socketio.emit('updateModelStatus', { 'id': net_metadata['id'], 'metadata': net_metadata, 'parameters': logger.num_parameters(net), 'trainingStatus': 'not_started'});

        # At this point, we should add this model to a queue, rather than train it explicitly in this thread.
        add_model_to_training_queue(net, NUM_EPOCHS_TRAIN)

@socketio.on('kill_queued_model')
def handle_kill_queued_model(message):
    print("message in kill queued models is ", message)
    # We actually rebuild the queue without the model id
    model_id = message['modelId']
    if delete_model_from_training_queue(model_id):
        return {'status': 'ok'}, 200, 'application/json'
    else:
        return {'status': 'bad_delete'}, 500, 'application/json'

@socketio.on('reorder_queued_model')
def handle_reorder_queued_model(message):
    # We actually rebuild the queue the model id spliced in the right place
    print("message is ", message)
    model_ids_order = message['modelIdOrderedList']
    if reorder_model_from_training_queue(model_ids_order):
        return {'status': 'ok'}, 200, 'application/json'
    else:
        return {'status': 'bad_delete'}, 500, 'application/json'

@socketio.on('generate_ablations')
def handle_generate_ablations(message):
    print("message is ", message)
    ablation_instructions = message['instructions']
    model_id = message['modelId']
    num_epochs = message['numEpochs']

    if generate_ablations_from_model(model_id, ablation_instructions, num_epochs=num_epochs):
        return {'status': 'ok'}, 200, 'application/json'
    else:
        return {'status': 'error'}, 500, 'application/json'

@socketio.on('generate_variations')
def handle_generate_variations(message):
    # print("message is ", message)
    variation_instructions = message['instructions']
    number_models = message['numModels'] or 5
    number_changes = message['numChanges'] or 1
    model_id = message['modelId']
    num_epochs = message['numEpochs']

    if generate_variations_from_model(model_id, number_models, variation_instructions, number_changes=number_changes, num_epochs=num_epochs):
        return {'status': 'ok'}, 200, 'application/json'
    else:
        return {'status': 'error'}, 500, 'application/json'

@socketio.on('generate_handcrafted')
def handle_generate_handcrafted(message):
    # print("message is ", message)
    handcrafted_instructions = message['instructions']
    model_id = message['modelId']
    num_epochs = message['numEpochs']
    changes = message['changes']
    print("handcrafted template is ", handcrafted_instructions)

    if generate_handcrafted_from_model(model_id, handcrafted_instructions, changes=changes, num_epochs=num_epochs):
        return {'status': 'ok'}, 200, 'application/json'
    else:
        return {'status': 'error'}, 500, 'application/json'

device = 'cuda'
# device = 'cpu'

if __name__ == "__main__":
    # app.run(host='0.0.0.0', port=80)
    # app.run(host='localhost', port=80)
    app.run(host='localhost', port=5000)
    # parser = argparse.ArgumentParser()

    # parser.add_argument("--method", default='none', help="either start_model_training or continue_model_training")
    # parser.add_argument("--model_id", default='none', help="ID of architecture to train")
    # parser.add_argument("--logfile", help="Logfile to save intermediate training info", default=LOG_FILE)
    # parser.add_argument("--cuda", help="use cuda if GPUs are available",
    #                     action="store_true")
    # parser.add_argument("--epochs", type=int, help="number of epochs each model trained", default=2)
    # args = parser.parse_args()

    # device = 'cpu'
    # if args.cuda:
    #     device = 'gpu'

    # if args.method == 'start_model_training':
    #     start_model_training(args.model_id, device=device, num_epochs=args.epochs)

    # if args.method == 'continue_model_training':
    #     continue_model_training(args.model_id, device=device, num_epochs=args.epochs)

reason: import numpy,from numpy,from scipy


text:
# -*- coding: utf-8 -*-

import tensorflow as tf
import numpy as np
import requests
import logging
import shutil
import os

from static import CHECKPOINTS_DIR

from grpc.beta import implementations
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2

# os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

data_save_dir = os.path.join(CHECKPOINTS_DIR, 'TensorFlowServing')
graph_save_dir = os.path.join(CHECKPOINTS_DIR, 'TensorFlowServing', 'graph')

# x_test = np.linspace(2 * -np.pi, 2 * np.pi, num=100).reshape((-1, 1))
# y_test = np.sin(x_test)

# x_train = x_test + np.random.normal(0.3, 0.003)
# y_train = np.sin(x_train) + np.random.normal(0.0, 0.00003)

# x_train = x_train.astype(np.float32)
# y_train = y_train.astype(np.float32)

x_train = np.load(os.path.join(data_save_dir, 'x_train.npy')).astype(np.float32)
y_train = np.load(os.path.join(data_save_dir, 'y_train.npy')).astype(np.float32)

np.save(os.path.join(data_save_dir, 'x_train.npy'), x_train)
np.save(os.path.join(data_save_dir, 'y_train.npy'), y_train)

def train():

    session = tf.Session()

    x_input = tf.placeholder(tf.float32, [None, 1], name='x_input')
    y_input = tf.placeholder(tf.float32, [None, 1], name='y_input')

    fc1 = tf.layers.dense(x_input, 10, tf.nn.relu)
    fc2 = tf.layers.dense(fc1, 10, tf.nn.relu)

    y_predict = tf.layers.dense(fc2, 1)

    loss_func = tf.losses.mean_squared_error(labels=y_input, predictions=y_predict)

    optimizer = tf.train.AdamOptimizer().minimize(loss_func)

    session.run(tf.global_variables_initializer())

    signature = tf.saved_model.signature_def_utils.build_signature_def(
        inputs={
            'x_input': tf.saved_model.utils.build_tensor_info(x_input),
            'y_input': tf.saved_model.utils.build_tensor_info(y_input)
        },
        outputs={
            'y_predict': tf.saved_model.utils.build_tensor_info(y_predict),
            'loss_func': tf.saved_model.utils.build_tensor_info(loss_func)
        },
        method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
    )

    for step in range(2000):
        session.run(optimizer, {
            x_input: x_train,
            y_input: y_train
        })
        if (step + 1) % 500 == 0:
            if os.path.exists(graph_save_dir):
                shutil.rmtree(graph_save_dir)
            builder = tf.saved_model.builder.SavedModelBuilder(graph_save_dir)
            builder.add_meta_graph_and_variables(session,
                                                 [tf.saved_model.tag_constants.SERVING],
                                                 {tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature})
            # builder.add_meta_graph([tf.saved_model.tag_constants.SERVING], {'signature': signature})
            builder.save()

    loss = session.run(loss_func, {
        x_input: x_train,
        y_input: y_train
    })

    logging.warning('Loss: {}'.format(loss))
    # builder.add_meta_graph_and_variables(session, [tf.saved_model.tag_constants.TRAINING], {'signature': signature})
    # builder.add_meta_graph([tf.saved_model.tag_constants.SERVING], {'signature': signature})
    # builder.save()

def test():
    # Session.
    session = tf.Session()
    # Load meta graph.
    meta_graph_def = tf.saved_model.loader.load(session, [tf.saved_model.tag_constants.SERVING], graph_save_dir)  # type: tf.MetaGraphDef
    # Get signature.
    signature_def = meta_graph_def.signature_def
    signature = signature_def[tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
    # Get input tensor.
    x_input_tensor = signature.inputs['x_input'].name
    y_input_tensor = signature.inputs['y_input'].name
    # Get output tensor.
    y_predict_tensor = signature.outputs['y_predict'].name
    # Get loss func.
    loss_op = signature.outputs['loss_func'].name

    _, loss = session.run([y_predict_tensor, loss_op], {
        x_input_tensor: x_train,
        y_input_tensor: y_train,
    })

    logging.warning('Loss: {}'.format(loss))

def inference_v1():
    # Init channel.
    channel = implementations.insecure_channel('localhost', 9000)
    # Init stub.
    stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
    # Init request.
    request = predict_pb2.PredictRequest()
    request.model_spec.name = 'test'
    request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
    request.inputs['x_input'].CopyFrom(
        tf.contrib.util.make_tensor_proto(x_train, shape=x_train.shape)
    )
    request.inputs['y_input'].CopyFrom(
        tf.contrib.util.make_tensor_proto(y_train, shape=y_train.shape)
    )
    # Predict.
    future = stub.Predict.future(request, 2.0)
    result = future.result().outputs['loss_func'].float_val
    logging.warning('Loss: {}'.format(result))

def inference_v2():
    # Init url.
    url = "http://localhost:9001/v1/models/test:predict"
    # url = 'http://172.16.11.43:10000/tool_list/test'
    # Init body.
    import json
    body = {
        # 'signature_name': tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
        'instances':  [
            {
                'x_input': json.dumps(x_train.tolist(), ensure_ascii=True)
            }
        ]
    }
    # Post.
    # response = requests.post(url, data=body)
    response = requests.post(url, json=json.dumps(body))
    logging.warning('{}'.format(response.text))
    return response

train()
# test()
# inference_v1()
# inference_v2()

# plt.plot(y_train)
# plt.plot(y_test)
# plt.show()

ext: py

max_stars_repo_name: Ceruleanacg/Learning-Notes

max_issues_repo_name: Ceruleanacg/Descent

max_forks_repo_name: Ceruleanacg/Descent

reason: import numpy


text:
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created for Python 3

@author: Sebastian M. Gaebel
@email: sgaebel@star.sr.bham.ac.uk
"""

from __future__ import division, print_function
import h5py
import numpy as np
import os
import scipy.interpolate

# Global variables to reuse objects for multiple calls
_random_thetas = None
_interpolator = None
_sens = None

def detection_probability(m1, m2, redshift, distance, snr_threshold,sensitivity='design'):
    """
    Returns the detection probability of a CBC event with given
    masses and distance.

    This function is a convenience function to generate the
    interpolator with 'SNR_Grid_IMRPhenomPv2_FD_all_noise.hdf5'
    and 'SimNoisePSDaLIGODesignSensitivityP1200087', redshift the
    masses, rescale the SNR using the distance, and then call
    'detection_probability_from_snr' to return the probability.

    Parameters
    ----------
    m1, m2 : float > 0
        Primary and secondary mass in source frame.
    redshift : float >= 0
        Redshift of the waveform through cosmology.
    distance : float > 0
        Luminosity Distance in Mpc to the source.
    snr_threshold : float > 0
        Threshold above which an event is considered to be detected.
    sensitivity : str
        Which detector sensitivity PSD to use. Options are 'design' and 'O1'

    Returns
    -------
    out : float
        Estimate of the detection probability.

    Notes
    -----
    The interpolator is only initialized once and then stored in a
    module level global variable to be reused.
    """

    global _interpolator
    global _sens
    if (_interpolator is None) or (_sens is not sensitivity):
        _sens = sensitivity
        _interpolator = SNRinterpolator(_sens)
    interpolated_snr = _interpolator(m1*(1+redshift), m2*(1+redshift))
    # SNR scales as 1/distance
    interpolated_snr /= distance

    return detection_probability_from_snr(snr_value=interpolated_snr,
                                          snr_threshold=snr_threshold)

def SNRinterpolator(sensitivity='design'):
    """
    Returns an Interpolator class instance for a given sensitivity

    This function is a convenience function to generate the
    interpolator with 'SNR_Grid_IMRPhenomPv2_FD_all_noise.hdf5'
    and 'SimNoisePSDaLIGODesignSensitivityP1200087'.

    Parameters
    ----------
    sensitivity : str
    Which detector sensitivity PSD to use. Options are 'design' and 'O1'

    Returns
    -------
    out : Interpolator
    Interpolator class instance

    Notes
    -----
    The interpolator is only initialized once and then stored in a
    module level global variable to be reused.
    """
    path = os.path.dirname(os.path.abspath(__file__))

    if sensitivity == 'design':
        hdfDatasetName = 'SimNoisePSDaLIGODesignSensitivityP1200087'
    elif sensitivity == 'O1':
        hdfDatasetName = 'P1500238_GW150914_H1-GDS-CALIB_STRAIN.txt'
    elif sensitivity == 'O3':
        hdfDatasetName = 'SimNoisePSDaLIGOMidHighSensitivityP1200087'

    global _interpolator
    global _sens
    if (_interpolator is None) or (_sens is not sensitivity):
        _sens = sensitivity
        _interpolator = Interpolator(
                                     path+'/SNR_Grid_IMRPhenomPv2_FD_all_noise.hdf5',
                                     hdfDatasetName, mode='scipy')
    return _interpolator

class Interpolator:
    """
    Interpolation class to estimate the values of arbitrary points
    from a given grid. The interpolator class is initialized with
    a mass axis defining the (symmetric) grid, a grid of values
    which are used as reference points, and a operation mode.

    The operation mode determines the interpolation method used and
    may be either 'scipy', which uses 'RectBivariateSpline' from
    'scipy.interpolate', or 'custom' where all non-NaN points adjacent
    to the call are averaged while being weighed by the inverse of
    the distance between the given corner point and queried position.

    The grid is generally assumed to be spaced uniformly in log,
    therefore mass_axis and evaluation masses are transformed to
    log space before interpolating.
    """
    def __init__(self, first_arg, second_arg, mode='scipy'):
        """
        Initialize the interpolator by definig the used reference
        values and interpolation mode.

        The interpolator may be initialized by providing the mass
        axis and SNR grid, or via a path to a stored precomputed
        grid and a string specifying the noise spectrum to use.

        Parameters
        ----------
        first_arg : mass axis as 1D ndarray or path as string
            If given as mass axis, must be a one dimensional
            ndarray. If given as string, must be a valid path
            to a HDF5 file containing mass axis and SNR grid
            as produced by 'generate_SNR_files.py'.
        second_arg : SNR grid as 2D ndarray or noise spectrum as string
            If given as SNR grid, must be a two dimensional
            ndarray with each dimension being the length of the mass
            axis. If given as noise curve, must be the string specifying
            a valid group of the HDF5 file given above.
        mode : str, optional
            Valid values are 'scipy' and 'custom', all other will
            raise a ValueError.

        Returns
        -------
        out : None
        """
        if isinstance(first_arg, str) and isinstance(second_arg, str):
            if not os.path.isfile(first_arg):
                raise FileNotFoundError('HDF5 file expected: %r' % first_arg)
            with h5py.File(first_arg, 'r') as hdf:
                mass_axis = hdf['mass_axis'][...]
                if second_arg not in hdf['snr_values']:
                    raise ValueError('Group %r not found.' % second_arg)
                snr_grid = hdf['snr_values'][second_arg][...]
        else:
            mass_axis = first_arg
            snr_grid = second_arg

        if mode == 'scipy':
            self.use_scipy = True
            self.interpolator = scipy.interpolate.RectBivariateSpline(
                np.log(mass_axis), np.log(mass_axis), snr_grid)
        elif mode == 'custom':
            self.use_scipy = False
            self.mass_axis = np.log(mass_axis)
            self.snr_grid = snr_grid
        else:
            raise ValueError('Invalid mode: %r' % mode)

    def __call__(self, m1, m2):
        """
        Obtain an interpolated value for the given position.

        Parameters
        ----------
        m1, m2 : float
            Positive values for which the interpolated SNR is calculated.

        Returns
        -------
        out : float
            Estimate for the SNR at (m1, m2).
        """
        if self.use_scipy:
            return self.interpolator(np.log(m1), np.log(m2), grid=False)
        else:
            return self._custom(np.log(m1), np.log(m2))

    def _custom(self, m1, m2):
        """
        Custom function to approximate the value of any given point
        within the limits of the grid. The value is calculated as the
        average value of the 4 corners of the grid, weighed by the
        inverse distance to (m1, m2). Points for which the SNR is
        NaN are ignored.

        Parameters
        ----------
        m1, m2 : float
            Positive values for which the interpolated SNR is calculated.

        Returns
        -------
        out : float
            Estimate for the SNR at (m1, m2).
        """
        # The point is in the lower triangle by definition
        if isinstance(m1, np.ndarray):
            ret_values = np.empty_like(m1)
            for i, (m1_, m2_) in enumerate(zip(m1, m2)):
                ret_values[i] = self._custom(m1_, m2_)
            return ret_values
        elif m1 < m2:
            m1, m2 = m2, m1
        assert np.min(self.mass_axis) < m1 < np.max(self.mass_axis)
        assert np.min(self.mass_axis) < m2 < np.max(self.mass_axis)
        # Find the indices of the next higher element in the axis
        # this ensures mass_axis[i-1] < m1 < mass_axis[i]
        # Edge cases are covered by the assertions, which ensure no pair are on
        # the outer bondary.
        i, j = np.searchsorted(self.mass_axis, [m1, m2])
        m1_selection = self.mass_axis[[i-1, i-1, i, i]]
        m2_selection = self.mass_axis[[j-1, j, j-1, j]]
        snr_selection = self.snr_grid[[i-1, i-1, i, i], [j-1, j, j-1, j]]
        distance = np.sqrt((m1_selection - m1)**2 + (m2_selection - m2)**2)
        # If the distance is zero, return the value at that position
        if np.any(distance == 0):
            return snr_selection[distance == 0]
        # Ensure there are no NaN in the SNR values,which may happen for the
        # upper triangle.
        distance = distance[~np.isnan(snr_selection)]
        snr_selection = snr_selection[~np.isnan(snr_selection)]
        if snr_selection.size == 0:
            raise ValueError('No non-NaN values surrounding (m1, m2).')
        return np.average(snr_selection, weights=1./distance)

def detection_probability_from_snr(snr_value, snr_threshold, n_thetas=1e6):
    """
    Compute the probability of detecting an CBC with given SNR and
    threshold, averaging over all orientations and sky positions.

    Based of Finn & Chernoff 1993 (https://arxiv.org/abs/gr-qc/9301003).

    Parameters
    ----------
    snr_value : float or array of floats
        SNR value of the event under ideal conditions. Must be positive.
    snr_threshold : float
        SNR threshold for detection. Must be positive.
    n_thetas : unsigned int
        Number of random realizations of inclination and sky position
        used to calculate the average probability of detecting the
        event. Default: 1e6.

    Returns
    -------
    out : float or array of floats (same as 'snr_value')
        Probability of the received signal to be above the threshold.
    """
    # Check input values
    assert np.all(snr_value > 0), repr(snr_value)
    assert snr_threshold > 0, repr(snr_threshold)
    assert n_thetas > 0, repr(n_thetas)
    n_thetas = int(n_thetas)

    # Generate thetas if necessary
    global _random_thetas
    if _random_thetas is None or _random_thetas.size != n_thetas:
        cos_thetas = np.random.uniform(low=-1, high=1, size=n_thetas)
        cos_incs = np.random.uniform(low=-1, high=1, size=n_thetas)
        phis = np.random.uniform(low=0, high=2*np.pi, size=n_thetas)
        zetas = np.random.uniform(low=0, high=2*np.pi, size=n_thetas)

        Fps = (0.5 * np.cos(2*zetas) * (1+cos_thetas**2)*np.cos(2*phis) -
               np.sin(2*zetas) * cos_thetas*np.sin(2*phis))  # 3.29a
        Fxs = (0.5 * np.sin(2*zetas) * (1+cos_thetas**2)*np.cos(2*phis) +
               np.cos(2*zetas) * cos_thetas*np.sin(2*phis))  # 3.29b
        _random_thetas = np.sqrt(0.25 * Fps**2 * (1 + cos_incs**2)**2 +
                                 Fxs**2 * cos_incs**2)  # 3.31
        _random_thetas = np.sort(_random_thetas)

    # Calculate and return the probability of detection
    # From Finn & Chernoff 1993, we have SNR ~ theta*integrand, assuming
    # that the polarisations are orthogonal.
    # Remember: snr_value is the maximum possible SNR
    theta_min = snr_threshold / snr_value
    # theta_min is converted to a 1-dimensional array so this function
    # can be used with array and scalar arguments.

    #I have replaced this easy to read for loop with an unreadable (but much faster) one liner. Leaving it commented so
    #that future people can understand what the hell it's doing

    # theta_min = np.atleast_1d(theta_min).flatten()
    # detection_prob = np.zeros_like(theta_min)
    # for i, theta_min_value in enumerate(theta_min):
    #     if theta_min_value <= 1:
    #         detection_prob[i] = np.mean(_random_thetas > theta_min_value)

    theta_min = np.atleast_1d(theta_min).flatten()
    detection_prob = np.zeros_like(theta_min)
    detection_prob[theta_min <= 1] = 1.-((np.digitize(theta_min[theta_min <= 1], _random_thetas)-1.)/float(n_thetas))

    if len(detection_prob) == 1 and np.array(snr_value).ndim == 0:
        # If the original arg was 0-dimensional (scalar), return a scalar.
        return detection_prob[0]
    return detection_prob.reshape(np.array(snr_value).shape)

reason: import numpy,import scipy


text:
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
import pandas as pd
import numpy as np
import itertools
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, roc_auc_score

def plot_confusion_matrix(df, classes = ["0", "1"],
           normalize=False,
           title='Confusion matrix',
           cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    print("F1 score      >> ", sklearn.metrics.f1_score(df["truth"], df["pred_label"]))
    print("ROC AUC score >> ", roc_auc_score(df["truth"], df["pred_label"]))
    cm = sklearn.metrics.confusion_matrix(df["truth"], df["pred_label"])
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')

def analyze_class_predictions(df,thr):
    """
        Plot the classwise prediction distributions
    """
    fig, (ax1, ax2) = plt.subplots(nrows=2, sharey=True ,figsize=(10,12))
    sns.distplot(df[df.truth == 0]["predicted_risk"], bins = 30, color = "navy", ax = ax1)
    #ax1.axvline(thr, 'k')
    ax1.set_title("Risks predicted for Negative records")
    sns.distplot(df[df.truth == 1]["predicted_risk"], bins = 30, color = "maroon", ax = ax2)
    ax2.set_title("Risks predicted for Positive records")
    #ax2.axvline(thr, 'k')
    plt.show()

def plot_tuning_stats(df, metric = "AUC", min_value_to_show = 0):
    if min_value_to_show:
        df = df[df[metric] > min_value_to_show]
    plt.figure(figsize=(6,6))
    plt.style.use('seaborn-whitegrid')
    FEATURES = list(df.columns)[1:]
    for feat in FEATURES:
        plt.figure(figsize=(5,5))
        plt.plot(df[feat], df[metric], '*')
        plt.title(feat)
        plt.show()

max_stars_repo_name: zeroknowledgediscovery/ehrzero

max_issues_repo_name: zeroknowledgediscovery/ehrzero

max_forks_repo_name: zeroknowledgediscovery/ehrzero


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import torch.nn as nn
import torch as T
from torch.autograd import Variable as var
import torch.nn.functional as F
import numpy as np
import math

from flann_index import FLANNIndex
from util import *
import time

class SparseTemporalMemory(nn.Module):

  def __init__(
      self,
      input_size,
      mem_size=512,
      cell_size=32,
      independent_linears=True,
      read_heads=4,
      sparse_reads=4,
      temporal_reads=4,
      num_lists=None,
      index_checks=32,
      gpu_id=-1,
      mem_gpu_id=-1
  ):
    super(SparseTemporalMemory, self).__init__()

    self.mem_size = mem_size
    self.cell_size = cell_size
    self.gpu_id = gpu_id
    self.mem_gpu_id = mem_gpu_id
    self.input_size = input_size
    self.independent_linears = independent_linears
    self.K = sparse_reads if self.mem_size > sparse_reads else self.mem_size
    self.KL = temporal_reads if self.mem_size > temporal_reads else self.mem_size
    self.read_heads = read_heads
    self.num_lists = num_lists if num_lists is not None else int(self.mem_size / 100)
    self.index_checks = index_checks

    m = self.mem_size
    w = self.cell_size
    r = self.read_heads
    # The visible memory size: (K * R read heads, forward and backward
    # temporal reads of size KL and least used memory cell)
    self.c = (r * self.K) + (self.KL * 2) + 1

    if self.independent_linears:
      self.read_query_transform = nn.Linear(self.input_size, w * r)
      self.write_vector_transform = nn.Linear(self.input_size, w)
      self.interpolation_gate_transform = nn.Linear(self.input_size, self.c)
      self.write_gate_transform = nn.Linear(self.input_size, 1)
      T.nn.init.orthogonal_(self.read_query_transform.weight)
      T.nn.init.orthogonal_(self.write_vector_transform.weight)
      T.nn.init.orthogonal_(self.interpolation_gate_transform.weight)
      T.nn.init.orthogonal_(self.write_gate_transform.weight)
    else:
      self.interface_size = (r * w) + w + self.c + 1
      self.interface_weights = nn.Linear(self.input_size, self.interface_size)
      T.nn.init.orthogonal_(self.interface_weights.weight)

    self.I = cuda(1 - T.eye(self.c).unsqueeze(0), gpu_id=self.gpu_id)  # (1 * n * n)
    self.δ = 0.005  # minimum usage
    self.timestep = 0
    self.mem_limit_reached = False

  def rebuild_indexes(self, hidden, erase=False):
    b = hidden['memory'].size(0)

    # if indexes already exist, we reset them
    if 'indexes' in hidden:
      [x.reset() for x in hidden['indexes']]
    else:
      # create new indexes
      try:
        from .faiss_index import FAISSIndex
        hidden['indexes'] = \
            [FAISSIndex(cell_size=self.cell_size,
                        nr_cells=self.mem_size, K=self.K, num_lists=self.num_lists,
                        probes=self.index_checks, gpu_id=self.mem_gpu_id) for x in range(b)]
      except Exception as e:
        print("\nFalling back to FLANN (CPU). \nFor using faster, GPU based indexes, install FAISS: `conda install faiss-gpu -c pytorch`")
        from flann_index import FLANNIndex
        hidden['indexes'] = \
            [FLANNIndex(cell_size=self.cell_size,
                        nr_cells=self.mem_size, K=self.K, num_kdtrees=self.num_lists,
                        probes=self.index_checks, gpu_id=self.mem_gpu_id) for x in range(b)]

    # add existing memory into indexes
    pos = hidden['read_positions'].squeeze().data.cpu().numpy()
    if not erase:
      for n, i in enumerate(hidden['indexes']):
        i.reset()
        i.add(hidden['memory'][n], last=pos[n][-1])
    else:
      self.timestep = 0
      self.mem_limit_reached = False

    return hidden

  def reset(self, batch_size=1, hidden=None, erase=True):
    m = self.mem_size
    w = self.cell_size
    b = batch_size
    r = self.read_heads
    c = self.c

    if hidden is None:
      hidden = {
          # warning can be a huge chunk of contiguous memory
          'memory': cuda(T.zeros(b, m, w).fill_(δ), gpu_id=self.mem_gpu_id),
          'visible_memory': cuda(T.zeros(b, c, w).fill_(δ), gpu_id=self.mem_gpu_id),
          'link_matrix': cuda(T.zeros(b, m, self.KL * 2), gpu_id=self.gpu_id),
          'rev_link_matrix': cuda(T.zeros(b, m, self.KL * 2), gpu_id=self.gpu_id),
          'precedence': cuda(T.zeros(b, self.KL * 2).fill_(δ), gpu_id=self.gpu_id),
          'read_weights': cuda(T.zeros(b, m).fill_(δ), gpu_id=self.gpu_id),
          'write_weights': cuda(T.zeros(b, m).fill_(δ), gpu_id=self.gpu_id),
          'read_vectors': cuda(T.zeros(b, r, w).fill_(δ), gpu_id=self.gpu_id),
          'least_used_mem': cuda(T.zeros(b, 1).fill_(c + 1), gpu_id=self.gpu_id).long(),
          'usage': cuda(T.zeros(b, m), gpu_id=self.gpu_id),
          'read_positions': cuda(T.arange(0, c).expand(b, c), gpu_id=self.gpu_id).long()
      }
      hidden = self.rebuild_indexes(hidden, erase=True)
    else:
      hidden['memory'] = hidden['memory'].clone()
      hidden['visible_memory'] = hidden['visible_memory'].clone()
      hidden['link_matrix'] = hidden['link_matrix'].clone()
      hidden['rev_link_matrix'] = hidden['link_matrix'].clone()
      hidden['precedence'] = hidden['precedence'].clone()
      hidden['read_weights'] = hidden['read_weights'].clone()
      hidden['write_weights'] = hidden['write_weights'].clone()
      hidden['read_vectors'] = hidden['read_vectors'].clone()
      hidden['least_used_mem'] = hidden['least_used_mem'].clone()
      hidden['usage'] = hidden['usage'].clone()
      hidden['read_positions'] = hidden['read_positions'].clone()
      hidden = self.rebuild_indexes(hidden, erase)

      if erase:
        hidden['memory'].data.fill_(δ)
        hidden['visible_memory'].data.fill_(δ)
        hidden['link_matrix'].data.zero_()
        hidden['rev_link_matrix'].data.zero_()
        hidden['precedence'].data.zero_()
        hidden['read_weights'].data.fill_(δ)
        hidden['write_weights'].data.fill_(δ)
        hidden['read_vectors'].data.fill_(δ)
        hidden['least_used_mem'].data.fill_(c + 1 + self.timestep)
        hidden['usage'].data.fill_(0)
        hidden['read_positions'] = cuda(
            T.arange(self.timestep, c + self.timestep).expand(b, c), gpu_id=self.gpu_id).long()

    return hidden

  def write_into_sparse_memory(self, hidden):
    visible_memory = hidden['visible_memory']
    positions = hidden['read_positions']

    (b, m, w) = hidden['memory'].size()
    # update memory
    hidden['memory'].scatter_(1, positions.unsqueeze(2).expand(b, self.c, w), visible_memory)

    # non-differentiable operations
    pos = positions.data.cpu().numpy()
    for batch in range(b):
      # update indexes
      hidden['indexes'][batch].reset()
      hidden['indexes'][batch].add(hidden['memory'][batch], last=(pos[batch][-1] if not self.mem_limit_reached else None))

    mem_limit_reached = hidden['least_used_mem'][0].data.cpu().numpy()[0] >= self.mem_size - 1
    self.mem_limit_reached = mem_limit_reached or self.mem_limit_reached

    return hidden

  def update_link_matrices(self, link_matrix, rev_link_matrix, write_weights, precedence, temporal_read_positions):
    write_weights_i = write_weights.unsqueeze(2)
    precedence_j = precedence.unsqueeze(1)

    (b, m, k) = link_matrix.size()
    I = cuda(T.eye(m, k).unsqueeze(0).expand((b, m, k)), gpu_id=self.gpu_id)

    # since only KL*2 entries are kept non-zero sparse, create the dense version from the sparse one
    precedence_dense = cuda(T.zeros(b, m), gpu_id=self.gpu_id)
    precedence_dense.scatter_(1, temporal_read_positions, precedence)
    precedence_dense_i = precedence_dense.unsqueeze(2)

    temporal_write_weights_j = write_weights.gather(1, temporal_read_positions).unsqueeze(1)

    link_matrix = (1 - write_weights_i) * link_matrix + write_weights_i * precedence_j

    rev_link_matrix = (1 - temporal_write_weights_j) * rev_link_matrix + \
        (temporal_write_weights_j * precedence_dense_i)

    return link_matrix * I, rev_link_matrix * I

  def update_precedence(self, precedence, write_weights):
    return (1 - T.sum(write_weights, dim=-1, keepdim=True)) * precedence + write_weights

  def write(self, interpolation_gate, write_vector, write_gate, hidden):

    read_weights = hidden['read_weights'].gather(1, hidden['read_positions'])
    # encourage read and write in the first timestep
    if self.timestep == 1: read_weights =  read_weights + 1
    write_weights = hidden['write_weights'].gather(1, hidden['read_positions'])

    hidden['usage'], I = self.update_usage(
        hidden['read_positions'],
        read_weights,
        write_weights,
        hidden['usage']
    )

    # either we write to previous read locations

    x = interpolation_gate * read_weights
    # or to a new location
    y = (1 - interpolation_gate) * I
    write_weights = write_gate * (x + y)

    # store the write weights
    hidden['write_weights'].scatter_(1, hidden['read_positions'], write_weights)

    # erase matrix
    erase_matrix = I.unsqueeze(2).expand(hidden['visible_memory'].size())

    # write into memory
    hidden['visible_memory'] = hidden['visible_memory'] * \
        (1 - erase_matrix) + T.bmm(write_weights.unsqueeze(2), write_vector)
    hidden = self.write_into_sparse_memory(hidden)

    # update link_matrix and precedence
    (b, c) = write_weights.size()

    # update link matrix
    temporal_read_positions = hidden['read_positions'][:, self.read_heads * self.K + 1:]
    hidden['link_matrix'], hidden['rev_link_matrix'] = \
        self.update_link_matrices(
        hidden['link_matrix'],
        hidden['rev_link_matrix'],
        hidden['write_weights'],
        hidden['precedence'],
        temporal_read_positions
    )

    # update precedence vector
    read_weights = hidden['read_weights'].gather(1, temporal_read_positions)
    hidden['precedence'] = self.update_precedence(hidden['precedence'], read_weights)

    # update least used memory cell
    hidden['least_used_mem'] = T.topk(hidden['usage'], 1, dim=-1, largest=False)[1]

    return hidden

  def update_usage(self, read_positions, read_weights, write_weights, usage):
    (b, _) = read_positions.size()
    # usage is timesteps since a non-negligible memory access
    u = (read_weights + write_weights > self.δ).float()

    # usage before write
    relevant_usages = usage.gather(1, read_positions)

    # indicator of words with minimal memory usage
    minusage = T.min(relevant_usages, -1, keepdim=True)[0]
    minusage = minusage.expand(relevant_usages.size())
    I = (relevant_usages == minusage).float()

    # usage after write
    relevant_usages = (self.timestep - relevant_usages) * u + relevant_usages * (1 - u)

    usage.scatter_(1, read_positions, relevant_usages)

    return usage, I

  def directional_weightings(self, link_matrix, rev_link_matrix, temporal_read_weights):
    f = T.bmm(link_matrix, temporal_read_weights.unsqueeze(2)).squeeze(2)
    b = T.bmm(rev_link_matrix, temporal_read_weights.unsqueeze(2)).squeeze(2)
    return f, b

  def read_from_sparse_memory(self, memory, indexes, keys, least_used_mem, usage, forward, backward, prev_read_positions):
    b = keys.size(0)
    read_positions = []

    # we search for k cells per read head
    for batch in range(b):
      distances, positions = indexes[batch].search(keys[batch])
      read_positions.append(positions)
    read_positions = T.stack(read_positions, 0)

    # add least used mem to read positions
    # TODO: explore possibility of reading co-locations or ranges and such
    (b, r, k) = read_positions.size()
    read_positions = var(read_positions).squeeze(1).view(b, -1)

    # no gradient here
    # temporal reads
    (b, m, w) = memory.size()
    # get the top KL entries
    max_length = int(least_used_mem[0, 0].data.cpu().numpy()) if not self.mem_limit_reached else (m-1)

    _, fp = T.topk(forward, self.KL, largest=True)
    _, bp = T.topk(backward, self.KL, largest=True)

    # differentiable ops
    # append forward and backward read positions, might lead to duplicates
    read_positions = T.cat([read_positions, fp, bp], 1)
    read_positions = T.cat([read_positions, least_used_mem], 1)
    read_positions = T.clamp(read_positions, 0, max_length)

    # use the indixes of the sparse reads, as well as the indixes for least used and ba/fwd and look them up in the memory
    visible_memory = memory.gather(1, read_positions.unsqueeze(2).expand(b, self.c, w))

    # take the vectors of the sparse reads and lru and let the read heads each look for the most similiar vector, then do softmax among all the vectors
    # for each head (b x r x (r*k + lru + fwd + ba))
    read_weights = σ(θ(visible_memory, keys), 2)
    # let each head return one vector based on the previous softmax (b x r x w)
    read_vectors = T.bmm(read_weights, visible_memory)
    read_weights = T.prod(read_weights, 1)

    return read_vectors, read_positions, read_weights, visible_memory

  def read(self, read_query, hidden):
    # get forward and backward weights
    temporal_read_positions = hidden['read_positions'][:, self.read_heads * self.K + 1:]
    read_weights = hidden['read_weights'].gather(1, temporal_read_positions)
    forward, backward = self.directional_weightings(hidden['link_matrix'], hidden['rev_link_matrix'], read_weights)

    # sparse read
    read_vectors, positions, read_weights, visible_memory = \
        self.read_from_sparse_memory(
            hidden['memory'],
            hidden['indexes'],
            read_query,
            hidden['least_used_mem'],
            hidden['usage'],
            forward, backward,
            hidden['read_positions']
        )

    hidden['read_positions'] = positions
    hidden['read_weights'] = hidden['read_weights'].scatter_(1, positions, read_weights)
    hidden['read_vectors'] = read_vectors
    hidden['visible_memory'] = visible_memory

    return hidden['read_vectors'], hidden

  def forward(self, ξ, hidden):
    t = time.time()

    # ξ = ξ.detach()
    m = self.mem_size
    w = self.cell_size
    r = self.read_heads
    c = self.c
    b = ξ.size()[0]

    if self.independent_linears:
      # r read keys (b * r * w)
      read_query = self.read_query_transform(ξ).view(b, r, w)
      # write key (b * 1 * w)
      write_vector = self.write_vector_transform(ξ).view(b, 1, w)
      # write vector (b * 1 * r)
      interpolation_gate = T.sigmoid(self.interpolation_gate_transform(ξ)).view(b, c)
      # write gate (b * 1)
      write_gate = T.sigmoid(self.write_gate_transform(ξ).view(b, 1))
    else:
      ξ = self.interface_weights(ξ)
      # r read keys (b * r * w)
      read_query = ξ[:, :r * w].contiguous().view(b, r, w)
      # write key (b * 1 * w)
      write_vector = ξ[:, r * w: r * w + w].contiguous().view(b, 1, w)
      # write vector (b * 1 * r)
      interpolation_gate = T.sigmoid(ξ[:, r * w + w: r * w + w + c]).contiguous().view(b, c)
      # write gate (b * 1)
      write_gate = T.sigmoid(ξ[:, -1].contiguous()).unsqueeze(1).view(b, 1)

    self.timestep += 1
    hidden = self.write(interpolation_gate, write_vector, write_gate, hidden)
    return self.read(read_query, hidden)

max_stars_repo_name: Xirider/tdnc

max_issues_repo_name: Xirider/tdnc

max_forks_repo_name: Xirider/tdnc


text:
from timemachines.skatertools.utilities.conventions import Y_TYPE, A_TYPE, E_TYPE, T_TYPE, wrap
from typing import Any
from timemachines.skatertools.components.residuals import residual

def residual_chaser_factory(y :Y_TYPE, s:dict, k:int, a:A_TYPE =None, t:T_TYPE =None, e:E_TYPE =None,
                            f1=None, f2=None, r1=None, r2=None)->([float] , Any , Any):
    """ Second model predicts k=1, k=k residuals of the first, and interpolates

          f1  - A skater making the primary prediction
          f2  - A skater designed to predict residuals ... both 1 step ahead and k-steps ahead
          r1  - hyper-params for f1, if any
          r2  - hyper-params for f2, if any

       It *may* make sense to choose an f2 that shrinks towards zero.
    """
    if k == 1:
        J = [1]
    else:
        J = [1,k]  # Determines horizons over which residual model is used.
                   # We'd rather not call the residual model k-times

    y0 = wrap(y)[0]
    if not s.get('s1'):
        s = {'sres': {},                      # Residual state ... used to determine the residual
             'x': y0,
             's1':{},                         # First model state
             's2':dict([(j,{}) for j in J]),  # Residual model states
             'n_obs':0}

    if y0 is None:
        return None, None, s
    else:
        # Use the first skater to predict
        if r1 is None:
            x1, x1_std, s['s1'] = f1(y=y,s=s['s1'],k=k, a=a,t=t,e=e)
        else:
            x1, x1_std, s['s1'] = f1(y=y, s=s['s1'], k=k, a=a, t=t, e=e, r=r1)
        resid1, s['sres'] = residual(s['sres'],y=y0,x=x1)

        s['n_obs']+=1

        # Use the second skater to predict j-step ahead residuals
        # There are two copies of the residual model employed.
        res_j_hat = [None for j in J]
        res_j_std = [None for j in J]
        for jpos,j in enumerate(J):
            j_ahead_residual = resid1[j-1]
            if r2 is None:
                _x, _std, s['s2'][j] = f2(y=j_ahead_residual, s=s['s2'][j], k=j, a=a, t=t, e=e)
            else:
                _x, _std, s['s2'][j] = f2(y=j_ahead_residual, s=s['s2'][j], k=j, a=a, t=t, e=e,r=r2)
            res_j_hat[jpos] = _x[jpos]
            res_j_std[jpos] = _std[jpos]

        # Interpolate
        if k==1:
            res_interp = res_j_hat
            res_interp_std = res_j_std
        else:
            import numpy as np
            ks = list(range(1,k+1))
            res_interp = np.interp( x=ks, xp=J, fp=res_j_hat )
            res_interp_std = np.interp(x=ks, xp=J, fp=res_j_std)

        # Residual   res =  y - x1,   so  x1+res ~ y  .... one hopes
        x_hat = [ resj+x1j for resj, x1j in zip( res_interp, x1) ]
        return x_hat, res_interp_std, s

max_stars_repo_name: iklasky/timemachines

max_issues_repo_name: iklasky/timemachines

max_forks_repo_name: iklasky/timemachines


text:
# Julio Bruce 1

from estrategias.jogadores import Jogador
import numpy as np

class MeuJogador(Jogador):

    def __init__(self):
        Jogador.__init__(self)

    def escolha_de_cacada(self, rodada, comida_atual, reputacao_atual, m, reputacoes_dos_jogadores):

        meta_reputacao = 0.64
        meta_rodada = 205

        if rodada > meta_rodada:
            escolhas = []
            for i in range(len(reputacoes_dos_jogadores)):
                if reputacoes_dos_jogadores[i] == reputacao_atual:
                    escolhas = escolhas + ['c']
                else:
                    escolhas = escolhas + ['d']
            return escolhas

        else:
            escolhas = []
            for i in range(len(reputacoes_dos_jogadores)):
                    if (rodada != 1) and (reputacoes_dos_jogadores[i] == 1 or reputacoes_dos_jogadores[i] == 0):
                        escolhas = escolhas + ['d']
                    elif reputacao_atual > meta_reputacao:
                        escolhas = escolhas + ['d']
                    else:
                        escolhas = escolhas + ['c']
            return escolhas


text:
from handwritingrecognition import neuralnetwork
from handwritingrecognition import data
import sys
import os
import numpy

def main():
    # Train Neural Network
    print('Loading data...')
    stuff = data.loaddata(sys.argv[1])

    Theta = neuralnetwork.randomtheta([63, 63, 26], stuff.X_train.shape[1])

    print('Training...')
    output = neuralnetwork.train(stuff.X_train, stuff.Y_train, Theta, 0.16, maxiterations = 9999)

    # Save correct Thetas
    print('Saving Thetas...')
    numpy.savez_compressed(os.path.abspath(sys.argv[2]), *output[0])

    print(output[1:])

max_stars_repo_name: rishihahs/handwritingrecognition

max_issues_repo_name: rishihahs/handwritingrecognition

max_forks_repo_name: rishihahs/handwritingrecognition


text:
import jax
from jax import lax
from jax import tree_util
import jax.numpy as np

def is_list_like(x):
  return isinstance(x, (list, tuple))

def call_fn(fn, args):
  if is_list_like(args):
    return fn(*args)
  return fn(args)

def call_fn_value_and_grad(fn, args):

  def _fn(args):
    if is_list_like(args):
      return fn(*args)
    return fn(args)

  output, vjp_fn = jax.vjp(_fn, args)
  grad = vjp_fn(np.ones_like(output))[0]
  return output, grad

def choose(is_accepted, proposed_state, state):

  def _choose(is_accepted, proposed_state, state):

    def _expand_is_accepted_like(x):
      if x.shape is not None and is_accepted.shape is not None:
        expand_shape = list(is_accepted.shape) + [1] * (
            len(x.shape) - len(is_accepted.shape))
      else:
        expand_shape = is_accepted.shape + (1,) * (x.ndim - is_accepted.ndim)

      return np.reshape(is_accepted, expand_shape)

    if is_list_like(proposed_state):
      assert is_list_like(state)
      return type(proposed_state)(*[
          np.where(_expand_is_accepted_like(p), p, s)
          for p, s in zip(proposed_state, state)
      ])
    else:
      return np.where(_expand_is_accepted_like(proposed_state), proposed_state, state)

  return tree_util.tree_multimap(lambda p, s: _choose(is_accepted, p, s),
                                 proposed_state, state)

def trace(state, fn, num_steps, trace_fn=None):

  if trace_fn is None:
    trace_fn = lambda state, extra: extra

  def wrapped_fn(state, _unused):
    next_state, aux = fn(state)
    return next_state, trace_fn(next_state, aux)

  (final_state, out) = lax.scan(wrapped_fn, state, xs=None, length=num_steps)
  return final_state, out

def split_rng_as(rng, structure):
  struct_flat, tree = tree_util.tree_flatten(structure)
  if len(struct_flat) == 1:
    rngs = (rng,)
  else:
    rngs = jax.random.split(rng, len(struct_flat))
  return tree_util.tree_unflatten(tree, rngs)

def block_until_ready(state):

  def _wait(s):
    if isinstance(s, np.ndarray):
      s.block_until_ready()

  return jax.tree_map(_wait, state)

max_stars_repo_name: ethanluoyc/sympais

max_issues_repo_name: ethanluoyc/sympais

max_forks_repo_name: ethanluoyc/sympais

reason: import jax,from jax


text:
if __name__ == '__main__':
    from typing import List, Tuple

    import numpy as np
    from bnet.util import create_data_dir
    from hbtfmt.model import HQAgent
    from hbtfmt.util import (ConfigClap, format_data1, generate_rewards,
                             parameter_product1, run_baseline_test)
    from networkx import to_numpy_matrix

    clap = ConfigClap()
    config = clap.config()
    params = parameter_product1(config)

    data_dir = create_data_dir(__file__, "bnet")
    filepath = data_dir.joinpath(config.filename)

    results: List[Tuple[int, float, float, float, int, int, float, float]] = []

    for param in params:
        n, ro, qop, qot = param
        rewards_baseline = generate_rewards(1., ro, n)
        rewards_test = generate_rewards(0., ro, n)

        print(f"n = {n}, ro = {ro}, qop = {qop}, qot  = {qot}")

        for i in range(config.loop_per_condition):
            agent = HQAgent(qop, qot, n)
            agent.construct_network(2)

            result = run_baseline_test(agent, rewards_baseline, rewards_test,
                                       n, config.loop_per_simulation)
            result_with_params = param + result
            results.append(result_with_params)

        mat = to_numpy_matrix(agent.network).astype(np.uint8)
        matpath = data_dir.joinpath(filepath.stem + f"-mat{-qop}.csv")
        np.savetxt(matpath, mat, delimiter=",", fmt="%d")

    output_data = format_data1(results)
    output_data.to_csv(filepath, index=False)

reason: import numpy,from networkx


text:
"""
Package to provide support for various objects and modules

This module contains functions that don't have an obvious home in a
submodule.
"""
import datetime
import logging
import os
import sys

from numpy import array
from os import environ, makedirs
from os.path import exists
from time import sleep, time

from numpy import array

logger = logging.getLogger(__name__)

__version__ = "1.2.0"

def python_version():
    return sys.version.split()[0]

######################### Classes with extra features #####################

class NamedClass(object):
    """
    A base class enhanced with methods for reporting self.
    """

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.base() + ' "' + str(self.name) + '"'

    def __repr__(self):
        return self.base() + ' "' + str(self.name) + '"'

    def base(self):
        """
        String representing the class instance type
        """
        return str(type(self)).split()[-1].strip('>').strip("'").split('.')[-1]

class PropertiedClass(NamedClass):
    """
    A subclass of MCobject enhanced with properties.
    """

    def __init__(self):
        self.data = {}

    def __setitem__(self, key, item):
        self.data[key] = item

    def __getitem__(self, key):
        return self.data[key]

    def keys(self):
        return self.data.keys()

    def has_key(self, key):
        if self.data.has_key(key):
            return True
        else:
            return False

########################## functions for numpy arrays ########################

def nearest_index(np_array, value):
    """
    Return the index of the element of the array with the nearest value

    Note
    ====
    This has a problem with arrays of datetime.datetime() objects.

    @param np_array : an array of value
    @type  np_array : numpy array

    @param value : a single value
    @type  value : the same type as values in np_array

    @return: index of item in np_array closest to value
    """
    # Convert to to numpy array if necessary
    if type(np_array) == list or type(np_array) == tuple:
        np_array = array(np_array)
    # find the index of the array value nearest the test value
    if type(value) == datetime.datetime:
        data_array = date2num(np_array)
        ref_value = date2num(value)
        index = abs(data_array - ref_value).argmin()
    else:
        index = abs(np_array - value).argmin()
    # value is beyond the ends of the array
    # this test assumes the data are monotonic, then result is unknown
    if (((np_array[0] < np_array[-1]) and
             (value < np_array[0] or value > np_array[-1])) or
            ((np_array[0] > np_array[-1]) and
                 (value > np_array[0] or value < np_array[-1]))):
        return -1
    else:
        return index

def trim_extremes(data):
  """
  Remove extreme values from a data array.

  Extreme values are those greater than 10x the standard deviation
  and are 'skinny'.: numpy array

  @param data : numpy array
  """
  data_array = array(data)
  amean   = mean(data_array)
  avar    = var(data_array)
  astdev  = sqrt(avar)
  amax = data_array.max()
  amin = data_array.min()
  # Check the maximum
  if abs(amax-amean) > 10*astdev:
    index = argmax(data_array)
    if is_skinny(data_array,index):
      data_array = clobber(data_array,index)
  # check the minimum
  if abs(amin-amean) > 10*astdev:
    index = argmin(data_array)
    if is_skinny(data_array,index):
      data_array = clobber(data_array,index)
  return data_array

def is_skinny(data_array,index):
  """
  Test whether a data value is an isolated outlier

  Returns True if the data values adjacent to the test value are
  less that 1/10 of the test value, i.e., the data point is a spike

  @param data_array : numpy array

  @param index : int

  @return: boolean
  """
  amean   = mean(data_array)
  test_value = abs(data_array[index]-amean)
  if index == 0:
    ref_value = abs(data_array[index+1] - amean)
  elif index == len(data_array)-1:
    ref_value = abs(data_array[index-1] - amean)
  else:
    ref_value = (data_array[index-1] + data_array[index+1])/2. - amean
  if test_value > 10 * ref_value:
    return True
  else:
    return False

def clobber(data_array,index):
  """
  Replace a data array value with the adjacent value(s)

  @param data_array : numpy array

  @param index : int
  """
  if index == 0:
    data_array[index] = data_array[index+1]
  elif index == len(data_array)-1:
    data_array[index] = data_array[index-1]
  else:
    data_array[index] = (data_array[index-1] + data_array[index+1])/2.
  return data_array

########################## Users and Permissions ############################

def get_user():
  """
  Returns the user running this session

  This will return the username of the logged-in user.  If the user is logged
  in as 'root' (or by doing 'su'), then there is no way to find out who the
  actual user is.
  """
  try:
    test = os.environ['USER']
    # there is a USER
    logger.debug("get_user: USER = %s", test)
  except KeyError:
    # probably was run by a cron job
    logger.debug("get_user: no USER; trying LOGNAME")
    try:
      test = os.environ['LOGNAME']
    except KeyError:
      logger.error("get_user: no USER or LOGNAME")
      logger.debug("get_user: environment is %s", os.environ)
      return None
  if test == "root":
    try:
      # maybe the user was using 'sudo'
      logger.debug("get_user: user is %s; trying SUDO_USER", test)
      user = os.environ['SUDO_USER']
    except KeyError:
      # no, this is really by 'root'
      logger.debug("get_user: SUDO_USER not set")
      user = "root"
  else:
    user = test
    logger.debug("get_user: settled on %s", user)
  return user

def check_permission(group):
  """
  Verify that the user has the necessary permissions.

  Permissions can be gained by being superuser, using 'sudo' or by belonging to
  the right group.

  Example::
    check_permission('dialout')
  """
  p = invoke("groups") # What groups does the user belong to?
  groups = p.stdout.readline().split()
  try:
    groups.index(group)
    return True
  except ValueError:
    if environ['USER'] != 'root':
      logger.error("You must be in group '%s', root or 'sudo'er", group)
      return False
  logger.info(" User permissions verified")

def sync_second():
    """
    Wait until the second changes

    This returns roughly 6.2 ms after the second changes. I can't seem to get the
    delay below this.

    Using 'sleep' rather than 'pass' requires less CPU cycles.
    """
    now = int(time())
    while not bool(int(time()) - now):
        sleep(0.0001)

def cpu_arch():
    """
    """
    p = invoke('uname -a')
    text = p.stdout.readline()
    return text.split()[-2]

def mkdir_if_needed(path):
    """
    python3 only
    """
    if exists(path) == False:
        logger.warning(" Creating %s", path)
        makedirs(path, mode=0o775)

max_stars_repo_name: SDRAST/support

max_issues_repo_name: SDRAST/support

max_forks_repo_name: SDRAST/support

reason: from numpy


text:
import argparse
import os

import numpy as np
from keras.losses import categorical_crossentropy
from keras.utils import np_utils
from sklearn.model_selection import KFold

from utils import *

SENSORS = 17
BATCH_SIZE = 64
EPOCHS = 100
NUM_CLASSES = 42    # TODO: Get this from the data

# Fix random seed for reproducibility
seed = 7
np.random.seed(seed)

# Turn off TensorFlow warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

# Parsing from terminal
parser = argparse.ArgumentParser(description='K-Fold validation script')
parser.add_argument('dataset_path', help='Path of datasets folder')
parser.add_argument('-m', '--model', help='Choose model to train on from [inception, seq_v1, seq_v2, functional]',
                    type=str, default='inception')
parser.add_argument('-k', '--kfolds', type=int, help='Number of folds', default=10)
args = parser.parse_args()

dataset_path = args.dataset_path
model_type = args.model
k = args.kfolds

# Data loading
# TODO: Tackle weird float conversion from pandas to numpy array
dataset = create_dataset(dataset_path, randomize=True)
y, x = np.hsplit(dataset, [1])
x = np.expand_dims(x, axis=2)
y = y - 1
y = np_utils.to_categorical(y, NUM_CLASSES)

# K-fold cross validation
kfold = KFold(n_splits=k, shuffle=True, random_state=seed)
cvscores = []
for train, test in kfold.split(x, y):
    # Model building
    model = build_model(model_type, SENSORS, NUM_CLASSES)
    # compile model
    model.compile(loss=categorical_crossentropy, optimizer='adam', metrics=['accuracy'])
    # fit model
    model.fit(x[train], y[train], batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=0, shuffle=True)
    # evaluate model
    scores = model.evaluate(x[test], y[test], verbose=0)
    print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
    cvscores.append(scores[1] * 100)

print("%.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))


text:
import os
import asyncio
import websockets
import pyroboy
import redis
import struct, pickle
from pyroboy.face_recognition import FaceRec
import numpy as np
import pdb

from scientio.ontology.ontology import Ontology
from scientio.session import Session
from scientio.ontology.node import Node

from ecdsa import VerifyingKey
import hashlib

class FaceOracle:
    def __init__(self):
        self.r = redis.Redis(host='bot.roboy.org', port=6379, password=os.environ['REDIS_PASSWORD'], db=0)
        self.start_server = websockets.serve(self.recognize, '0.0.0.0', 8765)

    def start(self):
        asyncio.get_event_loop().run_until_complete(self.start_server)
        asyncio.get_event_loop().run_forever()

    def get_known_faces(self):
        # key = neo4j node id
        # value = binary face encoding

        keys = self.r.keys()
        values = []
        for k in keys:
            value = self.r.get(k)
            #import pdb; pdb.set_trace()
            values.append(pickle.loads(value))
        return keys,values

    async def recognize(self, websocket, path):
        b_face_encodings = await websocket.recv()
        #print(b_face_encoding)
        face_encodings, h, signature = pickle.loads(b_face_encodings, encoding='bytes')#.decode()

        # check signature
        #to_hash = bytearray(face_encodings)#array.array('B', encodings).tostring()
        #h = hashlib.sha256(to_hash).digest()
        vk = VerifyingKey.from_pem(open("newpubkey.pem").read())
        #try:
        #    vk.verify_digest(signature, h)
        #except:
        #    print("Some dirty business here. Could not verify signature")

        #face_encoding = struct.unpack('%sd' % 128, b_face_encoding)
        ids, known_faces = self.get_known_faces()
        names = []
        confidences = []
        node_ids = []
        for face_encoding in face_encodings:
            idx, confidence = FaceRec.match_face(face_encoding, known_faces)
            if idx is not None:
                node = sess.retrieve(node_id=int(ids[idx].decode('utf-8')))[0]
                node_ids.append(int(ids[idx].decode('utf-8')))
                names.append(node.get_name())
                confidences.append(confidence)
            else:
                names.append("stranger")
                confidences.append(1.0)
                node_ids.append(-1)
        await websocket.send(pickle.dumps((names, confidences, node_ids), protocol=2))

if __name__ == '__main__':
    global sess
    onto = Ontology(path_to_yaml="ravestate_ontology.yml")
    sess = Session(
        ontology=onto,
        neo4j_address=os.environ['NEO4J_ADDRESS'],
        neo4j_username=os.environ['NEO4J_USERNAME'],
        neo4j_password=os.environ['NEO4J_PASSWORD'])
    #res = sess.retrieve(node_id=7)
    #pdb.set_trace()
    oracle = FaceOracle()
    oracle.start()


text:
# coding=utf-8
import numpy as np
import igraph

## Graph-aware measures (igraph version)
def gam(self, u, v, method="rand", adjusted=True):
    """
    Compute one of 11 graph-aware measures to compare graph partitions.

    Parameters
    ----------
    self: Graph of type 'igraph.Graph' on which the partitions are defined.

    u: Partition of type 'igraph.clustering.VertexClustering' on 'self', or a dictionary of node:community.

    v: Partition of type 'igraph.clustering.VertexClustering' on 'self', or a dictionary of node:community.

    method: 'str'
      one of 'rand', 'jaccard', 'mn', 'gmn', 'min' or 'max'

    adjusted: 'bool'
      if True, return adjusted measure (preferred). All measures can be adjusted except 'jaccard'.

    Returns
    -------
    A graph-aware similarity measure between vertex partitions u and v.

    Examples
    --------
    >>> g = ig.Graph.Famous('Zachary')
    >>> part1 = g.community_multilevel()
    >>> part2 = g.community_label_propagation()
    >>> print(g.GAM(part1, part2))

     Reference
    ---------
    Valérie Poulin and François Théberge, "Comparing Graph Clusterings: Set partition measures vs. Graph-aware measures", https://arxiv.org/abs/1806.11494.
    """
    if(type(u) is dict):
        d1 = u
    else:
        d1 = {val:idx for idx,part in enumerate(u) for val in part}
    if(type(v) is dict):
        d2 = v
    else:
        d2 = {val:idx for idx,part in enumerate(v) for val in part}
    bu = np.array([(d1[x.tuple[0]]==d1[x.tuple[1]]) for x in self.es])
    bv = np.array([(d2[x.tuple[0]]==d2[x.tuple[1]]) for x in self.es])
    su = np.sum(bu)
    sv = np.sum(bv)
    suv = np.sum(bu*bv)
    m = len(bu)
    ## all adjusted measures
    if adjusted:
        if method=="jaccard":
            print("no adjusted jaccard measure, set adjusted=False")
            return None
        if method=="rand" or method=="mn":
            return((suv-su*sv/m)/(np.average([su,sv])-su*sv/m))
        if method=="gmn":
            return((suv-su*sv/m)/(np.sqrt(su*sv)-su*sv/m))
        if method=="min":
             return((suv-su*sv/m)/(np.min([su,sv])-su*sv/m))
        if method=="max":
             return((suv-su*sv/m)/(np.max([su,sv])-su*sv/m))
        else:
            print('Wrong method!')

    ## all non-adjusted measures
    else:
        if method=="jaccard":
            union_b = sum((bu+bv)>0)
            return(suv/union_b)
        if method=="rand":
            return(1-(su+sv)/m+2*suv/m)
        if method=="mn":
            return(suv/np.average([su,sv]))
        if method=="gmn":
            return(suv/np.sqrt(su*sv))
        if method=="min":
            return(suv/np.min([su,sv]))
        if method=="max":
            return(suv/np.max([su,sv]))
        else:
            print('Wrong method!')

    return None

igraph.Graph.gam = gam

import igraph
import numpy as np

def community_ecg(self, weights=None, ens_size = 16, min_weight = 0.05, final='louvain'):
    """
    Stable ensemble-based graph clustering;
    the ensemble consists of single-level randomized Louvain;
    each member of the ensemble gets a "vote" to determine if the edges
    are intra-community or not;
    the votes are aggregated into ECG edge-weights in range [0,1];
    a final (full depth) Louvain is run using those edge weights;

    Parameters
    ----------
    self: graph of type 'igraph.Graph'
      Graph to define the partition on.
    weights: list of double, optional
      the edge weights
    ens_size: int
      the size of the ensemble of single-level Louvain
    min_weight: double in range [0,1]
      the ECG edge weight for edges with zero votes from the ensemble
    final: 'louvain' or 'leiden'
      the algorithm to run on the final re-weighted graph

    Returns
    -------
    partition
      The final partition, of type 'igraph.clustering.VertexClustering'
    partition.W
      The ECG edge weights
    partition.CSI
      The community strength index
    partition.original_modularity
      The modularity with respect to the original edge weights

    Notes
    -----
    The ECG edge weight function is defined as:

      min_weight + ( 1 - min_weight ) x (#votes_in_ensemble) / ens_size

    Edges outside the 2-core are assigned 'min_weight'.

    Examples
    --------
    >>> g = igraph.Graph.Famous('Zachary')
    >>> part = g.community_ecg(ens_size=25, min_weight = .1)
    >>> print(part.CSI)

    Reference
    ---------
    Valérie Poulin and François Théberge, "Ensemble clustering for graphs: comparisons and applications", Appl Netw Sci 4, 51 (2019).
    https://doi.org/10.1007/s41109-019-0162-z
    """
    W = [0]*self.ecount()
    ## Ensemble of level-1 Louvain
    for i in range(ens_size):
        p = np.random.permutation(self.vcount()).tolist()
        g = self.permute_vertices(p)
        l1 = g.community_multilevel(weights=weights, return_levels=True)[0].membership
        b = [l1[p[x.tuple[0]]]==l1[p[x.tuple[1]]] for x in self.es]
        W = [W[i]+b[i] for i in range(len(W))]
    W = [min_weight + (1-min_weight)*W[i]/ens_size for i in range(len(W))]
    ## Force min_weight outside 2-core
    core = self.shell_index()
    ecore = [min(core[x.tuple[0]],core[x.tuple[1]]) for x in self.es]
    w = [W[i] if ecore[i]>1 else min_weight for i in range(len(ecore))]
    if final=='leiden':
        part = self.community_leiden(weights=w, objective_function='modularity')
    else:
        part = self.community_multilevel(weights=w)
    part.W = w
    part.CSI = 1-2*np.sum([min(1-i,i) for i in w])/len(w)
    part.original_modularity = self.modularity(part.membership, weights=weights)
    return part

igraph.Graph.community_ecg = community_ecg

max_stars_repo_name: ftheberge/graph-partition-and-measures

max_issues_repo_name: ftheberge/graph-partition-and-measures

max_forks_repo_name: ftheberge/graph-partition-and-measures


text:
import socket
import cv2
import numpy
import face_recognition
from datetime import datetime, timedelta
import numpy as np
import platform
import pickle
import sys
import struct
import base64
from PIL import Image
Image.LOAD_TRUNCATED_IMAGES = True
from io import BytesIO
from datetime import datetime
import time
import RPi.GPIO as GPIO
import os
import sys
from playsound import playsound

Image.LOAD_TRUNCATED_IMAGES = True
from io import BytesIO

# jetson-nano 카메라 캡쳐 함수
def get_jetson_gstreamer_source(capture_width=720, capture_height=480, display_width=720, display_height=480,
                                framerate=60, flip_method=0):
    return (
            f'nvarguscamerasrc ! video/x-raw(memory:NVMM), ' +
            f'width=(int){capture_width}, height=(int){capture_height}, ' +
            f'format=(string)NV12, framerate=(fraction){framerate}/1 ! ' +
            f'nvvidconv flip-method={flip_method} ! ' +
            f'video/x-raw, width=(int){display_width}, height=(int){display_height}, format=(string)BGRx ! ' +
            'videoconvert ! video/x-raw, format=(string)BGR ! appsink'
    )

def recvall(sock, count):
    buf = b''
    while count:
        newbuf = sock.recv(count)
        if not newbuf: return None
        buf += newbuf
        count -= len(newbuf)
    return buf

def restart():
    # 문이 열렸으므로 재시작
    executable = sys.executable
    args = sys.argv[:]
    args.insert(0, sys.executable)
    time.sleep(1)
    print('restart program')
    os.execvp(executable, args)

def run():
    # 연결할 서버(수신단)의 ip주소와 port번호
    TCP_IP = '164.125.234.91'
    TCP_PORT = 5010

    # 송신을 위한 socket 준비
    sock = socket.socket()
    sock.connect((TCP_IP, TCP_PORT))
    cam = cv2.VideoCapture(get_jetson_gstreamer_source(), cv2.CAP_GSTREAMER)
    start_time = time.time()

    ## 0~100에서 90의 이미지 품질로 설정 (default = 95)
    encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]
    ret, frame = cam.read()
    while True:
        ret, frame = cam.read()
        small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
        rgb_small_frame = small_frame[:, :, ::-1]
        # cv2. imencode(ext, img [, params])
        # encode_param의 형식으로 frame을 jpg로 이미지를 인코딩한다.
        result, frame = cv2.imencode('.jpg', frame, encode_param)
        # frame을 String 형태로 변환
        data = numpy.array(frame)
        stringData = data.tostring()
        # 서버에 데이터 전송
        sock.sendall((str(len(stringData))).encode().ljust(16) + stringData)
        resp = sock.recv(1024)
        duration = time.time()-start_time
        if (resp.decode('utf-8') == 'I'):
            if duration >= 20:
                #print("20초 경과로 프로그램이 꺼집니다.")
                #cam.release()
                #cv2.destroyAllWindows()
                #restart()
                 continue
        elif(resp.decode('utf-8') == 'AA'):
            playsound("/home/iotlab/Desktop/unregistered.mp3")
            print('Intruder!')
            nowDate = datetime.now()
            length = recvall(sock, 16)
            stringdata = recvall(sock, int(length))
            img = base64.decodebytes(stringdata)
            im = Image.open(BytesIO(img))
            im.save(nowDate.strftime("/home/iotlab/dlib-19.17/recog/A.jpg"))
            showing = cv2.imread("/home/iotlab/dlib-19.17/recog/A.jpg", cv2.IMREAD_COLOR)
            show_img = cv2.resize(showing, dsize=(480, 400), interpolation=cv2.INTER_AREA)
            cv2.putText(show_img, "Not Registered Face", (5, 18), cv2.FONT_HERSHEY_DUPLEX, 0.8, (0, 0, 255), 1)
            cv2.imshow("Recog_Face", show_img)
            cv2.waitKey(3000)
            cv2.destroyAllWindows()
            continue
        elif(resp.decode('utf-8') == 'BB'):
            playsound("/home/iotlab/Desktop/registerd.mp3")
            print('****Welcome Home****')
            GPIO.setmode(GPIO.BOARD)

            # initialize EnA, In1 and In2
            GPIO.setup(ENA, GPIO.OUT, initial=GPIO.LOW)
            GPIO.setup(IN1, GPIO.OUT, initial=GPIO.LOW)
            GPIO.setup(IN2, GPIO.OUT, initial=GPIO.LOW)

            # Stop
            GPIO.output(ENA, GPIO.HIGH)
            GPIO.output(IN1, GPIO.LOW)
            GPIO.output(IN2, GPIO.LOW)
            time.sleep(0.2)

            # Backward
            GPIO.output(IN1, GPIO.LOW)
            GPIO.output(IN2, GPIO.HIGH)
            time.sleep(0.2)

            # Stop
            GPIO.output(ENA, GPIO.LOW)
            GPIO.output(IN1, GPIO.LOW)
            GPIO.output(IN2, GPIO.LOW)
            time.sleep(3)

            GPIO.cleanup()
            cam.release()
            cv2.destroyAllWindows()
            break
        elif cv2.waitKey(1) & 0xFF == ord('q'):
            cam.release()
            cv2.destroyAllWindows()
            break
        else:
            continue

    cam.release()

if __name__ == '__main__':
    GPIO.setmode(GPIO.BOARD)
    PIR_PIN = 26
    ENA = 33
    IN1 = 35
    IN2 = 37
    GPIO.setup(PIR_PIN, GPIO.IN)
    try:
        time.sleep(1)
        print('PIR Module Ready')

        while True:
            GPIO.setmode(GPIO.BOARD)
            GPIO.setup(PIR_PIN, GPIO.IN)
            PIR_PIN = 26
            if GPIO.input(PIR_PIN):
                t = time.localtime()
                print('% d: % d: % d Motion Detected!' % (t.tm_hour, t.tm_min, t.tm_sec))
                run()
                cv2.VideoCapture(get_jetson_gstreamer_source(), cv2.CAP_GSTREAMER).release()
                restart()
            else:
                continue
            time.sleep(1)

    except ConnectionRefusedError:
        cv2.VideoCapture(get_jetson_gstreamer_source(), cv2.CAP_GSTREAMER).release()
        cv2.destroyAllWindows()
        time.sleep(10)
        restart()
    except BrokenPipeError:
        cv2.VideoCapture(get_jetson_gstreamer_source(), cv2.CAP_GSTREAMER).release()
        cv2.destroyAllWindows()
        time.sleep(10)
        restart()
    except ConnectionResetError:
        cv2.VideoCapture(get_jetson_gstreamer_source(), cv2.CAP_GSTREAMER).release()
        cv2.destroyAllWindows()
        time.sleep(10)
        restart()