text:
import logging
from functools import reduce
from typing import Callable, Dict, List

import numpy as np
from scipy.stats import pearsonr, spearmanr
from seqeval.metrics import classification_report as token_classification_report
from sklearn.metrics import (
    matthews_corrcoef,
    f1_score,
    mean_squared_error,
    r2_score,
    classification_report
)

from haystack.modeling.model.prediction_head import PredictionHead
from haystack.modeling.utils import flatten_list

logger = logging.getLogger(__name__)

registered_metrics = {}
registered_reports = {}

def register_metrics(name: str, implementation: Callable):
    registered_metrics[name] = implementation

def register_report(name: str, implementation: Callable):
    """
    Register a custom reporting function to be used during eval.

    This can be useful:
    - if you want to overwrite a report for an existing output type of prediction head (e.g. "per_token")
    - if you have a new type of prediction head and want to add a custom report for it

    :param name: This must match the `ph_output_type` attribute of the PredictionHead for which the report should be used.
                 (e.g. TokenPredictionHead => `per_token`, YourCustomHead => `some_new_type`).
    :param implementation: Function to be executed. It must take lists of `y_true` and `y_pred` as input and return a
                           printable object (e.g. string or dict).
                           See sklearns.metrics.classification_report for an example.
    :type implementation: function
    """
    registered_reports[name] = implementation

def simple_accuracy(preds, labels):
    # works also with nested lists of different lengths (needed for masked LM task)
    if type(preds) == type(labels) == list:
        preds = np.array(list(flatten_list(preds)))
        labels = np.array(list(flatten_list(labels)))
    assert type(preds) == type(labels) == np.ndarray
    correct = preds == labels
    return {"acc": correct.mean()}

def acc_and_f1(preds, labels):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds)
    return {"acc": acc['acc'], "f1": f1, "acc_and_f1": (acc['acc'] + f1) / 2}

def f1_macro(preds, labels):
    return {"f1_macro": f1_score(y_true=labels, y_pred=preds, average="macro")}

def pearson_and_spearman(preds, labels):
    pearson_corr = pearsonr(preds, labels)[0]
    spearman_corr = spearmanr(preds, labels)[0]
    return {
        "pearson": pearson_corr,
        "spearman": spearman_corr,
        "corr": (pearson_corr + spearman_corr) / 2,
    }

def compute_metrics(metric: str, preds, labels):
    """
    Calculate the named metric values for the list of predictions vs list of labels.

    :param metric: The name of a predefined metric; a function that takes a prediction list and a label
        list and returns a dict from metric names to values, or recursively a list of metrics.
        Predefined metrics are: mcc, acc, acc_f1, pear_spear, seq_f1, f1_macro, squad, mse, r2,
        top_n_accuracy, text_similarity_metric.
    :param preds: list of predictions
    :param labels: list of target labels
    :return: a dictionary mapping metric names to values.
    """
    assert len(preds) == len(labels)
    if metric == "mcc":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif metric == "acc":
        return simple_accuracy(preds, labels)
    elif metric == "acc_f1":
        return acc_and_f1(preds, labels)
    elif metric == "pear_spear":
        return pearson_and_spearman(preds, labels)
    elif metric == "f1_macro":
        return f1_macro(preds, labels)
    elif metric == "squad":
        return squad(preds, labels)
    elif metric == "mse":
        return {"mse": mean_squared_error(preds, labels)}
    elif metric == "r2":
        return {"r2": r2_score(preds, labels)}
    elif metric == "top_n_accuracy":
        return {"top_n_accuracy": top_n_accuracy(preds, labels)}
    elif metric == "text_similarity_metric":
        return text_similarity_metric(preds, labels)
    elif isinstance(metric, list):
        ret = {}
        for m in metric:
            ret.update(compute_metrics(m, preds, labels))
        return ret
    elif metric in registered_metrics:
        metric_func = registered_metrics[metric]
        return metric_func(preds, labels)
    else:
        raise KeyError(metric)

def compute_report_metrics(head: PredictionHead, preds, labels):
    if head.ph_output_type in registered_reports:
        report_fn = registered_reports[head.ph_output_type]
    elif head.ph_output_type == "per_token":
        report_fn = token_classification_report
    elif head.ph_output_type == "per_sequence":
        report_fn = classification_report
    elif head.ph_output_type == "per_token_squad":
        report_fn = lambda *args, **kwargs: "Not Implemented"
    elif head.ph_output_type == "per_sequence_continuous":
        report_fn = r2_score
    else:
        raise AttributeError(f"No report function for head.ph_output_type '{head.ph_output_type}'. "
                             f"You can register a custom one via register_report(name='{head.ph_output_type}', implementation=<your_report_function>")

    # CHANGE PARAMETERS, not all report_fn accept digits
    if head.ph_output_type in ["per_sequence"]:
        # supply labels as all possible combination because if ground truth labels do not cover
        # all values in label_list (maybe dev set is small), the report will break
        if head.model_type == "text_similarity":
            labels = reduce(lambda x, y: x + list(y.astype('long')), labels, [])
            preds = reduce(lambda x, y: x + [0] * y[0] + [1] + [0] * (len(y) - y[0] - 1), preds, [])  # type: ignore
            all_possible_labels = list(range(len(head.label_list)))
        else:
            all_possible_labels = head.label_list
        return report_fn(
            labels,
            preds,
            digits=4,
            labels=all_possible_labels,
            target_names=head.label_list
        )
    else:
        return report_fn(labels, preds)

def squad_EM(preds, labels):
    """
    Count how often the pair of predicted start and end index exactly matches one of the labels
    """
    n_docs = len(preds)
    n_correct = 0
    for (pred, label) in zip(preds, labels):
        qa_candidate = pred[0][0]
        pred_start = qa_candidate.offset_answer_start
        pred_end = qa_candidate.offset_answer_end
        curr_labels = label
        if (pred_start, pred_end) in curr_labels:
            n_correct += 1
    return n_correct/n_docs if n_docs else 0

def squad_EM_start(preds, labels):
    """
    Count how often the predicted start index exactly matches the start index given by one of the labels
    """
    n_docs = len(preds)
    n_correct = 0
    for (pred, label) in zip(preds, labels):
        qa_candidate = pred[0][0]
        pred_start = qa_candidate.offset_answer_start
        curr_labels = label
        curr_labels_start = [curr_label[0] for curr_label in curr_labels]
        if pred_start in curr_labels_start:
            n_correct += 1
    return n_correct/n_docs if n_docs else 0

def squad_f1(preds, labels):
    f1_scores = []
    n_docs = len(preds)
    for i in range(n_docs):
        best_pred = preds[i][0]
        best_f1 = max([squad_f1_single(best_pred, label) for label in labels[i]])
        f1_scores.append(best_f1)
    return np.mean(f1_scores)

def squad_f1_single(pred, label, pred_idx: int = 0):
    label_start, label_end = label
    span = pred[pred_idx]
    pred_start = span.offset_answer_start
    pred_end = span.offset_answer_end

    if (pred_start + pred_end == 0) or (label_start + label_end == 0):
        if pred_start == label_start:
            return 1.0
        else:
            return 0.0
    pred_span = list(range(pred_start, pred_end + 1))
    label_span = list(range(label_start, label_end + 1))
    n_overlap = len([x for x in pred_span if x in label_span])
    if n_overlap == 0:
        return 0.0
    precision = n_overlap / len(pred_span)
    recall = n_overlap / len(label_span)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

def confidence(preds):
    conf = 0
    for pred in preds:
        conf += pred[0][0].confidence
    return conf/len(preds) if len(preds) else 0

def metrics_per_bin(preds, labels, num_bins: int = 10):
    pred_bins = [[] for _ in range(num_bins)]  # type: List
    label_bins = [[] for _ in range(num_bins)]  # type: List
    count_per_bin = [0]*num_bins
    for (pred, label) in zip(preds, labels):
        current_score = pred[0][0].confidence
        if current_score >= 1.0:
            current_score = 0.9999
        pred_bins[int(current_score*num_bins)].append(pred)
        label_bins[int(current_score*num_bins)].append(label)
        count_per_bin[int(current_score*num_bins)] += 1

    em_per_bin = [0]*num_bins
    confidence_per_bin = [0]*num_bins
    for i in range(num_bins):
        em_per_bin[i] = squad_EM_start(preds=pred_bins[i], labels=label_bins[i])
        confidence_per_bin[i] = confidence(preds=pred_bins[i])
    return em_per_bin, confidence_per_bin, count_per_bin

def squad_base(preds, labels):
    em = squad_EM(preds=preds, labels=labels)
    f1 = squad_f1(preds=preds, labels=labels)
    top_acc = top_n_accuracy(preds=preds, labels=labels)
    return {"EM": em, "f1": f1, "top_n_accuracy": top_acc}

def squad(preds, labels):
    """
    This method calculates squad evaluation metrics a) overall, b) for questions with text answer and c) for questions with no answer
    """
    # TODO change check for no_answer questions from using (start,end)==(-1,-1) to is_impossible flag in QAInput. This needs to be done for labels though. Not for predictions.
    overall_results = squad_base(preds, labels)

    preds_answer = [pred for (pred, label) in zip(preds, labels) if (-1, -1) not in label]
    labels_answer = [label for label in labels if (-1, -1) not in label]
    answer_results = squad_base(preds_answer, labels_answer)

    preds_no_answer = [pred for (pred, label) in zip(preds, labels) if (-1, -1) in label]
    labels_no_answer = [label for label in labels if (-1, -1) in label]
    no_answer_results = squad_base(preds_no_answer, labels_no_answer)

    return {"EM": overall_results["EM"], "f1": overall_results["f1"], "top_n_accuracy": overall_results["top_n_accuracy"],
            "EM_text_answer": answer_results["EM"], "f1_text_answer": answer_results["f1"], "top_n_accuracy_text_answer": answer_results["top_n_accuracy"],
            "Total_text_answer": len(preds_answer),
            "EM_no_answer": no_answer_results["EM"], "f1_no_answer": no_answer_results["f1"], "top_n_accuracy_no_answer": no_answer_results["top_n_accuracy"],
            "Total_no_answer": len(preds_no_answer)
            }

def top_n_accuracy(preds, labels):
    """
    This method calculates the percentage of documents for which the model makes top n accurate predictions.
    The definition of top n accurate a top n accurate prediction is as follows:
    For any given question document pair, there can be multiple predictions from the model and multiple labels.
    If any of those predictions overlap at all with any of the labels, those predictions are considered to be top n accurate.
    """
    answer_in_top_n = []
    n_questions = len(preds)
    for i in range(n_questions):
        f1_score = 0
        current_preds = preds[i][0]
        for idx, pred in enumerate(current_preds):
            f1_score = max([squad_f1_single(current_preds, label, pred_idx=idx) for label in labels[i]])
            if f1_score:
                break
        if f1_score:
            answer_in_top_n.append(1)
        else:
            answer_in_top_n.append(0)

    return np.mean(answer_in_top_n)

def text_similarity_acc_and_f1(preds, labels):
    """
    Returns accuracy and F1 scores for top-1(highest) ranked sequence(context/passage) for each sample/query

    :param preds: list of numpy arrays of dimension n1 x n2 containing n2 predicted ranks for n1 sequences/queries
    :type preds: List of numpy array containing similarity scores for each sequence in batch
    :param labels: list of arrays of dimension n1 x n2 where each array contains n2 labels(0/1) indicating whether the sequence/passage is a positive(1) passage or hard_negative(0) passage
    :type labels: List of list containing values(0/1)

    :return: predicted ranks of passages for each query
    """
    top_1_pred = reduce(lambda x, y: x + [0] * y[0] + [1] + [0] * (len(y) - y[0] - 1), preds, [])
    labels = reduce(lambda x, y: x + list(y.astype('long')), labels, [])
    res = acc_and_f1(top_1_pred, labels)
    return res

def text_similarity_avg_ranks(preds, labels) -> float:
    """
    Calculates average predicted rank of positive sequence(context/passage) for each sample/query

    :param preds: list of numpy arrays of dimension n1 x n2 containing n2 predicted ranks for n1 sequences/queries
    :type preds: List of numpy array containing similarity scores for each sequence in batch
    :param labels: list of arrays of dimension n1 x n2 where each array contains n2 labels(0/1) indicating whether the sequence/passage is a positive(1) passage or hard_negative(0) passage
    :type labels: List of list containing values(0/1)

    :return: average predicted ranks of positive sequence/passage for each sample/query
    """
    positive_idx_per_question = list(reduce(lambda x, y: x + list((y == 1).nonzero()[0]), labels, []))  # type: ignore
    rank = 0
    for i, idx in enumerate(positive_idx_per_question):
        # aggregate the rank of the known gold passage in the sorted results for each question
        gold_idx = (preds[i] == idx).nonzero()[0]
        rank += gold_idx.item()
    return float(rank / len(preds))

def text_similarity_metric(preds, labels) -> Dict[str, float]:
    """
    Returns accuracy, F1 scores and average rank scores for text similarity task

    :param preds: list of numpy arrays of dimension n1 x n2 containing n2 predicted ranks for n1 sequences/queries
    :type preds: List of numpy array containing similarity scores for each sequence in batch
    :param labels: list of arrays of dimension n1 x n2 where each array contains n2 labels(0/1) indicating whether the sequence/passage is a positive(1) passage or hard_negative(0) passage
    :type labels: List of list containing values(0/1)

    :return metrics(accuracy, F1, average rank) for text similarity task
    """
    scores = text_similarity_acc_and_f1(preds, labels)
    scores["average_rank"] = text_similarity_avg_ranks(preds, labels)
    return scores

reason: import numpy,from scipy


text:
# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow import keras

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt

print(tf.__version__)

fashion_mnist = keras.datasets.fashion_mnist

# Loading the dataset returns four NumPy arrays:
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# The train_images and train_labels arrays are the training set—the data the model uses to learn.
# The model is tested against the test set, the test_images, and test_labels arrays

# The images are 28x28 NumPy arrays, with pixel values ranging between 0 and 255.
# The labels are an array of integers, ranging from 0 to 9.
# These correspond to the class of clothing the image represents:

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# The data must be preprocessed before training the network.
# If you inspect the first image in the training set, you will see that the pixel values fall in the range of 0 to 255:

plt.figure()
plt.imshow(train_images[0])
plt.colorbar()
plt.gca().grid(False)

# We scale these values to a range of 0 to 1 before feeding to the neural network model.
# For this, cast the datatype of the image components from an integer to a float, and divide by 255.
# Here's the function to preprocess the images:
# It's important that the training set and the testing set are preprocessed in the same way:

train_images = train_images / 255.0
test_images = test_images / 255.0

# Display the first 25 images from the training set and display the class name below each image.
# Verify that the data is in the correct format and we're ready to build and train the network.

plt.figure(figsize=(10, 10))
for i in range(25):
    plt.subplot(5, 5, i + 1)
    plt.xticks([])
    plt.yticks([])
    plt.grid('off')
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])

# The basic building block of a neural network is the layer.
# Layers extract representations from the data fed into them.
# And, hopefully, these representations are more meaningful for the problem at hand.

# Most of deep learning consists of chaining together simple layers.
# Most layers, like tf.keras.layers.Dense, have parameters that are learned during training.

model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)),
    keras.layers.Dense(128, activation=tf.nn.relu),
    keras.layers.Dense(10, activation=tf.nn.softmax)
])

# The first layer in this network, tf.keras.layers.Flatten,
# transforms the format of the images from a 2d-array (of 28 by 28 pixels), to a 1d-array of 28 * 28 = 784 pixels.
# Think of this layer as unstacking rows of pixels in the image and lining them up.
# This layer has no parameters to learn; it only reformats the data.

# After the pixels are flattened, the network consists of a sequence of two tf.keras.layers.Dense layers.
# These are densely-connected, or fully-connected, neural layers.
# The first Dense layer has 128 nodes (or neurons).
# The second (and last) layer is a 10-node softmax layer—this returns an array of 10 probability scores that sum to 1.
# Each node contains a score that indicates the probability that the current image belongs to one of the 10 classes.

model.compile(optimizer=tf.train.AdamOptimizer(),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# Compile the model
# Before the model is ready for training, it needs a few more settings. These are added during the model's compile step:

# Loss function —This measures how accurate the model is during training.
# We want to minimize this function to "steer" the model in the right direction.
# Optimizer —This is how the model is updated based on the data it sees and its loss function.
# Metrics —Used to monitor the training and testing steps.
# The following example uses accuracy, the fraction of the images that are correctly classified.

model.fit(train_images, train_labels, epochs=5)

# Train the model
# Training the neural network model requires the following steps:

# Feed the training data to the model—in this example, the train_images and train_labels arrays.
# The model learns to associate images and labels.
# We ask the model to make predictions about a test set—in this example, the test_images array.
# We verify that the predictions match the labels from the test_labels array.
# To start training, call the model.fit method—the model is "fit" to the training data.

test_loss, test_acc = model.evaluate(test_images, test_labels)

print('Test accuracy:', test_acc)

predictions = model.predict(test_images)

# Plot the first 25 test images, their predicted label, and the true label
# Color correct predictions in green, incorrect predictions in red
plt.figure(figsize=(10, 10))
for i in range(25):
    plt.subplot(5, 5, i + 1)
    plt.xticks([])
    plt.yticks([])
    plt.grid('off')
    plt.imshow(test_images[i+6000], cmap=plt.cm.binary)
    predicted_label = np.argmax(predictions[i+6000])
    true_label = test_labels[i+6000]
    if predicted_label == true_label:
        color = 'green'
    else:
        color = 'red'
    plt.xlabel("{} ({})".format(class_names[predicted_label], class_names[true_label]), color=color)

ext: py

max_stars_repo_name: SpyRefused/componist

max_issues_repo_name: SpyRefused/componist

max_forks_repo_name: SpyRefused/componist

reason: import numpy


text:
import os
from gym import spaces
import numpy as np
import pybullet as p
from .env import AssistiveEnv

class TestEnv(AssistiveEnv):
    def __init__(self, robot, human):
        super(TestEnv, self).__init__(robot=robot, human=human, task='test', obs_robot_len=21, obs_human_len=(19 if human else 0))

    def step(self, action):
        # Compute action (do nothing)
        n_dof = len(action)
        # action = np.zeros(n_dof).ravel()
        # Execute action
        self.take_step(action)

        # Get observation
        obs = self._get_obs()
        return obs

    def _get_obs(self, agent=None):
        observation = np.zeros(20).ravel() # can be anything
        reward = 0
        done = False
        info = 'placeholder'
        return observation, reward, done, info

    def reset(self):
        super(TestEnv, self).reset()
        self.build_assistive_env()

        p.resetDebugVisualizerCamera(cameraDistance=1.10, cameraYaw=40, cameraPitch=-45, cameraTargetPosition=[-0.2, 0, 0.75], physicsClientId=self.id)

        p.configureDebugVisualizer(p.COV_ENABLE_RENDERING, 1, physicsClientId=self.id)

        return self._get_obs()

max_stars_repo_name: mlamsey/assistive-gym

max_issues_repo_name: mlamsey/assistive-gym

max_forks_repo_name: mlamsey/assistive-gym


text:
from __future__ import print_function

#import warnings

#from distutils.version import LooseVersion
#import tensorflow as tf

import numpy
from numpy import load, save
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

import os.path

import densenet
import numpy as np
import sklearn.metrics as metrics

from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.models import Model
from keras import backend as K

#assert LooseVersion(tf.__version__) >= LooseVersion('1.0')
#print('Tensorflow Version: {}'.format(tf.__version__))

#Check for a GPU
#if not tf.test.gpu_device_name():
#    warnings.warn('No GPU found')
#else:
#    print('Deafault GPU Device: {}'.format(tf.test.gpu_device_name()))

img_rows, img_cols = 64, 64
img_height_test, img_width_test = 64, 64

speckle_data = load('speckle_array_case0.npy')
print(speckle_data.shape)
#speckle_labels = load('speckle_labels.npy')
speckle_labels = load('symbol_array_case0.npy')
print(speckle_labels.shape)
#plt.imshow(speckle_labels[2], cmap='gray')
#plt.show()
#dictionary = {speckle_labels_n: speckle_labels_mn_n for speckle_labels_n, speckle_labels_mn_n in zip(speckle_labels, speckle_labels_mn)}

trainX, testX, Y_train, Y_test = train_test_split(speckle_data, speckle_labels, test_size=0.1, random_state=42)

trainX = trainX.reshape(-1, img_rows, img_cols, 1)
testX = testX.reshape(-1, img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)

Y_train = Y_train.reshape(-1, img_height_test, img_width_test, 1)
Y_test = Y_test.reshape(-1, img_height_test, img_width_test, 1)
input_shape_test = (img_height_test, img_width_test, 1)

batch_size = 100
nb_classes = 6
nb_epoch = 5

img_channels = 1

img_dim = (img_channels, img_rows, img_cols) if K.image_dim_ordering() == "th" else (img_rows, img_cols, img_channels)
print(img_dim)
depth = 40
nb_dense_block = 3
growth_rate = 12
nb_filter = -1
dropout_rate = 0.0 # 0.0 for data augmentation

# model = densenet.DenseNetFCN(input_shape=img_dim, depth=depth, include_top=True, nb_dense_block=nb_dense_block, classes=1, nb_filter=-1,
#                             growth_rate=16, dropout_rate=dropout_rate)

model = densenet.DenseNetFCN(input_shape=img_dim)

print("Model created")

model.summary()
optimizer = Adam(lr=1e-3) # Using Adam instead of SGD to speed up training
model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=["accuracy"])
print("Finished compiling")
print("Building model...")

print('After preprocess_input: {}'.format(trainX.shape))
print('After preprocess_input: {}'.format(testX.shape))

model.fit(trainX, Y_train,
          batch_size = 50,
          epochs = 10,
          verbose = 1,
          validation_data = (testX, Y_test)) # Data on which to evaluate the loss and any model metrics at the end of each epoch.
                                              # The model will not be trained on this data.
                                              # This could be a list (x_val, y_val) or a list (x_val, y_val, val_sample_weights).
                                              # validation_data will override validation_split.

score = model.evaluate(testX, Y_test, verbose = 0)

print('Test loss:', score[0])
print('Test acuracy:', score[1])

y_predicted = model.predict(testX)

print(y_predicted.shape)
save('challenge_predicted.npy', y_predicted)

# extract = Model(reconstruction.inputs, reconstruction.layers[-1].output) # Dense(128,...)
# features = extract.predict(testX)
# print(features.shape)
# save('features_data', features)
# save('features_predicted', y_predicted)

### ----------------------------------------------------------------------------------------------- ###
###                  Original Code                                                                  ###

# generator = ImageDataGenerator(rotation_range=15,
#                                width_shift_range=5./32,
#                                height_shift_range=5./32,
#                                horizontal_flip=True)

# generator.fit(trainX, seed=0)

# # Load model
# weights_file="weights/DenseNet-40-12-CIFAR10.h5"
# if os.path.exists(weights_file):
#     #model.load_weights(weights_file, by_name=True)
#     print("Model loaded.")

# out_dir="weights/"

# lr_reducer      = ReduceLROnPlateau(monitor='val_acc', factor=np.sqrt(0.1),
#                                     cooldown=0, patience=5, min_lr=1e-5)
# model_checkpoint= ModelCheckpoint(weights_file, monitor="val_acc", save_best_only=True,
#                                   save_weights_only=True, verbose=1)

# callbacks=[lr_reducer, model_checkpoint]

# model.fit_generator(generator.flow(trainX, Y_train, batch_size=batch_size),
#                     steps_per_epoch=len(trainX) // batch_size, epochs=nb_epoch,
#                     # callbacks=callbacks,
#                     validation_data=(testX, Y_test),
#                     validation_steps=testX.shape[0] // batch_size, verbose=1)

# yPreds = model.predict(testX)
# print(yPreds.shape)
# save('challenge_predicted.npy', yPreds)
# yPred = np.argmax(yPreds, axis=1)
# yTrue = testY

# accuracy = metrics.accuracy_score(yTrue, yPred) * 100
# error = 100 - accuracy
# print("Accuracy : ", accuracy)
# print("Error : ", error)

max_stars_repo_name: SpyroRock/DenseNet

max_issues_repo_name: SpyroRock/DenseNet

max_forks_repo_name: SpyroRock/DenseNet

reason: import numpy,from numpy


text:
"""
Wordserv - simple server application that provides mapping service to/from word vectors over HTTP

"""

from flask import Blueprint, jsonify, Flask, request

import numpy as np

from .word2vec import w2v

help_text =  """

<title>WordServ</title>

WordServ

A simple word2vec server By James Ravenscroft

GET /vector

Make a get request to /vector endpoint with a query variable 'word' to receive a wordvector for that word
or get an array of words and get an array of words back.

Example

curl http://localhost:5000/vector?word=hello&word=goodbye

GET /similar_by_word

Make a get request to /similar_by_word endpoint with a path variable representing the word you want to query.
Optionally specify a maximum number of results to return (default 10)

Example

curl http://localhost:5000/similar_by_word/cat/10

POST /similar_by_vector

Find a number of similar words via an input vector posted to the server. Optionally specify a maximum number of results to return (default 10)

Example

curl -X POST -H "Content-Type: application/json" -d "[0.12, -0.23, 0.53 ... ]" http://localhost:5000/similar_by_vector/10

"""

word_server_bp = Blueprint('wordserver', __name__)

@word_server_bp.route("/")
def index():
    return help_text

@word_server_bp.route("/vector", methods=['GET'] )
def vector():
    _wv = w2v.word2vec

    wordvecs = {}

    for word in request.args.getlist('word'):
        if word in _wv:
            wordvecs[word] = _wv[word].tolist()
        else:
            wordvecs[word] = [0] * 300

    return jsonify(wordvectors=wordvecs)

@word_server_bp.route("/similar_by_word/<string:word>/<int:topn>", methods=['GET'])
def similar_by_word(word,topn=10):
    _wv = w2v.word2vec

    if word in _wv:
        similar = _wv.similar_by_word(word,topn)
    else:
        similar = []

    return jsonify(similar=similar)

@word_server_bp.route("/similar_by_vector/<int:topn>", methods=['POST'])
def similar_by_vector(topn=10):

    _wv = w2v.word2vec

    vector = np.asarray(request.get_json())

    similar = _wv.similar_by_vector(vector, topn=topn)

    return jsonify(similar=similar)

@word_server_bp.route("/similarity/<string:word1>/<string:word2>", methods=['GET'])
def similarity(word1,word2):
    _wv = w2v.word2vec
    return jsonify(similarity=_wv.similarity(word1,word2))

@word_server_bp.route("/batch_similarity",methods=['GET'])
def batch_similarity():
    _wv = w2v.word2vec
    wordMap = {}
    words = request.args['words']
    pairs = words.split(',')
    for eachPair in pairs:
        p = eachPair.split('-')
        try :
            similarity = _wv.similarity(p[0],p[1])
        except KeyError:
            similarity = 0
        wordMap.update({eachPair:similarity})
    return jsonify(wordMap)

@word_server_bp.route("/n_similarity", methods=['GET'])
def n_similarity():
    _wv = w2v.word2vec
    return jsonify(similarity=_wv.n_similarity(request.args.getlist('ws1'),request.args.getlist('ws2')))

@word_server_bp.route("/doesnt_match", methods=['GET'])
def doesnt_match():
    _wv = w2v.word2vec

    return jsonify(doesnt_match=_wv.doesnt_match(request.args.getlist('word')))

@word_server_bp.route("/benchmark", methods=['GET'])
def benchmark():
    import time
    _wv = w2v.word2vec

    w2vdesc = "An estimated 90% of all data is unstructured and the amount of it is increasing at a daunting rate across our ever more connected world. Unlocking the value of big data is one of the biggest challenges for businesses today. Fortunately a new age of AI and cognitive computing is upon us which can help us make sense of unstructured data like never before."

    start = time.time()

    for x in w2vdesc.split(" "):
        if x in _wv:
            vec = _wv[x]

    end = time.time()
    wordtest = end - start

    start = time.time()

    for word in w2vdesc.split(" "):
        if x in _wv:
            _wv.similar_by_word(word)

    end = time.time()#
    simtest = end-start

    return jsonify(wordtest=wordtest, similar = simtest)

max_stars_repo_name: ravenscroftj/wordserve

max_issues_repo_name: ravenscroftj/wordserve

max_forks_repo_name: ravenscroftj/wordserve


text:
from __future__ import absolute_import

import multiprocessing
import collections
import numpy as np
from joblib import Parallel, delayed
from keras.preprocessing.sequence import pad_sequences

class TextAnalyzer(object):

    def __init__(self, word2inx, vectors, max_len, reverse=False,
                 min_word_hits=1, min_doc_hits=1, max_doc_freq=1.0,
                 process_oov_words=False, oov_window=5, oov_min_doc_hits=1):
        self.word2inx = word2inx
        self.vectors = vectors

        self.PAD_TOKEN = "_PAD_"
        self.UNK_TOKEN = "_UNK_"
        self.max_len = max_len
        self.reverse = reverse

        self.emb_size = vectors.shape[1]
        self.pad_vec = -1*np.ones(self.emb_size)
        self.unk_vec = np.zeros(self.emb_size)

        self.doc_counts = 0
        self.word_hits = collections.OrderedDict()
        self.doc_hits = collections.OrderedDict()

        self.min_word_hits = min_word_hits
        self.min_doc_hits = min_doc_hits
        self.max_doc_freq = max_doc_freq

        self.process_oov_words = process_oov_words
        self.oov_window = oov_window
        self.oov_min_doc_hits = oov_min_doc_hits
        self._oov_vec = collections.OrderedDict()
        self._oov_vec_sum = collections.OrderedDict()

        self.inx2emb = [self.PAD_TOKEN, self.UNK_TOKEN]
        self.emb_vectors = [self.pad_vec, self.unk_vec]
        self._embs = collections.OrderedDict()

    def fit_on_docs(self, docs):
        doc_len = []
        doc_ulen = []
        docs_seq = []

        self.doc_counts = len(docs)

        for doc in docs:
            for w in doc:
                if w in self.word_hits: self.word_hits[w] += 1
                else: self.word_hits[w] = 1
            for w in set(doc):
                if w in self.doc_hits: self.doc_hits[w] += 1
                else: self.doc_hits[w] = 1

            doc_len.append(len(doc))
            doc_ulen.append(len(set(doc)))

            for i, w in enumerate(doc):
                if w in self.word2inx:
                    self._embs[w] = self.vectors[self.word2inx[w]]
                elif self.process_oov_words:
                    emb = self.unk_vec
                    for pw in doc[max(i-self.oov_window,0):i]:
                        if pw in self.word2inx:
                            emb+= self.vectors[self.word2inx[pw]]
                            if w in self._oov_vec_sum: self._oov_vec_sum[w] += 1
                            else: self._oov_vec_sum[w] = 1
                    for pw in doc[i+1:i+self.oov_window]:
                        if pw in self.word2inx:
                            emb+= self.vectors[self.word2inx[pw]]
                            if w in self._oov_vec_sum: self._oov_vec_sum[w] += 1
                            else: self._oov_vec_sum[w] = 1

                    self._oov_vec[w] = emb

        if self.process_oov_words:
            for w, s in self._oov_vec_sum.items():
                if (self.doc_hits[w] >= self.oov_min_doc_hits):
                    self._embs[w] = self._oov_vec[w] / s

        for w,v in self._embs.items():
            if (self.word_hits[w] >= self.min_word_hits and \
                self.doc_hits[w] >= self.min_doc_hits and \
                self.doc_hits[w]/self.doc_counts <= self.max_doc_freq):
                self.inx2emb.append(w)
                self.emb_vectors.append(v)

        self.emb2inx = {word: inx for inx, word in enumerate(self.inx2emb)}
        self.emb_vectors = np.stack(self.emb_vectors)

        for doc in docs:
            if self.reverse:
                docs_seq.append([self.emb2inx[w] if w in self.emb2inx else self.emb2inx[self.UNK_TOKEN] for w in doc[::-1]])
            else:
                docs_seq.append([self.emb2inx[w] if w in self.emb2inx else self.emb2inx[self.UNK_TOKEN] for w in doc])

        self.summary()
        return [pad_sequences(docs_seq, maxlen=self.max_len), np.stack([np.array(doc_len), np.array(doc_ulen)], axis=-1)]

    def transform_docs(self, docs):
        doc_len = []
        doc_ulen = []
        docs_seq = []

        for doc in docs:
            doc_len.append(len(doc))
            doc_ulen.append(len(set(doc)))
            if self.reverse:
                docs_seq.append([self.emb2inx[w] if w in self.emb2inx else self.emb2inx[self.UNK_TOKEN] for w in doc[::-1]])
            else:
                docs_seq.append([self.emb2inx[w] if w in self.emb2inx else self.emb2inx[self.UNK_TOKEN] for w in doc])

        return [pad_sequences(docs_seq, maxlen=self.max_len), np.stack([np.array(doc_len), np.array(doc_ulen)], axis=-1)]

    def summary(self):
        print("Docs: {}".format(self.doc_counts))
        print("Selected words: {}".format(len(self.inx2emb)))
        print("Processed OOV words: {}".format(len(self._oov_vec_sum.keys() & set(self.inx2emb))))


text:
import numpy as np
import re

def decompressed_len(t):
    m = re.search('\((\d+)x(\d+)\)', t)
    if m is None:
        return len(t)
    else:
        i = int(m.group(1))
        j = int(m.group(2))

        return len(t[:t.index('(')]) + j * decompressed_len(t[t.index(')')+1:t.index(')')+1+i]) + decompressed_len(t[t.index(')')+1+i:])

def decompress(in_file):
    # read file
    f = open(in_file, 'r')

    text = f.read()
    text = re.sub('[ \n]', '', text)
    counter = 0
    decompressed_file_len = 0

    while counter < len(text):
        l = text[counter]
        if l == '(':
            m = re.search('(\d+)x(\d+)', text[counter+1:counter+10])
            i = m.group(1)
            j = m.group(2)
            counter += 3 + len(i) + len(j)
            i = int(i)
            j = int(j)
            data = text[counter:counter+i]
            decompressed_file_len += j * len(data)
            counter += i
        else:
            decompressed_file_len += 1
            counter += 1

    print('Length of decompressed file is {}'.format(decompressed_file_len))

    print('Length of decompressed file in format 2 is {}'.format(decompressed_len(text)))

if __name__ == "__main__":
    decompress('day9_input.txt')


text:
# <editor-fold desc="The following 6 lines of code are required to allow Heron to be able to see the Operation without
# package installation. Do not change.">
import sys
from os import path
import cv2
import numpy as np

current_dir = path.dirname(path.abspath(__file__))
while path.split(current_dir)[-1] != r'Heron':
    current_dir = path.dirname(current_dir)
sys.path.insert(0, path.dirname(current_dir))
# </editor-fold>

# <editor-fold desc="Extra imports if required">
from Heron.communication.socket_for_serialization import Socket
from Heron import general_utils as gu, constants as ct
import copy
# </editor-fold>

# <editor-fold desc="Global variables if required. Global variables operate obviously within the scope of the process

vis: bool
block_length_ranges: list
lengths_block: list
reward_contingencies: list
current_block: list
correct_licks: list
# </editor-fold>

def initialise(worker_object):
    global vis
    global block_length_ranges
    global lengths_block
    global current_block
    global reward_contingencies
    global correct_licks

    try:
        parameters = worker_object.parameters
        vis = parameters[0]
        block_length_ranges_str = parameters[1]
        reward_contingencies_block1_str = parameters[2]
        reward_contingencies_block2_str = parameters[3]
    except:
        return False

    block_length_ranges = [int(i) for i in block_length_ranges_str.split(',')]
    create_new_block_sizes()
    #current_block = [np.random.randint(0, 2) for i in np.arange(4)]
    current_block = [np.random.randint(0, 2) for i in np.arange(1)]
    reward_contingencies = [[float(i) for i in reward_contingencies_block1_str.split(',')],
                            [float(i) for i in reward_contingencies_block2_str.split(',')]]
    correct_licks = [0 for i in np.arange(4)]

    print(lengths_block, current_block, reward_contingencies)
    return True

def create_new_block_sizes(stim=None):
    global block_length_ranges
    global lengths_block

    if stim == None:
        lengths_block = [[np.random.randint(block_length_ranges[i], block_length_ranges[i + 1]) for i
                          in np.arange(0, 8, 2)] for i in np.arange(2)]
    else:
        s = int(stim)
        new_lengths = [np.random.randint(block_length_ranges[2 * s], block_length_ranges[2 * s + 1]) for i
                               in np.arange(2)]
        lengths_block[0][s] = new_lengths[0]
        lengths_block[1][s] = new_lengths[1]

def work_function(data, parameters):
    global vis
    global lengths_block
    global current_block
    global reward_contingencies
    global correct_licks

    vis = parameters[0]

    message = data[1:]  # data[0] is the topic
    message = Socket.reconstruct_array_from_bytes_message(message)

    # If the message comes from the Trial controller
    if len(message) == 2:
        # The Trial controller sends [previous_stim, correct_port_licks]
        previous_stim = message[0]
        correct_port_licks = message[1]

        # Add the number of correct licks to the current running sum
        correct_licks[previous_stim] = correct_licks[previous_stim] + correct_port_licks

        # If the number of correct licks reaches the block length of that stim then swap block
        # and zero the running sum of licks for that stim
        if correct_licks[previous_stim] == lengths_block[current_block[previous_stim]][previous_stim]:
            create_new_block_sizes(previous_stim)
            temp = copy.copy(current_block[previous_stim])
            current_block[previous_stim] = int(not current_block[previous_stim])
            correct_licks[previous_stim] = 0
            if vis:
                print('Changing block of stim {}, from block {} to block {}'.format(previous_stim,
                                                                                temp,
                                                                                current_block[previous_stim]))
                print('Current block lengths = {}'.format(lengths_block))

    #current_stim = np.random.randint(0, 4)
    current_stim = np.random.randint(0, 1)
    current_correct_reward_port_probability = reward_contingencies[current_block[current_stim]][current_stim]
    current_correct_reward_port = np.random.binomial(1, current_correct_reward_port_probability)

    result = [np.array([current_stim, current_correct_reward_port, current_block[current_stim]])]

    return result

# The on_end_of_life function must exist even if it is just a pass
def on_end_of_life():
    pass

if __name__ == "__main__":
    worker_object = gu.start_the_transform_worker_process(work_function=work_function,
                                                          end_of_life_function=on_end_of_life,
                                                          initialisation_function=initialise)
    worker_object.start_ioloop()


text:
from classes.layers.layer import Layer
from classes import functions as funcs

import numpy as np

class FcLayer(Layer):
    def __init__(self, h_size, i_size, o_size,
                 is_input, is_output, is_hidden,
                 u_type='adam', dropout=1.0, a_type="relu", need_activation=True, **kwargs):

        self._w, self._d, self._m = {}, {}, {}

        assert h_size == o_size

        self.h_size = h_size
        self.i_size = i_size
        self.o_size = o_size
        self.is_ouput = is_output
        self.is_input = is_input
        self.is_hidden = is_hidden
        self.u_type = u_type
        self.dropout = dropout
        self.a_type = a_type
        self.need_activation = need_activation

        # for cache
        self.x = []

        self._w_keys = ['w']
        self._b_keys = ['b']

        self._w, self._d, self._m = {}, {}, {}

        for k in self._w_keys:
            self._w[k] = np.random.randn(o_size, i_size).astype(np.float32) * np.sqrt(2.0 / i_size)

        for k in self._b_keys:
            self._w[k] = np.zeros((self.o_size, 1), np.float32)

        for k in self._w_keys + self._b_keys:
            self._m[k] = {'m': 0, 'v': 0, 't': 0}
            self._d[k] = np.zeros_like(self._w[k], np.float32)

    def forward(self, x):
        self.x = x
        y_prime = []

        for t in range(len(x)):
            y = self._w['w'].dot(x[t]) + self._w['b']

            if self.need_activation:
                if self.a_type == 'relu':
                    y = funcs.relu(y)
                elif self.a_type == 'sigmoid':
                    y = funcs.sigmoid(y)
                elif self.a_type == 'tanh':
                    y = funcs.sigmoid(y)

            y_prime.append(y)

        y_prime = np.array(y_prime)

        return y_prime, y_prime

    def backward(self, dy):
        output_d = []

        for t in range(len(self.x)):

            if self.need_activation:
                if self.a_type == 'relu':
                    dy = funcs.d_relu(dy)
                elif self.a_type == 'sigmoid':
                    dy = funcs.d_sigmoid(dy)
                elif self.a_type == 'tanh':
                    dy = funcs.d_tanh(dy)

            self._d['w'] += dy[t].dot(self.x[t].T)
            self._d['b'] += dy[t]
            dx = self._w['w'].T.dot(dy[t])

            output_d.append(dx)

        return np.array(output_d)

    def update(self, lr):
        for k in self._w_keys + self._b_keys:
            if self.u_type == 'adam':
                self._w[k] = funcs.adam_update(self._w[k], self._d[k], self._m[k], lr)
            elif self.u_type == 'adagrad':
                self._w[k] = funcs.adagrad_update(self._w[k], self._d[k], self._m[k], lr)

        for k in self._w_keys + self._b_keys:
            self._d[k] = np.zeros_like(self._w[k], np.float32)

    def predict(self, x):
        y_prime = []

        for t in range(len(x)):
            y = self._w['w'].dot(x[t]) + self._w['b']

            if self.need_activation:
                if self.a_type == 'relu':
                    y = funcs.relu(y)
                elif self.a_type == 'sigmoid':
                    y = funcs.sigmoid(y)
                elif self.a_type == 'tanh':
                    y = funcs.sigmoid(y)

            y_prime.append(y)

        y_prime = np.array(y_prime)

        return y_prime, y_prime

max_stars_repo_name: amsukdu/SimplePythonRNN

max_issues_repo_name: amsukdu/SimplePythonRNN

max_forks_repo_name: amsukdu/SimplePythonRNN


text:
import numpy as np

from sortedcontainers import SortedDict
from sklearn import manifold
from format import wordLocationAsDict, getRangeMiddle

def _getPairwiseDistances(wordsT1, model):
    dists = np.zeros((len(wordsT1), len(wordsT1)))
    for i, w1 in enumerate(wordsT1):
        for j, w2 in enumerate(wordsT1[i:]):
            try:
                dists[i, i+j] = 1-model.n_similarity(w1, w2)
            except:
                # If we cannot calculate the similarity, they are
                # as dissimilar as possible
                dists[i, i+j] = 1
            dists[i+j, i] = dists[i, i+j]
    return dists

def _getMDSEmbedding(dists):
    seed = np.random.RandomState(seed=3)
    mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed,
                       dissimilarity="precomputed", n_jobs=1)
    xyEmbedding = mds.fit(dists).embedding_
    return xyEmbedding

def _normalizeCloud(X):
    X -= X.min(axis=0)
    X /= X.max(axis=0)
    X -= X.mean(axis=0)
    return X

def _findTransform(wordsT0, locsT0, wordsT1, locsT1):
    matchingTerms = list(set(wordsT0).intersection(set(wordsT1)))

    # If we don't have any matching terms -- do not transform
    # (or transform by identity matrix)
    if len(matchingTerms) == 0:
        return np.eye(2)

    F0 = []
    F1 = []

    for word in matchingTerms:
        F0.append(locsT0[wordsT0.index(word)])
        F1.append(locsT1[wordsT1.index(word)])

    F0 = np.array(F0)
    F1 = np.array(F1)

    T, residuals, rank, s = np.linalg.lstsq(F1, F0)
    return T

def doSpaceEmbedding(monitor, results, aggMetadata):
    '''Create 2D word embedding from given set of results'''
    embeddedResults = SortedDict()

    wordsT0 = None
    locsT0 = None
    for label, r in results.items():
        model = monitor._models[label]
        wordsT1 = [w for w, _ in r]

        dists = _getPairwiseDistances(wordsT1, model)
        locsT1 = _getMDSEmbedding(dists)

        if wordsT0 is not None:
            T = _findTransform(wordsT0, locsT0, wordsT1, locsT1)
            locsT1 = locsT1.dot(T)
            locsT1 = _normalizeCloud(locsT1)

        wordsT0 = wordsT1
        locsT0 = locsT1

        str_label = str(int(getRangeMiddle(label)))
        embeddedResults[str_label] = [wordLocationAsDict(
            wordsT1[i], locsT1[i, :]) for i in range(len(wordsT1))]

    # Aggregation step (more like throwing away some years)
    embeddedResultsAgg = {year: embeddedResults[year] for year in aggMetadata if year in embeddedResults}
    embeddedResultsAgg = SortedDict(embeddedResultsAgg)

    return embeddedResultsAgg

max_stars_repo_name: ivanvladimir/ShiCo

max_issues_repo_name: ivanvladimir/ShiCo

max_forks_repo_name: ivanvladimir/ShiCo