text:
import numpy as np

def rsmat(arbmat):
    """ Convert an arbitrary matrix to its simplest format """
    arbmat = arbmat.astype(float)
    row_number, column_number = arbmat.shape
    if row_number == 1:
        if arbmat[0, 0] != 0:
            return arbmat / arbmat[0, 0]
        else:
            return arbmat
    else:
        rc_number = min(row_number, column_number)
        anarbmat = arbmat.copy()
        r = 0
        for n in range(rc_number):
            s_row = -1
            for i in arbmat[r:row_number, n]:
                s_row += 1
                if abs(i) > 1e-10:
                    anarbmat[r, :] = arbmat[s_row + r, :]
                    for j in range(r, row_number):
                        if j < s_row + r:
                            anarbmat[j + 1, :] = arbmat[j, :]
                    arbmat = anarbmat.copy()
            if abs(anarbmat[r, n]) > 1e-10:
                anarbmat[r, :] = anarbmat[r, :] / anarbmat[r, n]
                for i in range(row_number):
                    if i != r:
                        anarbmat[i, :] -= \
                            anarbmat[i, n] * anarbmat[r, :]
            arbmat = anarbmat.copy()
            if abs(arbmat[r, n]) < 1e-10:
                r = r
            else:
                r = r + 1
        for m in range(column_number):
            if abs(arbmat[-1, m]) > 1e-10:
                arbmat[-1, :] = arbmat[-1, :] / arbmat[-1, m]
                for i in range(row_number - 1):
                    arbmat[i, :] -= \
                        arbmat[i, m] * arbmat[-1, :]
                break

        return arbmat

if __name__ == '__main__':

    a = np.array([
            [1, 0, 2, -1, 1, 3],
            [2, 0, 3, 1, 4, 5],
            [2, 0, 3, 1, 4, 5]])

    a1 = rsmat(a)

    print(a1)

max_stars_repo_name: mathischeap/mifem

max_issues_repo_name: mathischeap/mifem

max_forks_repo_name: mathischeap/mifem


text:
import click
import numpy as np
import pickle
import open3d as o3d

import pcp_utils

@click.command()
@click.argument("pcd_file")#config
@click.option("--mode")#config
def main(pcd_file, mode):

    if mode == "generated_grasps":
        pcds = np.load(pcd_file, allow_pickle=True).item()

        scene_pcd = pcp_utils.np_vis.get_pcd_object(pcds['pcd'], clip_radius=10.0)

        #'generated_scores': generated_scores,
        #import ipdb; ipdb.set_trace()
        #scene_pcd = pcp_utils.np_vis.get_pcd_object(pcds['obj_pcd'], clip_radius=10.0)
        frame = o3d.geometry.TriangleMesh.create_coordinate_frame(origin=np.zeros(3), size=0.8)
        things_to_print = [scene_pcd, frame]

        num_grasps = len(pcds['generated_scores'])
        threshold = 0 #0.95

        good_grasps_ids = [id_ for id_ in range(num_grasps) if pcds['generated_scores'][id_] > threshold]
        pcds['generated_grasps'] = [pcds['generated_grasps'][id_] for id_ in good_grasps_ids]
        num_grasps = len(pcds['generated_grasps'])

        for grasp_id in range(num_grasps):
            rot = pcds['generated_grasps'][grasp_id]
            grasp_linset = pcp_utils.np_vis.make_lineset_4x4matrix(rot)
            things_to_print += [grasp_linset]

        o3d.visualization.draw_geometries(things_to_print)
    else:

        if mode == "pcds_color":
            out = np.load(pcd_file, allow_pickle=True).item()
            pcds = out["pts"]
            color = out["color"]

        else:
            pcds = np.load(pcd_file)

        object_pcd = pcp_utils.np_vis.get_pcd_object(pcds, clip_radius=10.0)
        frame = o3d.geometry.TriangleMesh.create_coordinate_frame(origin=np.zeros(3), size=0.8)

        things_to_print = [object_pcd, frame]
        o3d.visualization.draw_geometries(things_to_print)

if __name__=="__main__":
	main()

ext: py

reason: import numpy


text:
"""
The pre-trained models produce embeddings of size 512 - 1024. However, when storing a large
number of embeddings, this requires quite a lot of memory / storage.

In this example, we reduce the dimensionality of the embeddings to e.g. 128 dimensions. This significantly
reduces the required memory / storage while maintaining nearly the same performance.

For dimensionality reduction, we compute embeddings for a large set of (representative) sentence. Then,
we use PCA to find e.g. 128 principle components of our vector space. This allows us to maintain
us much information as possible with only 128 dimensions.

PCA gives us a matrix that down-projects vectors to 128 dimensions. We use this matrix
and extend our original SentenceTransformer model with this linear downproject. Hence,
the new SentenceTransformer model will produce directly embeddings with 128 dimensions
without further changes needed.
"""
from sklearn.decomposition import PCA
from sentence_transformers import SentenceTransformer, LoggingHandler, util, evaluation, models, InputExample
import logging
import os
import gzip
import csv
import random
import numpy as np
import torch

#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
#### /print debug information to stdout

#Model for which we apply dimensionality reduction
model = SentenceTransformer('roberta-base-nli-stsb-mean-tokens')

#New size for the embeddings
new_dimension = 128

#We use AllNLI as a source of sentences to compute PCA
nli_dataset_path = 'datasets/AllNLI.tsv.gz'

#We use the STS benchmark dataset to see how much performance we loose by using the dimensionality reduction
sts_dataset_path = 'datasets/stsbenchmark.tsv.gz'

if not os.path.exists(nli_dataset_path):
    util.http_get('https://sbert.net/datasets/AllNLI.tsv.gz', nli_dataset_path)

if not os.path.exists(sts_dataset_path):
    util.http_get('https://sbert.net/datasets/stsbenchmark.tsv.gz', sts_dataset_path)

# We measure the performance of the original model
# and later we will measure the performance with the reduces dimension size
logging.info("Read STSbenchmark test dataset")
eval_examples = []
with gzip.open(sts_dataset_path, 'rt', encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        if row['split'] == 'test':
            score = float(row['score']) / 5.0 #Normalize score to range 0 ... 1
            eval_examples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))

# Evaluate the original model on the STS benchmark dataset
stsb_evaluator = evaluation.EmbeddingSimilarityEvaluator.from_input_examples(eval_examples, name='sts-benchmark-test')

logging.info("Original model performance:")
stsb_evaluator(model)

######## Reduce the embedding dimensions ########

#Read sentences from NLI dataset
nli_sentences = set()
with gzip.open(nli_dataset_path, 'rt', encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        nli_sentences.add(row['sentence1'])
        nli_sentences.add(row['sentence2'])

nli_sentences = list(nli_sentences)
random.shuffle(nli_sentences)

#To determine the PCA matrix, we need some example sentence embeddings.
#Here, we compute the embeddings for 20k random sentences from the AllNLI dataset
pca_train_sentences = nli_sentences[0:20000]
train_embeddings = model.encode(pca_train_sentences, convert_to_numpy=True)

#Compute PCA on the train embeddings matrix
pca = PCA(n_components=new_dimension)
pca.fit(train_embeddings)
pca_comp = np.asarray(pca.components_)

# We add a dense layer to the model, so that it will produce directly embeddings with the new size
dense = models.Dense(in_features=model.get_sentence_embedding_dimension(), out_features=new_dimension, bias=False, activation_function=torch.nn.Identity())
dense.linear.weight = torch.nn.Parameter(torch.tensor(pca_comp))
model.add_module('dense', dense)

# Evaluate the model with the reduce embedding size
logging.info("Model with {} dimensions:".format(new_dimension))
stsb_evaluator(model)

# If you like, you can store the model on disc by uncommenting the following line
#model.save('models/bert-base-nli-stsb-mean-tokens-128dim')

# You can then load the adapted model that produces 128 dimensional embeddings like this:
#model = SentenceTransformer('models/bert-base-nli-stsb-mean-tokens-128dim')

max_stars_repo_name: dd-dos/sentence-transformers

max_issues_repo_name: dd-dos/sentence-transformers

max_forks_repo_name: dd-dos/sentence-transformers


text:
#-- coding: utf-8 --
#@Time : 2021/3/27 20:40
#@Author : HUANG XUYANG
#@Email : xhuang032@e.ntu.edu.sg
#@File : RVFL.py
#@Software: PyCharm

import numpy as np
import sklearn.datasets as sk_dataset

num_nides = 10  # Number of enhancement nodes.
regular_para = 1  # Regularization parameter.
weight_random_range = [-1, 1]  # Range of random weights.
bias_random_range = [0, 1]  # Range of random weights.

class RVFL:
    """A simple RVFL classifier.

    Attributes:
        n_nodes: An integer of enhancement node number.
        lam: A floating number of regularization parameter.
        w_random_vec_range: A list, [min, max], the range of generating random weights.
        b_random_vec_range: A list, [min, max], the range of generating random bias.
        random_weights: A Numpy array shape is [n_feature, n_nodes], weights of neuron.
        random_bias: A Numpy array shape is [n_nodes], bias of neuron.
        beta: A Numpy array shape is [n_feature + n_nodes, n_class], the projection matrix.
        activation: A string of activation name.
        data_std: A list, store normalization parameters for each layer.
        data_mean: A list, store normalization parameters for each layer.
        same_feature: A bool, the true means all the features have same meaning and boundary for example: images.
    """
    def __init__(self, n_nodes, lam, w_random_vec_range, b_random_vec_range, activation, same_feature=False):
        self.n_nodes = n_nodes
        self.lam = lam
        self.w_random_range = w_random_vec_range
        self.b_random_range = b_random_vec_range
        self.random_weights = None
        self.random_bias = None
        self.beta = None
        a = Activation()
        self.activation_function = getattr(a, activation)
        self.data_std = None
        self.data_mean = None
        self.same_feature = same_feature

    def train(self, data, label, n_class):
        """

        :param data: Training data.
        :param label: Training label.
        :param n_class: An integer of number of class.
        :return: No return
        """

        assert len(data.shape) > 1, 'Data shape should be [n, dim].'
        assert len(data) == len(label), 'Label number does not match data number.'
        assert len(label.shape) == 1, 'Label should be 1-D array.'

        data = self.standardize(data)  # Normalization data
        n_sample = len(data)
        n_feature = len(data[0])
        self.random_weights = self.get_random_vectors(n_feature, self.n_nodes, self.w_random_range)
        self.random_bias = self.get_random_vectors(1, self.n_nodes, self.b_random_range)

        h = self.activation_function(np.dot(data, self.random_weights) + np.dot(np.ones([n_sample, 1]), self.random_bias))
        d = np.concatenate([h, data], axis=1)
        d = np.concatenate([d, np.ones_like(d[:, 0:1])], axis=1)
        y = self.one_hot(label, n_class)
        if n_sample > (self.n_nodes + n_feature):
            self.beta = np.linalg.inv((self.lam * np.identity(d.shape[1]) + np.dot(d.T, d))).dot(d.T).dot(y)
        else:
            self.beta = d.T.dot(np.linalg.inv(self.lam * np.identity(n_sample) + np.dot(d, d.T))).dot(y)

    def predict(self, data, output_prob=False):
        """

        :param data: Predict data.
        :param output_prob: A bool number, if True return the raw predict probability, if False return predict class.
        :return: Prediction result.
        """
        data = self.standardize(data)  # Normalization data
        h = self.activation_function(np.dot(data, self.random_weights) + self.random_bias)
        d = np.concatenate([h, data], axis=1)
        d = np.concatenate([d, np.ones_like(d[:, 0:1])], axis=1)
        result = self.softmax(np.dot(d, self.beta))
        if not output_prob:
            result = np.argmax(result, axis=1)
        return result

    def eval(self, data, label):
        """

        :param data: Evaluation data.
        :param label: Evaluation label.
        :return: Accuracy.
        """

        assert len(data.shape) > 1, 'Data shape should be [n, dim].'
        assert len(data) == len(label), 'Label number does not match data number.'
        assert len(label.shape) == 1, 'Label should be 1-D array.'

        data = self.standardize(data)  # Normalization data
        h = self.activation_function(np.dot(data, self.random_weights) + self.random_bias)
        d = np.concatenate([h, data], axis=1)
        d = np.concatenate([d, np.ones_like(d[:, 0:1])], axis=1)
        result = np.dot(d, self.beta)
        result = np.argmax(result, axis=1)
        acc = np.sum(np.equal(result, label))/len(label)
        return acc

    def get_random_vectors(self, m, n, scale_range):
        x = (scale_range[1] - scale_range[0]) * np.random.random([m, n]) + scale_range[0]
        return x

    def one_hot(self, x, n_class):
        y = np.zeros([len(x), n_class])
        for i in range(len(x)):
            y[i, x[i]] = 1
        return y

    def standardize(self, x):
        if self.same_feature is True:
            if self.data_std is None:
                self.data_std = np.maximum(np.std(x), 1/np.sqrt(len(x)))
            if self.data_mean is None:
                self.data_mean = np.mean(x)
            return (x - self.data_mean) / self.data_std
        else:
            if self.data_std is None:
                self.data_std = np.maximum(np.std(x, axis=0), 1/np.sqrt(len(x)))
            if self.data_mean is None:
                self.data_mean = np.mean(x, axis=0)
            return (x - self.data_mean) / self.data_std

    def softmax(self, x):
        return np.exp(x) / np.repeat((np.sum(np.exp(x), axis=1))[:, np.newaxis], len(x[0]), axis=1)

class Activation:
    def sigmoid(self, x):
        return 1 / (1 + np.e ** (-x))

    def sine(self, x):
        return np.sin(x)

    def hardlim(self, x):
        return (np.sign(x) + 1) / 2

    def tribas(self, x):
        return np.maximum(1 - np.abs(x), 0)

    def radbas(self, x):
        return np.exp(-(x**2))

    def sign(self, x):
        return np.sign(x)

    def relu(self, x):
        return np.maximum(0, x)

def prepare_data(proportion):
    dataset = sk_dataset.load_breast_cancer()
    label = dataset['target']
    data = dataset['data']
    n_class = len(dataset['target_names'])

    shuffle_index = np.arange(len(label))
    np.random.shuffle(shuffle_index)

    train_number = int(proportion * len(label))
    train_index = shuffle_index[:train_number]
    val_index = shuffle_index[train_number:]
    data_train = data[train_index]
    label_train = label[train_index]
    data_val = data[val_index]
    label_val = label[val_index]
    return (data_train, label_train), (data_val, label_val), n_class

if __name__ == '__main__':
    train, val, num_class = prepare_data(0.8)
    rvfl = RVFL(num_nides, regular_para, weight_random_range, bias_random_range, 'relu', False)
    rvfl.train(train[0], train[1], num_class)
    prediction = rvfl.predict(val[0], output_prob=True)
    accuracy = rvfl.eval(val[0], val[1])
    print(accuracy)

max_stars_repo_name: Xuyang-Huang/Simple-RVFL-python

max_issues_repo_name: Xuyang-Huang/Simple-RVFL-python

max_forks_repo_name: Xuyang-Huang/Simple-RVFL-python


text:
# pylint: disable=g-bad-file-header
# Copyright 2021 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Sweeps for MC Dropout agent."""

import dataclasses
from typing import Sequence

from enn import base as enn_base
from enn import losses
from enn import networks
from neural_testbed import agents
from neural_testbed import base as testbed_base
from neural_testbed.agents.factories import base as factories_base
import numpy as np
import optax

@dataclasses.dataclass
class McDropoutConfig:
  """Configuration for mc dropout agent."""
  dropout_rate: float = 0.1  # Drop probability for each hidden unit
  length_scale: float = 1.  # Length scale used for weight regularization
  regularization_tau: float = 1.  # tau for scaling the weight regularizer
  dropout_input: bool = False  # Whether to have dropout for the input layer
  exclude_bias_l2: bool = False  # Whether to exclude bias from regularization
  adaptive_weight_scale: bool = True  # Whether to scale with prior
  hidden_sizes: Sequence[int] = (100, 100)  # Hidden sizes for neural network
  num_batches: int = 1000  # Number of SGD steps
  learning_rate: float = 1e-3  # Learning rate for adam optimizer
  seed: int = 0  # Initialization seed

def make_mc_dropout_agent(
    config: McDropoutConfig) -> agents.VanillaEnnAgent:
  """Factory method to create MC dropout agent."""

  def make_enn(prior: testbed_base.PriorKnowledge) -> enn_base.EpistemicNetwork:
    return networks.MLPDropoutENN(
        output_sizes=list(config.hidden_sizes) + [prior.num_classes],
        dropout_rate=config.dropout_rate,
        dropout_input=config.dropout_input,
        seed=config.seed,
    )

  def make_loss(prior: testbed_base.PriorKnowledge,
                enn: enn_base.EpistemicNetwork) -> enn_base.LossFn:
    del enn
    single_loss = losses.combine_single_index_losses_as_metric(
        # This is the loss you are training on.
        train_loss=losses.XentLoss(prior.num_classes),
        # We will also log the accuracy in classification.
        extra_losses={'acc': losses.AccuracyErrorLoss(prior.num_classes)},
    )

    # Averaging over index
    loss_fn = losses.average_single_index_loss(single_loss, num_index_samples=1)

    # Adding a special weight regularization based on paper "Dropout as a
    # Bayesian Approximation: Representing Model Uncertainty in Deep Learning",
    # https://github.com/yaringal/DropoutUncertaintyExps/blob/master/net/net.py#L72
    scale = (config.length_scale**2) * (1 - config.dropout_rate) / (
        2. * prior.num_train * config.regularization_tau)
    if config.adaptive_weight_scale:
      scale = config.length_scale * np.sqrt(
          prior.temperature) * prior.input_dim / prior.num_train
    if config.exclude_bias_l2:
      predicate = lambda module, name, value: name != 'b'
    else:
      predicate = lambda module, name, value: True
    loss_fn = losses.add_l2_weight_decay(loss_fn, scale, predicate)
    return loss_fn

  agent_config = agents.VanillaEnnConfig(
      enn_ctor=make_enn,
      loss_ctor=make_loss,
      optimizer=optax.adam(config.learning_rate),
      num_batches=config.num_batches,
      seed=config.seed,)

  return agents.VanillaEnnAgent(agent_config)

def droprate_sweep() -> Sequence[McDropoutConfig]:
  """Generates the dropout sweep over dropping parameters for paper."""
  sweep = []
  for dropout_rate in [0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]:
    for dropout_input in [True, False]:
      sweep.append(
          McDropoutConfig(
              dropout_rate=dropout_rate,
              dropout_input=dropout_input))
  return tuple(sweep)

def l2reg_sweep() -> Sequence[McDropoutConfig]:
  """Generates the dropout sweep over l2 regularization parameters for paper."""
  sweep = []
  for adaptive_weight_scale in [True, False]:
    for length_scale in [1, 3, 10]:
      sweep.append(
          McDropoutConfig(
              adaptive_weight_scale=adaptive_weight_scale,
              length_scale=length_scale))
  return tuple(sweep)

def network_sweep() -> Sequence[McDropoutConfig]:
  """Generates the dropout sweep over dropping parameters for paper."""
  sweep = []
  for hidden_sizes in [(50, 50), (100, 100), (50, 50, 50)]:
    sweep.append(McDropoutConfig(hidden_sizes=hidden_sizes))
  return tuple(sweep)

def combined_sweep() -> Sequence[McDropoutConfig]:
  return tuple(droprate_sweep()) + tuple(l2reg_sweep()) + tuple(network_sweep())

def paper_agent() -> factories_base.PaperAgent:
  return factories_base.PaperAgent(
      default=McDropoutConfig(),
      ctor=make_mc_dropout_agent,
      sweep=combined_sweep,
      )


text:
# Third Party
import numpy as np
import pytest
import xgboost as xgb

# First Party
from smdebug.xgboost.utils import parse_tree_model

@pytest.mark.slow
def test_parse_tree_model():
    np.random.seed(42)
    num_row, num_col = 20, 10
    train_data = np.random.rand(num_row, num_col)
    train_label = np.random.randint(2, size=num_row)
    dtrain = xgb.DMatrix(train_data, label=train_label)
    valid_data = np.random.rand(num_row, num_col)
    valid_label = np.random.randint(2, size=num_row)
    dvalid = xgb.DMatrix(valid_data, label=valid_label)
    params = {"objective": "binary:logistic"}
    num_boost_round = 10
    bst = xgb.train(params, dtrain, evals=[(dtrain, "train")], num_boost_round=num_boost_round)

    columns = ["Tree", "Node", "ID", "Feature", "Split", "Yes", "No", "Missing", "Gain", "Cover"]

    try:
        from pandas import DataFrame  # noqa

        pandas_installed = True
    except ImportError:
        pandas_installed = False

    for iteration in range(num_boost_round):
        tree = parse_tree_model(bst, iteration)
        if pandas_installed:
            df = bst.trees_to_dataframe()
            df = df.loc[df["Tree"] == iteration].reset_index(drop=True)
            for col in columns:
                x, y = tree[col], df[col].values
                if x.dtype != y.dtype:
                    y = y.astype(x.dtype)
                # We first check if arrays are equal. Due to the fact that
                # np.nan != np.nan, equality check will fail if we have any
                # np.nan's. If all items are not equal, then we also check for
                # inequality in the same positions because np.nan != np.nan.
                assert ((x == y) | ((x != x) & (y != y))).all()
        assert sorted(tree.keys()) == sorted(columns)
        assert len(tree["Tree"]) > 0
        assert all(len(tree["Tree"]) == len(tree[col]) for col in columns)
        assert all(x == iteration for x in tree["Tree"])
        assert (sorted(tree["Node"]) == tree["Node"]).all()

max_stars_repo_name: NRauschmayr/sagemaker-debugger

max_issues_repo_name: NRauschmayr/sagemaker-debugger

max_forks_repo_name: NRauschmayr/sagemaker-debugger


text:
import numpy as np
import pandas as pd
import keras
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator

def prepare_data(num_classes, train_test_ratio = 0.8):
    # Load the train data and split it .8/.2

    train_data = pd.read_csv("./data/train.csv")

    print('train_data.shape=', train_data.shape)

    train_size = int(len(train_data) * train_test_ratio)
    y_train = train_data['label'][:train_size]
    y_test = train_data['label'][train_size:]

    x_train = train_data.drop(labels=['label'], axis=1)[:train_size]
    x_test = train_data.drop(labels=['label'], axis=1)[train_size:]

    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255

    print(x_train.shape)

    x_train = x_train.values.reshape(-1, 28, 28, 1)
    x_test = x_test.values.reshape(-1, 28, 28, 1)

    print(x_train.shape[0], 'train samples')
    print(x_test.shape[0], 'test samples')

    # convert class vectors to binary class matrices
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    print('x_train.shape=', x_train.shape)
    print('y_train.shape=', y_train.shape)

    print('x_test.shape=', x_test.shape)
    print('y_test.shape=', y_test.shape)

    return x_train, y_train, x_test, y_test

def fit_model(m, kx_train, ky_train, kx_test, ky_test, batch_size=128, max_epochs=1000):

    early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',
                                                   min_delta=0.01,
                                                   patience=10,
                                                   verbose=1,
                                                   mode='max')

    reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
                                                  factor=0.5,
                                                  patience=10,
                                                  min_lr=0.0001,
                                                  verbose=1)

    m.fit(kx_train,
          ky_train,
          batch_size=batch_size,
          epochs=max_epochs,
          verbose=1,
          validation_data=(kx_test, ky_test),
          callbacks=[early_stopping, reduce_lr])

def fit_model_with_geneartor(m, x_train, y_train, x_test, y_test, batch_size=128, max_epochs=1000):

    early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',
                                                   min_delta=0.01,
                                                   patience=10,
                                                   verbose=1,
                                                   mode='max')

    reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
                                                  factor=0.5,
                                                  patience=10,
                                                  min_lr=0.0001,
                                                  verbose=1)

    gen = ImageDataGenerator(rotation_range=8,
                             width_shift_range=0.08,
                             shear_range=0.3,
                             height_shift_range=0.08,
                             zoom_range=0.08)

    test_gen = ImageDataGenerator()

    train_generator = gen.flow(x_train, y_train, batch_size=batch_size)
    test_generator = test_gen.flow(x_test, y_test, batch_size=batch_size)

    m.fit_generator(train_generator,
                    steps_per_epoch=x_train.shape[0] / batch_size,
                    epochs=max_epochs,
                    validation_data=test_generator,
                    validation_steps=x_test.shape[0] / batch_size,
                    callbacks=[early_stopping, reduce_lr])

def predict_results(m):
    test_data = pd.read_csv("./data/test.csv")
    test_data = test_data.astype('float32')
    test_data /= 255
    test_data = test_data.values.reshape(-1, 28, 28, 1)

    return m.predict(test_data)

batch_size = 128
num_classes = 10
epochs = 20
train_test_ratio = 0.8

# input image dimensions
img_rows, img_cols = 28, 28

# Load and preparing the train and test data
x_train, y_train, x_test, y_test = prepare_data(num_classes, train_test_ratio)

# Creating the model
model = Sequential()

model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))

model.summary()

model.compile(optimizer='adadelta',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Data augmentation

gen = ImageDataGenerator(rotation_range=8,
                         width_shift_range=0.08,
                         shear_range=0.3,
                         height_shift_range=0.08,
                         zoom_range=0.08)

test_gen = ImageDataGenerator()

train_generator = gen.flow(x_train, y_train, batch_size=batch_size)
test_generator = test_gen.flow(x_test, y_test, batch_size=batch_size)

# Fitting the model
# fit_model(model, x_train, y_train, x_test, y_test, batch_size, epochs)
fit_model_with_geneartor(model, x_train, y_train, x_test, y_test, batch_size, epochs)

# Predict the test results
results = predict_results(model)
results = np.argmax(results, axis=1)
results = pd.Series(results, name="Label")

submission = pd.concat([pd.Series(range(1, 28001), name="ImageId"), results], axis=1)
submission.to_csv("submission.csv", index=False)


text:
import os
import numpy as np
import shutil
import random
import time
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from tensorboardX import SummaryWriter
from tools import metric
from opts import parser
from ops.utils import get_logger, AverageMeter, accuracy
from archs.i3d_model import I3D

from ops import videotransforms
from archs.fusion_i3d import fusion
from ops.drive_dataset_with_keypoint_i3d import Drive as Dataset

SEED = 777
random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
np.random.seed(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

def main():
    global best_recall, args
    args = parser.parse_args()
    best_recall = 0
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
    if not os.path.isdir(args.root_model):
        os.makedirs(args.root_model)

    global logger, writer
    logger = get_logger(args)
    logger.info('SEED {}\nruntime args\n{}\n\n'.format(SEED, args))
    writer = SummaryWriter(args.root_model)

    if args.arch == 'i3d':
        model = I3D(num_classes=args.num_class)
        pretrained_dict = torch.load(args.tune_from)

        pretrained_dict = {k: v for k, v in pretrained_dict.items() if 'conv3d_0c_1x1' not in k}
        model_dict = model.state_dict()
        model_dict.update(pretrained_dict)
        model.load_state_dict(model_dict)
    elif args.arch == 'i3d_all':
        model = fusion(first=args.first, second=args.second,
                       stride=args.gcn_stride, patch_size=args.patch_size,
                       concat_layer=args.concat_layer, xyc=args.xyc, bn=args.bn)
        if args.gcn_pretrained or args.tune_from:
            model_dict = model.state_dict()
            replace_dict_gcn = []
            replace_dict_cnn = []
            new_sd = {}
            if args.gcn_pretrained:
                print(("=> fine-tuning from '{}'".format(args.gcn_pretrained)))
                gcn_ckpt = torch.load(args.gcn_pretrained)
                for k, v in gcn_ckpt.items():
                    if k == 'A' or 'data_bn' in k or 'edge_importance' in k:
                        continue
                    if k not in model_dict and 'st_gcn.' + k in model_dict:
                        # print('=> Load after add st_gcn.: ', k)
                        replace_dict_gcn.append((k, 'st_gcn.' + k))
                    else:
                        print(k)
            print(("=> fine-tuning from '{}'".format(args.tune_from)))
            sd = torch.load(args.tune_from)
            for k, v in sd.items():
                if k not in model_dict and 'cnn.' + k in model_dict:
                    # print('=> Load after add cnn: ', k)
                    replace_dict_cnn.append((k, 'cnn.' + k))
                else:
                    print(k)
            for k, k_new in replace_dict_gcn:
                new_sd[k_new] = gcn_ckpt.pop(k)
            for k, k_new in replace_dict_cnn:
                new_sd[k_new] = sd.pop(k)
            if new_sd == {} and args.arch != 'fusion':
                new_sd = sd
            keys1 = set(list(new_sd.keys()))
            keys2 = set(list(model_dict.keys()))
            set_diff = (keys1 - keys2) | (keys2 - keys1)
            print('#### Notice: keys that failed to load: {}'.format(set_diff))
            print('=> New dataset, do not load fc weights')
            new_sd = {k: v for k, v in new_sd.items() if 'fc' not in k}
            new_sd = {k: v for k, v in new_sd.items() if 'conv3d_0c_1x1.conv3d' not in k}
            model_dict.update(new_sd)
            model.load_state_dict(model_dict)
    model = nn.DataParallel(model).cuda()

    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0000001)

    if args.arch == 'c3d':
        lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 20, gamma=0.5)
    else:
        lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [50, 100], gamma=0.1)

    if args.resume:
        if os.path.isfile(args.resume):
            print(("=> loading checkpoint '{}'".format(args.resume)))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch'] - 1
            best_recall = checkpoint['best_recall']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print(("=> loaded checkpoint (epoch {})"
                   .format(checkpoint['epoch'])))
        else:
            print(("=> no checkpoint found at '{}'".format(args.resume)))

    train_transforms = transforms.Compose(
        [videotransforms.RandomCrop(args.input_size), videotransforms.RandomHorizontalFlip()]
    )
    test_transforms = transforms.Compose(
        [videotransforms.CenterCrop(args.input_size)]
    )

    dataset = Dataset(args.root, args.train_split, args.task, args.view, 'train', transforms=train_transforms)
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True
    )
    val_dataset = Dataset(args.root, args.val_split, args.task, args.view, 'val', transforms=test_transforms)
    val_dataloader = torch.utils.data.DataLoader(
        val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True
    )
    criterion = torch.nn.CrossEntropyLoss().cuda()
    ske_criterion = torch.nn.CrossEntropyLoss().cuda()
    epoch_time = AverageMeter()
    for epoch in range(args.start_epoch, args.epochs + 1):
        epoch_start = time.time()
        logger.info('Epoch {}/{}'.format(epoch, args.epochs))
        logger.info('-' * 10)
        train(model, dataloader, optimizer, criterion, ske_criterion, epoch)
        if epoch % 5 == 0 or epoch == 1:
            acc = val(model, val_dataloader, criterion, ske_criterion, epoch)
            is_best = acc > best_recall
            best_recall = max(acc, best_recall)
            logger.info(
                'epoch time: {:.3f}, {} epoch Val Recall: {:.4f} Best Val Recall: {:.4f}'.format(epoch_time.avg, epoch,
                                                                                acc, best_recall))
            save_checkpoint({
                'epoch': epoch + 1,
                'arch': args.arch,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(),
                'best_recall': best_recall,
            }, is_best)
        lr_scheduler.step()
        epoch_time.update(time.time() - epoch_start)

def train(model, train_loader, optimizer, criterion, ske_criterion, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    rgb_losses = AverageMeter()
    ske_losses = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()
    model.train()
    optimizer.zero_grad()
    end = time.time()
    for i, (input, target, ske_joint, boxes) in enumerate(train_loader):
        data_time.update(time.time() - end)
        input = input.cuda()
        target = target.cuda()
        boxes = boxes.cuda()
        output = model(input, ske_joint, boxes)
        if type(output) is tuple:
            rgb_loss = criterion(output[0], target)
            ske_loss = ske_criterion(output[1], target)
            rgb_losses.update(rgb_loss.item(), input.size(0))
            ske_losses.update(ske_loss.item(), input.size(0))
            loss = rgb_loss + ske_loss
            # measure accuracy and record loss
            prec1, prec5 = accuracy(output[0].data, target, topk=(1, 5))
        else:
            loss = criterion(output, target)

            # measure accuracy and record loss
            prec1, prec5 = accuracy(output.data, target, topk=(1, 5))

        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))
        loss.backward()

        optimizer.step()
        optimizer.zero_grad()
        batch_time.update(time.time() - end)
        writer.add_scalar('train_data/loss', losses.val, epoch * len(train_loader) + i + 1)
        end = time.time()
        iteration = epoch * len(train_loader) + i
        if i % 10 == 0:
            writer.add_scalar('train_loss', losses.avg, iteration)
            output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                      'Ske_Loss {ske_loss.val:.4f} ({ske_loss.avg:.4f})\t'
                      'RGB_Loss {rgb_loss.val:.4f} ({rgb_loss.avg:.4f})\t'
                      'Total_Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                      'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, ske_loss=ske_losses, rgb_loss=rgb_losses, top1=top1, top5=top5,
                lr=optimizer.param_groups[-1]['lr']))  # TODO
            logger.info(output + '\n')
            batch_time.reset()
            data_time.reset()

@torch.no_grad()
def val(model, val_loader, criterion, ske_criterion, epoch):
    batch_time = AverageMeter()
    rgb_losses = AverageMeter()
    ske_losses = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()
    data_time = AverageMeter()
    CM = metric.ConfusionMatrix(34)
    model.eval()
    end = time.time()
    for i, (input, target, ske_joint, boxes) in enumerate(val_loader):
        input = input.cuda()
        target = target.cuda()
        boxes = boxes.cuda()
        data_time.update(time.time() - end)
        output = model(input, ske_joint, boxes)
        if type(output) is tuple:
            rgb_loss = criterion(output[0], target)
            ske_loss = ske_criterion(output[1], target)
            rgb_losses.update(rgb_loss.item(), input.size(0))
            ske_losses.update(ske_loss.item(), input.size(0))
            loss = rgb_loss + ske_loss
            # measure accuracy and record loss
            prec1, prec5 = accuracy(output[0].data, target, topk=(1, 5))
            output = output[0]
        else:
            loss = criterion(output, target)
            # measure accuracy and record loss
            prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))

        CM.update(target, output)
        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()
        writer.add_scalar('val_data/loss', losses.val, epoch * len(val_loader) + i + 1)
        if i % 10 == 0:
            output = ('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                      'Ske_Loss {ske_loss.val:.4f} ({ske_loss.avg:.4f})\t'
                      'RGB_Loss {rgb_loss.val:.4f} ({rgb_loss.avg:.4f})\t'
                      'Total_Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                      'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                i, len(val_loader), batch_time=batch_time, data_time=data_time, loss=losses, ske_loss=ske_losses,
                rgb_loss=rgb_losses,
                top1=top1, top5=top5))
            if logger is not None:
                logger.info(output + '\n')
    recall = 0.0
    for p in CM.recall():
        recall += p
    recall_avg = recall / (args.num_class)

    if writer is not None:
        writer.add_scalar('loss/test', losses.avg, epoch)
        writer.add_scalar('acc/test_top1', top1.avg, epoch)
        writer.add_scalar('acc/test_top5', top5.avg, epoch)
    return recall_avg

def check_rootfolders():
    """Create log and model folder"""
    folders_util = [args.root_log, args.root_model]
    if os.path.exists(args.root_model) and args.root_model != 'exp/test':
        print('The directory already exists, if you want to cover it?[y/n]')
        is_cover = input()
        assert is_cover.upper() not in ['N', 'NOT'], 'need to select a new files path!'

    for folder in folders_util:
        if not os.path.exists(folder):
            print('creating folder ' + folder)
            os.makedirs(folder)

def save_checkpoint(state, is_best):
    dir = args.root_model
    if not os.path.isdir(dir):
        os.makedirs(dir)
    filename = os.path.join(dir, "checkpoint.pth")
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, filename.replace('pth', 'best.pth'))

if __name__ == '__main__':
    main()

max_stars_repo_name: SCUT-AILab/BPAI-Net

max_issues_repo_name: SCUT-AILab/BPAI-Net

max_forks_repo_name: SCUT-AILab/BPAI-Net


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# OpenData DWD FX Radar processing tool
# Author: Akio Hansen
# v0: First test version to read in FX binary data and create first static plot
# v1: Included automatic download of latest image and further improvements
# v2: Splittet program into single functions for better code structure
# --> not yet finished, has to be done

# Load necessary python modules
import numpy as np
from datetime import datetime
from sys import argv

# Define user inputs
temp_folder = '/work/bm0834/u233139/DWDRadar/temp/'               # Static link for debug options
#infile_fx  = '/work/bm0834/u233139/DWDRadar/FXLATEST_000_MF002'  # Static link for debug options
#infile_fx   = temp_folder+"FXLATEST_000_MF002"                    # Latest image update option
infile_fx  = sys.argv[1]                                         # Option for argument input
plot_path  = sys.argv[2]                                         # Option for argument input
#plot_path   = '/work/bm0834/u233139/DWDRadar/plots/'              # Static link for debug options

# Define DWD FX Radar download function
def download_dwd_radar(latest):
    "This function downloads latest DWD radar file."

    ## Download latest radar image to temporary directory
    # Check if temporary folder exists otherwise create one
    import os
    import urllib
    if not os.path.exists(temp_folder):
        os.makedirs(temp_folder)

    # Download file
    if latest == 1:
        latest_fx_file = urllib.URLopener()
        latest_fx_file.retrieve("https://opendata.dwd.de/weather/radar/composit/fx/FXLATEST_000_MF002", temp_folder+"FXLATEST_000_MF002")
    else:
        print "Error, no other function implemented yet."
        return 1

    # If everything runs without errors, return 0
    return 0

# Define DWD FX Radar processing function
def process_dwd_radar(infile_fx):
    "This function process DWD radar file."

    ### Read and process DWD FX data ###
    ## Documentation about DWD OpenData FX format is found at:
    ## https://www.dwd.de/DE/leistungen/radarprodukte/formatbeschreibung_fxdaten.pdf

    ### Read header information and binary data
    with open(infile_fx, 'r') as f:
        f.seek(0, 0)                                # Set reading cursor to start of file
        file_data   = f.read()                      # Read all lines and store to object
        h_line      = file_data.split('\x03', 1)[0] # Read header information till first ETX
        fx_bin_data = file_data.split('\x03', 1)[1] # Binary data is stored after header ETX

    # Process header information and save variables
    prod_type = h_line[0:2]           # Product type (FX)
    meas_time = h_line[2:8]           # Measurement time UTC (DDhhmm)
    meas_date = h_line[13:17]         # Measurement date (MMYY)
    byte_len  = np.int(h_line[19:26]) # Product length in bytes
    intv_min  = np.int(h_line[51:55]) # Intervall length in minutes
    dim_x     = np.int(h_line[57:61]) # Number of Pixels (x-direction)
    dim_y     = np.int(h_line[62:66]) # Number of Pixels (y-direction)

    # Convert header time information to python time object
    datetime_meas = datetime.strptime(meas_date+'-'+meas_time, '%m%y-%d%H%M')

    # Read binary data from buffer variable and clean buffer
    fx_data = np.frombuffer(fx_bin_data, dtype='<i2').copy()
    del file_data

    # Set missing values according to file format specs.
    fx_data[fx_data > 4095] = 0
    # Data handling for optimized postprocessing
    fx_data = np.asfarray(fx_data)          # Convert data to float
    fx_data = fx_data.reshape(dim_x, dim_y) # Reshape vector to array
    # Convert pixel to dBz values
    fx_data = fx_data/10.0
    fx_data = (fx_data/2.0) - 32.5
    # Set all values below to NaN, non physical values
    fx_data[fx_data < 0.0] = np.nan

    # If everything runs without errors, return dictonary with variables
    return {'dim_x': dim_x, 'dim_y': dim_y, 'datetime_meas': datetime_meas, 'fx_data': fx_data}

# Define DWD FX Radar radar grid function
def create_radar_grid(dim_x, dim_y):
    "This function downloads latest DWD radar file."

    ### Geolocation of pixels
    # Creating array for positions
    grid_pos    = np.empty((dim_x, dim_y,))
    grid_pos[:] = np.nan

    # Reference for Coordinates Calculation see:
    # https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf?__blob=publicationFile&v=7
    #  page 13 and following

    # Set constants
    earthRadius = 6370.04 # km

    junctionNorth = 60.0 # N
    junctionEast  = 10.0 # E

    # Equidistant cartesian coordinates
    dx_dist = 1.0 # km
    dy_dist = 1.0 # km

    # Coordinates of corners of cartesian grid
    lowerleft_x = -523.4622
    lowerleft_y = -4658.645

    lowerright_x = 376.5378
    lowerright_y = -4658.645

    upperright_x = 376.5378
    upperright_y = -3758.645

    upperleft_x = -523.4622
    upperleft_y = -3758.645

    # Create x and y equidistant vectors
    x_cord_cart = np.arange(lowerleft_x,lowerright_x,dx_dist)
    y_cord_cart = np.arange(lowerleft_y,upperleft_y,dy_dist)

    # Convert cartesian coordinates to lat lon coordinates
    x_vec_cord = np.rad2deg( np.arctan(-x_cord_cart[0:900]/y_cord_cart[0]) + np.deg2rad( junctionEast ) ) # see p.13, formula 1.4a

    y_vec_cord = np.rad2deg( np.arcsin(( \
        ((earthRadius**2) * ((1+np.sin(np.deg2rad(junctionNorth)))**2) - ( (x_cord_cart[0:900]**2) + (y_cord_cart[0:900]**2))) / \
        ((earthRadius**2) * ((1+np.sin(np.deg2rad(junctionNorth)))**2) + ( (x_cord_cart[0:900]**2) + (y_cord_cart[0:900]**2))) )) )

    # Create meshgrid to have coordinates matrix
    xm_ll, ym_ll = np.meshgrid(x_vec_cord, y_vec_cord)

    # If everything runs without errors, return dictonary with grid variables
    return {'xm_ll': xm_ll, 'ym_ll': ym_ll}

# Define DWD FX Radar plot function function
def plot_dwd_radar(xm_ll, ym_ll, fx_data, datetime_meas, plot_path, temp_folder):
    "This function gets DWD radar and grid information and creates a plot."

    # Import map data for plotting
    ### Plot DWD Open Radardata on a map
    import matplotlib.pyplot as plt
    from mpl_toolkits.basemap import Basemap

    # Create formatted time string
    time_string = datetime.strftime(datetime_meas, '%d/%m/%Y %H:%M UTC')
    time_file   = datetime.strftime(datetime_meas, '%d%m%Y_%H%M')

    # Create new figure
    fig = plt.figure(num=None, facecolor='w', edgecolor='k')

    plt.contourf(xm_ll, ym_ll, fx_data, interpolation='none')
    plt.title("Rain at: "+time_string, fontweight="bold", size=18)

    map = Basemap(llcrnrlon=3.5,llcrnrlat=46.5,urcrnrlon=15.5,urcrnrlat=55.5,
                 resolution='i')
    map.drawcoastlines()
    map.drawcountries()
    lat_ticks = [55.0,53.0,51.0,49.0,47.0]
    map.drawparallels(lat_ticks, labels=[1,0,0,0], linewidth=0.0)
    lon_ticks = [4.0,6.0,8.0,10.0,12.0,14.0]
    map.drawmeridians(lon_ticks, labels=[0,0,0,1], linewidth=0.0)

    # Insert colorbar
    cbar = plt.colorbar()
    cbar.set_label('Radar reflectivity (dBz)')

    # Save plot to file
    plt.savefig(plot_path + 'DWD_OpenData_FX_5min_precip_'+ time_file + '.png', dpi=100, format='png')
    plt.close()
    #plt.show()

    # Clean temporary directory
    import os
    os.remove(temp_folder+"FXLATEST_000_MF002")

    # If everything runs without errors, return 0
    return 0

# Main control function
def main():
    # Define parameters
    print "Starting DWD OpenData Radar FX plot generation..."

    # Download latest Radar file
    radar_download = download_dwd_radar(1) # 1 = latest radar image

    # Process Radar file
    radar_process  = process_dwd_radar(infile_fx)

    # Generate DWD RADOLAN grid
    #print(radar_process["dim_x"], radar_process["dim_y"]) # Debug option for grid size
    radar_grid = create_radar_grid(radar_process["dim_x"], radar_process["dim_y"])

    # Generate latest DWD FX Radar plot
    radar_plot = plot_dwd_radar(radar_grid["xm_ll"], radar_grid["ym_ll"], radar_process["fx_data"], \
                    radar_process["datetime_meas"], plot_path, temp_folder)

    # End of main

# Call main routine
main()
print "Finished processing DWD OpenData for radar."

max_stars_repo_name: akiohansen/DWDOpenData

max_issues_repo_name: akiohansen/DWDOpenData

max_forks_repo_name: akiohansen/DWDOpenData


text:
# -*- encoding: utf-8 -*-
import time
import argparse

import numpy as np

from benchmark import bench_suite

def num_to_str(num_array, digits):
    res_list = []
    for i in num_array:
        template = '%0' + str(digits) + 'd'
        res_list.append(template % i)
    return np.array(res_list)

def run(bs, seed, size, loops):
    np.random.seed(seed)
    print(f'Array size: {size}. Loop num: {loops}')
    for name, func in bs.funcs.items():
        base_time = None
        print(f'Testing {name} array:')
        for kind in bs.sort_kinds:
            times = []
            for i in range(loops):
                arr, answer = func(size)
                str_arr = num_to_str(arr, int(np.log10(size)) + 1)
                time1 = time.time()
                res = str_arr.argsort(kind=kind)
                time2 = time.time()
                times.append(time2 - time1)
                assert np.allclose(np.int64(str_arr[res]), answer)
            times = np.array(times) * 1e6
            mean, std = times.mean(), times.std()
            if base_time is None:
                base_time = times.mean()
            print(f'    {kind}: {mean:.3f}±{std:.3f} us per loop. Relative: {mean/base_time*100:.0f}%')
        print()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--random_seed', type=int, default=123)
    parser.add_argument('--array_size', type=int, default=5000)
    parser.add_argument('--loops', type=int, default=50)
    args = parser.parse_args()
    run(bench_suite, args.random_seed, args.array_size, args.loops)


text:
""" 함수의 미분2

도함수의 오차와 검증
"""
import numpy as np
import matplotlib.pyplot as plt

# 데이터 간격 h
h = 1.0

def f(x):
    return np.sin(x)

def g(func, xmin, xmax, h=0.01):
    x = np.arange(xmin, xmax, h)
    y = []
    y0 = func(xmin)
    for _x in x:
        y0 += func(_x + h) - func(_x)
        y.append(y0)
    return x, np.array(y)

if __name__ == "__main__":
    x = np.linspace(0, np.pi * 10, 100)
    plt.plot(x, f(x), "k--")
    plt.plot(*g(f, 0, np.pi * 10, h), "r")
    plt.grid(True)
    plt.show()


text:
import numpy as np
from math import sin, cos, radians

def RotZ(theta):
  c = cos(theta)
  s = sin(theta)
  return np.array([
    [c, -s, 0, 0],
    [s, c, 0, 0],
    [0, 0, 1, 0],
    [0, 0, 0, 1]]).astype(np.float32)

def RotY(theta):
  c = cos(theta)
  s = sin(theta)
  return np.array([
    [c, 0, s, 0],
    [0, 1, 0, 0],
    [-s, 0, c, 0],
    [0, 0, 0, 1]]).astype(np.float32)

def RotX(theta):
  c = cos(theta)
  s = sin(theta)
  return np.array([
    [1, 0, 0, 0],
    [0, c, -s, 0],
    [0, s, c, 0],
    [0, 0, 0, 1]]).astype(np.float32)

def Translate(x, y, z):
  return np.array([
    [1, 0, 0, x],
    [0, 1, 0, y],
    [0, 0, 1, z],
    [0, 0, 0, 1]]).astype(np.float32)

def forwardKinematics(lengths, angles):
  """
  Given a series of lengths and angles, calculate the endpoints with the
  following joint configurations:
    angle 1: z
    angle 2: x
    angle 3: x
    angle 4: x
    angle 5: z
    angle 6: x
    angle 7: doesn't matter
  Arguments:
  - lengths: the lengths of each of the links
  - angles: the angles of each of the joints in degrees
  Return:
  - a 7x3 numpy array containing all the positions
  """
  # convert all the angles into radians
  angles = [radians(theta) for theta in angles]
  transformations = [
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[0]), RotZ(angles[0])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[1]), RotX(angles[1])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[2]), RotX(angles[2])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[3]), RotX(angles[3])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[4]), RotZ(angles[4])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[5]), RotX(angles[5])), v),
      lambda v: np.dot(np.dot(Translate(0, 0, lengths[6]), RotY(angles[6])), v)]
  positions = []
  for i in range(0, len(transformations)):
    positions = [np.array([0, 0, 0, 1])] + positions
    ind = len(transformations) - i - 1
    for j, pos in enumerate(positions):
      positions[j] = transformations[ind](pos)
  return np.array(positions)[:,:3]

max_stars_repo_name: timrobot/ArmRL

max_issues_repo_name: timrobot/ArmRL

max_forks_repo_name: timrobot/ArmRL


text:
import unittest

import numpy as np
from prediction.fixtures import generate_mock_training_data, simple_mock_data
from test_utils import (PLACES_PRECISION, matrices_almost_equal,
                        sequences_almost_equal, ensure_str)

from linkedin.learner.ds.indexed_model import IndexedModel
from linkedin.learner.prediction.evaluator import evaluate
from linkedin.learner.prediction.hessian_type import HessianType
from linkedin.learner.prediction.linear_scorer import score_linear_model
from linkedin.learner.prediction.trainer_sequential_bayesian_logistic_loss_with_l2 import \
    TrainerSequentialBayesianLogisticLossWithL2

class TrainerSequentialBayesianLogisticLossWithL2Test(unittest.TestCase):
    def test_incremental_lr_loss_and_gradient(self):
        """Test the loss and gradient functions.

        The expected values in this test are set using this implementation, so this
        is a test against regression, rather than strictly a test of correctness.
        """
        indexed_data, _ = generate_mock_training_data()

        theta_0 = np.array([4, 100, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
        initial_model = IndexedModel(theta=theta_0)
        #  Discrepancies with offline: (1) Labels are 0/1; in reality the data processing will set it to -1/+1.
        #  (2) Intercept gets regularized nearline but not offline, (3) This test should ideally only set initial_theta
        #  without changing hist_theta, but current implementation does not allow that.

        lr = TrainerSequentialBayesianLogisticLossWithL2(training_data=indexed_data, initial_model=initial_model, penalty=10, delta=0.8)
        initial_loss = lr.loss(theta_0)

        self.assertAlmostEqual(initial_loss, 112946.61089443817, PLACES_PRECISION, "Expect loss is calculated correctly.")

        initial_gradient = lr.gradient(theta_0)

        self.assertTrue(
            sequences_almost_equal(initial_gradient, [1006.0, 1198.0, 871.0, 871.0, 1000.0, 1000.0, 126.0, 126.0, 7.0, 7.0]),
            "Expect gradient is calculated correctly.",
        )

        final_model, final_loss, metadata = lr.train()

        self.assertTrue(
            sequences_almost_equal(
                final_model.theta,
                [
                    -13.37418015,
                    63.42582255,
                    -7.97896804,
                    -7.97896804,
                    -15.77418024,
                    -15.77418024,
                    -6.49519736,
                    -6.49519736,
                    0.29998522,
                    0.29998522,
                ],
            ),
            "Expect theta after optimization is correct.",
        )
        self.assertAlmostEqual(final_loss, 15104.873256903807, PLACES_PRECISION, "Expect lost after optimization is correct.")
        self.assertEqual(metadata["warnflag"], 0, "Expect convergence status is 0, meaning successful convergence.")
        self.assertEqual(metadata["nit"], 20, "Expect number of iterations is correct.")
        self.assertEqual(metadata["funcalls"], 28, "Expect number of function calls is correct.")
        self.assertEqual(ensure_str(metadata["task"]), "CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH", f"Expect convergence reason is as expected")

        # Score (on the training data)
        initial_scores = score_linear_model(model=initial_model, test_data=indexed_data)
        initial_auc = evaluate(metric_list=["auc"], y_scores=initial_scores, y_targets=indexed_data.y)["auc"]
        final_scores = score_linear_model(model=final_model, test_data=indexed_data)
        final_auc = evaluate(metric_list=["auc"], y_scores=final_scores, y_targets=indexed_data.y)["auc"]

        expected_initial_auc = 0.5290581162324649
        expected_final_auc = 0.7059118236472945

        self.assertAlmostEqual(
            initial_auc, expected_initial_auc, PLACES_PRECISION, f"Initial AUC: expected {expected_initial_auc} == actual {initial_auc}"
        )
        self.assertAlmostEqual(
            final_auc, expected_final_auc, PLACES_PRECISION, f"Final AUC: expected {expected_final_auc} == actual {final_auc}"
        )

    def test_incremental_lr_update_hessian(self):
        """Test the Hessian update when using a full Hessian."""
        indexed_data, model = simple_mock_data()
        lr = TrainerSequentialBayesianLogisticLossWithL2(
            training_data=indexed_data, initial_model=model, hessian_type=HessianType.FULL, penalty=10, delta=1.0
        )
        hessian = lr._update_hessian(model.theta)
        expected_hessian = np.array(
            [[1.076006603, 0.007826689, 0.167666585], [0.007826689, 1.003913344, 0.023480067], [0.167666585, 0.023480067, 1.382293306]]
        )
        self.assertTrue(
            matrices_almost_equal(hessian, expected_hessian),
            f"(Full) Hessian computation is correct. Actual {hessian} == Expected {expected_hessian}.",
        )

    def test_incremental_lr_update_hessian_diagonal(self):
        """Test the Hessian update when using a diagonal Hessian."""
        indexed_data, model = simple_mock_data()
        lr = TrainerSequentialBayesianLogisticLossWithL2(
            training_data=indexed_data, initial_model=model, hessian_type=HessianType.DIAGONAL, penalty=10, delta=1.0
        )
        hessian = lr._update_hessian(model.theta).toarray()
        expected_hessian = np.diag([1.076006603, 1.003913344, 1.382293306])
        self.assertTrue(
            matrices_almost_equal(hessian, expected_hessian),
            f"(Diagonal) Hessian computation is correct. Actual {hessian} == Expected {expected_hessian}.",
        )

    def test_incremental_lr_update_hessian_identity(self):
        """Test the Hessian update when using an identity Hessian."""
        indexed_data, model = simple_mock_data()
        lr = TrainerSequentialBayesianLogisticLossWithL2(
            training_data=indexed_data, initial_model=model, hessian_type=HessianType.IDENTITY, penalty=10, delta=1.0
        )
        hessian = lr._update_hessian(model.theta).toarray()
        expected_hessian = np.diag([1.0, 1.0, 1.0])
        self.assertTrue(
            matrices_almost_equal(hessian, expected_hessian),
            f"Hessian computation is correct. Actual {hessian} == Expected {expected_hessian}.",
        )

    def test_hessian_computation_is_optional(self):
        """Test that the Hessian update is optional."""
        indexed_data, model = simple_mock_data()

        trainer = TrainerSequentialBayesianLogisticLossWithL2(
            training_data=indexed_data, initial_model=model, penalty=10, hessian_type=HessianType.NONE
        )
        final_model, _, _ = trainer.train(precision=1e4, num_corrections=7, max_iterations=50)

        self.assertTrue(final_model.hessian is None, "Hessian computation optional.")

max_stars_repo_name: linkedin/lambda-learner

max_issues_repo_name: linkedin/lambda-learner

max_forks_repo_name: linkedin/lambda-learner


text:
#
#  File:
#    xy2.py
#
#  Synopsis:
#    Illustrates how to add primitives and text to an XY plot.
#
#  Categories:
#    paneling
#    xy plots
#    polygons
#    polylines
#    polymarkers
#
#  Author:
#    Mary Haley
#
#  Date of initial publication:
#    January 2006
#
#  Description:
#    This example shows how to attach primitives and text to an XY
#    plot so that when the plot is resized and drawn, the primitives
#    and text are automatically resized and drawn.

#  Effects illustrated:
#    o  Attaching primitives to a plot.
#    o  Attaching text to a plot.
#    o  Panelling plots.
#
#  Output:
#     A single visualization with two plots is produced.
#
#  Notes:
#

#
#  Import NumPy.
#
from __future__ import print_function
import numpy

#
#  Import Ngl support functions.
#
import Ngl

PI = 3.14159
x = numpy.arange(1,65,1)       # X points for both XY plots
y = numpy.sin(PI*x/32.)        # Y points for first XY plot

#
# Set up arrays for polygons, polymarkers, and polylines.
#
xx = numpy.array([1.,32.,64.],'f')
ytopline = [ 0.5, 0.5, 0.5]     # Top line
ymidline = [ 0.0, 0.0, 0.0]     # Middle line
ybotline = [-0.5,-0.5,-0.5]     # Bottom line

xdots = x[26:37:5]     # X and Y coordinates for polymarkers.
ydots = y[26:37:5]

xsquare = [16.0,48.0,48.0,16.0,16.0]    # X and Y coordinates
ysquare = [-0.5,-0.5, 0.5, 0.5,-0.5]    # for polygon.

# Send output to PNG file
wks_type = "png"
wks = Ngl.open_wks(wks_type,"xy2")

#
# Set up common resources for the XY plots.
#
xyres          = Ngl.Resources()
xyres.nglDraw  = False
xyres.nglFrame = False
xyres.trXMaxF  = 64

# Create first XY plot, but don't draw it yet.

xy1 = Ngl.xy(wks,x,y,xyres)

#
# Set some resources for the primitives to be drawn.
#
gsres                   = Ngl.Resources()

# Polyline resources.

gsres.gsLineColor       = "Blue"
gsres.gsLineThicknessF  = 3.0      # thrice thickness

# Polygon resources.

gsres.gsFillColor       = "Yellow"
gsres.gsEdgesOn         = True    # Line will be drawn around polygon.

# Polymarker resources.

gsres.gsMarkerIndex     = 16       # dots
gsres.gsMarkerColor     = "HotPink"
gsres.gsMarkerSizeF     = 0.014    # twice normal size

#
# Add primitives to plot.  They will be drawn in the order
# they are added, so some primitives may cover other ones.
#
# Be sure to use unique variable names on the left-hand side
# for the Ngl.add_polyxxxx routines.
#
prim1 = Ngl.add_polyline(wks,xy1,xx,ymidline,gsres)
prim2 = Ngl.add_polygon(wks,xy1,xsquare,ysquare,gsres)
prim3 = Ngl.add_polyline(wks,xy1,xx,ytopline,gsres)
prim4 = Ngl.add_polyline(wks,xy1,xx,ybotline,gsres)

gsres.gsLineColor      = "Green"
prim5 = Ngl.add_polyline(wks,xy1,x[26:37],y[26:37],gsres)
prim6 = Ngl.add_polymarker(wks,xy1,xdots,ydots,gsres)

#
# New Y points for second XY plot.
#
y = numpy.cos(3.14159*x/32.)

# Create second XY plot, but don't draw it yet.

xy2 = Ngl.xy(wks,x,y,xyres)

#
# Add primitives to second plot.
#
gsres.gsLineColor      = "Blue"

prim7  = Ngl.add_polyline(wks,xy2,xx,ymidline,gsres)
prim8  = Ngl.add_polygon(wks,xy2,xsquare,ysquare,gsres)
prim9  = Ngl.add_polyline(wks,xy2,xx,ytopline,gsres)
prim10 = Ngl.add_polyline(wks,xy2,xx,ybotline,gsres)

gsres.gsLineColor      = "Green"

xdots2 = x[15:21:5]
ydots2 = y[15:21:5]
prim11 = Ngl.add_polyline(wks,xy2,x[15:21],y[15:21],gsres)
prim12 = Ngl.add_polymarker(wks,xy2,xdots2,ydots2,gsres)

xdots2 = x[42:48:5]
ydots2 = y[42:48:5]
prim13 = Ngl.add_polyline(wks,xy2,x[42:48],y[42:48],gsres)
prim14 = Ngl.add_polymarker(wks,xy2,xdots2,ydots2,gsres)

#
# Now add some text strings.
#
txres               = Ngl.Resources()
txres.txFont        = "helvetica-bold"
txres.txFontColor   = "coral4"
txres.txFontHeightF = 0.037
txres.txJust        = "BottomCenter"

text1  = Ngl.add_text(wks,xy1,"this is a text string",32.,0.5,txres)

#
# Slanted strings
#
txres.txJust    = "BottomRight"
txres.txAngleF  = -45.
text2b = Ngl.add_text(wks,xy2,"some slanted",32.,.5,txres)
txres.txJust    = "BottomLeft"
txres.txAngleF  =  45.
text2b = Ngl.add_text(wks,xy2,"text strings",32.,.5,txres)

#
# Filled text box.
#
txres.txAngleF              = 0.
txres.txJust                = "CenterLeft"
txres.txPerimOn             = True
txres.txFontColor           = "black"
txres.txBackgroundFillColor = "gray75"
text4  = Ngl.add_text(wks,xy1,"filled text box",3.,-1.0,txres)

#
# Now that our primitives have been added to both plots, panel them.
# Note that by adding the primitives, they will get drawn when the
# XY plots are drawn, and they will be automatically resized with
# the plot.
#
Ngl.panel(wks,[xy1,xy2],[2,1])

Ngl.end()


text:
from gtable.lib import merge_table, sort_table
from gtable import Table
import numpy as np

class TimeSuite:
    def setup(self):
        self.a = np.arange(10, dtype=np.double)
        self.b = np.arange(5, 15, dtype=np.double)
        self.table_a = Table({'a': self.a, 'b': self.a})
        self.table_b = Table({'b': self.b})

    def time_concatenate(self):
        np.concatenate((self.a, self.b))

    # def time_concatenate_fast(self):
    #     concatenate_float(self.a, self.b)

    def time_merge(self):
        merge_table(self.table_a, self.table_b, 'b')

    def time_sort(self):
        sort_table(self.table_a, 'a')

if __name__ == '__main__':
    import timeit
    import inspect
    import sys
    import os
    from statistics import mean, stdev
    t = TimeSuite()
    t.setup()
    print('Getting runtimes')

    for method in inspect.getmembers(t, predicate=inspect.ismethod):
        if method[0].startswith('time_'):
            stats = timeit.repeat(
                "t.{}()".format(method[0]),
                globals=globals(),
                number=100,
                repeat=10)

            mean_t = mean(stats[1:]) * 10
            stdev_t = stdev(stats[1:]) * 10
            sys.stdout.write(method[0] + ': ')
            sys.stdout.write(str(mean_t) + ' ± (std) ' +
                             str(stdev_t) + ' [ms]')
            sys.stdout.write(os.linesep)

max_stars_repo_name: guillemborrell/gtable

max_issues_repo_name: guillemborrell/gtable

max_forks_repo_name: guillemborrell/gtable


text:
#
# Copyright (c) 2021, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import hugectr
from hugectr.inference import InferenceParams, CreateInferenceSession
import hugectr2onnx
import onnxruntime as ort
from utils import read_samples_for_dcn, compare_array_approx
import numpy as np

def hugectr2onnx_dcn_test(batch_size, num_batches, data_source, data_file, graph_config, dense_model, sparse_models, onnx_model_path, model_name):
    hugectr2onnx.converter.convert(onnx_model_path, graph_config, dense_model, True, sparse_models)
    label, dense, keys = read_samples_for_dcn(data_file, batch_size*num_batches, slot_num = 26)
    sess = ort.InferenceSession(onnx_model_path)
    res = sess.run(output_names=[sess.get_outputs()[0].name],
                  input_feed={sess.get_inputs()[0].name: dense, sess.get_inputs()[1].name: keys})
    res = res[0].reshape(batch_size*num_batches,)

    inference_params = InferenceParams(model_name = model_name,
                                    max_batchsize = batch_size,
                                    hit_rate_threshold = 0.6,
                                    dense_model_file = dense_model,
                                    sparse_model_files = sparse_models,
                                    device_id = 0,
                                    use_gpu_embedding_cache = True,
                                    cache_size_percentage = 0.6,
                                    i64_input_key = False)
    inference_session = CreateInferenceSession(graph_config, inference_params)
    predictions = inference_session.predict(num_batches, data_source, hugectr.DataReaderType_t.Norm, hugectr.Check_t.Sum)
    compare_array_approx(res, predictions, model_name, 1e-3, 1e-2)

if __name__ == "__main__":
    hugectr2onnx_dcn_test(64, 100, "./dcn_data/file_list_test.txt",
                    "./dcn_data/val/sparse_embedding0.data",
                    "/onnx_converter/graph_files/dcn.json",
                    "/onnx_converter/hugectr_models/dcn_dense_2000.model",
                    ["/onnx_converter/hugectr_models/dcn0_sparse_2000.model"],
                    "/onnx_converter/onnx_models/dcn.onnx",
                    "dcn")
    hugectr2onnx_dcn_test(64, 100, "./dcn_data/file_list_test.txt",
                    "./dcn_data/val/sparse_embedding0.data",
                    "/onnx_converter/graph_files/deepfm.json",
                    "/onnx_converter/hugectr_models/deepfm_dense_2000.model",
                    ["/onnx_converter/hugectr_models/deepfm0_sparse_2000.model"],
                    "/onnx_converter/onnx_models/deepfm.onnx",
                    "deepfm")
    hugectr2onnx_dcn_test(64, 100, "./dcn_data/file_list_test.txt",
                    "./dcn_data/val/sparse_embedding0.data",
                    "/onnx_converter/graph_files/dlrm.json",
                    "/onnx_converter/hugectr_models/dlrm_dense_2000.model",
                    ["/onnx_converter/hugectr_models/dlrm0_sparse_2000.model"],
                    "/onnx_converter/onnx_models/dlrm.onnx",
                    "dlrm")

max_stars_repo_name: Chunshuizhao/HugeCTR

max_issues_repo_name: Chunshuizhao/HugeCTR

max_forks_repo_name: Chunshuizhao/HugeCTR


text:
# Author: Danilo Motta  -- <ddanilomotta@gmail.com>

# This is an implementation of the technique described in:
# Sparse multidimensional scaling using landmark points
# http://graphics.stanford.edu/courses/cs468-05-winter/Papers/Landmarks/Silva_landmarks5.pdf

import numpy as np
import scipy as sp

def MDS(D,dim=[]):
	# Number of points
	n = len(D)

	# Centering matrix
	H = - np.ones((n, n))/n
	np.fill_diagonal(H,1-1/n)
	# YY^T
	H = -H.dot(D**2).dot(H)/2

	# Diagonalize
	evals, evecs = np.linalg.eigh(H)

	# Sort by eigenvalue in descending order
	idx   = np.argsort(evals)[::-1]
	evals = evals[idx]
	evecs = evecs[:,idx]

	# Compute the coordinates using positive-eigenvalued components only
	w, = np.where(evals > 0)
	if dim!=[]:
		arr = evals
		w = arr.argsort()[-dim:][::-1]
		if np.any(evals[w]<0):
			print('Error: Not enough positive eigenvalues for the selected dim.')
			return []
	L = np.diag(np.sqrt(evals[w]))
	V = evecs[:,w]
	Y = V.dot(L)
	return Y

def landmark_MDS(D, lands, dim):
	Dl = D[:,lands]
	n = len(Dl)

	# Centering matrix
	H = - np.ones((n, n))/n
	np.fill_diagonal(H,1-1/n)
	# YY^T
	H = -H.dot(Dl**2).dot(H)/2

	# Diagonalize
	evals, evecs = np.linalg.eigh(H)

	# Sort by eigenvalue in descending order
	idx   = np.argsort(evals)[::-1]
	evals = evals[idx]
	evecs = evecs[:,idx]

	# Compute the coordinates using positive-eigenvalued components only
	w, = np.where(evals > 0)
	if dim:
		arr = evals
		w = arr.argsort()[-dim:][::-1]
		if np.any(evals[w]<0):
			print('Error: Not enough positive eigenvalues for the selected dim.')
			return []
	if w.size==0:
		print('Error: matrix is negative definite.')
		return []

	V = evecs[:,w]
	L = V.dot(np.diag(np.sqrt(evals[w]))).T
	N = D.shape[1]
	Lh = V.dot(np.diag(1./np.sqrt(evals[w]))).T
	Dm = D - np.tile(np.mean(Dl,axis=1),(N, 1)).T
	dim = w.size
	X = -Lh.dot(Dm)/2.
	X -= np.tile(np.mean(X,axis=1),(N, 1)).T

	_, evecs = sp.linalg.eigh(X.dot(X.T))

	return (evecs[:,::-1].T.dot(X)).T

max_stars_repo_name: danilomotta/LMDS

max_issues_repo_name: danilomotta/LMDS

max_forks_repo_name: danilomotta/LMDS

reason: import numpy,import scipy