text:
import tensorflow as tf
import numpy as np

import gpflow

from elfi.methods.bo.iwvi.layers import RegularizerType

class DGP_VI(gpflow.models.GPModel):
    def __init__(self, X, Y, layers, likelihood,
                 num_samples=1,
                 minibatch_size=None,
                 name=None):
        gpflow.Parameterized.__init__(self, name=name)

        self.likelihood = likelihood

        self.num_data = X.shape[0]
        self.num_samples = num_samples

        if minibatch_size is None:
            self.X = gpflow.params.DataHolder(X)
            self.Y = gpflow.params.DataHolder(Y)
        else:
            self.X = gpflow.params.Minibatch(X, batch_size=minibatch_size, seed=0)
            self.Y = gpflow.params.Minibatch(Y, batch_size=minibatch_size, seed=0)

        self.layers = gpflow.params.ParamList(layers)

    @gpflow.params_as_tensors
    def propagate(self, X, full_cov=False, inference_amorization_inputs=None, is_sampled_local_regularizer=False):

        samples, means, covs, kls, kl_types = [X, ], [], [], [], []

        for layer in self.layers:
            sample, mean, cov, kl = layer.propagate(samples[-1],
                                                    full_cov=full_cov,
                                                    inference_amorization_inputs=inference_amorization_inputs,
                                                    is_sampled_local_regularizer=is_sampled_local_regularizer)
            samples.append(sample)
            means.append(mean)
            covs.append(cov)
            kls.append(kl)
            kl_types.append(layer.regularizer_type)

        return samples[1:], means, covs, kls, kl_types

    @gpflow.params_as_tensors
    def _build_likelihood(self):
        X_tiled = tf.tile(self.X, [self.num_samples, 1])  # SN, Dx
        Y_tiled = tf.tile(self.Y, [self.num_samples, 1])  # SN, Dy

        XY = tf.concat([X_tiled, Y_tiled], -1)  # SN, Dx+Dy

        # Following Salimbeni 2017, the sampling is independent over N
        # The flag is_sampled_local_regularizer=False means that the KL is returned for the regularizer

        samples, means, covs, kls, kl_types = self.propagate(X_tiled,
                                                             full_cov=False,
                                                             inference_amorization_inputs=XY,
                                                             is_sampled_local_regularizer=False)

        local_kls = [kl for kl, t in zip(kls, kl_types) if t is RegularizerType.LOCAL]
        global_kls = [kl for kl, t in zip(kls, kl_types) if t is RegularizerType.GLOBAL]

        var_exp = self.likelihood.variational_expectations(means[-1], covs[-1], Y_tiled)  # SN, Dy

        # Product over the columns of Y
        L_SN = tf.reduce_sum(var_exp, -1)  # SN, Dy -> SN

        shape_S_N = [self.num_samples, tf.shape(self.X)[0]]
        L_S_N = tf.reshape(L_SN, shape_S_N)

        if len(local_kls) > 0:
            local_kls_SN_D = tf.concat(local_kls, -1)  # SN, sum(W_dims)
            local_kls_SN = tf.reduce_sum(local_kls_SN_D, -1)
            local_kls_S_N = tf.reshape(local_kls_SN, shape_S_N)
            L_S_N -= local_kls_S_N  # SN

        scale = tf.cast(self.num_data, gpflow.settings.float_type)\
                / tf.cast(tf.shape(self.X)[0], gpflow.settings.float_type)

        # This line is replaced with tf.reduce_logsumexp in the IW case
        logp = tf.reduce_mean(L_S_N, 0)

        return tf.reduce_sum(logp) * scale - tf.reduce_sum(global_kls)

    @gpflow.params_as_tensors
    def _build_predict(self, X, full_cov=False):
        fs, means, covs, _, _ = self.propagate(X, full_cov=full_cov)
        return fs[-1], means[-1], covs[-1]

    @gpflow.params_as_tensors
    def _build_predict_decomp(self, X, full_cov=False):
        fs, means, covs, _, _ = self.propagate(X, full_cov=full_cov)
        return fs[-1], means[-1], covs

    @gpflow.params_as_tensors
    @gpflow.autoflow((gpflow.settings.float_type, [None, None]), (gpflow.settings.int_type, ()))
    def predict_f_multisample(self, X, S):
        X_tiled = tf.tile(X[None, :, :], [S, 1, 1])
        _, means, covs, _, _ = self.propagate(X_tiled)
        return means[-1], covs[-1]

    @gpflow.params_as_tensors
    @gpflow.autoflow((gpflow.settings.float_type, [None, None]), (gpflow.settings.int_type, ()))
    def predict_y_samples(self, X, S):
        X_tiled = tf.tile(X[None, :, :], [S, 1, 1])
        _, means, covs, _, _ = self.propagate(X_tiled)
        m, v = self.likelihood.predict_mean_and_var(means[-1], covs[-1])
        z = tf.random_normal(tf.shape(means[-1]), dtype=gpflow.settings.float_type)

        '''res_mean = list()
        res_var = list()
        for i in range(0, len(covs)):
            m1, v1 = self.likelihood.predict_mean_and_var(means[i], covs[i])
            res_mean.append(m1)
            res_var.append(v1)'''

        return m + z * v**0.5

class DGP_IWVI(DGP_VI):
    @gpflow.params_as_tensors
    def _build_likelihood(self):
        X_tiled = tf.tile(self.X[:, None, :], [1, self.num_samples, 1])  # N, S, Dx
        Y_tiled = tf.tile(self.Y[:, None, :], [1, self.num_samples, 1])  # N, S, Dy

        XY = tf.concat([X_tiled, Y_tiled], -1)  # N, S, Dx+Dy

        # While the sampling independent over N follows just as in Salimbeni 2017, in this
        # case we need to take full cov samples over the multisample dim S.
        # The flag is_sampled_local_regularizer=True means that the log p/q is returned
        # for the regularizer, rather than the KL
        samples, means, covs, kls, kl_types = self.propagate(X_tiled,
                                                             full_cov=True,  # NB the full_cov is over the S dim
                                                             inference_amorization_inputs=XY,
                                                             is_sampled_local_regularizer=True)

        local_kls = [kl for kl, t in zip(kls, kl_types) if t is RegularizerType.LOCAL]
        global_kls = [kl for kl, t in zip(kls, kl_types) if t is RegularizerType.GLOBAL]

        # This could be made slightly more efficient by making the last layer full_cov=False,
        # but this seems a small price to pay for cleaner code. NB this is only a SxS matrix, not
        # an NxN matrix.
        cov_diag = tf.transpose(tf.matrix_diag_part(covs[-1]), [0, 2, 1])  # N,Dy,K,K -> N,K,Dy
        var_exp = self.likelihood.variational_expectations(means[-1], cov_diag, Y_tiled)  # N, K, Dy

        # Product over the columns of Y
        L_NK = tf.reduce_sum(var_exp, 2)  # N, K, Dy -> N, K

        if len(local_kls) > 0:
            local_kls_NKD = tf.concat(local_kls, -1)  # N, K, sum(W_dims)
            L_NK -= tf.reduce_sum(local_kls_NKD, 2)  # N, K

        scale = tf.cast(self.num_data, gpflow.settings.float_type) \
                / tf.cast(tf.shape(self.X)[0], gpflow.settings.float_type)

        # This is reduce_mean in the VI case.
        logp = tf.reduce_logsumexp(L_NK, 1) - np.log(self.num_samples)

        return tf.reduce_sum(logp) * scale - tf.reduce_sum(global_kls)

ext: py

max_stars_repo_name: alexaushev/LFIwithDGPs

max_issues_repo_name: alexaushev/LFIwithDGPs

max_forks_repo_name: alexaushev/LFIwithDGPs

reason: import numpy


text:
import os
from sysconfig import get_paths
from setuptools import Extension, setup, find_packages
from setuptools.command.build_ext import build_ext

from msitrees import __version__

def get_long_desc():
    with open('README.md', 'r') as file:
        description = file.read()
    return description

def include_numpy():
    """Get path to numpy headers after setup ensured install"""
    import numpy as np
    return np.get_include()

def include_pybind():
    """Get path to pybind headers after setup ensured install"""
    import pybind11
    return pybind11.get_include()

def get_ext_modules():
    """Get external modules to compile"""
    ext = Extension(
        name='msitrees._core',
        sources=[os.path.join('msitrees', 'core', 'core.cpp')],
        include_dirs=[
            get_paths()['include'],
            include_numpy(),
            include_pybind(),
            os.path.join(os.getcwd(), 'msitrees', 'core', 'include')
        ],
        language='c++',
        # extra flags for linux dist
        # https://github.com/xtensor-stack/xtensor/issues/1704
        # on windows -ffast-math is enabled by default with
        # equivalent /fp:precise flag
        extra_link_args=['-lstdc++'],
        extra_compile_args=['-std=c++14', '-ffast-math', '-mavx2']
    )
    return [ext]

setup(
    name='msitrees',
    version=__version__,
    author='xadrianzetx',
    url='https://github.com/xadrianzetx/msitrees',
    description='MSI based machine learning algorithms',
    long_description=get_long_desc(),
    long_description_content_type='text/markdown',
    project_urls={
        'Documentation': 'https://msitrees.readthedocs.io/en/latest/index.html',
        'Source Code': 'https://github.com/xadrianzetx/msitrees/tree/master'
    },
    packages=find_packages(include=['msitrees']),
    ext_modules=get_ext_modules(),
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Operating System :: Microsoft :: Windows',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: C++',
        'Programming Language :: Python :: 3',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Scientific/Engineering :: Mathematics',
        'Topic :: Software Development :: Libraries'
    ],
    python_requires='>=3.5',
    install_requires=[
        'numpy>=1.18',
        'pandas>=1.0.0',
        'joblib>=0.14.0'
    ],
    cmdclass={'build_ext': build_ext},
    zip_safe=False
)

max_stars_repo_name: xadrianzetx/msitrees

max_issues_repo_name: xadrianzetx/msitrees

max_forks_repo_name: xadrianzetx/msitrees


text:
import os
import random
import datetime
import numpy as np
import pandas as pd
from gensim.models import word2vec
from keras import backend, optimizers, initializers
from keras.utils import to_categorical
from keras.models import Model, Sequential
from keras.layers import AveragePooling1D, Conv1D, concatenate, Dense, Dropout, Embedding, Flatten, GRU, Input, MaxPooling1D
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

E = ['AllTogether', 'Baseball', 'Boy-Girl', 'CVS', 'C_chat', 'GameSale', 'GetMarry', 'Lifeismoney', 'LoL', 'MH', 'MLB', 'Mobilecomm', 'movie', 'MuscleBeach', 'NBA', 'SENIORHIGH', 'Stock', 'Tennis', 'Tos', 'WomenTalk']

def loss(y_true, y_pred):
	return backend.mean(backend.abs(y_true - y_pred))

def build_w2v():
	sentences = [open('./Testing/{index}.txt'.format(index=str(index)), 'r', encoding='utf-8').read().strip().split(' / ') for index in range(1000)]
	for i in E:
		for j in os.listdir('./Training/{key}_cut/'.format(key=i)):
			sentences.append(open("./Training/{key}_cut/{file}".format(key=i, file=j), 'r', encoding='utf-8').read().split(' / '))
	random.shuffle(sentences)
	for min_count in range(2, 3): # <--- min_count
		word2Vec = word2vec.Word2Vec(sentences, min_count=min_count, size=270, iter=10, sg=1, workers=10)
		word2Vec.save("word2vec{count}.model".format(count=min_count))

if __name__ == '__main__':
	# build_w2v()

	max_length = 300
	now = datetime.datetime.now()
	word2Vec = word2vec.Word2Vec.load("word2vec2.model")

	train_list = []
	for category, i in enumerate(E):
		category_index = open("./Training/{key}.txt".format(key=i), 'r', encoding='utf-8').read().split('\n')
		for j in range(1800):
			train_word = open("./Training/{key}_cut/{key}_cut{index}.txt".format(key=i, index=str(j)), 'r', encoding='utf-8').read().strip().split(' / ')
			train_up, train_down = category_index[j].split('推')[-1].split('噓')
			train_list.append([category, train_word, train_up, train_down])
	test_list = [[open('./Testing/{index}.txt'.format(index=str(index)), 'r', encoding='utf-8').read().strip().split(' / ')] for index in range(4000)]

	train_df = pd.DataFrame(train_list, columns=["category", "text", "good", "bad"]).sample(frac=1)
	test_df = pd.DataFrame(test_list, columns=["text"])

	tokenizer = Tokenizer()
	tokenizer.fit_on_texts(train_df['text'].append(test_df['text']))

	train_x = tokenizer.texts_to_sequences(train_df['text'])
	train_x = pad_sequences(train_x, maxlen=max_length)

	train_y_category = to_categorical(np.asarray(list(train_df["category"])))
	train_y_good = train_df["good"]
	train_y_bad = train_df["bad"]

	test_x = tokenizer.texts_to_sequences(test_df['text'])
	test_x = pad_sequences(test_x, maxlen=max_length)

	word_index = tokenizer.word_index

	embedding_matrix = np.zeros((len(word_index)+1, word2Vec.vector_size))
	for word, i in word_index.items():
		try:
			embedding_matrix[i] = word2Vec[word]
		except:
			continue

	inputs = Input(shape=(max_length,))
	embedding = Embedding(name="embedding", input_dim=len(word_index)+1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs)

	category_gru = GRU(name="category_gru", units=16, kernel_initializer=initializers.glorot_uniform(seed=1), activation='tanh', dropout=.2, recurrent_dropout=.1)(embedding)
	category_dense1 = Dense(name="category_dense1", units=100, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(category_gru)
	category_dense2 = Dense(name="category_dense2", units=100, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(category_dense1)
	category_output = Dense(name="category", units=20, kernel_initializer=initializers.glorot_uniform(seed=1), activation='softmax')(category_dense2)

	good_1x1 = Conv1D(name="good_1x1", filters=64, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	good_3x3_reduce = Conv1D(name="good_3x3_reduce", filters=96, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	good_3x3 = Conv1D(name="good_3x3", filters=64, kernel_size=3, strides=3, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(good_3x3_reduce)
	good_5x5_reduce = Conv1D(name="good_5x5_reduce", filters=16, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	good_5x5 = Conv1D(name="good_5x5", filters=64, kernel_size=5, strides=5, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(good_5x5_reduce)
	good_pool = MaxPooling1D(name="good_pool", pool_size=3, strides=1, padding='same')(embedding)
	good_pool_proj = Conv1D(name="good_pool_proj", filters=64, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(good_pool)
	good_merge = concatenate(name="good_merge", inputs=[good_1x1, good_3x3, good_5x5, good_pool_proj], axis=1)
	good_ave_pool = AveragePooling1D(name="good_ave_pool", pool_size=5, strides=3)(good_merge)
	good_conv = Conv1D(name="good_conv", filters=128, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(good_ave_pool)
	good_flat = Flatten(name="good_flat")(good_conv)
	good_dense = Dense(name="good_dense", output_dim=256, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(good_flat)
	good_drop = Dropout(name="good_drop", rate=.2)(good_dense)
	good_output = Dense(name="good", output_dim=1, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(good_drop)

	bad_1x1 = Conv1D(name="bad_1x1", filters=64, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	bad_3x3_reduce = Conv1D(name="bad_3x3_reduce", filters=96, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	bad_3x3 = Conv1D(name="bad_3x3", filters=64, kernel_size=3, strides=3, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(bad_3x3_reduce)
	bad_5x5_reduce = Conv1D(name="bad_5x5_reduce", filters=16, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(embedding)
	bad_5x5 = Conv1D(name="bad_5x5", filters=64, kernel_size=5, strides=5, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(bad_5x5_reduce)
	bad_pool = MaxPooling1D(name="bad_pool", pool_size=3, strides=1, padding='same')(embedding)
	bad_pool_proj = Conv1D(name="bad_pool_proj", filters=64, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(bad_pool)
	bad_merge = concatenate(name="bad_merge", inputs=[bad_1x1, bad_3x3, bad_5x5, bad_pool_proj], axis=1)
	bad_ave_pool = AveragePooling1D(name="bad_ave_pool", pool_size=5, strides=3)(bad_merge)
	bad_conv = Conv1D(name="bad_conv", filters=128, kernel_size=1, strides=1, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same', activation='relu')(bad_ave_pool)
	bad_flat = Flatten(name="bad_flat")(bad_conv)
	bad_dense = Dense(name="bad_dense", output_dim=256, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(bad_flat)
	bad_drop = Dropout(name="bad_drop", rate=.2)(bad_dense)
	bad_output = Dense(name="bad", output_dim=1, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(bad_drop)

	model = Model(inputs=inputs, outputs=[category_output, good_output, bad_output])
	model.compile(loss='mae', optimizer=optimizers.Adam(lr=.001), metrics=[])
	model.summary()

	epochs = 100

	callback = model.fit(x=train_x, y=[train_y_category, train_y_good, train_y_bad], epochs=epochs, validation_split=.3, batch_size=1000, verbose=1).history

	test_y = model.predict(x=test_x, batch_size=10, verbose=1)

	test_y_good = np.rint(test_y[1]).astype('int')
	test_y_bad = np.rint(test_y[2]).astype('int')

	seconds = str((datetime.datetime.now() - now).seconds)

	print(seconds)

	with open('test{seconds}.txt'.format(seconds=seconds), 'w') as file:
		file.write('id,good,bad\n')
		for index, good, bad in zip(range(4000), test_y_good, test_y_bad):
			file.write('{},{},{}\n'.format(index, good[0], bad[0]))

	with open('record{seconds}.log'.format(seconds=seconds), 'w') as file:
		file.write('result\t\n\n')
		file.write('\t'.join(['index', 'loss\t\t', 'category_loss\t', 'good_loss\t\t', 'bad_loss\t\t', 'val_loss\t\t', 'val_category_loss\t', 'val_good_loss\t\t', 'val_bad_loss']) + '\n')
		for index, loss, category_loss, good_loss, bad_loss, val_loss, val_category_loss, val_good_loss, val_bad_loss in zip(range(1, epochs+1), callback['loss'], callback['category_loss'], callback['good_loss'], callback['bad_loss'], callback['val_loss'], callback['val_category_loss'], callback['val_good_loss'], callback['val_bad_loss']):
			file.write('\t'.join([str(index) + '\t', '{:.12f}'.format(loss), '{:.12f}'.format(category_loss), '{:.12f}'.format(good_loss), '{:.12f}'.format(bad_loss), '{:.12f}'.format(val_loss), '{:.12f}'.format(val_category_loss), '{:.12f}'.format(val_good_loss), '{:.12f}'.format(val_bad_loss)]) + '\n')
		file.write('\nmax_length={max_length}\nmin_count=2, size=270, iter=10, sg=1, workers=10\n'.format(max_length=max_length))


text:
import time
import os
import sys

import numpy as np
import scipy.io
import scipy.misc

import torch
import torchvision

from nams import *

NAME = 'cars196.triplet.fc6'
DESCRIPTION = ''

batch_size = 32
test_frequency = 1000
loss_frequency = 100

###############################################################
# Read cars196 dataset
###############################################################

dataset_path = '../cars196/'
cars_annos = scipy.io.loadmat(dataset_path + '/cars_annos.mat')

train_imgs = []
test_imgs = []
train_labels = []
test_labels = []
for d in cars_annos['annotations'][0]:
    im_path = d[0][0]
    bbox_x1 = d[1][0][0]
    bbox_y1 = d[2][0][0]
    bbox_x2 = d[3][0][0]
    bbox_y2 = d[4][0][0]
    imclass = d[5][0][0]
    test = d[6][0][0]
    test = imclass > 98

    if test:
        test_imgs.append(dataset_path + '/' + im_path)
        test_labels.append(imclass)
    else:
        train_imgs.append(dataset_path + '/' + im_path)
        train_labels.append(imclass)

print 'Done reading data'
print np.min(train_labels)
print np.max(test_labels)
print str(len(train_imgs)) + ' images for training'
print str(len(test_imgs)) + ' images for testing'

train_class2ids = []
while len(train_class2ids) <= np.max(train_labels):
    train_class2ids.append([])
for i in range(len(train_labels)):
    train_class2ids[train_labels[i]].append(i)

class_probs = []
for i in range(len(train_class2ids)):
    n = len(train_class2ids[i])
    if n == 1:
        n = 0
    class_probs.append(0.0 + n)
class_probs = np.array(class_probs) / np.sum(class_probs)

def construct_minibatch(all_imgs, all_labels = None, batch_size = None, indices = None, random_crop = False):
    if batch_size == None:
        batch_size = len(indices)
    data = np.zeros([batch_size, 3, 224, 224])
    if all_labels is not None:
        labels = np.zeros([batch_size])
    for i in range(batch_size):
        j = np.random.randint(low=0, high=len(all_imgs))
        if indices is not None:
            j = indices[i]

        # set label
        if all_labels is not None:
            labels[i] = all_labels[j]

        # read image
        img = scipy.misc.imread(all_imgs[j])
        if len(img.shape) < 3:
            img1 = img
            img = np.zeros((img.shape[0], img.shape[1], 3))
            img[:,:,0] = img1
            img[:,:,1] = img1
            img[:,:,2] = img1

        # resize
        if random_crop:
            s = max(224.0/img.shape[0], 224.0/img.shape[1])
            s *= 1.0 + np.random.rand() * 0.1
            img = scipy.misc.imresize(img, s)
            if np.random.rand() > 0.5:
                img = np.fliplr(img)
            img = my_random_crop(img, (224, 224))
        else:
            img = my_resize_image(img, (224, 224), mode=1)

        img = img / 255.0
        img[:,:,0] = (img[:,:,0] - 0.485) / 0.229
        img[:,:,1] = (img[:,:,1] - 0.456) / 0.224
        img[:,:,2] = (img[:,:,2] - 0.406) / 0.225
        data[i,:,:,:] = img.transpose([2, 0, 1])

    if all_labels is not None:
        return data, labels
    else:
        return data

###############################################################
# Create network, loss function, optimizer
###############################################################

net_feature = torchvision.models.vgg16_bn(pretrained=True)
net_feature.features = torch.nn.Sequential(
    net_feature.features,
    torch.nn.MaxPool2d(7),
    torch.nn.Dropout(p=0.01)
)
net_feature.classifier = torch.nn.Sequential(
    torch.nn.Linear(512, 512)
)
myloss = MyTripletLoss(normalize_scale=4.0, learn_scale=True, DBL=True)

net_feature = net_feature.cuda()
myloss = myloss.cuda()

optimizer = torch.optim.SGD(
    [
        {'params': net_feature.features.parameters(), 'lr': 0.01, 'weight_decay': 5e-4},
        {'params': net_feature.classifier.parameters(), 'lr': 0.01, 'weight_decay': 5e-4},
        {'params': myloss.parameters(), 'lr': 0.01, 'weight_decay': 5e-5}
    ],
    lr=0.01, momentum=0.9, weight_decay=5e-4
)

nams_logger = NamLogger(NAME + '.' +  str(time.time()) + '.namslog')
nams_logger.log('description', DESCRIPTION)

train_losses = []
rank1recalls = []
it = 0

###############################################################
# Some functions to do feature extraction & testing
###############################################################

def vgg_forward_all(input_var, vgg_model, out_layer = None):

    assert(len(net_feature.features[0]) == 44)
    assert(len(net_feature.classifier) <= 5)

    data = []
    x = input_var

    # get pool4
    for i in range(40):
        x = vgg_model.features[0][i](x)
    y = vgg_model.features[0][43](x)
    y = vgg_model.features[1](y)
    y = y.view(x.size(0), -1)
    data.append(y.data.cpu().numpy())

    # get pool5
    x = vgg_model.features[0][40](x)
    x = vgg_model.features[0][41](x)
    x = vgg_model.features[0][42](x)
    x = vgg_model.features[0][43](x)
    x = vgg_model.features[1](x)
    x = x.view(x.size(0), -1)
    data.append(x.data.cpu().numpy())

    # get fc6
    if len(net_feature.classifier) >= 1:
      x = vgg_model.classifier[0](x)
      data.append(x.data.cpu().numpy())

    # get fc7
    if len(net_feature.classifier) >= 3:
        x = vgg_model.classifier[1](x)
        x = vgg_model.classifier[2](x)
        data.append(x.data.cpu().numpy())

    # get fc8
    if len(net_feature.classifier) >= 5:
       x = vgg_model.classifier[3](x)
       x = vgg_model.classifier[4](x)
       data.append(x.data.cpu().numpy())

    return data

def perform_all_features_extraction(net, all_imgs, do_normalization=True):
    all_features = [[]]
    while len(all_features[0]) < len(all_imgs):
        indices = np.arange(len(all_features[0]), min(len(all_features[0])+batch_size, len(all_imgs)))
        data = construct_minibatch(all_imgs, indices=indices, random_crop=False)
        data = torch.autograd.Variable(torch.from_numpy(data).float().cuda())
        output = vgg_forward_all(data, net)
        for i in range(len(output)):
            try:
                all_features[i];
            except:
                all_features.append([])

            for j in range(output[i].shape[0]):
                all_features[i].append(output[i][j,:])
    print len(all_features[0])
    if do_normalization:
      for i in range(len(all_features)):
        all_features[i] = np.array(all_features[i])
        for j in range(all_features[i].shape[0]):
            all_features[i][j,:] = 100.0 * all_features[i][j,:] / np.linalg.norm(all_features[i][j,:])
    return all_features

def test_r1_full_all():

    print 'Perform feature extraction on test set'
    imgs = test_imgs[::1]
    theirlabels = test_labels[::1]

    net_feature.train(mode=False)
    all_features = perform_all_features_extraction(
        net_feature,
        imgs
    )
    r1 = test_retrieval_r1(all_features[0], theirlabels)
    print 'R@1-full-pool4:', r1
    nams_logger.log('R@1-full-pool4', r1, step=it)

    r1 = test_retrieval_r1(all_features[1], theirlabels)
    print 'R@1-full-pool5:', r1
    nams_logger.log('R@1-full-pool5', r1, step=it)

    if len(all_features) >= 3:
      r1 = test_retrieval_r1(all_features[2], theirlabels)
      print 'R@1-full-fc6:', r1
      nams_logger.log('R@1-full-fc6', r1, step=it)

    if len(all_features) >= 4:
        r1 = test_retrieval_r1(all_features[3], theirlabels)
        print 'R@1-full-fc7:', r1
        nams_logger.log('R@1-full-fc7', r1, step=it)

    if len(all_features) >= 5:
        r1 = test_retrieval_r1(all_features[4], theirlabels)
        print 'R@1-full-fc8:', r1
        nams_logger.log('R@1-full-fc8', r1, step=it)

    print 'Perform feature extraction on train set'
    imgs = train_imgs[::1]
    theirlabels = train_labels[::1]

    net_feature.train(mode=False)
    all_features = perform_all_features_extraction(
        net_feature,
        imgs
    )
    r1 = test_retrieval_r1(all_features[0], theirlabels)
    print 'R@1-full-train-pool4:', r1
    nams_logger.log('R@1-full-train-pool4', r1, step=it)

    r1 = test_retrieval_r1(all_features[1], theirlabels)
    print 'R@1-full-train-pool5:', r1
    nams_logger.log('R@1-full-train-pool5', r1, step=it)

    if len(all_features) >= 3:
      r1 = test_retrieval_r1(all_features[2], theirlabels)
      print 'R@1-full-train-fc6:', r1
      nams_logger.log('R@1-full-train-fc6', r1, step=it)

    if len(all_features) >= 4:
        r1 = test_retrieval_r1(all_features[3], theirlabels)
        print 'R@1-full-train-fc7:', r1
        nams_logger.log('R@1-full-train-fc7', r1, step=it)

    if len(all_features) >= 5:
        r1 = test_retrieval_r1(all_features[4], theirlabels)
        print 'R@1-full-train-fc8', r1
        nams_logger.log('R@1-full-train-fc8', r1, step=it)

    nams_logger.flush()

###############################################################
# Training loop
###############################################################

tic = time.time()
for it in range(it, 99999):

    # learning rate scheduling
    if it < 100:
        optimizer.param_groups[0]['lr'] = 0.00
        optimizer.param_groups[1]['lr'] = 0.01
        optimizer.param_groups[2]['lr'] = 0.01
    elif it <= 10000:
        optimizer.param_groups[0]['lr'] = 0.01
        optimizer.param_groups[1]['lr'] = 0.01
        optimizer.param_groups[2]['lr'] = 0.01
    elif it <= 20000:
        optimizer.param_groups[0]['lr'] = 0.001
        optimizer.param_groups[1]['lr'] = 0.001
        optimizer.param_groups[2]['lr'] = 0.001
    else:
        optimizer.param_groups[0]['lr'] = 0.0001
        optimizer.param_groups[1]['lr'] = 0.0001
        optimizer.param_groups[2]['lr'] = 0.0001

    # construct mini batch
    c = np.random.choice(range(len(train_class2ids)), replace=False, size=batch_size/2, p=class_probs)
    indices = []
    for i in c:
        indices = indices + np.random.choice(train_class2ids[i], replace=False, size=2).tolist()

    data, labels = construct_minibatch(train_imgs, train_labels, indices=indices, random_crop=True)
    data = torch.autograd.Variable(torch.from_numpy(data).float().cuda())
    labels = torch.autograd.Variable(torch.from_numpy(labels).long().cuda())

    # forward, backward & udpate
    net_feature.train(mode=True)
    optimizer.zero_grad()
    output = net_feature(data)
    loss = myloss(output, labels)
    loss.backward()
    optimizer.step()
    train_losses.append(loss.data[0])

    # print
    if len(train_losses) > loss_frequency * 0 and (it % loss_frequency == 0):
        print 'Iter %d, elapsed time %f, loss %f' % (it, time.time() - tic, np.mean(train_losses[-loss_frequency:]))
        nams_logger.log('time_elapsed', time.time() - tic, step=it)
        nams_logger.log('train_loss_mean', np.mean(train_losses[-loss_frequency:]), step=it)
        nams_logger.flush()
        tic = time.time()

    # save & run test
    if it % (test_frequency) == 0:
        torch.save({
            'it': it,
            'model': net_feature,
            'loss': myloss,
            'optimizer': optimizer
        }, NAME + '.' + str(it) + '.pth')

        test_r1_full_all()

max_stars_repo_name: lugiavn/generalization-dml

max_issues_repo_name: lugiavn/generalization-dml

max_forks_repo_name: lugiavn/generalization-dml

reason: import numpy,import scipy


text:
# -*- coding: utf8 -*-
"""
    This is part of shot detector.
    Produced by w495 at 2017.05.04 04:18:27
"""

from __future__ import absolute_import, division, print_function, \
    unicode_literals

import logging

import numpy as np

from .math_filter import MathFilter

class ColourFilter(MathFilter):
    """
        ...
    """
    __logger = logging.getLogger(__name__)

    def filter_feature_item(self,
                            feature,
                            **kwargs):
        """

        :param feature:
        :param kwargs:
        :return:
        """
        return self.extract_item_colour(feature, **kwargs)

    @staticmethod
    def extract_item_colour(feature,
                            pattern=None,
                            y=None,
                            cr=None,
                            cb=None,
                            red=0,
                            green=0,
                            blue=0,
                            summand=0,
                            factor=1,
                            **_):
        """

        :param feature:
        :param pattern:
        :param y:
        :param cr:
        :param cb:
        :param red:
        :param green:
        :param blue:
        :param summand:
        :param factor:
        :param _:
        :return:
        """

        if y is not None:
            pattern = (299, 587, 114)
            summand = 0
            factor = y

        if cr is not None:
            pattern = (500000, -418688, -81312)
            summand = 0.5
            factor = cr

        if cb is not None:
            pattern = (-168736, -331264, 500000)
            summand = 0.5
            factor = cb

        if pattern is None:
            pattern = (red, green, blue)

        pattern_sum = sum(pattern)
        y_pb_pr = np.inner(feature, pattern) / pattern_sum
        result = summand + factor * y_pb_pr
        return result


text:
import cv2
import numpy as np
img = cv2.imread('prof.jpeg',0)
rows,cols = img.shape
M = np.float32([[1,0,100],[0,1,50]])
dst = cv2.warpAffine(img,M,(cols,rows))
cv2.imshow('img',dst)
cv2.waitKey(0)
cv2.destroyAllWindows()


text:
# we are going to test the forward model, reverse model and multiple vector functions here.
import sys
sys.path.append("./src/")
import numpy as np
from undefined.GraphGenerator import UDGraph, GeneratorHelper
from undefined.Calculator import sin, cos, exp, tan, sqrt, log
from undefined.UDFunction import UDFunction
from undefined.API import trace
import unittest

class TestTrace(unittest.TestCase):

    def setUp(self):
        self.f1 = lambda x: sqrt(exp(sin(x)))
        self.f2 = lambda x: exp(x**2 + sin(x))
        self.f3 = lambda x: sin(exp(2**x))
        self.f4 = lambda x: x**sin(x+2)
        self.f5 = lambda x, y: exp(1-6*x) * tan(4*x + 2*y)
        self.f6 = lambda x: 1 / (8 + cos(2*np.pi*x))
        self.f7 = lambda x, y, z: x*exp(y**2 - z**2)
        self.f8 = lambda x: log(x*3 + tan(2*x), np.e) + 2**x
        self.f9 = lambda x, y: x**y
        self.f10 = lambda x: log(x*3, np.e)
        self.f11 = lambda x: cos(exp(2*x))
        self.f12 = lambda x, y: log(1 + 6*x) * tan(4*x + 2*y)
        self.f13 = lambda x: log(6*x, 10) * tan(4*x)
        self.f1000 = lambda x, y: exp(1-6*x) * tan(4*x + 2*y) + x**2*y
        self.f1001 = lambda x, y: exp(1-6*x) / tan(4*x + 2*y) + x**2*y
        self.f1002 = lambda x, y: exp(1-6*x) // tan(4*x + 2*y) + x**2*y
        self.f1003 = lambda x: x//192
        self.f1004 = lambda x: x/192
        self.f1005 = lambda x: x**2 + x**5
        self.f1006 = lambda x: 2**x + 5**x
        self.f1007 = lambda x, y: x**y

    def assertNumpyArraysEqual(self, o1, o2):
        if o1.shape != o2.shape:
            raise AssertionError("Shapes don't match")
        if not np.allclose(o1, o2):
            raise AssertionError("Elements don't match!")

    def tearDown(self):
        pass

    def test_forward(self):
        self.assertEqual(trace(self.f1, x=2), (1.58, -0.328))
        self.assertEqual(trace(self.f2, x=3), (9331.210, 46749.434))
        self.assertEqual(trace(self.f3, x=3), (0.40, -15137.189))
        self.assertEqual(trace(self.f4, x=3), (0.35, -0.003))

        self.assertEqual(trace(self.f5, x=2, y=3), (0.0, [[0.003], [0.002]]))
        self.assertEqual(trace(self.f7, x=1, y=2, z=3),
                         (0.01, [[0.007],  [0.027], [-0.04]]))
        self.assertEqual(trace(self.f8, x=3), (10.16, 6.139))
        self.assertEqual(trace(self.f9, x=5, y=3), (125, [[75.], [201.18]]))
        self.assertEqual(trace(self.f10, x=2), (1.79, 0.5))

        def f22(x):
            return x+cos(2*x)
        with self.assertRaises(TypeError):
            trace(f22(2), x=2)

    def test_forward_trace(self):
        result1 = trace(self.f1, x=np.array([[2]]))
        self.assertEqual(result1, (np.array([[1.58]]), np.array([[-0.328]])))
        self.assertEqual(trace(self.f2, x=np.array(
            [[3]])), (np.array([[9331.21]]), np.array([[46749.434]])))
        self.assertEqual(trace(self.f8, x=np.array([[3]])), (np.array(
            [[10.16]]), np.array([[6.139]])))  # raise error
        self.assertEqual(trace(self.f11, x=np.array(
            [[2]])), (np.array([[-0.37]]), np.array([[101.418]])))
        self.assertEqual(trace(self.f13, x=np.array(
            [[3]])), (np.array([[-0.8]]), np.array([[6.959]])))
        self.assertEqual(trace(self.f13, x=np.array(
            [[3]])), (np.array([[-0.8]]), np.array([[6.959]])))
        with self.assertRaises(TypeError):
            trace(self.f1, x=np.array([]))

        # test multiple x values at the same time
        self.assertNumpyArraysEqual(trace(self.f1, x=np.array([[1, 2]]))[
                                    0], np.array([[1.52, 1.58]]))

        self.assertEqual(trace(self.f1, x=np.array([[1, 2]]))[
                         1], [[0.411, -0.328]])

        with self.assertRaises(TypeError):
            trace(self.f1, x="2")

        self.assertNumpyArraysEqual(trace([self.f1, self.f2], x=2)[
                                    0], np.array([1.58, 135.54]))
        self.assertNumpyArraysEqual(trace([self.f1, self.f2], x=2)[
                                    1], np.array([-3.2800e-01,  4.8577e+02]))

        g = UDGraph(np.array([[2], [1]]))
        with self.assertRaises(ValueError):
            g.__rpow__(UDGraph(np.array([[np.pi/2]])))

        with self.assertRaises(TypeError):
            g.__rpow__("str")

        g = UDGraph(np.array([[2], [1]]))
        g.__hash__()

    def test_reverse(self):
        g = UDGraph(9999.909)
        g2 = g.__radd__(UDGraph(np.pi/2))
        GeneratorHelper._radd(g2, g, seed_dic={})

        g = UDGraph(999.09)
        g2 = g.__rmul__(UDGraph(np.pi/2))
        GeneratorHelper._rmul(g2, g, seed_dic={})

        g = UDGraph(np.pi/2)
        g2 = g.__neg__()
        GeneratorHelper._neg(g2, g, seed_dic={})

        g = UDGraph(9999.909)
        g2 = g.__rsub__(UDGraph(np.pi/2))
        GeneratorHelper._rsub(g2, g, seed_dic={})

        g = UDGraph(9999.909)
        g2 = g.__rtruediv__(UDGraph(np.pi/2))
        GeneratorHelper._rtruediv(g2, g, seed_dic={})

        g = UDGraph(9999.909)
        g2 = g.__rfloordiv__(UDGraph(np.pi/2))
        GeneratorHelper._rfloordiv(g2, g, seed_dic={})

        g = UDGraph(3.5)
        g2 = g.__rpow__(UDGraph(np.pi/2))
        GeneratorHelper._rpow(g2, g, seed_dic={})

        g = UDGraph(3.5)
        g2 = g.__rpow__(UDGraph(np.array([[np.pi/2]])))
        GeneratorHelper._rpow(g2, g, seed_dic={})

        g = UDGraph(np.array(3.5))
        g2 = g.__rpow__(UDGraph(3.5))
        GeneratorHelper._rpow(g2, g, seed_dic={})

        g1 = UDGraph(np.array([[3.5]]))
        g2 = g1.__rpow__(UDGraph(np.array([[np.pi/2]])))
        GeneratorHelper._rpow(g2, g1, seed_dic={})

        self.assertEqual(repr(g), "Computational Graph (3.5, UDPrimitive.VAR)")

        result1001 = trace(self.f1001, mode="reverse", x=np.pi, y=998)

        self.assertEqual(result1001, (9849.87, [[6270.619], [9.87]]))

        result1002 = trace(self.f1002, mode="reverse", x=np.pi, y=998)

        self.assertEqual(result1002, (9849.87, [[6269.619], [8.87]]))

        result1003 = trace(self.f1003, mode="reverse", x=1343)

        self.assertEqual(result1003, (6, 0))

        result1004 = trace(self.f1004, mode="reverse", x=1343)

        self.assertEqual(result1004, (6.99, 0.005))

        result1005 = trace(self.f1005, mode="reverse", x=3)

        self.assertEqual(result1005, (252, 411))

        result1006 = trace(self.f1006, mode="reverse", x=5)

        self.assertEqual(result1006, (3157, 5051.674))

        result1007 = trace(self.f1005, mode="reverse", x=np.array([[1343]]))

        self.assertEqual(result1005, (252, 411))

        result1006 = trace(self.f1006, mode="reverse", x=np.array([[3]]))

        self.assertEqual(result1006, (np.array([[133]]), 206.725))

        result1007 = trace(self.f1007, mode="reverse",
                           x=np.array([[3]]), y=np.array([[3]]))

        self.assertEqual(result1007, (np.array(
            [[27]]), [[27.0], [29.663]]), 206.725)

        result1007 = trace(self.f1007, mode="reverse", x=3, y=np.array([[3]]))

        self.assertEqual(result1007, (np.array(
            [[27]]), [[27.0], [29.663]]), 206.725)

        result1007 = trace(self.f1007, mode="reverse", x=np.array([[3]]), y=3)

        self.assertEqual(result1007, (np.array(
            [[27]]), [[27.0], [29.663]]), 206.725)

        result1007 = trace(self.f1007, mode="reverse", x=3, y=3)

        self.assertEqual(result1007, (np.array(
            [[27]]), [[27.0], [29.663]]), 206.725)
        result1 = trace(self.f1, mode="reverse", x=2)

        self.assertEqual(result1, (1.58, [-0.328]))

        result2 = trace(self.f2, mode="reverse", x=2)
        self.assertEqual(result2, (135.54, [485.77]))

        self.function_ = lambda x: x+1

        result3 = trace(self.function_, mode="reverse", x=2)
        self.assertEqual(result3, (3, [1]))

        self.function_2 = lambda x: 3*x - 1
        result4 = trace(self.function_2, mode="reverse", x=2)
        self.assertEqual(result4, (5, [3]))

        self.function_3 = lambda x: 1 - x*3
        result5 = trace(self.function_3, mode="reverse", x=2)
        self.assertEqual(result5, (-5, [-3]))

        self.function_4 = lambda x: 1 + x

        result6 = trace(self.function_4, mode="reverse", x=2)
        self.assertEqual(result6, (3, [1]))

        result7 = trace(self.f5, mode="reverse", x=2, y=1)
        self.assertEqual(result6, (3, [1]))

        result8 = trace(self.f6, mode="reverse", x=2)
        self.assertEqual(result8, (0.11, [-0.0]))

        result9 = trace(self.f7, mode="reverse", x=2, y=1, z=3)
        self.assertEqual(result6, (3, [1]))

        # result10 = trace(self.f8,mode = "reverse",x = 2)
        # self.assertEqual(result10,(3, [1]))

        # result11 = trace(self.f9,mode = "reverse",x = 2,y =1)
        # self.assertEqual(result11,(3, [1]))

        result12 = trace(self.f10, mode="reverse", x=2)
        self.assertEqual(result12, (1.79, [0.5]))

        result13 = trace(self.f11, mode="reverse", x=2)
        self.assertEqual(result13, (-0.37, [101.418]))

        result14 = trace(self.f12, mode="reverse", x=2, y=1)
        self.assertEqual(result14, (1.66, [[14.872], [7.286]]))

        result15 = trace(self.f13, mode="reverse", x=2)
        self.assertEqual(result15, (-7.34, [[202.429]]))

        self.f100 = lambda x: 1+x**2
        result16 = trace(self.f100, mode="reverse", x=2)
        self.assertEqual(result16, (5, [4]))

        self. f101 = lambda x, y: x*y
        result17 = trace(self.f101, mode="reverse", x=2, y=1)
        self.assertEqual(result17, (2, [[1], [2]]))

        self.f102 = lambda x, y: -x*y
        result18 = trace(self.f102, mode="reverse", x=2, y=1)
        self.assertEqual(result18, (-2, [[-1], [-2]]))

        self.f103 = lambda x: - x + 1
        result19 = trace(self.f103, mode="reverse", x=2)
        self.assertEqual(result19, (-1, [-1]))

        self.fcs = lambda x: cos(x)+1
        result19 = trace(self.fcs, mode="reverse", x=np.array([[2]]))
        self.assertEqual(result19, (np.array([[0.58]]), -0.909))

        with self.assertRaises(TypeError):
            trace(self.fcs, mode="reverse", x="str")

        self.sine = lambda x: sin(x)+1
        result19 = trace(self.sine, mode="reverse", x=np.array([[20]]))
        self.assertEqual(result19, (np.array([[1.91]]), 0.408))

        with self.assertRaises(TypeError):
            trace(self.sine, mode="reverse", x="str")

        self.tange = lambda x: sin(x)+1
        result19 = trace(self.tange, mode="reverse", x=np.array([[50]]))
        self.assertEqual(result19, (np.array([[0.74]]), 0.965))

        with self.assertRaises(TypeError):
            trace(self.tange, mode="reverse", x="str")

    def test_multiple_function(self):
        def f1(x): return sqrt(exp(sin(x)))
        def f2(x): return 2*x + sqrt(x)

        self.assertEqual(trace([f1, f2], x=np.array([[1, 2]]))[
                         1].tolist(), [[[0.411, -0.328]], [[2.5,  2.354]]])
        self.assertEqual(trace([f1, f2], x=np.array([[1, 2, 3]]))[0].tolist(), [
                         [[1.52, 1.58, 1.07]], [[3., 5.41, 7.73]]])
        self.assertEqual(trace([f1, f2], x=np.array([[1, 2, 3]]))[1].tolist(), [
                         [[0.411, -0.328, -0.531]], [[2.5,  2.354,  2.289]]])

    def test_multiple_variable_function(self):

        def f3(x, y): return x**2 + 2**y
        def f4(x, y): return 2*x - 2/y

        self.assertEqual(trace([f3, f4], x=np.array([[1, 2]]), y=4)[
                         0].tolist(), [[[17., 20.]], [[1.5,  3.5]]])
        self.assertEqual(trace([f3, f4], x=np.array([[1, 2]]), y=4)[1].tolist(), [
                         [[2.,  4.], [11.09, 11.09]], [[2.,  2.], [0.125,  0.125]]])

        self.assertEqual(trace([f3, f4], mode="reverse", x=np.array([[1, 2]]), y=4)[
                         0].tolist(), [[[17., 20.]], [[1.5,  3.5]]])
        self.assertEqual(trace([f3, f4], mode="reverse", x=np.array([[1, 2]]), y=4)[
                         1].tolist(), [[[2.,  4.], [11.09, 11.09]], [[2.,  2.], [0.125,  0.125]]])

        self.assertEqual(trace([f3, f4], x=4, y=np.array([[1, 2]]))[
                         0].tolist(), [[[18., 20.]], [[6.,  7.]]])
        self.assertEqual(trace([f3, f4], x=4, y=np.array([[1, 2]]))[1].tolist(), [
                         [[8., 8.], [1.386, 2.773]], [[2., 2.], [2., 0.5]]])

        self.assertEqual(trace([f3, f4], mode="reverse", x=4, y=np.array([[1, 2]]))[
                         0].tolist(), [[[18., 20.]], [[6.,  7.]]])
        self.assertEqual(trace([f3, f4], mode="reverse", x=4, y=np.array([[1, 2]]))[
                         1].tolist(), [[[8., 8.], [1.386, 2.773]], [[2., 2.], [2., 0.5]]])

if __name__ == "__main__":
    unittest.main()


text:
# ---------------------------*-python-*----------------------------------------#
# file   plot_dictionary.py
# author mathew cleveland
# date   February 2019
# note   Copyright (C) 2019, Los Alamos National Security, LLC.
#        All rights reserved.
# -----------------------------------------------------------------------------#
'''
Dictionary Plotting class

.. autosummary::

  plot_dictionary

'''
import matplotlib.pyplot as PyPloter
import matplotlib.axes as axes
import string, sys, os
import numpy as np
import re
from math import *
import argparse
import shlex

from opppy.plotting_help import *

class plot_dictionary():
    '''
    This class encapsolates a basic dictionary plotter. The important class
    options include:
        setup_parser
        parse_input_string
        is_data_available
        plot_dict

    '''
    def __init__(self):
        self.parser = argparse.ArgumentParser(description="Dictionary ploter")
        self.setup_parser(self.parser)

    def parse_input_string(self, input_string):
        '''
        Parse an input string and return a plotting argument list
        '''
        return self.parser.parse_args(shlex.split(input_string))

    def is_data_available(self, args, dictionary):
        '''
        Check the dictionary for valid plotting data

        arguments:
            args parsed dictionary plotting arguments
            dictonary the dictonary to search for the x and y data value names
        '''
        try:
            data = dictionary[args.dictionary_name]
            x = data[args.x_value_name]
            if not (args.y_value_names[0] == "select_key"):
                for yname in args.y_value_names:
                    y = data[yname]
            return True
        except:
            return False

    def setup_parser(self, parser):
        '''
        setup the minimum parsing information need to plot a dictionary

        Input arguments:
            parser from argparse
        '''
        parser.add_argument('-dn','--dictionary_name', dest='dictionary_name', help='dictionary that the plotting data is contained in', required=True, type=str)
        parser.add_argument('-x','--x_data', dest='x_value_name', help='dictionary data to be plotted on the x axis.', required=True)
        parser.add_argument('-y','--y_data', dest='y_value_names', help='dictionary data to be plotted on the y axis.', required=True, action='append')
        add_plot_options(parser);

    def plot_dict(self, args, dictionaries, data_names):
        '''
        argument based dictionary plotting tool.

        arguments:
            args parsed dictionary plotting arguments
            dictionaries a list of dictionaries to be plotted
            data_names names associated with the dictionaries to be plotted
        '''
        if(args.hide_plot):
            PyPloter.switch_backend('agg')

        if len(dictionaries) is not len(data_names):
            print("Error: len of dictionaries do not match length of associated data names")
            sys.exit(0)

        dictionary_name = args.dictionary_name
        yname = ''
        xname = ''

        if len(args.scale_x) != len(dictionaries):
            if len(args.scale_x) == 0:
                args.scale_x = [1.0]*len(dictionaries)
            else:
                args.scale_x = [args.scale_x[-1]]*len(dictionaries)

        if len(args.scale_y) != len(dictionaries):
            if len(args.scale_y) == 0:
                args.scale_y = [1.0]*len(dictionaries)
            else:
                args.scale_y = [args.scale_y[-1]]*len(dictionaries)

        for dictionary, filename, scale_x, scale_y in zip(dictionaries,data_names,args.scale_x,args.scale_y):
            data = dictionary[dictionary_name]
            new_file = True
            last_x = []
            last_y = []
            last_z = []
            for yname in args.y_value_names:
                # clean file names
                if(args.data_file_name is not None):
                    outputfile = open(args.data_file_name+'_'+filename.split('/')[-1].strip('.p')+'.'+dictionary_name.replace(' ','_').replace('/','_').replace('#','num')+"."+re.sub(r'[^\w]','',yname)+'.dat', 'w')
                xname = args.x_value_name
                x = np.array(data[xname])*scale_x
                y = np.array(data[yname])*scale_y
                if args.last_time_value is not None:
                    for time, x_value, y_value in zip(data['time'],data[xname],data[yname]):
                        if value > last_time_value:
                            last_y.append(y_value*scale_y)
                            last_x.append(x_value*scale_x)
                elif args.last_point_only:
                    last_y.append(data[yname][-1]*scale_y)
                    last_x.append(data[xname][-1]*scale_x)

                if(args.data_file_name is not None):
                    if(args.x_label is not None):
                        header_xlabel = args.x_label
                    else:
                        header_xlabel = xname
                    if(args.y_label is not None):
                        header_ylabel = args.y_label
                    else:
                        header_ylabel = yname

                    print('# ', header_xlabel, header_ylabel, file=outputfile)
                    for x_value, y_value in zip(x, y):
                        outstring = "%.9e"%(x_value*scale_x)+" %.9e"%(y_value*scale_y)+"\n"
                        outputfile.write(outstring)
                data_name = ''
                if(args.data_file_name is not None):
                    print("data saved as -- "+args.data_file_name+'_'+filename.split('/')[-1].strip('.p')+'.'+dictionary_name.replace(' ','_').replace('/','_').replace('#','num')+"."+re.sub(r'[^\w]','',yname)+'.dat')
                    outputfile.close()
                if(args.line_names is not None):
                    if(len(args.line_names) != len(args.y_value_names)):
                        print("number of line names doesn't match the number of plotted lines")
                        print("line names have to be specified either for ALL or NONE of the plotted lines")
                        print("number of line names = ", len(args.tracer_names))
                        print("number of y_value_names = ", len(args.tracer_numbers))
                        sys.exit()
                    for temp_y_name, line_name in zip(args.y_value_names, args.line_names):
                            if(temp_y_name == yname):
                                data_name = filename.split('/')[-1] + " - " + line_name
                elif(args.no_y_names or args.last_point_only):
                    if(new_file or args.last_point_only):
                        data_name = filename.split('/')[-1]
                        new_file = False
                    else:
                        data_name = ''
                else:
                    data_name = filename.split('/')[-1] + " " +str(yname)

                data_line_type = ''
                if(args.line_types is not None):
                    if(len(args.line_types) != len(args.y_value_names)):
                        print("number of line types doesn't match the number of plotted lines")
                        print("line types have to be specified either for ALL or NONE of the lines")
                        print("number of line types = ", len(args.line_types))
                        print("number of y_value_names = ", len(args.tracer_numbers))
                        sys.exit()
                    for temp_y_name, line_type in zip(args.y_value_names, args.line_types):
                        if(temp_y_name == yname):
                            data_line_type = line_type.strip(' ')

                data_line_color = ''
                if(args.line_colors is not None):
                    if(len(args.line_colors) != len(args.filenames)):
                        print("number of line colors doesn't match the number of inputs plotted")
                        print("line colors have to be specified either for ALL or NONE of the lines")
                        print("number of line colors = ", len(args.line_colors))
                        print("number of files plotted = ", len(data_names))
                        sys.exit()
                    for temp_filename, line_color in zip(data_names, args.line_colors):
                        if(temp_filename == filename):
                            data_line_color = line_color

                if(args.last_point_only):
                    if(len(x)>0):
                        last_x.append(x[-1])
                        last_y.append(y[-1])

                if(args.plot_max):
                    last_y.append(sum(sorted(y,reverse=True)[0:2])/3.0)
                    last_x.append(x[-1])
                elif(args.plot_arrival):
                    continue
                elif(data_line_color != '' and data_line_type != ''):
                    PyPloter.plot(x,y,label = data_name, linestyle = data_line_type, color = data_line_color)
                elif(data_line_color != '' ):
                    PyPloter.plot(x,y,label = data_name, color = data_line_color)
                elif(data_line_type != '' ):
                    PyPloter.plot(x,y,label = data_name, linestyle = data_line_type)
                else:
                    PyPloter.plot(x,y,label = data_name)

                if(args.plot_arrival):
                    last_x = x[0]
                    last_y = y[0]
                    for x_value, y_value in zip(x,y):
                        if(y_value>args.y_exceeds_value):
                            dy = (args.y_exceeds_value - last_y)/(y_value - last_y)
                            interp_x = last_x + dy*(x_value-last_x)
                            print(data_name, "first exceeds ", y_exceeds_value, " at ", interp_x)
                            break
                        last_x = x_value
                        last_y = y_value

                if(args.find_max_y):
                    print(data_name, "max y value ", x[y.index(max(y))], max(y))
                if(args.find_min_y):
                    print(data_name, "max y value ", x[y.index(min(y))], min(y))

            if(args.last_point_only):
                PyPloter.plot(last_x, last_y, label = data_name)
            elif(args.plot_arrival or args.plot_max):
                print(last_x, last_y)
                PyPloter.plot(last_x, last_y, label = data_name)

        if(args.x_label is not None):
            PyPloter.xlabel(args.x_label)
        else:
            PyPloter.xlabel(xname)

        if(args.x_limits is not None):
            PyPloter.xlim(args.x_limits)

        if(args.y_label is not None):
            PyPloter.ylabel(args.y_label)
        else:
            PyPloter.ylabel(dictionary_name)

        if(args.y_limits is not None):
            PyPloter.ylim(args.y_limits)

        if(args.plot_grid):
            PyPloter.grid()

        PyPloter.legend(loc='best')
        if(args.log_x):
            PyPloter.xscale("log")
        if(args.log_y):
            PyPloter.yscale("log")
        if(args.figure_name is not None):
            fig = PyPloter.savefig(args.figure_name, dpi=args.figure_resolution)
            print("Plot save as -- "+args.figure_name)
        elif(not args.hide_plot):
            PyPloter.show()

max_stars_repo_name: lowrie/OPPPY

max_issues_repo_name: lowrie/OPPPY

max_forks_repo_name: lowrie/OPPPY