text:
import matplotlib.pyplot as plt

import pandas as pd
import seaborn as sns
import numpy as np

def plot_3d(x, y, z, figure=None, x_label='X', y_label='Y', z_label='Z'):
    """Creates a 3D plot using matplotlib

    Parameters
    ----------
    x : list
        X values (can be :class:`numpy.array` or :class:`Pandas.Series`)
    y : list
        Y values (can be :class:`numpy.array` or :class:`Pandas.Series`)
    z : list
        Z values (can be :class:`numpy.array` or :class:`Pandas.Series`)
    figure : matplotlib.figure.Figure, optional
        Existing figure to add to (the default is None, which creates a new figure)
    x_label : str, optional
        X axis label (the default is 'X')
    y_label : str, optional
        Y axis label (the default is 'Y')
    z_label : str, optional
        Z axis label (the default is 'Z')

    Returns
    -------
    matplotlib.figure.Figure
        3D plot of x, y, and z

    """
    if figure is None:
        figure = plt.figure()
    axis = figure.gca(projection='3d')
    axis.plot(x, y, z, zdir='z')
    axis.set_xlabel(x_label)
    axis.set_ylabel(y_label)
    axis.set_zlabel(z_label)
    axis.axis('equal')

    return figure

def heatmap(x, y, **kwargs):
    """From https://github.com/drazenz/heatmap

    """
    if 'color' in kwargs:
        color = kwargs['color']
    else:
        color = [1]*len(x)

    if 'palette' in kwargs:
        palette = kwargs['palette']
        n_colors = len(palette)
    else:
        n_colors = 256 # Use 256 colors for the diverging color palette
        palette = sns.color_palette("Blues", n_colors)

    if 'color_range' in kwargs:
        color_min, color_max = kwargs['color_range']
    else:
        color_min, color_max = min(color), max(color) # Range of values that will be mapped to the palette, i.e. min and max possible correlation

    def value_to_color(val):
        if color_min == color_max:
            return palette[-1]
        else:
            val_position = float((val - color_min)) / (color_max - color_min) # position of value in the input range, relative to the length of the input range
            val_position = min(max(val_position, 0), 1) # bound the position betwen 0 and 1
            ind = int(val_position * (n_colors - 1)) # target index in the color palette
            return palette[ind]

    if 'size' in kwargs:
        size = kwargs['size']
    else:
        size = [1]*len(x)

    if 'size_range' in kwargs:
        size_min, size_max = kwargs['size_range'][0], kwargs['size_range'][1]
    else:
        size_min, size_max = min(size), max(size)

    size_scale = kwargs.get('size_scale', 500)

    def value_to_size(val):
        if size_min == size_max:
            return 1 * size_scale
        else:
            val_position = (val - size_min) * 0.99 / (size_max - size_min) + 0.01 # position of value in the input range, relative to the length of the input range
            val_position = min(max(val_position, 0), 1) # bound the position betwen 0 and 1
            return val_position * size_scale
    if 'x_order' in kwargs:
        x_names = [t for t in kwargs['x_order']]
    else:
        x_names = [t for t in sorted(set([v for v in x]))]
    x_to_num = {p[1]:p[0] for p in enumerate(x_names)}

    if 'y_order' in kwargs:
        y_names = [t for t in kwargs['y_order']]
    else:
        y_names = [t for t in sorted(set([v for v in y]))]
    y_to_num = {p[1]:p[0] for p in enumerate(y_names)}

    plot_grid = plt.GridSpec(1, 15, hspace=0.2, wspace=0.1) # Setup a 1x10 grid
    ax = plt.subplot(plot_grid[:,:-1]) # Use the left 14/15ths of the grid for the main plot

    marker = kwargs.get('marker', 's')

    kwargs_pass_on = {k:v for k,v in kwargs.items() if k not in [
         'color', 'palette', 'color_range', 'size', 'size_range', 'size_scale', 'marker', 'x_order', 'y_order'
    ]}

    ax.scatter(
        x=[x_to_num[v] for v in x],
        y=[y_to_num[v] for v in y],
        marker=marker,
        s=[value_to_size(v) for v in size],
        c=[value_to_color(v) for v in color],
        **kwargs_pass_on
    )
    ax.set_xticks([v for k,v in x_to_num.items()])
    ax.set_xticklabels([k for k in x_to_num], rotation=45, horizontalalignment='right')
    ax.set_yticks([v for k,v in y_to_num.items()])
    ax.set_yticklabels([k for k in y_to_num])

    ax.grid(False, 'major')
    ax.grid(True, 'minor')
    ax.set_xticks([t + 0.5 for t in ax.get_xticks()], minor=True)
    ax.set_yticks([t + 0.5 for t in ax.get_yticks()], minor=True)

    ax.set_xlim([-0.5, max([v for v in x_to_num.values()]) + 0.5])
    ax.set_ylim([-0.5, max([v for v in y_to_num.values()]) + 0.5])
    ax.set_facecolor('#F1F1F1')

    # Add color legend on the right side of the plot
    if color_min < color_max:
        ax = plt.subplot(plot_grid[:,-1]) # Use the rightmost column of the plot

        col_x = [0]*len(palette) # Fixed x coordinate for the bars
        bar_y=np.linspace(color_min, color_max, n_colors) # y coordinates for each of the n_colors bars

        bar_height = bar_y[1] - bar_y[0]
        ax.barh(
            y=bar_y,
            width=[5]*len(palette), # Make bars 5 units wide
            left=col_x, # Make bars start at 0
            height=bar_height,
            color=palette,
            linewidth=0
        )
        ax.set_xlim(1, 2) # Bars are going from 0 to 5, so lets crop the plot somewhere in the middle
        ax.grid(False) # Hide grid
        ax.set_facecolor('white') # Make background white
        ax.set_xticks([]) # Remove horizontal ticks
        ax.set_yticks(np.linspace(min(bar_y), max(bar_y), 3)) # Show vertical ticks for min, middle and max
        ax.yaxis.tick_right() # Show vertical ticks on the right

def corrplot(data, size_scale=500, marker='s'):
    """From https://github.com/drazenz/heatmap

    """
    corr = pd.melt(data.reset_index(), id_vars='index')
    corr.columns = ['x', 'y', 'value']
    corr.fillna(0, inplace=True)
    heatmap(
        corr['x'], corr['y'],
        color=corr['value'], color_range=[-1, 1],
        palette=sns.diverging_palette(20, 220, n=256),
        size=corr['value'].abs(), size_range=[0,1],
        marker=marker,
        x_order=data.columns,
        y_order=data.columns[::-1],
        size_scale=size_scale
    )


text:
"""
k-means clustering
"""

import argparse

import numpy as np
from sklearn.decomposition import PCA

import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
from sklearn.cluster import KMeans

parser = argparse.ArgumentParser(description='Kmeans clustering')
# parameters
parser.add_argument('--clusters', type=int, default=4, help='number of clusters')
args = parser.parse_args()

class k_means():
    def __init__(self, num_clusters, tol):
        self.num_clusters = num_clusters
        self.tol = tol

    def distance(self, x, y):
        """
        Return the euclidian distance between two points
        """
        return np.linalg.norm(x-y)

    def get_cluster(self, data):
        old_centroids = []

        sample_size, feature_size = data.shape
        init_centorid_idx = np.random.randint(0, sample_size-1, size=self.num_clusters)
        est_centorid = data[init_centorid_idx]                                             # randomly fetch the centroids

        old_centroids.append(est_centorid)                                                 # record the initial estimation
        prev_centroids = np.zeros(est_centorid.shape)                                      # initialize centroid history

        classify = np.zeros((sample_size, ))
        diff = self.distance(est_centorid, prev_centroids)
        num_iter = 0
        loss = []

        while diff > self.tol:
            diff = self.distance(est_centorid, prev_centroids)
            loss.append(diff*2)                                                             # record the losses
            num_iter += 1
            print(f'Iter: {num_iter} | diff: {diff}')

            for ii, instance in enumerate(data):
                dist = np.zeros((self.num_clusters,1))
                # for each centroid:
                for jj, centroids in enumerate(est_centorid):
                    dist[jj] = self.distance(centroids, instance)
                # find the minimum distance
                classify[ii] = np.argmin(dist)
            tmp_est = np.zeros((self.num_clusters, feature_size))

            # for each cluster
            for idx in range(len(est_centorid)):
                # all points that classified to a cluster
                instance_close = [k for k in range(len(classify)) if classify[k] == idx]
                centorid = np.mean(data[instance_close], axis=0)
                tmp_est[idx, :] = centorid

            prev_centroids = est_centorid
            est_centorid = tmp_est
            old_centroids.append(tmp_est)

        return est_centorid, old_centroids, classify, loss, num_iter

def compute_inertia(X, est_centorid, predict_labels, num_clusters):
    total_inertia = 0
    for ii in range(num_clusters):
        data = X[predict_labels == ii]
        inertia = 0
        centroid = est_centorid[ii]

        for jj in range(data.shape[0]):
            data_point = data[jj, :]
            inertia += np.linalg.norm(data_point - centroid)

        total_inertia += inertia

    print(f'Clusters: {num_clusters} | interia = {total_inertia}')
    return total_inertia

def plotting(predict_labels, data, num_clusters, centroids):
    color = ['lightgreen', 'orange', 'lightblue', 'steelblue', 'red', 'blueviolet', 'aqua', 'g', 'tan', 'darkcyan', 'darkblue']
    markers = ['s', 'o', 'v', '^', 'x', 'D', 'P', 'X', 'h', '+']

    plt.figure(figsize=(8,8), dpi=300)

    for ii in range(num_clusters):
        plt.scatter(
            data[predict_labels == ii, 0], data[predict_labels == ii, 1],
            s=50, c=color[ii],
            marker=markers[ii], edgecolor='black',
            label=f'cluster {ii+1}'
        )

        plt.scatter(
            centroids[ii, 0],
            centroids[ii, 1],
            marker='X',
            s=100,
            c=color[ii],
            label=f'centroid {ii+1}'
        )

    plt.title(f'Kmeans: after clustering | Number of clusters={args.clusters}')
    plt.legend(loc='best')
    plt.savefig(f'./figs/kmeans_cluster_{num_clusters}_genderFalse.png', bbox_inches = 'tight', pad_inches = 0)
    plt.close()

def main():
    clusters = args.clusters
    if clusters not in [3, 4, 5, 6, 7, 8, 9, 10]:
        raise ValueError("Number of clusters must be 4, 6, 8, or 10!")

    # Load the data samples and the corresponding labels
    samples = np.load('./data/customer_data_original_genderFalse.npy', allow_pickle=True)

    # print(f"Empty data: {samples.isnull().any().any()}")
    print(f'Size of the data samples: {samples.shape}')

    # data preprocessing: PCA
    pca = PCA(n_components=2)
    principleComp = pca.fit_transform(samples)

    # visualize the transformed data:
    plt.figure(figsize=(8,8), dpi=300)
    sns.scatterplot(principleComp[:, 0], principleComp[:, 1])
    plt.title('Before clustering')
    plt.savefig('./figs/pca_process.png', bbox_inches = 'tight', pad_inches = 0)
    plt.close()

    # Performing kmeans (with sklearn)
    # km = KMeans(n_clusters=3, max_iter=200)
    # predict_labels = km.fit_predict(samples)

    km = k_means(num_clusters=clusters, tol=1e-4)
    est_centroid, history_centroids, predict_labels, loss, num_iter = km.get_cluster(samples)

    centroid_pca = pca.fit_transform(est_centroid)

    # plotting:
    plotting(predict_labels, principleComp, clusters, centroid_pca)

    inertia = compute_inertia(samples, est_centroid, predict_labels, clusters)
    # losses
    plt.figure(figsize=(8,8), dpi=300)
    plt.plot(np.arange(0, num_iter, 1), loss)
    plt.xlabel('Iteration', fontsize=16)
    plt.ylabel('Cost')
    plt.xticks(np.arange(0, num_iter, 1))
    plt.title(f'Kmeans: Cost in learning | Number of clusters={clusters}')
    plt.savefig('./figs/Kmeans_loss.png', bbox_inches = 'tight', pad_inches = 0)
    plt.close()

    return inertia

if __name__ == '__main__':
    main()

ext: py

reason: import numpy


text:
import os
import json
import requests
import tensorflow as tf
import numpy as np
import sys
import feedparser
import time
from datetime import datetime, timedelta
import requests
import base64
from working.fakedata import create_authors, get_random_school, get_random_department, get_date
from ttp import ttp

sys.path.append('../')
from lm.modeling import GroverConfig, sample
from sample.encoder import get_encoder, _tokenize_article_pieces, extract_generated_target
import random

def generate_html_from_article_text(article_title, article_text, author_list, publish_date, university, department, image_url=None, caption = ""):
    # parse the article text
    p = ttp.Parser()
    result = p.parse(article_text)
    article_text = result.html
    image = None

    #  split article on newlines
    lines = article_text.split("\n")

    new_lines = []
    for line in lines:
        new_lines.append(line)

    #  add paragraph between each new line
    article_text = "
".join(new_lines)

    print(article_text)

    # if an image url exists, add it to the returned html
    #

    #
    #
Fig.1 - Trulli, Puglia, Italy.

    #

    if image_url is not None:
        image = "

" + caption + "

"

    #  create html from article text

    # body { margin: 20px; text-align: center; } h1 {color: green; } img { float: left; margin: 5px; } p { text-align: justify; font-size: 25px; }
    #

    style = " hr.solid { border-top: 3px solid #bbb;} figure { display: inline-block; border: 0px dotted gray; margin: 20px;text-align: center; } figure img { vertical-align: top; }"
    article_title = "
" + article_title + "
\n"
    authors = "
Authors: "
    for author in author_list:
        authors += author + " "
    authors += "
\n"
    publish_date = "Published Date: " + publish_date + "
\n"
    university = "
" + university + "
\n"
    department = department + "
\n"
    abstract = ""
    # abstract = "
Abstract
\n
" + article_summary + "
\n"
    sep = "
\n"
    body = article_title + authors + publish_date + sep + university + department + sep + abstract + "

" + article_text
    article_html = "" + body + image + ""

    # return the article as html
    return article_html

def generate_article_attribute(sess, encoder, tokens, probs, article, target='article'):

    # Tokenize the raw article text from sample grover encoder
    article_pieces = _tokenize_article_pieces(encoder, article)

    # Grab the article elements the model careas about - domain, date, title, etc.
    context_formatted = []
    for key in ['domain', 'date', 'authors', 'title', 'article']:
        if key != target:
            context_formatted.extend(article_pieces.pop(key, []))

    # Start formatting the tokens in the way the model expects them, starting with
    # which article attribute we want to generate.
    context_formatted.append(encoder.__dict__['begin_{}'.format(target)])
    # Tell the model which special tokens (such as the end token) aren't part of the text
    ignore_ids_np = np.array(encoder.special_tokens_onehot)
    ignore_ids_np[encoder.__dict__['end_{}'.format(target)]] = 0

    # We are only going to generate one article attribute with a fixed
    # top_ps cut-off of 95%. This simple example isn't processing in batches.
    gens = []
    article['top_ps'] = [0.95]

    # Run the input through the TensorFlow model and grab the generated output
    tokens_out, probs_out = sess.run(
        [tokens, probs],
        feed_dict={
            # Pass real values for the inputs that the
            # model needs to be able to run.
            initial_context: [context_formatted],
            eos_token: encoder.__dict__['end_{}'.format(target)],
            ignore_ids: ignore_ids_np,
            p_for_topp: np.array([0.95]),
        }
    )

    # The model is done! Grab the results it generated and format the results into normal text.
    for t_i, p_i in zip(tokens_out, probs_out):
        extraction = extract_generated_target(output_tokens=t_i, encoder=encoder, target=target)
        gens.append(extraction['extraction'])

    # Return the generated text.
    return gens[-1]

def create_fake(article, sess, encoder, tokens, probs, count):
    print(f"Building article from headline '{article['title']}'")
    # generate fake text based on the original paper
    article['text'] = generate_article_attribute(sess, encoder, tokens, probs, article, target="article")
    # Generate a fake title that fits the generated paper text
    article['title'] = generate_article_attribute(sess, encoder, tokens, probs, article, target="title")
    # Generate a fake abstract that fits the generated paper text
    # article['summary'] = generate_article_attribute(sess, encoder, tokens, probs, article, target="title")

    article_title = article['title']
    article_text = article['text']

    #
    article_image_url = "http://mnicosia.tech/images/samples_5_" + str(count) + ".png"
    # article_summary = article['summary']

    authors = create_authors()
    university = get_random_school()
    department = get_random_department()
    publish_date = get_date(365 * 2, 365 * 8)
    # caption = TODO
    caption = "Test Caption"
    article_text = generate_html_from_article_text(
        article_title, article_text, authors, publish_date, university, department, article_image_url, caption)

    print(f" - Generated fake article titled '{article_title}'")
    article_title = article_title.replace(" ", "-")
    filename = "/content/gdrive/My Drive/Articles/generated_fake-" + str(count) + ".html"
    with open(filename, 'w' ) as f:
        f.write(article_text)
    count += 1

model_type = "mega"

model_dir = os.path.join('/content/gdrive/MyDrive/grover-fork2/grover/models', model_type)
if not os.path.exists(model_dir):
    os.makedirs(model_dir)

for ext in ['data-00000-of-00001', 'index', 'meta']:
    r = requests.get(f'https://storage.googleapis.com/grover-models/{model_type}/model.ckpt.{ext}', stream=True)
    with open(os.path.join(model_dir, f'model.ckpt.{ext}'), 'wb') as f:
        file_size = int(r.headers["content-length"])
        if file_size < 1000:
            raise ValueError("File doesn't exist? idk")
        chunk_size = 1000
        for chunk in r.iter_content(chunk_size=chunk_size):
            f.write(chunk)
    print(f"Just downloaded {model_type}/model.ckpt.{ext}!", flush=True)

articles = list()

for filename in os.listdir("/content/gdrive/MyDrive/grover-fork2/grover/working/inputs/json"):
    fn = os.path.join("/content/gdrive/MyDrive/grover-fork2/grover/working/inputs/json/", filename)
    # checking if it is a file
    if os.path.isfile(fn) and ".jsonl" not in fn:
        try:
            f = open(fn)
            article = json.load(f)
            articles.append(article)
        except:
            print("unable to parse json file " + fn)
        finally:
            f.close()

# Load the pre-trained "huge" Grover model with 1.5 billion params
model_config_fn = '/content/gdrive/MyDrive/grover-fork2/grover/lm/configs/mega.json'
model_ckpt = '/content/gdrive/MyDrive/grover-fork2/grover/models/mega/model.ckpt'
encoder = get_encoder()
news_config = GroverConfig.from_json_file(model_config_fn)

# Set up TensorFlow session to make predictions
tf_config = tf.ConfigProto(allow_soft_placement=True)

with tf.Session(config=tf_config, graph=tf.Graph()) as sess:
    # Create the placehodler TensorFlow input variables needed to feed data to Grover model
    # to make new predictions.
    initial_context = tf.placeholder(tf.int32, [1, None])
    p_for_topp = tf.placeholder(tf.float32, [1])
    eos_token = tf.placeholder(tf.int32, [])
    ignore_ids = tf.placeholder(tf.bool, [news_config.vocab_size])

    # Load the model config to get it set up to match the pre-trained model weights
    tokens, probs = sample(
        news_config=news_config,
        initial_context=initial_context,
        eos_token=eos_token,
        ignore_ids=ignore_ids,
        p_for_topp=p_for_topp,
        do_topk=False
    )

    # Restore the pre-trained Grover 'huge' model weights
    saver = tf.train.Saver()
    saver.restore(sess, model_ckpt)

    fakes_made = 0
    paper_index = 0
    # process each research paper to make fake papers
    for article in articles:
        paper_index += 1
        #  create first
        create_fake(article, sess, encoder, tokens, probs, fakes_made)
        fakes_made += 1
        #  create second
        create_fake(article, sess, encoder, tokens, probs, fakes_made)
        fakes_made += 1
        if paper_index < 104:
            #  for first 104 articles create third
            continue
        # create third
        create_fake(article, sess, encoder, tokens, probs, fakes_made)
        fakes_made += 1


text:
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""custom callbacks for ema and loss"""
from copy import deepcopy

import numpy as np
from mindspore.train.callback import Callback
from mindspore.common.parameter import Parameter
from mindspore.train.serialization import save_checkpoint
from mindspore.nn import Loss, Top1CategoricalAccuracy, Top5CategoricalAccuracy
from mindspore.train.model import Model
from mindspore import Tensor

def load_nparray_into_net(net, array_dict):
    """
    Loads dictionary of numpy arrays into network.

    Args:
        net (Cell): Cell network.
        array_dict (dict): dictionary of numpy array format model weights.
    """
    param_not_load = []
    for _, param in net.parameters_and_names():
        if param.name in array_dict:
            new_param = array_dict[param.name]
            param.set_data(Parameter(Tensor(deepcopy(new_param)), name=param.name))
        else:
            param_not_load.append(param.name)
    return param_not_load

class EmaEvalCallBack(Callback):
    """
    Call back that will evaluate the model and save model checkpoint at
    the end of training epoch.

    Args:
        network: tinynet network instance.
        ema_network: step-wise exponential moving average of network.
        eval_dataset: the evaluation daatset.
        decay (float): ema decay.
        save_epoch (int): defines how often to save checkpoint.
        dataset_sink_mode (bool): whether to use data sink mode.
        start_epoch (int): which epoch to start/resume training.
    """

    def __init__(self, network, ema_network, eval_dataset, loss_fn, decay=0.999,
                 save_epoch=1, dataset_sink_mode=True, start_epoch=0):
        self.network = network
        self.ema_network = ema_network
        self.eval_dataset = eval_dataset
        self.loss_fn = loss_fn
        self.decay = decay
        self.save_epoch = save_epoch
        self.shadow = {}
        self.ema_accuracy = {}

        self.best_ema_accuracy = 0
        self.best_accuracy = 0
        self.best_ema_epoch = 0
        self.best_epoch = 0
        self._start_epoch = start_epoch
        self.eval_metrics = {'Validation-Loss': Loss(),
                             'Top1-Acc': Top1CategoricalAccuracy(),
                             'Top5-Acc': Top5CategoricalAccuracy()}
        self.dataset_sink_mode = dataset_sink_mode

    def begin(self, run_context):
        """Initialize the EMA parameters """
        for _, param in self.network.parameters_and_names():
            self.shadow[param.name] = deepcopy(param.data.asnumpy())

    def step_end(self, run_context):
        """Update the EMA parameters"""
        for _, param in self.network.parameters_and_names():
            new_average = (1.0 - self.decay) * param.data.asnumpy().copy() + \
                self.decay * self.shadow[param.name]
            self.shadow[param.name] = new_average

    def epoch_end(self, run_context):
        """evaluate the model and ema-model at the end of each epoch"""
        cb_params = run_context.original_args()
        cur_epoch = cb_params.cur_epoch_num + self._start_epoch - 1

        save_ckpt = (cur_epoch % self.save_epoch == 0)
        load_nparray_into_net(self.ema_network, self.shadow)
        model = Model(self.network, loss_fn=self.loss_fn, metrics=self.eval_metrics)
        model_ema = Model(self.ema_network, loss_fn=self.loss_fn,
                          metrics=self.eval_metrics)
        acc = model.eval(
            self.eval_dataset, dataset_sink_mode=self.dataset_sink_mode)
        ema_acc = model_ema.eval(
            self.eval_dataset, dataset_sink_mode=self.dataset_sink_mode)
        print("Model Accuracy:", acc)
        print("EMA-Model Accuracy:", ema_acc)

        output = [{"name": k, "data": Tensor(v)}
                  for k, v in self.shadow.items()]
        self.ema_accuracy[cur_epoch] = ema_acc["Top1-Acc"]
        if self.best_ema_accuracy < ema_acc["Top1-Acc"]:
            self.best_ema_accuracy = ema_acc["Top1-Acc"]
            self.best_ema_epoch = cur_epoch
            save_checkpoint(output, "ema_best.ckpt")

        if self.best_accuracy < acc["Top1-Acc"]:
            self.best_accuracy = acc["Top1-Acc"]
            self.best_epoch = cur_epoch

        print("Best Model Accuracy: %s, at epoch %s" %
              (self.best_accuracy, self.best_epoch))
        print("Best EMA-Model Accuracy: %s, at epoch %s" %
              (self.best_ema_accuracy, self.best_ema_epoch))

        if save_ckpt:
            # Save the ema_model checkpoints
            ckpt = "{}-{}.ckpt".format("ema", cur_epoch)
            save_checkpoint(output, ckpt)
            save_checkpoint(output, "ema_last.ckpt")

            # Save the model checkpoints
            save_checkpoint(cb_params.train_network, "last.ckpt")

        print("Top 10 EMA-Model Accuracies: ")
        count = 0
        for epoch in sorted(self.ema_accuracy, key=self.ema_accuracy.get,
                            reverse=True):
            if count == 10:
                break
            print("epoch: %s, Top-1: %s)" % (epoch, self.ema_accuracy[epoch]))
            count += 1

class LossMonitor(Callback):
    """
    Monitor the loss in training.

    If the loss is NAN or INF, it will terminate training.

    Note:
        If per_print_times is 0, do not print loss.

    Args:
        lr_array (numpy.array): scheduled learning rate.
        total_epochs (int): Total number of epochs for training.
        per_print_times (int): Print the loss every time. Default: 1.
        start_epoch (int): which epoch to start, used when resume from a
        certain epoch.

    Raises:
        ValueError: If print_step is not an integer or less than zero.
    """

    def __init__(self, lr_array, total_epochs, per_print_times=1, start_epoch=0):
        super(LossMonitor, self).__init__()
        if not isinstance(per_print_times, int) or per_print_times < 0:
            raise ValueError("print_step must be int and >= 0.")
        self._per_print_times = per_print_times
        self._lr_array = lr_array
        self._total_epochs = total_epochs
        self._start_epoch = start_epoch

    def step_end(self, run_context):
        """log epoch, step, loss and learning rate"""
        cb_params = run_context.original_args()
        loss = cb_params.net_outputs
        cur_epoch_num = cb_params.cur_epoch_num + self._start_epoch - 1
        if isinstance(loss, (tuple, list)):
            if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray):
                loss = loss[0]

        if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray):
            loss = np.mean(loss.asnumpy())
        global_step = cb_params.cur_step_num - 1
        cur_step_in_epoch = global_step % cb_params.batch_num + 1

        if isinstance(loss, float) and (np.isnan(loss) or np.isinf(loss)):
            raise ValueError("epoch: {} step: {}. Invalid loss, terminating training.".format(
                cur_epoch_num, cur_step_in_epoch))

        if self._per_print_times != 0 and cur_step_in_epoch % self._per_print_times == 0:
            print("epoch: %s/%s, step: %s/%s, loss is %s, learning rate: %s"
                  % (cur_epoch_num, self._total_epochs, cur_step_in_epoch,
                     cb_params.batch_num, loss, self._lr_array[global_step]),
                  flush=True)

max_stars_repo_name: GuoSuiming/mindspore


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Apr  2 00:11:20 2015

@author: hoseung
"""

#sim_dir = ['C29195', 'C01605_hi/', 'C04466/', 'G2/kisti'][3]
#base_dir = path.join(base, '', sim_dir, '')
base_dir = '/home/hoseung/Work/data/29172/'

nout=187

import matplotlib.pyplot as plt

fig = plt.figure()
ax1 = fig.add_subplot(111)

import pickle
snout = str(nout).zfill(3)
fin = base_dir + snout + 'map.pickle'
with open(fin, 'rb') as f:
    img = pickle.load(f)

ptimg = img.ptden2d
fout = base_dir + snout + "dmmap_" + ptimg.proj + ".png"
img.ptden2d.plot_2d_den(save=False, show=False, vmin=1e13, vmax=1e20, dpi=200, axes=ax1)
#%%

import tree
import load
import numpy as  np
import utils.sampling as smp

#s = load.sim.Sim(nout, base_dir)
info = load.info.Info(nout=nout, base=base_dir, load=True)
hall = tree.halomodule.Halo(nout=nout, base=base_dir, halofinder="HM", info=info)
hall.load()

i_center = np.where(hall.data['np'] == max(hall.data['np']))
h = tree.halomodule.Halo()
h.derive_from(hall, [i_center])

region = smp.set_region(xc=h.data.x, yc=h.data.y, zc=h.data.z, radius = h.data.rvir * 2)

#%%
from draw import pp
npix = 400

ind = np.where(hall.data.mvir > 5e11)
h_sub = tree.halomodule.Halo()
h_sub.derive_from(hall, ind)
#x = hall.data.x#[ind]
#y = hall.data.y#[ind]
#r = hall.data.rvir#[ind]
#pp.circle_scatter(ax1, x*npix, y*npix, r*30, facecolor='none', edgecolor='b', label='555')

#ax1.set_xlim(right=npix).
#ax1.set_ylim(top=npix)
pp.pp_halo(h_sub, npix, region=img.ptden2d.region, axes=ax1, rscale=40, name=True)

plt.show()

#%%

max_stars_repo_name: Hoseung/pyRamAn

max_issues_repo_name: Hoseung/pyRamAn

max_forks_repo_name: Hoseung/pyRamAn


text:
from scipy.stats import t

def P_value(population_list,confidence_level):
    degree_of_freedom = len(population_list) - 1
    alpha = (1 - confidence_level) / 2
    my_t = t.ppf(confidence_level, degree_of_freedom)
    my_p = 1 - t.cdf(my_t, degree_of_freedom)
    return my_p

reason: from scipy


text:
import copy
import os
import os.path as osp
import shutil

import chainer
from chainercv.utils import apply_to_iterator
import cv2
import fcn
import numpy as np
import six

import chainer_mask_rcnn as mrcnn

class InstanceSegmentationVisReport(chainer.training.extensions.Evaluator):

    def __init__(self, iterator, target, label_names,
                 file_name='visualizations/iteration=%08d.jpg',
                 shape=(3, 3), copy_latest=True):
        super(InstanceSegmentationVisReport, self).__init__(iterator, target)
        self.label_names = np.asarray(label_names)
        self.file_name = file_name
        self._shape = shape
        self._copy_latest = copy_latest

    def __call__(self, trainer):
        iterator = self._iterators['main']
        target = self._targets['main']

        if hasattr(iterator, 'reset'):
            iterator.reset()
            it = iterator
        else:
            it = copy.copy(iterator)

        in_values, out_values, rest_values = apply_to_iterator(
            target.predict, it)
        imgs, = in_values
        pred_bboxes, pred_masks, pred_labels, pred_scores = out_values[:4]
        gt_bboxes, gt_labels, gt_masks = rest_values[:3]

        # visualize
        vizs = []
        for img, gt_bbox, gt_label, gt_mask, \
            pred_bbox, pred_label, pred_mask, pred_score \
                in six.moves.zip(imgs, gt_bboxes, gt_labels, gt_masks,
                                 pred_bboxes, pred_labels, pred_masks,
                                 pred_scores):
            keep = pred_score >= 0.7
            pred_bbox = pred_bbox[keep]
            pred_label = pred_label[keep]
            pred_mask = pred_mask[keep]
            pred_score = pred_score[keep]

            # organize input
            img = img.transpose(1, 2, 0)  # CHW -> HWC

            n_fg_class = len(self.label_names)

            # bg mask
            gt_viz_bg = mrcnn.utils.draw_instance_bboxes(
                img, gt_bbox, gt_label, n_class=n_fg_class,
                masks=gt_mask == 0, captions=self.label_names[gt_label],
                bg_class=-1)
            # visible mask
            gt_viz_vis = mrcnn.utils.draw_instance_bboxes(
                img, gt_bbox, gt_label, n_class=n_fg_class,
                masks=gt_mask == 1, captions=self.label_names[gt_label],
                bg_class=-1)
            # invisible mask
            gt_viz_inv = mrcnn.utils.draw_instance_bboxes(
                img, gt_bbox, gt_label, n_class=n_fg_class,
                masks=gt_mask == 2, captions=self.label_names[gt_label],
                bg_class=-1)
            gt_viz = np.hstack([gt_viz_bg, gt_viz_vis, gt_viz_inv])

            captions = []
            for p_score, l_name in zip(pred_score,
                                       self.label_names[pred_label]):
                caption = '{:s} {:.1%}'.format(l_name, p_score)
                captions.append(caption)
            assert np.isin(np.unique(pred_mask), [-1, 0, 1, 2]).all()
            pred_viz_bg = mrcnn.utils.draw_instance_bboxes(
                img, pred_bbox, pred_label, n_class=n_fg_class,
                masks=pred_mask == 0, captions=captions, bg_class=-1)
            pred_viz_vis = mrcnn.utils.draw_instance_bboxes(
                img, pred_bbox, pred_label, n_class=n_fg_class,
                masks=pred_mask == 1, captions=captions, bg_class=-1)
            pred_viz_inv = mrcnn.utils.draw_instance_bboxes(
                img, pred_bbox, pred_label, n_class=n_fg_class,
                masks=pred_mask == 2, captions=captions, bg_class=-1)
            pred_viz = np.hstack([pred_viz_bg, pred_viz_vis, pred_viz_inv])

            viz = np.vstack([gt_viz, pred_viz])
            vizs.append(viz)
            if len(vizs) >= (self._shape[0] * self._shape[1]):
                break

        viz = fcn.utils.get_tile_image(vizs, tile_shape=self._shape)
        file_name = osp.join(
            trainer.out, self.file_name % trainer.updater.iteration)
        try:
            os.makedirs(osp.dirname(file_name))
        except OSError:
            pass
        cv2.imwrite(file_name, viz[:, :, ::-1])

        if self._copy_latest:
            shutil.copy(file_name,
                        osp.join(osp.dirname(file_name), 'latest.jpg'))


text:
import sys
sys.path.append("..")

import tensorflow as tf
import numpy as np
import os
import SNN

input = tf.placeholder(tf.float32)
input_exp = tf.exp(input)
groundtruth = tf.placeholder(tf.float32)

layer_in = SNN.SNNLayer(784,800)
layer_out = SNN.SNNLayer(800,10)

layerin_out = layer_in.forward(input_exp)
layerout_out = layer_out.forward(layerin_out)

layerout_groundtruth = tf.concat([layerout_out,groundtruth],1)
loss = tf.reduce_mean(tf.map_fn(SNN.loss_func,layerout_groundtruth))

wsc = layer_in.w_sum_cost() + layer_out.w_sum_cost()
l2c = layer_in.l2_cost() + layer_out.l2_cost()

K = 100
K2 = 1e-3
learning_rate = 1e-4
TRAINING_BATCH = 10

SAVE_PATH = os.getcwd() + '/weight_mnist'

cost = loss + K*wsc + K2*l2c

opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = opt.minimize(cost)

config = tf.ConfigProto(
    device_count={'GPU': 1}
)
config.gpu_options.allow_growth = True
sess = tf.Session()
sess.run(tf.global_variables_initializer())

scale = 2
mnist = SNN.MnistData(path=["MNIST/train-images-idx3-ubyte","MNIST/train-labels-idx1-ubyte"])

print('training started')
step = 1
while(True):
    xs, ys = mnist.next_batch(TRAINING_BATCH, shuffle=True)
    xs = scale*(-xs + 1)
    [out,c,_] = sess.run([layerout_out,cost,train_op],{input:xs,groundtruth:ys})
    if step % 20 == 1:
        print('step '+repr(step) +', cost='+repr(c))
        w1 = sess.run(layer_in.weight)
        w2 = sess.run(layer_out.weight)
        np.save(SAVE_PATH + '1', w1)
        np.save(SAVE_PATH + '2', w2)
    step = step + 1


text:
# -*- coding: utf-8 -*-
"""
The MIT License (MIT)
Copyright (c) 2014 Leonardo Kewitz

Exemplo de leitura e processamento de um arquivo CSV no Python. Neste exemplo
um arquivo obtido no analisador de espéctro é lido, convertido e plotado.
"""

# Importa bibliotecas necessárias.
from numpy import *
import matplotlib.pyplot as plt

# Com o arquivo CSV aberto na variável "f"...
with open("./src/FSP_RADIO.CSV") as f:
    # Leia todas as linhas e as separe nos caracteres ";"
    lines = [l.strip().split(';') for l in f.readlines()]
    # Transforma essas linhas num array Numpy, para trabalharmos depois.
    lines = array(lines)

# Cria um array chamado "famp" convertendo as 2 primeiras colunas de lines para
# ponto flutuante. No caso teremos um array (501,2) onde as colunas são respect-
# ivamente frequência e amplitude do sinal capturado no analisador de espectro.
famp = array([[float(a[0]), float(a[1])] for a in lines[:,:2]])

# Plota um gráfico frequência x amplitude.
plt.plot(famp[:,0], famp[:,1])
# Adiciona grade no gráfico.
plt.grid()

# Vamos criar o eixo X a partir de 7 pontos equidistantes nas frequências.
ticks = famp[::len(lines)//7,0]
# Vamos formatar para MHz
x = ["{0:.2f}".format(t/1E6) for t in ticks]
# Seta o novo eixo X.
plt.xticks(ticks, x)
# Adiciona nome para os eixos.
plt.xlabel("Frequência [MHz]")
plt.ylabel("Potência [dBm]")

# Destaca algumas frequências conhecidas...
plt.axvline(107.1E6, label="FURB", color="orange")
plt.axvline(102.7E6, label="Atl.", color="red")
plt.axvline(105.3E6, label="Diplo.", color="blue")

# Plota legenda.
plt.legend(fontsize="medium", fancybox=True, framealpha=.5, ncol=3, loc=9)

reason: from numpy


text:
from scipy.stats import gaussian_kde
import pandas as pd
import numpy as np
import copy
import joypy
from matplotlib import cm
from math import floor, ceil
import matplotlib.pyplot as plt
import pylab as plot
import itertools
import seaborn as sns

class DopplerRadar(object):

    def __init__(self, data_frame=None, drep_readings=[-0.3,0.1,0.4,0.75, 1,1.4,1.9,2.1,2.3,2.65,3,3.25,3.8], intv_width=float(0.25), number_samples=int(10)):
        # stats.gaussian_kde.__init__(self,data_frame,drep_readings,intv_width)

        self.data_frame = data_frame
        self.drep_readings = drep_readings
        self.intv_width = float(intv_width)
        self.number_samples = int(number_samples)

    def sample_drep_gep(self):
        sample_drep_dict = {}
        for i in self.drep_readings:
            j = self.drep_readings.index(i)
            x = np.array(self.data_frame[(self.data_frame[self.data_frame.columns[1]] < (i + self.intv_width)) & (
                    self.data_frame[self.data_frame.columns[1]] > (i - self.intv_width))][self.data_frame.columns[0]])
            my_kde = gaussian_kde(x)
            sample = my_kde.resample(self.number_samples)
            sample = sample.tolist()[0]
            sample_drep_dict[j] = sample

        sample_gep_lst = []
        for k in range(self.number_samples):
            sample = []
            for i in range(len(self.drep_readings)):
                sample.append(sample_drep_dict[i][k])
            sample_gep_lst.append(sample)
        return sample_drep_dict, sample_gep_lst

    def get_kde(self):
        x_val=[]
        my_kde_val=[]
        for i in self.drep_readings:
            j = self.drep_readings.index(i)
            x = np.array(self.data_frame[(self.data_frame[self.data_frame.columns[1]] < (i + self.intv_width)) & (
                    self.data_frame[self.data_frame.columns[1]] > (i - self.intv_width))][
                             self.data_frame.columns[0]])
            my_kde = gaussian_kde(x)
            x_val.append(x)
            my_kde_val.append(my_kde)
        return x_val, my_kde_val

    def get_percentiles(self):
        GEP_simulations,_=self.sample_drep_gep()

        per85 = []
        per50 = []
        per25 = []
        dict_len = len(GEP_simulations)
        for i in range(dict_len):
            GEP_simulations[i].sort()
            index85 = floor(len(GEP_simulations[i]) * 0.85)
            index50 = floor(len(GEP_simulations[i]) * 0.5)
            index25 = floor(len(GEP_simulations[i]) * 0.25)
            per85.append(GEP_simulations[i][index85])
            per50.append(GEP_simulations[i][index50])
            per25.append(GEP_simulations[i][index25])
        df_percent = pd.DataFrame([per25, per50, per85], columns=self.drep_readings, index=['25th Percentile', '50th Percentile', '85th Percentile'])
        return df_percent, per85, per50, per25

    def plot_percentiles(self, per25=None, per50=None,per85=None,figsize=(12,8),xlabel="DREP Daily Readings (StDev)",
                         ylabel="GEP Realizations (StDev)", xlabel_fontsize=int(14),ylabel_fontsize=int(14), title=None,
                         title_fontsize=int(18),legend_location='upper right', label=None,s=2000, lw=3, gep_marker='o',tick_param_labelsize=12,
                         marker_85='^',marker_50='s', marker_25='D',color_85='r',color_50='b',color_25='y',legend_fontsize=16, gep_markersize=int(8)):

        self.per25=per25
        self.per50=per50
        self.per85=per85
        #self.Generated_GEP_strings=Generated_GEP_strings
        self.figsize=figsize
        self.xlabel = xlabel
        self.ylabel = ylabel
        self.title = title
        self.xlabel_fontsize = xlabel_fontsize
        self.ylabel_fontsize = ylabel_fontsize
        self.title_fontsize = title_fontsize
        self.label = label
        self.lw = lw
        self.gep_marker = gep_marker
        self.marker_85=marker_85
        self.marker_25=marker_25
        self.marker_50=marker_50
        self.color_85=color_85
        self.color_50=color_50
        self.color_25=color_25
        self.gep_markersize = gep_markersize
        self.s=s
        self.legend_location=legend_location
        self.legend_fontsize=legend_fontsize
        self.tick_param_labelsize=tick_param_labelsize

        _,Generated_GEP_strings=self.sample_drep_gep()

        plt.figure(figsize=self.figsize)
        for i in range(self.number_samples):
            plt.scatter(self.drep_readings, Generated_GEP_strings[i],label=None, marker=self.gep_marker, s=self.gep_markersize)
            plt.xlabel(xlabel=self.xlabel, fontsize=self.xlabel_fontsize)
            plt.ylabel(ylabel=self.ylabel, fontsize=self.ylabel_fontsize)
            #plt.title(title=self.title, fontsize=self.title_fontsize)

        plt.scatter(self.drep_readings, self.per85, marker=self.marker_85, s=self.s, color=self.color_85, lw=self.lw, label='85th Percentile')
        plt.scatter(self.drep_readings, self.per50, marker=self.marker_50, s=self.s, color=self.color_50, lw=self.lw, label='50th Percentile')
        plt.scatter(self.drep_readings, self.per25, marker=self.marker_25, s=self.s, color=self.color_25, lw=self.lw, label='25th Percentile')
        plt.xticks(self.drep_readings)
        plt.legend(loc=self.legend_location, fontsize=self.legend_fontsize)
        plt.tick_params(labelsize=self.tick_param_labelsize)

    def get_maxGEP(self):

        sample_drep_dict, sample_gep_lst=self.sample_drep_gep()
        Max_gep = []
        for item in sample_gep_lst:
            Max_gep.append(max(item))
        return Max_gep

    def plot_maxGEP_Dist(self,figsize=(8,6),domain=np.linspace(-1.5,6,1000),title=None,title_fontsize=14,alpha=0.3, axis_on_off='on',color='red', tick_parm_labelsize=14):

        self.figsize=figsize
        self.domain=domain
        self.alpha=alpha
        self.color=color
        self.tick_param_labelsize=tick_parm_labelsize
        self.axis_on_off=axis_on_off
        self.title=title
        self.title_fontsize=title_fontsize

        Max_gep=self.get_maxGEP()

        fig = plt.figure(figsize=self.figsize)
        ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
        ax.set_title(self.title, fontsize=self.title_fontsize)
        my_kde = gaussian_kde(Max_gep)
        ax.plot(self.domain, my_kde(self.domain))
        ax.fill_between(self.domain, my_kde(self.domain), alpha=self.alpha, color=self.color)
        ax.plot(Max_gep, np.full_like(Max_gep, -0.01), '|k', markeredgewidth=1)
        ax.axis(self.axis_on_off)
        plt.tick_params(labelsize=self.tick_param_labelsize)

class DopplerSensitivityAnalysis(object):

    def __init__(self,data_frame=None):

        self.data_frame=data_frame

    def plot_GepDist_DiscScale(self, DiscScale_list=[0.05,0.1,0.125,0.25],drep_pt=2.5,num_rows=1,num_columns=4, x_d=np.linspace(-2,7,1000),
                               alpha=0.5, markeredgewidth=1, figsize=(12,4), legend_loc=1, frameon=False, legend_fontsize=9,legend_handle_length=0.2):
        self.DiscScale_list = DiscScale_list
        self.drep_pt = drep_pt
        self.num_rows=num_rows
        self.num_columns=num_columns
        self.x_d=x_d
        self.alpha=alpha
        self.markeredgewidth=markeredgewidth
        self.legend_loc=legend_loc
        self.figsize=figsize
        self.frameon=frameon
        self.legend_fontsize=legend_fontsize
        self.legend_handle_length=legend_handle_length

        sub_gep = {}
        for j,i in enumerate(self.DiscScale_list):
            x = np.array(self.data_frame[(self.data_frame[self.data_frame.columns[1]] < ( self.drep_pt+i)) & (self.data_frame[self.data_frame.columns[1]] > (self.drep_pt-i))][self.data_frame.columns[0]])
            sub_gep[j]=x

        fig, ax = plt.subplots(self.num_rows, self.num_columns, figsize=self.figsize)

        color = iter(cm.rainbow(np.linspace(0, 1, len(self.DiscScale_list))))

        for j in range(self.num_columns):
            c = next(color)
            # ax[i,j].hist(sub_gep[ct],bins=30, label="Width {}".format(width_lst[ct]), density=True, color=c, alpha=0.5)
            mean=np.round(sub_gep[j].mean(),3)
            stdev=np.round(sub_gep[j].std(),3)
            n=len(sub_gep[j])
            my_kde = gaussian_kde(sub_gep[j])
            ax[j].plot(self.x_d, my_kde(self.x_d), color='k')
            ax[j].fill_between(self.x_d, my_kde(self.x_d), alpha=self.alpha, color=c, label="$\epsilon$= {} \n Mean={}\n StDev={}\n n= {} ".format(self.DiscScale_list[j], mean,stdev,n))
            ax[j].plot(sub_gep[j], np.full_like(sub_gep[j], -0.01), '|k', markeredgewidth=self.markeredgewidth)
            ax[j].legend(loc=self.legend_loc,frameon=self.frameon)
            ax[j].title.set_text("DREP reading: {}".format(self.drep_pt))
        params = {'legend.fontsize': self.legend_fontsize,'legend.handlelength': self.legend_handle_length}
        plot.rcParams.update(params)

class Runoff(DopplerRadar):
    def __init__(self,data_frame=None, drep_readings=[-0.3,0.1,0.4,0.75, 1,1.4,1.9,2.1,2.3,2.65,3,3.25,3.8], intv_width=float(0.25), number_samples=int(10),
                 CN=60,gauge_mean=23.88,gauge_stdev=28.19):
        DopplerRadar.__init__(self,data_frame, drep_readings, intv_width, number_samples)
        #self.data_frame = data_frame
        #self.Generated_GEP = Generated_GEP
        self.CN = CN
        #self.drep_readings=drep_readings
        self.gauge_mean=gauge_mean
        self.gauge_stdev=gauge_stdev

    def compute_runoff(self):

        _, Generated_GEP = self.sample_drep_gep()
        sorted_gep = copy.deepcopy(Generated_GEP)
        for item in sorted_gep:
            item.sort()
        P = [(np.multiply(i, self.gauge_stdev) + self.gauge_mean)/25.4 for i in sorted_gep]

        for lst in P:
            for i, val in enumerate(lst):
                lst[i] = max(val, 0)
        S = np.full_like(P, ((1000 / self.CN) - 10))  # fills array S with same length as P above
        Q = (np.square(P - 0.2 * S)) / (P + 0.8 * S)  # SCS run off formula

        return P, Q

    def plot_runoff(self,x_axis=12,y_axis=8,xlabel="Time in Days", ylabel="Run Off Estimates (cm)",xlabel_fontsize=int(14),
                    ylabel_fontsize=int(14), title="DualPol Run Off Estimates Over a Thirteen Day Period",
                    title_fontsize=int(22),label=None, lw=float(0.5), marker='o', markersize=int(8)):

        self.x_axis=x_axis
        self.y_axis=y_axis
        self.xlabel=xlabel
        self.ylabel=ylabel
        self.title=title
        self.xlabel_fontsize=xlabel_fontsize
        self.ylabel_fontsize=ylabel_fontsize
        self.title_fontsize=title_fontsize
        self.label=label
        self.lw=lw
        self.marker=marker
        self.markersize=markersize

        _, Generated_GEP = self.sample_drep_gep()

        _, Q = self.compute_runoff()

        drep_inch=[(np.multiply(i,self.gauge_stdev) + self.gauge_mean)/25.4 for i in self.drep_readings]
        drep_inch=[round(i,2) for i in drep_inch]

        plt.figure(figsize=(self.x_axis, self.y_axis))
        for i in range(len(Generated_GEP)):
            plt.plot(drep_inch, Q[i], label=self.label, lw=self.lw, marker=self.marker, markersize=self.markersize)
            plt.xlabel(xlabel=self.xlabel, fontsize=self.xlabel_fontsize)
            plt.ylabel(ylabel=self.ylabel, fontsize=self.ylabel_fontsize)
            plt.title(label=self.title, fontsize=self.title_fontsize)
        plt.xticks(drep_inch)

    def plot_RunoffvsGEP(self,figsize=(8,6), N=1,marker='o',markerfacecolor='r', xlabel='GEP (in)', ylabel='Runoff (in)',
                         xlabel_size=14,ylabel_size=14, title=None,title_size=18, tick_param_size=12):

        self.figsize=figsize
        self.N=N
        self.marker=marker
        self.markerfacecolor=markerfacecolor
        self.xlabel=xlabel
        self.ylabel=ylabel
        self.xlabel_size=xlabel_size
        self.ylabel_size=ylabel_size
        self.title=title
        self.title_size=title_size
        self.tick_param_size=tick_param_size

        P, Q = self.compute_runoff()

        plt.figure(figsize=self.figsize)
        for i in range(N):
            plt.scatter(P[i], Q[i], marker=self.marker)
            plt.xlabel(self.xlabel, fontsize=self.xlabel_size)
            plt.ylabel(self.ylabel, fontsize=self.ylabel_size)
            plt.title(self.title, fontsize=self.title_size)

        plt.tick_params(labelsize=self.tick_param_size)

class DopplerJoyPlot():

    def __init__(self, data_frame=None, drep_readings=[-0.3,0.1,0.4,0.75, 1,1.4,1.9,2.1,2.3,2.65,3,3.25,3.8],intv_width=float(0.125),kind="kde",
                      range_style='own', tails=0.2, yrot=22, ylabelsize=14, xlabelsize=14, linecolor='k',
                      overlap=5, linewidth=2, colormap=cm.gist_rainbow, fade=False, legend=False,
                      labels=[0.1,0.2].sort(), grid='both', figsize=(10, 10), alpha=0.5):
        self.data_frame = data_frame
        self.drep_readings = drep_readings
        self.intv_width=intv_width
        self.kind=kind
        self.range_style=range_style
        self.tails=tails
        self.yrot=yrot
        self.ylabelsize=ylabelsize
        self.xlabelsize=xlabelsize
        self.linecolor=linecolor
        self.overlap=overlap
        self.linewidth=linewidth
        self.colormap=colormap
        self.fade=fade
        self.legend=legend
        self.labels=labels
        self.grid=grid
        self.figsize=figsize
        self.alpha=alpha

    def data_frame_for_joyplot(self):
        df_lst=[]

        for i in self.drep_readings:
            x = list(self.data_frame[(self.data_frame[self.data_frame.columns[1]] < (i + self.intv_width)) & (
                    self.data_frame[self.data_frame.columns[1]] > (i - self.intv_width))][
                             self.data_frame.columns[0]])
            df = pd.DataFrame(np.array([[i for k in x], x]).T, columns=[self.data_frame.columns[1],self.data_frame.columns[0]])
            df_lst.append(df)
        return pd.concat(df_lst)

    def display_joy_plot(self):
        df_lst = []

        for i in self.drep_readings:
            x = list(self.data_frame[(self.data_frame[self.data_frame.columns[1]] < (i + self.intv_width)) & (
                    self.data_frame[self.data_frame.columns[1]] > (i - self.intv_width))][
                         self.data_frame.columns[0]])
            df = pd.DataFrame(np.array([[i for k in x], x]).T,
                              columns=[self.data_frame.columns[1], self.data_frame.columns[0]])
            df_lst.append(df)

        merged=pd.concat(df_lst)

        return joypy.joyplot(data=merged, by=self.data_frame.columns[1], column=self.data_frame.columns[0],
                             kind=self.kind,
                             range_style=self.range_style, tails=self.tails, yrot=self.yrot, ylabelsize=self.ylabelsize,
                             xlabelsize=self.xlabelsize, linecolor=self.linecolor,
                             overlap=self.overlap, linewidth=self.linewidth, colormap=self.colormap, fade=self.fade,
                             legend=self.legend,
                             labels=self.labels, grid=self.grid, figsize=self.figsize, alpha=self.alpha)

max_stars_repo_name: vbucaj/DopplerRadar

max_issues_repo_name: vbucaj/DopplerRadar

max_forks_repo_name: vbucaj/DopplerRadar

reason: import numpy,from scipy


text:
import numpy as np
from numpy.linalg import norm
from scipy.optimize import linear_sum_assignment
from sklearn.preprocessing import normalize

def puncture(new_t, cent):
    mui = cent/(-new_t)
    c = mui[mui>0]

    if len(c) == 0:
        print 'Warning: can\'t puncture'
        new_t = new_t + cent
        new_t[new_t < 0] = 0
        new_t = normalize(new_t, 'l1')
        return new_t

    c = c.min()

    return new_t*c + cent

def cosine(a, b):
    return np.dot(a, b)/(norm(a)*norm(b))

def matching_upd_j(cosac_topics_j, global_topics, topic_counts, J, gamma, tau1):

    K = global_topics.shape[0]

    param_cost = np.zeros((cosac_topics_j.shape[0], K))

    for i in range(global_topics.shape[0]):
        param_cost[:,i] = norm(global_topics[i] + tau1 * cosac_topics_j, axis=1) - norm(global_topics[i]) + \
                          np.log(min(topic_counts[i],10)) - np.log(J-min(topic_counts[i], 10))

    # Nonparametric Cost
    K = global_topics.shape[0]
    Kj = cosac_topics_j.shape[0]
    max_new = min(Kj, max(250-K, 1))
    nonparam_cost = np.ones((Kj, max_new))*tau1
    nonparam_cost -= np.log(np.arange(1, max_new+1))
    nonparam_cost += np.log(gamma/J)

    cost = np.hstack((param_cost, nonparam_cost))

    row_ind, col_ind = linear_sum_assignment(-cost)

    assignment_j = []
    new_K = K

    for k, i in zip(row_ind, col_ind):
        if i < K:
            global_topics[i] += tau1*cosac_topics_j[k]
            topic_counts[i] += 1
            assignment_j.append(i)
        # New Topic
        else:
            global_topics = np.vstack((global_topics, tau1*cosac_topics_j[k]))
            topic_counts += [1]
            assignment_j.append(new_K)
            new_K += 1

    return global_topics, topic_counts, assignment_j

def init_matching(cosac_topics, tau1, gamma):

    J = len(cosac_topics)
    group_order = sorted(range(J), key=lambda x: -cosac_topics[x].shape[0])

    global_topics = tau1*cosac_topics[group_order[0]]
    topic_counts = [1]*global_topics.shape[0]

    assignment = [[] for _ in range(J)]

    assignment[group_order[0]] = range(global_topics.shape[0])

    J_count = 1
    for j in group_order[1:]:
        J_count += 1
        global_topics, topic_counts, assignment_j = matching_upd_j(cosac_topics[j], global_topics, topic_counts,
                                                                   J=J_count, gamma=gamma, tau1=tau1)
        assignment[j] = assignment_j

    return global_topics, topic_counts, assignment

def matching_upd(cosac_topics, cent, tau1=3., gamma=1., it=3):
    """
    Input: list of cosac topic matrices for groups;
           global topic matrix;
           list of counts for topic popularity for groups;
           Bessel ratio
    Output: updated global topics and updated list of topic counts
    """

    # All topics should be centered at same point and normalized to unit ball times tau0
    cosac_topics = [normalize(cosac_topic_j - cent) for cosac_topic_j in cosac_topics]

    # Initialize
    global_topics, topic_counts, assignment = init_matching(cosac_topics, tau1, gamma)
    print 'Initialization objective is %f; number of topics is %d\n' % \
          (norm(global_topics, axis=1).sum(), global_topics.shape[0])

    J = len(cosac_topics)

    # Iterate over groups
    for iteration in range(it):
        random_order = np.random.permutation(J)
        for j in random_order:
            to_delete = []
            # Remove j
            Kj = len(assignment[j])
            for k, i in sorted(zip(range(Kj), assignment[j]), key=lambda x: -x[1]):
                topic_counts[i] -= 1
                if topic_counts[i] == 0:
                    del topic_counts[i]
                    to_delete.append(i)
                    for j_clean in range(J):
                        for idx, k_ind in enumerate(assignment[j_clean]):
                            if i < k_ind and j_clean != j:
                                assignment[j_clean][idx] -= 1
                            elif i == k_ind and j_clean != j:
                                print('Warning - weird unmatching')

                else:
                    global_topics[i] -= tau1*cosac_topics[j][k]

            global_topics = np.delete(global_topics, to_delete, axis=0)

            global_topics, topic_counts, assignment[j] = \
                matching_upd_j(cosac_topics[j], global_topics, topic_counts, J, gamma, tau1)

        print 'Iteration %d objective is %f; number of topics is %d\n' % \
              (iteration, norm(global_topics, axis=1).sum(), global_topics.shape[0])

    # Puncture & Uncenter
    for i in range(global_topics.shape[0]):
        global_topics[i] = puncture(global_topics[i], cent)

    return global_topics, topic_counts

max_stars_repo_name: moonfolk/SDDM

max_issues_repo_name: moonfolk/SDDM

max_forks_repo_name: moonfolk/SDDM

reason: import numpy,from numpy,from scipy


text:
"""
Throughout this file, the following notations are consistently used for variables:
    * `x` always refers to user histories, and will generally be a tensor of dimension
      num_interactions x num_users x compress_dim
    * `y` always refers to the output probabilities of the neural net, and will generally
      be of the dimensions num_interactions x num_users x num_questions
    * `t` will always refer to the "true" response of the student on the *next* question.
      This is used for training and testing purposes. It is generally of dimension
      num_interactions x num_users
    * `m` will always represent a *mask* on the truth values, since users have different
      history lengths. The dimension is usually num_interactions x num_users
    * `h` will always refer to the hidden layer
    * w_hh/w_xh/w_hy refer to weights with which neural network layers are convolved
    * w_bh/w_by are the bias weights added within the various relevant nonlinearities
"""
from __future__ import division

from collections import namedtuple
import logging
import pickle
import time

import numpy as np
import theano
from theano import tensor as T

from .common import Results
from .data.rnn import build_batches
from .irt.metrics import Metrics

_logger = logging.getLogger(__name__)

def build_grad_and_test_fns(sigmoid_fn=T.nnet.hard_sigmoid, recurrent=True, num_type='float32',
                            mask_type='int8', probability_clip=1e-12, compressed_output=False):
    """
    Actually build the Theano functions for the recurrent neural net. These are *independent*
    of the data passed into the neural net, so whatever it produces should be cached in the
    actual state of the class.

    :param function sigmoid_fn: A function that performs an element-wise sigmoid operation
        on a theano tensor. These will likely be one of
        - `T.nnet.sigmoid`
        - `T.nnet.hard_sigmoid`
        - `T.nnet.ultra_fast_sigmoid`
    :param bool recurrent: If True, use a recurrent architecture
    :param str num_type: A valid floating point numeric type to pass to Theano. Should
        be a numpy-esque name, but Theano requires the stringy version.
    :param str mask_type: A valid "boolean" type to pass to Theano. Should be a numpy-esque
        name for an integer type, but Theano requires the stringy version. Also note
        that Theano does *not* support bools currently.
    :param float probability_clip: In order to avoid nans when taking logs, we clip the
        predicted probabilities to lie in (probability_clip, 1 - probability_clip)
    :param bool compressed_output: Is the output of the RNN compressed? If so, we will run
        the final layer through a sigmoid. Else, we'll assume the final layer are already
        probabilities.
    :rtype: (function, function)
    :return: The main Theano functions. The first function is the "gradient" function,
        whose signature looks like::
            inputs: [x, y, t, m, h_0, w_hh, w_xh, w_hy, w_bh, w_by]
            outputs: [error, num_pred_correct, g_hh, g_xh, g_hy, g_bh, g_by]
        For a further explanation of these variable names see the class docstring.

        The second function is a simpler "test" function, that doesn't go through the
        trouble of computing gradients. Thus, its signature looks like::
            inputs: [x, y, t, m, h_0, w_hh, w_xh, w_hy, w_bh, w_by],
            outputs: [num_pred_correct, next_qn_prob]
    """

    ######################################################
    #                    Tensor input                    #
    ######################################################
    # (Compressed) one hot encoding for input x
    x = T.tensor3('x', dtype=num_type)            # (NUM INTERACTIONS x BATCH x COMPRESS)
    # Mask if uncompressed or "to dot with" if compressed
    # of the probabilities of y to only contain probabilities for the next question
    y = T.tensor3('y', dtype=num_type)            # (NUM INTERACTIONS x BATCH x NUM QN)
    # Correctness of the next question
    t = T.matrix('t', dtype=mask_type)            # (NUM INTERACTIONS x BATCH)
    # Mask for non-existing interactions within a rectangle
    m = T.matrix('m', dtype=mask_type)            # (NUM INTERACTIONS x BATCH)
    # Initial hidden state
    h_0 = T.matrix('h_0', dtype=num_type)         # (BATCH x HIDDEN)
    # Dropout mask for (compressed) input
    x_drop = T.vector('x_drop', dtype=mask_type)  # (COMPRESS)
    # Dropout mask for hidden layer
    h_drop = T.vector('h_drop', dtype=mask_type)  # (HIDDEN)

    ######################################################
    #                       Weights                      #
    ######################################################
    # Recurrent weight from layer H to layer H
    w_hh = T.matrix('w_hh', dtype=num_type)  # (HIDDEN x HIDDEN)
    # Input weights from layer X to layer H
    w_xh = T.matrix('w_xh', dtype=num_type)  # (COMPRESS x HIDDEN)
    # Output weights from layer H to layer Y
    w_hy = T.matrix('w_hy', dtype=num_type)  # (HIDDEN x NUM QN)
    # Bias weights from bias to layer H
    w_bh = T.vector('w_bh', dtype=num_type)  # (HIDDEN)
    # Bias weights from bias to layer Y
    w_by = T.vector('w_by', dtype=num_type)  # (NUM QN)

    # Recurrent function
    def step(x_t, y_t, h_tm1, x_drop, h_drop, w_hh, w_xh, w_hy, w_bh, w_by):
        """
        :param x_t: (Compressed) one hot encoding for input x for time t (BATCH x COMPRESS)
        :param y_t: Mask of the probabilities of y for time t            (BATCH x NUM QN)
        :param h_tm1: Previous hidden state for time t-1                 (BATCH x HIDDEN)
        :param w_hh: Recurrent weight from layer H to layer H            (HIDDEN x HIDDEN)
        :param w_xh: Input weights from layer X to layer H               (COMPRESS x HIDDEN)
        :param w_hy: Output weights from layer H to layer Y              (HIDDEN x NUM QN)
        :param w_bh: Bias weights from bias to layer H                   (HIDDEN)
        :param w_by: Bias weights from bias to layer Y                   (NUM QN)
        :return: h_t: Hidden state for time t
                 all_qn_prob_t: Probabilities for all questions for time t
                 next_qn_prob_t: Probabilities for next question for time t
        """
        # Broadcast over batches
        x_drop_batch = x_drop.dimshuffle(('x', 0))  # BATCH x COMPRESS
        h_drop_batch = h_drop.dimshuffle(('x', 0))  # BATCH x HIDDEN
        w_bh_batch = w_bh.dimshuffle(('x', 0))  # BATCH x HIDDEN
        w_by_batch = w_by.dimshuffle(('x', 0))  # BATCH x NUM QN

        # Dropout on inputs
        x_t_dropped = x_t * x_drop_batch

        if recurrent:
            # BATCH x HIDDEN
            h_t = T.tanh(T.dot(x_t_dropped, w_xh) + T.dot(h_tm1, w_hh) + w_bh_batch)
        else:
            # In this case, we don't care about the previous state, but Theano is a bit
            # smarter than we are and notices if we don't include w_hh. This trains up
            # fast enough as is, so let's just dot it with 0s to get rid of it.
            h_t = T.tanh(T.dot(x_t_dropped, w_xh) + T.dot(T.zeros_like(h_tm1), w_hh) + w_bh_batch)

        # Dropout on hidden layer
        h_t_dropped = h_t * h_drop_batch

        # Compute predicted pass rate for all questions, taking into account
        # whether we have compressed the output dimensions or not
        if compressed_output:
            # If compressed, run the final output through a sigmoid
            final_layer = T.dot(h_t, w_hy) + w_by_batch
            next_qn_prob_t = sigmoid_fn((final_layer * y_t).sum(axis=1))
        else:
            # If not compressed, mask probabilities of all questions except the next one
            all_qn_prob_t = sigmoid_fn(T.dot(h_t, w_hy) + w_by_batch)
            next_qn_prob_t = (all_qn_prob_t * y_t).sum(axis=1)

        return h_t_dropped, next_qn_prob_t

    # This is the scan function to apply the forward propagation for each of the time slice
    # For each time slice:
    #   A time slice of the x and y are passed in as arguments
    #   The previous output of the function is also passed in
    #       (the initial output is passed into outputs_info)
    #   The weights are also passed in but not sliced in time
    #
    # h: Hidden state for all time                               (NUM INTERACTIONS x BATCH x HIDDEN)
    # next_qn_prob: Probabilities for next question for all time (NUM INTERACTIONS x BATCH)
    [h, next_qn_prob], _ = theano.scan(fn=step,
                                       sequences=[x, y],
                                       outputs_info=[h_0, None],
                                       non_sequences=[x_drop, h_drop, w_hh, w_xh, w_hy,
                                                      w_bh, w_by])

    # The negative cross entropy will be what we minimize
    # Note that we multiply by our mask (m) since not all students have the same history length
    next_qn_prob = T.clip(next_qn_prob, probability_clip, 1 - probability_clip)
    cross_entropy = t * T.log(next_qn_prob) + (1 - t) * T.log(1 - next_qn_prob)
    error = -((cross_entropy * m).sum())

    # Prediction accuracy at a cutoff of 0.5
    num_pred_correct = (T.eq(T.ge(next_qn_prob, 0.5), t) * m).sum()

    # BP Gradients for all weights
    g_hh, g_xh, g_hy, g_bh, g_by = T.grad(error, [w_hh, w_xh, w_hy, w_bh, w_by])

    # After all the symbols are linked correctly
    # The expression is compiled using the theano.function to a function
    # Hence grad_fn and test_fn are actual functions to pass in real data
    grad_fn = theano.function(inputs=[x, y, t, m, h_0, x_drop, h_drop, w_hh, w_xh, w_hy, w_bh,
                                      w_by],
                              outputs=[error, g_hh, g_xh, g_hy, g_bh, g_by])

    # Note that in testing we use all the edges in the graph regardless of dropout.
    # In order to make this clear to the user, we do not require them to pass the
    # relevant vector of ones and instead simply make it "given"
    test_fn = theano.function(inputs=[x, y, t, m, h_0, w_hh, w_xh, w_hy, w_bh, w_by],
                              givens=[(x_drop, T.ones((x.shape[2],), dtype=mask_type)),
                                      (h_drop, T.ones_like(w_bh, dtype=mask_type))],
                              outputs=[num_pred_correct, next_qn_prob])
    return grad_fn, test_fn

# A namedtuple for keeping track of weights by name
Weights = namedtuple('Weights', ['w_hh', 'w_xh', 'w_hy', 'w_bh', 'w_by'])

class RnnOpts(namedtuple('RnnOpts', ['max_compress_dim', 'hidden_dim', 'recurrent', 'num_iters',
                                     'dropout_prob', 'grad_norm_limit', 'first_learning_rate',
                                     'decay_rate', 'largest_grad', 'batch_threshold',
                                     'max_output_compress_dim'])):
    """ An immutable container class for RNN structure and optimization options. Note that we take
    an unconventional approach here - we extend namedtuple to get both immutability and
    initialization with default arguments. """
    def __new__(cls, max_compress_dim=10, hidden_dim=10, recurrent=True, num_iters=10,
                dropout_prob=0., grad_norm_limit=0.01, first_learning_rate=30.0, decay_rate=0.99,
                largest_grad=2. ** 20, batch_threshold=0.9, max_output_compress_dim=None):
        """
        RNN STRUCTURE PARAMETERS

        :param int|None max_compress_dim: The max. dimension to which to compress the data using the
            compressed sensing technique. If this is None or the data has length less than
            max_compress_dim, then no compression is performed.
        :param int hidden_dim: The number of hidden units in the RNN.
        :param bool recurrent: Should we actually use a recurrent network or a nonrecurrent one?
        :param int|None max_output_compress_dim: The maximum dimension to which to compress the
            *output* of the RNN (as opposed to the input, controlled by max_compress_dim). None
            simply means no compression.

        LEARNING PARAMETERS

        :param int num_iters: Number of training iterations
        :param float dropout_prob: The probability that a node is dropped in dropout training
        :param float grad_norm_limit: Rescale the gradient so that its sup norm is at most this
        :param float first_learning_rate: What is the first learning rate. Will be decayed
            according to `decay_rate`
        :param float decay_rate: The rate at which the learning_rate will be annealed
        :param float largest_grad: The largest allowable gradient. Any gradient larger than this
            will be clipped to this value.
        :param float batch_threshold: The threshold at which to batch responses
        """
        return super(RnnOpts, cls).__new__(cls, max_compress_dim, hidden_dim, recurrent, num_iters,
                                           dropout_prob, grad_norm_limit, first_learning_rate,
                                           decay_rate, largest_grad, batch_threshold,
                                           max_output_compress_dim)

class SimpleRnn(object):

    @property
    def grad_fn(self):
        if self._grad_fn is None:
            self._grad_fn, self._test_fn = build_grad_and_test_fns(
                recurrent=self.opts.recurrent, compressed_output=self.output_basis is not None)
        return self._grad_fn

    @property
    def test_fn(self):
        if self._test_fn is None:
            self._grad_fn, self._test_fn = build_grad_and_test_fns(recurrent=self.opts.recurrent)
        return self._test_fn

    def __init__(self, train_data, opts, test_data=None, basis=None, weights=None, results=None,
                 num_questions=None, num_type=np.float32, mask_type=np.int8, data_opts=None,
                 output_basis=None, input_dim=None):
        """
        OPTIONS INVOLVING STRUCTURE

        :param list[UserData] train_data: The training data to use for this RNN. Should
            be a list of `UserData` objects, e.g., that comes from `data.rnn.build_nn_data`
        :param RnnOpts opts: learning options (these are not mutated)
        :param list[UserData]|None test_data: The testing data to use for this RNN. Should
            be a list of `UserData` objects, e.g., that comes from `data.rnn.build_nn_data`
        :param np.ndarray|None basis: Which basis to use for compression. If not None, then a basis
            will be created using the `max_compress_dim` option. Should be of size
            `num_q_dims x max_compress_dim`, where `num_q_dims` is likely the number of questions
            or twice that depending on whether you're using correct.
        :param int|None input_dim: What is the dimension of the input. Useful for when you are
            dumping and loading. You likely won't need this.
        :param Weights weights: What weights to use for the RNN if you are reinstantiating
        :param list[Results]|None results: A list of Results to populate the data structure with.
            Probably should only be used by `load`.
        :param int num_questions: the size of the prediction vector (number of questions), useful
            when instantiating the network with empty data. Inferred from data if passed.
        :param num_type: The numeric type to be used in computations. Should be a valid numpy type
        :param mask_type: The mask type to be used in computations. Should be a valid numpy type
        :param DataOpts data_opts: data pre-processing parameters, to be saved with the RNN.
        """
        # Setup data
        self.train_data = train_data
        self.test_data = test_data if test_data is not None else []

        self.opts = opts
        self.data_opts = data_opts

        # Compute number of questions from data if not supplied
        if num_questions is None:
            num_questions = max(np.max(d.next_answer) for d in train_data) + 1
            if test_data is not None:
                num_questions = max(num_questions,
                                    max(np.max(d.next_answer) for d in test_data) + 1)
        self.num_questions = num_questions

        # Setup compression
        if basis is not None:
            # Basis is supplied, so infer the compression dimension from it.
            self.basis = basis
            self.compress_dim = basis.shape[1]
            self.input_dim = basis.shape[0]
            _logger.debug("Using supplied basis with compression "
                          "dimension %d" % self.compress_dim)
        else:
            # Compute input dimension from data if not supplied
            if input_dim is None:
                self.input_dim = max(np.max(d.history) for d in train_data) + 1
                if test_data is not None:
                    self.input_dim = max(self.input_dim,
                                         max(np.max(d.history) for d in test_data) + 1)
            else:
                self.input_dim = input_dim
            self.basis = self._generate_basis(self.input_dim,
                                              self.opts.max_compress_dim, num_type)
            if self.basis is None:
                self.compress_dim = self.input_dim
            else:
                self.compress_dim = self.basis.shape[1]
            _logger.debug("Generated a basis with compression dimension %d" % self.compress_dim)

        _logger.debug("Initialized with input dimension %d", self.compress_dim)

        if output_basis is not None:
            self.output_basis = output_basis
            self.output_compress_dim = output_basis.shape[1]
            _logger.debug("Using supplied output basis with compression "
                          "dimension %d" % self.num_questions)
        else:
            self.output_basis = self._generate_basis(num_questions,
                                                     self.opts.max_output_compress_dim,
                                                     num_type)
            if self.output_basis is not None:
                self.output_compress_dim = self.output_basis.shape[1]
            else:
                self.output_compress_dim = num_questions
            _logger.debug("Generated an output basis with "
                          "compression dimension %d" % self.output_compress_dim)

        # Build batches
        self.train_batches = build_batches(self.train_data, self.output_compress_dim, self.basis,
                                           threshold=self.opts.batch_threshold,
                                           output_basis=self.output_basis,
                                           compress_dim=self.compress_dim)
        self.test_batches = build_batches(self.test_data, self.output_compress_dim, self.basis,
                                          threshold=self.opts.batch_threshold,
                                          output_basis=self.output_basis,
                                          compress_dim=self.compress_dim)

        # Type choices in case you want to optimize
        self.num_type = num_type
        self.mask_type = mask_type
        if weights is None:
            # Weights are uniformly initialized
            w_hh = np.random.uniform(size=(self.opts.hidden_dim, self.opts.hidden_dim),
                                     low=-.01, high=.01).astype(self.num_type)
            w_xh = np.random.uniform(size=(self.compress_dim, self.opts.hidden_dim),
                                     low=-.01, high=.01).astype(self.num_type)
            w_hy = np.random.uniform(size=(self.opts.hidden_dim, self.output_compress_dim),
                                     low=-.01, high=.01).astype(self.num_type)
            w_bh = np.random.uniform(size=self.opts.hidden_dim,
                                     low=-.01, high=.01).astype(self.num_type)
            w_by = np.random.uniform(size=self.output_compress_dim,
                                     low=-.01, high=.01).astype(self.num_type)
            self.weights = Weights(w_hh, w_xh, w_hy, w_bh, w_by)
        else:
            self.weights = weights

        # Setup scaled down weights for testing. See page 2 and 3 of
        # http://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf
        self.test_weights = Weights(*[np.empty_like(getattr(self.weights, key))
                                      for key in Weights._fields])
        self._copy_and_scale_weights()

        # To be instantiated on first call
        self._grad_fn = None
        self._test_fn = None

        # Keep track of test runs
        self.results = [] if results is None else results

        # Initialize the learning rate
        self.learning_rate = self.opts.first_learning_rate

    def _generate_basis(self, input_dimension, max_option_dim, num_type):
        """
        Generate a linear compression map from R^{input_dimension} to R^N where
        N = min(input_dimension, max_option_dim). The type of the returned matrix will
        be num_type.

        If the returned matrix would be the identity matrix, we return None.

        :param int input_dimension: The "from" dimension
        :param int max_option_dim: The maximum "to" dimension
        :param dtype num_type: The type to use for the compression matrix
        :return: The compression matrix or None
        :rtype: np.ndarray | None
        """
        if max_option_dim and max_option_dim < input_dimension:
            _logger.debug("Using random compression with supplied max_compress_dim "
                          "(%d)" % max_option_dim)
            basis = np.random.normal(size=(input_dimension, max_option_dim)).astype(num_type)
            normalizer = np.sqrt((basis * basis).sum(axis=1))
            basis = basis / normalizer[:, np.newaxis]
        else:
            if max_option_dim:
                _logger.debug("Not using compression because max history size (%d)"
                              " <= maximum allowed compression dimension (%d)",
                              input_dimension, max_option_dim)
            else:
                _logger.debug("Not using compression because dimension not specified.")
            basis = None
        return basis

    def train_and_test(self, num_iters, test_spacing=5):
        """
        Train the RNN on the train data, and test on the test data.
        Note the learning rate is reset to self.opts.first_learning_rate
        before training commences.

        :param int num_iters: number of training iterations
        :param int test_spacing: evaluate RNN on the test data every
                                 this many training iterations.
        :return: the MAP accuracy, AYC, predicted probabilities of correct,
                 and boolean correct values on the test data.
        :rtype: float, float, np.ndarray(float), np.ndarray(float)
        """
        # Initialize the learning rate based on options.
        self.learning_rate = self.opts.first_learning_rate

        def run_test(num_iter, num_iters):
            _logger.info("Computing metrics for training data on iteration %3d/%d",
                         num_iter + 1, num_iters)
            train_acc, train_auc, _, _ = self.test_on_training()
            _logger.info("Train Accuracy: %.5f; Train AUC: %.5f", train_acc, train_auc)
            _logger.info("Running tests for iteration %3d/%d", num_iter + 1, num_iters)
            test_acc, test_auc, test_prob_correct, test_corrects = self.test()
            self.results.append(Results(num_iter=num_iter, accuracy=test_acc, auc=test_auc))
            _logger.info("Test Accuracy: %.5f; Test AUC: %.5f", test_acc, test_auc)
            return test_acc, test_auc, test_prob_correct, test_corrects

        for num_iter in range(num_iters):
            _logger.info("Performing training iteration %3d/%d", num_iter + 1, num_iters)
            start_time = time.time()
            self.step()
            end_time = time.time()
            _logger.info("Iteration %3d/%d took %.2fs", num_iter + 1, num_iters,
                         end_time - start_time)

            # DECAY!!!!
            self.learning_rate *= self.opts.decay_rate

            if (num_iter + 1) % test_spacing == 0:
                test_acc, test_auc, test_prob_correct, test_corrects = run_test(num_iter,
                                                                                num_iters)

        # If we didn't run a test on the last iteration, run one now
        if num_iters % test_spacing != 0:
            test_acc, test_auc, test_prob_correct, test_corrects = run_test(num_iters - 1,
                                                                            num_iters)
        return test_acc, test_auc, test_prob_correct, test_corrects

    def step(self):
        """
        Take a single step using `self.train_batches`
        """
        np.random.shuffle(self.train_batches)
        num_batches = len(self.train_batches)
        print_on = {num_batches * i // 10 for i in range(1, 11)}
        for batch_num, batch in enumerate(self.train_batches):
            # Initial hidden state
            h_0 = np.zeros((batch.length, len(self.weights.w_bh)), dtype=self.num_type)

            # Dropout (1s for *retained* nodes)
            h_drop = (np.random.random(self.opts.hidden_dim) > self.opts.dropout_prob).astype(
                self.mask_type)
            x_drop = (np.random.random(self.compress_dim) > self.opts.dropout_prob).astype(
                self.mask_type)

            # Run RNN grad function
            results = self.grad_fn(batch.history, batch.next_answer, batch.truth, batch.mask,
                                   h_0, x_drop, h_drop, *self.weights)
            error, g_hh, g_xh, g_hy, g_bh, g_by = results

            grads = Weights(w_hh=g_hh, w_xh=g_xh, w_hy=g_hy, w_bh=g_bh, w_by=g_by)

            # Update gradient parameters
            self.update_weights(grads)

            if batch_num in print_on:
                _logger.debug("Batch %3d/%d complete", batch_num, num_batches)

    def _copy_and_scale_weights(self):
        """ Copy and scale the weights for testing according to the dropout probability """
        for key in Weights._fields:
            np.multiply(1.0 - self.opts.dropout_prob, getattr(self.weights, key),
                        getattr(self.test_weights, key))

    def update_weights(self, grads):
        """
        Update the weights using (modified) gradient descent. Done in place.

        :param Weights grads: The gradients of the error function w.r.t. the weights
        """
        # Set invalid values to 0 and clip extreme gradients
        for grad in grads:
            grad[~np.isfinite(grad)] = 0
            np.clip(grad, -self.opts.largest_grad, self.opts.largest_grad, out=grad)

        # Gather the gradient norms
        grad_norms = [np.linalg.norm(grad) for grad in grads]
        max_grad_norm = max(grad_norms)
        if max_grad_norm > self.opts.grad_norm_limit:
            offset = self.opts.grad_norm_limit / max_grad_norm
        else:
            offset = 1.0

        # Gradient descent
        for weight, grad in zip(self.weights, grads):
            weight -= self.learning_rate * grad * offset

        self._copy_and_scale_weights()

    def _test_on_batches(self, batches):
        """ Compute validation metrics on the specified batches.

        :return: accuracy (at a 0.5 cutoff) and AUC, and the response probability
                 and correctness values from the passed data.
        :rtype: (float, float, np.ndarray(float), np.ndarray(bool))
        """
        # Setup test stats variables
        test_questions = 0
        test_correct = 0

        # AUC probs and truth
        p = np.zeros(0, dtype=self.num_type)
        t = np.zeros(0, dtype=self.num_type)

        num_batches = len(batches)
        print_on = {num_batches * i // 10 for i in range(1, 11)}
        for batch_num, batch in enumerate(batches):
            # Initial hidden state and broadcast bias weights to all batches
            h_0 = np.zeros((batch.length, len(self.weights.w_bh)), dtype=self.num_type)

            # Run RNN test function
            num_pred_correct, next_qn_prob = self.test_fn(batch.history, batch.next_answer,
                                                          batch.truth, batch.mask, h_0,
                                                          *self.test_weights)

            # Update stats
            test_questions += batch.num_interactions
            test_correct += num_pred_correct

            # Update points for AUC
            interaction_mask = batch.mask.astype(np.bool).flatten()
            p = np.concatenate((p, np.ravel(next_qn_prob)[interaction_mask]), axis=0)
            t = np.concatenate((t, np.ravel(batch.truth)[interaction_mask]), axis=0)

            if batch_num in print_on:
                _logger.debug("Batch %3d/%d complete", batch_num, num_batches)

        # Find test accuracy and AUC
        test_acc = test_correct / test_questions
        test_auc = Metrics.auc_helper(t, p)
        return test_acc, test_auc, p, t

    def test(self):
        """ Compute validation metrics on the test set.

        :return: test accuracy (at a 0.5 cutoff) and test AUC, and the response probability
                 and correctness values from the test set.
        :rtype: (float, float, np.ndarray(float), np.ndarray(bool))
        """
        return self._test_on_batches(self.test_batches)

    def test_on_training(self):
        """ Compute validation metrics on the training set.

        :return: training accuracy (at a 0.5 cutoff) and training AUC, and the response probability
                 and correctness values from the training set.
        :rtype: (float, float, np.ndarray(float), np.ndarray(bool))
        """
        return self._test_on_batches(self.train_batches)

    def dump(self, f):
        """ Dump to a file handler `f`. Load with `load`. """
        output = {
            'train_data': [],
            'opts': self.opts,
            'data_opts': self.data_opts,
            'basis': self.basis,
            'weights': self.weights,
            'results': self.results,
            'num_questions': self.num_questions,
            'input_dim': self.input_dim,
            'num_type': self.num_type,
            'mask_type': self.mask_type,
            'output_basis': self.output_basis
        }
        pickle.dump(output, f)

    @classmethod
    def load(cls, f):
        """ Load from a file handle `f`. Should have been dumped with `dump`. """
        d = pickle.load(f)
        return cls(**d)

reason: import numpy,import theano,from theano


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Sep  3 12:09:39 2019

@author: autol
"""
#%%
from scipy import sparse,stats
import numpy as np

#%% Matrix 1
def init_A1(n=5,m=3):
    n,m=int(n),int(m)
    A = sparse.random(n,m, density=.8,
                      data_rvs=stats.randint(1,6).rvs).toarray()
    return A

#%% Matrix 2
def init_A2(n=18,m=15,d=.4,rstat=1212): #密度0~1,1是全部数存在
    np.random.seed(rstat)
    n,m=int(n),int(m)
    d_nan = round((1-d)*n*m)
    A = np.random.randint(1,5,(n,m)).astype(np.float)
    A.ravel()[np.random.choice(A.size,d_nan,replace=0)] = np.nan
    return A

#%% Matrix 3
# [soft impute referense](https://cran.r-project.org/web/packages/softImpute/vignettes/softImpute.html)

def init_A3():
    A = np.array([[0.8654889,0.01565179,0.1747903,np.nan, np.nan],
                    [-0.6004172,np.nan,-0.2119090,np.nan,np.nan],
                    [-0.7169292,np.nan, np.nan,0.06437356,-0.09754133],
                    [0.6965558,-0.50331812,0.5584839 ,1.54375663 ,np.nan],
                    [1.2311610,-0.34232368,-0.8102688 ,-0.82006429 ,-0.13256942],
                    [0.2664415,0.14486388,np.nan,np.nan, -2.24087863]
                    ])
    return A

def init_A4(n=5,m=3):
    n,m=int(n),int(m)
    A = np.random.randint(1,100,n*m).reshape(n,m).astype(np.float)
    return A

max_stars_repo_name: autolordz/gradient-descent-optimization

max_issues_repo_name: autolordz/gradient-descent-optimization

max_forks_repo_name: autolordz/gradient-descent-optimization


text:
import os
import sys
sys.path.append(os.path.realpath(__file__ + "/../../"))
from Utils import read_data, util_functions
from simple_model import NetModel
import numpy as np

def _main():

	WORK_DIR = os.path.abspath(__file__+ '/../../' )

	if (len(sys.argv)) < 3:
		print("Missing arguments, call should be : python main.py <pbm> <selex0> <selex1> <selex2> ... <selex5>")
		exit(1)

	# Reads external json file with parameters.
	params_file_name = os.path.abspath(__file__ + '/../') + '/Utils/config.json'
	parameters = util_functions.get_model_parameters(params_file_name)

	# Create data pipeline obj
	dataPipe = read_data.DataPipeline(sys.argv,WORK_DIR)

	# If multi_selex mode is enabled, we train over each selex with the same model
	if parameters["multi_selex"]:
		number_of_selex = dataPipe.number_of_selex
		steps_per_epoch = parameters['steps_per_epoch']/(number_of_selex-1)*np.linspace(0.5,2,number_of_selex-1)

		for i in range(parameters['n_epochs']):
			for selex in range(number_of_selex-1):
				if selex==0 and i==0:
					model = NetModel(numOfModel=2,input_shape=(dataPipe.input_shape))
				model.train(tain_generator=dataPipe.train_generator[selex],
	 		   		        validation_generator=None,
	    		    	    steps_per_epoch=steps_per_epoch[selex]/(i+1),
	    		    	    n_epochs=1,
	    		    	    n_workers=parameters['n_workers'])

	# If multi_selex mode is not enabled, we train over the first and last selex
	else:
		model = NetModel(numOfModel=2, input_shape=(dataPipe.input_shape))
		model.train(tain_generator=dataPipe.train_generator,
					validation_generator=None,
					steps_per_epoch=parameters['steps_per_epoch'],
					n_epochs=parameters['n_epochs'],
					n_workers=parameters['n_workers'])

	# Calculate PBM predictions and AUPR
	predictions = model.predict(dataPipe.test_generator, parameters['n_workers'])
	AUPR = util_functions.getAUPR(dataPipe.testData, predictions)
	print(AUPR)

	# Sort and save fo files the PBM predictions
	sortedPBMarray = util_functions.sortPBM(dataPipe.testData, predictions)
	TF_index = dataPipe.TF_index
	util_functions.dumpPBMtoFile(os.path.realpath(__file__ + "/../"), 'sortedPBM_TF' + str(TF_index), sortedPBMarray)

if __name__ == '__main__':
	_main()

max_stars_repo_name: RoyHirsch/BioComp

max_issues_repo_name: RoyHirsch/BioComp

max_forks_repo_name: RoyHirsch/BioComp


text:
import os
import cv2
import tqdm
import torch
import numpy as np

from utils import results_common_utils as common_utils
from seg_kanezaki.code.utils.data_preprocessing import process_img

def get_iou_model(args, predictions, labels, n_classes, labels_name=None):
	"""
	Do matching between the values of the labels and the values of the output
	of the model and measure IOU metric for each matching.
	Save the mask of the prediction with the best matching applied.
	Args:
		args: Config dict
		predictions: Numpy array of predictions from the model
		labels: Numpy array with the labels
		n_classes: Int with number of classe of dataset
		labels_name: Optional list with filename info
	Return:
		IOU metrics
	"""
	# iou_model = None
	masks_list = []
	iou_imgs = []
	# Get IOU of each image
	for id_img in range(len(predictions)):
		iou, mask, _ = common_utils.get_iou(labels[id_img], predictions[id_img], n_classes, None)
		if labels_name:
			iou['img_name'] = os.path.basename(labels_name[id_img])
		iou_imgs.append(iou)
		masks_list.append(mask)
	iou_model = {
		'overall_mean': np.mean(np.array([x['mean_IOU'] for x in iou_imgs])),
		'variance': np.var(np.array([x['mean_IOU'] for x in iou_imgs])),
		'all': iou_imgs}
	# Save masks
	folder_path = os.path.join(args.DIR_SAVE_PATH, 'masks')
	if not os.path.exists(folder_path):
		print(folder_path)
		# Set up folder
		os.makedirs(folder_path)
	if labels_name:
		[cv2.imwrite(os.path.join(folder_path, "mask_" + os.path.basename(labels_name[id_x]).split('.')[0] + '.png'),
		             masks_list[id_x]) for id_x in range(len(masks_list))]
		[cv2.imwrite(os.path.join(folder_path, "label_" + os.path.basename(labels_name[id_x]).split('.')[0] + '.png'),
		             labels[id_x]) for id_x in range(len(labels))]
	return iou_model

def inference_folder(args, model, test_img_list, label_colours=None):
	"""
	Run inference over the given images
	Args:
		args: Config dict
		model: Torch model
		test_img_list: List of paths of the images to be predicted
		label_colours: Colour palette applied to the predictions for viasualization purposes
	"""
	print('Testing ' + str(len(test_img_list)) + ' images.')
	for img_file in tqdm.tqdm(test_img_list):
		# Load image
		args.IMG_PATH = img_file
		im = process_img(args, validation=True)
		data = torch.from_numpy(np.array([im.transpose((2, 0, 1)).astype('float32') / 255.]))
		data = data.cuda()
		data = torch.autograd.Variable(data)
		# Run inference
		output = model(data)[0]
		output = output.permute(1, 2, 0).contiguous().view(-1, args.COMMON.NCHANNEL)
		ignore, target = torch.max(output, 1)
		inds = target.data.cpu().numpy().reshape((im.shape[0], im.shape[1]))
		# Apply colours
		if label_colours is not None:
			inds = np.array([label_colours[c % args.COMMON.NCHANNEL] for c in inds])
			inds = inds.reshape(im.shape).astype(np.uint8)
			inds = cv2.cvtColor(inds, cv2.COLOR_BGR2GRAY)
		else:
			inds = inds.reshape((im.shape[0], im.shape[1])).astype(np.uint8)
		# Save mask
		pred_path = os.path.join(args.DIR_SAVE_PATH, os.path.basename(img_file).split('.')[0]+'.png')
		cv2.imwrite(pred_path, inds)