text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import argparse
import logging
import time
import mxnet as mx
import numpy as np
from get_data import get_movielens_iter, get_movielens_data
from model import matrix_fact_model_parallel_net

logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser(description="Run model parallel version of matrix factorization",
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num-epoch', type=int, default=3,
                    help='number of epochs to train')
parser.add_argument('--batch-size', type=int, default=256,
                    help='number of examples per batch')
parser.add_argument('--print-every', type=int, default=100,
                    help='logging interval')
parser.add_argument('--factor-size', type=int, default=128,
                    help="the factor size of the embedding operation")
parser.add_argument('--num-gpus', type=int, default=2,
                    help="number of gpus to use")

MOVIELENS = {
    'dataset': 'ml-10m',
    'train': './ml-10M100K/r1.train',
    'val': './ml-10M100K/r1.test',
    'max_user': 71569,
    'max_movie': 65135,
}

if __name__ == '__main__':
    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.INFO, format=head)

    # arg parser
    args = parser.parse_args()
    logging.info(args)
    num_epoch = args.num_epoch
    batch_size = args.batch_size
    optimizer = 'sgd'
    factor_size = args.factor_size
    print_every = args.print_every
    num_gpus = args.num_gpus

    momentum = 0.9
    learning_rate = 0.1

    # prepare dataset and iterators
    max_user = MOVIELENS['max_user']
    max_movies = MOVIELENS['max_movie']
    get_movielens_data(MOVIELENS['dataset'])
    train_iter = get_movielens_iter(MOVIELENS['train'], batch_size)
    val_iter = get_movielens_iter(MOVIELENS['val'], batch_size)

    # construct the model
    net = matrix_fact_model_parallel_net(factor_size, factor_size, max_user, max_movies)

    # construct the module
    # map the ctx_group attribute to the context assignment
    group2ctxs={'dev1':[mx.cpu()]*num_gpus, 'dev2':[mx.gpu(i) for i in range(num_gpus)]}

    # Creating a module by passing group2ctxs attribute which maps
    # the ctx_group attribute to the context assignment
    mod = mx.module.Module(symbol=net, context=[mx.cpu()]*num_gpus, data_names=['user', 'item'],
        label_names=['score'], group2ctxs=group2ctxs)

    # the initializer used to initialize the parameters
    initializer = mx.init.Xavier(factor_type="in", magnitude=2.34)

    # the parameters for the optimizer constructor
    optimizer_params = {
        'learning_rate': learning_rate,
        'wd': 1e-4,
        'momentum': momentum,
        'rescale_grad': 1.0/batch_size}

    # use MSE as the metric
    metric = mx.gluon.metric.create(['MSE'])

    speedometer = mx.callback.Speedometer(batch_size, print_every)

    # start training
    mod.fit(train_iter,
            val_iter,
            eval_metric        = metric,
            num_epoch          = num_epoch,
            optimizer          = optimizer,
            optimizer_params   = optimizer_params,
            initializer        = initializer,
            batch_end_callback = speedometer)

max_stars_repo_name: tkameyama/incubator-mxnet

max_issues_repo_name: tkameyama/incubator-mxnet

max_forks_repo_name: tkameyama/incubator-mxnet


text:
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np

def depth_to_color(depth):
    cmap = plt.cm.jet
    d_min = np.min(depth)
    d_max = np.max(depth)
    depth_relative = (depth - d_min) / (d_max - d_min)
    return 255 * cmap(depth_relative)[:, :, :3]  # H, W, C

def error_to_color(depth, gt):
    mask = gt <= 0.0
    cmap = plt.cm.Greys
    err = np.abs(depth - gt)
    err[mask] = 0.0
    err_min = np.min(err)
    err_max = np.max(err)
    err_rel = (err - err_min) / (err_max - err_min)
    return 255 * cmap(err_rel)[:, :, :3]

ext: py

reason: import numpy


text:
# --------------
import pandas as pd
from sklearn.model_selection import train_test_split

data = pd.read_csv(path)

X = data.drop(columns = ["customer.id", "paid.back.loan"])
y = data["paid.back.loan"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3,
                                                    random_state = 0)

# --------------
#Importing header files
import matplotlib.pyplot as plt

# Code starts here
fully_paid = y_train.value_counts()
fully_paid.plot(kind = "bar")
for num, count in enumerate(fully_paid):
    plt.text(x = num, y = count, s = str(count))
plt.show()

# Code ends here

# --------------
#Importing header files
import numpy as np
from sklearn.preprocessing import LabelEncoder

# Code starts here
X_train["int.rate"] = X_train["int.rate"].map(lambda x: float(x.strip("%"))/100)
X_test["int.rate"] = X_test["int.rate"].map(lambda x: float(x.strip("%"))/100)

num_df = X_train.select_dtypes(exclude = "object")
cat_df = X_train.select_dtypes(include = "object")
# Code ends here

# --------------
#Importing header files
import seaborn as sns

# Code starts here
cols = num_df.columns
fig, axes = plt.subplots(9,1)
for i in range(9):
    sns.boxplot(x=y_train, y=num_df[cols[i]], ax=axes[i])
plt.show()

# Code ends here

# --------------
# Code starts here
cols = cat_df.columns
fig, axes = plt.subplots(2,2)
for i in range(2):
    for j in range(2):
        sns.countplot(x=X_train[cols[i*2+j]], hue=y_train, ax=axes[i,j])
plt.show()

# Code ends here

# --------------
#Importing header files
from sklearn.tree import DecisionTreeClassifier

# Code starts here
le = LabelEncoder()
X_test.fillna("NA", inplace = True)
for col in cat_df.columns:
    X_train[col].fillna("NA", inplace = True)
    X_train[col] = le.fit_transform(X_train[col])
    X_test[col] = le.transform(X_test[col])

y_train = y_train.map(lambda x: 0 if x == "No" else 1)
y_test = y_test.map(lambda x: 0 if x == "No" else 1)

model = DecisionTreeClassifier(random_state = 0)
model.fit(X_train, y_train)

acc = np.round(model.score(X_test, y_test),2)
print(acc)
# Code ends here

# --------------
#Importing header files
from sklearn.model_selection import GridSearchCV

#Parameter grid
parameter_grid = {'max_depth': np.arange(3,10), 'min_samples_leaf': range(10,50,10)}

# Code starts here
model_2 = DecisionTreeClassifier(random_state = 0)
p_tree = GridSearchCV(estimator = model_2, param_grid = parameter_grid, cv=5)
p_tree.fit(X_train, y_train)
acc_2 = np.round(p_tree.score(X_test, y_test),2)
print(acc_2)

# Code ends here

# --------------
#Importing header files

from io import StringIO
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn import metrics
from IPython.display import Image
import pydotplus

# Code starts here
dot_data = export_graphviz(decision_tree=p_tree.best_estimator_, out_file = None,
                            feature_names = X.columns, filled = True,
                            class_names = ['loan_paid_back_yes','loan_paid_back_no']                            )
graph_big = pydotplus.graph_from_dot_data(dot_data)

# show graph - do not delete/modify the code below this line
img_path = user_data_dir+'/file.png'
graph_big.write_png(img_path)

plt.figure(figsize=(20,15))
plt.imshow(plt.imread(img_path))
plt.axis('off')
plt.show()

# Code ends here


text:
"""
------------------------------------------------------------------
FILE:   save.py
AUTHOR: Edo Altamura
DATE:   20-11-2019
------------------------------------------------------------------
In order to make the data post-processing more manageable, instead
of calculating quantities from the simulations every time, just
compute them once and store them in a hdf5 file.
This process of data reduction level condenses data down to a few KB
or MB and it is possible to transfer it locally for further analysis.
-------------------------------------------------------------------
"""

import sys
import os
import itertools
import time
import numpy as np
import warnings
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from matplotlib import colors
from matplotlib.patches import Patch

exec(open(os.path.abspath(os.path.join(
        os.path.dirname(__file__), os.path.pardir, 'visualisation', 'light_mode.py'))).read())

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
from import_toolkit.simulation import Simulation
from import_toolkit.progressbar import ProgressBar
from import_toolkit._cluster_retriever import redshift_str2num

class SimulationOutput(Simulation):

    # How the directory is structured
    directory_levels = ['simulation_name',
                        'cluster_ID',
                        'redshift',
                        'data.hdf5 && other_outputs.*']

    def __init__(self, simulation_name: str = None, run_dircheck: bool = False):

        super().__init__(simulation_name=simulation_name)
        if run_dircheck:
            self.check_data_structure()

    @ProgressBar()
    def check_data_structure(self):

        if not os.path.exists(os.path.join(self.pathSave, self.simulation_name)):
            os.makedirs(os.path.join(self.pathSave, self.simulation_name))
        length_operation = len(self.clusterIDAllowed) * len(self.redshiftAllowed)
        counter = 0

        for cluster_number, cluster_redshift in itertools.product(self.clusterIDAllowed, self.redshiftAllowed):
            out_path = os.path.join(self.pathSave,
                                    self.simulation_name,
                                    f'halo{self.halo_Num(cluster_number)}',
                                    f'halo{self.halo_Num(cluster_number)}_{cluster_redshift}')

            if (not os.path.exists(out_path) and
                self.sample_completeness[cluster_number, self.redshiftAllowed.index(cluster_redshift)]):
                os.makedirs(out_path)
            yield ((counter + 1) / length_operation)  # Give control back to decorator
            counter += 1

    @staticmethod
    def list_files(startpath):
        for root, dirs, files in os.walk(startpath):
            dirs.sort()
            level = root.replace(startpath, '').count(os.sep)
            indent = ' ' * 4 * (level)
            print('{}{}/'.format(indent, os.path.basename(root)))
            subindent = ' ' * 4 * (level + 1)
            files.sort()
            for f in files:
                print('{}{}'.format(subindent, f))

    def print_directory_tree(self):
        print(self.pathSave)
        self.list_files(os.path.join(self.pathSave, self.simulation_name))

    def dir_tree_to_dict(self, path_):
        file_token = ''
        for root, dirs, files in os.walk(path_):
            tree = {d: self.dir_tree_to_dict(os.path.join(root, d)) for d in dirs}
            tree.update({f: file_token for f in files})
            return tree  # note we discontinue iteration trough os.walk

    @staticmethod
    def draw_pie(dist,
                 xpos,
                 ypos,
                 size,
                 ax=None):
        if ax is None:
            fig, ax = plt.subplots(figsize=(11, 12))

        # for incremental pie slices
        cumsum = np.cumsum(dist)
        cumsum = cumsum / cumsum[-1]
        pie = [0] + cumsum.tolist()
        colors = ['lime', 'red']
        for r1, r2, c in zip(pie[:-1], pie[1:], colors):
            angles = np.linspace(2 * np.pi * r1, 2 * np.pi * r2)
            x = [0] + np.cos(angles).tolist()
            y = [0] + np.sin(angles).tolist()

            xy = np.column_stack([x, y])

            ax.scatter([xpos], [ypos], marker=xy, s=size, facecolor=c, alpha = 0.5)

        return ax

    @ProgressBar()
    def status_plot(self):
        warnings.filterwarnings("ignore")
        fig = plt.figure(figsize=(10, 10))
        ax = fig.add_subplot(111)
        report_matrix = np.zeros((len(self.clusterIDAllowed), len(self.redshiftAllowed)), dtype=np.int)
        length_operation = np.product(report_matrix.shape)
        counter = 0
        for cluster_number, cluster_redshift in itertools.product(self.clusterIDAllowed, self.redshiftAllowed):

            out_path = os.path.join(self.pathSave,
                                    self.simulation_name,
                                    f'halo{self.halo_Num(cluster_number)}',
                                    f'halo{self.halo_Num(cluster_number)}_{cluster_redshift}')

            if self.sample_completeness[cluster_number, self.redshiftAllowed.index(cluster_redshift)]:
                num_of_files = len([name for name in os.listdir(out_path) if os.path.isfile(os.path.join(out_path, name))])
                report_matrix[cluster_number, self.redshiftAllowed.index(cluster_redshift)] = num_of_files
            else:
                report_matrix[cluster_number, self.redshiftAllowed.index(cluster_redshift)] = -1
            yield ((counter + 1) / length_operation)  # Give control back to decorator
            counter += 1

        expected_total_files = 13
        timestr = time.strftime("%d%m%Y-%H%M%S")
        fraction_complete = np.sum(report_matrix[report_matrix>0]) / (np.product(report_matrix.shape) *
                                                                     expected_total_files) * 100

        cmap   = colors.ListedColormap(['white', 'black', 'red', 'orange', 'lime'])
        bounds = [-1, 0, 0.5, 3.5, expected_total_files-0.5, expected_total_files]
        norm   = colors.BoundaryNorm(bounds, cmap.N)
        ax.imshow(report_matrix, interpolation='nearest', cmap=cmap, norm=norm, origin='lower',
                  aspect = report_matrix.shape[1]/report_matrix.shape[0],
                  extent=(0, len(self.redshiftAllowed),
                          0, self.totalClusters))

        patch_1 = Patch(facecolor='black', label='0 files', edgecolor='k', linewidth=1)
        patch_2 = Patch(facecolor='red', label='1 - 3 files', edgecolor='k', linewidth=1)
        patch_3 = Patch(facecolor='orange', label=f'4 - {expected_total_files - 1} files', edgecolor='k', linewidth=1)
        patch_4 = Patch(facecolor='lime', label=f'{expected_total_files} files', edgecolor='k', linewidth=1)
        patch_5 = Patch(facecolor='white', label='Excluded', edgecolor='k', linewidth=1)
        ax.legend(handles=[patch_4, patch_3, patch_2, patch_1, patch_5],
                  loc='upper right',
                  labelspacing=1.5,
                  handlelength=1,
                  fontsize=20,
                  bbox_to_anchor=(1.3, 0.75))

        ax.set_title(f'{self.simulation:s} output status: {fraction_complete:.0f}\% complete \qquad{timestr:s}',
                     size=20)
        ax.set_xlabel(r'redshift', size=20)
        ax.set_ylabel(r'Cluster ID', size=20)
        ax.set_xticks(list(range(0, len(self.redshiftAllowed), 4)))
        redhifts_ticks = [self.redshiftAllowed[i] for i in range(0, len(self.redshiftAllowed), 4)]
        redhifts_ticks = [f"{redshift_str2num(z):.1f}" for z in redhifts_ticks]
        ax.set_xticklabels(redhifts_ticks)
        plt.tight_layout()
        plt.savefig(os.path.join(self.pathSave,
                                 self.simulation_name,
                                 f"{self.simulation_name}_OutputStatusReport_{timestr}.png"), dpi=400)

if __name__ == '__main__':
    for sim in ['celr_e']:
        out = SimulationOutput(simulation_name = sim, run_dircheck = False)
        print(f"{out.simulation:=^100s}")
        out.status_plot()

max_stars_repo_name: LBJ-Wade/C-Eagle-analysis

max_issues_repo_name: LBJ-Wade/C-Eagle-analysis

max_forks_repo_name: LBJ-Wade/C-Eagle-analysis


text:
import numpy as np

from tqdm import tqdm
from plipy.dpdpl import DeepPrimalDualPriorLearningConv1d
from pathlib import Path

RESULTS = Path(__file__).resolve().parents[1] / "results"
N_TRAINING = 20

def theta(t, theta0, w0):
    return theta0 * np.sin(w0 * t)

def learning_1d(freq, sampling_rate, time=10):

    dt = 1 / sampling_rate
    N = 1000
    theta0 = np.random.random(N)
    T = np.arange(0, time, dt)

    xx, yy = np.meshgrid(T, theta0)

    thetas = theta(xx, yy, 2 * np.pi * freq)
    noise = np.random.normal(scale=thetas.std() / 5, size=thetas.shape)
    thetas_noisy = thetas + noise

    results = []

    while len(results) != N_TRAINING:
        dpdpl_ed = DeepPrimalDualPriorLearningConv1d(n_components=1,
                                                     n_iter=10,
                                                     lambd=0.5,
                                                     kernel_size=3,
                                                     device="cuda:1")
        dpdpl_ed.fit(thetas_noisy.T)

        kernel_ed = dpdpl_ed.get_prior()[0][0]
        kernel_ed / kernel_ed[0]
        w = np.sqrt(2 - (- kernel_ed / kernel_ed[0])[1]) / dt

        if not np.isnan(w):
            results.append(w / (2 * np.pi))

    return np.array(results).mean()

freq = np.arange(1, 11, 1)
sampling_rate = np.arange(10, 110, 10)
results = np.zeros((len(freq), len(sampling_rate)))

for i in tqdm(range(len(freq))):
    for j in range(len(sampling_rate)):
        print(sampling_rate[j])
        w_result = learning_1d(freq[i], sampling_rate[j])
        results[i, j] = w_result

np.save(str(RESULTS / "pde_learning_freq.npy"), freq)
np.save(str(RESULTS / "pde_learning_sampling.npy"), sampling_rate)
np.save(str(RESULTS / "pde_learning_results.npy"), results)

max_stars_repo_name: bmalezieux/plipy

max_issues_repo_name: bmalezieux/plipy

max_forks_repo_name: bmalezieux/plipy


text:
import numpy as np
import threading
import gym
import time  # Thread delay
import random
from utils import init_state, update_state
from utils import save_pickle
import os

class Agent():
    """ An Agent class which collects experience for the brain
    Args:
      config: configuration dictionary
      config['decision_func']: function pointer for policy logits from the brain
      config['push_to_brain']: function pointer for appending training examples
      config['get_epsilon']: function pointer to the epsilon getter
    """

    def __init__(self, config):
        self.memory = []  # for n-step return
        self.G = 0.0  # return
        self.decision_func = config['decision_func']
        self.push_to_brain = config['push_to_brain']
        self.get_epsilon = config['get_epsilon']
        self.config = config

    def act(self, s):
        """ Act epsilon-greedly using epsilon from the brain"""
        eps = self.get_epsilon()
        rand = random.random()
        if rand < eps:
            return random.randint(0, self.config['ACTION_SPACE'] - 1)
        else:
            return self.decision_func(s)

    def get_sample(self, n):
        """ sample n-step predictions, states and returns
        """
        s, a, _, _ = self.memory[0]
        _, _, _, sp = self.memory[n - 1]
        return s, a, self.G, sp

    def train(self, s, a, r, sp):
        """ manipulate memory and push to brain for training
        Args:
          s: state
          a: actions
          r: immediate reward after a
          sp: immediate state after a
        """
        # Turn a into a one-hot vector and add to memory
        a_onehot = np.zeros(self.config['ACTION_SPACE'])
        a_onehot[a] = 1
        self.memory.append([s, a_onehot, r, sp])
        self.G = (self.G + r * self.config['GAMMA_N']) / self.config['GAMMA']

        # Calculate n-step return if the episode terminates
        # This calucate is not accurate
        if sp is None:
            # Terminal State this calculation is inaccurate
            while len(self.memory) > 0:
                n = len(self.memory)
                s, a, r, sp = self.get_sample(n)
                self.push_to_brain(s, a, r, sp)
                self.G = (self.G - self.memory[0][2]) / self.config['GAMMA']
                self.memory.pop(0)
            self.G = 0.0

        # Calulcate n-step return according to config
        if len(self.memory) >= self.config['N_STEPS']:
            s, a, r, sp = self.get_sample(self.config['N_STEPS'])
            self.push_to_brain(s, a, r, sp)
            self.G = self.G - self.memory[0][2]
            self.memory.pop(0)

class Environment(threading.Thread):
    """ An contrainer class with an instance of Gym environments and Agent each

    Args:
      config: configuration dictoinary
      render: bool, this will not work with multi-threading
      debug: bool, this will save details of each epsiode into a pickle file
    """
    env_id = 0  # Evnironment ID
    scores = []  # Global scores record for reporting. this may cause mem overflow

    def __init__(self, config, render=False):
        threading.Thread.__init__(self)
        self.stop_signal = False
        self.render = render
        self.config = config
        self.env = gym.make(self.config['ENV_NAME'])
        self.id = Environment.env_id
        Environment.env_id += 1
        self.agent = Agent(config)
        self.episode_number = 0
        self.debug = config['DEBUG']
        self.debug_log = []

    def run_episode(self):
        """ Game loop """
        # reset
        obs = self.env.reset()
        s = init_state(obs)
        R = 0  # total reward this episode
        self.debug_log = []

        while True:
            time.sleep(self.config['THREAD_DELAY'])  # yield

            if self.render:
                self.env.render()

            a = self.agent.act(s)
            obs, r, done, info = self.env.step(a)
            sp = update_state(s, obs)

            if done:  # terminal state
                sp = None

            self.agent.train(s, a, r, sp)
            if self.debug:
                self.debug_log.append([s, a, r, sp, done])

            s = sp
            R += r

            if done or self.stop_signal:
                Environment.scores.append(R)
                self.episode_number += 1
                # Static purge for now
                Environment.scores = Environment.scores[-100:]

                if self.debug:  # Save logs
                    # TODO: folder restructure
                    save_pickle(self.debug_log, os.path.join(
                        'debug_logs', "ENV{}_EPISODE{}".format(self.id, self.episode_number)))
                if self.render:  # Demo mode
                    print("ENV_{} INFO: total reward this episode: {}".format(self.id, R))
                break

    def run(self):
        while not self.stop_signal:
            self.run_episode()

    def stop(self):
        self.stop_signal = True


text:
import sys
from functools import partial
import matplotlib
import matplotlib.pyplot as plt

matplotlib.use('Qt5Agg')

from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.figure import Figure

# Import QApplication and the required widgets from PyQt5.QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QMainWindow

from PyQt5.QtWidgets import QWidget
from PyQt5.QtWidgets import QSlider
from qtrangeslider import QRangeSlider
from qtrangeslider import QLabeledRangeSlider
from PyQt5.QtWidgets import QLineEdit
from PyQt5.QtWidgets import QPushButton
from PyQt5.QtWidgets import QCheckBox
from PyQt5.QtWidgets import QComboBox

from PyQt5.QtWidgets import QGridLayout
from PyQt5.QtWidgets import QVBoxLayout
from PyQt5.QtWidgets import QHBoxLayout
from PyQt5.QtWidgets import QFormLayout
from PyQt5.QtWidgets import QGroupBox

import numpy as np

# Create a subclass of QMainWindow to setup the calculator's GUI
class DepthMapUi(QMainWindow):
    """PyCalc's View (GUI)."""

    def __init__(self):
        """View initializer."""
        super().__init__()
        # Set some main window's properties
        self.setWindowTitle('depth mapper')
        # self.setFixedSize(235, 235)
        # Set CSD canvas properties
        self.width = 5
        self.height = 20
        self.dpi = 100
        # separator info
        self.layerBorders = ['1/2', '2/3', '3/4', '4/5', '5/6']
        self.separatorBoolean = {border: False for border in self.layerBorders}
        self.separatorPosition = {border: 0 for border in self.layerBorders}
        # draw lines info
        self.lines = {border: None for border in self.layerBorders}
        self.labels = {border: None for border in self.layerBorders}
        # general widget info
        # Set the central widget and the general layout
        self.generalLayout = QGridLayout()
        self._centralWidget = QWidget(self)
        self.setCentralWidget(self._centralWidget)
        self._centralWidget.setLayout(self.generalLayout)
        # Create the display and the buttons
        self._createCanvas()
        self._createSlider()
        self._createCheckList()
        self._createDropDown()
        self._createActionButtons()
        self._createDisplay()

    def _createCanvas(self):
        # creates MPL figure as QT compatible canvas
        self.fig = Figure(figsize=(self.width, self.height), dpi=self.dpi)
        self.ax = self.fig.add_subplot(111)
        self.canvas = FigureCanvasQTAgg(self.fig)
        self.generalLayout.addWidget(self.canvas, 1, 1, 4, 1)

    def _createSlider(self):
        self.slider = QSlider(Qt.Vertical)
        self.slider.setDisabled(True)
        self.slider.setTracking(False)
        self.generalLayout.addWidget(self.slider, 1, 2, 4, 1)

    def _selectSlider(self, index,):
        name = self.dropDown.itemText(index)
        position = self.separatorPosition[name]
        print(f'{index}: border {name} slider set at {position}')
        self.slider.setDisabled(False)
        self.slider.setValue(position)

    def _updateSlider(self, value):
        currentSlider = self.dropDown.currentText()
        self.separatorPosition[currentSlider] = value
        for sep, pos in self.separatorPosition.items():
            print(f'sep{sep} pos{pos}')
        print('\n')

    def _createCheckList(self):
        checkGroup = QGroupBox('Visible layer borders')
        checkLayout = QHBoxLayout()
        self.layerCheckBoxes = {f'{l}':'' for l in self.layerBorders}
        for layer in self.layerCheckBoxes.keys():
            checkbox = QCheckBox(layer)
            self.layerCheckBoxes[layer] = checkbox
            checkLayout.addWidget(checkbox)
        checkGroup.setLayout(checkLayout)
        self.generalLayout.addWidget(checkGroup, 1, 3, 1, 1)

    def _createDropDown(self):
        self.dropDown = QComboBox()
        self.generalLayout.addWidget(self.dropDown, 2, 3, 1, 1)

    def _updateDropDown(self, sepName, checkbox):
        print(f'updating dropdown with {sepName}')
        self.separatorBoolean[sepName] = checkbox.isChecked()
        self.dropDown.clear()
        for sepName, state in self.separatorBoolean.items():
            if state is True:
                self.dropDown.addItem(sepName)

        # set slider right acording to current dropdown state
        if not any(self.separatorBoolean.values()):
            self.slider.setValue(0)
            self.slider.setDisabled(True)
        else:
            self._selectSlider(self.dropDown.currentIndex())

        # erase lines and text if box unchecked
        for sepName, state in self.separatorBoolean.items():
            if state is False and self.lines[sepName] is not None:
                print(f'removing line for {sepName}')
                self.lines[sepName].remove()
                self.lines[sepName] = None
                self.labels[sepName].remove()
                self.labels[sepName] = None
                self.canvas.draw()

    def _createActionButtons(self):
        buttonGroup = QGroupBox('cellDB actions')
        buttonLayout = QHBoxLayout()

        self.saveButton = QPushButton('save layers')

        buttonLayout.addWidget(self.saveButton)
        buttonGroup.setLayout(buttonLayout)
        self.generalLayout.addWidget(buttonGroup, 3, 3, 1, 1)

    def _createDisplay(self):

        self.display = QLineEdit()
        self.display.setAlignment(Qt.AlignLeft)
        self.display.setReadOnly(True)
        self.generalLayout.addWidget(self.display, 4, 3, 1, 1)

    def _updateDisplay(self):
        return None

    def _drawSeparator(self, value):
        currentSlider = self.dropDown.currentText()

        # catches when the dropdown is empty
        if currentSlider == '':
            return

        # draw or redraw the line
        if self.lines[currentSlider] is not None:
            print('erasing old')
            self.lines[currentSlider].remove()
            self.labels[currentSlider].remove()

        print(f'drawing {currentSlider} at {value}')
        self.lines[currentSlider] = self.ax.axhline(value, color='white', linewidth=5)
        top, bottom = currentSlider.split('/')
        labelText = f'{top}\n{bottom}'
        self.labels[currentSlider] = self.ax.text(0, value-4, labelText, color='white', fontsize=40)

        self.canvas.draw()

class DepthMapCtrl():
    def __init__(self, model, view):
        self._view = view
        self._model = model
        self._model._drawCSD(self._view.ax)
        self._connectSignals()

    def testState(self, boxName):
        checkBox  = self._view.layerCheckBoxes[boxName]
        if checkBox.isChecked() == True:
            t = f'{boxName} on'
        else:
            t = f'{boxName} off'

        print(t)
        self._view.display.setText(t)
        return boxName

    def _connectSignals(self):
        # set the subset of separators to consider
        for boxName, checkBox in  self._view.layerCheckBoxes.items():
            checkBox.stateChanged.connect(partial(self._view._updateDropDown,
                                                  boxName, checkBox))

        self._view.dropDown.activated.connect(self._view._selectSlider)

        self._view.slider.valueChanged.connect(self._view._updateSlider)
        self._view.slider.valueChanged.connect(self._view._drawSeparator)

class DepthMapModel():
    def __init__(self):
        self.layerBorders = ['1/2', '2/3', '3/4', '4/5', '5/6']
        self.separatorBoolean = {border: False for border in self.layerBorders}
        self.separatorPosition = {border: 0 for border in self.layerBorders}
        self.lines = {}
        self.labels = {}
        self.file = ''
        self.csd = []
        self.currenteSeparator=''

    def _drawCSD(self, ax):
        # ax.imshow(np.random.rand(100, 25), aspect='equal', origin='lower', cmap='inferno')
        ax.imshow(np.zeros((100, 25)), aspect='equal', origin='lower', cmap='inferno')
        ax.get_xaxis().set_visible(False)
        ax.figure.tight_layout(pad=0)

    # def _drawSeparators(self, ax, separators):
    #     lines = list()
    #     labels = list()
    #     for separator,  position in separators.values():
    #         lines.append(ax.axhline(position))
    #         labels.append(ax.text(0, position, separator))
    #
    # def _clearSeparators(self, lines, lables):
    #     for line, label in zip(lines, lables):
    #         line.remove()
    #         label.remove()

# Client code
def main():
    """Main function."""
    # Create an instance of QApplication
    pycalc = QApplication(sys.argv)
    # Show the calculator's GUI
    view = DepthMapUi()
    view.show()
    # get model functions
    model = DepthMapModel()
    # create instance of the controller
    DepthMapCtrl(model=model, view=view)
    # Execute the calculator's main loop
    sys.exit(pycalc.exec_())

if __name__ == '__main__':
    main()


text:
import cvxpy as cvx
import numpy as np
from scipy.linalg import sqrtm

import matplotlib
matplotlib.use('tkagg')
from matplotlib import pyplot as plt
plt.rc('figure', figsize=[10, 6])

class ControlModel(object):
    """
    Models used to find optimal investment strategies.
    """

    def __init__(self):
        pass

    def run(self, data):
        pass

    def optima(self):
        pass

    def variables(self):
        pass

    def get_input(self, L, past_data, ar_projections, ar_errors, bank_rate=1.0):
        pass

    def apply_model_results(self, true_x):
        pass

class NormModel(ControlModel):

    def __init__(self, num_assets, gamma=1.0, norm=1, nu=0.01):
        super(ControlModel, CovarianceModel).__init__(self)
        self.num_assets = num_assets
        self.gamma = gamma
        self.norm = norm
        self.nu = nu
        self.x = None
        self.x0 = None
        self.problem = None
        self._optima = None

    def run(self, data):
        x0, mu, sigma = data
        self.x0 = x0

        self.x = cvx.Variable(self.num_assets+1)

        objective = self.x.T*mu - self.gamma*cvx.norm(self.x, self.norm)

        self.problem = cvx.Problem(cvx.Maximize(objective),
                           [
                               cvx.norm(self.x, 1) <= 1 - self.nu * cvx.norm(self.x - x0, 1),
                               self.x >= 0
                           ])
        self._optima = self.problem.solve()

    def optima(self):
        return self._optima

    def variables(self):
        return self.x.value.flatten()

    def get_input(self, L, past_data, ar_projections, ar_errors, bank_rate=1.0):
        proj = ar_projections.T[:,0]
        proj = np.concatenate((proj, np.array([bank_rate])))
        if len(ar_errors.shape) == 3:
            # janky way to tell what kind of error we have available.
            return proj, ar_errors[0]
        else:
            return proj, ar_errors.reshape(ar_errors.shape[0], 1)[:,0]

    def apply_model_results(self, true_x):
        new_x = self.variables()
        updated_x = true_x.copy()
        # modify updated_x with sell/buy changes.
        sell, buy = np.maximum(0, -(new_x - true_x)), np.maximum(0, (new_x - true_x))
        updated_x = updated_x - sell + buy
        return updated_x

class CovarianceModel(ControlModel):

    def __init__(self, num_assets, gamma=1.0, nu=0.01):
        super(ControlModel, CovarianceModel).__init__(self)
        self.num_assets = num_assets
        self.gamma = gamma
        self.nu = nu
        self.x = None
        self.x0 = None
        self.problem = None
        self._optima = None

    def run(self, data):
        x0, mu, sigma = data
        self.x0 = x0

        self.x = cvx.Variable(self.num_assets+1)

        objective = self.x.T*mu - self.gamma*cvx.quad_form(self.x, sigma)

        self.problem = cvx.Problem(cvx.Maximize(objective),
                           [
                               cvx.norm(self.x, 1) <= 1 - self.nu * cvx.norm(self.x - x0, 1),
                               self.x >= 0
                           ])
        self._optima = self.problem.solve()

    def optima(self):
        return self._optima

    def variables(self):
        return self.x.value.flatten()

    def get_input(self, L, past_data, ar_projections, ar_errors, bank_rate=1.0):
        proj = ar_projections.T[:,0]
        proj = np.concatenate((proj, np.array([bank_rate])))

        if len(ar_errors.shape) == 3:
            # janky way to tell what kind of error we have available.
            sigma_small = ar_errors[0]
            sigma = np.zeros(shape=(sigma_small.shape[0] + 1, sigma_small.shape[1] + 1))
            sigma[:-1, :-1] = sigma_small
            sigma[-1, -1] = 1.0
            return proj, sigma
        else:
            sigma = ar_errors.reshape(ar_errors.shape[0], 1)[:, 0]
            sigma = np.concatenate((sigma, np.array([0.0])))
            sigma = np.diag(sigma)
        return proj, sigma

    def apply_model_results(self, true_x):
        new_x = self.variables()
        updated_x = true_x.copy()
        # modify updated_x with buy/sell changes.
        sell, buy = np.maximum(0, -(new_x - true_x)), np.maximum(0, (new_x - true_x))
        updated_x = updated_x - sell + buy
        return updated_x

class MultiPeriodModel(ControlModel):
    """
    http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.116.559&rep=rep1&type=pdf
    page 4
    """
    def __init__(self, num_assets, L, theta, nu):
        self.L = L # planning horizon
        self.theta = theta # safety margin on std dev
        self.nu = nu # transaction cost
        self.num_assets = num_assets
        self.R = None
        self.xi = cvx.Variable((num_assets+1, L+1))
        self.eta = cvx.Variable((num_assets+1, L))
        self.zeta = cvx.Variable((num_assets+1, L))
        self.omega = cvx.Variable()
        self.problem = None
        self._optima = None

    def run(self, data):
        # x0 n x 1 initial state of portfolio,
        # returns n x L expected return at each time step,
        # sigmas n x n x L variance at each time step
        x0, returns, _ = data
        self.R = np.cumprod(returns, axis=1)
        # print("R:",self.R)
        objective = cvx.Maximize(self.omega)
        constraints = [self.omega <= self.R[:,self.L].T @ self.xi[:,-1],
                       self.zeta >= 0, self.xi >= 0, self.eta >= 0,
                       self.xi[:,0] == np.divide(x0, self.R[:,0]),
                       self.xi[-1,1:] == 0]
        A = (1 - self.nu) * self.R
        B = (1 + self.nu) * self.R
        for l in range(1, self.L + 1):
            # Equation 1.9
            constraints += [0 == -self.xi[:,l] + self.xi[:,l-1] - self.eta[:,l-1] + self.zeta[:,l-1],
                            0 <= A[:,l-1].T @ self.eta[:,l-1] - B[:,l-1] @ self.zeta[:,l-1]]
        self.problem = cvx.Problem(objective, constraints)
        self._optima = self.problem.solve()
        # print(self.problem.status)

    def optima(self):
        return self._optima

    def variables(self):
        zeta = self.zeta.value
        eta = self.eta.value
        xi = self.xi.value
        R = self.R
        return xi * R, eta * R[:,1:], zeta * R[:, 1:]

    def get_input(self, L, past_data, ar_projections, ar_errors, bank_rate=1.0):
        num_assets = self.num_assets

        ar_variances = np.zeros((num_assets, L))
        ar_variances[:, 1:] = np.repeat(ar_errors.reshape(-1, 1), L - 1, axis=1)
        variances = np.zeros((num_assets + 1, L))
        variances[:-1, :] = ar_variances[:, :]

        projections = np.ones((num_assets + 1, L)) * bank_rate
        projections[:-1, 0] = past_data.T[:, -1]
        projections[:-1, 1:] = ar_projections.T[:, :-1]
        # assert np.allclose(projections[-1, :], bank_rate)

        return projections, variances

    def apply_model_results(self, true_x):
        updated_x = true_x.copy()
        # print(true_x.shape)
        # modify updated_x with buy/sell changes.
        _, sell_all, buy_all = self.variables()
        sell, buy = sell_all[:, 0], buy_all[:, 0]
        updated_x = true_x - sell + buy
        return updated_x

class RobustMultiPeriodModel(ControlModel):
    """
    http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.116.559&rep=rep1&type=pdf
    page 15

    Note: Here we use theta as a scaling parameter on the variance, xi^T V xi,
    rather than on the standard deviation sqrt(xi^T V xi) to ease implementation in CVX
    """
    def __init__(self, num_assets, L, theta, nu):
        self.L = L # planning horizon
        self.theta = theta # safety margin on std dev
        self.nu = nu # transaction cost
        self.num_assets = num_assets
        self.R = None
        self.xi = cvx.Variable((num_assets+1, L+1))
        self.zeta = cvx.Variable((num_assets+1, L))
        self.eta = cvx.Variable((num_assets+1, L))
        self.omega = cvx.Variable()
        self.problem = None
        self._optima = None

    def run(self, data):
        # x0 n x 1 initial state of portfolio,
        # log_returns n x L expected log return at each time step,
        # sigmas n x L expected log variance at each time step
        x0, log_returns, log_vars = data

        # Expectations
        ExpR = np.exp(np.cumsum(log_returns + 0.5 * log_vars, axis=1))
        self.R = ExpR
        pl = [ np.concatenate([(1-self.nu) * ExpR[:,l], -(1+self.nu) * ExpR[:,l]]) for l in range(self.L) ]
        pLp1 = ExpR[:,-1]

        # Covariances
        VarR = (np.exp(np.cumsum(2 * log_returns + log_vars, axis=1)) * np.exp(np.cumsum(log_vars, axis=1)) -
                np.exp(np.cumsum(2 * log_returns + log_vars, axis=1)))
        Cl = [ np.diag(VarR[:,l]) for l in range(1, self.L+1) ]
        Vl = [ np.bmat( [[(1 - self.nu) ** 2 * C, -(1-self.nu) * (1 + self.nu) * C],
                         [-(1-self.nu) * (1 + self.nu) * C, (1 + self.nu) ** 2 * C]] ) for C in Cl ]
        VLp1 = Cl[-1]

        # print(Vl[0])
        # np.save("vl0", Vl[0])
        # print(sqrtm(Vl[0]))
        # print(np.linalg.norm(Vl[0]))

        objective = cvx.Maximize(self.omega)
        constraints = [self.omega <= pLp1 @ self.xi[:,self.L] - self.theta * cvx.norm(sqrtm(VLp1 + np.eye(VLp1.shape[0])*1e-8) @ self.xi[:,-1], 2),
                       self.xi >= 0, self.eta >= 0, self.zeta >= 0,
                       self.xi[:,0] == np.divide(x0, self.R[:,0]),
                       self.xi[-1,1:] == 0]
        alpha = (1 - self.nu) * self.R
        beta = (1 + self.nu) * self.R

        for l in range(1, self.L + 1):
            # Equation 1.9
            constraints += [0 == -self.xi[:,l] + self.xi[:,l-1] - self.eta[:,l-1] + self.zeta[:,l-1],
                            0 <= (alpha[:,l-1].T @ self.eta[:,l-1] - beta[:,l-1] @ self.zeta[:,l-1] -
                                  self.theta * cvx.norm(sqrtm(Vl[l-1] + np.eye(Vl[l-1].shape[0])*1e-8) @ cvx.bmat([[self.eta[:,l-1], self.zeta[:,l-1]]]).T, 2))
                            ]
        self.problem = cvx.Problem(objective, constraints)
        self._optima = self.problem.solve()
        print(self.problem.status)

    def optima(self):
        return self._optima

    def variables(self):
        zeta = self.zeta.value
        eta = self.eta.value
        xi = self.xi.value
        R = self.R
        return xi * R, eta * R[:,1:], zeta * R[:, 1:]

    def get_input(self, L, past_data, ar_projections, ar_errors, bank_rate=1.0):
        num_assets = self.num_assets

        ar_variances = np.zeros((num_assets, L-1))
        ar_variances[:, :] = np.repeat(ar_errors.reshape(-1, 1), L - 1, axis=1)

        # We want projections and variances in log space
        projections = np.zeros((num_assets + 1, L))
        projections[:-1, 0] = np.log(past_data.T[:, -1])
        # Technically, if you have normal data there would be a variance term here too,
        # but I'm not sure that makes sense for the autoregression
        projections[:-1, 1:] = np.log(ar_projections.T[:, :-1])

        variances = np.zeros((num_assets + 1, L))
        # This is a first-order approximation for normally distributed data, but it's the best we have for now
        variances[:-1, 1:] = ar_variances[:, :] / np.square(ar_projections.T[:, :-1])

        return projections, variances

    def apply_model_results(self, true_x):
        updated_x = true_x.copy()
        if self.problem.status != cvx.OPTIMAL:
            return updated_x
        # modify updated_x with buy/sell changes.
        _, sell_all, buy_all = self.variables()
        sell, buy = sell_all[:, 0], buy_all[:, 0]
        updated_x = updated_x - sell + buy
        return updated_x

def main():
    data = RealData()
    all_samples = data.sample()
    num_samples = 50
    num_assets = all_samples.shape[1]
    samples = all_samples[:num_samples]
    for i in range(samples.shape[1]):
        print(samples.T[i])

    L = 10
    ar = AutoRegression(L)
    ar.fit(samples)
    ar_projections, ar_errors = ar.predict(samples, L)
    projection_truth = all_samples[num_samples:num_samples+L]
    print("projection_errors", np.abs(projection_truth - ar_projections))

    # Run models
    mpc = MultiPeriodModel(num_assets, L-1, 2, .1)
    rmpc = RobustMultiPeriodModel(num_assets, L-1, 2, .1)

    projections, variances = mpc.get_input(L, samples, ar_projections, ar_errors)

    x0 = np.zeros((num_assets+1,))
    x0[-1] = 1.0

    mpc.run(data=(x0, projections, None))

    x, y, z = mpc.variables()
    print("x:",x)
    print("y:",y)
    print("z:",z)
    print(mpc.optima())

    rmpc.run(data=(x0, np.log(projections), variances))

    rx, ry, rz = rmpc.variables()
    print("rx:",rx)
    print("ry:",ry)
    print("rz:",rz)
    print(rmpc.optima())

    plt.plot(x.T,label='x')
    plt.plot(rx.T,':',label='rx')
    plt.legend()
    plt.show()

    plt.plot(y.T,label='y')
    plt.plot(ry.T,':',label='ry')
    plt.legend()
    plt.show()

    plt.plot(z.T,label='z')
    plt.plot(rz.T,':',label='rz')
    plt.legend()
    plt.show()

if __name__ == "__main__":
    from data_models import GaussianNoise, NoisySine, RealData
    from prediction_models import UnbiasGaussianEstimator, AutoRegression

    # num_samples = 1000
    # num_assets = 3

    # mu_truth = np.ones(num_assets)
    # sigma_truth = np.diag([0.5, 0.3, 0.2])
    #
    # sampler = GaussianNoise()
    # data = np.zeros(shape=(num_samples, num_assets))
    #
    # for i in range(num_samples):
    #     data[i] = sampler.sample((mu_truth, sigma_truth))
    #
    # sample_mean, sample_covar = UnbiasGaussianEstimator().predict(data)
    #
    # for i in range(num_assets):
    #     print(sample_mean[i], sample_covar[i])

    # mpc = MultiPeriodModel(num_assets, 2, 2, .1)
    # x0 = np.ones((num_assets,)) / num_assets
    # sample_mean[0] = 1.1
    # sample_mean[1] = 0.9
    # means = np.repeat(sample_mean.reshape(-1, 1), 3, 1)
    # covs = np.repeat(sample_covar.reshape(-1, 1), 3, 1)
    # mpc.run(data=(x0, means, covs))
    #
    # x, y, z = mpc.variables()
    # print("x:", x)
    # print("y:", y)
    # print("z:", z)
    # print(mpc.optima())
    #
    # cov_model = CovarianceModel(num_assets=num_assets)
    # cov_model.run(data=(sample_mean, sample_covar), gamma=1.0)

    # print(cov_model.variables())
    # print(cov_model.optima())

    # noisy sine
    # data = NoisySine()
    # phase = np.array([1., .5, 2.])
    # noise = np.array([0.1, 0.03, 0.2])
    # samples = data.sample((phase, noise, 20))
    # for i in range(samples.shape[1]):
    #     print(samples.T[i])
    main()

reason: import numpy,from scipy,import cvxpy


text:
# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
.. module:: centering360
   :platform: Unix
   :synopsis: A plugin to find the center of rotation per frame
.. moduleauthor:: Nghia Vo, <scientificsoftware@diamond.ac.uk>
"""
from savu.plugins.driver.cpu_plugin import CpuPlugin
from savu.plugins.utils import register_plugin
from savu.plugins.filters.base_filter import BaseFilter
import savu.core.utils as cu

import logging
import numpy as np
import scipy.ndimage as ndi
from scipy import stats

@register_plugin
class Centering360(BaseFilter, CpuPlugin):

    def __init__(self):
        super(Centering360, self).__init__("Centering360")

    def find_overlap(self, mat1, mat2, win_width, side=None, denoise=True, norm=False,
                     use_overlap=False):
        """
        Find the overlap area and overlap side between two images (Ref. [1]) where
        the overlap side referring to the first image.

        Parameters
        ----------
        mat1 : array_like
            2D array. Projection image or sinogram image.
        mat2 :  array_like
            2D array. Projection image or sinogram image.
        win_width : int
            Width of the searching window.
        side : {None, 0, 1}, optional
            Only there options: None, 0, or 1. "None" corresponding to fully
            automated determination. "0" corresponding to the left side. "1"
            corresponding to the right side.
        denoise : bool, optional
            Apply the Gaussian filter if True.
        norm : bool, optional
            Apply the normalization if True.
        use_overlap : bool, optional
            Use the combination of images in the overlap area for calculating
            correlation coefficients if True.

        Returns
        -------
        overlap : float
            Width of the overlap area between two images.
        side : int
            Overlap side between two images.
        overlap_position : float
            Position of the window in the first image giving the best
            correlation metric.

        References
        ----------
        .. [1] https://doi.org/10.1364/OE.418448
        """
        (_, ncol1) = mat1.shape
        (_, ncol2) = mat2.shape
        win_width = np.int16(np.clip(win_width, 6, min(ncol1, ncol2) // 2))
        if side == 1:
            (list_metric, offset) = self.search_overlap(mat1, mat2, win_width, side,
                                                   denoise, norm, use_overlap)
            (_, overlap_position) = self.calculate_curvature(list_metric)
            overlap_position = overlap_position + offset
            overlap = ncol1 - overlap_position + win_width // 2
        elif side == 0:
            (list_metric, offset) = self.search_overlap(mat1, mat2, win_width, side,
                                                   denoise, norm, use_overlap)
            (_, overlap_position) = self.calculate_curvature(list_metric)
            overlap_position = overlap_position + offset
            overlap = overlap_position + win_width // 2
        else:
            (list_metric1, offset1) = self.search_overlap(mat1, mat2, win_width, 1, norm,
                                                     denoise, use_overlap)
            (list_metric2, offset2) = self.search_overlap(mat1, mat2, win_width, 0, norm,
                                                     denoise, use_overlap)
            (curvature1, overlap_position1) = self.calculate_curvature(list_metric1)
            overlap_position1 = overlap_position1 + offset1
            (curvature2, overlap_position2) = self.calculate_curvature(list_metric2)
            overlap_position2 = overlap_position2 + offset2
            if curvature1 > curvature2:
                side = 1
                overlap_position = overlap_position1
                overlap = ncol1 - overlap_position + win_width // 2
            else:
                side = 0
                overlap_position = overlap_position2
                overlap = overlap_position + win_width // 2
        return overlap, side, overlap_position

    def search_overlap(self, mat1, mat2, win_width, side, denoise=True, norm=False,
                       use_overlap=False):
        """
        Calculate the correlation metrics between a rectangular region, defined
        by the window width, on the utmost left/right side of image 2 and the
        same size region in image 1 where the region is slided across image 1.

        Parameters
        ----------
        mat1 : array_like
            2D array. Projection image or sinogram image.
        mat2 : array_like
            2D array. Projection image or sinogram image.
        win_width : int
            Width of the searching window.
        side : {0, 1}
            Only two options: 0 or 1. It is used to indicate the overlap side
            respects to image 1. "0" corresponds to the left side. "1" corresponds
            to the right side.
        denoise : bool, optional
            Apply the Gaussian filter if True.
        norm : bool, optional
            Apply the normalization if True.
        use_overlap : bool, optional
            Use the combination of images in the overlap area for calculating
            correlation coefficients if True.

        Returns
        -------
        list_metric : array_like
            1D array. List of the correlation metrics.
        offset : int
            Initial position of the searching window where the position
            corresponds to the center of the window.
        """
        if denoise is True:
            mat1 = ndi.gaussian_filter(mat1, (2, 2), mode='reflect')
            mat2 = ndi.gaussian_filter(mat2, (2, 2), mode='reflect')
        (nrow1, ncol1) = mat1.shape
        (nrow2, ncol2) = mat2.shape
        if nrow1 != nrow2:
            raise ValueError("Two images are not at the same height!!!")
        win_width = np.int16(np.clip(win_width, 6, min(ncol1, ncol2) // 2 - 1))
        offset = win_width // 2
        win_width = 2 * offset  # Make it even
        ramp_down = np.linspace(1.0, 0.0, win_width)
        ramp_up = 1.0 - ramp_down
        wei_down = np.tile(ramp_down, (nrow1, 1))
        wei_up = np.tile(ramp_up, (nrow1, 1))
        if side == 1:
            mat2_roi = mat2[:, 0:win_width]
            mat2_roi_wei = mat2_roi * wei_up
        else:
            mat2_roi = mat2[:, ncol2 - win_width:]
            mat2_roi_wei = mat2_roi * wei_down
        list_mean2 = np.mean(np.abs(mat2_roi), axis=1)
        list_pos = np.arange(offset, ncol1 - offset)
        num_metric = len(list_pos)
        list_metric = np.ones(num_metric, dtype=np.float32)
        for i, pos in enumerate(list_pos):
            mat1_roi = mat1[:, pos - offset:pos + offset]
            if use_overlap is True:
                if side == 1:
                    mat1_roi_wei = mat1_roi * wei_down
                else:
                    mat1_roi_wei = mat1_roi * wei_up
            if norm is True:
                list_mean1 = np.mean(np.abs(mat1_roi), axis=1)
                list_fact = list_mean2 / list_mean1
                mat_fact = np.transpose(np.tile(list_fact, (win_width, 1)))
                mat1_roi = mat1_roi * mat_fact
                if use_overlap is True:
                    mat1_roi_wei = mat1_roi_wei * mat_fact
            if use_overlap is True:
                mat_comb = mat1_roi_wei + mat2_roi_wei
                list_metric[i] = (self.correlation_metric(mat1_roi, mat2_roi)
                                  + self.correlation_metric(mat1_roi, mat_comb)
                                  + self.correlation_metric(mat2_roi, mat_comb)) / 3.0
            else:
                list_metric[i] = self.correlation_metric(mat1_roi, mat2_roi)
        min_metric = np.min(list_metric)
        if min_metric != 0.0:
            list_metric = list_metric / min_metric
        return list_metric, offset

    def correlation_metric(self, mat1, mat2):
        """
        Calculate the correlation metric. Smaller metric corresponds to better
        correlation.

        Parameters
        ---------
        mat1 : array_like
        mat2 : array_like

        Returns
        -------
        float
            Correlation metric.
        """
        metric = np.abs(
            1.0 - stats.pearsonr(mat1.flatten('F'), mat2.flatten('F'))[0])
        return metric

    def calculate_curvature(self, list_metric):
        """
        Calculate the curvature of a fitted curve going through the minimum
        value of a metric list.

        Parameters
        ----------
        list_metric : array_like
            1D array. List of metrics.

        Returns
        -------
        curvature : float
            Quadratic coefficient of the parabola fitting.
        min_pos : float
            Position of the minimum value with sub-pixel accuracy.
        """
        radi = 2
        num_metric = len(list_metric)
        min_pos = np.clip(
            np.argmin(list_metric), radi, num_metric - radi - 1)
        list1 = list_metric[min_pos - radi:min_pos + radi + 1]
        (afact1, _, _) = np.polyfit(np.arange(0, 2 * radi + 1), list1, 2)
        list2 = list_metric[min_pos - 1:min_pos + 2]
        (afact2, bfact2, _) = np.polyfit(
            np.arange(min_pos - 1, min_pos + 2), list2, 2)
        curvature = np.abs(afact1)
        if afact2 != 0.0:
            num = - bfact2 / (2 * afact2)
            if (num >= min_pos - 1) and (num <= min_pos + 1):
                min_pos = num
        return curvature, np.float32(min_pos)

    def _downsample(self, image, dsp_fact0, dsp_fact1):
        """Downsample an image by averaging.

        Parameters
        ----------
            image : 2D array.
            dsp_fact0 : downsampling factor along axis 0.
            dsp_fact1 : downsampling factor along axis 1.

        Returns
        ---------
            image_dsp : Downsampled image.
        """
        (height, width) = image.shape
        dsp_fact0 = np.clip(np.int16(dsp_fact0), 1, height // 2)
        dsp_fact1 = np.clip(np.int16(dsp_fact1), 1, width // 2)
        height_dsp = height // dsp_fact0
        width_dsp = width // dsp_fact1
        if dsp_fact0 == 1 and dsp_fact1 == 1:
            image_dsp = image
        else:
            image_dsp = image[0:dsp_fact0 * height_dsp, 0:dsp_fact1 * width_dsp]
            image_dsp = image_dsp.reshape(
                height_dsp, dsp_fact0, width_dsp, dsp_fact1).mean(-1).mean(1)
        return image_dsp

    def pre_process(self):
        self.win_width = np.int16(self.parameters['win_width'])
        self.side = self.parameters['side']
        self.denoise = self.parameters['denoise']
        self.norm = self.parameters['norm']
        self.use_overlap = self.parameters['use_overlap']

        self.broadcast_method = str(self.parameters['broadcast_method'])
        self.error_msg_1 = ""
        self.error_msg_2 = ""
        self.error_msg_3 = ""
        if not ((self.broadcast_method == 'mean')
                or (self.broadcast_method == 'median')
                or (self.broadcast_method == 'linear_fit')
                or (self.broadcast_method == 'nearest')):
            self.error_msg_3 = "!!!WARNING!!! Selected broadcasting method" \
                               " is out of the list. Use the default option:" \
                               " 'median'"
            logging.warning(self.error_msg_3)
            cu.user_message(self.error_msg_3)
            self.broadcast_method = 'median'
        in_pData = self.get_plugin_in_datasets()[0]
        data = self.get_in_datasets()[0]
        starts, stops, steps = data.get_preview().get_starts_stops_steps()[0:3]
        start_ind = starts[1]
        stop_ind = stops[1]
        step_ind = steps[1]
        name = data.get_name()
        pre_start = self.exp.meta_data.get(name + '_preview_starts')[1]
        pre_stop = self.exp.meta_data.get(name + '_preview_stops')[1]
        pre_step = self.exp.meta_data.get(name + '_preview_steps')[1]
        self.origin_prev = np.arange(pre_start, pre_stop, pre_step)
        self.plugin_prev = self.origin_prev[start_ind:stop_ind:step_ind]
        num_sino = len(self.plugin_prev)
        if num_sino > 20:
            warning_msg = "\n!!!WARNING!!! You selected to calculate the " \
                          "center-of-rotation using '{}' sinograms.\n" \
                          "This is computationally expensive. Considering to " \
                          "adjust the preview parameter to use\na smaller " \
                          "number of sinograms (< 20).\n".format(num_sino)
            logging.warning(warning_msg)
            cu.user_message(warning_msg)

    def process_frames(self, data):
        """
        Find the center-of-rotation (COR) in a 360-degree scan with offset COR use
        the method presented in Ref. [1].

        Parameters
        ----------
        data : array_like
            2D array. 360-degree sinogram.

        Returns
        -------
        cor : float
            Center-of-rotation.

        References
        ----------
        .. [1] https://doi.org/10.1364/OE.418448
        """
        sino = data[0]
        (nrow, ncol) = sino.shape
        nrow_180 = nrow // 2 + 1
        sino_top = sino[0:nrow_180, :]
        sino_bot = np.fliplr(sino[-nrow_180:, :])
        overlap, side, overlap_position =\
            self.find_overlap(sino_top, sino_bot, self.win_width, self.side,
                              self.denoise, self.norm, self.use_overlap)
        #overlap : Width of the overlap area between two halves
        #           of the sinogram.
        # side : Overlap side between two halves of the sinogram.
        # overlap_position : Position of the window in the first
        #           image giving the best correlation metric."""
        if side == 0:
            cor = overlap / 2.0 - 1.0
        else:
            cor = ncol - overlap / 2.0 - 1.0
        return [np.array([cor]), np.array([cor])]

    def post_process(self):
        in_datasets, out_datasets = self.get_datasets()
        cor_prev = out_datasets[0].data[...]
        cor_broad = out_datasets[1].data[...]
        cor_broad[:] = np.median(np.squeeze(cor_prev))
        self.cor_for_executive_summary = np.median(cor_broad[:])
        if self.broadcast_method == 'mean':
            cor_broad[:] = np.mean(np.squeeze(cor_prev))
            self.cor_for_executive_summary = np.mean(cor_broad[:])
        if (self.broadcast_method == 'linear_fit') and (len(cor_prev) > 1):
            afact, bfact = np.polyfit(self.plugin_prev, cor_prev[:, 0], 1)
            list_cor = self.origin_prev * afact + bfact
            cor_broad[:, 0] = list_cor
            self.cor_for_executive_summary = cor_broad[:]
        if (self.broadcast_method == 'nearest') and (len(cor_prev) > 1):
            for i, pos in enumerate(self.origin_prev):
                minpos = np.argmin(np.abs(pos - self.plugin_prev))
                cor_broad[i, 0] = cor_prev[minpos, 0]
            self.cor_for_executive_summary = cor_broad[:]
        out_datasets[1].data[:] = cor_broad[:]
        self.populate_meta_data('cor_preview', np.squeeze(cor_prev))
        self.populate_meta_data('centre_of_rotation',
                                out_datasets[1].data[:].squeeze(axis=1))

    def populate_meta_data(self, key, value):
        datasets = self.parameters['datasets_to_populate']
        in_meta_data = self.get_in_meta_data()[0]
        in_meta_data.set(key, value)
        for name in datasets:
            self.exp.index['in_data'][name].meta_data.set(key, value)

    def setup(self):
        self.exp.log(self.name + " Start calculating center of rotation")
        # set up the output dataset that is created by the plugin
        in_dataset, out_dataset = self.get_datasets()
        in_pData, out_pData = self.get_plugin_datasets()
        in_pData[0].plugin_data_setup('SINOGRAM', self.get_max_frames())
        slice_dirs = list(in_dataset[0].get_slice_dimensions())
        self.orig_full_shape = in_dataset[0].get_shape()

        # reduce the data as per data_subset parameter
        self.set_preview(in_dataset[0], self.parameters['preview'])
        total_frames = \
            self._calc_total_frames(in_dataset[0].get_preview(), slice_dirs)

        # copy all required information from in_dataset[0]
        fullData = in_dataset[0]
        new_shape = (np.prod(np.array(fullData.get_shape())[slice_dirs]), 1)
        self.orig_shape = \
            (np.prod(np.array(self.orig_full_shape)[slice_dirs]), 1)
        out_dataset[0].create_dataset(shape=new_shape,
                                      axis_labels=['x.pixels', 'y.pixels'],
                                      remove=True,
                                      transport='hdf5')
        out_dataset[0].add_pattern("METADATA", core_dims=(1,), slice_dims=(0,))

        out_dataset[1].create_dataset(shape=self.orig_shape,
                                      axis_labels=['x.pixels', 'y.pixels'],
                                      remove=True,
                                      transport='hdf5')
        out_dataset[1].add_pattern("METADATA", core_dims=(1,), slice_dims=(0,))
        out_pData[0].plugin_data_setup('METADATA', self.get_max_frames())
        out_pData[1].plugin_data_setup('METADATA', self.get_max_frames())
        out_pData[1].meta_data.set('fix_total_frames', total_frames)
        self.exp.log(self.name + " End")

    def _calc_total_frames(self, preview, slice_dims):
        starts, stops, steps, _ = preview.get_starts_stops_steps()
        lengths = [len(np.arange(starts[i], stops[i], steps[i]))
                   for i in range(len(starts))]
        return np.prod([lengths[i] for i in slice_dims])

    def nOutput_datasets(self):
        return 2

    def get_max_frames(self):
        return 'single'

    def fix_transport(self):
        return 'hdf5'

    def executive_summary(self):
        if ((self.error_msg_1 == "")
                and (self.error_msg_2 == "")):
            msg = "Centre of rotation is : %s" % (
                str(self.cor_for_executive_summary))
        else:
            msg = "\n" + self.error_msg_1 + "\n" + self.error_msg_2
            msg2 = "(Not well) estimated centre of rotation is : %s" % (str(
                self.cor_for_executive_summary))
            cu.user_message(msg2)
        return [msg]

max_stars_repo_name: elainehoml/Savu

max_issues_repo_name: elainehoml/Savu

max_forks_repo_name: elainehoml/Savu

reason: import numpy,import scipy,from scipy


text:
import os
import jieba
import ipdb
import pickle
from tqdm import tqdm
import numpy as np

def load_w2v(path):
    '''
    Load the `text` embedding file, such as glove and tencent chinese word embedding
    '''
    if os.path.exists(f'{path}.pkl'):
        with open(f'{path}.pkl', 'rb') as f:
            w2v = pickle.load(f)
            print(f'[!] load w2v from {path}.pkl, size: {len(w2v)}')
            return w2v
    with open(f'{path}.txt') as f:
        w2v = {}
        for line in tqdm(f.readlines()):
            line = line.split()
            word, vec = line[0], line[1:]
            vec = np.array(list(map(float, vec)))
            assert len(vec) == 300, f'[!] except the vector 300, got {len(vec)}'
            w2v[word] = vec
    with open(f'{path}.pkl', 'wb') as f:
        pickle.dump(w2v, f)
    print(f'[!] write w2v into {path}.pkl, size: {len(w2v)}')
    return w2v

def convert_text_embedding(w2v, texts):
    '''
    w2v is the word embedding dict obtained by `load_w2v`
    return the 300 dimension embedding of the texts, the [UNK] tokens are not considered
    The embedding is much faster than the Bert-as-service
    '''
    e = []
    for text in texts:
        words = list(jieba.cut(text))
        vectors = []
        for w in words:
            if w in w2v:
                vectors.append(w2v[w])
        if not vectors:
            vectors.append(np.random.randn(300))
        vectors = np.stack(vectors).mean(axis=0)    # [words, 300] -> [300]
        e.append(vectors)
    return e    # [batch, 300]

if __name__ == "__main__":
    w2v = load_w2v('data/chinese_w2v')
    e = convert_text_embedding(w2v, ['我非常喜欢看电影', '我不认为电影很好看'])
    ipdb.set_trace()

max_stars_repo_name: zhongerqiandan/OpenDialog

max_issues_repo_name: zhongerqiandan/OpenDialog

max_forks_repo_name: gmftbyGMFTBY/OpenDialog


text:
import numpy as np
import matplotlib.pylab as plt
from dataset.mnist import load_mnist
from two_layer_net import TwoLayerNet

(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)
train_loss_list = []

print("x_train=", x_train.shape)
print("t_train=", t_train.shape)

print("x_test=", x_test.shape)
print("t_test=", t_test.shape)

iters_num = 1
train_size = x_train.shape[0]
batch_size = 10
print("train_size=", train_size, "  batch_size=", batch_size)
learning_rate = 0.1
network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)

for i in range(iters_num) :
    batch_mask = np.random.choice(train_size, batch_size)
    print("batch_mask=", batch_mask.shape)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]
    print("x_batch=", x_batch.shape)
    print("t_batch=", t_batch.shape)

    grad = network.numerical_gradient(x_batch, t_batch)
    print("grad W1=", grad['W1'].shape)
    print("grad b1=", grad['b1'].shape)
    print("grad W2=", grad['W2'].shape)
    print("grad b2=", grad['b2'].shape)
    for key in ('W1', 'b1', 'W2', 'b2') :
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch, t_batch)
    print("loss=", loss)
    train_loss_list.append(loss)
    print("loss list=", train_loss_list)


text:
import gym
from stable_baselines.deepq.policies import MlpPolicy
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines import DQN
import numpy as np

def evaluate(model, num_episodes= 1000, num_steps=1000):
  """
  Evaluate a RL agent
  :param model: (BaseRLModel object) the RL Agent
  :param num_steps: (int) number of timesteps to evaluate it
  :return: (float) Mean reward for the last 100 episodes
  """
  episode_rewards = [0.0]
  for x in range(num_episodes):
    obs = env.reset()
    for i in range(num_steps):
        # _states are only useful when using LSTM policies
        action, _states = model.predict(obs)
        # here, action, rewards and dones are arrays
        # because we are using vectorized env
        obs, rewards, dones, info = env.step(action)

        episode_rewards[-1] += rewards[0]
        if dones[0]:
            episode_rewards.append(0.0)
            break

  mean_reward = round(np.mean(episode_rewards), 3)
  print("Mean reward: {:}, Num successfull episodes: {:}".format(mean_reward, len(episode_rewards)))

  return mean_reward

env = gym.make('CartPole-v1')
env = DummyVecEnv([lambda: env])  # The algorithms require a vectorized environment to run

model = DQN(MlpPolicy, env, verbose=1, tensorboard_log="/tmp/DQN_test_cartpole_tensorboard/")

mean_reward_before_train = evaluate(model, num_steps=10000)

model.learn(total_timesteps=10000)
model.save("test_cartpole_model")
print("Finish Training and saving the model")

del model

model = DQN.load("test_cartpole_model")
mean_reward = evaluate(model, num_steps=10000)

# obs = env.reset()
# for i in range(1000):
#     action, _states = model.predict(obs)
#     obs, rewards, dones, info = env.step(action)
#     env.render()


text:
import spacy
import argparse
import numpy as np
from collections import Counter
from collections import defaultdict
import datetime
import operator

parser = argparse.ArgumentParser()

#To run spacy, in command line: pip install spacy
#python -m spacy download en

nlp = spacy.load('en', disable=['parser', 'tagger', 'textcat', 'ner'])
nlp.add_pipe(nlp.create_pipe('sentencizer'))

parser.add_argument('-d', type=str, required=True, dest = 'document')
parser.add_argument('-o', type=str, required=True, dest = 'output')
parser.add_argument('-l', type=int, required=True, dest = 'summary_length')

args = parser.parse_args()

def get_sentences(article):

	# Input:	riginal document
	# Returns:	List of sentences, sgmented using spaCy

	sentences = []

	tokens = nlp(article)
	for sentence in tokens.sents:
		sentences.append(str(sentence))

	return sentences

def word_weights(sentences):

	# Input:	List of sentences in the document
	# Returns:	A dictionary of the normalized weight of each word in the vocabulary
	# 			Weights for stop words are set to zero

	dictionary = defaultdict(int)
	freq_dict = defaultdict(int)

	for sentence in sentences:
		for word in sentence.split():
			if(nlp.vocab[word].is_stop == False):
				dictionary[word] += 1

	summed = 0

	for k,v in dictionary.items():
		summed += v

	for k, v in dictionary.items():
		freq_dict[k] = v/summed

	return freq_dict

def sentence_weights(sentences, word_weights, n):

	# Inputs:	List of sentences in the document
	# 			The dictionary of the normalized weight for each of the words in the vocabulary
	# 			Number of sentences required in the summary
	# Returns:	The top 'n' words by their importance due to frequency of non-stop words

	importance = defaultdict(int)

	for sentence in sentences:
		weight = 0
		for word in sentence.split():
			weight += word_weights[word]
		importance[sentence] = weight

	sorted_x = sorted(importance.items(), key=operator.itemgetter(1), reverse=True)[:n]

	return sorted_x

def summarize(article, n):

	# Inputs:	Original document
	# 			Number of sentences required in the summary
	# Returns:	The summary of the document

	sentences = get_sentences(article)
	word_weights_dict = word_weights(sentences)
	summary_sentences = sentence_weights(sentences, word_weights_dict, n)

	summary = ""

	for sentence, importance in summary_sentences:
		summary = summary + sentence + " "

	return summary.replace("\n", "").replace("  ", " ")

# Main

'''
Inputs:			Document which needs to be summarized
				Number of sentences expected in the summary
				Output file to append the summary to

Outputs:		Appends the summary to output file

Example usage:	python3 extension.py -d original_document.txt -l 3 -o computed_summaries.txt
'''

R = open(args.document, "r")
W = open(args.output,"a")

index = 1

for article in R:
	predicted_summary = summarize(article, args.summary_length)
	print("Summary:", predicted_summary)

	print("Completed:", index)
	index += 1

	W.write(predicted_summary)
	W.write("\n")

max_stars_repo_name: shivasuri/textsummarization

max_issues_repo_name: shivasuri/textsummarization

max_forks_repo_name: shivasuri/textsummarization


text:
# coding=utf-8
# Copyright 2019 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Berkeley (BAIR) robot pushing dataset.

Self-Supervised Visual Planning with Temporal Skip Connections
Frederik Ebert, Chelsea Finn, Alex X. Lee, and Sergey Levine.
https://arxiv.org/abs/1710.05268
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

from absl import logging
import numpy as np
import tensorflow as tf

import tensorflow_datasets.public_api as tfds

DATA_URL = (
    "http://rail.eecs.berkeley.edu/datasets/bair_robot_pushing_dataset_v0.tar")

# There are exactly 30 frames in each video.
FRAMES_PER_VIDEO = 30
IMG_SHAPE = (64, 64, 3)

_CITATION = """\
@inproceedings{conf/nips/FinnGL16,
  added-at = {2016-12-16T00:00:00.000+0100},
  author = {Finn, Chelsea and Goodfellow, Ian J. and Levine, Sergey},
  biburl = {https://www.bibsonomy.org/bibtex/230073873b4fe43b314724b772d0f9256/dblp},
  booktitle = {NIPS},
  crossref = {conf/nips/2016},
  editor = {Lee, Daniel D. and Sugiyama, Masashi and Luxburg, Ulrike V. and Guyon, Isabelle and Garnett, Roman},
  ee = {http://papers.nips.cc/paper/6161-unsupervised-learning-for-physical-interaction-through-video-prediction},
  interhash = {2e6b416723704f4aa5ad0686ce5a3593},
  intrahash = {30073873b4fe43b314724b772d0f9256},
  keywords = {dblp},
  pages = {64-72},
  timestamp = {2016-12-17T11:33:40.000+0100},
  title = {Unsupervised Learning for Physical Interaction through Video Prediction.},
  url = {http://dblp.uni-trier.de/db/conf/nips/nips2016.html#FinnGL16},
  year = 2016
}
"""

class BairRobotPushingSmall(tfds.core.GeneratorBasedBuilder):
  """Robot pushing dataset from BAIR (Small 64x64 version)."""

  VERSION = tfds.core.Version("1.0.0")

  def _info(self):
    # The Bair dataset consist of a sequence of frames (video) with associated
    # metadata (action and position)
    features = tfds.features.SequenceDict({
        "image_main": tfds.features.Image(shape=IMG_SHAPE),
        "image_aux1": tfds.features.Image(shape=IMG_SHAPE),
        "action": tfds.features.Tensor(shape=(4,), dtype=tf.float32),
        "endeffector_pos": tfds.features.Tensor(shape=(3,), dtype=tf.float32),
    }, length=FRAMES_PER_VIDEO)

    return tfds.core.DatasetInfo(
        builder=self,
        description="This data set contains roughly 44,000 examples of robot "
        "pushing motions, including one training set (train) and "
        "two test sets of previously seen (testseen) and unseen "
        "(testnovel) objects. This is the small 64x64 version.",
        features=features,
        urls=["https://sites.google.com/site/brainrobotdata/home/push-dataset"],
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    files = dl_manager.download_and_extract(DATA_URL)
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=10,
            gen_kwargs={
                "filedir": os.path.join(files, "softmotion30_44k", "train"),
            }),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=4,
            gen_kwargs={
                "filedir": os.path.join(files, "softmotion30_44k", "test"),
            }),
    ]

  def _generate_examples(self, filedir):
    logging.info("Reading data from %s.", filedir)
    files = tf.io.gfile.listdir(filedir)
    logging.info("%d files found.", len(files))

    # For each file
    for filename in sorted(tf.io.gfile.listdir(filedir)):
      filepath = os.path.join(filedir, filename)

      # For each video inside the file
      for example_str in tf.compat.v1.io.tf_record_iterator(filepath):
        example = tf.train.SequenceExample.FromString(example_str)

        # Merge all frames together
        all_frames = []
        for frame_id in range(FRAMES_PER_VIDEO):
          # Extract all features from the original proto context field
          frame_feature = {
              out_key: example.context.feature[in_key.format(frame_id)]
              for out_key, in_key in [
                  ("image_main", "{}/image_main/encoded"),
                  ("image_aux1", "{}/image_aux1/encoded"),
                  ("endeffector_pos", "{}/endeffector_pos"),
                  ("action", "{}/action"),
              ]
          }

          # Decode float
          for key in ("endeffector_pos", "action"):
            values = frame_feature[key].float_list.value
            frame_feature[key] = [values[i] for i in range(len(values))]

          # Decode images (from encoded string)
          for key in ("image_main", "image_aux1"):
            img = frame_feature[key].bytes_list.value[0]
            img = np.frombuffer(img, dtype=np.uint8)
            img = np.reshape(img, IMG_SHAPE)
            frame_feature[key] = img

          all_frames.append(frame_feature)

        # Encode the sequence (list) of frames (feature dicts)
        # yield [
        #     {'action': [...], 'image_main': img_frame0, ...},  # Frame 0
        #     {'action': [...], 'image_main': img_frame1, ...},  # Frame 1
        #     ...,
        # ]
        yield all_frames

max_stars_repo_name: suvarnak/datasets

max_issues_repo_name: rsepassi/datasets

max_forks_repo_name: rsepassi/datasets


text:
# coding=utf-8
import numpy as np
import networkx as nx
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn import metrics
import matplotlib.pyplot as plt
from statsmodels.tsa.arima_model import ARIMA
from scipy import stats
import sys

class Graph:
    def __init__(self, nx_G, list_G, test_G, is_directed):
        self.G = nx_G
        self.is_directed = is_directed
        self.list_G = list_G
        self.test_G = test_G
        self.n = len(nx_G)
        self.edge_info = []

    def get_time_matrix(self, lam, time_count):
        # 初始化权重，lam是每个时刻图对下一个时刻的影响参数，指数衰减
        matrix_time = np.zeros((self.n, self.n))
        for g in self.list_G:
            for edge in g.edges():
                if g[edge[0]][edge[1]]['weight'] > 1:
                    g[edge[0]][edge[1]]['weight'] = 1
        for i, g in enumerate(self.list_G):
            t = np.array(nx.adjacency_matrix(g).todense())
            matrix_time = matrix_time + lam ** (time_count - i) * t
        return matrix_time

    def get_similarity(self, lam, time_count):
        Matrix_time = self.get_time_matrix(lam, time_count)
        Matrix_similarity = Matrix_time * Matrix_time
        max_num = np.max(Matrix_similarity)
        return Matrix_similarity / max_num

    def predict(self, lam, time_count):
        matrix_p = self.get_time_matrix(lam, time_count)

        fpr, tpr, thresholds = metrics.roc_curve(np.array(nx.adjacency_matrix(self.test_G).todense()).flatten(),
                                                 matrix_p.flatten(), pos_label=1)
        plt.plot(fpr, tpr, marker='o')
        plt.show()
        auc_score = roc_auc_score(np.array(nx.adjacency_matrix(self.test_G).todense()).flatten(), matrix_p.flatten())
        print auc_score
        return

reason: import numpy,from scipy,from statsmodels,import networkx


text:
import os
import cv2
import numpy as np

def mask_generator():
    pass

def contour_detection():
    pass

def hist_equilizer():
    pass

def canny_edge(img_path):

    """ Detect edges in image"""

    img = cv2.imread(img_path)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = cv2.Canny(img, 100, 200)
    return img


text:
import random

import gym
import csv
import os
import gym_gvgai
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from IPython import display

from tensorflow import keras
from tensorflow.keras import Model, Sequential
from tensorflow.keras.layers import Dense, Input, Reshape, Flatten

from enum import Enum
from collections import deque
from PIL import Image
from numpy import asarray

# Move to module

class Experience():
    def __init__(self, state, action, reward, next_state):
        self.state = state
        self.action = action
        self.reward = reward
        self.next_state = next_state

    def __str__(self):
        return str(self.__class__) + ": " + str(self.__dict__)

class ReplayMemory():
    def __init__(self, max_capacity):
        self.max_capacity = max_capacity
        self.memory = deque(maxlen=max_capacity)

    def __str__(self):
        return str(self.__class__) + ": " + str(self.__dict__)

    def __len__(self):
        return len(self.memory)

    def remember_experience(self, experience):
        if len(self.memory) == self.max_capacity:
            self.memory.pop()
        self.memory.append(experience)

    def get_batch(self, batchSize):
        if batchSize > len(self.memory):
            return random.sample(self.memory, len(self.memory))
        else:
            return random.sample(self.memory, batchSize)

# Move to module

class EpsilonStrategy():
    def __init__(self, initial_epsilon=1, min_epsilon=0.01, epsilon_decay_rate=0.001):
        self.epsilon = initial_epsilon
        self.min_epsilon = min_epsilon
        self.epsilon_decay_rate = epsilon_decay_rate

    def __str__(self):
        return str(self.__class__) + ": " + str(self.__dict__)

    def get_movement_strategy(self):
        random_number = random.uniform(0, 1)
        if self.epsilon > self.min_epsilon:
            self.epsilon -= self.epsilon_decay_rate
        if random_number > self.epsilon:
            return 'EXPLOIT'
        return 'EXPLORE'

# Utils / Constants
IMG_H = 9
IMG_W = 13

def show_state(env, step=0, name="", info=""):
    plt.figure(3)
    plt.clf()
    plt.imshow(env.render(mode='rgb_array'))
    plt.title("{} | Step: {} {}".format(name, step, info))
    plt.axis("off")

    display.clear_output(wait=True)
    display.display(plt.gcf())

def show_loss_history(loss_history):
    plt.plot(loss_history)

    display.clear_output(wait=True)
    plt.show()

def grayToArray(array):
    result = np.zeros((IMG_H, IMG_W))
    for i in range(int(array.shape[0]/10)):
        for j in range(int(array.shape[1]/10)):
            result[i][j] = int(array[10*i+5, 10*j+5])
    return result

def grayConversion(image):
    b = image[..., 0]
    g = image[..., 1]
    r = image[..., 2]
    return 0.21 * r + 0.72 * g + 0.07 * b

# Hyperparameters

N_EPISODES = 20
N_TIMESTAMPS = 2000
N_INITIAL_POPULATION_TIMESTAMPS = 2500

MIN_REPLAY_MEMORY_SIZE = 10
EXPERIENCE_BATCH_SIZE = 100
MAX_REPLAY_MEMORY_SIZE = 200
STEPS_TO_UPDATE_NETWORK = 5
GAMMA = 0.9

# Agent

class Agent():

    def __init__(self):
        self.name = "Q-Learning agent"

        self.replay_memory = None
        self.policy_network = None
        self.target_network = None

        if os.path.exists("networks/policy_network"):
            print("Model loaded!")
            self.policy_network = keras.models.load_model(
                "networks/policy_network")
            print(self.policy_network.summary())
            self.target_network = keras.models.load_model(
                "networks/policy_network")

        self.movementStrategy = EpsilonStrategy()

    def _init_network(self, input_shape, n_outputs):
        inputs = Input(input_shape, name='state')
        x = Flatten()(inputs)
        x = Dense(30, activation='relu')(x)
        x = Dense(64, activation='relu')(x)
        outputs = Dense(n_outputs, activation='softmax')(x)

        model = Model(inputs=inputs, outputs=outputs, name='game')

        model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(
            learning_rate=0.001))
        return model

    def _preprocess_state(self, raw_state):
        return grayToArray(grayConversion(raw_state))

    def _preprocess_reward(self, current_state, next_state, action_id, raw_reward, is_over, info):
        # This isn't a generic logic, it's exclusive for Zelda's levels
        actions_list = ['ACTION_NIL', 'ACTION_USE', 'ACTION_LEFT',
                        'ACTION_RIGHT', 'ACTION_DOWN', 'ACTION_UP']
        action = actions_list[action_id]

        is_winner = info["winner"] != "PLAYER_LOSES" and info["winner"] != "NO_WINNER"

        if is_over and is_winner:
            return 200

        if is_over and not is_winner:
            return -100

        if raw_reward > 0:
            return raw_reward * 10

        if action in ['ACTION_NIL', 'ACTION_USE']:
            return -100

        return 0

    def train(self, env):
        # Models
        self.policy_network = self._init_network(
            (IMG_H, IMG_W), env.action_space.n)
        self.target_network = self._init_network(
            (IMG_H, IMG_W), env.action_space.n)

        # Replay memory
        self.replay_memory = ReplayMemory(MAX_REPLAY_MEMORY_SIZE)
        population_steps = 0

        # Metrics
        metrics = []
        loss_history = [0]

        for e in range(N_EPISODES):
            current_state = self._preprocess_state(env.reset())
            next_state = None
            sum_reward = 0

            for t in range(N_TIMESTAMPS):

                while population_steps < N_INITIAL_POPULATION_TIMESTAMPS:

                    # Take random action
                    action_id = env.action_space.sample()

                    # Emulate action
                    next_state, reward, is_over, debug = env.step(action_id)

                    # If the game is over while populating the replay memory we want to restart it
                    if is_over:
                        next_state = env.reset()

                    next_state = self._preprocess_state(next_state)
                    reward = self._preprocess_reward(
                        current_state, next_state, action_id, reward, is_over, debug)

                    print('reward', reward)
                    sum_reward += reward

                    info = "action {0: <1}/{1: <15} | tick {2: <10} | reward {3: <5} | sum {4: <5} | is_over {5: <10} | status {6: <10}".format(
                        action_id, env.unwrapped.get_action_meanings()[action_id], t, reward, sum_reward, is_over, debug["winner"])

                    print("Populating replay memory: {}".format(info))

                    # Remember experience
                    experience = Experience(
                        current_state, action_id, reward, next_state)
                    self.replay_memory.remember_experience(experience)

                    population_steps += 1

                tensor_state = tf.convert_to_tensor([current_state])

                if self.movementStrategy.get_movement_strategy() == 'EXPLOIT':
                    q_values = self.policy_network.predict(tensor_state)
                    action_id = np.argmax(q_values[0])

                else:
                    action_id = env.action_space.sample()

                next_state, reward, is_over, debug = env.step(action_id)
                next_state = self._preprocess_state(next_state)
                reward = self._preprocess_reward(
                    current_state, next_state, action_id, reward, is_over, debug)
                sum_reward += reward

                info = "action {0: <1}/{1: <15} | tick {2: <10} | reward {3: <5} | sum {4: <5} | is_over {5: <10} | status {6: <10}".format(
                    action_id, env.unwrapped.get_action_meanings()[action_id], t, reward, sum_reward, is_over, debug["winner"])

                metrics.append([
                    action_id,
                    env.unwrapped.get_action_meanings()[action_id],
                    e,
                    t,
                    reward,
                    sum_reward,
                    is_over
                ])

                print(info)
                # show_state(env, t, "Zelda", info)
                # show_loss_history(loss_mean_history)

                if is_over:
                    break

                experience = Experience(
                    current_state, action_id, reward, next_state)
                self.replay_memory.remember_experience(experience)

                if len(self.replay_memory) > MIN_REPLAY_MEMORY_SIZE:
                    experiences_batch = self.replay_memory.get_batch(
                        EXPERIENCE_BATCH_SIZE)
                    for experience in experiences_batch:
                        tensor_state = tf.convert_to_tensor([experience.state])
                        tensor_next_state = tf.convert_to_tensor(
                            [experience.next_state])

                        policy_q_values = self.policy_network.predict(
                            tensor_state)
                        next_q_values = self.target_network.predict(
                            tensor_next_state)

                        policy_q_values[0][experience.action] = experience.reward + \
                            GAMMA * np.amax(next_q_values)

                        result = self.policy_network.fit(
                            tensor_state, policy_q_values, epochs=1, verbose=0)

                        loss = result.history['loss'][0]
                        loss_history.append(loss)

                current_state = next_state

                if t % STEPS_TO_UPDATE_NETWORK == 0:
                    self.target_network.set_weights(
                        self.policy_network.get_weights())

            self.save_model("policy_network_{}".format(e))
            np.savetxt("metrics_loss_history_{}.csv".format(e), loss_history, delimiter=",")

        np.savetxt("metrics_loss_history.csv", loss_history, delimiter=",")
        with open("metrics_metrics.csv", 'w') as csv_file:
            wr = csv.writer(csv_file, delimiter=',')
            for metric in metrics:
                wr.writerow(list(metric))

    def save_model(self, name="policy_network"):
        self.policy_network.save("networks/{}".format(name))

    def act(self, stateObs, actions):
        tensor_state = tf.convert_to_tensor([self._preprocess_state(stateObs)])
        q_values = self.policy_network.predict(tensor_state)
        action_id = np.argmax(q_values[0])
        return action_id

def train_agent(env, agent):
    agent.train(env)
    agent.save_model()

def play_agent(env, agent):
    stateObs = env.reset()
    actions = env.unwrapped.get_action_meanings()
    for t in range(5000):
        action_id = agent.act(stateObs, actions)
        stateObs, diffScore, done, debug = env.step(action_id)
        print("Action " + str(action_id) + " tick " + str(t+1) +
              " reward " + str(diffScore) + " win " + debug["winner"])
        if done:
            break

def main():
    zelda_env = gym.make("gvgai-zelda-lvl0-v0")
    agent = Agent()

    train_agent(zelda_env, agent)
    # play_agent(zelda_env, agent)

main()

max_stars_repo_name: AlfaroMiguel/zelda-deep-q-learning

max_issues_repo_name: AlfaroMiguel/zelda-deep-q-learning

max_forks_repo_name: AlfaroMiguel/zelda-deep-q-learning

reason: import numpy,from numpy