text:
import os
import sys
import argparse
import importlib
import numpy as np
import tensorflow as tf
import pdb
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR, '..'))
sys.path.append(os.path.join(BASE_DIR, '../utils'))
import provider
import provider_riconv
import tf_util
import pdb
import time
import scipy
import re
import pickle
import gc

parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0, help='GPU to use [default: GPU 0]')
parser.add_argument('--model', default='rinet', help='Model name')
parser.add_argument('--load_dir', required=True, default='rinet')
parser.add_argument('--num_point', type=int, default=1024, help='Point Number [256/512/1024/2048] [default: 1024]')
parser.add_argument('--max_epoch', type=int, default=251, help='Epoch to run [default: 251]')
parser.add_argument('--batch_size', type=int, default=4, help='Batch Size during training [default: 32]')
parser.add_argument('--learning_rate', type=float, default=0.001, help='Initial learning rate [default: 0.001]')
parser.add_argument('--momentum', type=float, default=0.9, help='Initial learning rate [default: 0.9]')
parser.add_argument('--optimizer', default='adam', help='adam or momentum [default: adam]')
parser.add_argument('--decay_step', type=int, default=200000, help='Decay step for lr decay [default: 200000]')
parser.add_argument('--decay_rate', type=float, default=0.7, help='Decay rate for lr decay [default: 0.7]')
parser.add_argument('--rotation', action='store_true', help='Whether to apply rotation during training [default: False]')
parser.add_argument('--finetune', action='store_true', help='Whether to finetune [default: False]')
parser.add_argument('--checkpoint', default='log/model.ckpt', help='Checkpoint directory to finetune [default: log/model.ckpt]')
parser.add_argument('--num_pool', type=int, default=256, help='Number of pooling [default: 64]')
parser.add_argument('--pool_knn1', type=int, default=64, help='Number of neighbors for lf [default: 128]')
parser.add_argument('--num_votes', type=int, default=12)
parser.add_argument('--so3', action='store_true', default=True, help='Whether training in SO3 setting')
parser.add_argument('--azi', action='store_true', help='Whether training in azimuthal rotation')

FLAGS = parser.parse_args()
LOAD_DIR = FLAGS.load_dir
BATCH_SIZE = FLAGS.batch_size
NUM_POINT = FLAGS.num_point
MAX_EPOCH = FLAGS.max_epoch
BASE_LEARNING_RATE = FLAGS.learning_rate
GPU_INDEX = FLAGS.gpu
MOMENTUM = FLAGS.momentum
OPTIMIZER = FLAGS.optimizer
DECAY_STEP = FLAGS.decay_step
DECAY_RATE = FLAGS.decay_rate
FINETUNE = FLAGS.finetune
CHECKPOINT = FLAGS.checkpoint
NUM_VOTES = FLAGS.num_votes

sys.path.append(os.path.join(BASE_DIR, FLAGS.load_dir))
MODEL = importlib.import_module(FLAGS.model)
MODEL_FILE = os.path.join(BASE_DIR, 'models', FLAGS.model+'.py')

print(MODEL_FILE)
LOG_FOUT = open(os.path.join(LOAD_DIR, 'log_test.txt'), 'w')
LOG_FOUT.write(str(FLAGS)+'\n')

MAX_NUM_POINT = 2048
NUM_CLASSES = 40

BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

BASE_REG_WEIGHT = 0.001
REG_WEIGHT_DECAY_RATE = 0.5
REG_WEIGHT_DECAY_STEP = float(DECAY_STEP)

# ModelNet40 official train/test split
TRAIN_FILES = provider.getDataFiles(
    os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/train_files.txt'))
TEST_FILES = provider.getDataFiles(
    os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/test_files.txt'))

# Load data beforehand
KEYS = ['data', 'label']
TRAIN_DATA, TRAIN_LABEL = \
    zip(*[provider.loadDataFile_with_keys(fn, KEYS) for fn in TRAIN_FILES])
TEST_DATA, TEST_LABEL = \
    zip(*[provider.loadDataFile_with_keys(fn, KEYS) for fn in TEST_FILES])

# concatenate batches
TRAIN_DATA = np.concatenate(TRAIN_DATA, axis=0)
TRAIN_LABEL = np.squeeze(np.concatenate(TRAIN_LABEL, axis=0))

TEST_DATA = np.concatenate(TEST_DATA, axis=0)
TEST_LABEL = np.squeeze(np.concatenate(TEST_LABEL, axis=0))

def log_string(out_str):
    LOG_FOUT.write(out_str+'\n')
    LOG_FOUT.flush()
    print(out_str)

def get_learning_rate(batch):
    learning_rate = tf.train.exponential_decay(
                        BASE_LEARNING_RATE,  # Base learning rate.
                        batch * BATCH_SIZE,  # Current index into the dataset.
                        DECAY_STEP,          # Decay step.
                        DECAY_RATE,          # Decay rate.
                        staircase=True)
    learning_rate = tf.maximum(learning_rate, 0.00001) # CLIP THE LEARNING RATE!
    return learning_rate

def get_bn_decay(batch):
    bn_momentum = tf.train.exponential_decay(
                      BN_INIT_DECAY,
                      batch*BATCH_SIZE,
                      BN_DECAY_DECAY_STEP,
                      BN_DECAY_DECAY_RATE,
                      staircase=True)
    bn_decay = tf.minimum(BN_DECAY_CLIP, 1 - bn_momentum)
    return bn_decay

def get_reg_weight(batch):
    reg_weight = tf.train.exponential_decay(
                      BASE_REG_WEIGHT,
                      batch * BATCH_SIZE,
                      REG_WEIGHT_DECAY_STEP,
                      REG_WEIGHT_DECAY_RATE,
                      staircase=False)
    reg_weight = tf.maximum(reg_weight, 0.00001)
    return reg_weight

def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    return [atoi(c) for c in re.split('(\d+)', text)]

def train():
    with tf.Graph().as_default():
        with tf.device('/gpu:'+str(GPU_INDEX)):
            pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, 1024)
            input_graph = tf.placeholder(tf.float32, shape = (BATCH_SIZE, NUM_POINT, NUM_POINT))
            is_training_pl = tf.placeholder(tf.bool, shape=())
            flag_pl = tf.placeholder(tf.int32, shape=())
            flag1 =  tf.placeholder(tf.int32, shape=())
            flag2 =  tf.placeholder(tf.int32, shape=())
            flag3 =  tf.placeholder(tf.int32, shape=())
            dilation = tf.placeholder(tf.int32, shape=())

            gcn1 = tf.placeholder(tf.int32, shape=())
            gcn2 = tf.placeholder(tf.int32, shape=())
            gcn3 = tf.placeholder(tf.int32, shape=())

            print(is_training_pl)
            # Note the global_step=batch parameter to minimize.
            # That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
            batch = tf.Variable(0, trainable=False)
            bn_decay = get_bn_decay(batch)
            tf.summary.scalar('bn_decay', bn_decay)

            # Get model and loss
            pred, end_points  = MODEL.get_model(pointclouds_pl, FLAGS.num_pool, FLAGS.pool_knn1,
                                     is_training_pl, bn_decay=bn_decay, flag=flag_pl, flag2=flag2, flag3=flag3,gcn1=gcn1, gcn2=gcn2, gcn3=gcn3, dilation=dilation)
            reg_weight = get_reg_weight(batch)
            loss = MODEL.get_loss(pred, labels_pl, end_points)
            tf.summary.scalar('loss', loss)

            correct = tf.equal(tf.argmax(pred, 1), tf.to_int64(labels_pl))
            accuracy = tf.reduce_sum(tf.cast(correct, tf.float32)) / float(BATCH_SIZE)
            tf.summary.scalar('accuracy', accuracy)

            # Get training operator
            learning_rate = get_learning_rate(batch)
            tf.summary.scalar('learning_rate', learning_rate)
            if OPTIMIZER == 'momentum':
                optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=MOMENTUM)
            elif OPTIMIZER == 'adam':
                optimizer = tf.train.AdamOptimizer(learning_rate)

            if FINETUNE:
                """THIS IS NOT WORKING CURRENTLY"""
                finetune_var_names = ['fc1', 'fc2', 'fc3']
                finetuning_vars = [v for v in tf.trainable_variables() if v.name.split('/')[0] in finetune_var_names]
                orig_vars = [v for v in tf.trainable_variables() if v.name.split('/')[0] not in finetune_var_names]
                gvs = optimizer.compute_gradients(loss, [orig_vars, finetuning_vars])
                scaled_gvs = [(grad * 0.1, var) for (grad, var) in gvs[:len(orig_vars)]] + gvs[len(orig_vars):]
                train_op = optimizer.apply_gradients(scaled_gvs, global_step=batch)
            else:
                gvs = optimizer.compute_gradients(loss)
                train_op = optimizer.apply_gradients(gvs, global_step=batch)

            # Add ops to save and restore all the variables.
            saver = tf.train.Saver()

            # Load parameters before finetuning
            if FINETUNE:
                variables_to_restore = [v for v in tf.all_variables() if 'rel' not in v.name.split('/')[0]]
                variables_to_restore = [v for v in variables_to_restore if not v.name == 'batch']
                pre_saver = tf.train.Saver(variables_to_restore)

        # Create a session
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        config.log_device_placement = False
        sess = tf.Session(config=config)

        saver.restore(sess, LOAD_DIR + '/model.ckpt')

        # Add summary writers
        #merged = tf.merge_all_summaries()
        merged = tf.summary.merge_all()
        test_writer = tf.summary.FileWriter(os.path.join(LOAD_DIR, 'test'))

        # Init variables
        init = tf.global_variables_initializer()
        ops = {'pointclouds_pl': pointclouds_pl,
               'labels_pl': labels_pl,
               'is_training_pl': is_training_pl,
               'pred': pred,
               'loss': loss,
               'gvs': gvs,
               'train_op': train_op,
               'merged': merged,
               'step': batch,
               'flag': flag_pl,
               'flag2': flag2,
               'flag3': flag3,
               'dilation' : dilation,
               'gcn1' : gcn1,
               'gcn2' : gcn2,
               'gcn3' : gcn3
               }

        acc, cls_avg = eval_one_epoch(sess, ops, test_writer, NUM_VOTES)
        print('Overall accuracy: ', acc)

def eval_one_epoch(sess, ops, test_writer, num_votes):
    """ ops: dict mapping from string to tf ops """
    is_training = False
    current_data = TEST_DATA[:, 0:NUM_POINT, :]
    current_label = TEST_LABEL
    file_size = current_data.shape[0]
    num_batches = file_size // BATCH_SIZE
    pred_conf = np.zeros((0, 40))

    total_correct = 0
    total_seen = 0
    loss_sum = 0
    total_seen_class = [0 for _ in range(NUM_CLASSES)]
    total_correct_class = [0 for _ in range(NUM_CLASSES)]

    shape_txt = open('data/modelnet40_ply_hdf5_2048/shape_names.txt', 'r')
    label_to_class = shape_txt.read().split('\n')

    flag1 = 64
    flag2 = 32
    flag3 = 16
    gcn1 = 16
    gcn2 = 8
    gcn3 = 4

    log_string('----------------')
    for batch_idx in range(num_batches):
        start_idx = batch_idx * BATCH_SIZE
        end_idx = (batch_idx + 1 ) * BATCH_SIZE
        batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES))
        for vote_idx in range(NUM_VOTES):
            shuffle = np.arange(NUM_POINT)
            np.random.shuffle(shuffle)
            rot_data = provider_riconv.so3_rotate(current_data[start_idx:end_idx, shuffle, :])
            feed_dict = {
                ops['pointclouds_pl']: rot_data,
                ops['labels_pl']: current_label[start_idx:end_idx],
                ops['is_training_pl']: is_training,
                ops['flag'] : flag1,
                ops['flag2'] : flag2,
                ops['flag3'] : flag3,
                ops['dilation'] : 3,
                ops['gcn1'] : gcn1,
                ops['gcn2'] : gcn2,
                ops['gcn3'] : gcn3
            }

            summary, step, loss_val, pred_val = sess.run([ops['merged'], ops['step'],
                                                      ops['loss'], ops['pred']], feed_dict=feed_dict)

            batch_pred_sum += pred_val

        pred_conf = np.argmax(batch_pred_sum, 1)
        test_writer.add_summary(summary, step)
        correct = np.sum(pred_conf == current_label[start_idx:end_idx])
        total_correct += correct
        total_seen += BATCH_SIZE
        loss_sum += (loss_val*BATCH_SIZE)
        for i in range(start_idx, end_idx):
            l = current_label[i]
            total_seen_class[l] += 1
            total_correct_class[l] += (pred_conf[i-start_idx] == l)

    # Handle remaining
    if file_size - num_batches * BATCH_SIZE > 0:

        start_idx = num_batches * BATCH_SIZE
        end_idx = file_size

        input_data = np.zeros((BATCH_SIZE, 1024, 3))
        input_label = np.zeros(BATCH_SIZE)
        input_label[0:end_idx-start_idx] = current_label[start_idx:end_idx]
        batch_pred_sum = np.zeros((BATCH_SIZE, NUM_CLASSES))
        for vote_idx in range(NUM_VOTES):
            shuffle = np.arange(NUM_POINT)
            np.random.shuffle(shuffle)
            input_data[0:end_idx - start_idx, ...] = provider_riconv.so3_rotate(current_data[start_idx:end_idx, 0:NUM_POINT, :])

            feed_dict = {
                ops['pointclouds_pl']: input_data,
                ops['labels_pl']: input_label,
                ops['is_training_pl']: is_training,
                ops['flag'] : flag1,
                ops['flag2'] : flag2,
                ops['flag3'] : flag3,
                ops['dilation'] : 3,
                ops['gcn1'] : gcn1,
                ops['gcn2'] : gcn2,
                ops['gcn3'] : gcn3

            }

            summary, step, loss_val, pred_val= sess.run([ops['merged'], ops['step'],
                                                      ops['loss'], ops['pred']], feed_dict=feed_dict)

            batch_pred_sum += pred_val

        pred_conf = np.argmax(batch_pred_sum, 1)

        test_writer.add_summary(summary, step)
        pred_val = np.argmax(pred_val, axis=1)
        correct = np.sum(pred_conf[0:end_idx-start_idx] == current_label[start_idx:end_idx])
        total_correct += correct
        total_seen += end_idx - start_idx
        loss_sum += (loss_val * (end_idx - start_idx))
        for i in range(start_idx, end_idx):
            l = current_label[i]
            total_seen_class[l] += 1
            total_correct_class[l] += (pred_conf[i - start_idx] == l)

    return (total_correct / float(total_seen)), np.mean(np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float))

if __name__ == "__main__":
    start_time = time.time()
    train()
    end_time = time.time()

    LOG_FOUT.close()

reason: import numpy,import scipy


text:
# Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Predict extractor for scikit-learn models."""

import copy
import os
import pickle
from typing import Dict, Iterable, List, Text

import apache_beam as beam
import numpy as np
import tensorflow as tf
import tensorflow_model_analysis as tfma
from tensorflow_model_analysis import constants, model_util, types
from tensorflow_model_analysis.extractors import extractor
from tfx_bsl.tfxio import tensor_adapter

_PREDICT_EXTRACTOR_STAGE_NAME = 'SklearnPredict'

def _make_sklearn_predict_extractor(
    eval_shared_model: tfma.EvalSharedModel, ) -> extractor.Extractor:
  """Creates an extractor for performing predictions using a scikit-learn model.

  The extractor's PTransform loads and runs the serving pickle against
  every extract yielding a copy of the incoming extracts with an additional
  extract added for the predictions keyed by tfma.PREDICTIONS_KEY. The model
  inputs are searched for under tfma.FEATURES_KEY.

  Args:
    eval_shared_model: Shared model (single-model evaluation).

  Returns:
    Extractor for extracting predictions.
  """
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model)
  return extractor.Extractor(
      stage_name=_PREDICT_EXTRACTOR_STAGE_NAME,
      ptransform=_ExtractPredictions(  # pylint: disable=no-value-for-parameter
          eval_shared_models={m.model_name: m
                              for m in eval_shared_models}))

@beam.typehints.with_input_types(types.Extracts)
@beam.typehints.with_output_types(types.Extracts)
class _TFMAPredictionDoFn(model_util.DoFnWithModels):
  """A DoFn that loads the models and predicts."""
  def __init__(self, eval_shared_models: Dict[Text, types.EvalSharedModel]):
    super(_TFMAPredictionDoFn, self).__init__(
        {k: v.model_loader
         for k, v in eval_shared_models.items()})

  def setup(self):
    """Setup DoFn for sklearn"""
    super(_TFMAPredictionDoFn, self).setup()
    self._feature_keys = None
    self._label_key = None
    for loaded_model in self._loaded_models.values():
      if self._feature_keys and self._label_key:
        assert self._feature_keys == loaded_model.feature_keys, (
            f'Features mismatch in loaded models. Expected {self._feature_keys}'
            f', got {loaded_model.feature_keys} instead.')
        assert self._label_key == loaded_model.label_key, (
            f'Label mismatch in loaded models. Expected "{self._label_key}"'
            f', got "{loaded_model.label_key}" instead.')
      elif loaded_model.feature_keys and loaded_model.label_key:
        self._feature_keys = loaded_model.feature_keys
        self._label_key = loaded_model.label_key
      else:
        raise ValueError('Missing feature or label keys in loaded model.')

  def process(self, elem: types.Extracts) -> Iterable[types.Extracts]:
    """Uses loaded models to make predictions on batches of data.

    Args:
      elem: An extract containing batched features.

    Yields:
      Copy of the original extracts with predictions added for each model. If
      there are multiple models, a list of dicts keyed on model names will be
      added, with each value corresponding to a prediction for a single sample.
    """
    # Build feature and label vectors because sklearn cannot read tf.Examples.
    features = []
    labels = []
    result = copy.copy(elem)
    for features_dict in result[constants.FEATURES_KEY]:
      features_row = [features_dict[key] for key in self._feature_keys]
      features.append(np.concatenate(features_row))
      labels.append(features_dict[self._label_key])
    result[constants.LABELS_KEY] = np.concatenate(labels)

    # Generate predictions for each model.
    for model_name, loaded_model in self._loaded_models.items():
      preds = loaded_model.predict(features)
      if len(self._loaded_models) == 1:
        result[constants.PREDICTIONS_KEY] = preds
      elif constants.PREDICTIONS_KEY not in result:
        result[constants.PREDICTIONS_KEY] = [{
            model_name: pred
        } for pred in preds]
      else:
        for i, pred in enumerate(preds):
          result[constants.PREDICTIONS_KEY][i][model_name] = pred
    yield result

@beam.ptransform_fn
@beam.typehints.with_input_types(types.Extracts)
@beam.typehints.with_output_types(types.Extracts)
def _ExtractPredictions(  # pylint: disable=invalid-name
    extracts: beam.pvalue.PCollection,
    eval_shared_models: Dict[Text, types.EvalSharedModel],
) -> beam.pvalue.PCollection:
  """A PTransform that adds predictions and possibly other tensors to extracts.

  Args:
    extracts: PCollection of extracts with inputs keyed by tfma.INPUTS_KEY.
    eval_shared_models: Shared model parameters keyed by model name.

  Returns:
    PCollection of Extracts updated with the predictions.
  """
  return extracts | 'Predict' >> beam.ParDo(
      _TFMAPredictionDoFn(eval_shared_models))

def _custom_model_loader_fn(model_path: Text):
  """Returns a function that loads a scikit-learn model."""
  return lambda: pickle.load(tf.io.gfile.GFile(model_path, 'rb'))

# TFX Evaluator will call the following functions.
def custom_eval_shared_model(eval_saved_model_path, model_name, eval_config,
                             **kwargs) -> tfma.EvalSharedModel:
  """Returns a single custom EvalSharedModel."""
  model_path = os.path.join(eval_saved_model_path, 'model.pkl')
  return tfma.default_eval_shared_model(
      eval_saved_model_path=model_path,
      model_name=model_name,
      eval_config=eval_config,
      custom_model_loader=types.ModelLoader(
          construct_fn=_custom_model_loader_fn(model_path)),
      add_metrics_callbacks=kwargs.get('add_metrics_callbacks'))

def custom_extractors(
    eval_shared_model: tfma.MaybeMultipleEvalSharedModels,
    eval_config: tfma.EvalConfig,
    tensor_adapter_config: tensor_adapter.TensorAdapterConfig,
) -> List[tfma.extractors.Extractor]:
  """Returns default extractors plus a custom prediction extractor."""
  predict_extractor = _make_sklearn_predict_extractor(eval_shared_model)
  return tfma.default_extractors(eval_shared_model=eval_shared_model,
                                 eval_config=eval_config,
                                 tensor_adapter_config=tensor_adapter_config,
                                 custom_predict_extractor=predict_extractor)

ext: py

max_stars_repo_name: casassg/tfx-addons

max_issues_repo_name: casassg/tfx-addons

max_forks_repo_name: casassg/tfx-addons

reason: import numpy


text:
import os
import logging
import sys
import subprocess

from position_rank import position_rank_mod
from tokenizer import StanfordCoreNlpTokenizer

import pke
from pke.utils import compute_document_frequency

from keybert import KeyBERT

import spacy
import en_core_web_sm
from textacy.ke import sgrank
from textacy.ke import scake
from textacy.ke import textrank, yake

from bert_serving.client import BertClient
path_uke = "unsupervised_keyword_extraction/" #path for the dir from  https://github.com/AnzorGozalishvili/unsupervised_keyword_extraction
sys.path.append(path_uke)
from model.embedrank_transformers import EmbedRankTransformers #uncomment to use EmbedRank

import numpy as np
import pickle
import nltk
from nltk.corpus import stopwords

# import silence_tensorflow.auto
# from tensorflow.python.keras.models import Model, Sequential, model_from_json, load_model
# from tensorflow.python.keras.preprocessing.text import Tokenizer
# from tensorflow.python.keras.preprocessing.sequence import pad_sequences

class KeyModel:
    """ Main Model class
    """
    def __init__(self):
        pass

    def normalize_weights(self, weights):
        max_value = max(weights)
        min_val = min(weights)
        normalized =[]
        for w in weights:
            new_w = (w-min_val)/(max_value-min_val)
            normalized.append(new_w)
        return  normalized

class PKE (KeyModel):
    """Class for pke models developed by Florian Boudin
    """

    def __init__(self, model_name, models=None, _df_counts=None, frequency_path = None):
        self.df = None

        self.model_name = "PKE_"+model_name
        if model_name.lower() == "topicrank":
            self.model = pke.unsupervised.TopicRank()
        elif model_name.lower() == "textrank":
            self.model = pke.unsupervised.TextRank()
        elif model_name.lower() == "singlerank":
            self.model = pke.unsupervised.SingleRank()
        elif model_name.lower() == "topicalpagerank":
            self.model = pke.unsupervised.TopicalPageRank()
        elif model_name.lower() == "positionrank":
            self.model = pke.unsupervised.PositionRank()
        elif model_name.lower() == "multipartiterank":
            self.model = pke.unsupervised.MultipartiteRank()
        elif model_name.lower() == "tfidf":
            self.model = pke.unsupervised.TfIdf()
            if frequency_path:
                self.df = pke.load_document_frequency_file(input_file=frequency_path)

        elif model_name.lower() == "kpminer":
            self.model = pke.unsupervised.KPMiner()
        elif model_name.lower() == "yake":
            self.model = pke.unsupervised.YAKE()
        elif model_name.lower() == "kea":
            self.model = pke.supervised.Kea()
        elif model_name.lower() == "wingnus":
            self.model = pke.supervised.WINGNUS()
        else:
            raise Exception("{} is not a model name defined in PKE library".format(model_name))
        if models:
            self.model._models = models
        else:
            self.model._models = os.path.join(os.path.dirname(pke.__file__), 'models')

        if _df_counts:
            self.model._df_counts = _df_counts

        else:
            self._df_counts = os.path.join(self.model._models, "df-semeval2010.tsv")

    def get_keywords(self, text, n=10, normalization='None'):
        self.model.load_document(text, language='en', normalization=normalization)

        # keyphrase candidate selection, in the case of TopicRank: sequences of nouns
        # and adjectives (i.e. `(Noun|Adj)*`)
        self.model.candidate_selection()

        # candidate weighting, in the case of TopicRank: using a random walk algorithm
        if not self.df:
            self.model.candidate_weighting()
        else:
            self.model.candidate_weighting(df = self.df)

        # N-best selection, keyphrases contains the 10 highest scored candidates as
        # (keyphrase, score) tuples
        try:
            keyphrases, weights = list(map(list, zip(*self.model.get_n_best(n=n))))
        except:
            raise Warning("model was not able to retrieve keywords")
            keyphrases, weights = None, None
        return keyphrases, weights

    def create_doc_frequency(self, texts, outpath= "output.txt"):
        path = os.path.join(os.path.dirname(pke.__file__), 'models/temp/')
        for i in range(len(texts)):
            fpath = os.path.join(path,"f"+str(i)+".txt")
            open(fpath,"w",encoding="utf-8").write(texts[i])
        opath = os.path.join(path, outpath)
        compute_document_frequency(path, opath,"txt")

class keyBert (KeyModel):
    def __init__(self):
        self.model = KeyBERT ('allenai/scibert_scivocab_uncased') #('distilbert-base-nli-mean-tokens')
        self.model_name = "keyBert"
    def get_keywords(self, text, n=10):
        try:
            keyphrases, weights = list(map(list, zip(*self.model.extract_keywords(text,keyphrase_ngram_range=(1, 3), top_n=n))))

        except:
            logging.warning("model was not able to retrieve keywords")
            keyphrases, weights = None, None

        return keyphrases, weights

class Textacy (KeyModel):
    def __init__(self, model_name):
        self.nlp = en_core_web_sm.load()
        self.model_name = "Textacy_"+model_name

    def get_keywords(self, text, n=10):
        try:
            doc = self.nlp(text)
        except:
            raise Exception("Couldn't load Spacy model")

        try:
            if "sgrank" in self.model_name :
                keyphrases, weights = list(map(list, zip(*sgrank(doc, topn=n))))
            elif "scake" in self.model_name:
                keyphrases, weights = list(map(list, zip(*scake(doc, topn=n))))
            elif "textrank" in self.model_name:
                keyphrases, weights = list(map(list, zip(*textrank(doc, topn=n))))
            elif "yake" in self.model_name:
                keyphrases, weights = list(map(list, zip(*yake(doc, topn=n))))
            else:
                raise Exception("{} is not a model name defined in textacy library".format(self.model_name))
        except:
            logging.warning("model was not able to retrieve keywords")
            keyphrases, weights = None, None
        return keyphrases, weights

class BertEmbedRank (KeyModel):
    def __init__(self, pre_load=True, n=10, bert_path=None):
        """

        :param pre_load:
        :param n:

        bert-serving-start -model_dir /tmp/english_L-12_H-768_A-12/ -num_worker=4 (to start the service)
        """
        self.model_name = "BertEmbedRank"
        if not bert_path:
            bert_path = "unsupervised_keyword_extraction/cased_L-12_H-768_A-12/"
        #try:
        #with open("stdout.txt", "wb") as out, open("stderr.txt", "wb") as err:
        #    self.p = subprocess.Popen(["bert-serving-start", "-model_dir", bert_path, "-num_worker=4"], stdout=out, stderr=err)
        self.bc = BertClient(output_fmt='list', timeout=120000)
        #except:
        #    logging.warning("BertClient is not working")

        self.nlp = spacy.load("en_core_web_lg", disable=['ner'])
        if pre_load:
            self.model = EmbedRankTransformers(nlp=self.nlp,
                                          dnn=self.bc,
                                          perturbation='replacement',
                                          emb_method='subtraction',
                                          mmr_beta=0.55,
                                          top_n=n,
                                          alias_threshold=0.8)
        else:
            self.model=None

    def get_keywords(self, text, n=10):
        if not self.model:
            self.model = EmbedRankTransformers(nlp=self.nlp,
                                   dnn=self.bc,
                                   perturbation='replacement',
                                   emb_method='subtraction',
                                   mmr_beta=0.55,
                                   top_n=n,
                                   alias_threshold=0.8)

        #marked_target, keywords, keyword_relevance = self.model.fit(text)

        keyphrases, weights = list(map(list, zip(*self.model.extract_keywords(text))))

        return  keyphrases, weights

    def exit(self):
        self.p.terminate()

class BiLSTM (KeyModel):
    def __init__(self, json_config_f, model_f, tokenizer_f):
        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
        self.stop_words = set(stopwords.words('english'))
        with open(json_config_f) as json_file:
            json_config = json_file.read()
        self.model = model_from_json(json_config)
        self.model_name = "BiLSTM"
        # Load weights
        self.model.load_weights(model_f)

        tokenizer = Tokenizer()
        with open(tokenizer_f, 'rb') as handle:
            self.tokenizer = pickle.load(handle)

    def get_keywords(self, text, n=10):
        new_t = Tokenizer()
        new_t.fit_on_texts([text])
        tokens = [i for i in new_t.word_index.keys()]

        actual_tokens = new_t.texts_to_sequences([text])
        inv_map_tokens = {v: k for k, v in new_t.word_index.items()}
        actual_tokens = [inv_map_tokens[i] for i in actual_tokens[0]]
        tokens = actual_tokens
        input_ = self.tokenizer.texts_to_sequences([text])
        input_ = pad_sequences(input_, padding="post", truncating="post", maxlen=25, value=0)
        output = self.model.predict([input_])
        output = np.argmax(output, axis=-1)
        where_ = np.where(output[0] == 1)[0]
        output_keywords = np.take(tokens, where_)
        output_keywords = [i for i in output_keywords if i not in self.stop_words]
        output_keywords = list(set(output_keywords))
        k = output_keywords[:n]
        return k, [0.0]*len(k)

class BibRank(KeyModel):
    """
    java -mx4g -cp "*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port 9000 -timeout 15000
    """
    def __init__(self, weights=None, wiki=False):
        self.tokenizer = StanfordCoreNlpTokenizer("http://localhost", port = 9000)
        self.weights = weights
        self.wiki = wiki
        self.model_name = "PositionRankMod"
    def get_keywords(self, text, n=10):
        keyphrases_p = position_rank_mod(text.split(), self.tokenizer, alpha=0.85, window_size=2, num_keyphrase=n, weights=self.weights,
                                     wiki=self.wiki)
        return list(map(list, zip(*keyphrases_p)))

class CollabRank (KeyModel):
    def __init__(self, collab_documents):
        self.collab_documents = collab_documents
        self.model_name = "CollabRank"

    def get_keywords(self, text, n=10):
        pos = {'NOUN', 'PROPN', 'ADJ'}
        extractor = pke.unsupervised.CollabRank()

        extractor.load_document(text,
                                language='en',
                                normalization=None)
        extractor.candidate_selection(pos=pos)
        extractor.candidate_weighting(collab_documents=self.collab_documents, window=10,
                                      pos=pos)

        keyphrases, weights = list(map(list, zip(*extractor.get_n_best(n=n))))
        return keyphrases, weights

text = '''
   Over the past few years, there has been a strong and growing interest
   in faster network technologies such as FDDI and ATM. However, the
   perceived throughput at the application level has not always increased
   accordingly. Various performance bottlenecks have been encountered
   each of which has to be analysed and corrected.

   This paper presents a performance evaluation of continuous video data
   streams over ATM networks and compares them with similar experiments
   over Ethernet networks. The ATM LAN testbed for these experiments
   consists of three SG R4000 Indigos workstations connected by ATM Fore
   first generation interface cards and a Synoptics ATM switch. It is
   expected that video applications would run faster on ATM networks than
   Ethernet networks. A preliminary examination suggested to us that data
   movements through the protocol stack and the processing overheads were
   considerably high and that with some parameter settings, experiments
   with Ethernet would perform four to five times better than those with
   ATM.

   To address and explore this issue, a packetisation process has been
   added within the video application that would split each video frame
   into a number of packets. We show that by implementing a packetisation
   process at the application level, the end host is able to deliver
   cells at a faster rate. The sensitivity of these parameters, and also
   the overheads involved are discussed. The results are consistent over
   a range of video frame sizes: CIF(25K), QCIF(101K), and SCIF(405K).

'''

'''
extractor = Textacy ("yake")
k,w = extractor.get_keywords(text)
print(k)
print(w)
'''


text:
import numpy as np
from Variable import Variable, GLOBAL_VARIABLE_SCOPE

class Operator(object):
    def __init__(self, name, input_variables, output_variables):

        # init input check
        if GLOBAL_VARIABLE_SCOPE.has_key(name):
            raise Exception("Operator %s has exists !"%name)

        if not isinstance(input_variables, Variable) and not isinstance(input_variables[0], Variable):
            raise Exception("Operator %s 's input_variables is not instance(or list) of Variable!")

        if not isinstance(output_variables, Variable) and not isinstance(output_variables[0], Variable):
            raise Exception("Operator %s 's output_variables is not instance(or list) of Variable!")

        # register in GLOBAL_OP_SCOPE
        self.name = name
        GLOBAL_VARIABLE_SCOPE[self.name] = self

        self.child = []
        self.parent =[]

        # register for input Variable's child and output Variable's parents
        register_graph(input_variables, output_variables, self)

        self.wait_forward = True
        # self.wait_backward = not self.wait_forward

    def forward(self):
        pass
        # if self.wait_forward == True:
        #     1.check_parent_eval()
        #         for variable in self.parent:
        #             variable.eval()
        #     2.do forward_cal()
        #     3.set wait_forward()
        #         self.wait_forward = False
        # else:
        #     pass

    def backward(self):
        pass
        # if self.wait_forward == True:
        #     pass
        # else:
        #     1.check_child_diffeval()
        #         for variable in self.child:
        #             variable.diff_eval()
        #     2.do backward_cal()
        #     3.set wait forward()
        #         self.wait_forward=True
        #

class Conv2D(Operator):

    def __init__(self, kernel_shape=list, input_variable=Variable, name=str, stride=1, padding='SAME'):
        # kernel_shape = [ksize, ksize, input_channels, output_channels]
        for i in kernel_shape:
            if not isinstance(i, int):
                raise Exception("Operator Conv2D name: %s kernel shape is not list of int" % self.name)

        if not isinstance(input_variable, Variable):
            raise Exception("Operator Conv2D name: %s's input_variable is not instance of Variable" % name)

        if len(input_variable.shape)!=4:
            raise Exception("Operator Conv2D name: %s's input_variable's shape != 4d Variable!" % name)

        self.ksize = kernel_shape[0]
        self.stride = stride
        self.output_num = kernel_shape[-1]
        self.padding = padding
        self.col_image = []

        self.weights = Variable(kernel_shape, scope=name, name='weights',learnable=True)
        self.bias = Variable([self.output_num], scope=name, name='bias', learnable=True)
        self.batch_size = input_variable.shape[0]

        if self.padding == 'SAME':
            _output_shape = [self.batch_size, input_variable.shape[1] / stride, input_variable.shape[2] / stride,
                             self.output_num]
        if self.padding == 'VALID':
            _output_shape = [self.batch_size, (input_variable.shape[1] - self.ksize + 1) / stride,
                             (input_variable.shape[2] - self.ksize + 1) / stride, self.output_num]

        self.output_variables = Variable(_output_shape, name='out', scope=name)  # .name
        self.input_variables = input_variable
        Operator.__init__(self, name, self.input_variables, self.output_variables)

    def forward(self):
        if self.wait_forward:
            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()
            self._conv(self.input_variables, self.output_variables, self.weights.data, self.bias.data)
            self.wait_forward = False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()
            self._deconv(self.input_variables, self.output_variables, self.weights, self.bias)
            self.wait_forward = True
            return

    def _deconv(self, input=Variable, output=Variable, weights=Variable, bias=Variable):
        col_eta = np.reshape(output.diff, [self.batch_size, -1, self.output_num])
        for i in range(self.batch_size):
            weights.diff += np.dot(self.col_image[i].T, col_eta[i]).reshape(self.weights.shape)
        bias.diff += np.sum(col_eta, axis=(0, 1))

        # deconv of padded eta with flippd kernel to get next_eta
        if self.padding == 'VALID':
            pad_eta = np.pad(output.diff, (
                (0, 0), (self.ksize - 1, self.ksize - 1), (self.ksize - 1, self.ksize - 1), (0, 0)),
                             'constant', constant_values=0)

        if self.padding == 'SAME':
            pad_eta = np.pad(output.diff, (
                (0, 0), (self.ksize / 2, self.ksize / 2), (self.ksize / 2, self.ksize / 2), (0, 0)),
                             'constant', constant_values=0)

        col_pad_eta = np.array([im2col(pad_eta[i][np.newaxis, :], self.ksize, self.stride) for i in range(self.batch_size)])
        flip_weights = np.flipud(np.fliplr(weights.data))
        flip_weights = flip_weights.swapaxes(2,3)
        col_flip_weights = flip_weights.reshape([-1, weights.shape[2]])
        next_eta = np.dot(col_pad_eta, col_flip_weights)
        next_eta = np.reshape(next_eta, input.shape)
        input.diff = next_eta
        return

    def _conv(self, input=Variable, output=Variable, weights=np.ndarray, bias=np.ndarray):
        # reshape weights to col
        col_weights = weights.reshape(-1, self.output_num)

        # padding input_img according to method
        if self.padding == 'SAME':
            batch_img = np.pad(input.data, (
                (0, 0), (self.ksize / 2, self.ksize / 2), (self.ksize / 2, self.ksize / 2), (0, 0)),
                               'constant', constant_values=0)
        else:
            batch_img = input.data

        # malloc tmp output_data
        conv_out = np.zeros(output.data.shape)

        self.col_image = []
        # do dot for every image in batch by im2col dot col_weight
        for i in range(self.batch_size):
            img_i = batch_img[i][np.newaxis, :]
            col_image_i = im2col(img_i, self.ksize, self.stride)
            conv_out[i] = np.reshape(np.dot(col_image_i, col_weights) + bias, output.data[0].shape)
            self.col_image.append(col_image_i)
        self.col_image = np.array(self.col_image)

        output.data = conv_out
        return

class MaxPooling(Operator):
    def __init__(self, ksize=2, input_variable=Variable, name=str, stride=2):

        if not isinstance(input_variable, Variable):
            raise Exception("Operator Conv2D name: %s's input_variable is not instance of Variable" % name)

        self.ksize = ksize
        self.stride = stride
        self.batch_size = input_variable.shape[0]
        self.output_channels = input_variable.shape[-1]
        self.index = np.zeros(input_variable.shape)

        self.input_variables = input_variable
        _output_shape = [self.batch_size, input_variable.shape[2] / stride, input_variable.shape[2] / stride,
                         self.output_channels]
        self.output_variables = Variable(_output_shape, name='out', scope=name)
        Operator.__init__(self, name, self.input_variables, self.output_variables)

    def forward(self):
        if self.wait_forward:
            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()
            self._pool()
            self.wait_forward = False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()
            self.input_variables.diff = np.repeat(np.repeat(self.output_variables.diff, self.stride, axis=1),
                                                  self.stride, axis=2) * self.index
            self.wait_forward = True
            return

    def _pool(self):
        _out = np.zeros(self.output_variables.shape)
        for b in range(self.input_variables.shape[0]):
            for c in range(self.output_channels):
                for i in range(0, self.input_variables.shape[1], self.stride):
                    for j in range(0, self.input_variables.shape[2], self.stride):
                        _out[b, i / self.stride, j / self.stride, c] = np.max(
                            self.input_variables.data[b, i:i + self.ksize, j:j + self.ksize, c])
                        index = np.argmax(self.input_variables.data[b, i:i + self.ksize, j:j + self.ksize, c])
                        self.index[b, i+index/self.stride, j + index % self.stride, c] = 1
        self.output_variables.data = _out
        return

class FullyConnect(Operator):
    def __init__(self, output_num, input_variable=Variable, name=str):
        if not isinstance(input_variable, Variable):
            raise Exception("Operator Conv2D name: %s's input_variable is not instance of Variable" % name)

        self.batch_size = input_variable.shape[0]
        input_len = reduce(lambda x, y: x * y, input_variable.shape[1:])
        self.output_num = output_num
        self.weights = Variable([input_len, self.output_num], name='weights', scope=name, init='const' ,learnable=True)
        self.bias = Variable([self.output_num], name='bias', scope=name, init='const',learnable=True)

        self.output_variables = Variable([self.batch_size, self.output_num], name='out', scope=name)
        self.input_variables = input_variable
        Operator.__init__(self, name, self.input_variables, self.output_variables)

    def forward(self):
        if self.wait_forward:

            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()
            self.flatten_x = self.input_variables.data.reshape([self.batch_size, -1])
            self.output_variables.data = np.dot(self.flatten_x, self.weights.data)+self.bias.data
            self.wait_forward = False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()

            for i in range(self.batch_size):
                col_x = self.flatten_x[i][:, np.newaxis]
                diff_i = self.output_variables.diff[i][:, np.newaxis].T
                self.weights.diff += np.dot(col_x, diff_i)
                self.bias.diff += diff_i.reshape(self.bias.shape)
            next_diff = np.dot(self.output_variables.diff, self.weights.data.T)
            self.input_variables.diff = np.reshape(next_diff, self.input_variables.shape)

            self.wait_forward = True
            return

class SoftmaxLoss(Operator):
    def __init__(self, predict = Variable, label=Variable, name=str):
        self.batch_size = predict.shape[0]
        self.input_variables = [predict, label]
        self.loss = Variable([1], name='loss', scope=name, init='None')
        self.prediction = Variable(predict.shape, name='prediction', scope=name)
        self.softmax = np.zeros(self.prediction.shape)

        self.output_variables = [self.loss, self.prediction]
        Operator.__init__(self, name, self.input_variables, self.output_variables)

    def forward(self):
        if self.wait_forward:
            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()

            predict = self.input_variables[0].data
            label = self.input_variables[1].data

            self.prediction.data = self.predict(predict)

            self.loss.data = 0
            for i in range(self.batch_size):
                self.loss.data += np.log(np.sum(np.exp(predict[i]))) - predict[i, label[i]]

            self.wait_forward = False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()
            self.input_variables[0].diff = self.softmax.copy()
            for i in range(self.batch_size):
                self.input_variables[0].diff[i, self.input_variables[1].data[i]] -= 1
            self.wait_forward = True
            return

    def predict(self, prediction):
        exp_prediction = np.zeros(prediction.shape)
        self.softmax = np.zeros(prediction.shape)
        for i in range(self.batch_size):
            prediction[i, :] -= np.max(prediction[i, :])
            exp_prediction[i] = np.exp(prediction[i])
            self.softmax[i] = exp_prediction[i]/np.sum(exp_prediction[i])
        return self.softmax

class DropOut(Operator):
    def __init__(self, name, phase, input_variable=Variable, prob=0.5):
        self.input_variables = input_variable
        self.output_variables = Variable(shape=input_variable.shape, scope=name, name='out')
        self.prob = prob
        self.phase = phase
        self.index = np.ones(input_variable.shape)

        Operator.__init__(self, name, self.input_variables, self.output_variables)

    def forward(self):
        if self.wait_forward:
            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()
            if self.phase == 'train':
                self.index = np.random.random(self.input_variables.shape) < self.prob
                self.output_variables.data = self.input_variables.data * self.index
                self.output_variables.data /= self.prob
            elif self.phase == 'test':
                self.output_variables.data = self.input_variables.data
            else:
                raise Exception('Operator %s phase is not in test or train'% self.name)

            self.wait_forward=False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()
            if self.phase == 'train':
                self.input_variables.diff = self.output_variables.diff * self.index / self.prob
            elif self.phase == 'test':
                self.output_variables.diff = self.input_variables.diff
            else:
                raise Exception('Operator %s phase is not in test or train'% self.name)

            self.wait_forward = True
            return

def register_graph(input_variable, output_variable, operator):
    if isinstance(input_variable,Variable) and isinstance(output_variable, Variable):
        input_variable.child.append(operator.name)
        output_variable.parent.append(operator.name)
        operator.parent.append(input_variable.name)
        operator.child.append(output_variable.name)

    elif isinstance(input_variable, Variable) and len(output_variable)>1:
        for output in output_variable:
            input_variable.child.append(operator.name)
            output.parent.append(operator.name)
            operator.parent.append(input_variable.name)
            operator.child.append(output.name)

    elif isinstance(output_variable, Variable) and len(input_variable)>1:
        for _input in input_variable:
            _input.child.append(operator.name)
            output_variable.parent.append(operator.name)
            operator.parent.append(_input.name)
            operator.child.append(output_variable.name)

    elif len(output_variable)> 1 and len(input_variable)> 1:
        for _input in input_variable:
            _input.child.append(operator.name)
            operator.parent.append(_input.name)
        for output in output_variable:
            output.parent.append(operator.name)
            operator.child.append(output.name)

    else:
        raise Exception('Operator name %s input,output list error'% operator.name)

def im2col(image, ksize, stride):
    # image is a 4d tensor([batchsize, width ,height, channel])
    # print image.shape
    # print image
    image_col = []
    for i in range(0, image.shape[1] - ksize + 1, stride):
        for j in range(0, image.shape[2] - ksize + 1, stride):
            col = image[:, i:i + ksize, j:j + ksize, :].reshape([-1])

            image_col.append(col)
    image_col = np.array(image_col)

    return image_col

if __name__ == "__main__":
    A = Variable((2,2,3,3),'A')
    B = Variable((3,3,4,4),'B')
    print np.zeros((1,1))


text:
from read import read_odyssey_tuples
import numpy

class Codec:
    def __init__(self, tuplesize, num_words):
        self.tuplesize = tuplesize
        self.num_words = num_words
        (self.tuples, self.words) = read_odyssey_tuples(tuplesize, num_words)
    def tuples_to_matrix(self):
        ntuples = len(self.tuples)
        num_visible = self.tuplesize*self.num_words
        res = numpy.zeros((ntuples, num_visible),
                          dtype=numpy.float32)
        for i in range(ntuples):
            tup = self.tuples[i]
            for t in range(len(tup)):
                word = tup[t]
                res[i, word + self.num_words*t] = 1.0
        return res

max_stars_repo_name: malie/theano-rbm-on-word-tuples

max_issues_repo_name: malie/theano-rbm-on-word-tuples

max_forks_repo_name: malie/theano-rbm-on-word-tuples


text:
import time
from options.train_options import TrainOptions
from data.custom_dataset_data_loader import CustomDatasetDataLoader
from util.visualizer import Visualizer
import copy
from tqdm import tqdm
import numpy as np
import torch
from models.base_model import BaseModel

torch.manual_seed(0)

opt = TrainOptions().parse()

data_loader = CustomDatasetDataLoader(opt)
dataset = data_loader.load_data()

opt_for_eval = copy.deepcopy(opt)
opt_for_eval.isTrain = False
opt_for_eval.max_dataset_size = 1000
val_loader = CustomDatasetDataLoader(opt_for_eval)
valset = val_loader.load_data()

dataset_size = len(data_loader)
print('#training samples = %d' % dataset_size)

model = BaseModel(opt)

visualizer = Visualizer(opt)
total_steps = 0

best_l1 = np.inf

for epoch in range(opt.epoch_count, opt.niter + opt.niter_decay + 1):
    epoch_start_time = time.time()
    epoch_iter = 0

    iter_start_time = 0

    for i, data in enumerate(dataset):

        total_steps += opt.batchSize
        epoch_iter += opt.batchSize

        model.set_input(data)
        model.forward()

        if total_steps % opt.display_freq == 0:
            save_result = total_steps % opt.update_html_freq == 0
            visualizer.display_current_results(model.get_current_visuals(), epoch, save_result)

        model.optimize_parameters()

        if total_steps % opt.print_freq == 0:
            errors = model.get_current_errors()
            t = (time.time() - iter_start_time) / opt.batchSize
            visualizer.print_current_errors(epoch, epoch_iter, errors, t)
            if opt.display_id > 0:
                visualizer.plot_current_errors(epoch, float(epoch_iter)/dataset_size, opt, errors)

        iter_start_time = time.time()

    if epoch % 100 == 0:
        print('saving the model (epoch %d, total_steps %d)' % (epoch, total_steps))
        model.save(epoch)

        model.switch_mode('eval')

        l1s = []

        for i, data in enumerate(tqdm(valset)):
            model.set_input(data)
            model.forward()
            res = model.evaluate()
            l1s.append(res['L1'])
        print('eval metric: L1 %f' %(np.mean(l1s)))
        if np.mean(l1s) < best_l1:
            model.save('best')
            message = 'Update best checkpoint at end of epoch %d' % epoch
            with open(visualizer.log_name, "a") as log_file:
                log_file.write('%s l1: %f\n' % (message, np.mean(l1s)))
            print(message)
            best_l1 = np.mean(l1s)

        model.switch_mode('train')

    print('End of epoch %d / %d \t Time Taken: %d sec' %
          (epoch, opt.niter + opt.niter_decay, time.time() - epoch_start_time))
    model.update_learning_rate()


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 26 19:40:13 2019

@author: Titus
"""

import numpy as np
from scipy.optimize import curve_fit
import uncertainties as unc
import matplotlib.pyplot as plt
import uncertainties.unumpy as unp
from scipy.integrate import odeint
import pandas as pd
from scipy import stats

# calculate lower and upper prediction bands
def predband(x, xd, yd, f_vars, conf=0.95):
    """
    Code adapted from Rodrigo Nemmen's post:
    https://astropython.blogspot.com.ar/2011/12/calculating-prediction-band-
    of-linear.html

    Calculates the prediction band of the regression model at the
    desired confidence level.

    Clarification of the difference between confidence and prediction bands:

    "The prediction bands are further from the best-fit line than the
    confidence bands, a lot further if you have many data points. The 95%
    prediction band is the area in which you expect 95% of all data points
    to fall. In contrast, the 95% confidence band is the area that has a
    95% chance of containing the true regression line."
    (from https://www.graphpad.com/guides/prism/6/curve-fitting/index.htm?
    reg_graphing_tips_linear_regressio.htm)

    Arguments:
    - x: array with x values to calculate the confidence band.
    - xd, yd: data arrays.
    - a, b, c: linear fit parameters.
    - conf: desired confidence level, by default 0.95 (2 sigma)

    References:
    1. https://www.JerryDallal.com/LHSP/slr.htm, Introduction to Simple Linear
    Regression, Gerard E. Dallal, Ph.D.
    """
    alpha = 1. - conf    # Significance
    N = xd.size          # data sample size
    var_n = len(f_vars)  # Number of variables used by the fitted function.
    # Quantile of Student's t distribution for p=(1 - alpha/2)
    q = stats.t.ppf(1. - alpha / 2., N - var_n)
    # Std. deviation of an individual measurement (Bevington, eq. 6.15)
    se = np.sqrt(1. / (N - var_n) * np.sum((yd - simulate(xd, *f_vars)) ** 2))
    # Auxiliary definitions
    sx = (x - xd.mean()) ** 2
    sxd = np.sum((xd - xd.mean()) ** 2)
    # Predicted values (best-fit model)
    yp = simulate(x, *f_vars)
    # Prediction band
    dy = q * se * np.sqrt(1. + (1. / N) + (sx / sxd))
    # Upper & lower prediction bands.
    lpb, upb = yp - dy, yp + dy
    return lpb, upb

# generate data file from TCLab or get sample data file from:
#  https://apmonitor.com/pdc/index.php/Main/ArduinoEstimation2
# Import data file
# Column 1 = time (t)
# Column 2 = input (u)
# Column 3 = output (yp)
data = np.loadtxt('Heater_lab_trial_1.txt',delimiter=',')
# extract data columns
t = data[:,0].T
Q1 = data[:,1].T
Q2 = data[:,2].T
T1meas = data[:,3].T
T2meas = data[:,4].T
ind = np.linspace(0,np.size(t),np.size(t))

# number of time points
ns = len(t)

# define energy balance model
def heat(x,t,Q1,Q2,p):
    # Optimized parameters
    U,alpha1,alpha2 = p

    # Parameters
    Ta = 23 + 273.15   # K
    m = 4.0/1000.0     # kg
    Cp = 0.5 * 1000.0  # J/kg-K
    A = 10.0 / 100.0**2 # Area in m^2
    As = 2.0 / 100.0**2 # Area in m^2
    eps = 0.9          # Emissivity
    sigma = 5.67e-8    # Stefan-Boltzman

    # Temperature States
    T1 = x[0] + 273.15
    T2 = x[1] + 273.15

    # Heat Transfer Exchange Between 1 and 2
    conv12 = U*As*(T2-T1)
    rad12  = eps*sigma*As * (T2**4 - T1**4)

    # Nonlinear Energy Balances
    dT1dt = (1.0/(m*Cp))*(U*A*(Ta-T1) \
            + eps * sigma * A * (Ta**4 - T1**4) \
            + conv12 + rad12 \
            + alpha1*Q1)
    dT2dt = (1.0/(m*Cp))*(U*A*(Ta-T2) \
            + eps * sigma * A * (Ta**4 - T2**4) \
            - conv12 - rad12 \
            + alpha2*Q2)

    return [dT1dt,dT2dt]

def simulate(tm,U,alpha1,alpha2):
    T = np.zeros((len(t),2))
    T[0,0] = T1meas[0]
    T[0,1] = T2meas[0]
    T0 = T[0]
    p = (U,alpha1,alpha2)
    for i in range(len(t)-1):
        ts = [t[i],t[i+1]]
        y = odeint(heat,T0,ts,args=(Q1[i],Q2[i],p))
        T0 = y[-1]
        T[i+1] = T0
    z = np.empty((len(t)*2))
    z[0:len(t)] = T[:,0]
    z[len(t):] = T[:,1]
    return z

def simulate2(p):
    T = np.zeros((len(t),2))
    T[0,0] = T1meas[0]
    T[0,1] = T2meas[0]
    T0 = T[0]
    for i in range(len(t)-1):
        ts = [t[i],t[i+1]]
        y = odeint(heat,T0,ts,args=(Q1[i],Q2[i],p))
        T0 = y[-1]
        T[i+1] = T0
    return T

# Parameter initial guess
U = 10.0           # Heat transfer coefficient (W/m^2-K)
alpha1 = 0.0100    # Heat gain 1 (W/%)
alpha2 = 0.0075    # Heat gain 2 (W/%)
pinit = [U,alpha1,alpha2]

x = []
y = np.empty((len(t)*2))
y[0:len(t)] = T1meas
y[len(t):] = T2meas

popt, pcov = curve_fit(simulate, x, y)

Uu, alpha1u, alpha2u = unc.correlated_values(popt, pcov)

# create prediction band
lpb, upb = predband(y, y, y, popt, conf=0.95)
lpb1 = np.empty((len(t)))
lpb2 = np.empty((len(t)))
upb1 = np.empty((len(t)))
upb2 = np.empty((len(t)))
lpb1[0:len(t)] = lpb[0:len(t)]
lpb2[0:len(t)] = lpb[len(t):]
upb1[0:len(t)] = upb[0:len(t)]
upb2[0:len(t)] = upb[len(t):]

# optimized parameter values with uncertainties
print('Optimal Parameters with Uncertanty Range')
print('U: ' + str(Uu))
print('alpha1: ' + str(alpha1u))
print('alpha2: ' + str(alpha2u))

# calculate model with updated parameters
Ti  = simulate2(pinit)
Tp  = simulate2(popt)

# Plot results
plt.figure(1)

plt.subplot(3,1,1)
plt.plot(t/60.0,Ti[:,0],'y:',label=r'$T_1$ initial')
plt.plot(t/60.0,T1meas,'b-',label=r'$T_1$ measured')
plt.plot(t/60.0,Tp[:,0],'r--',label=r'$T_1$ optimized')
plt.plot(t/60.0,lpb1,'k:',label=r'$T_1$ prediction band')
plt.plot(t/60.0,upb1,'k:')

plt.ylabel('Temperature (degC)')
plt.legend(loc='best')

plt.subplot(3,1,2)
plt.plot(t/60.0,Ti[:,1],'y:',label=r'$T_2$ initial')
plt.plot(t/60.0,T2meas,'b-',label=r'$T_2$ measured')
plt.plot(t/60.0,Tp[:,1],'r--',label=r'$T_2$ optimized')
plt.plot(t/60.0,lpb2,'k:',label=r'$T_2$ prediction band')
plt.plot(t/60.0,upb2,'k:')
plt.ylabel('Temperature (degC)')
plt.legend(loc='best')

plt.subplot(3,1,3)
plt.plot(t/60.0,Q1,'g-',label=r'$Q_1$')
plt.plot(t/60.0,Q2,'k--',label=r'$Q_2$')
plt.ylabel('Heater Output')
plt.legend(loc='best')

plt.xlabel('Time (min)')
plt.show()

max_stars_repo_path: code/Lab C/param_example.py

max_issues_repo_path: code/Lab C/param_example.py

max_forks_repo_path: code/Lab C/param_example.py

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
import tensorflow as tf
import numpy as np
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import node_def_pb2

#Device, filter dimension calculator, arch block helpers

def _conv_dim(orig,filt,dilation=1,stride=1):
    return np.ceil((orig-(filt-1)*dilation)/stride)

def desambiguatename(opnamescope, scope):
  '''
   Returns a new name according to the givem op namescope and the scope to embed the op namescope
  '''
  istag = scope.find('_')
  first = opnamescope.find('/')
  appendname = scope.find('//')
  if scope == '' or (appendname != -1 and scope.split('//')[0] == orgname.split('/')[0]):
    new_name = orgname
  elif istag != -1:
    if appendname != -1:
      names = orgname.split('/')
      if names[0] == scope[:istag]:
        new_name = names[0]+scope[istag:appendname]+orgname[first:]
      elif scope[:istag] != "":
        new_name = scope[:istag]+orgname[first:]
      else:
        new_name = names[0]+scope+orgname[first:]
    else:
      names = orgname.split('/')
      if names[0] == scope[:istag]:
        new_name = names[0]+scope[istag:]+orgname[first:]
      elif scope[:istag] != "":
        new_name = scope[:istag]+orgname[first:]
      else:
        new_name = names[0]+scope+orgname[first:]
  elif appendname != -1 and scope.split('//')[0] != orgname.split('/')[0]:
    new_name = scope[:appendname]+'/'+ orgname
  else:
    new_name = scope + orgname[orgname.find('/'):]

  return new_name

def get_listeners(graph, scope = 'signal_in/'):
  '''
   Returns the name of the inputs from scopes that has the given scope as input.
  '''
  inputmap = {}
  for node in graph.as_graph_def().node:
    for inp in node.input:
      if inp.startswith(scope) and (node.name.startswith(scope) == False):
        if inputmap.get(inp) == None:
          inputmap[inp] = [node.name]
        else:
          inputmap[inp].append(node.name)
  if len(inputmap.items()) == 0:
    raise NameError('{} is not input of any Arch-Block'.format(scope) )

  return inputmap

def get_op_fromscope(graph,scope='signal_in/',opname = 'init', **kwargs):
  '''
   Return operation inside a namescope, independently of how many name scope the op is nested in.
  '''

  try:
    op = graph.get_operation_by_name(scope+opname)
    return op
  except:
    notfound = True

  if notfound:
    op = []
    for node in graph.as_graph_def().node:
      if (node.name.startswith(scope) == True) and node.name.split('/')[-1] == opname:
        op.append(node.name)
    if len(op) != 0:
      return op
    else:
      raise NameError('Operation not found')

def calculatefilter(initialdimsizes, filterlist):
  '''
   Given the dimension sizes of data structure and a list of convolutional
   filter sizes, to recursively pass the data input, calculates the output of
   size the filtering.
   Returns:
       - size of the output.
       - vector, of the size of data structure dimension, containing the index
       of the filter that is bigger than the input dimension size such the
       filtering is not possible. Once one knows this index, filter size of
       previous filters can be updated.
  '''

  if hasattr(initialdimsizes,'__iter__'):
    size = len(initialdimsizes)
    a = initialdimsizes
  else:
    size = 1
    a = [initialdimsizes]
  b = [0]*size
  c = [0]*size
  ant = 0
  stop = [-1]*size
  for i in range(size):
    for fir in range(len(filterlist)):
      ant = a[i]
      a[i] = _conv_dim(a[i],filterlist[fir][i],1,1)

      if a[i] <= 0:
        b[i] = ant
        c[i] = 1
        stop[i] = fir
        break
    if c[i] == 1:
      a[i] = np.int32(b[i])
  return a,stop

def get_tensor_list_from_scopes(graph, tensor_scope_name):

    names = []
    for i in range(len(tensor_scope_name)):
      names.append(graph.get_tensor_by_name(tensor_scope_name[i]+':0'))
    if len(names) > 1:
      return tf.keras.layers.concatenate( names, axis = 1)
    else:
      return names[0]

def find_softmax(graph, **kwargs):

  names = []
  for op in graph.get_operations():
    name = op.name
    a = name.find('softmax')
    if a != -1:
      b = name.split('/')
      if b[-2].find('softmax') != -1 and b[-1] == 'transpose_1':
        names.append(op.name)

  return names

class dev_selector:

  def __init__(self,**kwargs):
    self.arg1 = kwargs['arg1']

  def __call__(self,op):
    if op.find('gcnn2d') != -1:
      return '/device:CPU:0'
    else:
      return '/device:CPU:0'

  def _conv2D_on_gpu(op):
    if op.type == "Conv2D":
      return "/device:GPU:*"
    else:
      return "/device:CPU:*"