text:
import recommender
import random
import pandas as pd
import numpy as np
import sys
from recommender.model.Rating import Rating
from sklearn.metrics import mean_squared_error

class TestRecommendations:
    user_ratings = []
    training_ratings = []
    test_ratings = []

    def __init__(self):
        recommender.Database.initialize()

    # Extract 60 % of user ratings for training and 40 % for testing
    def test_recommendations(self):
        print("Preparing Recommendation Test")
        self.user_ratings = recommender.Database.session.query(Rating).all()
        num_extract = len(self.user_ratings) * 60 // 100
        indexes = random.sample(xrange(len(self.user_ratings)), num_extract)
        self.training_ratings = [self.user_ratings[i] for i in indexes]
        self.test_ratings = [self.user_ratings[i] for i in xrange(len(self.user_ratings)) if i not in indexes]

        if len(self.training_ratings) != len(indexes):
            print("Training set dosen\'t match the expected number of instances")
            exit(1)

        if len(self.test_ratings) != (len(self.user_ratings) - len(indexes)):
            print("Test set dosen\'t match the expected number of instances")
            exit(1)

        user_test_ratings = {}
        for rating in self.test_ratings:
            user_test_ratings.setdefault(rating.user_id, []).append(
                {'movie_id': rating.movie_id, 'value': rating.value})

        predicted_ratings = pd.DataFrame(columns=['user_id', 'movie_id', 'value', 'timestamp', 'predicted'])
        user_ids = pd.Series([rating.user_id for rating in self.test_ratings])
        movie_ids = pd.Series([rating.movie_id for rating in self.test_ratings])
        actual_values = pd.Series([rating.value for rating in self.test_ratings])
        timestamps = pd.Series([rating.timestamp for rating in self.test_ratings])

        predicted_ratings['user_id'] = user_ids
        predicted_ratings['movie_id'] = movie_ids
        predicted_ratings['value'] = actual_values
        predicted_ratings['timestamp'] = timestamps

        # Remove test data from database
        recommender.Database.session.query(Rating).filter(
            Rating.id.in_([rating.id for rating in self.test_ratings])).delete(synchronize_session=False)
        recommender.Database.session.commit()

        print("Running Recommendation Test")
        recom = recommender.Recommender()
        recom.sim_pearsons()
        recom.recommendations()

        # Calculate number of accurate recommendations
        estimated_rating = 0
        movie_id = 1
        predicted_recommendations = recom.recoms

        print("Parsing recommendations")
        num_rows = len(predicted_recommendations)
        print("num recom " + str(num_rows))
        index = 0

        for user, recoms in predicted_recommendations.iteritems():
            # Recommended movie is in the test set
            for item in recoms:
                if item[movie_id] in predicted_ratings.index and user.userId in predicted_ratings.user_id.values:
                    row_index = predicted_ratings.loc[(predicted_ratings['user_id'] == user.userId) & (
                    predicted_ratings['movie_id'] == item[movie_id])]
                    if row_index.empty is not True:
                        # print("Found match, inserting " + str(row_index))
                        predicted_ratings.set_value(row_index.index[0], 'predicted', item[estimated_rating])

            progress = (float(index) / float(num_rows)) * 100
            sys.stdout.write("Progress: %f%%    \r" % progress)
            sys.stdout.flush()
            index += 1

        print("Inserting TestData")
        list_new_ratings = []
        for index, row in predicted_ratings.iterrows():
            rating = Rating(value=row['value'], timestamp=row['timestamp'], user_id=row['user_id'],
                            movie_id=row['movie_id'])
            list_new_ratings.append(rating)

        print("Number new ratings " + str(len(list_new_ratings)))
        recommender.Database.session.bulk_save_objects(list_new_ratings)
        recommender.Database.session.commit()

        predicted_ratings = predicted_ratings.dropna(how='any')
        rms = np.sqrt(mean_squared_error(predicted_ratings['value'], predicted_ratings['predicted']))
        target = open('results', 'a')
        target.write(str(rms))
        target.write("\n")
        target.close()

        print("Finished Testing")


text:
# reader.py

import os
import tempfile

import numpy

from utils.file import shuffle

# lowest-level stream
class textreader:
    def __init__(self, name, shuffle=False, strict=False):
        if not isinstance(name, (list, tuple)):
            name = [name]

        self.eos = False
        self.count = 0
        self.names = name
        self.stream = []
        self._shuffle = shuffle
        self.strict = strict
        self.tempfiles = None
        self.indices = None

        self.reset()

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def readline(self):
        # read from file
        texts = [fd.readline() for fd in self.stream]
        flag = any([line == "" for line in texts])

        if flag:
            return None

        self.count += 1
        texts = [text.strip() for text in texts]

        return texts

    def next(self):
        data = self.readline()

        if data is None:
            self.reset()
            raise StopIteration

        return data

    def shuf(self, indices=None):
        """
        dump shuffled files to temp files and return sorting indices
        """
        # read original file
        stream = [open(item, "r") for item in self.names]
        texts = [fd.readlines() for fd in stream]

        if indices is None:
            linecnt = min([len(text) for text in texts])
            indices = numpy.arange(linecnt)
            numpy.random.shuffle(indices)
        shuffle(self.names, [name for _, name in self.tempfiles], indices, self.strict)

        return indices

    def reset(self):
        self.count = 0
        self.eos = False

        if not self.stream:
            self.stream = [open(item, "r") for item in self.names]

        for fd in self.stream:
            fd.seek(0)

        if self._shuffle:
            self.close()
            self.tempfiles = [tempfile.mkstemp() for _ in self.names]

            self.indices = self.shuf()
            self.stream = [open(name, "r") for (_, name) in self.tempfiles]

    def close(self):
        for fd in self.stream:
            fd.close()
        if self._shuffle and self.tempfiles:
            for fd, name in self.tempfiles:
                # on Windows, it's required to release both file object and descriptor
                os.close(fd)
                os.remove(name)
            self.tempfiles = None
            self.stream = []

    def get_indices(self):
        return self.indices

    def set_indices(self, indices):
        self.indices = indices

    def __getstate__(self):
        d = self.__dict__.copy()
        del d["stream"]
        del d["tempfiles"]
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        if self._shuffle:
            self.tempfiles = [tempfile.mkstemp() for _ in self.names]
            self.shuf(self.indices)
            self.stream = [open(name, "r") for (_, name) in self.tempfiles]
        else:
            self.stream = [open(name, "r") for name in self.names]
        for _ in xrange(self.count):
            for stream in self.stream:
                stream.readline()

ext: py

max_stars_repo_name: jtianesq/protein-prediction-nmt-evo

max_issues_repo_name: jtianesq/protein-prediction-nmt-evo

max_forks_repo_name: jtianesq/protein-prediction-nmt-evo

reason: import numpy


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-

import unittest

import numpy as np
from numpy.testing import assert_allclose
from scipy.stats import sem as standard_error
from symengine import symbols

from jitcode import jitcode, y, jitcode_lyap, UnsuccessfulIntegration, test
from jitcode._jitcode import _is_C, _is_lambda

from scenarios import (
		y0, f_of_y0, jac_of_y0, y1, lyaps, vanilla, n,
	)

class TestOrders(unittest.TestCase):
	"""
	tests that the derivative and Jacobian are compiled/generated as intended when calling the several methods of the jitcode object in a certain order.
	"""
	def test_standard_order(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.set_integrator("dopri5")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_C(self.ODE.f))
		self.assertIsNone(self.ODE.jac)

	def test_standard_order_with_jac(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.set_integrator("lsoda")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_C(self.ODE.f))
		self.assertTrue(_is_C(self.ODE.jac))

	def test_initialise_first(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.set_initial_value(y0)
		self.ODE.set_integrator("dop853")
		self.assertTrue(_is_C(self.ODE.f))

	def test_initalise_with_dict(self):
		self.ODE = jitcode(**vanilla)
		initial_value = {y(i):y0[i] for i in range(n)}
		self.ODE.set_initial_value(initial_value)
		self.ODE.set_integrator("dop853")
		self.assertTrue(_is_C(self.ODE.f))

	def test_lambdas_with_jac(self):
		self.ODE = jitcode(wants_jacobian=True,**vanilla)
		self.ODE.generate_lambdas()
		self.ODE.set_integrator("lsoda")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_lambda(self.ODE.f))
		self.assertTrue(_is_lambda(self.ODE.jac))

	def test_lambdas_without_jac(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.generate_lambdas()
		self.assertIsNone(self.ODE.jac)
		self.ODE.set_integrator("vode")
		self.ODE.set_initial_value(y0)

	def test_compile_without_jac(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.compile_C()
		self.ODE.set_integrator("LSODA")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_C(self.ODE.f))
		self.assertTrue(_is_C(self.ODE.jac))

	def test_generate_jac_manually(self):
		self.ODE = jitcode(**vanilla)
		self.ODE.generate_jac_C()
		self.ODE.set_integrator("vode")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_C(self.ODE.f))
		self.assertTrue(_is_C(self.ODE.jac))

	def test_save_and_load(self):
		self.ODE = jitcode(**vanilla)
		filename = self.ODE.save_compiled(overwrite=True)
		self.ODE = jitcode(n=n,module_location=filename)
		self.ODE.set_integrator("RK45")
		self.ODE.set_initial_value(y0)
		self.assertTrue(_is_C(self.ODE.f))
		self.assertIsNone(self.ODE.jac)

	def tearDown(self):
		if isinstance(self.ODE,jitcode) and self.ODE.f_sym():
			self.ODE.check()
		self.assertIsNotNone(self.ODE.f)
		assert_allclose( self.ODE.f(0.0,y0), f_of_y0, rtol=1e-5 )
		if not self.ODE.jac is None:
			assert_allclose( self.ODE.jac(0.0,y0), jac_of_y0, rtol=1e-5)
		assert_allclose( self.ODE.integrate(1.0), y1, rtol=1e-4 )
		for i in reversed(range(n)):
			assert_allclose( self.ODE.y_dict[y(i)], y1[i], rtol=1e-4 )

integrators = [
		#  name     with_jac
		("dopri5"  , False),
		("dop853"  , False),
		("lsoda"   , True ),
		("vode"    , True ),
		("RK23"    , False),
		("RK45"    , False),
		("Radau"   , True ),
		("BDF"     , True ),
		("LSODA"   , True ),
	]

class TestIntegrators(unittest.TestCase):
	"""
		Tests for every known integrator that its properties are identified correctly and the derivative and Jacobian are generated as needed.
	"""
	def test_normal_integration(self):
		for lambdas in (True,False):
			for integrator in integrators:
				with self.subTest(integrator=integrator,lambdas=lambdas):
					ODE = jitcode(**vanilla)
					if lambdas:
						ODE.generate_f_lambda()
						ODE.generate_jac_lambda()
					ODE.set_integrator(integrator[0])
					assert ODE._wants_jacobian == integrator[1]
					assert not  _is_C(ODE.f) == lambdas
					assert _is_lambda(ODE.f) == lambdas
					if integrator[1]:
						assert _is_lambda(ODE.jac) == lambdas
						assert not  _is_C(ODE.jac) == lambdas
					ODE.set_initial_value(y0,0.0)
					assert_allclose( ODE.integrate(1.0), y1, rtol=1e-3 )

class TestLyapunov(unittest.TestCase):
	"""
		Integration test for jitcode_lyap.
	"""
	def test_regular(self):
		self.ODE = jitcode_lyap(**vanilla,n_lyap=n)
		self.ODE.set_integrator("dopri5")

	def test_jac(self):
		self.ODE = jitcode_lyap(**vanilla,n_lyap=n)
		self.ODE.set_integrator("lsoda")

	def test_save_and_load_with_jac(self):
		self.ODE = jitcode_lyap(**vanilla,n_lyap=n,wants_jacobian=True)
		filename = self.ODE.save_compiled(overwrite=True)
		self.ODE = jitcode_lyap((),n=n,n_lyap=n,module_location=filename)
		self.ODE.set_integrator("vode")
		self.assertTrue(_is_C(self.ODE.f))
		self.assertTrue(_is_C(self.ODE.jac))

	def tearDown(self):
		if isinstance(self.ODE,jitcode) and self.ODE.f_sym():
			self.ODE.check()
		self.ODE.set_initial_value(y0,0.0)
		times = range(10,100000,10)
		data = np.vstack([ self.ODE.integrate(time)[1] for time in times ])
		result = np.average(data[1000:], axis=0)
		margin = standard_error(data[1000:], axis=0)
		self.assertLess( np.max(margin), 0.003 )
		for i in range(n):
			self.assertLess( result[i]-lyaps[i], 5*margin[i] )

class TestErrors(unittest.TestCase):
	def test_duplicate_error(self):
		ODE1 = jitcode(**vanilla)
		ODE2 = jitcode(**vanilla)
		ODE1.compile_C(modulename="foo")
		with self.assertRaises(NameError):
			ODE2.compile_C(modulename="foo")

	def test_wrong_n(self):
		with self.assertRaises(ValueError):
			ODE = jitcode(**vanilla,n=2*n)

	def test_dimension_mismatch(self):
		ODE = jitcode(**vanilla)
		with self.assertRaises(ValueError):
			ODE.set_initial_value(np.array([1.,2.,3.]),0.0)

	def test_check_index_negative(self):
		ODE = jitcode([y(-1)])
		with self.assertRaises(ValueError):
			ODE.check()

	def test_check_index_too_high(self):
		ODE = jitcode([y(1)])
		with self.assertRaises(ValueError):
			ODE.check()

	def test_check_undefined_variable(self):
		x = symbols("x")
		ODE = jitcode([x])
		with self.assertRaises(ValueError):
			ODE.check()

	def test_no_integrator(self):
		ODE = jitcode(**vanilla)
		with self.assertRaises(RuntimeError):
			ODE.integrate(1.0)

	def test_no_interpolation_LSODA(self):
		ODE = jitcode(**vanilla)
		with self.assertRaises(NotImplementedError):
			ODE.set_integrator("LSODA",interpolate=False)

if __name__ == "__main__":
	unittest.main(buffer=True)

max_stars_repo_name: neurophysik/jitcode

max_issues_repo_name: neurophysik/jitcode

max_forks_repo_name: neurophysik/jitcode

reason: import numpy,from numpy,from scipy


text:
import glob
import os

import funcy
import numpy as np
import tensorflow as tf

from .feed_dict import simple_feed_builder
from ..batches import generate_batches
from ..data_types import Results
from ..util import get_number_of_samples

# This is in the form {output variable name: tf graph variable name}
DEFAULT_PRED_DICT = {'y': 'output/predictions'}

def train_model_with_stopping(
    model_builder,
    preprocessor,
    train_data,
    val_data=None,
    test_data=None,
    feed_builder=simple_feed_builder,
    batch_generator=generate_batches,
    predict_batch_generator=None,
    n_stop=-1,
    verbose=1,
    losses=None,
    train_str='output/training',
    pred_dict=DEFAULT_PRED_DICT,
    model_file=None
):
    """Train a Tensorflow model with rudimentary early stopping.

    The preprocessor takes in train_data, val_data, and test_data
    and outputs (train, val, test, feature_names), where train, val, and test
    are Datasets. It does any additional processing that hasn't been done yet.

    The model_builder takes in the output of the preprocessor and outputs
    a tensorflow graph.

    The feed_builder takes in a batch and a boolean flag to designate whether
    we are on a training step or not and outputs a tensorflow feed dict.

    The batch_generator takes in a Dataset and outputs a BatchData instance.

    Args:
        model_builder: callable, builds the tensorflow graph
        preprocessor: callable,
        train_data: Dataset instance
        val_data: maybe(Dataset) instance
        test_data: maybe(Dataset) instance
        feed_builder: callable, creates a feed dict
        batch_generator: callable, generates batches
        predict_batch_generator: maybe(callable), an alternative batch generator
                                 for predictions on the test & validation sets
        n_stop: int, number of epochs with no improvement before stopping.
                If 0 or negative, the stopping is disabled.
        verbose: int, verbosity level
        losses: list of Loss tuples
        train_str: str, the training operation for the graph
        pred_dict: dict of str -> str, gives the mapping of output variable name to
                   tensorflow graph variable name.
        model_file: maybe(str or callable)

    Returns:
        Results instance with predictions on any validation or test data.
    """
    best_scores, best_epochs = _initialize_stopping(losses)
    if predict_batch_generator is None:
        predict_batch_generator = batch_generator

    train, val, test, features = preprocessor(
        train_data, val_data, test_data
    )
    model = model_builder(train, val, test, features)

    if model_file is None:
        output_file = None
    elif isinstance(model_file, str):
        output_file = model_file
    elif callable(model_file):
        output_file = model_file()
    else:
        raise TypeError('model_file must be None, str, or callable.')

    with tf.Session(graph=model) as sess:
        if output_file is None:
            saver = None
        else:
            saver = tf.train.Saver()
        tf.global_variables_initializer().run()

        stop_test_counter = 0
        for batch in batch_generator(train):
            train_feed_dict = feed_builder(batch, True)

            sess.run(train_str, feed_dict=train_feed_dict)
            if batch.epoch_done and losses is not None and val is not None:

                if verbose == 0 and n_stop <= 0:
                    continue

                scores = _get_validation_losses(
                    sess, val, losses, predict_batch_generator, feed_builder, verbose
                )

                (
                    best_epochs, best_scores, is_updated, stop_training
                ) = _check_early_stopping(
                    losses, scores, best_epochs, best_scores, stop_test_counter, n_stop
                )

                _save_models(sess, saver, output_file, is_updated)

                if stop_training:
                    if verbose > 0:
                        print('Stopping training at epoch {}'.format(batch.epoch))
                    break

                stop_test_counter += 1

        _restore_model(sess, saver, output_file, best_epochs)

        if verbose > 0 and val is not None:
            print('Making validation set predictions')
        val_preds = _run_predictions(
            sess,
            pred_dict,
            data=val,
            batch_generator=predict_batch_generator,
            feed_builder=feed_builder
        )
        if verbose > 0 and test is not None:
            print('Making test set predictions')
        test_preds = _run_predictions(
            sess,
            pred_dict,
            data=test,
            batch_generator=predict_batch_generator,
            feed_builder=feed_builder
        )

    results = Results(validation=val_preds, test=test_preds)
    return results

def _run_predictions(sess, pred_dict, data, batch_generator, feed_builder):
    """With the current active session, make predictions from a given dataset.

    A feed dict may be passed directly into this function but otherwise, a Dataset
    and a function to build a feed dict must be used instead.

    Args:
        sess: tensorflow.Session instance
        pred_dict: dict of str -> str, gives the mapping of output variable name to
                   tensorflow graph variable name.
        data: Dataset
        batch_generator: callable
        feed_builder: callable

    Returns:
        dict of str to numpy.ndarray giving a mapping of output field name
             to predicted values
    """
    if data is None:
        return None

    n_total = get_number_of_samples(data)
    # Get the tensor names to be sent to tensorflow
    pred_names = [pred + ':0' for pred in pred_dict.values()]
    # Get the correct output shapes
    pred_shapes = [
        tf.get_default_graph().get_tensor_by_name(pred_name).get_shape().as_list()
        for pred_name in pred_names
    ]
    for pred_shape in pred_shapes:
        pred_shape[0] = n_total
    # Get the output labels and build the output arrays
    pred_labels = [label for label in pred_dict.keys()]
    output_dict = {
        label: np.zeros(shape, dtype=np.float32)
        for label, shape in zip(pred_labels, pred_shapes)
    }

    # Loop through batches
    for batch in batch_generator(data):
        feed_dict = feed_builder(batch, False)
        preds = sess.run(pred_names, feed_dict=feed_dict)
        for label, pred in zip(pred_labels, preds):
            output_dict[label][batch.index] = pred
        if batch.epoch_done:
            break

    return output_dict

def _get_validation_losses(
        sess,
        data,
        losses,
        batch_generator,
        feed_builder,
        verbose):
    """Calculate losses for a given dataset.

    Args:
        sess: tensorflow.Session
        data: Dataset instance
        losses: list of Loss tuples
        batch_generator: callable
        feed_builder: callable
        verbose: int

    Returns:
        dict of str -> float
    """
    loss_tensor_names = funcy.ldistinct([loss.tensor for loss in losses])
    tensor_dict = {name: name for name in loss_tensor_names}
    result_dict = _run_predictions(
        sess, tensor_dict, data, batch_generator, feed_builder
    )

    loss_dict = {
        loss.name: loss.function(
            data.outputs[loss.field], result_dict[loss.tensor]
        )
        for loss in losses
    }

    if verbose > 0:
        print('Validation Scores:')
        for loss_name, loss in loss_dict.items():
            print('{}: {:0.5}'.format(loss_name, loss))
    return loss_dict

def _check_early_stopping(losses, scores, best_epochs, best_scores, current_epoch, n_stop):
    """Check when to stop training due to early stopping.

    The training will stop once one or more loss functions fail to improve for
    a set number of epochs.

    Args:
        losses: list of Loss tuples
        scores: dict of str -> float, keys are the loss function names, values
                are the current values of these losses.
        best_epochs: dict of str -> int, keys are the loss functions
        best_scores: dict of str -> float, keys are the loss functions
        current_epoch: int, the number of the current epoch.
        n_stop: int, how many epochs to wait before stopping trainng.

    Returns:
        3-tuple:
        updated_epochs: best_epochs after updating for this stopping round
        updated_scores: best_scores after updating for this stopping round
        is_updated: list of str, the losses that were updated
        stop_training: bool, True if training should be stopped at this point
    """
    if n_stop <= 0:
        return best_epochs, best_scores, False

    updated_epochs = {}
    updated_scores = {}
    stop_training = False
    is_updated = []
    for loss in losses:
        loss_name = loss.name
        greater_is_better = loss.greater_is_better
        new_score = scores[loss_name]
        best_epoch = best_epochs[loss_name]
        best_score = best_scores[loss_name]

        if (((greater_is_better is True) and (new_score > best_score)) or
                ((greater_is_better is False) and (new_score < best_score))):
            updated_epochs[loss_name] = current_epoch
            updated_scores[loss_name] = new_score
            is_updated.append(loss_name)
        else:
            updated_epochs[loss_name] = best_epoch
            updated_scores[loss_name] = best_score
            if current_epoch - best_epoch >= n_stop:
                stop_training = True

    return updated_epochs, updated_scores, is_updated, stop_training

def _initialize_stopping(losses):
    """Initialize early stopping variables for a set of losses/scores.

    Args:
        losses: list of Loss tuples.

    Returns:
        2-tuple giving:
        best_losses: str -> float, keys are the loss functions, values
                     are the best scores evaluated for each loss function.
        best_epochs: str -> int, keys are the loss functions, values
                     are the epochs with the best score for that loss.
    """
    best_losses = {
        loss.name: (1 - 2 * loss.greater_is_better) * np.inf
        for loss in losses
    }
    best_epochs = {loss.name: 0 for loss in losses}
    return best_losses, best_epochs

def _restore_model(
        sess,
        saver,
        output_file,
        best_epochs,
):
    """Restore and clean up models.

    Args:
        sess: tensorflow Session instance
        saver: tensorflow Saver instance
        output_file: str, the output file path
        best_epochs: dict, str -> int
    """
    if output_file is None:
        return

    model_to_restore = min(
        list(best_epochs.items()),
        key=lambda x: x[1]
    )[0]

    restore_file = output_file + '_' + model_to_restore
    saver.restore(sess, restore_file)
    saver.save(sess, output_file)
    for loss in best_epochs.keys():
        file_path = output_file + '_' + loss
        files = glob.glob(file_path + '*')
        for file in files:
            os.remove(file)

def _save_models(sess, saver, output_file, is_updated):
    """

    Args:
        sess: tensorflow Session instance
        saver: tensorflow Saver instance
        output_file: str, the name of the expected output file
        is_updated: list of str, models to update
    """
    if output_file is None:
        return
    for loss in is_updated:
        saver.save(
            sess,
            output_file + '_' + loss
        )


text:
import random
from typing import Any, Dict, List, Tuple
from unittest.mock import Mock

import numpy as np
import pytest

# Assertions
from eddington import fitting_function

def assert_calls(mock_object: Mock, calls: List[Tuple[List[Any], Dict[str, Any]]], rel):
    expected_number_of_calls = len(calls)
    assert mock_object.call_count == expected_number_of_calls, (
        f"Lists should have the same length. Expected {expected_number_of_calls}, "
        f"but got {mock_object.call_count}"
    )
    for i, (args, kwargs) in enumerate(calls):
        assert_list_equal(mock_object.call_args_list[i][0], args, rel)
        assert_dict_equal(mock_object.call_args_list[i][1], kwargs, rel)

def assert_list_equal(list1: List[Any], list2: List[Any], rel):
    length1, length2 = len(list1), len(list2)
    assert (
        length1 == length2
    ), f"Lists should have the same length. {length1} != {length2}"
    for item1, item2 in zip(list1, list2):
        assert_equal_item(item1, item2, rel)

def assert_dict_equal(dict1: Dict[Any, Any], dict2: Dict[Any, Any], rel):
    assert set(dict1.keys()) == set(dict2.keys()), "Keys are different than expected"
    for key, value in dict1.items():
        value2 = dict2[key]
        assert_equal_item(value, value2, rel)

def assert_equal_item(value1, value2, rel):
    if isinstance(value1, (list, np.ndarray)):
        assert_numpy_array_equal(value1, value2, rel)
    elif isinstance(value1, float):
        assert value1 == pytest.approx(
            value2, rel=rel
        ), f"{value1} should be the same as {value2}"
    else:
        assert value1 == value2, f"{value1} should be the same as {value2}"

def assert_numpy_array_equal(array1, array2, rel):
    if len(np.shape(array1)) <= 1:
        assert array1 == pytest.approx(array2, rel=rel)
        return
    assert np.shape(array1) == np.shape(array2)
    for i in range(np.shape(array1)[0]):
        assert array1[i] == pytest.approx(array2[i], rel=rel)

# Additional methods

def dummy_function(name, syntax, save=True):
    value = random.random()

    @fitting_function(n=2, name=name, syntax=syntax, save=save)
    def dummy_func(a, x):  # pylint: disable=W0613
        return value

    return dummy_func

max_stars_repo_name: sonicrs/eddington

max_issues_repo_name: sonicrs/eddington

max_forks_repo_name: sonicrs/eddington


text:
from skimage import io
import numpy as np
from tqdm import tqdm
import shutil
import os
from aicsimageio import AICSImage, imread
import shutil
import time
import numpy
import random
from aicsimageio import AICSImage, imread
from aicsimageio.writers import png_writer
from tqdm import tqdm
from google.colab.patches import cv2_imshow
from aicsimageio.writers.ome_tiff_writer import OmeTiffWriter
from tqdm import tqdm
from timeit import default_timer as timer
import imageio
import tifffile
from aicsimageio.transforms import reshape_data
from datetime import datetime

def make_folder_with_date(save_location, name):
  today = datetime.now()
  if today.hour < 12:
    h = "00"
  else:
    h = "12"
  sub_save_location = save_location + "/" + today.strftime('%Y%m%d%H')+ "_"+ today.strftime('%H%M%S')+ "_%s"%name
  os.mkdir(sub_save_location)
  return sub_save_location

def diplay_img_info(img, divisor):
  ### display image data
    image_resolution = img.shape[-1]
    nr_z_slices = img.shape[2]
    nr_channels = img.shape[0]
    nr_timepoints = img.shape[1]
    x_dim = img.shape[-1]
    y_dim = img.shape[-2]
    x_div = x_dim//divisor
    y_div = y_dim//divisor
    print(img.shape)
    print("The Resolution is: " + str(image_resolution))
    print("The number of z-slizes is: " + str(nr_z_slices))
    print("The number of timepoints: " + str(nr_timepoints))
    print("The number of channels: " + str(nr_channels))
    return nr_z_slices, nr_channels, nr_timepoints, x_dim, y_dim, x_div, y_div

def rotation_aug(source_img, name, path, flip=False):
    print(source_img.shape)
    # Source Rotation
    source_img_90 = np.rot90(source_img,axes=(2,3))
    source_img_180 = np.rot90(source_img_90,axes=(2,3))
    source_img_270 = np.rot90(source_img_180,axes=(2,3))
    # Add a flip to the rotation
    if flip == True:
      source_img_lr = np.fliplr(source_img)
      source_img_90_lr = np.fliplr(source_img_90)
      source_img_180_lr = np.fliplr(source_img_180)
      source_img_270_lr = np.fliplr(source_img_270)

      #source_img_90_ud = np.flipud(source_img_90)
    # Save the augmented files
    # Source images
    with OmeTiffWriter(path + "/" + name + ".tif") as writer2:
        writer2.save(source_img, dimension_order='TZYX')
    with OmeTiffWriter(path + "/" + name +'_90.tif') as writer2:
        writer2.save(source_img_90, dimension_order='TZYX')
    with OmeTiffWriter(path + "/" + name +'_180.tif') as writer2:
        writer2.save(source_img_180, dimension_order='TZYX')
    with OmeTiffWriter(path + "/" + name +'_270.tif') as writer2:
      writer2.save(source_img_270, dimension_order='TZYX')
    # Target images

    if flip == True:
      with OmeTiffWriter(path + "/" + name + '_lr.tif') as writer2:
        writer2.save(source_img_lr, dimension_order='TZYX')
      with OmeTiffWriter(path + "/" + name + '_90_lr.tif') as writer2:
          writer2.save(source_img_90_lr, dimension_order='TZYX')
      with OmeTiffWriter(path + "/" + name + '_180_lr.tif') as writer2:
          writer2.save(source_img_180_lr, dimension_order='TZYX')
      with OmeTiffWriter(path + "/" + name + '_270_lr.tif') as writer2:
        writer2.save(source_img_270_lr, dimension_order='TZYX')

def flip(source_img, name, path):
    source_img_lr = np.fliplr(source_img)
    with OmeTiffWriter(path + "/" + name + ".tif") as writer2:
        writer2.save(source_img, dimension_order='TZYX')
    with OmeTiffWriter(path + "/" + name + '_lr.tif') as writer2:
      writer2.save(source_img_lr, dimension_order='TZYX')

max_stars_repo_name: bencoster/DAIN

max_issues_repo_name: bencoster/DAIN

max_forks_repo_name: bencoster/DAIN


text:
"""
Test density estimation techniques
"""
import numpy as np
from numpy.testing import assert_allclose
from scipy.stats import norm
from astroML.density_estimation import KDE, KNeighborsDensity

def check_1D_density(clf, X, X2, true_dens, atol):
    clf.fit(X)
    dens = clf.eval(X2)

    assert_allclose(dens, true_dens, atol=atol)

def test_1D_density():
    np.random.seed(0)
    dist = norm(0, 1)

    X = dist.rvs((5000, 1))
    X2 = np.linspace(-5, 5, 10).reshape((10, 1))
    true_dens = dist.pdf(X2[:, 0]) * X.shape[0]

    classifiers = [KDE('gaussian', h=0.1),
                   KDE('tophat', h=0.2),
                   KDE('exponential', h=0.1),
                   KDE('quadratic', h=0.2),
                   KNeighborsDensity(method='simple', n_neighbors=250),
                   KNeighborsDensity(method='bayesian', n_neighbors=250)]

    for clf in classifiers:
        yield (check_1D_density, clf, X, X2, true_dens, 100)

max_stars_repo_name: nhuntwalker/astroML

max_issues_repo_name: nhuntwalker/astroML

max_forks_repo_name: nhuntwalker/astroML


text:
# Copyright 2021 Xanadu Quantum Technologies Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Labudde tests"""
import math
import numpy as np
import thewalrus.charpoly

import pytest

@pytest.mark.parametrize("phi", [0.1, 0.2, 0.3])
def test_labudde_2by2(phi):
    """Test that the La Budde algorithm produces the correct characteristic polynomial
    from https://en.wikipedia.org/wiki/Characteristic_polynomial."""
    sinh_phi = math.sinh(phi)
    cosh_phi = math.cosh(phi)
    mat = np.array([[cosh_phi, sinh_phi], [sinh_phi, cosh_phi]])
    charpoly = thewalrus.charpoly.charpoly(mat)
    assert np.allclose(charpoly[0], -2 * cosh_phi)
    assert np.allclose(charpoly[1], 1)

@pytest.mark.parametrize("n", [1, 2, 3])
def test_powtrace_2by2(n):
    """Consistency test between power_trace_eigen and powertrace"""
    phi = 0.1 * math.pi
    sinh_phi = math.sinh(phi)
    cosh_phi = math.cosh(phi)
    mat = np.array([[cosh_phi, sinh_phi], [sinh_phi, cosh_phi]])
    pow_trace_lab = thewalrus.charpoly.powertrace(mat, n + 1)

    # Use wolfram alpha to verify with:
    # Trace[[[1.04975523, 0.31935254], [0.31935254, 1.04975523]]^1]
    # Trace[[[1.04975523, 0.31935254], [0.31935254, 1.04975523]]^2]
    # Trace[[[1.04975523, 0.31935254], [0.31935254, 1.04975523]]^3]

    if n == 1:
        assert np.allclose(pow_trace_lab[-1], 2.09951)
    if n == 2:
        assert np.allclose(pow_trace_lab[-1], 2.40794)
    if n == 3:
        assert np.allclose(pow_trace_lab[-1], 2.95599)

@pytest.mark.parametrize("n", [1, 2, 4])
def test_powtrace_4by4(n):
    """Consistency test between power_trace_eigen and powertrace"""

    mat = np.array(
        [
            [1.04975523, 0.31935254, 1, 2],
            [0.31935254, 1.04975523, 3, 4],
            [0.31635254, 2.444, 5, 6],
            [21.31935254, 3.14975523, 7, 8],
        ]
    )
    pow_trace_lab = thewalrus.charpoly.powertrace(mat, n + 1)
    if n == 1:
        assert np.allclose(pow_trace_lab[-1], 15.0995)
    if n == 2:
        assert np.allclose(pow_trace_lab[-1], 301.18)
    if n == 4:
        assert np.allclose(pow_trace_lab[-1], 81466.1)

max_stars_repo_name: GregoryMorse/thewalrus

max_issues_repo_name: GregoryMorse/thewalrus

max_forks_repo_name: GregoryMorse/thewalrus


text:
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.colors as colors

from ..utils.plotting_utils import match_loc_data_map_data
import pickle
import cartopy
import cartopy.crs as ccrs
#TODO put in utils/eval...? function necessary for what? -> eval?

def plot_cluster_freq_maps(config,
                           cluster_frequency,
                           n_rows=2,
                           tag='loc'):
    c_label = r"frequency [%]"

    n_cols = int(np.ceil(cluster_frequency.shape[0]/n_rows))
    cm = 1/2.54
    figsize = (n_cols*3.5*cm+.2, n_rows*7*cm+.2)

    mrc = ccrs.Mercator()
    color_map = plt.get_cmap('coolwarm')  # 'YlOrRd')
    fig, ax = plt.subplots(n_rows, n_cols, figsize=figsize,
                           subplot_kw={'projection': mrc})
    wspace = 0.001
    plt.subplots_adjust(top=1-0.01*2/n_rows, bottom=0.04*2/n_rows, left=0.01, right=0.99,
                        hspace=0.01*2/n_rows, wspace=wspace)

    # Get min/max for each row -> colorbar
    min_max = np.zeros((n_rows, 2))
    min_freq = np.min(cluster_frequency, axis=1)
    max_freq = np.max(cluster_frequency, axis=1)
    for i in range(n_rows):
        min_max[i, :] = (np.min(min_freq[i*n_cols: (i+1)*n_cols]),
                         np.max(max_freq[i*n_cols: (i+1)*n_cols]))

    for i in range(cluster_frequency.shape[0]):
        freq = cluster_frequency[i]
        k = i % n_cols
        j = i // n_cols

        # ax[j, k].set_extent([config.Data.lon_range[0],
        #                     config.Data.lon_range[1],
        #                     config.Data.lat_range[0],
        #                     config.Data.lat_range[1]])
        ax[j, k].coastlines(zorder=4)
        # TODO resolution='50m', color='black', linewidth=1)

        normalize = mpl.colors.Normalize(vmin=min_max[j, 0],
                                         vmax=min_max[j, 1])

        lons_grid, lats_grid = np.meshgrid(config.Data.all_lons,
                                           config.Data.all_lats)
        data_grid = match_loc_data_map_data(config, freq)
        # Compute map projection coordinates.
        # TODO does this work now? threw error too many values to unpack
        # - works locally
        pcm = ax[j, k].pcolormesh(lons_grid, lats_grid, data_grid,
                                  cmap=color_map, norm=normalize,
                                  transform=cartopy.crs.PlateCarree(),
                                  zorder=3)
        # Colorbar below each row
        if k == n_cols-1 or i == cluster_frequency.shape[0]-1:
            fig.colorbar(pcm, ax=ax[j, :], shrink=0.6, location='bottom',
                         label=c_label, pad=0.001 + 0.05*2/n_rows)
        # fig.colorbar(pcm, ax=ax[j, k], shrink=0.6, location='bottom',
        #              label=c_label)

        # Add cluster profile ID tag
        txt = '${}$'.format(int(i+1))
        cmap = plt.get_cmap("gist_ncar")
        if config.Clustering.n_clusters > 25:
            if i % 2 == 1:
                if config.Clustering.n_clusters % 2 == 1:
                    shift = -1
                else:
                    shift = 0
                i_c = -i + shift
            else:
                i_c = i
        else:
            i_c = i
        clrs = cmap(np.linspace(0.03, 0.97, config.Clustering.n_clusters))
        ax[j, k].plot(0.1, 0.1, 'o', mfc=clrs[i_c], alpha=1, ms=14, mec='k',
                      transform=ax[j, k].transAxes, zorder=4)  # "white"
        ax[j, k].plot(0.1, 0.1, marker=txt, alpha=1, ms=7, mec='k',
                      transform=ax[j, k].transAxes, zorder=4)
        # TODO Add matching background color cluster name -> mfc to power curves colors
    if not config.Plotting.plots_interactive:
        plt.savefig(config.IO.plot_output
                    .format(title='cluster_map_projections_rel_to_{}'
                            .format(tag)))

def plot_location_map(config):
    locations = config.Data.locations
    # Prepare the general map plot.
    with open(config.IO.labels, 'rb') as f:
        labels_file = pickle.load(f)
    labels = labels_file['labels [-]']
    locations = labels_file['locations']
    n_samples_per_loc = labels_file['n_samples_per_loc']

    # Evaluate locations data
    loc_data_cluster = np.zeros(len(locations))
    max_cluster_found = np.zeros(config.Clustering.n_clusters)
    for i_loc in range(len(locations)):
        loc_labels = labels[
            n_samples_per_loc*i_loc:n_samples_per_loc*(i_loc+1)]
        cluster_frequency = np.zeros(config.Clustering.n_clusters)
        for i_cluster in range(config.Clustering.n_clusters):
            cluster_frequency[i_cluster] = sum(loc_labels == i_cluster+1)
        max_cluster = max(zip(cluster_frequency,
                              np.arange(config.Clustering.n_clusters)+1),
                          key=lambda t: t[0])[1]
        loc_data_cluster[i_loc] = max_cluster
        max_cluster_found[max_cluster-1] = 1

    # print(loc_data_cluster)

    # Compute relevant map projection coordinates
    lats = [lat for lat, _ in locations]
    lons = [lon for _, lon in locations]

    cm = 1/2.54
    fig = plt.figure(figsize=(13*cm, 14.5*cm))

    mrc = ccrs.Mercator()
    ax = plt.axes(projection=mrc)
    ax.coastlines()  # TODO resolution='50m', color='black', linewidth=1)
    plt.title("Cluster mapping")
    # Compute map projection coordinates.

    # TODO what do I use this for, improve plotting/labelling
    if sum(max_cluster_found) <= 8:
        color_map = plt.get_cmap('Dark2')
    elif sum(max_cluster_found) <= 20:
        color_map = plt.get_cmap('tab20')
    else:
        # continuous
        # TODO define custom discrete color map depending on found clusters
        color_map = plt.get_cmap('nipy_spectral')

    # Map found max clusters to cluster id
    max_cluster = [i+1 for i, m in enumerate(max_cluster_found) if m == 1]

    loc_data = [max_cluster.index(c)+1 for c in loc_data_cluster]

    normalize = mpl.colors.Normalize(vmin=1, vmax=len(max_cluster))

    plt.scatter(lons, lats, c=color_map(normalize(loc_data)),
                transform=cartopy.crs.PlateCarree(),
                zorder=0.5)

    cax, _ = mpl.colorbar.make_axes(ax)
    cbar = mpl.colorbar.ColorbarBase(cax,
                                     cmap=color_map,
                                     norm=normalize,
                                     ticks=np.arange(len(max_cluster))+1,
                                     label='Cluster')
    cbar.set_ticklabels(max_cluster)
    if not config.Plotting.plots_interactive:
        plt.savefig(config.IO.plot_output.format(title='map_max_cluster_id'))

if __name__ == '__main__':
    from ..config import Config
    config = Config()
    plot_location_map(config)
    plt.show()

max_stars_repo_name: lthUniBonn/AWERA

max_issues_repo_name: lthUniBonn/AWERA

max_forks_repo_name: lthUniBonn/AWERA


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 31 23:10:58 2020

@author: kissami
"""
from collections import OrderedDict
from mpi4py import MPI
import numpy as np
from numba import njit, literal_unroll
from numba.typed import  Dict

COMM = MPI.COMM_WORLD
SIZE = COMM.Get_size()
RANK = COMM.Get_rank()

__all__ = ["generate_mesh"]

class Cells:
    nodeid = []
    faceid = []
    center = []
    volume = []
    cellfid = []
    nf = [] #normal sortante des faces
    cellnid = []
    globalindex = OrderedDict()
    father = Dict()
    son = Dict()
    iadiv = Dict()

    def __init__(self, nodeid, faceid, center, volume, father, son, iadiv, globalindex, cellfid, cellnid,
                 nf):
        self.nodeid = nodeid    # instance variable unique to each instance
        self.faceid = faceid
        self.center = center
        self.volume = volume
        self.father = father
        self.son = son
        self.iadiv = iadiv
        self.globalindex = globalindex
        self.cellfid = cellfid
        self.cellnid = cellnid
        self.nf = nf

class Nodes:
    vertex = []
    name = []
    cellid = []
    globalindex = OrderedDict()
    halonid = []
    ghostcenter = []

    def __init__(self, vertex, name, cellid, globalindex, halonid, ghostcenter):
        self.vertex = vertex    # instance variable unique to each instance
        self.name = name
        self.cellid = cellid
        self.globalindex = globalindex
        self.halonid = halonid
        self.ghostcenter = ghostcenter

class Faces:
    nodeid = []
    cellid = []
    name = []
    halofid = []
    normal = []
    mesure = []
    bound = 0
    center = []
    ghostcenter = []

    def __init__(self, nodeid, cellid, name, normal, mesure, center, bound, halofid, ghostcenter):
        self.nodeid = nodeid    # instance variable unique to each instance
        self.cellid = cellid
        self.name = name
        self.normal = normal
        self.mesure = mesure
        self.bound = bound
        self.halofid = halofid
        self.center = center
        self.ghostcenter = ghostcenter

class Halo:
    halosint = []
    halosext = []
    neigh = []
    centvol = []
    faces = OrderedDict()
    nodes = OrderedDict()

    def __init__(self, halosint, halosext, centvol, neigh, faces, nodes):
        self.halosint = halosint    # instance variable unique to each instance
        self.halosext = halosext
        self.neigh = neigh
        self.centvol = centvol
        self.faces = faces
        self.nodes = nodes

@njit
def normalvector(node_a, node_b, bary):

    norm = np.zeros(2)#[None]*2
    snorm = np.zeros(2)#[None]*2
    center = np.zeros(2)
    normal = np.zeros(2)#[None]*2

    norm[0] = node_a[1] - node_b[1]
    norm[1] = node_b[0] - node_a[0]

    center[0] = 0.5 * (node_a[0] + node_b[0])
    center[1] = 0.5 * (node_a[1] + node_b[1])

    snorm[0] = bary[0] - center[0]
    snorm[1] = bary[1] - center[1]

    if (snorm[0] * norm[0] + snorm[1] * norm[1]) > 0:
        normal[0] = -1*norm[0]
        normal[1] = -1*norm[1]
    else:
        normal[0] = norm[0]
        normal[1] = norm[1]

    normal[0] = normal[0]
    normal[1] = normal[1]

    mesure = np.sqrt(normal[0]**2 + normal[1]**2)

    return normal, mesure

def create_local_mesh(file):

    clear_class(Cells, Nodes, Faces)
    #Lecture des cellules à partir du fichier mesh..txt
    #file = open("mesh"+str(RANK)+".txt","r")
    for line in file:
        #read elements
        if line == "elements\n":
            continue
        if line == "endelements\n":
            continue
        if line == "nodes\n":
            break
        Cells.nodeid.append([int(x) for x in line.split()])
    #Lecture des coordonnées des noeuds à partir du fichier mesh..txt
    for line in file:
        #read Nodes
        if line == "nodes\n":
            continue
        if line == "endnodes\n":
            continue
        if line == "halosint\n":
            break
        Nodes.vertex.append([np.double(x) for x in line.split()])
        Nodes.name = [0]*len(Nodes.vertex)

    for i in range(len(Nodes.vertex)):
        Nodes.name[i] = int(Nodes.vertex[i][3])

    #calcul du barycentre
    for i in range(len(Cells.nodeid)):
        s_1 = Cells.nodeid[i][0]
        s_2 = Cells.nodeid[i][1]
        s_3 = Cells.nodeid[i][2]

        x_1 = Nodes.vertex[s_1][0]
        y_1 = Nodes.vertex[s_1][1]
        x_2 = Nodes.vertex[s_2][0]
        y_2 = Nodes.vertex[s_2][1]
        x_3 = Nodes.vertex[s_3][0]
        y_3 = Nodes.vertex[s_3][1]

        Cells.center.append((1./3 * (x_1 + x_2 + x_3), 1./3*(y_1 + y_2 + y_3)))
        Cells.volume.append((1./2) * abs((x_1-x_2)*(y_1-y_3)-(x_1-x_3)*(y_1-y_2)))
        var1 = (x_2-x_1)*(y_3-y_1)-(y_2-y_1)*(x_3-x_1)
        if var1 < 0:
            Cells.nodeid[i][0] = s_1;   Cells.nodeid[i][1] = s_3; Cells.nodeid[i][2] = s_2

    tmp = [[] for i in range(len(Nodes.vertex))]
    longn = [0]*len(Nodes.vertex)
    for i in range(len(Cells.nodeid)):
        for j in range(3):
            tmp[Cells.nodeid[i][j]].append(i)
            longn[Cells.nodeid[i][j]] = longn[Cells.nodeid[i][j]] + 1

    longc = [0]*len(Cells.nodeid)
    tmp2 = [[] for i in range(len(Cells.nodeid))]
    for i in range(len(Cells.nodeid)):
        for j in range(3):
            for k in range(len(tmp[Cells.nodeid[i][j]])):
                if (tmp[Cells.nodeid[i][j]][k] not in tmp2[i] and  tmp[Cells.nodeid[i][j]][k] != i):
                    tmp2[i].append(tmp[Cells.nodeid[i][j]][k])
                    longc[i] = longc[i] + 1

    maxlongn = max(longn)
    Nodes.cellid = [[-1]*maxlongn for i in range(len(Nodes.vertex))]

    for i in range(len(tmp)):
        for j in range(len(tmp[i])):
            Nodes.cellid[i][j] = tmp[i][j]

    maxlongc = max(longc)
    Cells.cellnid = [[-1]*maxlongc for i in range(len(Cells.nodeid))]

    for i in range(len(tmp2)):
        for j in range(len(tmp2[i])):
            Cells.cellnid[i][j] = tmp2[i][j]

    #Création des faces
    cellule = Cells.nodeid
    cellf = []
    faces = []
    k = 0
    for i in range(len(cellule)):
        faces.append([cellule[i][0], cellule[i][1]])
        faces.append([cellule[i][1], cellule[i][2]])
        faces.append([cellule[i][2], cellule[i][0]])
        cellf.append([faces[k], faces[k+1], faces[k+2]])
        k = k+3
    for i in range(len(faces)):
        faces[i].sort()
    faces = set(tuple(x) for x in faces)
    faces = list(faces)

    facesdict = OrderedDict()
    for i in range(len(faces)):
        facesdict[faces[i]] = i
        Faces.nodeid.append(faces[i])
        Faces.cellid.append((-1, -1))

    #Création des 3 faces de chaque cellule
    for i in range(len(cellf)):
        Cells.faceid.append([facesdict.get(tuple(cellf[i][0])), facesdict.get(tuple(cellf[i][1])),
                             facesdict.get(tuple(cellf[i][2]))])

    #Faces.cellid = [(-1, -1)]*len(faces)
    for i in range(len(Cells.faceid)):
        for j in range(3):
            if Faces.cellid[Cells.faceid[i][j]] == (-1, -1):
                Faces.cellid[Cells.faceid[i][j]] = (i, -1)

            if Faces.cellid[Cells.faceid[i][j]][0] != i:
                Faces.cellid[Cells.faceid[i][j]] = (Faces.cellid[Cells.faceid[i][j]][0], i)

    Cells.cellfid = [[] for i in range(len(Cells.nodeid))]
    #Création des 3 triangles voisins par face
    for i in range(len(Cells.faceid)):
        for j in range(3):
            f = Cells.faceid[i][j]
            if Faces.cellid[f][1] != -1:
                if i == Faces.cellid[f][0]:
                    Cells.cellfid[i].append(Faces.cellid[f][1])
                else:
                    Cells.cellfid[i].append(Faces.cellid[f][0])
            else:
                Cells.cellfid[i].append(-1)

#    for i in range(len(faces)):
#        if Faces.cellid[i][1] != -1:
#            Cells.cellfid[Faces.cellid[i][1]].append(Faces.cellid[i][0])
#            Cells.cellfid[Faces.cellid[i][0]].append(Faces.cellid[i][1])
#
#    for i in range(len(Cells.cellfid)):
#        if len(Cells.cellfid[i]) == 2 :
#            Cells.cellfid[i].append(-1)

    #Faces aux bords (1,2,3,4), Faces à l'interieur 0    A VOIR !!!!!
    for i in range(len(faces)):
        Faces.name.append(0)
        if (Faces.cellid[i][1] == -1 and Faces.cellid[i][1] != -10):
            if Nodes.name[faces[i][0]] == Nodes.name[faces[i][1]]:
                Faces.name[i] = Nodes.name[faces[i][0]]
            if ((Nodes.name[faces[i][0]] == 3 and Nodes.name[faces[i][1]] != 0) or
                    (Nodes.name[faces[i][0]] != 0 and Nodes.name[faces[i][1]] == 3)):
                Faces.name[i] = 3
            if ((Nodes.name[faces[i][0]] == 4 and Nodes.name[faces[i][1]] != 0) or
                    (Nodes.name[faces[i][0]] != 0 and Nodes.name[faces[i][1]] == 4)):
                Faces.name[i] = 4
        normal, mesure = normalvector(np.asarray(Nodes.vertex[faces[i][0]]),
                                         np.asarray(Nodes.vertex[faces[i][1]]),
                                         Cells.center[Faces.cellid[i][0]])
        Faces.normal.append(normal)
        Faces.mesure.append(mesure)
        Faces.center.append([0.5 * (Nodes.vertex[faces[i][0]][0] + Nodes.vertex[faces[i][1]][0]),
                             0.5 * (Nodes.vertex[faces[i][0]][1] + Nodes.vertex[faces[i][1]][1])])
    for i in range(len(faces)):
        if Faces.name[i] != 0:
            Faces.bound += 1

    #compute the outgoing normal faces for each cell
    for i in range(len(Cells.nodeid)):
        ss = np.zeros((3, 2))

        G = np.asarray(Cells.center[i])

        for j in range(3):
            f = Cells.faceid[i][j]
            c = Faces.center[f]

            if np.dot(G-c, Faces.normal[f]) < 0.0:
                ss[j] = Faces.normal[f]
            else:
                ss[j] = -1.0*Faces.normal[f]

        Cells.nf.append(ss)

    Faces.ghostcenter = [[] for i in range(len(Faces.name))]
    Nodes.ghostcenter = [[] for i in range(len(Nodes.name))]

    #compute the ghostcenter for each face and each node
    for i in range(len(Faces.name)):
        nod1 = Faces.nodeid[i][1]
        nod2 = Faces.nodeid[i][0]
        if Faces.name[i] != 0 and Faces.name[i] != 10:

            x_1 = Nodes.vertex[nod1]
            x_2 = Nodes.vertex[nod2]

            c_left = Faces.cellid[i][0]
            v_1 = Cells.center[c_left]
            gamma = ((v_1[0] - x_2[0])*(x_1[0]-x_2[0]) + (v_1[1]-x_2[1])*(x_1[1]-x_2[1]))/((x_1[0]-x_2[0])**2 + (x_1[1]-x_2[1])**2)

            kk = np.array([gamma * x_1[0] + (1 - gamma) * x_2[0], gamma * x_1[1] + (1 - gamma) * x_2[1]])

            v_2 = np.array([2 * kk[0] + ( -1 * v_1[0]), 2 * kk[1] + ( -1 * v_1[1])])

            Faces.ghostcenter[i] = v_2
            Nodes.ghostcenter[nod1].append([v_2[0], v_2[1], i])
            Nodes.ghostcenter[nod2].append([v_2[0], v_2[1], i])

        else:
            Faces.ghostcenter[i] = [0,0]
            #Nodes.ghostcenter[nod1].append([0,0])
            #Nodes.ghostcenter[nod2].append([0,0])

    for i in range(len(Nodes.name)):
        if len(Nodes.ghostcenter[i]) == 0 :
            Nodes.ghostcenter[i].append([0,0,0])
            Nodes.ghostcenter[i].append([0,0,0])
        elif len(Nodes.ghostcenter[i]) == 1 :
            Nodes.ghostcenter[i].append([0,0,0])

def create_halo_structure(file):

    #txt_file = open(filename)
    for line in file:
        if "endhalosint" in line:
            break
        Halo.halosint.append(int(line))#.append(int(line))#for x in line.split()])

    for line in file:
        # process the line
        if "halosext" in line:
            continue
        if "centvol" in line:
            break
        Halo.halosext.append([int(x) for x in line.split()])

    for line in file:
        if "centvol" in line:
            continue
        if "globalcelltolocal" in line:
            break
        Halo.centvol.append([float(x) for x in line.split()])

    #Cells.globalindex = [-1]*len(Cells.nodeid)
    cmpt = 0
    for line in file:
    #read Global cell to local
        if line == "globalcelltomocal\n":
            continue
        if line == "endglobalcelltolocal\n":
            break
        Cells.globalindex[int(line)] = cmpt
        cmpt += 1

    cmpt = 0
    for line in file:
        #read Local Node To Global
        if line == "localnodetoglobal\n":
            continue
        if line == "endlocalnodetoglobal\n":
            break
        Nodes.globalindex[cmpt] = int(line)
        cmpt += 1
    for line in file:
        #read LocalToGlobal
        if line == "neigh\n":
            continue
        if line == "endneigh\n":
            break
        Halo.neigh.append([int(x) for x in line.split()])

    #halofaces = []
    Faces.halofid = np.zeros(len(Faces.nodeid), dtype=int)
    if SIZE > 1:
        k = 1
        for i in range(len(Halo.halosext)):
            Halo.faces[tuple([Halo.halosext[i][0], Halo.halosext[i][1]])] = k
            Halo.faces[tuple([Halo.halosext[i][1], Halo.halosext[i][2]])] = k + 1
            Halo.faces[tuple([Halo.halosext[i][0], Halo.halosext[i][2]])] = k + 2
            k = k+3

        for i in range(len(Faces.nodeid)):
            n1 = Nodes.globalindex[Faces.nodeid[i][0]]
            n2 = Nodes.globalindex[Faces.nodeid[i][1]]
            if  Halo.faces.get(tuple([n1, n2])):

                Faces.cellid[i] = (Faces.cellid[i][0], -10)
                Faces.name[i] = 10
                Nodes.name[Faces.nodeid[i][0]] = 10
                Nodes.name[Faces.nodeid[i][1]] = 10
                Faces.halofid[i] = int((-1+Halo.faces.get(tuple([n1, n2])))/3)

            if Halo.faces.get(tuple([n2, n1])):

                Faces.cellid[i] = (Faces.cellid[i][0], -10)
                Faces.name[i] = 10
                Nodes.name[Faces.nodeid[i][0]] = 10
                Nodes.name[Faces.nodeid[i][1]] = 10

                Faces.halofid[i] = int((-1+Halo.faces.get(tuple([n2, n1])))/3)

        longueur = 0
        tmp = [[] for i in range(len(Nodes.name))]
        for i in range(len(Nodes.name)):
            if Nodes.name[i] == 10:
                Halo.nodes[i] = Nodes.globalindex[i]
                arg = np.where(np.asarray(Halo.halosext) == Nodes.globalindex[i])
                tmp[i].append(arg[0])
                longueur = max(longueur, len(arg[0]))
            else:
                tmp[i].append(np.array([-1]))

        Nodes.halonid = [[-1]*longueur for i in range(len(Nodes.name))]

        for i in range(len(tmp)):
            for j in range(len(tmp[i][0])):
                Nodes.halonid[i][j] = tmp[i][0][j]

    cells = Cells(np.asarray(Cells.nodeid), np.asarray(Cells.faceid), np.asarray(Cells.center),
                  np.asarray(Cells.volume), Cells.father, Cells.son, Cells.iadiv, Cells.globalindex,
                  np.asarray(Cells.cellfid), np.asarray(Cells.cellnid), np.asarray(Cells.nf))

    nodes = Nodes(np.asarray(Nodes.vertex), np.asarray(Nodes.name), np.asarray(Nodes.cellid),
                  Nodes.globalindex, np.asarray(Nodes.halonid), np.asarray(Nodes.ghostcenter))

    faces = Faces(np.asarray(Faces.nodeid), np.asarray(Faces.cellid), np.asarray(Faces.name),
                  np.asarray(Faces.normal), np.asarray(Faces.mesure), np.asarray(Faces.center),
                  Faces.bound, np.asarray(Faces.halofid), np.asarray(Faces.ghostcenter))

    halos = Halo(np.asarray(Halo.halosint), np.asarray(Halo.halosext), np.asarray(Halo.centvol),
                 np.asarray(Halo.neigh), Halo.faces, Halo.nodes)

    return cells, nodes, faces, halos

@njit
def define_halosend(w_c, w_halosend, indsend):
    if SIZE > 1:
        for i in range(len(w_halosend)):
            w_halosend[i] = w_c[indsend[i]]

    return w_halosend

def all_to_all(w_halosend, taille, mystruct, variables, scount, sdepl, rcount, rdepl):

    w_halorecv = np.zeros(taille, dtype=mystruct)

    for var in literal_unroll(variables):
        s_msg = r_msg = 0
        s_msg = [np.array(w_halosend[var]), (scount, sdepl), MPI.DOUBLE_PRECISION]
        r_msg = [np.array(w_halorecv[var]), (rcount, rdepl), MPI.DOUBLE_PRECISION]

        COMM.Alltoallv(s_msg, r_msg)

        w_halorecv[var] = r_msg[0]

    return w_halorecv

@njit
def halo_value(w_halo, w_halorecv):

    for i in range(len(w_halo)):
        w_halo[i] = w_halorecv[i]

    return w_halo

def prepare_comm(cells, halos):

    scount = np.zeros(SIZE, dtype=int)
    sdepl = np.zeros(SIZE, dtype=int)
    rcount = np.zeros(SIZE, dtype=int)
    rdepl = np.zeros(SIZE, dtype=int)
    taille = 0
    indsend = 0

    if SIZE > 1:
        for i in range(len(halos.neigh[0])):
            scount[halos.neigh[0][i]] = halos.neigh[1][i]

        for i in range(SIZE):
            if i > 0:
                sdepl[i] = sdepl[i-1] + scount[i-1]

        rcount = COMM.alltoall(scount)

        for i in range(SIZE):
            if i > 0:
                rdepl[i] = rdepl[i-1] + rcount[i-1]

        for i in range(SIZE):
            taille += rcount[i]

        taille = int(taille)

        indsend = np.zeros(0, dtype=int)
        for i in range(len(halos.halosint)):
            indsend = np.append(indsend, cells.globalindex[Halo.halosint[i]])

#    indRecv = np.zeros(len(faces.cellid), dtype = int)
#    for i in range(len(faces.cellid)):
#        if faces.cellid[i][1] == -10:
#            indRecv[i] = int((-1+halos.faces.get(
#                    tuple([nodes.globalindex[Faces.nodeid[i][0]],
#                           nodes.globalindex[Faces.nodeid[i][1]]])))/3)

    return scount, sdepl, rcount, rdepl, taille, indsend

def generate_mesh():

    filename = 'mesh'+str(RANK)+'.txt'
    txt_file = open(filename)

    create_local_mesh(txt_file)

    cells, nodes, faces, halos = create_halo_structure(txt_file)

    grid = {}

    grid["cells"] = cells
    grid["nodes"] = nodes
    grid["faces"] = faces
    grid["halos"] = halos

    txt_file.close()

    return grid

def clear_class(cells, nodes, faces):

    cells.nodeid = []
    cells.volume = []
    cells.center = []
    cells.faceid = []
    cells.father = Dict()
    cells.son = Dict()
    cells.iadiv = Dict()
    cells.globalindex = OrderedDict()
    cells.cellfid = []
    cells.cellnid = []
    cells.nf = []

    faces.nodeid = []
    faces.name = []
    faces.cellid = []
    faces.normal = []
    faces.bound = 0
    faces.center = []
    faces.ghostcenter = []

    nodes.cellid = []
    nodes.name = []
    nodes.vertex = []
    nodes.globalindex = OrderedDict()
    nodes.halonid = []
    nodes.ghostcenter = []

max_stars_repo_name: pyccel/manapy

max_issues_repo_name: pyccel/manapy

max_forks_repo_name: pyccel/manapy

reason: import numpy,from numba


text:
# -*- coding: utf-8 -*-
#/usr/bin/python2
'''
By kyubyong park. kbpark.linguist@gmail.com.
https://www.github.com/kyubyong/tacotron_asr
'''

from __future__ import print_function

import os

import librosa
from tqdm import tqdm

from data import load_vocab, load_train_data
from data_load import get_batch
from hyperparams import Hyperparams as hp
from modules import *
from networks import encode, decode
import numpy as np
import tensorflow as tf
from utils import shift_by_one

char2idx, idx2char = load_vocab()

class Graph:
    def __init__(self, is_training=True):
        self.graph = tf.Graph()
        self.is_training=is_training
        with self.graph.as_default():
            if is_training:
                self.x, self.y, self.num_batch = get_batch()
            else: # Evaluation
                self.x = tf.placeholder(tf.float32, shape=(None, None, hp.n_mels*hp.r))
                self.y = tf.placeholder(tf.int32, shape=(None, hp.max_len))

            self.decoder_inputs = embed(shift_by_one(self.y), len(char2idx), hp.embed_size) # (N, T', E)

            with tf.variable_scope('net'):
                # Encoder
                self.memory = encode(self.x, is_training=is_training) # (N, T, hp.n_mels*hp.r)

                # Decoder
                self.outputs = decode(self.decoder_inputs, self.memory, is_training=is_training) # (N, T', E)
                self.logprobs = tf.log(tf.nn.softmax(self.outputs)+1e-10)
                self.preds = tf.arg_max(self.outputs, dimension=-1)

            if is_training:
                # Loss
                self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.y, logits=self.outputs)

                # Target masking
                self.istarget = tf.to_float(tf.not_equal(self.y, 0))
                self.mean_loss = tf.reduce_sum(self.loss*self.istarget) / (tf.reduce_sum(self.istarget) + 1e-7)

                # Training Scheme
                self.global_step = tf.Variable(0, name='global_step', trainable=False)
                self.optimizer = tf.train.AdamOptimizer(learning_rate=hp.lr)
                self.train_op = self.optimizer.minimize(self.mean_loss, global_step=self.global_step)

                # Summary
                tf.summary.scalar('mean_loss', self.mean_loss)
                self.merged = tf.summary.merge_all()

def main():
    g = Graph(); print("Training Graph loaded")

    with g.graph.as_default():
        # Training
        sv = tf.train.Supervisor(logdir=hp.logdir,
                                 save_model_secs=0)
        with sv.managed_session() as sess:
            for epoch in range(1, hp.num_epochs+1):
                if sv.should_stop(): break
                for step in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'):
                    sess.run(g.train_op)

                # Write checkpoint files at every epoch
                l, gs = sess.run([g.mean_loss, g.global_step])
                sv.saver.save(sess, hp.logdir + '/model_gs_%d_%.2f' % (gs, l))

if __name__ == '__main__':
    main()
    print("Done")


text:
# -*- coding: utf-8 -*-
"""
Dividend Discount Model(s)

Series of functions for solving dividend discount model problems. Primarily

- dividend_discount_model_irr() is a sovler for given px, divident stream & G
- multi_stage_irr() is a 3-stage model which estimates g = ROE * (1-PO)

"""

# %% REQUIRED PACKAGES

import pandas as pd
import numpy as np
from scipy.optimize import minimize_scalar

# %% Class Module Test

class dividend_discount_models(object):
    """

    """

    def __init__(self, **kwargs):

        # Test thing
        self.test = "badgers"

        # Kwargs & General Settings
        self.data = kwargs['data'] if 'data' in kwargs else None
        self.data_gdp = kwargs['data_gdp'] if 'data_gdp' in kwargs else None
        self.eom = kwargs['eom'] if 'eom' in kwargs else True

        # Do some data scrubbing if requested
        # Options for a few different types
        if 'scrub' in kwargs:

            # data_dict_fields is where data has multi df's for each field
            # minor statistical adjustments
            if kwargs['scrub'] == 'data_dict_fields':
                self._scrub_data_dict_fields()

        return

    # %% Admin & Data Srubbing

    # Collect Equity Metrics 2 Single Dataframe
    def data_from_ticker(self, ticker, dd=None, eom=True):
        """ Collect Equity Metrics (PX, PE etc...) 2 single DF

        requires input dictionary of form {'PX':pe_dataframe}
        where:
            pe_dataframe needs the ticker in the column varnames

        if eom == True all dates will be set to End-of-Month
        """

        # pull from internal if data dictionary not supplied
        dd = self.data if dd == None else dd

        # iterate through each key in dict of field-dataframes
        df = pd.DataFrame(columns=dd.keys())     # Dummy Dataframe
        for i, key in enumerate(dd.keys()):

            ix = dd[key][ticker]
            if eom:
                ix.index = ix.index + pd.tseries.offsets.MonthEnd(0)
            df[key] = ix    # populate dataframe

        self.ticker_data = df
        return df

    # Terminal Growth Trends
    def terminal_gdp_per_capita(self, gdp=None, w=10, smoothing=3,
                                pull2us=0, pull2par=0, par=0.02):
        """ Calculate Terminal Growth Rates from GDP Per Capita """

        # use class attribute if available (and None specified)
        gdp = gdp if isinstance(gdp, pd.DataFrame) else self.data_gdp

        # Calculate geo mean of Real GDP per capita & smooth if req.
        x = (1 + gdp.pct_change(1)).rolling(w).apply(np.prod).apply(lambda x: np.power(x, 1/w))-1
        x = x.rolling(smoothing).mean()  # smoothing if required

        # Form quick EM Composite
        wgtCNY = 0.4    # China Weight
        x['EM'] = (1-wgtCNY)*x.loc[:,['BRL','RUB','INR','MXN','ZAR']].mean(axis=1) + x['CNY']*wgtCNY

        # Pull-to-par if required
        if pull2par > 0:
            for c in x:
                x.loc[:,c] = (x.loc[:, c]*(1-pull2par)) + ((pull2par * par))

        # pull-to-US if specified
        if pull2us > 0:
            for c in x:
                x.loc[:,c] = (x.loc[:, c]*(1-pull2us)) + ((pull2us * x.loc[:,'USD']))

        self.terminal_G = x
        return x

    def _scrub_data_dict_fields(self, lim_dy=[0, 100], lim_roe=0, winsorize_lim=[0.05, 0.05]):
        """ Helper function for MSCI Equity Field (ROE, DY etc..) data

        Function uses self.data when a dict and makes a few minor adjustments,
        such as winsorising, no negative dividends etc...

        NB/ May need multiple data scrubbing functions """

        from scipy.stats.mstats import winsorize

        # ensure all data are floats
        # Replace None with np.nan & set all data to floats (maybe more later)
        for k in self.data.keys():
            self.data[k] = self.data[k].replace('None', np.nan).astype(float)

        # Limit Dividend Yields to 0-100%
        DY = self.data['DY']
        DY[DY < lim_dy[0]] = lim_dy[0]
        DY[DY > lim_dy[1]] = lim_dy[1]
        self.data['DY'] = DY

        # Floor ROE at 0
        ROE = self.data['ROE']
        ROE[ROE < lim_roe] = lim_roe
        self.data['ROE'] = ROE

        # Winsorize if available
        for k in ['ROE', 'PE', 'FwdPE']:
            # first check if field is a key in the data dict
            if k in self.data.keys():

                # then iterate over columns winsorising (doesn't work in 2D)
                x = self.data[k]
                for t in x:
                    x[t] = winsorize(x[t], winsorize_lim)
                self.data[k] = x

        # Interpolate Missing Data
        for k in self.data.keys():
            self.data[k] = self.data[k].interpolate(method='linear', limit=3,
                                                    axis=0, limit_area='inside')

        return self.data

    # %% DDM Solver Functions - GIVEN DIVIDEND STREAM
    # Build a dividend stream using another function - solve it here

    def ddm_irr(self, px, vDividend, G):
        """ IRR finder for Dividend Discount Model in SciPy

        Uses the scipy.optimise minimize_scalar optimisation function
            minimize_scalar(f, args=(d, px, G)).x

        Where d - vector of dividend stream from d0 to dTerminal
              px - current index level associated with dividend stream
              G = perpetual growth rate
        """
        x = minimize_scalar(self._solver_ddm_irr, args=(px, vDividend, G)).x
        return x

    def _solver_ddm_irr(self, x, px, d, G):
        """ solver function for dividend_discount_model_irr"""
        pv = 0     # Set present value of div stream to 0 to t0
        for i, v in enumerate(d):

            if i == 0:              # skip d0 which isn't included in DDM calc
                continue
            elif i < len(d) - 1:    # len(d)-1 == terminal year
                pv += v / ((1 + x) ** i)
            else:
                pv += (v / ((1 + x) ** i)) * ( 1 / (x - G))

        return np.abs(px - pv)    # minimise in optimiser

    # %% Sustainable Return Model

    # Multi-Ticker Latest
    def sustainable_rtn_multi(self, ticker_list=['MXUS'],
                              roe_trend=120, po_trend=120,
                              trend_start=10, terminal=21,
                              fwd_pe = False):

        """ Table of Most Recent Observations for MULTI tickers

        NB/ This is NOT an efficient method for doing this FURTHER WORK """

        ### MOVE THIS ELSEWHERE
        mapping = dict(MXWO = {'toto':'World', 'fx':'USD'},
                       MXEF = {'toto':'World', 'fx':'EM'},
                       MXUS = {'toto':'United States', 'fx':'USD'},
                       MXEA = {'toto':'', 'fx':'USD'},
                       MXGB = {'toto':'United Kingdom', 'fx':'GBP'},
                       MXEUG = {'toto':'Europe', 'fx':'EUR'},
                       MXEU = {'toto':'Europe', 'fx':'EUR'},
                       MXZA = {'toto':'South Africa', 'fx':'ZAR'},
                       MXJP = {'toto':'Japan', 'fx':'JPY'},
                       SPX = {'toto':'United States', 'fx':'USD'},
                       UKX = {'toto':'United Kingdom', 'fx':'GBP'},
                       JALSH = {'toto':'South Africa', 'fx':'ZAR'},
                       SXXP = {'toto':'Europe', 'fx':'EUR'},
                       TPX = {'toto':'Japan', 'fx':'JPY'},)

        # Dummy Dataframes
        ExRtn, ERP = [pd.DataFrame(columns=ticker_list)] * 2
        vn = ['PX','ExRtn','DY','PE','ROE','PO','g','G','ROE_trend','PO_trend']
        t0 = pd.DataFrame(index=ticker_list, columns=vn)

        # iterate through each ticker
        for t in ticker_list:

            fx = mapping[t]['fx']
            df = self.data_from_ticker(t)
            ddm, _ = self.sustainable_rtn_ts(df=df, G_fx=fx,
                                             roe_trend=roe_trend,
                                             po_trend=po_trend,
                                             trend_start=trend_start,
                                             terminal=terminal,
                                             fwd_pe = False)

            #populate ts ExRtns & ERP as well as t0 current input data
            ExRtn.loc[:, t] = ddm['ExRtn']
            vn = ['PX','ExRtn','DY','PE','ROE','PO','g','G','ROE_trend','PO_trend']
            t0.loc[t, vn] = ddm.loc[ddm.index[-1], vn]

        # Save to self for useful storage
        self.output_table = t0
        self.output_ExRtn = ExRtn

        return t0, ExRtn

    # Full timeseries for SINGLE Ticker
    def sustainable_rtn_ts(self, df=None, G_fx='USD',
                           roe_trend=120, po_trend=120,
                           trend_start=10, terminal=21,
                           fwd_pe = False,):

        """ Timeseries of Sustainable Returns for Single Ticker

        Sustainable Return model using current PX, ROE & Payout-Ratios. Where
        we estimate g = ROE * (1-PO). Currently 3 steps:
            1. converge to trend
            2. time in trend
            3. terminal growth rate

        INPUTS:
            df = dataframe for 1 name, with fields as column names
            G_fx = given as FX code. REQUIRES terminal_G df within class
            trend (in months) for ROE & Payout Ratios
            trend & terminal years.

        """

        # determine input dataframe to use - order of if's important here
        if type(df) == str:
            # if a string provided assumes its a ticker & build from scratch
            # will require self.data dictionary to have been populated!!!
            df = self.data_from_ticker(df)
        elif isinstance(df, pd.DataFrame):
            df = df   # THIS MAKES NO SENSE (HOW TO KILL THIS LINE?)
        else:
            # class attribute - comes 3rd or will fail if string input (above)
            df = self.ticker_data             # class attribute

        x = df.copy()    # avoid editing original data

        x['E0'] = x.PX / x.PE        # PE implied Earnings (pre-Inflation adj)

        # Dividend Stuff
        x['DY'] = x.DY / 100      # convert to %
        x['D0'] = x.PX * x.DY        # Dividned Stream (POST inflation adj)
        x['PO'] = x.DY / (1/x.PE)    # PO = DY / EY
        x['PO_trend'] = x.PO.rolling(po_trend).median()

        # Include forward earnings
        if fwd_pe == True:
            x['E1'] = x.PX / x.FwdPE       # implied earnings next year
            x['POi'] = x.DY / (1/x.FwdPE)  # implied PO at t1

        # Growth Stuff
        x.ROE = x.ROE / 100    # convert to %
        x['ROE_trend'] = x.ROE.rolling(roe_trend).median()  # find rolling average ROE
        x['g'] = x.ROE * (1 - x.PO)                         # EPS growth from g = EPS * (1 - PO)
        x['g_trend'] = x.ROE_trend * (1 - x.PO_trend)       # same but finding trend growth

        # Terminal Growth Rate
        # class terminal_G table used (see admin) & "FX" code to be used
        G = self.terminal_G[G_fx]
        idx = x.index.intersection(G.index)    # find index of matching dates
        x.loc[idx, 'G'] = G[idx]               # populate Real G from table
        x['G'] = x['G'].ffill()                # ffill annual data

        ### Iterate through point in time calculate IRR
        # remeber the point of the above is so that we have all the data we need for the calc in one row
        x.dropna(how='any', inplace=True)   # bin rows with missing data

        for i, v in enumerate(x.index):

            # Run DDM code
            ddm, res = self.sustainable_return_calc(x.iloc[i,:],
                                                    trend_start=trend_start,
                                                    terminal=terminal)
            x.loc[v, 'ExRtn'] = res        # populate table

        # x is the summary table; ddm is the actual dividend calculation
        self.output_table = x
        self.output_divstream = ddm

        return x, ddm

    def sustainable_return_calc(self, dv,
                                trend_start=10,
                                terminal=21, **kwargs):

        """ Multi-Stage Dividend Discount Model solving for IRR

        Assumtion is 3 phases:
            1. Convergence to trend
            2. Time in trend
            3. Perpetual growth

        growth with growth estimated as g = ROE * (1 - Payout_Ratio)

        INPUTS:
            dv = data-vector which can either be a list or a pd.Series; must contain:
                 ['PX', 'D0', 'ROE', 'PO', 'ROE_trend', 'PO_trend', 'G']
            trend_start = int of year (default=10) entering trend growth
            terminal = int or year (default=21) entering perpetual growth phase

        """
        if isinstance(dv, list):
            vn = ['PX', 'D0', 'ROE', 'PO', 'ROE_trend', 'PO_trend', 'G']
            dv = pd.Series(data=dv, index=vn)#.T    # Transpose as lists form column df

        # Set up output Dataframe, with the Index representing years
        ddm = pd.DataFrame(index=range(terminal+1), columns=['D','ROE','PO','g','PX'])

        # T0
        ddm.loc[0,['D', 'ROE', 'PO', 'PX']] = dv.D0, dv.ROE, dv.PO, dv.PX

        # Phase 1 - converge to trend
        ddm.loc[0:trend_start, 'ROE'] = np.linspace(ddm.loc[0,'ROE'], dv.ROE_trend, trend_start+1).ravel()
        ddm.loc[0:trend_start, 'PO'] = np.linspace(ddm.loc[0,'PO'], dv.PO_trend, trend_start+1).ravel()
        # Phase 2 - time in trend
        ddm.loc[trend_start:, 'ROE'] = dv.ROE_trend
        ddm.loc[trend_start:, 'PO'] = dv.PO_trend

        # implied g, Terminal G & update dividends
        ddm['g'] = ddm.ROE * (1 - ddm.PO)
        for i, v in enumerate(ddm.index[:-1]):
            ddm.loc[i+1, 'D'] = ddm.loc[i, 'D'] * (1 + ddm.loc[i, 'g'])

        ddm.loc[ddm.index[-1], 'g'] = dv['G']    # set Terminal Growth Rate in table

        r = self.ddm_irr(dv.PX, ddm.D, dv.G)

        # Show present value in table
        for i, v in enumerate(ddm.index):
            if i == 0:
                continue
            elif v < terminal:
                ddm.loc[v,'PX'] = -1 * ddm.loc[v,'D']/((1+r)**i)
            else:
                ddm.loc[v,'PX'] = (ddm.loc[v,'D'] /((1+r)**i)) * -1 * (1 / (r - ddm.g.iloc[-1]))

        return ddm, r

 # %% TESTING

def _sustainable_equity_return_testing():

    # import test data from Excel
    import xlwings as xlw

    wb = xlw.Book('Expected Returns.xlsm')
    pxlw = lambda a, b: wb.sheets[a].range(b).options(pd.DataFrame, expand='table').value
    px = pxlw('PX', 'D1').iloc[3:,:]
    pe = pxlw('PE', 'D1').iloc[3:,:]
    dy = pxlw('DY', 'D1').iloc[3:,:]
    roe = pxlw('ROE', 'D1').iloc[3:,:]
    fwdpe = pxlw('MX_FwdPE', 'D1').iloc[3:,:]
    tr = pxlw('TR', 'D1').iloc[3:,:]
    gdp_real = pxlw('GDP_RealPerCap', 'D1').iloc[3:,:]
    gdp_nom = pxlw('GDP_NomPerCap', 'D1').iloc[3:,:]

    # setup DDM instance
    dd = {'PX': px, 'DY': dy, 'PE': pe, 'ROE': roe, 'TR':tr, 'FwdPE':fwdpe}
    ddm = dividend_discount_models(data=dd) # scrub='data_dict_fields'

    # Test Area
    ddm.terminal_gdp_per_capita(gdp=gdp_nom, w=10, smoothing=1)
    ddm._scrub_data_dict_fields()
    x, y = ddm.sustainable_rtn_ts('MXZA', 'ZAR')

    return x, y

# x, y = _sustainable_equity_return_testing()

max_stars_repo_name: muntumdwara/TopGun

max_issues_repo_name: muntumdwara/TopGun

max_forks_repo_name: muntumdwara/TopGun

reason: import numpy,from scipy


text:
# ==============================================================================
# Copyright (c) 2018, Yamagishi Laboratory, National Institute of Informatics
# Author: Yusuke Yasuda (yasuda@nii.ac.jp)
# All rights reserved.
# ==============================================================================
"""
Preprocess VCTK dataset
usage: preprocess_vctk.py [options] <in_dir> <out_dir> <data> <version> <num_codes>

options:
    --hparams=<parmas>                  Ad-hoc replacement of hyper parameters. [default: ].
    --version=<version>                 Version number of VCTK.
    --hparam-json-file=<path>           JSON file contains hyper parameters.
    --source-only                       Process source only.
    --target-only                       Process target only.
    -h, --help                          Show help message.

"""

import csv
import os, sys
import numpy as np
import json
#from pyspark import SparkContext
from docopt import docopt
from hparams import hparams, hparams_debug_string

if __name__ == "__main__":
    args = docopt(__doc__)
    in_dir = args["<in_dir>"]
    out_dir = args["<out_dir>"]
    data = args["<data>"]
    version = args["<version>"]
    num_codes = args["<num_codes>"]
    source_only = args["--source-only"]
    target_only = args["--target-only"]

    if args["--hparam-json-file"]:
        with open(args["--hparam-json-file"]) as f:
            hparams_json = "".join(f.readlines())
            hparams.parse_json(hparams_json)

    hparams.parse(args["--hparams"])
    print(hparams_debug_string())

    if source_only:
        process_source = True
        process_target = False
    elif target_only:
        process_source = False
        process_target = True
    else:
        process_source = True
        process_target = True

    if data == "vctk":
        from preprocess.codes import CODES
        instance = CODES(in_dir, out_dir, version, num_codes, hparams)
    if data == "siwis":
        from preprocess.siwis_codes import CODES
        instance = CODES(in_dir, out_dir, version, num_codes, hparams)

#    sc = SparkContext()

    record_rdd = instance.list_files()

    if process_source:
        keys = instance.process_sources(record_rdd)

    if process_target:
        keys = instance.process_targets(record_rdd)

    with open(os.path.join(out_dir, 'list.csv'), 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        for path in keys:
            writer.writerow([path])

max_stars_repo_name: rhoposit/self-attention-tacotron

max_issues_repo_name: rhoposit/self-attention-tacotron

max_forks_repo_name: rhoposit/self-attention-tacotron


text:
"""
Copyright © 2021 Wallbreaker5th

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""

from typing import Tuple
from manimlib import *
import numpy as np
import time
import numpy.fft as nf

timestamp = 0
DEFAULT_STROKE_WIDTH = 2

def get_rand():
    return 1+np.random.random()

def calc(p: np.ndarray, q: np.ndarray, r: float) -> Tuple[np.ndarray, np.ndarray]:
    time0 = time.time()
    dis = np.linalg.norm(p-q)
    if r < (dis/2)*(1-1e-8):
        # assert(0)
        pass
    h = np.sqrt(max(r**2-(dis/2)**2, 0))
    v = (q-p)/dis*h
    v = np.array([v[1], -v[0], 0])
    mid = (p+q)/2
    p1, p2 = mid+v, mid-v
    return (p1, p2)

def reflect(p: np.ndarray, a: np.ndarray, b: np.ndarray) -> np.ndarray:
    l = np.linalg.norm(a-b)-np.dot(p-a, b-a)/np.linalg.norm(b-a)*2
    return p+l*(b-a)/np.linalg.norm(b-a)

def set_line(mob, p, q):
    mob.data["points"][0] = p
    mob.data["points"][1] = q
    mob.data["points"][2] = q
    # mob.data["points"]=np.array([p, midpoint(p, q), q])

class Fold(VMobject):
    opacity = -1
    color = -1
    stroke_with = -1

    def set_color(self, color, recurse=False, escape=None):
        global timestamp
        self.color = timestamp
        for i in self.submobjects:
            if i.color != timestamp:
                if (recurse or not isinstance(i, Fold)) and (not escape or not i in escape):
                    if isinstance(i, Fold):
                        i.set_color(color, recurse, escape)
                    else:
                        i.set_color(color, recurse)
        return self

    def set_opacity(self, opacity, recurse=False, escape=None):
        global timestamp
        self.opacity = timestamp
        self.opacity_val = opacity
        self.data["opacity"] = opacity
        for i in self.submobjects:
            if i.opacity != timestamp:
                if (recurse or not isinstance(i, Fold)) and (not escape or not i in escape):
                    if isinstance(i, Fold):
                        i.set_opacity(opacity, recurse, escape)
                    else:
                        i.set_opacity(opacity, recurse)
        return self

    def set_stroke_width(self, width, recurse=False, escape=None):
        global timestamp
        self.stroke_width = timestamp
        for i in self.submobjects:
            if i.stroke_width != timestamp:
                if (recurse or not isinstance(i, Fold)) and (not escape or not i in escape):
                    if isinstance(i, Fold):
                        i.set_stroke_width(width, recurse, escape)
                    else:
                        i.set_stroke(width=width)

class Funiop(Fold):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.timestamp = -1

    def is_newest(self):
        global timestamp, qaq
        return self.timestamp == timestamp

    def get_output(self):
        return self.output

    def get_input(self):
        global timestamp
        if isinstance(self.pre, Fold) and not self.pre.is_newest():
            self.pre.fupdate()
            self.pre.timestamp = timestamp
        self.input = self.pre.get_output() if isinstance(self.pre, Fold) else self.pre()
        return self.input

    def set_self_color(self, color):
        self.set_color(color, recurse=True, escape={self.pre})
        return self

    def set_self_opacity(self, opacity):
        self.set_opacity(opacity, recurse=True, escape={self.pre})
        return self

    def set_self_stroke_width(self, stroke_width):
        self.set_stroke_width(stroke_width, recurse=True, escape={self.pre})
        return self

class Frotate(Funiop):
    def fupdate(self):
        t = self.get_input()*self.rate
        pX = np.array([0, 0, 0])
        r = self.r
        pO = np.array([r*np.cos(t), r*np.sin(t), 0])
        self.output = pO
        set_line(self.submobjects[1], pX, pO)

    def __init__(self, r, rate, pre, **kwargs):
        super().__init__(**kwargs)
        self.r = r
        self.rate = rate
        self.pre = pre

        self.add(SmallDot(**kwargs))
        self.add(Line(**kwargs))

        self.fupdate()

class Frotate_2(Funiop):
    def fupdate(self):
        self.submobjects[0].fupdate()
        self.submobjects[1].fupdate()
        set_line(self.submobjects[2], self.submobjects[0].output,
                 self.submobjects[1].output)
        self.output = self.submobjects[1].output

    def __init__(self, v0, r, rate, pre, **kwargs):
        super().__init__(**kwargs)
        self.r = abs(v0)
        self.rate = rate
        self.pre = pre

        self.add(Frotate(r, rate, pre, **kwargs))
        self.add(Frotate(self.r, 1, lambda: pre()*rate +
                 np.angle(v0), **kwargs).set_color(RED))
        self.add(Line(**kwargs).set_color(YELLOW))

        self.fupdate()

class Fmulti(Funiop):
    def fupdate(self):
        t = self.get_input()
        pA = np.array([self.r1*np.cos(t*self.rate1),
                      self.r1*np.sin(t*self.rate1), 0])
        pB = np.array([self.r2*np.cos(t*self.rate2),
                      self.r2*np.sin(t*self.rate2), 0])
        pC = np.array([self.r3*np.cos(t*self.rate3),
                      self.r3*np.sin(t*self.rate3), 0])
        pO = np.array([0, 0, 0])

        pD = reflect(pO, pA, pB)
        pE = reflect(pO, pB, pC)
        set_line(self[0], pA, pD)
        set_line(self[1], pB, pD)
        set_line(self[2], pC, pE)

    def __init__(self, r1, r2, r3, rate1, rate2, rate3, pre, **kwargs):
        super().__init__(**kwargs)
        self.r1 = r1
        self.r2 = r2
        self.r3 = r3
        self.rate1 = rate1
        self.rate2 = rate2
        self.rate3 = rate3
        self.pre = pre

        self.add(Line(**kwargs))
        self.add(Line(**kwargs))
        self.add(Line(**kwargs))

        self.fupdate()

class Fbinop(Fold):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.timestamp = -1

    def is_newest(self):
        global timestamp, qaq
        return self.timestamp == timestamp

    def get_output(self):
        return self.output

    def get_input(self):
        global timestamp
        if isinstance(self.in1, Fold) and not self.in1.is_newest():
            self.in1.fupdate()
            self.in1.timestamp = timestamp
        if isinstance(self.in2, Fold) and not self.in2.is_newest():
            self.in2.fupdate()
            self.in2.timestamp = timestamp
        self.input = (self.in1.get_output() if isinstance(self.in1, Fold) else self.in1(),
                      self.in2.get_output() if isinstance(self.in2, Fold) else self.in2())
        return self.input

    def set_self_color(self, color):
        self.set_color(color, recurse=True, escape={self.in1, self.in2})
        return self

    def set_self_opacity(self, opacity):
        self.set_opacity(opacity, recurse=True, escape={self.in1, self.in2})
        return self

    def set_self_stroke_width(self, stroke_width):
        self.set_stroke_width(stroke_width, recurse=True,
                              escape={self.in1, self.in2})
        return self

def pluser_updater(self, pI1, pI2, r1=0, r2=0):
    pX = np.array([0, 0, 0])
    pO = np.array(pI1+pI2)
    pA = calc(pX, pI1, self.a)[r1]
    pB = calc(pX, pI2, self.a)[r2]
    pC = pA+pB
    pD = pB+pI1
    pE = pI2+pA
    set_line(self.submobjects[1], pX, pB)
    set_line(self.submobjects[2], pB, pI2)
    set_line(self.submobjects[3], pA, pC)
    set_line(self.submobjects[4], pC, pE)
    set_line(self.submobjects[5], pI1, pD)
    set_line(self.submobjects[6], pD, pO)
    set_line(self.submobjects[7], pX, pA)
    set_line(self.submobjects[8], pA, pI1)
    set_line(self.submobjects[9], pB, pC)
    set_line(self.submobjects[10], pC, pD)
    set_line(self.submobjects[11], pI2, pE)
    set_line(self.submobjects[12], pE, pO)

class Fplus(Fbinop):
    def fupdate(self):
        pX = np.array([0, 0, 0])
        I1, I2 = self.get_input()
        self.output = I1+I2
        pluser_updater(self, I1, I2, 0, 0)

    def __init__(self, a, in1, in2, **kwargs):
        super().__init__(**kwargs)
        self.a = a
        self.in1 = in1
        self.in2 = in2

        self.add(SmallDot(**kwargs))  # X
        for i in range(12):
            self.add(Line(**kwargs))

        if isinstance(in1, Fold):
            self.add(in1)
            self.in1 = self.submobjects[-1]
        if isinstance(in2, Fold):
            self.add(in2)
            self.in2 = self.submobjects[-1]

        self.fupdate()

class Ffourier(Funiop):
    def fupdate(self):
        for i in self.submobjects:
            i.fupdate()
        self.output = self.res.get_output()

    def __init__(self, fft, r0, k, pre, **kwargs):
        super().__init__(**kwargs)
        self.fft = fft
        self.n = fft.shape[0]//2
        n = self.n
        for i in range(-n, n):
            self.add(Frotate_2(fft[i+n], r0*k**i, i, pre, **kwargs).set_color(
                interpolate_color(WHITE, WHITE, (i+n)/(n*2)), recurse=True
            ))
        for i in range(-n, n-2):
            self.add(Fmulti(r0*k**(i+0), r0*k**(i+1), r0*k **
                     (i+2), i+0, i+1, i+2, pre, **kwargs).set_color(
                interpolate_color(ORANGE, ORANGE, (i+n)/(n*2))
            ))
        s = abs(fft[0])
        for i in range(-n, n-1):
            s += abs(fft[i+n+1])
            self.add(Fplus(s+1, self[i+n if i == -n else -1], self[i+n+1], **kwargs).set_color(
                interpolate_color('#0080FF', '#8080FF', (i+n)%4/3)
            ).set_opacity(0.3))

        self.res = self.submobjects[-1]
        self.submobjects = self.submobjects[n*4-2:]+self.submobjects[:n*4-2]

        self.fupdate()

class Fscene(Scene):
    def construct(self):
        global timestamp
        time_total = 10
        speed = 1
        self.camera.frame.scale(0.5)

        N = 4
        svg = SVGMobject(file_name="tri.svg")[0].set_height(2).move_to(ORIGIN)
        x = np.array([svg.point_from_proportion(i/N)[0] for i in range(N)])
        y = np.array([svg.point_from_proportion(i/N)[1] for i in range(N)])
        a = np.array([i+1j*j for i, j in zip(x, y)])
        a = a/np.array([np.exp(i*(-N//2)*np.pi*2j/N) for i in range(N)])
        fa = nf.fft(a)/N

        fres = Ffourier(fa, 1, 0.8, lambda: timestamp, stroke_width=2)
        self.add(fres)
        print(sum(isinstance(i, Line) for i in fres.family))

        path = VMobject(stroke_width=4)
        path.set_points([fres.get_output()])

        pen = SmallDot(color='#0080ff', radius=0.02)
        self.add(pen)

        def path_upd(mob):
            return mob.add_line_to(fres.get_output())
        self.add(path)

        def pen_upd(mob):
            return mob.move_to(fres.get_output())
        self.add(pen)

        def time_updater(mob, dt):
            global timestamp
            timestamp += dt/time_total*TAU*speed
            mob.fupdate()

        fres.add_updater(time_updater)
        path.add_updater(path_upd)
        pen.add_updater(pen_upd)

        self.wait(time_total)

        path.remove_updater(path_upd)
        fres.remove_updater(time_updater)
        pen.remove_updater(pen_upd)

        self.wait(3)


text:
# 124
#
# https://adventofcode.com/2016/day/13

import sys
import pprint
import re
import string
import numpy as np

# test
inputKey = 10 # test
dest = (4, 7) # (row, col)

# real
inputKey = 1350
dest = (39, 31) # (row, col)

def isWall(x, y, key):
    #Find x*x + 3*x + 2*x*y + y + y*y.
    calc = (x * x) + (3 * x) + (2 * x * y) + y + (y * y)
    #Add the office designer's favorite number (your puzzle input).
    calc += key
    #Find the binary representation of that sum; count the number of bits that are 1.
    binary = "{0:b}".format(calc)
    #If the number of bits that are 1 is even, it's an open space.
    #If the number of bits that are 1 is odd, it's a wall.`
    ones = binary.count('1')
    return (ones % 2) != 0

size = 100
array = np.zeros((size,size), dtype=np.int8)
for row in range(size):
    for col in range(size):
        array[row, col] = 1 if isWall(col, row, inputKey) else 0

print array[0:10, 0:10]
print array[dest[0], dest[1]]
visited = set()
frontier = set()
frontier.add((1,1))

neighbors = [(0, 1), (-1, 0), (1, 0), (0, -1)]
count = 0
found = False
while not found:
    # remove items from the frontier
    if dest in frontier:
        found = True
        break

    newFrontier = set()
    for row, col in frontier:
        # find all the places you can go from there and add them to the new frontier
        for rowOff, colOff in neighbors:
            newRow, newCol = row + rowOff, col + colOff
            print newRow, newCol
            if (newRow >= 0) and (newCol >= 0) and (array[newRow, newCol] == 0):
                if (not (newRow, newCol) in visited) and (not (newRow, newCol) in newFrontier):
                    newFrontier.add((newRow, newCol))
        visited.add((row, col))

    #print 'count:', count
    #print 'frontier:', frontier
    #print 'new frontier:', newFrontier
    #print 'visited:', visited

    if dest in frontier:
        found = True
    else:
        frontier = newFrontier

    count += 1
    if count == 50:
        for rowF, colF in newFrontier:
            visited.add((rowF, colF))
        print len(visited)
        exit()