text:
from util.integration_models import Simulator, sigmoid

from scipy.stats import norm
import numpy as np

class BiasedWeightedSum(Simulator):
    """
    My naming here has been a bit poor. Angularly, a weighted sum is equivalent
    to weighted averaging and MurrayMorgenstern-based simulators. This particular
    implementation is to represent a parameterised version of the model which can
    represent a "pure" weighted summation with no bias, but allows the addition
    of bias (drawn from a bias distribution) which is applied within a variable
    window. We can vary both of these parameters to change how bias is applied
    and how bias is distributed which lets us evaluate the likelihood of
    different bias applications.

    Variations on the bias distribution mean are not investigated here, though
    they would have an effect; the assumption is that individuals most likely
    have no bias and if they do it will be evenly distributed between the cues.
    We can change the bias distribution variance which makes it more likely that
    individuals will have biases and makes the potential biases more extreme.

    Variations on the bias window indicate where bias takes effect. A
    bias window with width zero means individual bias is never taken into effect.
    Amaximum bias window measn full bias is always applied. Any other bias window
    means the bias will be scaled depending on how close together the cues are.
    I.e. if the difference between cue weights is large, the effect will be
    small, if the difference is small, the effect will be large.
    """

    def __init__(self, output=False, bias_window=0, bias_variance=0, bias_mu=0):
        """
        Initialise this simulator; default settings make this equivalent to
        MMCS (pure weighted vector summation).
        """
        super().__init__(output)

        self.__bias_mu = bias_mu
        self.__bias_variance = bias_variance
        self.__bias_sigma = np.sqrt(self.__bias_variance)
        self.__bias_window = bias_window
        self.set_name("BiasedWS-{}-{}".format(bias_window, bias_variance))

    def compute_integration(self,
                            light,
                            light_weight,
                            wind,
                            wind_weight,
                            offset):
        # Account for any offset from the initial presentation
        adj_wind = wind - offset

        # Eq. 16 from Murray and Morgenstern, 2010
        l = adj_wind + np.arctan2(
            np.sin(light - adj_wind),
            (wind_weight / light_weight) + np.cos(light - adj_wind))

        return l

    def compute_cue_relationship(self, initial_cues):
       if len(initial_cues) == 1:
           return 0

       wind = [x for x in initial_cues if x.get_type() == "wind"][0]
       light = [x for x in initial_cues if x.get_type() == "light"][0]

       return wind.get_azimuth() - light.get_azimuth()

    def simulate_treatment(self, treatment):
        if self.check_options(treatment):
            light = [x for x in treatment.get_initial_cues()
                     if x.get_type() == "light"][0]
            wind = [x for x in treatment.get_initial_cues()
                    if x.get_type() == "wind"][0]

            conflict_light = [x for x in treatment.get_conflict_cues()
                              if x.get_type() == "light"][0]
            conflict_wind = [x for x in treatment.get_conflict_cues()
                             if x.get_type() == "wind"][0]

            wind_samples = None
            light_samples = None
            conflict_wind_samples = None
            conflict_light_samples = None

            # If this treatment had preset samples for testing,
            # use those rather than generating fresh ones
            if treatment.preset_samples():
                samples = treatment.get_samples()
                initial = samples["initial"]
                conflict = samples["conflict"]

                wind_samples = initial["wind"]
                light_samples = initial["light"]
                conflict_wind_samples = conflict["wind"]
                conflict_light_samples = conflict["light"]
            else:
                wind_samples = wind.sample(treatment.get_n())
                light_samples = light.sample(treatment.get_n())

                conflict_wind_samples = conflict_wind.sample(treatment.get_n())
                conflict_light_samples = conflict_light.sample(treatment.get_n())

            # If bias window is 0, then set all biases to zero; there
            # is no window of weight difference in which bias is to be
            # applied.  If bias window is max, scale = 1 for any
            # weight difference; this is somewhat arbitrary, however I
            # wanted to include the case where full bias is included across
            # any weight difference. Maybe only if it's greater than max.

            # If bias variance is zero, then all biases are zero; this is
            # the same as a zero window.

            # Initialise biases to be zero (i.e. no biases)
            biases = np.zeros(treatment.get_n())

            # Draw r.v. samples if bias can be used.
            if self.__bias_sigma != 0 and self.__bias_window != 0:
                print("Biases generated")
                biases = norm.rvs(self.__bias_mu,
                                  self.__bias_sigma,
                                  treatment.get_n())

            print("{}: simulating {} trials for treatment {}".format(
                self.name(),
                treatment.get_n(),
                treatment.get_id()
                )
            )

            changes = []
            for trial in range(treatment.get_n()):
                light_sample = light_samples[trial]
                wind_sample = wind_samples[trial]
                conflict_light_sample = conflict_light_samples[trial]
                conflict_wind_sample = conflict_wind_samples[trial]
                bias = biases[trial]

                # Retrieve cue weights - these are the kappa parameters
                light_weight_clean = light.get_weight()
                wind_weight_clean = wind.get_weight()
                conflict_light_weight_clean = conflict_light.get_weight()
                conflict_wind_weight_clean = conflict_wind.get_weight()

                # Normalise weights
                light_weight = light_weight_clean / (light_weight_clean + wind_weight_clean)
                wind_weight = wind_weight_clean / (light_weight_clean + wind_weight_clean)

                conflict_light_weight = conflict_light_weight_clean / (conflict_light_weight_clean + conflict_wind_weight_clean)
                conflict_wind_weight = conflict_wind_weight_clean / (conflict_light_weight_clean + conflict_wind_weight_clean)

                weight_difference = abs(light_weight - wind_weight)

                # Remap inverse differences into 0,0.1 to act as scaling factors
                # One could argue this should be (1 - window) on the denomenator
                # however in practice, the range was better approximated by 0.99
                window = 1 - self.__bias_window
                inv_diff = 1 - weight_difference
                scale = 1 # (inv_diff - window) / (0.99 - window)

                if weight_difference < self.__bias_window:
                    bias = scale * bias
                    print("Bias applied: {}".format(bias))
                else:
                    bias = 0

                # print("W_Diff: {}".format(weight_difference))
                # print("I_Diff: {}".format(inv_diff))
                # print("Bias: {}".format(bias))

                # Include bias
                light_weight = light_weight + bias
                wind_weight = wind_weight - bias
                conflict_light_weight = conflict_light_weight + bias
                conflict_wind_weight = conflict_wind_weight - bias

                # Weight adjustment
                light_weight = sigmoid(light_weight)
                wind_weight = sigmoid(wind_weight)
                conflict_light_weight = sigmoid(conflict_light_weight)
                conflict_wind_weight = sigmoid(conflict_wind_weight)

                # Angular offset between cues has to be taken into account
                offset = wind_sample - light_sample

                # Initial "roll"
                bearing = self.compute_integration(
                    light_sample,
                    light_weight,
                    wind_sample,
                    wind_weight,
                    offset)

                # Conflict "roll"
                conflict_bearing = self.compute_integration(
                    conflict_light_sample,
                    conflict_light_weight,
                    conflict_wind_sample,
                    conflict_wind_weight,
                    offset)

                # Change in bearing
                change = conflict_bearing - bearing

                if self.output():
                    print("Trial #{}".format(treatment.get_n()))
                    print("light_sample angle: " + str(np.degrees(light_sample)))
                    print("light weight: " + str(conflict_light_weight))
                    print("wind_sample angle: " + str(np.degrees(wind_sample)))
                    print("wind weight: " + str(conflict_wind_weight))
                    print("Initial integration: " + str(np.degrees(bearing)))
                    print("Conflict integration: " + str(np.degrees(conflict_bearing)))
                    print("Change: " + str(np.degrees(change)))

                changes.append(change)

            treatment.set_changes_in_bearing(changes)

reason: import numpy,from scipy


text:
# MIT License
#
# Copyright (c) 2020 Gabriel Nogueira (Talendar)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ==============================================================================

""" Tests the implementation in :mod:`.fixed_topology.layers`.
"""

# pylint: disable=wrong-import-position
import os
os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "1"
# pylint: enable=wrong-import-position
from timeit import default_timer as timer

import numpy as np
import tensorflow as tf

from nevopy.genetic_algorithm.config import GeneticAlgorithmConfig
from nevopy.fixed_topology.layers import mating
from nevopy.fixed_topology.layers.base_layer import IncompatibleLayersError
from nevopy.fixed_topology.layers.tf_layers import TensorFlowLayer
from nevopy.fixed_topology.layers.tf_layers import TFConv2DLayer
import test_utils

config = GeneticAlgorithmConfig(  # weight mutation
                                weight_mutation_chance=(0.7, 0.9),
                                weight_perturbation_pc=(0.1, 0.4),
                                weight_reset_chance=(0.1, 0.3),
                                new_weight_interval=(-2, 2))

def test_mutate_weights(layer, num_tests=100, verbose=False):
    mutation_time = 0
    mutated_weights_pc = 0
    reset_weights_pc = 0

    for _ in range(num_tests):
        test_info = {}
        weights_old = layer.weights

        start_time = timer()
        layer.mutate_weights(_test_info=test_info)
        mutation_time += timer() - start_time

        weights_new = layer.weights

        for w_num, (w_old, w_new) in enumerate(zip(weights_old, weights_new)):
            if verbose:
                print(f"\n[WEIGHT MATRIX AT INDEX {w_num}]")

            mutate_idx = test_info[f"w{w_num}_mutate_idx"]
            perturbation = test_info[f"w{w_num}_perturbation"] - 1
            assert len(mutate_idx) == len(perturbation)

            if len(mutate_idx) > 0:
                mutate_idx, perturbation = (
                    np.array(list(t))
                    for t in zip(
                        *sorted(zip(test_info[f"w{w_num}_mutate_idx"],
                                    test_info[f"w{w_num}_perturbation"] - 1))
                    )
                )

            reset_idx = test_info[f"w{w_num}_reset_idx"]
            diff_pc = np.divide(w_new - w_old, w_old,
                                out=np.full_like(w_old, fill_value=None),
                                where=w_old != 0)

            mutated_pc = len(mutate_idx) / w_old.size
            reset_pc = len(reset_idx) / w_old.size

            mutated_weights_pc += mutated_pc / len(weights_old)
            reset_weights_pc += reset_pc / len(weights_old)

            count_reset = 0
            count_pert = 0
            for i in range(w_old.size):
                d_real = diff_pc.flat[i] \
                    # pylint: disable=unsubscriptable-object
                if verbose:
                    print(f"[ABS CHANGE] {w_old.flat[i]:.5f} "
                          f"-> {w_new.flat[i]:.5f}  <>  "
                          f"Reset: {i in reset_idx} | "
                          f"Mutate: {i in mutate_idx}"
                          f"  <>  real = {d_real:.2%}", end="")

                if i not in mutate_idx:
                    if verbose:
                        print()

                    if i not in reset_idx:
                        assert w_new.flat[i] == w_old.flat[i]
                    else:
                        assert w_new.flat[i] != w_old.flat[i]

                    if not np.isnan(d_real):
                        assert d_real == 0 or i in reset_idx

                    continue

                d_calc = perturbation.flat[count_pert]
                count_pert += 1

                if verbose:
                    print(f" | calc = {d_calc:.2%}")

                if np.isnan(d_real):
                    continue

                if abs(d_real - d_calc) > 1e-5:
                    if i in reset_idx:
                        count_reset += 1
                        continue
                    print(perturbation)
                    raise RuntimeError("Incorrect weight value!")

            assert count_reset <= len(reset_idx)
            assert count_pert == len(mutate_idx)

            if verbose:
                print(f"\nMutated weights idx: {mutate_idx}")
                print(f"Reset weights idx: {reset_idx}")
                print(f"Mutation pc: {mutated_pc:.2%}")
                print(f"Reset pc: {reset_pc:.2%}\n\n")

    if verbose:
        print("\n" + "="*50)
    print(f"> Mutation time: {1000 * mutation_time / num_tests:.4f}ms")
    print(f"> Mutated weights pc: {mutated_weights_pc / num_tests:.2%}")
    print(f"> Reset weights pc: {reset_weights_pc / num_tests:.2%}")

def test_immutable_layer_mutation(layer, num_tests=100, verbose=False):
    mutable = layer.mutable
    layer.mutable = False

    mutation_time = 0
    for _ in range(num_tests):
        old_weights = layer.weights

        start_time = timer()
        layer.mutate_weights()
        mutation_time += timer() - start_time

        new_weights = layer.weights
        for w_new, w_old in zip(new_weights, old_weights):
            assert (w_new == w_old).all()

    layer.mutable = mutable
    if verbose:
        print("\n" + "=" * 50)
    print("> Immutable layer mutation time: "
          f"{1000 * mutation_time / num_tests:.4f}ms")

def test_deep_copy(layer, num_tests=100, verbose=False):
    dc_time = 0
    for _ in range(num_tests):
        layer.mutate_weights()

        start_time = timer()
        new_layer = layer.deep_copy()
        dc_time += timer() - start_time

        # Checking if weights are equal
        for w_new, w_old in zip(new_layer.weights, layer.weights):
            if verbose:
                for _ in range(50):
                    i = np.random.randint(low=0, high=w_old.size)
                    print(f"[W] old = {w_old.flat[i]}, new = {w_new.flat[i]}")

            assert w_new.shape == w_old.shape
            assert (w_new == w_old).all()

        # Checking if outputs are equal
        for _ in range(3):
            random_input = tf.random.uniform(shape=layer.input_shape,
                                             dtype=float)
            y_new = new_layer(random_input).numpy()
            y_old = layer(random_input).numpy()

            if verbose:
                print(f"[OUTPUT] diff = {np.abs(y_old - y_new).sum()}")

            assert (y_new == y_old).all()

    if verbose:
        print("\n" + "="*50)
    print(f"> Deep copy time: {1000 * dc_time / num_tests:.4f}ms")

def test_random_copy(layer, num_tests=100, verbose=False):
    rc_time = 0
    for _ in range(num_tests):
        layer.mutate_weights()

        start_time = timer()
        new_layer = layer.random_copy()
        rc_time += timer() - start_time

        # Checking if weights are different
        for w_old, w_new in zip(layer.weights, new_layer.weights):
            if verbose:
                for _ in range(50):
                    i = np.random.randint(low=0, high=w_old.size)
                    print(f"[W] old = {w_old.flat[i]}, new = {w_new.flat[i]}")

            assert w_old.shape == w_new.shape
            assert not (w_old == w_new).all()

        # Checking if outputs are different
        for _ in range(3):
            random_input = tf.random.uniform(shape=layer.input_shape,
                                             dtype=float)
            y_new = new_layer(random_input).numpy()
            y_old = layer(random_input).numpy()

            if verbose:
                print(f"[OUTPUT] diff = {np.abs(y_old - y_new).sum()}")

            assert not (y_new == y_old).all()

    if verbose:
        print("\n" + "=" * 50)
    print(f"> Random copy time: {1000 * rc_time / num_tests:.4f}ms")

def test_exchange_units_mating(layer1, layer2, num_tests=100, verbose=False):
    layer1.mating_func = layer2.mating_func = mating.exchange_units_mating
    mating_time = mutation_time = 0
    units_from_p1 = units_from_p2 = 0
    for t in range(num_tests):
        if verbose:
            print("\n" + "#"*50)
            print(f"----- Test {t} of {num_tests} -----\n")

        # Mutating layers
        start_time = timer()
        layer1.mutate_weights()
        layer2.mutate_weights()
        mutation_time += (timer() - start_time) / 2

        # Mating layers
        start_time = timer()
        new_layer = layer1.mate(layer2)
        mating_time += timer() - start_time

        # Checking units inheritance
        for w_new, w_p1, w_p2 in zip(new_layer.weights,
                                     layer1.weights,
                                     layer2.weights):
            assert w_new.shape == w_p1.shape == w_p2.shape
            for i in range(w_p1.shape[-1]):
                unit_new = w_new[..., i]
                unit_p1 = w_p1[..., i]
                unit_p2 = w_p2[..., i]

                if (unit_new == unit_p1).all():
                    units_from_p1 += 1
                    if verbose:
                        print(f"[{i}] Unit from parent 1 <> "
                              f"diff_sum = {np.abs(unit_new - unit_p1).sum()}")
                elif (unit_new == unit_p2).all():
                    units_from_p2 += 1
                    if verbose:
                        print(f"[{i}] Unit from parent 2 <> "
                              f"diff_sum = {np.abs(unit_new - unit_p2).sum()}")
                else:
                    raise AssertionError(
                        f"New unit at index {i} doesn't match any "
                        f"of the parents units at the same index!"
                        f"\n> New unit = \n{unit_new}\n\n"
                        f"> Parent 1 unit = \n{unit_p1}\n\n"
                        f"> Parent 2 unit = \n{unit_p2}")

    if verbose:
        print("\n" + "=" * 50)
        total_units = units_from_p1 + units_from_p2
        print(f"Units from parent 1: {units_from_p1 / total_units:.2%}")
        print(f"Units from parent 2: {units_from_p2 / total_units:.2%}\n")

    print(f"> Exchange units mating:\n"
          f"\t. Mating time: {1000 * mating_time / num_tests:.4f}ms\n"
          f"\t. Mutation time: {1000 * mutation_time / num_tests:.4f}ms")

def test_exchange_weights_mating(layer1, layer2, num_tests=100, verbose=False):
    layer1.mating_func = layer2.mating_func = mating.exchange_weights_mating
    mating_time = mutation_time = 0
    w_from_p1 = w_from_p2 = 0
    for t in range(num_tests):
        if verbose:
            print("\n" + "#"*50)
            print(f"----- Test {t} of {num_tests} -----\n")

        # Mutating layers:
        start_time = timer()
        layer1.mutate_weights()
        layer2.mutate_weights()
        mutation_time += (timer() - start_time) / 2

        # Mating layers:
        start_time = timer()
        new_layer = layer1.mate(layer2)
        mating_time += timer() - start_time

        # Checking individual weights inheritance:
        for w_new, w_p1, w_p2 in zip(new_layer.weights,
                                     layer1.weights,
                                     layer2.weights):
            assert w_new.shape == w_p1.shape == w_p2.shape
            for i in range(w_p1.size):
                if w_new.flat[i] == w_p1.flat[i]:
                    w_from_p1 += 1
                    if verbose:
                        print(f"[{i}] Weight from parent 1 <> "
                              f"parent = {w_p1.flat[i]}, "
                              f"child = {w_new.flat[i]}")
                elif w_new.flat[i] == w_p2.flat[i]:
                    w_from_p2 += 1
                    if verbose:
                        print(f"[{i}] Weight from parent 2 <> "
                              f"parent = {w_p2.flat[i]}, "
                              f"child = {w_new.flat[i]}")
                else:
                    raise AssertionError(
                        f"New weight at index {i} doesn't match any of the "
                        "parents weights at the same index!\n"
                        f"> New weight = {w_new.flat[i]}\n"
                        f"> Parent 1 weight = {w_p1.flat[i]}\n"
                        f"> Parent 2 weight = {w_p2.flat[i]}"
                    )

    if verbose:
        print("\n" + "=" * 50)
        total_w = w_from_p1 + w_from_p2
        print(f"Weights from parent 1: {w_from_p1 / total_w:.2%}")
        print(f"Weights from parent 2: {w_from_p2 / total_w:.2%}\n")

    print(f"> Exchange weights mating:\n"
          f"\t. Mating time: {1000 * mating_time / num_tests:.4f}ms\n"
          f"\t. Mutation time: {1000 * mutation_time / num_tests:.4f}ms")

def test_weights_avg_mating(layer1, layer2, num_tests=100, verbose=False):
    layer1.mating_func = layer2.mating_func = mating.weights_avg_mating
    mating_time = mutation_time = 0
    for t in range(num_tests):
        if verbose:
            print(f"> Test {t} of {num_tests}.")

        # Mutating layers:
        start_time = timer()
        layer1.mutate_weights()
        layer2.mutate_weights()
        mutation_time += (timer() - start_time) / 2

        # Mating layers:
        start_time = timer()
        new_layer = layer1.mate(layer2)
        mating_time += timer() - start_time

        # Checking individual weights inheritance:
        for w_new, w_p1, w_p2 in zip(new_layer.weights,
                                     layer1.weights,
                                     layer2.weights):
            assert w_new.shape == w_p1.shape == w_p2.shape
            for i in range(w_p1.size):
                assert w_new.flat[i] == (w_p1.flat[i] + w_p2.flat[i]) / 2

    if verbose:
        print("\n" + "=" * 50)

    print(f"> Weights averaging mating:\n"
          f"\t. Mating time: {1000 * mating_time / num_tests:.4f}ms\n"
          f"\t. Mutation time: {1000 * mutation_time / num_tests:.4f}ms")

def test_immutable_layer_mating(layer1, layer2, num_tests=100, verbose=False):
    mutable1 = layer1.mutable
    mutable2 = layer2.mutable

    mating_time = 0
    for _ in range(num_tests):
        # Valid mating
        layer1.mutable = layer2.mutable = False
        start_time = timer()
        new_layer1 = layer1.mate(layer2)
        new_layer2 = layer2.mate(layer1)
        mating_time += (timer() - start_time) / 2

        for old, new in [(layer1, new_layer1), (layer2, new_layer2)]:
            for w_old, w_new in zip(old.weights, new.weights):
                assert (w_old == w_new).all()

        # Invalid mating
        layer2.mutable = True
        try:
            layer1.mate(layer2)
            layer2.mate(layer1)
            raise AssertionError("Invalid mating occurred!")
        except IncompatibleLayersError:
            pass

    layer1.mutable = mutable1
    layer2.mutable = mutable2
    if verbose:
        print("\n" + "=" * 50)

    print("> Immutable layer valid mating time: "
          f"{1000 * mating_time / num_tests:.4f}ms")

def test_save_and_load(layer, num_tests=10, pkl=True):
    saving_time = loading_time = file_size = 0
    for _ in range(num_tests):
        file_name = (f"{test_utils.TEMP_SAVE_DIR}/"
                     f"saved_layer_{int(1e6 * timer())}")

        start_time = timer()
        layer.save(file_name)
        saving_time += timer() - start_time

        file_size += os.path.getsize(file_name + ".pkl" if pkl else "")

        start_time = timer()
        loaded_layer = TensorFlowLayer.load(file_name)
        loading_time += timer() - start_time

        for w0, w1 in zip(layer.weights, loaded_layer.weights):
            assert w0.shape == w1.shape
            assert (w0 == w1).all()

        for _ in range(3):
            test_input = tf.random.uniform(shape=layer.input_shape, dtype=float)
            h0, h1 = layer(test_input), loaded_layer(test_input)
            assert (h0.numpy() == h1.numpy()).all()

    print(f"> Saving time: {1000 * saving_time / num_tests:.4f}ms")
    print(f"> Loading time: {1000 * loading_time / num_tests:.4f}ms")
    print(f"> File size: {file_size / (num_tests * 1024):.2f}KB")

    w_kb = np.sum([w.nbytes for w in layer.weights]) / 1024
    print(f"> Weights size: {w_kb:.2f}KB")

def run_all_tests(test_layer1, test_layer2):
    if test_layer1.mutable:
        test_mutate_weights(test_layer1, num_tests=10, verbose=False)
        test_exchange_units_mating(test_layer1, test_layer2,
                                   num_tests=10, verbose=False)
        test_exchange_weights_mating(test_layer1, test_layer2,
                                     num_tests=10, verbose=False)
        test_weights_avg_mating(test_layer1, test_layer2,
                                num_tests=10, verbose=False)

    test_deep_copy(test_layer1, num_tests=10, verbose=False)
    test_random_copy(test_layer1, num_tests=10, verbose=False)
    test_immutable_layer_mating(test_layer1, test_layer2,
                                num_tests=10, verbose=False)
    test_immutable_layer_mutation(test_layer1, num_tests=10, verbose=False)
    test_save_and_load(test_layer1, num_tests=10)

def test_conv2d():
    input_shape = (4, 256, 256, 3)
    test_layer1 = TensorFlowLayer(filters=128,
                                  kernel_size=(3, 3),
                                  layer_type=tf.keras.layers.Conv2D,
                                  mating_func=mating.exchange_units_mating,
                                  config=config,
                                  input_shape=input_shape)
    test_layer2 = TFConv2DLayer(filters=128,
                                kernel_size=(3, 3),
                                config=config,
                                input_shape=input_shape)
    run_all_tests(test_layer1, test_layer2)

def test_dense():
    input_shape = (32, 1024)
    test_layer1 = TensorFlowLayer(units=128,
                                  activation="relu",
                                  layer_type=tf.keras.layers.Dense,
                                  mating_func=mating.exchange_units_mating,
                                  config=config,
                                  input_shape=input_shape)
    test_layer2 = TensorFlowLayer(units=128,
                                  activation="relu",
                                  layer_type=tf.keras.layers.Dense,
                                  mating_func=mating.exchange_units_mating,
                                  config=config,
                                  input_shape=input_shape)
    run_all_tests(test_layer1, test_layer2)

def test_flatten():
    flatten_layer = TensorFlowLayer(layer_type=tf.keras.layers.Flatten,
                                    config=config,
                                    mutable=False)

    for _ in range(100):
        dc = flatten_layer.deep_copy()
        rc = flatten_layer.random_copy()
        child = flatten_layer.mate(rc)

        test_input = tf.random.uniform(shape=[16, 32, 64, 100], dtype=float)
        orig_out = flatten_layer(test_input).numpy()
        deep_out = dc(test_input).numpy()
        rand_out = rc(test_input).numpy()
        child_out = child(test_input).numpy()

        assert ((deep_out == orig_out).all()
                and (deep_out == orig_out).all()
                and (rand_out == orig_out).all()
                and (child_out == orig_out).all())
        assert len(orig_out.shape) == 2 and orig_out.shape[1] == (32*64*100)

    test_save_and_load(flatten_layer)

def test_sequential(verbose=False):
    layers = [
        # Conv2D
        TensorFlowLayer(filters=64,
                        kernel_size=(3, 3),
                        layer_type=tf.keras.layers.Conv2D,
                        mating_func=mating.exchange_units_mating,
                        config=config),
        # Flatten
        TensorFlowLayer(layer_type=tf.keras.layers.Flatten,
                        config=config,
                        mutable=False),
        # Dense hidden
        TensorFlowLayer(units=64,
                        activation="relu",
                        layer_type=tf.keras.layers.Dense,
                        mating_func=mating.exchange_units_mating,
                        config=config),
        # Dense output
        TensorFlowLayer(units=10,
                        activation="softmax",
                        layer_type=tf.keras.layers.Dense,
                        mating_func=mating.exchange_units_mating,
                        config=config),
    ]

    for n in range(10):
        out = tf.random.uniform(shape=[3, 256, 256, 3], dtype=float)
        for layer in layers:
            out = layer(out)

        if verbose:
            print(f"\n\n[OUT {n}]")
            print(out.numpy(), [-1])

if __name__ == "__main__":
    test_utils.clear_temp()

    # Conv2D:
    print("\n[CONV2D]")
    test_conv2d()
    print("[CONV2D] Passed all assertions!")

    # Dense:
    print("\n[DENSE]")
    test_dense()
    print("[DENSE] Passed all assertions!")

    # Flatten:
    print("\n[FLATTEN]")
    test_flatten()
    print("[FLATTEN] Passed all assertions!")

    # Sequential layers processing:
    test_sequential(verbose=False)

    test_utils.clear_temp()

ext: py

max_stars_repo_name: Talendar/nevo-py

max_issues_repo_name: Talendar/nevo-py

max_forks_repo_name: Talendar/nevo-py

reason: import numpy


text:
#http://webbook.nist.gov/cgi/cbook.cgi?ID=C1333740&Units=SI&Mask=1#Thermo-Gas

#selected properties on hydrogen
#http://www.boulder.nist.gov/div838/Hydrogen/PDFs/McCarty.Hord.Weber.1981.monograph168.pdf
#selected topics on hydrogen fuel
#http://www.boulder.nist.gov/div838/Hydrogen/PDFs/Hord(ed.)1975.NBSIR%2075-803.pdf
#polarizability of the hydrogen molecule
#http://jcp.aip.org/resource/1/jcpsa6/v46/i4/p1426_s1

#low temperature specific heat (ortho vs para)
#http://ocw.tudelft.nl/fileadmin/ocw/courses/AdvancedStatisticalMechanics/res00030/!486973746f7279206f6620746865206465736372697074696f6e206f662074686520687964726f67656e20676173.pdf
#ionization of hydrogen
#http://csep10.phys.utk.edu/astr162/lect/stars/spectra.html

#less useful links
#http://www.wag.caltech.edu/home/jsu/Thesis/node31.html

from pylab import *
from scipy.constants import *
from scipy.misc import derivative as deriv
from scipy.integrate import quad
#from PyQuante.Molecule import Molecule
#from PyQuante.hartree_fock import rhf

#brute force anharmonic specific heat calculation with partition function
def anharmoniccvib(T):
    k2 = 0.782
    k3 = -34.1
    k4 = 34.88
    def logZ(T):
        foo = lambda x: exp(( -k2*x*x -k3*x*x*x - k4*(x**4) ) /  T )
        return log(quad(foo,-Inf,Inf)[0])
    energy = lambda x: x**2 * deriv(logZ,x,dx=1e-6)
    #return R*( 1/2. + deriv(energy,T,dx=1e-6 )) # 1/2 from kinetic energy
    #from perturbation theory
    return R * (1 - (k*T/1.6e-19) * 9. / 4 * k4 / (k2**2))

#def bondpotential(x):
    #h2 = Molecule('h2',[(1,(0,0,0)),(1,(x,0,0))])
    #return rhf(h2)[0]

def NISTcp(T):
    param = []
    if 298 < T < 1001:
        param += [33.066178, -11.363417, 11.432816, -2.772874, -0.158558]
    elif 1000 < T < 2501:
        param += [ 18.563083, 12.257357, -2.859786, 0.268238, 1.977990  ]
    elif 2500 < T < 6000:
        param += [ 43.413560, -4.293079, 1.272428, -0.096876, -20.533862 ]
    t = T/1000.
    return param[0] + param[1]*t + param[2]*t*t + param[3]*t*t*t + param[4]/(t*t)

def atomfraction(T):
    l = 1.47 * 5e-11
    sigma = l * l
    coeff = .5 * 1e5 /(k*298) * l * sigma
    if T > 950:
        b = coeff * exp( 52452.3639093575 /T) #4.52eV/k_B
        return (sqrt(4*b + 1) -1)/(2*b)
    else: return 0

cHO = lambda x: R * (x**2) * exp(x)/((exp(x)-1)**2)

def theorycpatom(T):
    # angular frequency from polarizability 6.9e16 Hz
    ratiovibatom = 527037/T #hbar * 6.9e16 / k
    return 5.*R/2 #+ cHO(ratiovibatom)

def crot(ratiorot):
    A = sum([(2*l+1)* ((l*(l+1))**2) * exp(-l*(l+1)*ratiorot) for l in range(30) ])
    B = sum([(2*l+1)* (l*(l+1)) * exp(-l*(l+1)*ratiorot) for l in range(30) ])
    Z = sum([(2*l+1)*  exp(-l*(l+1)*ratiorot) for l in range(30) ])
    return (A/Z- (B/Z)**2) * ratiorot * ratiorot * R

def theorycpmolecule(T):
    ratiorot = 85./T
    ratiorotelectron = 1000./T
    ratiovib = 6215. /T
    ratiodipole = 26000. /T
    cvib = cHO(ratiovib)
    cdipole = cHO(ratiodipole)
    return R + 1.5*R + crot(ratiorot) + 2* cdipole + cvib  #+ anharmoniccp(T)  #+cvib

def theorycp(T):
    f = atomfraction(T)
    return (1-f)*theorycpmolecule(T) + f*theorycpatom(T)

xaxis = range(300,6000,20)
p1= plot(xaxis, [NISTcp(i) for i in xaxis])
#p2= plot(xaxis, [theorycp(i) for i in xaxis])
p2 = plot(xaxis,[theorycpmolecule(i) for i in xaxis])
xlabel("Temperature (K)")
ylabel("Specific heat (J/(kg mol))")
title("Hydrogen molecule specific heat with Ctrans,Crot,Cvib,Cdip")
#legend([p1, p2], ["exp", "theory"], loc = "lower right")
show()

reason: from scipy


text:
""" Q learning """

import numpy as np

import plot

Q = np.array(np.zeros([11, 3]))

GAMMA = 0.9
ALPHA = 0.1

def action_select(s_s):
    """ action select """
    return np.random.choice([i for i in range(1, 4) if i + s_s < 11])

for i in range(10000):
    S_STATE = 0
    while S_STATE != 10:
        a_state = action_select(S_STATE)
        R = 0.001
        s_state_dash = S_STATE + a_state
        if s_state_dash == 10:
            R = -10
        else:
            s_state_dash = action_select(s_state_dash)+s_state_dash
            if s_state_dash == 10:
                R = 10

        Q[S_STATE, a_state-1] = Q[S_STATE, a_state-1]+ALPHA * \
            (R+GAMMA * Q[s_state_dash,
             np.argmax(Q[s_state_dash, ])] - Q[S_STATE, a_state-1])
        S_STATE = s_state_dash

plot.plot_func(Q)


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 10 10:13:39 2020

@author: muril
"""

def scatter_plot(sim,
                 obs,
                 fn_out,
                 vnam    = None,
                 units   = None,
                 p_index = False,
                 p_idx   = None,
                 save_fig= True,
                 v_sub   = None,
                 p_sub   = ["#999999", "#E69F00", "#56B4E9", "#009E73", "#F0E442", "#0072B2", "#D55E00", "#CC79A7"],
                 p_size  = 5):

    #----------------------------------------------------------#
    #---------------------- scatter_plot ----------------------#
    #----------------------------------------------------------#
    #--- Goal:
    #---    Plot Simulated vs Observed scatter plot
    #--- Parameters:
    #--- 	sim      	: Simulated data as vector
    #--- 	obs      	: Observed data as vector
    #--- 	fn_out   	: Filename of figure output
    #--- 	vnam     	: Name of variable
    #--- 	units    	: Units of the variable
    #--- 	p_index  	: Plot Indexes (True/False)
    #--- 	p_idx    	: Indexes to be plotted as dataframe with single row
    #--- 	save_fig 	: Save Figure (True/False)
    #--- Author:
    #---    Murilo Vianna (murilodsv@gmail.com)
    #----------------------------------------------------------#

    import matplotlib.pyplot as plt
    import statsmodels.api as sm
    import numpy as np

    #--- Linear regression model
    model = sm.OLS(sim, sm.add_constant(obs)).fit()

    #--- Regression parameters
    try:
        a     = model.params[0]
        b     = model.params[1]
    except:
        print('Warning: Only one value to compare, not all statistical indexes can be computed.')
        b     = model.params[0]
        a     = None

    #--- check a and b
    if type(a) == type(None):   a = 0 # None
    if a != a:                  a = 0 # nan
    if np.isinf(a):             a = 0 # inf

    if type(b) == type(None):   b = 0 # None
    if b != b:                  b = 0 # nan
    if np.isinf(b):             b = 0 # inf

    #--- scatter axis-limits
    min_vals = min(np.append(sim,obs))
    max_vals = max(np.append(sim,obs))
    bord     = (max_vals - min_vals) * 0.04

    #--- regression and 1:1 lines
    x_reg   = np.linspace(min_vals, max_vals, 2)
    one_one = np.linspace(min_vals - (max_vals - min_vals), max_vals + (max_vals - min_vals), 2)

    plt.figure(figsize=(p_size,p_size))
    plt.plot(one_one, one_one, '--', c = 'red', linewidth=1, label="1:1 Line")

    if type(v_sub) == type(None):

        plt.scatter(obs,sim,
            edgecolors='black',
            facecolors='green',
            linewidths= 0.5,
            label="Simulated")

    else:

        #--- Do not plot stat indexes
        p_index = False

        #--- Check size of indexed vector
        if type(v_sub) == type(None):
            print("Warning: None indexer vector given for scatter plot. All Data will be plotted with same color.")
            v_sub = ['Simulated'] * len(obs)
        if len(v_sub) != len(obs):
            print("Warning: Size of indexer vector given for scatter plot differ from observations. All Data will be plotted with same color.")
            v_sub = ['Simulated'] * len(obs)

        col_n = 0
        for v in list(v_sub.unique()):

            if col_n > (len(p_sub) -1):
                print("Warning: Number of color in pallete provided is lower than number of sub indexes to plot. Pallete will reboot.")
                col_n = 0

            obs_v = obs[v_sub == v]
            sim_v = sim[v_sub == v]
            col   = p_sub[col_n]

            plt.scatter(obs_v,sim_v,
                        edgecolors='black',
                        facecolors=col,
                        linewidths= 0.5,
                        label=v)
            col_n += 1

    plt.plot(x_reg, a + b * x_reg, c = 'blue', linewidth=1, label="Regression")

    plt.xlim(min_vals - bord,max_vals + bord)
    plt.ylim(min_vals - bord,max_vals + bord)
    plt.ylabel('Simulated '+str(vnam)+' ('+str(units)+')')
    plt.xlabel('Observed ' +str(vnam)+' ('+str(units)+')')

    if p_index:

        #--- size and format of strings
        max_s   = len(max(p_idx.keys(), key=len))
        fmt     = "%"+str(max_s)+"s"

        l_idx = []
        for k in p_idx.keys():
            lab_i   = k
            val_i   = p_idx[k].values[0]
            print_v = ((fmt) % str(lab_i))+' = '+(("%6.2f") % float(val_i))
            l_idx.append(print_v)
        idx_text = "\n".join(l_idx)

        #--- Add Text to plot
        plt.text(0.05, 0.95,
                 idx_text,
                 horizontalalignment = 'left',
                 verticalalignment   = 'top',
                 transform=plt.gca().transAxes)

    else:

        #--- Add legend instead of indexes
        plt.legend()

    #--- Save figure
    if save_fig:

        plt.savefig(fn_out+'.scatter_plot.png', dpi = 500)

    #plt.show()
    plt.close()

max_stars_repo_name: Murilodsv/py-jules

max_issues_repo_name: Murilodsv/py-jules

max_forks_repo_name: Murilodsv/py-jules

reason: import numpy,import statsmodels


text:
# speed and power...solves many many things. -Jeremy Clarkson
# name:         make_tsne.py
# author:       Timo Kats
# last update:  20/05/2021
# description:  makes tsne graphs per news source.

import sklearn
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import gensim

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn import svm, preprocessing
from sklearn.manifold import TSNE
from sklearn.model_selection import cross_val_score, train_test_split

def get_stopwords():
    file = open('data/stopwords', 'r+', encoding='utf-8')
    stop_words = file.read().split('\n')
    return stop_words

def get_data(dataframe):
    dataframe['is_sponsored'] = np.where(dataframe['sponsor'] != 'none', 1, 0)
    return dataframe['is_sponsored']

def get_classes(dataframe):
    dataframe['class'] = np.where(dataframe['sponsor'] != 'none', 'advertorial', 'article')
    return dataframe['class']

if __name__ == '__main__':

    sources = ['nu', 'telegraaf', 'nrc', 'ondernemer']

    for source in sources:
        data_ads = pd.read_csv('data/ads/ads_' + source + '.csv')
        data_articles = pd.read_csv('data/articles/articles_' + source + '.csv')
        data = pd.DataFrame(data_ads).append(data_articles, ignore_index=True)
        data['input'] = data['introduction'] + ' ' + data['body']
        is_sponsored = get_classes(data).tolist()
        source = data['source'].tolist()
        y = get_data(data)

        data = data['input'].values.astype(str)

        stop_words = get_stopwords()
        cv = TfidfVectorizer(stop_words=stop_words, lowercase=True, max_features=5000)
        cv.fit(data)

        X = cv.transform(data)

        clf = svm.SVC(max_iter=5000, cache_size=50000)
        clf.fit(X, y)
        clf.predict(X)

        model = TSNE(random_state=2)
        tsne_features = model.fit_transform(X)

        df = pd.DataFrame()
        df['x'] = tsne_features[:,0]
        df['y'] = tsne_features[:,1]
        df['class'] = is_sponsored
        df['source'] = source
        print(df)

        sns.scatterplot(x='x', y='y', hue='class', data=df, palette=['blue', 'red'])
        plt.show()


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.1.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # s_spectral_representation [<img src="https://www.arpm.co/lab/icons/icon_permalink.png" width=30 height=30 style="display: inline;">](https://www.arpm.co/lab/redirect.php?code=s_spectral_representation&codeLang=Python)
# For details, see [here](https://www.arpm.co/lab/redirect.php?permalink=ex-cross-spectr-propp-copy-5).

# +
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from scipy.linalg import toeplitz

from arpym.statistics import simulate_normal, simulate_var1
from arpym.tools import pca_cov, add_logo
# -

# ## [Input parameters](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-parameters)

t_ = 10**3  # length of process
j_ = 10  # number of simulations of paths
b = 0.6  # autoregression parameter of the AR(1)
mu_eps = 0  # expectation of shocks in AR(1)
sigma2_eps = 1-b**2  # variance of shocks in AR(1)
t = 300  # lags

# ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step01): Simulation of AR(1)

# +
if b == 0:
    b = 10**-6

mu_x = mu_eps/(1-b)  # expectation of (stationary) AR(1)
sigma2_x = sigma2_eps/(1-b**2)  # variance of (stationary) AR(1)

x = np.zeros((t_, j_))
for j in range(j_):
    x0 = simulate_normal(mu_x, sigma2_x, 1)
    x[:, j] = simulate_var1(x0,
                            np.atleast_2d(b),
                            np.atleast_2d(mu_eps),
                            np.atleast_2d(sigma2_eps),
                            t_-1,
                            j_=1).squeeze()
# -

# ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step02): Choose subsequent observations

t0 = np.int(t_/2)  # choose far from initial point to have more stationarity
x_vec = x[t0-t:t0+t+1, :]
mu_x_vec = mu_x*np.ones((2*t+1, j_))
t_vec = np.linspace(-t, t, 2*t+1)

# ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step03): Compute covariance matrix of random vector x_vec

k_x = b**(np.arange(2*t+1))/(1-b**2)*sigma2_eps  # autocovariance fun
cv_x = toeplitz(k_x)  # autocovariance matrix

# ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step04): Compute eigenvectors/eigenvalues

# +
e, lambda2 = pca_cov(cv_x)

if b < 0:
    ind_asc = np.argsort(lambda2)
    lambda2 = lambda2[ind_asc]
    e = e[:, ind_asc]

lambda2_new = []
ind_e = []*(2*t+1)
for n in range(1, 2*t+2):
    if n % 2 == 1:
        lambda2_new = np.append(lambda2_new, lambda2[n-1])
        ind_e = np.append(ind_e, n-1)
    else:
        lambda2_new = np.append(lambda2[n-1], lambda2_new)
        ind_e = np.append(n-1, ind_e)
ind_e1 = [int(i) for i in ind_e]
lambda2 = lambda2_new
e = e[:, ind_e1]

delta_omega = 2*np.pi/(2*t+1)
eta = np.sqrt(np.pi/(delta_omega))*e
# -

# ## [Step 5](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step05): Compute spectral density of the AR(1)

# +
# frequences
omega_vec = np.zeros((2*t+1, 1))
for j in range(1, 2*t+2):
    omega_vec[j-1] = ((-1)**(j-1))*j*np.pi/(2*t+1)
omega_vec = np.sort(omega_vec, axis=None)

ktilde_x = sigma2_eps/(1-2*b*np.cos(omega_vec)+b**2)
# -

# ## [Step 6](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step06): Compute principal factors

z_pc_omega = e.T@(x_vec - mu_x_vec)

# ## [Step 7](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step07): Compute rescaled principal factors

delta_y_omega = np.sqrt(delta_omega)*z_pc_omega

# ## [Step 8](https://www.arpm.co/lab/redirect.php?permalink=s_spectral_representation-implementation-step08): Compute the orthogonal increments process

y_omega = np.cumsum(delta_y_omega, 0)  # cumulative variable

# ## Plots

# +
plt.style.use('arpm')
mydpi = 72.0
f = plt.figure(figsize=(1280.0/mydpi, 720.0/mydpi), dpi=mydpi)
lightgrey = [.8, .8, .8]
darkgrey = [.1, .1, .1]
darkgreen = [0, 0.7, 0]
lightred = [.9, .6, .6]
darkred = [.9, 0, 0]
lightblue = [181/255, 209/225, 223/225]
omegalim = [-np.pi, np.pi]
taulim = [t_vec[0], t_vec[-1]]

gs0 = gridspec.GridSpec(2, 2)

ax1 = plt.Subplot(f, gs0[0, 0])
f.add_subplot(ax1)
ax1.tick_params(labelsize=14)
for j in range(1, j_):
    plt.plot(t_vec, np.squeeze(x_vec[:, j]), color=lightgrey, linewidth=0.2)
plt.ylabel('$X_t$', fontsize=17)
plt.xlabel('$t$', fontsize=17)
p1 = plt.plot(t_vec, np.squeeze(x_vec[:, j]), color='k', linewidth=1)
plt.title('AR(1) process, b = ' + str(b), fontsize=20)

ax2 = plt.Subplot(f, gs0[0, 1])
f.add_subplot(ax2)
ax2.tick_params(labelsize=14)
for j in range(1, j_):
    plt.plot(omega_vec, np.real(np.squeeze(y_omega[:, j])), color=lightgrey,
             linewidth=0.2)
plt.plot(omega_vec, np.real(np.squeeze(y_omega[:, 0])), color=darkgrey,
         linewidth=1, label='Orth. incr. process')
plt.plot(omega_vec, 2*np.sqrt(np.cumsum(lambda2*delta_omega)),
         color=darkgreen, linewidth=0.9, label='2std')
plt.plot(omega_vec, -2*np.sqrt(np.cumsum(lambda2*delta_omega)),
         color=darkgreen, linewidth=0.9)
plt.legend()
plt.title('Orthogonal increments process', fontsize=20)
plt.xlabel('$\omega$', fontsize=17)
plt.xticks([-np.pi, -np.pi/2, 0, np.pi/2, np.pi],
           [r'$-\pi$', r'$-\pi/2$', r'$0$', r'$\pi/2$', r'$\pi$'])

ax3 = plt.Subplot(f, gs0[1, 0])
f.add_subplot(ax3)
ax3.tick_params(labelsize=14)
plt.plot(t_vec, eta[:, t], 'b', linewidth=2, label=r'$\eta_{\omega_1}$')
plt.plot(t_vec, eta[:, t+1], 'm', linewidth=2, label=r'$\eta_{\omega_3}$')
plt.plot(t_vec, eta[:, t-1], 'y', linewidth=2, label=r'$\eta_{\omega_{2}}$')
plt.plot(t_vec, eta[:, t+2], 'c', linewidth=2, label=r'$\eta_{\omega_5}$')
plt.plot(t_vec, eta[:, t-2], 'g', linewidth=2, label=r'$\eta_{\omega_{4}}$')
plt.legend()
plt.xlabel(r'$\omega$', fontsize=17)
plt.xticks([-300, -150, 0, 150, 300],
           [r'$-\pi$', r'$-\pi/2$', r'$0$', r'$\pi/2$', r'$\pi$'])
plt.title('Rescaled eigenvectors', fontsize=20)

ax4 = plt.Subplot(f, gs0[1, 1])
f.add_subplot(ax4)
ax4.tick_params(labelsize=14)
plt.bar(t_vec, lambda2, color='lightblue', label=r'$\lambda^2_\omega$')
plt.plot(t_vec, ktilde_x, color=darkred, linewidth=0.4, label=r'$\tilde{k}_X(\omega)$')
plt.legend()
plt.xticks([-300, -150, 0, 150, 300],
           [r'$-\pi$', r'$-\pi/2$', r'$0$', r'$\pi/2$', r'$\pi$'])
plt.xlabel(r'$\omega$', fontsize=17)
plt.title('Spectrum', fontsize=20)

add_logo(f, location=4)
plt.tight_layout()

max_stars_repo_name: dpopadic/arpmRes

max_issues_repo_name: dpopadic/arpmRes

max_forks_repo_name: dpopadic/arpmRes


text:
"""SoftWebauthnDevice usage for (yubico's) fido2 example webserver/application"""

# Disable the HTTPs verify warning
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

import requests
import pytest
import json
import pickle

from multiprocessing import Process, Queue, Pipe
import time
import os

import numpy as np

# Hacky way to import `soft_webauthn`
import sys
sys.path.append("..")

from soft_webauthn import SoftWebauthnDevice

ADDRESS = 'https://localhost:8081'

def findTrim(s, findStr):
    return s.find(findStr) + len(findStr)

def getCookie(setCookie, cookieName):
    idx1 = findTrim(setCookie, cookieName) + 1 # +1 to rid of =
    idx2 = findTrim(setCookie[idx1:], ';') - 1 # -1 to rid of ;
    return setCookie[idx1:idx1+idx2]

def get(url):
    return requests.get(ADDRESS + url,
                        verify=False)

def post(url, data, cookies=dict()):
    return requests.post(ADDRESS + url,
                         files=data,
                         cookies=cookies,
                         verify=False)

class User():
    def __init__(self, username, password, useWebauthn):
        self.cookies = dict()

        # Remember the input arguments
        self.username = username
        self.password = password
        self.useWebauthn = useWebauthn

        # Generate a new device for this `User`
        self.device = SoftWebauthnDevice()

        # Persistent variables for the functions
        self._change_email_prev_t0 = time.time()
        self._change_email_prev_t1 = time.time()

        # Log the user in and webauthn register them (if necessary)
        self.log_in_and_register()

    def log_in_and_register(self):
        self.gogs_server_login()

        # Register only if the user does not have webauthn enabled
        if self.useWebauthn and not self.is_webauthn_enabled():
            self.gogs_server_registration()

    def is_webauthn_enabled(self):
        resp = get('/webauthn/is_enabled/{}'.format(self.username))

        if resp.status_code != 200:
            print("Error with webauthn is enabled")
            return False

        return resp.json()['webauthn_is_enabled']

    def gogs_server_login(self):
        # Get the `i_like_gogs` session cookie
        resp = get('/user/login')

        # Set the `i_like_gogs` session cookie
        setCookie = resp.headers['Set-Cookie']
        self.cookies['i_like_gogs'] = getCookie(setCookie, 'i_like_gogs')

        attestation = "no-attestation"
        # If webauthn is enabled, generate an `assertion` object
        if self.useWebauthn and self.is_webauthn_enabled():
            resp = post('/webauthn/begin_login',
                        {"username": (None, self.username)},
                        cookies=self.cookies,
            )

            if resp.status_code != 200:
                print("Error in begin_login")
                return

            # Set the `webauthn-session` cookie
            setCookie = resp.headers['Set-Cookie']
            self.cookies['webauthn-session'] = getCookie(setCookie, 'webauthn-session')

            options = {'publicKey': resp.json()}

            #  User holds an authenticator
            attestation = self.device.get(options, 'https://localhost:8081')

        resp = post('/user/login',
                    {"user_name": (None, self.username),
                     "username": (None, self.username),
                     "password": (None, self.password),
                     "assertion": (None, json.dumps(attestation)),
                    },
                    cookies=self.cookies,
        )

        if resp.status_code != 200:
            print("Error in login")
            return

        resp = get('/')

        # Set the `csrf` cookie
        setCookie = resp.headers['Set-Cookie']
        self.cookies['_csrf'] = getCookie(setCookie, '_csrf')

        #print("Successfully logged in user!")

    def gogs_server_registration(self):  # pylint: disable=redefined-outer-name
        """Registration example"""
        # Begin registration
        resp = post('/webauthn/begin_register',
                    {"username": (None, self.username)},
                    cookies=self.cookies
        )

        if resp.status_code != 200:
            print("Error in begin_register")
            return

        # Set the `webauthn-session` cookie
        setCookie = resp.headers['Set-Cookie']
        self.cookies['webauthn-session'] = getCookie(setCookie, 'webauthn-session')

        options = {'publicKey': resp.json()}

        # User holds an authenticator.
        #
        # NOTE: SoftWebauthnDevice emulates mixed client and authenticator behavior
        # and can be used for testing Webauthn/FIDO2 enabled applications during
        # continuous integration test-cases.
        attestation = self.device.create(options, 'https://localhost:8081')

        resp = post('/webauthn/finish_register',
                    {"username": (None, self.username),
                     "credentials": (None, json.dumps(attestation)),
                    },
                    cookies=self.cookies,
        )

        if resp.status_code == 200:
            #print("Successfully registered webauthn user!")
            pass
        else:
            print("Error in register webauthn user")

    def gogs_server_change_email(self, newEmail):
        auth_text = "Confirm profile details: username {} email {}".format(self.username, newEmail)

        attestation = "no-attestation"
        if self.useWebauthn:
            resp = post('/webauthn/begin_attestation',
                        {"auth_text": (None, auth_text)},
                        cookies=self.cookies
            )

            if resp.status_code != 200:
                print("Error in begin_attestation")
                return

            # Set the `webauthn-session` cookie
            setCookie = resp.headers['Set-Cookie']
            self.cookies['webauthn-session'] = getCookie(setCookie, 'webauthn-session')

            options = {'publicKey': resp.json()}

            #  User holds an authenticator
            attestation = self.device.get(options, 'https://localhost:8081')

        prev_run_time = self._change_email_prev_t1 - self._change_email_prev_t0
        time_since_last_POST = time.time() - self._change_email_prev_t0

        # Sleep so that each POST request is approximately sent every 500ms
        #time.sleep(max(0.5 - (time_since_last_POST + prev_run_time), 0))

        # Time this POST request
        t0 = time.time()
        self._change_email_prev_t0 = t0
        resp = post('/user/settings',
                    {"_csrf": (None, self.cookies['_csrf']),
                     "name": (None, self.username),
                     "full_name": (None, ""),
                     "email": (None, newEmail),
                     "website": (None, ""),
                     "location": (None, ""),
                     "auth_text": (None, auth_text),
                     "assertion": (None, json.dumps(attestation)),
                    },
                    cookies=self.cookies,
        )

        #resp = get('/user/settings')

        t1 = time.time()
        self._change_email_prev_t1 = t1

        # Save the `elapsed_time`
        elapsed_time = t1 - t0
        self._change_email_prev_run_time = elapsed_time

        if resp.status_code != 200:
            print("Error in change user email")

        # In milliseconds
        return elapsed_time * 1000, t1
        #return resp.elapsed.total_seconds() * 1000, t1

def change_emails(user, n_iters, queue):
    run_times = []
    diff_post_times = []
    prev_post_time = time.time()

    for i in range(n_iters):
        rt, end_post_time = user.gogs_server_change_email("email@email{}.com".format(i))

        # Append the timings
        run_times.append(rt)
        diff_post_times.append((end_post_time - prev_post_time) * 1000)
        prev_post_time = end_post_time

        if i % 50 == 0:
            #print("=== User {} \t Iters {} \t Avg time: {:0.2f} ms \t Avg between POST: {:0.2f} ms ===".format(
            #    user.username, i, np.mean(run_times), np.mean(diff_post_times)))
            pass

    # Place all of the `run_times` into the `queue`
    for rt in run_times:
        queue.put(rt)

def run_times_aggregator(percentile, queue, pipe):
    # Collect the `run_times`
    run_times = []

    # The `pipe` is still empty, so no kill signal received yet
    while not pipe.poll():
        # Continue emptying the `queue`
        while not queue.empty():
            run_times.append(queue.get())

    # Get the `percentile` from the `run_times`
    print("{}th percentile: {}".format(
        percentile,
        np.percentile(run_times, percentile))
    )

def create_accounts(lo, hi):
    """ Create user accounts 'durian{id}' where `id` is in [lo, hi) """
    for i in range(lo, hi):
        username = 'durian{}'.format(i)
        email = 'test{}@email.com'.format(i)

        resp = post('/user/sign_up',
                    {"_csrf": (None, "no-csrf"),
                     "user_name": (None, username),
                     "email": (None, email),
                     "password": (None, "password"),
                     "retype": (None, "password"),
                    }
        )

        if resp.status_code != 200:
            print("Error in creating new user")

        #print("Created user: {}".format(username))

def main():
    # Create a bunch of new users
    if False:
        create_accounts(18, 128)
        return

    useWebauthn = False
    n_users = int(sys.argv[1])
    n_iters = 300
    percentile = 95

    print("Running {} user(s)".format(n_users))

    users = []

    # Load the `users` if they exist
    usersFiles = './users.pkl'
    if os.path.exists(usersFiles):
        with open(usersFiles, 'rb') as infile:
            users = pickle.load(infile)
    else:
        # Initialize the new `User`s
        for u in range(n_users):
            username = 'durian{}'.format(u)
            users.append(User(username, 'password', useWebauthn))

    # Initialize a `queue` to collect all of the run times
    queue = Queue()
    procs = [Process(target=change_emails,
                     args=(users[u], n_iters, queue,))
             for u in range(n_users)]

    parent_conn, child_conn = Pipe()
    aggregator = Process(target=run_times_aggregator,
                         args=(percentile, queue, child_conn,))
    # procs = []
    # for u in range(n_users):
    #     user = users[0]

    #     def target():
    #         print("User: ", user.username)
    #         change_emails(user, n_iters, queue)

    #     procs.append(Process(target=target))

    for p in procs:
        p.start()

    aggregator.start()

    for p in procs:
        p.join()

    # Send over the kill signal to the `aggregator`
    parent_conn.send(True)

    # Wait for the aggregator to finish
    aggregator.join()

    # durian = users['durian']
    # print(durian.is_webauthn_enabled())

    # for i in range(100):
    #     durian.gogs_server_change_email("email@email{}.com".format(i))

    # TODO: Pickle does not want to serialize the private_key
    #
    # Write the `users` to file
    #with open(usersFiles, 'wb') as outfile:
    #    pickle.dump(users, outfile)

if __name__ == '__main__':
    main()

max_stars_repo_name: JSmith-BitFlipper/soft-webauthn

max_issues_repo_name: JSmith-BitFlipper/soft-webauthn

max_forks_repo_name: JSmith-BitFlipper/soft-webauthn


text:
import platform
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import cv2
import gc
import matplotlib.pyplot as plt

import torch
import timm
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader

from sklearn.metrics import roc_auc_score

def train_one_epoch(model, device, optimizer, dataloader, loss_fn, scheduler=None):
    """Trains a given model for 1 epoch on the given data

    Args:
        model: Main model
        device: Device on which model will be trained
        optimizer: Optimizer that will optimize during training
        dataloader: Training Dataloader
        loss_fn: Training Loss function. Will be optimized
        scheduler (optional): Scheduler for the learning rate. Defaults to None.
    """
    prog_bar = tqdm(enumerate(dataloader), total=len(dataloader))
    model.train()
    running_loss = 0
    for idx, (img, target) in prog_bar:
        img = img.to(device, torch.float)
        target = target.to(device, torch.float)

        output = model(img).view(-1)
        loss = loss_fn(output, target)

        # Sending the data from GPU to CPU in a numpy form (using .item()) consumes memory
        # So only do it once
        loss_item = loss.item()
        prog_bar.set_description('loss: {:.2f}'.format(loss_item))

        loss.backward()
        optimizer.step()

        if scheduler:
            scheduler.step()

        optimizer.zero_grad(set_to_none=True)

        running_loss += loss_item

    return running_loss / len(dataloader)

@torch.no_grad()
def valid_one_epoch(model, device, dataloader, loss_fn):
    """Validates the model on the validation set through all batches

    Args:
        model: Main model
        device: Device on which model will be validated
        dataloader: Validation Dataloader
        loss_fn: Validation Loss function. Will NOT be optimized
    """
    prog_bar = tqdm(enumerate(dataloader), total=len(dataloader))
    all_targets, all_predictions = [], []
    running_loss = 0
    model.eval()
    for idx, (img, target) in prog_bar:
        img = img.to(device, torch.float)
        target = target.to(device, torch.float)

        output = model(img).view(-1)

        loss = loss_fn(output, target)
        loss_item = loss.item()

        prog_bar.set_description('val_loss: {:.2f}'.format(loss_item))

        all_targets.extend(target.cpu().detach().numpy().tolist())
        all_predictions.extend(torch.sigmoid(output).cpu().detach().numpy().tolist())

        running_loss += loss_item

    val_roc_auc = roc_auc_score(all_targets, all_predictions)
    return val_roc_auc, running_loss / len(dataloader)

max_stars_repo_name: heytanay/alien-signal-detection

max_issues_repo_name: heytanay/alien-signal-detection

max_forks_repo_name: heytanay/alien-signal-detection


text:
import pdb
import os
import time
import sys
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, mean_absolute_percentage_error
from torch.utils.data import DataLoader
sys.path.insert(1,"../")
from model_utils import create_data_variables, HRMASDataset_Type_B, summary, EarlyStopping, absolute_percentage_error
from plot_loss import plot_loss
from config import *

# cross validation loop
process_start = time.time()
for outer_idx, (train_idx, vald_idx, test_idx, train_data, vald_data, test_data, all_data) in enumerate(generator(K, fold_dct, statistics, spectra, ppm_spectra, quant, class_labels)):
    print(f"Test fold {outer_idx}")
    torch.cuda.manual_seed_all(weight_seed)

    # normalize losses with respect to maximum quantification per metabolite
    max_quant_values_in_cohort = torch.from_numpy(np.mean(train_data["quant"], axis = 0))

    # generate data variables
    train_dataset = HRMASDataset_Type_B(train_data)
    vald_dataset = HRMASDataset_Type_B(vald_data)
    test_dataset = HRMASDataset_Type_B(test_data)
    all_dataset = HRMASDataset_Type_B(all_data)
    all_dataloader = HRMASDataset_Type_B(all_dataset)
    train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True)
    vald_dataloader = DataLoader(vald_dataset, batch_size=len(vald_dataset), shuffle=True)
    test_dataloader = DataLoader(test_dataset, batch_size=len(test_dataset), shuffle=True)
    all_dataloader = DataLoader(all_dataset, batch_size=len(all_dataset))

    # initialize model and constructs
    model = Model(len(metabolite_names)).to(device)
    model.apply(initialize_weights)
    optimizer = torch.optim.Adam(model.parameters(), lr=hp_space["ETA"], weight_decay=hp_space["weight_decay"])
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=50, min_lr=1e-4, verbose=True, cooldown=25, factor=0.2)
    loss_fn_1 = nn.MSELoss()
    early_stopper = EarlyStopping(patience=100)
    print(summary(model))

    # training loop
    train_losses = {}
    for idx, metabolite in enumerate(metabolite_names):
        train_losses[metabolite] = []
    vald_losses = {}
    for idx, metabolite in enumerate(metabolite_names):
        vald_losses[metabolite] = []
    train_start = time.time()
    for epoch in range(num_epochs):
        # training
        train_loss = {}
        for idx, metabolite in enumerate(metabolite_names):
            train_loss[metabolite] = 0
        for batch_idx, (_, ppm_spectra, target, _) in enumerate(train_dataloader):
            ppm_spectra = ppm_spectra.float().to(device)
            target = target.float().to(device)
            optimizer.zero_grad()
            preds = list(model(ppm_spectra))
            loss = None
            for idx, metabolite in enumerate(metabolite_names):
                temp = loss_fn_1(preds[idx], target[:,idx].squeeze()) / max_quant_values_in_cohort[idx]
                train_loss[metabolite] += temp.item()
                if loss is None:
                    loss = temp
                else:
                    loss += temp
            loss.backward()
            optimizer.step()
        for idx, metabolite in enumerate(metabolite_names):
            train_loss[metabolite] /= (batch_idx+1)
            train_losses[metabolite].append(train_loss[metabolite])
        total_train_loss = 0
        for idx, metabolite in enumerate(metabolite_names):
            total_train_loss += train_loss[metabolite]
        # validation
        vald_loss = {}
        for idx, metabolite in enumerate(metabolite_names):
            vald_loss[metabolite] = 0
        for batch_idx, (_, ppm_spectra, target, _) in enumerate(vald_dataloader):
            ppm_spectra = ppm_spectra.float().to(device)
            target = target.float().to(device)
            with torch.no_grad():
                preds = list(model(ppm_spectra))
                loss = None
                for idx, metabolite in enumerate(metabolite_names):
                    temp = loss_fn_1(preds[idx], target[:,idx].squeeze()) / max_quant_values_in_cohort[idx]
                    vald_loss[metabolite] += temp.item()
                    if loss is None:
                        loss = temp
                    else:
                        loss += temp
        for idx, metabolite in enumerate(metabolite_names):
            vald_loss[metabolite] /= (batch_idx+1)
            vald_losses[metabolite].append(vald_loss[metabolite])
        total_vald_loss = 0
        for idx, metabolite in enumerate(metabolite_names):
            total_vald_loss += vald_loss[metabolite]

        scheduler.step(total_vald_loss)
        if epoch % 5 == 0:
            print(f"Epoch: {epoch}\tTrain Loss (MSE): {total_train_loss}\tValidation Loss (MSE): {total_vald_loss}")
        # early stopping
        early_stopper(total_vald_loss )
        if early_stopper.early_stop == True:
            print(f"Early stopping at epoch {epoch}")
            break
    train_end = time.time()

    # save model
    savepath = os.path.join(model_base_path, f"test_fold_{outer_idx}.pth")
    torch.save(model.state_dict(), savepath)

    # test model on test dataset
    test_start = time.time()
    for batch_idx, (_, ppm_spectra, target, _) in enumerate(test_dataloader):
        ppm_spectra = ppm_spectra.float().to(device)
        target = target.float().to(device)
        with torch.no_grad():
            preds = list(model(ppm_spectra))
            test_label = target
    test_end = time.time()
    test_preds = []
    for idx, metabolite in enumerate(metabolite_names):
        test_preds.append(preds[idx].detach().cpu().numpy())
    test_label = test_label.detach().cpu().numpy()
    test_pred = np.vstack(tuple(test_preds)).T

    # calculate and save metrics
    mse_ = mean_squared_error(test_label, test_pred, multioutput="raw_values")
    mae_ = mean_absolute_error(test_label, test_pred, multioutput="raw_values")
    r2_ = r2_score(test_label, test_pred, multioutput="raw_values")
    mape_ = mean_absolute_percentage_error(test_label, test_pred, multioutput="raw_values")
    absolute_percent_error_per_sample =  absolute_percentage_error(test_label, test_pred)
    for idx, metabolite in enumerate(metabolite_names):
        metrics["mse"][metabolite].append(mse_[idx])
        metrics["mae"][metabolite].append(mae_[idx])
        metrics["r2"][metabolite].append(r2_[idx])
        metrics["mape"][metabolite].append(mape_[idx])
        metrics["absolute_percentage_error"][metabolite] += absolute_percent_error_per_sample[:,idx].tolist()
    runtime["test"].append(test_end - test_start)
    runtime["train"].append(train_end - train_start)

    # plot and save loss graph
    plot_loss(train_losses, vald_losses, outer_idx, plot_base_path, metabolite_names, model_name)
    print(f"Test fold {outer_idx}:\tMAPE: {mape_}")

# save metrics and timings
for metabolite_ind, metabolite in enumerate(metabolite_names):
    with open(os.path.join(log_base_path, f"{metabolite}__mape.txt"), "w") as f:
        for mape in metrics["mape"][metabolite]:
            f.write("%f\n" % mape)
for metabolite_ind, metabolite in enumerate(metabolite_names):
    with open(os.path.join(log_base_path, f"{metabolite}__r2.txt"), "w") as f:
        for r2 in metrics["r2"][metabolite]:
            f.write("%f\n" % r2)
for metabolite_ind, metabolite in enumerate(metabolite_names):
    with open(os.path.join(log_base_path, f"{metabolite}__mse.txt"), "w") as f:
        for mse in metrics["mse"][metabolite]:
            f.write("%f\n" % mse)
for metabolite_ind, metabolite in enumerate(metabolite_names):
    with open(os.path.join(log_base_path, f"{metabolite}__mae.txt"), "w") as f:
        for mae in metrics["mae"][metabolite]:
            f.write("%f\n" % mae)
for metabolite_ind, metabolite in enumerate(metabolite_names):
    with open(os.path.join(log_base_path, f"{metabolite}__absolute_percentage_error.txt"), "w") as f:
        for ape in metrics["absolute_percentage_error"][metabolite]:
            f.write("%f\n" % ape)
with open(os.path.join(log_base_path, "train_time.txt"), "w") as f:
    for time in runtime["train"]:
        f.write("%f\n" % time)
with open(os.path.join(log_base_path, "test_time.txt"), "w") as f:
    for time in runtime["test"]:
        f.write("%f\n" % time)


text:
# Compatibility Python 2/3
from __future__ import division, print_function, absolute_import
from builtins import range
# ----------------------------------------------------------------------------------------------------------------------

from dotmap import DotMap
import matplotlib.pyplot as plt
import numpy as np

class linearTransformation():
    """
    Linear transformation
    y = A x + B
    """
    def __init__(self, A, B):
        """

        :param A: np.array [M, N]
        :param B: np.array [M]
        """
        self.A = np.array(A)
        self.B = np.array(B)
        # TODO: assert size A and B
        assert A.ndim == 2
        assert A.shape[0] == B.shape[0], 'Inconsistent size'

    def transform(self, X):
        """

        :param X: np.array [N, D]
        :return:
        """
        assert X.shape[0] == self.A.shape[1], 'Inconsistent size X %d - %d' % (X.shape[0], self.A.shape[1])
        return np.dot(self.A, X).T + self.B

    def invert(self, Y):
        """
        x = (y - B)/A
        :param Y: np.array [M, D]
        :return: X np.array []
        """
        assert Y.ndim == 2
        assert Y.shape[0] == self.A.shape[0], 'Inconsistent size Y %d - %d' % (Y.shape[0], self.A.shape[1])
        return np.linalg.solve(self.A, Y - np.matrix(self.B).T)

max_stars_repo_name: tholiao/opto

max_issues_repo_name: tholiao/opto

max_forks_repo_name: tholiao/opto


text:
import sys
import logging
import sys
from lxml import etree
from whoosh.fields import TEXT, ID, SchemaClass, KEYWORD
from whoosh import index
from parsing.combinators import ParseError
from pagerank.pagerank import PageRank, Adjacency, normalize_title
from .analyzer import WikimediaAnalyzer
import config
from parsing.compiler import Compiler, ParseTypes
import shutil
from parsing.utils import MalformedTag
from parsing.lexer import RedirectFound
from config import ASSETS_DATA, N_PROC
import networkx as nx

logger = logging.getLogger('preprocessing')

WAnalyzer = WikimediaAnalyzer(cachesize=-1)

class WikiSchema(SchemaClass):
    id = ID(stored=True)
    title = TEXT(stored=True, analyzer=WAnalyzer)
    text = TEXT(stored=True, analyzer=WAnalyzer)
    categories = KEYWORD(stored=True, analyzer=WAnalyzer, scorable=True, lowercase=True, commas=True)

# TODO singleton
class WikiIndex:
    def __init__(self, namespace='http://www.mediawiki.org/xml/export-0.10/'):
        self.schema = WikiSchema
        self.xml_parser = WikiXML(namespace=namespace)
        self.index = None
        self.reader = None

    def clear(self):
        pass

    def get(self, name='__indexdir', dump=config.DUMP_FOLDER, destructive=False):
        index_path = config.ROOT.joinpath(name)
        path = str(index_path)

        if destructive and index_path.exists():
            shutil.rmtree(path)
            while index_path.exists():
                pass

        if destructive or (not index_path.exists() or not index.exists_in(path)):
            try:
                index_path.mkdir()
                self.index = index.create_in(path, WikiSchema())
                logging.info('Index newly created, adding documents')
                self.build(directory=dump)
            except (FileExistsError, FileNotFoundError) as e:
                logger.error('Index already exist or parent not found')
                sys.exit(0)
        self.index = index.open_dir(path)
        print(' * Bootstrap index reader')
        self.reader = self.index.reader()

        return self

    def build(self, directory=config.DUMP_FOLDER):
        if not self.index:
            raise FileNotFoundError('Index not initialized')

        writer = self.index.writer(limitmb=2048, procs=N_PROC, multisegment=True)

        compiler = Compiler()

        categories = []
        reverse_graph = []

        file_adjlist = open(ASSETS_DATA / 'graphs' / 'graph.adjlist.tmp', "w")

        def parse_link(node, article):
            category = node.value.category()

            if category:
                categories.append(category.group())
            else:
                article_link = normalize_title(node.value.text)
                if article_link not in reverse_graph:
                    reverse_graph.append(article_link)

        miss = 0
        count = 0
        for wiki in directory.iterdir():
            if wiki.is_file() and wiki.stem.startswith('enwiki'):
                for root in self.xml_parser.from_xml(str(wiki)):
                    count += 1

                    if count > 20000:
                        writer.commit()
                        writer = self.index.writer(limitmb=2048, procs=N_PROC, multisegment=True)
                        count = 0

                    listener = None
                    try:
                        id, title, text = self.xml_parser.get(root)
                        listener = compiler.on(lambda node: parse_link(node, title.text), ParseTypes.LINK)
                        logger.info(f'{title.text} compiling')
                        article = compiler.compile(text.text)
                        writer.add_document(title=title.text, text=article, categories=','.join(categories), id=f'{id}')
                        logger.info(f'{title.text} indexed')
                        listener and listener()  # Remove listeners
                        article_title = normalize_title(title.text)
                        adj_graph_str = " ".join(reverse_graph)
                        file_adjlist.write(article_title + " " + adj_graph_str + "\n")
                        categories.clear()
                        reverse_graph.clear()
                    except (ParseError, MalformedTag, RedirectFound) as e:
                        miss += 1
                        listener and listener()  # Remove listeners
                        logger.warning(f'{title.text} {e.type}, skipping')
                        continue

        if miss > 0:
            logger.warning(f'{miss} articles ignored')

        writer.commit()

        file_adjlist.close()

        adj = Adjacency()

        adj.load_from_index()

        adj.write_adjlist_clean()

        pr = PageRank()

        pr.load_adjlist()

        pr.load_graphml()

        pr.generate_rank()

class WikiXML:
    """XML dump wikipedia parser"""

    def __init__(self, namespace):
        self._prefix = 'W'
        self.TITLE = '{0}:title'.format(self._prefix)
        self.TEXT = '{0}:revision/{0}:text'.format(self._prefix)
        self.ID = '{0}:id'.format(self._prefix)

        self.namespaces = {self._prefix: namespace}
        self._base_tag = f'{{{namespace}}}' + 'page'

    def from_xml(self, path):
        context = etree.iterparse(path, events=('end',), tag=self._base_tag, huge_tree=True)
        # TODO improve with iterchildren/iterdescendants instead of xpath
        for _, root in context:
            yield root
            while root.getprevious() is not None:
                del root.getparent()[0]
            root.clear()

    def title(self, root):
        return root.xpath(self.TITLE, namespaces=self.namespaces)[0]

    def text(self, root):
        return root.xpath(self.TEXT, namespaces=self.namespaces)[0]

    def id(self, root):
        return root.xpath(self.ID, namespaces=self.namespaces)[0]

    def get(self, root):
        # TODO use a high order func
        return self.id(root), self.title(root), self.text(root)

max_stars_repo_name: iwasingh/Wikoogle

max_issues_repo_name: iwasingh/Wikoogle

max_forks_repo_name: iwasingh/Wikoogle

reason: import networkx


text:
import numpy as np
import matplotlib.pyplot as plt

Map1=[[1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1],
     [1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1],
     [1,0,0,1,1,0,0,0,1,1,0,1,1,1,0,1,1,0,0,0,1,1,0,0,1],
     [1,0,1,1,1,0,0,0,1,1,0,1,0,1,0,1,1,0,0,0,1,1,1,0,1],
     [1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,1],
     [1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],
     [0,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0],
     [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
     [0,0,1,1,0,0,0,0,1,1,1,1,0,1,1,1,1,0,0,0,0,1,1,0,0],
     [0,0,1,1,0,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0],
     [0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],
     [0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0],
     [0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0],
     [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
     [0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0],
     [0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],
     [0,0,1,1,0,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0],
     [0,0,1,1,0,0,0,0,1,1,1,1,0,1,1,1,1,0,0,0,0,1,1,0,0],
     [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
     [1,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,1],
     [1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,1],
     [1,0,1,1,1,0,0,0,1,1,0,1,0,1,0,1,1,0,0,0,1,1,1,0,1],
     [1,0,0,1,1,0,0,0,1,1,0,1,1,1,0,1,1,0,0,0,1,1,0,0,1],
     [1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],
     [1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1]]

if __name__=='__main__':
    plt.imshow(Map1)
    plt.show()


text:
import unittest
import numpy as np
import sys
import mung.feature_helpers
import ltprg.data.feature_helpers
from os.path import join

input_data_dir = sys.argv[1]
output_feature_dir = sys.argv[2]
partition_file = sys.argv[3]

# Necessary to allow unittest.main() to work
del sys.argv[3]
del sys.argv[2]
del sys.argv[1]

np.random.seed(1)

class FeaturizeSUA(unittest.TestCase):

    # Constructs sequences of integer indices representing utterances
    def test_utterance_indices(self):
        print "Featurizing utterance indices"
        mung.feature_helpers.featurize_path_enum_seqs(
            input_data_dir, # Source data set
            join(output_feature_dir, "utterance_idx"), # Output directory
            partition_file, # Data partition
            lambda d : d.get("id"), # Function that partitions the data
            "utterance_idx", # Name of the feature
            ["utterance.nlp.token_strs.strs"], # JSON path into data examples
            30, # Maximum utterance length
            token_fn=lambda x : x, # Function applied to tokens to construct the vocabulary
            indices=True) # Indicates that indices will be computed instead of one-hot vectors

    # Constructs sequences of integer indices representing utterances
    def test_premise_indices(self):
        print "Featurizing premise indices"
        mung.feature_helpers.featurize_path_enum_seqs(
            input_data_dir, # Source data set
            join(output_feature_dir, "premise_idx"), # Output directory
            partition_file, # Data partition
            lambda d : d.get("id"), # Function that partitions the data
            "premise_idx", # Name of the feature
            ["state.nlp.token_strs.strs"], # JSON path into data examples
            30, # Maximum utterance length
            token_fn=lambda x : x, # Function applied to tokens to construct the vocabulary
            indices=True) # Indicates that indices will be computed instead of one-hot vectors

    # Constructs an index of the target color index (0, 1, or 2) that
    # the speaker observed
    def test_speaker_target_indices(self):
        print "Featurizing speaker target indices"
        mung.feature_helpers.featurize_path_scalars(
            input_data_dir,
            join(output_feature_dir, "speaker_target_idx"),
            partition_file,
            lambda d : d.get("id"),
            "speaker_target_idx",
            ["state.sTarget"])

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: forkunited/ltprg

max_issues_repo_name: forkunited/ltprg

max_forks_repo_name: forkunited/ltprg


text:
## Required packages

import os
import sys
import pytest

import numpy as np
import pandas as pd
import inspect

sys.path.insert(0, os.path.abspath("../"))
sys.path.insert(0, os.path.abspath("p_toolkit/"))

print(sys.path)

import core

###Functionality tests

def test_p_bonnferoni_helper():
    """
    The purpose of this test is performing numerical tests on the outputs from the p_bonnferoni_helper function. We test
    different combinations of inputs and check their corresponding outputs:

    - The first block tests the Bonferroni adjustment.
    - The second one handls the cases with one of the values higher than 1, impossible for p-values.
    - The third block checks how the maximum value of 1 is handled in the function.
    """
    ###basic tests
    assert p_bonferroni_helper([0.07]) == [0.07], "a single value doesn't change under bonnferoni"
    assert p_bonferroni_helper([0.07,0.2]) == [0.14,0.4], "p_bonnferoni_helper((0.07,0.2))== (0.14,0.4)"
    assert p_bonferroni_helper([0.2,0.07]) == [0.4,0.14], "p_bonnferoni_helper((0.2,0.07))== (0.4,0.14)"
    assert p_bonferroni_helper([0.01, 0.02, 0.03]) == [0.03, 0.06,0.09], "p_bonnferoni_helper((0.01, 0.02, 0.03))==(0.03, 0.06,0.09)"

    ### all values are between 0 and 1
    with pytest.raises(ValueError):
          p_bonferroni_helper([-3])
          p_bonferroni_helper([-3, .05])
          p_bonferroni_helper([.05, -3])
          p_bonferroni_helper([8])
          p_bonferroni_helper([8, .05])
          p_bonferroni_helper([.05,8])

    ###maximum return is 1
    assert p_bonferroni_helper([0.01, 0.7]) == [0.02,1], "test_p_bonnferoni_helper has max of 1"

	###formatting tests
	# https://github.com/datalyze-solutions/pandas-qt/blob/master/tests/test_DataFrameModel.py
	# https://data-lessons.github.io/library-python/03-data-types-and-format/
	assert not data.empty
	assert data is dataFrame
	assert pvals == dtype(int32)
	assert alpha == dtype(int32)

def test_p_bonn():
    """
    The purpose of this test is performing numerical tests on the outputs from the p_bh_helper function. We test
    different combinations of inputs and check their corresponding outputs:

    - The first block tests the BH adjustment.
    - The second one handles the cases with one of the values higher than 1, impossible for p-values.
    """
    assert p_bh_helper([0.07]) == [0.07], "a single value doesn't change under bh"
    assert p_bh_helper([0.07,0.2]) == [0.14,0.2], "p_bh_helper((0.07,0.2))== (0.14,0.2)"
    assert p_bh_helper([0.2,0.07]) == [0.2,0.14], "p_bh_helper((0.2,0.07))== (0.2,0.14)"
    assert p_bh_helper([0.01, 0.02, 0.03]) == [0.03, 0.03,0.03], "p_bh_helper((0.01, 0.02, 0.03))==(0.03, 0.03,0.03)"
    assert p_bh_helper([0.02, 0.03, 0.01]) == [0.03, 0.03,0.03], "p_bh_helper((0.02, 0.03, 0.01))==(0.03, 0.03,0.03)"
    assert p_bh_helper([.02,.12,.24,.56,.6]) == [0.1,0.6, 0.4,0.7, 0.6], "p_bh_helper((.02,.12,.24,.56,.6))==(0.1,0.6, 0.4,0.7, 0.6)"
    assert p_bh_helper([0.04, 0.04, 0.04,0.08]) == [0.16,0.16,0.16,0.08], "p_bh_helper treats rep[eated values correctly"

    ### all valid probabilities
    with pytest.raises(ValueError):
     p_bh_helper([-3])
     p_bh_helper([-3, .05])
     p_bh_helper([.05, -3])
     p_bh_helper([8])
     p_bh_helper([8, .05])
     p_bh_helper([.05,8])

	###formatting tests
	# https://github.com/datalyze-solutions/pandas-qt/blob/master/tests/test_DataFrameModel.py
	assert not data.empty
	assert data is dataFrame
	assert BH_significant == type(numpy.bool_)
	assert Bonf_significant == type(numpy.bool_)

	###formatting tests
	# https://github.com/datalyze-solutions/pandas-qt/blob/master/tests/test_DataFrameModel.py
	assert not data.empty
	assert data is dataFrame
	assert pvals == dtype(int32)
	assert alpha == dtype(int32)

####p_adjust functionality
def test_p_adjust():
    """
    The purpose of this test is evaluating the p_adjust with more real world data using Pandas dataframes under
    different environments.
    """
    ##basic vector functionality"
    d = { "p_value": [0.07], "adjusted": [0.07]}
    df = pd.DataFrame(data = d)
    assert p_adjust(data =[0.07], method = "bonf") == df, "p_adjust 1 value vector for bonnferoni"
    assert p_adjust(data =[0.07], method = "bh") == df, "p_adjust single value vector for bh"

    d = { "p_value": [0.07,0.2], "adjusted": [0.14,0.4]}
    df = pd.DataFrame(data = d)
    assert p_adjust(data =[0.07,0.2], method = "bonf") == df, "p_adjust 2 values vector for bonnferoni"

    d = { "p_value": [0.07,0.2], "adjusted": [0.14,0.2]}
    df = pd.DataFrame(data = d)
    assert p_adjust(data =[0.07,0.2], method = "bh") == df, "p_adjust 2 values vector value for bh"

    ###basic dataframe fuinctionality
    d = {"test":["test 1"], "p_value": [0.05]}
    df = pd.DataFrame(data = d)
    ad = {"test":["test 1"], "p_value": [0.05],"adjusted": [0.05]}
    adf = pd.DataFrame(data =ad)
    assert p_adjust(data =df, method = "bonf") == adf, "bonferroni single value df under p_adjust"
    assert p_adjust(data =df, method = "bh") == adf, "bh single value df under p_adjust"

    d = {"test":["test 1", "test 2"], "p_value": [0.07,0.2]}
    df = pd.DataFrame(data = d)
    ad = {"test":["test 1", "test 2"], "p_value": [0.07,0.2],"adjusted": [0.14, 0.4]}
    adf = pd.DataFrame(data =ad)
    assert p_adjust(data =df, method = "bonf") == adf, "bonferroni 2 value df under p_adjust"

    ad = {"test":["test 1", "test 2"], "p_value": [0.07,0.2],"adjusted": [0.14, 0.2]}
    adf = pd.DataFrame(data =ad)
    assert p_adjust(data =df, method = "bh") == adf, "bh 2 value df under p_adjust"

	###formatting tests
	# https://github.com/datalyze-solutions/pandas-qt/blob/master/tests/test_DataFrameModel.py
	assert not data.empty
	assert data is dataFrame

def test_p_methods():
    """
    The purpose of this test is evaluating the p_adjust with more real world data using Pandas dataframes under
    different environments.
    """
    d = { "p_value": [0.07], "Bonnferoni_critical_value": [0.05],"Bonnferoni_reject" :[False],"BH_critical_value": [0.05], "BH_reject":[False] }
    df = pd.DataFrame(data = d)
    assert p_methods(data =[0.07], alpha =0.05) == df, "p_methods 1 value vector, FALSE"

    d = { "p_value": [0.01], "Bonnferoni_critical_value": [0.05],"Bonnferoni_reject" :[True],"BH_critical_value": [0.05], "BH_reject":[True] }
    df = pd.DataFrame(data = d)
    assert p_methods(data =[0.01], alpha =0.05) == df, "p_methods 1 value vector, TRUE"

    d = {"Test":["test 1", "test 2"], "p_value": [0.01,0.03], "Bonnferoni_critical_value": [0.025,0.025],"Bonnferoni_reject" :[True,False],"BH_critical_value": [0.025,0.05], "BH_reject":[True, True] }
    df = pd.DataFrame(data = d)
    assert p_methods(data =[0.01,0.03], alpha =0.05) == df, "p_methods 2 values vector "

    ###dataframe tests
    d = {"Test":["test 1"], "p_value": [0.01] }
    df = pd.DataFrame(data = d)
    ad = {"Test":["test 1"], "p_value": [0.01], "Bonnferoni_critical_value": [0.05],"Bonnferoni_reject" :[True],"BH_critical_value": [0.05], "BH_reject":[True] }
    adf = pd.DataFrame(data = ad)
    assert p_methods(data =df, column = "p_value", alpha =0.05) == adf, "p_methods 2 values dataframe "

    d = {"Test":["test 1"], "p_value": [0.1] }
    df = pd.DataFrame(data = d)
    ad = {"Test":["test 1"], "p_value": [0.1], "Bonnferoni_critical_value": [0.05],"Bonnferoni_reject" :[False],"BH_critical_value": [0.05], "BH_reject":[False] }
    adf = pd.DataFrame(data = ad)
    assert p_methods(data =df,column = "p_value", alpha =0.05) == adf, "p_methods 2 values dataframe "

    d = {"Test":["test 1", "test 2"], "p_value": [0.01,0.03] }
    df = pd.DataFrame(data = d)
    ad = {"Test":["test 1", "test 2"], "p_value": [0.01,0.03], "Bonnferoni_critical_value": [0.025,0.025],"Bonnferoni_reject" :[True,False],"BH_critical_value": [0.025,0.05], "BH_reject":[True, True] }
    adf = pd.DataFrame(data = ad)
    assert p_methods(data =df, column = "p_value", alpha =0.05) == adf, "p_methods 2 values dataframe "

	###formatting tests
	# https://github.com/datalyze-solutions/pandas-qt/blob/master/tests/test_DataFrameModel.py
	assert not data.empty
	assert data is dataFrame

###Plotting tests

def test_p_qq():
    """
    The purpose of this test is evaluating if the matplotlib object created with p_qq has the correct layers compared to the required
    plot. The test will cover the same things we checked with R:

    - The output is a matplotlib object.
    - The axis labels are correct. In this case if the labels are "Observed -log(p)" and "Expected -log(p)".
    - The chart type used is correct. In this case, if it is a scatter plot combined with a line.
    - The series used for plotting are the correct ones. "theoretical_pvalues" and "theoretical_pvalues".

    ## We couldn't find a way to extract information from the axes on the matplotlib object. With R, everything is stored
    ## on a list. We will get deeper this week to solve this issue and add the test announced before.
    """

    ##fig = p_qq(X,y)
    ##assert type(fig) == matplotlib.figure.Figure, "the object includes a matplotlib figure "

def test_p_plot():
    """
    The purpose of this test is evaluating if the matplotlib object created with p_plot has the correct layers compared to the required
    plot. The test will cover the same things we checked with R:

    - The output is a matplotlib object.
    - The axis labels are correct. In this case if the labels are "p(k)" and "k".
    - The chart type used is correct. In this case, if it is a scatter plot combined with two lines.
    - The series used for plotting are the correct ones. "pvalue" and "k".

    ## We couldn't find a way to extract information from the axes on the matplotlib object. With R, everything is stored
    ## on a list. We will get deeper this week to solve this issue and add the test announced before.
    """

    ##fig = p_plot(X,y)
    ##assert type(fig) == matplotlib.figure.Figure, "the object includes a matplotlib figure "