text:
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for data utils."""

import math
import os

from absl.testing import absltest
from absl.testing import parameterized
from language.mentionmemory.utils import data_utils
import numpy as np
import tensorflow.compat.v2 as tf

class DataTest(parameterized.TestCase):

  data_files = ['test1.tfrecord', 'test2.tfrecord']
  dataset_size = 32
  data_dim = 16
  samples_per_example = 4
  name_to_features = {
      'x': tf.io.FixedLenFeature([data_dim], tf.int64),
  }

  @staticmethod
  def preprocess_fn(sample):
    return sample

  def _get_test_data_path(self, file_name):
    path = os.path.join('language/mentionmemory/utils/testdata', file_name)
    return path

  def setUp(self):
    super().setUp()

    self.decode_fn = data_utils.make_decode_fn(self.name_to_features,
                                               self.samples_per_example)

    self.data1 = np.random.randint(
        0,
        10,
        size=(self.dataset_size // self.samples_per_example,
              self.data_dim * self.samples_per_example),
        dtype=np.int64)

    self.data2 = np.random.randint(
        0,
        10,
        size=(self.dataset_size // self.samples_per_example,
              self.data_dim * self.samples_per_example),
        dtype=np.int64)

    self.workdir_obj = self.create_tempdir()
    self.workdir = self.workdir_obj.full_path
    self.data_patterns = [
        os.path.join(self.workdir, filename) for filename in self.data_files
    ]

    for dataset_idx, data in enumerate([self.data1, self.data2]):
      writer = tf.io.TFRecordWriter(self.data_patterns[dataset_idx])
      for row in data:
        features = tf.train.Features(feature={
            'x': tf.train.Feature(int64_list=tf.train.Int64List(value=row)),
        })

        record_bytes = tf.train.Example(features=features).SerializeToString()
        writer.write(record_bytes)

  @parameterized.parameters(
      (1, 1, 1, 0, False),
      (1, 1, 1, 0, True),
      (1, 1, 3, 0, True),
      (1, 1, 3, 2, True),
      (1, 3, 3, 2, True),
      (3, 2, 2, 1, True),
  )
  def test_reproduce_eval_data(
      self,
      per_device_batch_size,
      local_device_count,
      host_count,
      host_id,
      pad_eval,
  ):
    """Check whether loading dataset gets back original data."""

    loaded_data = data_utils.load_dataset(
        patterns=self.data_patterns[0],
        decode_fn=self.decode_fn,
        preprocess_fn=DataTest.preprocess_fn,
        collater_fn=lambda x: x,
        is_training=False,
        per_device_batch_size=per_device_batch_size,
        local_device_count=local_device_count,
        host_count=host_count,
        host_id=host_id,
        pad_eval=pad_eval,
    )
    data_list = list(loaded_data)
    recovered_data = np.array([batch['x'] for batch in data_list])

    # Compute how many non-zero elements the host should have
    host_bsz = per_device_batch_size * local_device_count
    total_bsz = host_bsz * host_count
    total_remainder = self.dataset_size % total_bsz
    remainder_per_host = total_remainder // host_count
    host_remainder = total_remainder % host_count
    n_nonzero_elements = self.dataset_size // total_bsz * host_bsz + remainder_per_host
    if host_id < host_remainder:
      n_nonzero_elements += 1
    n_batches = math.ceil(self.dataset_size / total_bsz)

    # Check data has correct shape
    self.assertEqual(
        recovered_data.shape,
        (n_batches, local_device_count, per_device_batch_size, self.data_dim))
    reshaped_data = recovered_data.reshape(-1, self.data_dim)

    # # Check if all samples in dataset are present in original data
    comparison_data = reshaped_data[:n_nonzero_elements]
    reshaped_original_data = self.data1.reshape(-1, self.data_dim)
    arr_eq = np.expand_dims(comparison_data,
                            1) == np.expand_dims(reshaped_original_data, 0)
    self.assertTrue(np.all(np.any(np.all(arr_eq, axis=-1), axis=1), axis=0))

    if pad_eval:
      # Check if nonzero sample weights correspond to expected nonzero elements
      sample_weights = np.array(
          [batch['sample_weights'] for batch in data_list]).reshape(-1)
      self.assertTrue(np.all(sample_weights[:n_nonzero_elements] == 1))
      self.assertTrue(np.all(sample_weights[n_nonzero_elements:] == 0))

      # Check if values are zero for sample weight zero
      self.assertTrue(np.all(reshaped_data[n_nonzero_elements:] == 0))

  def test_reproduce_train_data(self):
    loaded_data = data_utils.load_dataset(
        patterns=self.data_patterns[0],
        decode_fn=self.decode_fn,
        preprocess_fn=DataTest.preprocess_fn,
        collater_fn=lambda x: x,
        is_training=False,
        per_device_batch_size=1,
        local_device_count=1,
        host_count=1,
        host_id=0,
    )
    data_iter = iter(loaded_data)
    recovered_data = np.array(
        [data_iter.get_next()['x'] for _ in range(self.dataset_size)])

    recovered_data = recovered_data.reshape(
        self.dataset_size // self.samples_per_example,
        self.data_dim * self.samples_per_example)
    # We shuffle, so we just check that all arrays are present without regard
    # to order.
    arr_eq = np.expand_dims(recovered_data, 1) == np.expand_dims(self.data1, 0)
    self.assertTrue(np.all(np.any(np.all(arr_eq, axis=-1), axis=1), axis=0))

  @parameterized.parameters(
      {'is_training': True},
      {'is_training': False},
  )
  def test_training_data_deterministic(self, is_training=True):
    """Make sure loading data is deterministic given seed."""
    data_arrays = []
    for _ in range(2):
      loaded_dataset = data_utils.load_dataset(
          patterns=self.data_patterns,
          decode_fn=self.decode_fn,
          preprocess_fn=DataTest.preprocess_fn,
          collater_fn=lambda x: x,
          is_training=is_training,
          per_device_batch_size=4,
          local_device_count=1,
          host_count=2,
          host_id=0,
          seed=0,
      )
      data_iterator = iter(loaded_dataset)
      data_array = np.array([
          data_iterator.get_next()['x'] for _ in range(self.dataset_size // 4)
      ])
      data_arrays.append(data_array)

    self.assertTrue(np.array_equal(*data_arrays))

  @parameterized.parameters(
      ('eae_paper-00000-of-00001', 1, 1, False),
      ('eae_paper-00000-of-00001', 1, 20, False),
      ('eae_paper-00000-of-00001', 1, 1, True),
      ('eae_paper-00000-of-00001', 1, 4, True),
      ('mtb.v5-00000-of-00001', 4, 1, False),
      ('mtb.v5-00000-of-00001', 4, 10, False),
      ('mtb.v5-00000-of-00001', 4, 1, True),
      ('mtb.v5-00000-of-00001', 4, 2, True),
  )
  def test_load_real_data(self, filename, samples_per_example,
                          per_device_batch_size, is_training):
    num_total_samples = 100
    self.assertEqual(num_total_samples % per_device_batch_size, 0)
    data_path = self._get_test_data_path(filename)

    name_to_features = {
        'text_ids': tf.io.FixedLenFeature(128, tf.int64),
        'text_mask': tf.io.FixedLenFeature(128, tf.int64),
        'dense_span_starts': tf.io.FixedLenFeature(128, tf.int64),
        'dense_span_ends': tf.io.FixedLenFeature(128, tf.int64),
        'dense_mention_mask': tf.io.FixedLenFeature(128, tf.int64),
        'dense_mention_ids': tf.io.FixedLenFeature(128, tf.int64),
    }

    decode_fn = data_utils.make_decode_fn(name_to_features, samples_per_example)

    loaded_dataset = data_utils.load_dataset(
        patterns=data_path,
        decode_fn=decode_fn,
        preprocess_fn=DataTest.preprocess_fn,
        collater_fn=lambda x: x,
        is_training=is_training,
        per_device_batch_size=per_device_batch_size,
        local_device_count=1,
        host_count=1,
        host_id=0,
        seed=0,
    )
    data_list = iter(loaded_dataset)
    for _ in range(num_total_samples // per_device_batch_size):
      batch = next(data_list)
      self.assertIsNotNone(batch)

  @parameterized.parameters(
      (1, False),
      (2, False),
      (20, False),
      (100, False),
      (1, True),
      (2, True),
      (20, True),
      (100, True),
  )
  def test_load_multi_dataset_real_data(self, per_device_batch_size,
                                        is_training):
    self.assertEqual(100 % per_device_batch_size, 0)
    num_total_samples = 200
    datasets_config = [
        {
            'patterns': self._get_test_data_path('eae_paper-00000-of-00001'),
            'samples_per_example': 1,
        },
        {
            'patterns': self._get_test_data_path('mtb.v5-00000-of-00001'),
            'samples_per_example': 4,
        },
    ]
    name_to_features = {
        'text_ids': tf.io.FixedLenFeature(128, tf.int64),
        'text_mask': tf.io.FixedLenFeature(128, tf.int64),
        'dense_span_starts': tf.io.FixedLenFeature(128, tf.int64),
        'dense_span_ends': tf.io.FixedLenFeature(128, tf.int64),
        'dense_mention_mask': tf.io.FixedLenFeature(128, tf.int64),
        'dense_mention_ids': tf.io.FixedLenFeature(128, tf.int64),
    }
    loaded_dataset = data_utils.load_multi_dataset(
        datasets_config=datasets_config,
        name_to_features=name_to_features,
        preprocess_fn=DataTest.preprocess_fn,
        collater_fn=lambda x: x,
        is_training=is_training,
        per_device_batch_size=per_device_batch_size,
        local_device_count=1,
        host_count=1,
        host_id=0,
        seed=0,
    )
    data_list = iter(loaded_dataset)
    for _ in range(num_total_samples // per_device_batch_size):
      batch = next(data_list)
      self.assertIsNotNone(batch)

class LoadTest(absltest.TestCase):
  n_splits = 8
  data_per_split = 16

  def test_loaded_arrays_match_saved(self):
    workdir_obj = self.create_tempdir()
    workdir = workdir_obj.full_path
    pattern = os.path.join(workdir, 'array*')
    array = np.random.rand(self.n_splits * self.data_per_split)
    save_array = array.reshape(self.n_splits, self.data_per_split)
    for split in range(self.n_splits):
      path = os.path.join(workdir, 'array' + str(split))
      np.save(path, save_array[split])

    self.assertTrue(
        np.all(data_utils.load_sharded_array(pattern, 1, 0) == array))
    self.assertTrue(
        np.all(
            data_utils.load_sharded_array(pattern, 1, 1) ==
            save_array[1:].reshape(-1)))
    self.assertTrue(
        np.all(
            data_utils.load_sharded_array(pattern, self.n_splits, 0) ==
            save_array[0]))

class SaveShardedArrayTest(parameterized.TestCase):

  @parameterized.parameters(
      ((100, 3), 16, 1),
      ((100, 3), 16, 8),
      ((100,), 16, 1),
      ((100,), 16, 2),
      ((100, 3, 5), 16, 1),
      ((100, 3, 5), 16, 16),
      ((100, 3), 100, 1),
      ((100, 3), 100, 10),
      ((100,), 99, 1),
      ((100,), 99, 11),
  )
  def test_save_sharded_array(self, array_shape, num_shards, stride):
    shard_size_divisible = 3

    self.assertEqual(num_shards % stride, 0)

    arrays_per_offset = [np.random.random(array_shape) for _ in range(stride)]

    tmp_dir = self.create_tempdir()
    prefix = os.path.join(tmp_dir.full_path, 'test')

    for offset in range(stride):
      data_utils.save_sharded_array(arrays_per_offset[offset], prefix,
                                    num_shards, stride, offset,
                                    shard_size_divisible)

    loaded_array_first_dim = None
    for offset in range(stride):
      loaded_array = data_utils.load_sharded_array(
          prefix + '-?????-of-%05d' % num_shards, stride, offset)
      all_axis_except_first = list(range(1, len(array_shape)))
      sum_all_except_first_axis = np.apply_over_axes(np.sum,
                                                     np.abs(loaded_array),
                                                     all_axis_except_first)
      sum_all_except_first_axis = sum_all_except_first_axis.reshape(-1)
      is_not_pad = sum_all_except_first_axis > 0
      actual_array = loaded_array[is_not_pad]
      self.assertTrue(np.all(actual_array == arrays_per_offset[offset]))

      if loaded_array_first_dim is None:
        loaded_array_first_dim = loaded_array.shape[0]
      else:
        self.assertEqual(loaded_array_first_dim, loaded_array.shape[0])

if __name__ == '__main__':
  absltest.main()

max_stars_repo_name: urikz/language

max_issues_repo_name: urikz/language

max_forks_repo_name: urikz/language


text:
# -*- coding:utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

class Blackjack(object):
    def __init__(self):
        self.action_hit = 1
        self.action_stop = 0
        self.actions = [self.action_hit, self.action_stop]
        self.policy_player = np.ones(22, dtype=int)
        self.policy_dealer = np.ones(22, dtype=int)

        self.policy_player[20:] = self.action_stop
        self.policy_dealer[17:] = self.action_stop

    def target_policy_player(self, sum_player, usable_ace, dealer_card):
        return self.policy_player[sum_player]

    def behavior_policy_player(self, sum_player, usable_ace, dealer_card):
        rand_num = np.random.binomial(1, 0.5)
        if rand_num:
            return self.action_hit
        else:
            return self.action_stop

    def get_card(self):
        card_num = np.random.randint(1, 14)
        return min(card_num, 10)

    def initial_game(self, initial_state=None):
        sum_player = 0   #sum of player's card
        usable_ace_player = False  # type: bool #whether player count ace as 11 or not
        card1_dealer = 0  # the first card of dealer is showing card
        card2_dealer = 0  # the second card of deale

        if initial_state is None:
            num_of_ace = 0  # the number of ace card
            # if the sum of player's card less than 12, always hits
            while sum_player < 12:
                card = self.get_card()
                # if getting an ace card, counting it as 11
                if card == 1:
                    num_of_ace += 1
                    card = 11
                    usable_ace_player = True
                sum_player += card
            if sum_player > 21:
                sum_player -= 10
                if num_of_ace == 1:
                    usable_ace_player = False

            card1_dealer += self.get_card()
            card2_dealer += self.get_card()
        else:
            sum_player, usable_ace_player, card1_dealer = initial_state
            card2_dealer = self.get_card()
        state = [sum_player, usable_ace_player, card1_dealer]  # initialize the state of game
        return state, card2_dealer

    def generate_episode(self, policy_player, initial_state=None, initial_action=None):
        #@policy_player: type:function
        sum_dealer = 0  # sum of dealer's card
        trajectory_player = []  # the player's trajectory
        usable_ace_dealer = False  #type: bool #whether dealer count ace as 11 or not

        state, card2_dealer = self.initial_game(initial_state)
        [sum_player, usable_ace_player, card1_dealer] = state
        # @sum_player: sum of player's card
        # @usable_ace_player:  type: bool #whether player count ace as 11 or not
        # @card1_dealer: the first card of dealer is showing card
        # @card2_dealer: the second card of dealer

        if card1_dealer == 1 and card2_dealer == 1:
            sum_dealer += 11 + 1
            usable_ace_dealer = True
        elif card1_dealer == 1 and card2_dealer != 1:
            sum_dealer += 11 + card2_dealer
            usable_ace_dealer = True
        elif card1_dealer != 1 and card2_dealer == 1:
            sum_dealer += card1_dealer + 11
            usable_ace_dealer = True
        else:
            sum_dealer += card1_dealer + card2_dealer

        while True:
            if initial_action is not None:
                action = initial_action
                initial_action = None
            else:
                action = policy_player(sum_player, usable_ace_player, card1_dealer)
                #get an action base on current sum of player's card
            trajectory_player.append([(sum_player, usable_ace_player, card1_dealer), action])

            if action == self.action_stop:
                break

            new_card = self.get_card()
            if new_card == 1 and sum_player+11 < 21:
                sum_player += 11
                usable_ace_player = True
            else:
                sum_player += new_card
            if sum_player > 21:
                if usable_ace_player:
                    sum_player -= 10
                    usable_ace_player = False
                else:
                    return state, -1, trajectory_player

        while True:
            action = self.policy_dealer[sum_dealer]   #get an action base on current sum of dealer's card
            if action == self.action_stop:
                break
            new_card = self.get_card()
            if new_card == 1 and sum_dealer+11 < 21:
                sum_dealer += 11
                usable_ace_dealer = True
            else:
                sum_dealer += new_card

            if sum_dealer > 21:
                if usable_ace_dealer:
                    sum_dealer -= 10
                    usable_ace_dealer = False
                else:
                    return state, 1, trajectory_player

        if sum_player > sum_dealer:
            return state, 1, trajectory_player
        elif sum_player == sum_dealer:
            return state, 0, trajectory_player
        else:
            return state, -1, trajectory_player

    def off_policy_MC(self, episodes=10000):
        sum_player = 13
        usable_ace = True
        card1 = 2

        rhos = []
        rewards = []
        initial_state = [sum_player, usable_ace, card1]

        for ite in np.arange(episodes):
            numerator = 1   #the numerator of importance-sampling ratio
            denominator = 1   #the denominator of importance-sampling ratio

            _, reward, trajectory = self.generate_episode(self.behavior_policy_player, initial_state)
            for (sum_player, usable_ace, dealer_showing), action in trajectory:
                if action == self.target_policy_player(sum_player, usable_ace, dealer_showing):
                    denominator *= 0.5
                else:
                    numerator = 0
                    break
            rho = numerator / denominator
            rhos.append(rho)
            rewards.append(reward)

        rhos = np.asarray(rhos)
        rewards = np.asarray(rewards)
        weighted_rewards = rhos * rewards
        rhos = np.add.accumulate(rhos)   #to accumulate the importance-sampling ratio
        weighted_returns = np.add.accumulate(weighted_rewards)   #to accumulate the reward

        ordinary_importance_sampling = weighted_returns / np.arange(1, episodes+1)
        weighted_importance_sampling = weighted_returns / (rhos+1e-7)   #to avoid dividing zero

        return ordinary_importance_sampling, weighted_importance_sampling

if __name__ == "__main__":
    true_value = -0.27726
    iteration = 10000
    runs = 100
    error_ordinary_sampling = np.zeros(iteration)
    error_weighted_sampling = np.zeros(iteration)

    blackjack_game = Blackjack()
    print("The Blackjack Game is starting......")
    for iters in np.arange(runs):
        ordinary_sampling, weighted_sampling = blackjack_game.off_policy_MC(episodes=iteration)
        error_ordinary_sampling += np.power(ordinary_sampling-true_value, 2)
        error_weighted_sampling += np.power(weighted_sampling-true_value, 2)
    print("Game Over !!!\n")
    error_ordinary_sampling /= runs
    error_weighted_sampling /= runs

    print("Starting drawing......")
    plt.plot(np.arange(1, iteration+1), error_ordinary_sampling, label="Ordinary importance sampling")
    plt.plot(np.arange(1, iteration+1), error_weighted_sampling, label="Weighted importance sampling")
    plt.xlabel("Episodes (log scale)")
    plt.ylabel("Mean Square Error")
    plt.xscale("log")
    plt.legend()
    plt.savefig("./images/Figure5-3.jpg")
    plt.show()

    print("Completed !!! You can check it in the 'images' directory.")
    plt.close()

ext: py

max_stars_repo_name: liloganle/Reinforcement-Learning

max_issues_repo_name: liloganle/Reinforcement-Learning

max_forks_repo_name: liloganle/Reinforcement-Learning

reason: import numpy


text:
import numpy as np
import pandas as pd

"""get_data definition does the following

1. Read the train/properties data
2. Merge the datasets
3. Remove the duplicate parcels
4. Convert the categorical features to "object" data type.

"""

def read_data(data_string):
    """
    Read the train/properties data set

    Parameters:
    data_string -- "train_2016",  "train_2017", "properties_2016", "properties_2017"

    Returns:
    df - dataframe from the "data/raw" folder

    """
    df = pd.read_csv("../data/raw/{0}.csv".format(data_string))

    return df

def remove_duplicate_parcels(df):
    """
    Remove the records with duplicate parcelid in the merged train data set.

    Parameters:
    df -- merged data frame

    Returns:
    unique_df -- a dataframe with unique parcelid's

    """
    parcel_count = df.groupby(["parcelid"]).size()
    unique_parcel = df[df["parcelid"].isin(parcel_count[parcel_count == 1].index)]
    duplicated_parcel = df[df["parcelid"].isin(parcel_count[parcel_count > 1].index)]
    duplicated_parcel_made_unique = (
        duplicated_parcel.sample(frac=1, random_state=42).groupby(["parcelid"]).head(1)
    )
    unique_df = pd.concat([unique_parcel, duplicated_parcel_made_unique], axis=0)

    return unique_df

def get_data(data_string):
    """
    Read the train/test dataset and merge with properties data set
    and remove duplicate parcelid's in train

    Parameters:
    data_string -- "train" or "test"

    Returns:
    X, y -- a tuple of dataframe X and Series y

    """
    year = 2016 if data_string == "train" else 2017

    train = read_data("train_{0}".format(year))
    properties = read_data("properties_{0}".format(year))
    merged = pd.merge(train, properties, on="parcelid", how="left")

    if data_string == "train":
        merged = remove_duplicate_parcels(merged)

    y = merged["logerror"]
    merged = merged.drop(columns=["logerror"], axis=1)

    id_col = ["parcelid"]
    cat_cols = [
        "airconditioningtypeid",
        "architecturalstyletypeid",
        "buildingclasstypeid",
        "buildingqualitytypeid",
        "decktypeid",
        "fips",
        "fireplaceflag",
        "hashottuborspa",
        "heatingorsystemtypeid",
        "pooltypeid10",
        "pooltypeid2",
        "pooltypeid7",
        "propertycountylandusecode",
        "propertylandusetypeid",
        "propertyzoningdesc",
        "rawcensustractandblock",
        "censustractandblock",
        "regionidcounty",
        "regionidcity",
        "regionidzip",
        "regionidneighborhood",
        "storytypeid",
        "typeconstructiontypeid",
        "yearbuilt",
        "assessmentyear",
        "taxdelinquencyflag",
        "taxdelinquencyyear",
    ]

    # convert all categorical variables to categorical type
    for col in id_col + cat_cols:
        merged[col] = merged[col].astype("object")

    # convert transactiondate column to datetime type
    merged["transactiondate"] = pd.to_datetime(merged["transactiondate"])

    return merged, y

max_stars_repo_name: firozgit/Zillow-Home-Value-Prediction

max_issues_repo_name: firozgit/Zillow-Home-Value-Prediction

max_forks_repo_name: firozgit/Zillow-Home-Value-Prediction


text:
from sktime.utils.time_series import time_series_slope
from sktime.transformers.tests.test_RandomIntervalFeatureExtractor import generate_df_from_array
import numpy as np
from scipy.stats import linregress

N_ITER = 100

def test_time_series_slope():
    Y = np.array(generate_df_from_array(np.random.normal(size=10), n_rows=100).iloc[:, 0].tolist())
    y = Y[0, :]

    # Compare with scipy's linear regression function
    x = np.arange(y.size) + 1
    a = linregress(x, y).slope
    b = time_series_slope(y)
    np.testing.assert_almost_equal(a, b, decimal=10)

    # Check computations over axis
    a = np.apply_along_axis(time_series_slope, 1, Y)
    b = time_series_slope(Y, axis=1)
    np.testing.assert_equal(a, b)

    a = time_series_slope(Y, axis=1)[0]
    b = time_series_slope(y)
    np.testing.assert_equal(a, b)

    # Check linear and constant cases
    for step in [-1, 0, 1]:
        y = np.arange(1, 4) * step
        np.testing.assert_almost_equal(time_series_slope(y), step, decimal=10)

max_stars_repo_name: fkiraly/sktime

max_issues_repo_name: fkiraly/sktime

max_forks_repo_name: jasonlines/sktime-workshop-shapelets

reason: import numpy,from scipy


text:
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal, QSocketNotifier, QSize, QRect, QRectF
from PyQt5.QtWidgets import QGraphicsView, QGraphicsScene
from PyQt5.QtGui import QBrush, QColor, QImage, QPixmap, QTransform
import numpy as np

try:
    import cv2
    cv2_available = True
except ImportError:
    cv2_available = False

class QPicamera2(QGraphicsView):
    done_signal = pyqtSignal()
    update_overlay_signal = pyqtSignal(object)

    def __init__(self, picam2, parent=None, width=640, height=480, bg_colour=(20, 20, 20), keep_ar=True):
        super().__init__(parent=parent)
        self.picamera2 = picam2
        picam2.have_event_loop = True
        self.keep_ar = keep_ar
        self.image_size = None
        self.last_rect = QRect(0, 0, 0, 0)

        self.size = QSize(width, height)
        self.pixmap = None
        self.overlay = None
        self.scene = QGraphicsScene()
        self.setScene(self.scene)
        self.setBackgroundBrush(QBrush(QColor(*bg_colour)))
        self.resize(width, height)
        self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.enabled = True

        self.update_overlay_signal.connect(self.update_overlay)
        self.camera_notifier = QSocketNotifier(self.picamera2.camera_manager.efd,
                                               QSocketNotifier.Read, self)
        self.camera_notifier.activated.connect(self.handle_requests)

    def cleanup(self):
        del self.scene
        del self.overlay
        del self.camera_notifier

    def signal_done(self, picamera2):
        self.done_signal.emit()

    def image_dimensions(self):
        # The dimensions of the camera images we're displaying.
        camera_config = self.picamera2.camera_config
        if camera_config is not None and camera_config['display'] is not None:
            # This works even before we receive any camera images.
            size = self.picamera2.stream_map[camera_config['display']].configuration.size
        elif self.image_size is not None:
            # If the camera is unconfigured, stick with the last size (if available).
            size = self.image_size
        else:
            # Otherwise pretend the image fills everything.
            rect = self.viewport().rect()
            size = rect.width(), rect.height()
        self.image_size = size
        return size

    def set_overlay(self, overlay):
        camera_config = self.picamera2.camera_config
        if camera_config is None:
            raise RuntimeError("Camera must be configured before using set_overlay")

        new_pixmap = None
        if overlay is not None:
            overlay = np.copy(overlay, order='C')
            shape = overlay.shape
            qim = QImage(overlay.data, shape[1], shape[0], QImage.Format_RGBA8888)
            new_pixmap = QPixmap(qim)
            # No scaling here - we leave it to fitInView to set that up.
        self.update_overlay_signal.emit(new_pixmap)

    @pyqtSlot(object)
    def update_overlay(self, pix):
        if pix is None:
            # Delete overlay if present
            if self.overlay is not None:
                self.scene.removeItem(self.overlay)
                self.overlay = None
                return
        elif self.overlay is None:
            # Need to add the overlay to the scene
            self.overlay = self.scene.addPixmap(pix)
            self.overlay.setZValue(100)
        else:
            # Just update it
            self.overlay.setPixmap(pix)
        self.fitInView()

    @pyqtSlot(bool)
    def set_enabled(self, enabled):
        self.enabled = enabled

    def fitInView(self):
        # Reimplemented fitInView to remove fixed border
        image_w, image_h = self.image_dimensions()
        rect = QRectF(0, 0, image_w, image_h)
        self.setSceneRect(rect)
        # I get one column of background peeping through on the right without this:
        viewrect = self.viewport().rect().adjusted(0, 0, 1, 1)
        self.resetTransform()
        factor_x = viewrect.width() / image_w
        factor_y = viewrect.height() / image_h
        if self.keep_ar:
            factor_x = min(factor_x, factor_y)
            factor_y = factor_x
        self.scale(factor_x, factor_y)

        # This scales the overlay to be on top of the camera image.
        if self.overlay:
            rect = self.overlay.boundingRect()
            self.overlay.resetTransform()
            factor_x = image_w / rect.width()
            factor_y = image_h / rect.height()
            self.overlay.setTransform(QTransform.fromScale(factor_x, factor_y), True)

    def resizeEvent(self, event):
        self.fitInView()

    @pyqtSlot()
    def handle_requests(self):
        request = self.picamera2.process_requests()
        if not request:
            return

        camera_config = self.picamera2.camera_config
        if self.enabled and self.picamera2.display_stream_name is not None and camera_config is not None:
            stream_config = camera_config[self.picamera2.display_stream_name]
            img = request.make_array(self.picamera2.display_stream_name)
            if stream_config["format"] == "YUV420":
                if cv2_available:
                    img = cv2.cvtColor(img, cv2.COLOR_YUV420p2BGR)
                    width = stream_config["size"][0]
                    if width != stream_config["stride"]:
                        img = img[:, :width, :]  # this will make it even more expensive!
                else:
                    raise RuntimeError("Qt preview cannot display YUV420 without cv2")
            img = np.ascontiguousarray(img[..., :3])
            shape = img.shape
            qim = QImage(img.data, shape[1], shape[0], QImage.Format_RGB888)
            pix = QPixmap(qim)
            # Add the pixmap to the scene if there wasn't one, or replace it if the images have
            # changed size.
            if self.pixmap is None or pix.rect() != self.last_rect:
                if self.pixmap:
                    self.scene.removeItem(self.pixmap)
                self.last_rect = pix.rect()
                self.pixmap = self.scene.addPixmap(pix)
                self.fitInView()
            else:
                # Update pixmap
                self.pixmap.setPixmap(pix)
        request.release()


text:
import matplotlib.pyplot as plt
import DataGen, IterativeSinModel
from utils import plotting
from scipy.interpolate import make_interp_spline
import numpy as np

degree = 1
num_datapoints = 20
num_test = 500
minimum = -1
maximum = 1
train_noise = 0.0 # std
test_noise = 0.
max_capacity = 2000
N_SKIP = 1
REPEAT = 1 # Repeat with same coefficients
EPSILON = 1

data_gen = DataGen.TrigDataGen()

noise_to_accuracy = {}
test_noises = [0., 1e-5, 1e-1, 1e0]
train_data = data_gen.generate_data(num_datapoints, train_noise, minimum, maximum)
test_data_0 = data_gen.generate_data(num_test, test_noise, minimum=min(train_data[0]), maximum=max(train_data[0]))
all_capacities = []
all_test_rmses = []
all_train_rmses = []
for test_noise_idx in range(len(test_noises)):
    test_noise = test_noises[test_noise_idx]
    test_data = (test_data_0[0], data_gen.add_noise(test_data_0[1], test_noise))
    # plotting.plot_data(test_data, "Test")
    # plotting.plot_data(train_data, "Train")

    capacities = sorted(set(list(range(1,num_datapoints+5))+list(range(1, max_capacity + 1, N_SKIP))))
    train_rmses = []
    test_rmses = []
    adversarial_rmses = []
    model =  IterativeSinModel.IterativeSinModel(max_capacity + 1)
    model.feed(train_data[0])
    model.feed_test(test_data[0])
    thetas = {}

    for i in range(len(capacities)): # n
        capacity = capacities[i]
        print('.', end='')
        model.fit(*train_data, capacity)
        train_rmse = model.score_train(train_data[1], capacity)
        test_rmse = model.score_test(test_data[1], capacity)
        adversarial_rmse = model.score_adversarial(test_data[1], capacity, EPSILON)
        train_rmses.append(train_rmse)
        test_rmses.append(test_rmse)
        adversarial_rmses.append(adversarial_rmse)
        thetas[capacity] = model.a
        # if i%(len(capacities)//3) == 1:
        #     plotting.plot_fn(lambda x: model.predict(capacity, model.generate_features(x)), min(train_data[0]), max(train_data[0]), f"capacity: {capacity}")
    print()
    all_capacities.append(capacities)
    all_train_rmses.append(train_rmses)
    all_test_rmses.append(test_rmses)
    # Choose Y limits
    # y_low = min(min(train_data[1]), min(train_data[1])*2)
    # y_high = max(train_data[1])*2
    # plt.ylim([y_low, y_high])
    # plt.show()
    # plt.clf()

    # plt.plot(capacities, train_rmses, label='Train')
    # plt.plot(capacities, test_rmses, label='Test')
    # plt.axvline(x=num_datapoints, color='r', linestyle='--')
    # plt.legend(); plt.show()
    # plt.plot(capacities, train_rmses, label='Train')
    # plt.plot(capacities, test_rmses, label='Test')
    # plt.xscale('log')
    # plt.axvline(x=num_datapoints, color='r', linestyle='--')
    # plt.legend(); plt.show()
    # plt.plot(capacities, train_rmses, label='Train')
    # plt.xscale('log')
    # plt.axvline(x=num_datapoints, color='r', linestyle='--')
    # plt.legend(); plt.show()
    # plt.plot(capacities, test_rmses, label='Test')
    # plt.xscale('log')
    # plt.axvline(x=num_datapoints, color='r', linestyle='--')
    # plt.legend(); plt.show()

plt.style.use("seaborn")
plt.xscale("log")
for idx in range(len(test_noises)):
    test_noise = test_noises[idx]
    X_Y_Spline = make_interp_spline(all_capacities[idx], all_test_rmses[idx])
    X_ = np.linspace(min(all_capacities[idx]), max(all_capacities[idx]), 500)
    Y_ = X_Y_Spline(X_)
    plt.plot(X_, Y_, label='{:.0e}'.format(test_noise))
    # plt.plot(all_capacities[idx], all_test_rmses[idx], label='{:.2e}'.format(test_noise))
plt.legend()
plt.show()

plt.xscale("log")
for idx in range(len(test_noises)):
    test_noise = test_noises[idx]
    plt.plot(all_capacities[idx], all_train_rmses[idx], label='{:.2e}'.format(test_noise))
plt.legend()
plt.show()

max_stars_repo_name: layjain/Deep-Multiple-Descent

max_issues_repo_name: layjain/Deep-Multiple-Descent

max_forks_repo_name: layjain/Deep-Multiple-Descent


text:
import numpy as np

# RT is a 3x4 matrix
def se3_inverse(RT):
    R = RT[0:3, 0:3]
    T = RT[0:3, 3].reshape((3,1))
    RT_new = np.zeros((3, 4), dtype=np.float32)
    RT_new[0:3, 0:3] = R.transpose()
    RT_new[0:3, 3] = -1 * np.dot(R.transpose(), T).reshape((3))
    return RT_new

def se3_mul(RT1, RT2):
    R1 = RT1[0:3, 0:3]
    T1 = RT1[0:3, 3].reshape((3,1))

    R2 = RT2[0:3, 0:3]
    T2 = RT2[0:3, 3].reshape((3,1))

    RT_new = np.zeros((3, 4), dtype=np.float32)
    RT_new[0:3, 0:3] = np.dot(R1, R2)
    T_new = np.dot(R1, T2) + T1
    RT_new[0:3, 3] = T_new.reshape((3))
    return RT_new


text:
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import torch
import numpy as np
from .cython_bbox import bbox_overlaps as bbox_overlaps_np

def bbox_overlaps(anchors, gt_boxes):
    if isinstance(anchors, np.ndarray):
        return bbox_overlaps_np(anchors, gt_boxes)
    elif isinstance(anchors, torch.Tensor):
        raise NotImplementedError
    else:
        raise ValueError("Unknown type of 'anchors' {}".format(type(anchors)))

max_stars_repo_name: songheony/SPM-Tracker

max_issues_repo_name: songheony/SPM-Tracker

max_forks_repo_name: songheony/SPM-Tracker


text:
# -*- coding：utf-8 -*-
# &Author  AnFany

import pandas as pd
import numpy as np

#  训练数据文件路径
train_path = r'C:\Users\lenovo\Desktop\Adult_Train.csv'

#  预测数据文件路径
pre_path = r'C:\Users\lenovo\Desktop\Adult_Test.csv'

# 读取并且处理确缺失数据
def ReadHandle(filepath, miss='fill'):  # 定义处理数据的函数
    data = pd.read_csv(r'%s' % filepath)
    data = data.replace('?', np.nan)
    #  处理缺失值
    if miss == 'del':  # 删除掉缺失值
        miss_data = data.dropna(how='any')
    else:
        miss_data = data.fillna(method='ffill')
    return miss_data

# 读取的数据
read_trai = ReadHandle(train_path)
read_pr = ReadHandle(pre_path)

# 因为目标字段"Money"中，预测数据较训练数据多了一个点，需要处理
read_pr["Money"] = [ii[:-1] for ii in read_pr["Money"]]

# 和Stacking回归一样，用字典储存数据，保证了代码的复用
data_dict = {}
data_dict['train'] = read_trai
data_dict['predict'] = read_pr

max_stars_repo_name: Anfany/Machine-Learning-for-Beginner


text:
# -------------------------------------------------------
# CSCI 561, Spring 2021
# Homework 1
# The Oregon Trail
# Author: Joseph Ko
# This module holds utility functions
# -------------------------------------------------------
import numpy as np
from collections import deque
from Node import Node

# Function to find height of cell
def find_height(grid, coord):
    """ This function outputs the height in a cell """
    x = coord[0]
    y = coord[1]
    if (grid[y, x] < 0):
        return abs(grid[y, x])
    else:
        return 0

# Check if coord is in bound of grid
def bound_check(coord, w, h):
    """ Checks if coord is within grid """
    if (0 <= coord[1] < h) and (0 <= coord[0] < w):
        return True
    else:
        return False

# Check if elevation change condition is satisfied
def height_check(h1, h2, max_height_delta):
    """ Checks if elevation change condition """
    if (abs(h1 - h2) <= max_height_delta):
        return True
    else:
        return False

# Simple loop detector
def simple_loop_check(neighbor, closed):
    """ Checks if neighbor is in closed list """
    for key in closed:
        if (np.array_equal(closed[key].coord, neighbor)):
            return False
    return True

# Loop detector (more sophisticated version)
def loop_check(neighbor, closed, open_queue):
    """ Checks if there is a loop, returns open and closed list """
    open_list = list(open_queue) # populate list of open node (i.e. nodes in queue)
    open_matches = 0
    closed_matches = 0
    # check for open node with same state
    for node in open_list:
        if (np.array_equal(node.coord, neighbor.coord)):
            open_matches += 1
            if neighbor.cost < node.cost:
                open_list.remove(node) # delete existing
                open_list.append(neighbor) # append neighbor
            break
    # check for closed node with same state
    for key in closed:
        node = closed[key]
        if (np.array_equal(node.coord, neighbor.coord)):
            closed_matches += 1
            if neighbor.cost < node.cost:
                del closed[node.id] # delete existing
                open_list.append(neighbor) # append neighbor
            break
    # if there are no matches found at all in open or closed
    if (open_matches + closed_matches == 0):
        open_list.append(neighbor) # append neighbor
    open_queue = deque(open_list)
    return open_queue, closed

# Function print solution path
def get_path(end, closed):
    """ This function prints the path from start to target """
    path_q= deque()
    path = []
    node = end # start at target and trace back
    path_q.append(node) # append target node to initialize
    while (node.parent != 0): # backtracking
        node = closed[node.parent] # set parent to node
        path_q.append(node) # append to right side of queue
    while path_q: # convert to list
        popped_node = path_q.pop()
        path.append(popped_node.coord)
    return path # return list of nodes

# Function returns unit path cost
def calc_unit_path_cost(unit_coord):
    """ returns 14 if diagonal, 10 if not diagonal """
    if (sum(np.abs(unit_coord)) == 2): # sum of magnitudes of coords == 2 if diagonal
        return 14
    else:
        return 10

# Function sorts a queue by path cost
def sort_by_path_cost(open_queue):
    """ sort queue by path cost """
    open_list = list(open_queue)
    open_queue = deque(sorted(open_list, key = lambda node: node.cost))
    return open_queue

# Function that prints results to output file
def print_to_output(result, output, i, n):
    """ prints result from algorithm to output file """
    if i == (n - 1):
        if type(result) == str:
            print(result, file = output, end = '')
        else:
            for i in range(len(result)):
                if i == (len(result) - 1):
                    print(result[i][0], ',', result[i][1], file = output, sep = '', end = '')
                else:
                    print(result[i][0], ',', result[i][1], file = output, sep = '', end = ' ')
    else:
        if type(result) == str:
            print(result, file = output)
        else:
            for i in range(len(result)):
                if i == (len(result) - 1):
                    print(result[i][0], ',', result[i][1], file = output, sep = '')
                else:
                    print(result[i][0], ',', result[i][1], file = output, sep = '', end = ' ')

# Function that calculates the mud cost of moving into a cell
def find_mud_cost(grid, neighbor_coord, neighbor_height):
    """ returns cost of muddiness in cell you travel into """
    x = neighbor_coord[0]
    y = neighbor_coord[1]
    if neighbor_height > 0:
        return 0
    else:
        return grid[y, x]

# Function that calculates heuristic cost from node n to target
def calc_heuristic(neighbor_coord, target):
    """ Returns hueristic cost. Chebyshev distance used here. """
    x1 = neighbor_coord[0]
    x2 = target[0]
    y1 = neighbor_coord[1]
    y2 = target[1]
    return max(abs(y2 - y1), abs(x2 - x1)) # Chebyshev distance

# Function sorts a queue by path cost
def astar_sort(open_queue):
    """ sort queue by path cost + heuristic """
    open_list = list(open_queue)
    open_queue = deque(sorted(open_list, key = lambda node: (node.cost + node.heuristic, node.heuristic)))
    return open_queue


text:
import numpy as np
import pytest

import mesh_join

def test_simple_join():
    vertices1 = np.array([
        [0.0, 0.0, 1.0],
        [0.0, 0.0, 2.0],
        [0.0, 0.0, 3.0],
    ], dtype='f4')

    vertices2 = np.array([
        [0.0, 0.0, 4.0],
        [0.0, 0.0, 5.0],
        [0.0, 0.0, 6.0],
    ], dtype='f4')

    index1 = np.array([0, 1, 2], dtype='i4')
    index2 = np.array([0, 1, 2], dtype='i4')

    vdata, idata = mesh_join.join([vertices1, vertices2], [index1, index2], stride=12)

    vertices = np.frombuffer(vdata, 'f4').reshape(-1, 3)
    index = np.frombuffer(idata, 'i4')

    np.testing.assert_equal(vertices, [
        [0.0, 0.0, 1.0],
        [0.0, 0.0, 2.0],
        [0.0, 0.0, 3.0],
        [0.0, 0.0, 4.0],
        [0.0, 0.0, 5.0],
        [0.0, 0.0, 6.0],
    ])

    np.testing.assert_equal(index, [0, 1, 2, 3, 4, 5])

def test_join_with_restart_index():
    vertices1 = np.array([
        [0.0, 0.0, 1.0],
        [0.0, 0.0, 2.0],
        [0.0, 0.0, 3.0],
        [0.0, 0.0, 4.0],
    ], dtype='f4')

    vertices2 = np.array([
        [0.0, 0.0, 5.0],
        [0.0, 0.0, 6.0],
    ], dtype='f4')

    index1 = np.array([0, 1, 2, -1, 2, 3], dtype='i4')
    index2 = np.array([0, 1], dtype='i4')

    vdata, idata = mesh_join.join([vertices1, vertices2], [index1, index2], stride=12, separator=True)

    vertices = np.frombuffer(vdata, 'f4').reshape(-1, 3)
    index = np.frombuffer(idata, 'i4')

    np.testing.assert_equal(vertices, [
        [0.0, 0.0, 1.0],
        [0.0, 0.0, 2.0],
        [0.0, 0.0, 3.0],
        [0.0, 0.0, 4.0],
        [0.0, 0.0, 5.0],
        [0.0, 0.0, 6.0],
    ])

    np.testing.assert_equal(index, [0, 1, 2, -1, 2, 3, -1, 4, 5])

if __name__ == '__main__':
    pytest.main([__file__])


text:
# External imports
from dataclasses import dataclass
import numpy as np
from numpy.linalg import norm

@dataclass
class Light:
    x: int
    y: int
    z: int

    @property
    def origin(self):
        return np.array([self.x, self.y, self.z])

@dataclass
class Camera:
    x: int
    y: int
    z: int

    @property
    def origin(self):
        return np.array([self.x, self.y, self.z])

@dataclass
class Ray:
    _origin: list
    _direction: list
    _vector: list

    def __init__(self, origin, pixel_origin):
        self.origin = origin
        self.vector = pixel_origin - origin
        self.direction = self.vector / norm(self.vector)

    @property
    def origin(self):
        return np.array(self._origin)

    @origin.setter
    def origin(self, value):
        self._origin = value

    @property
    def direction(self):
        return np.array(self._direction)

    @direction.setter
    def direction(self, value):
        self._direction = value

    @property
    def vector(self):
        return self._vector

    @vector.setter
    def vector(self, value):
        self._vector = value

    @property
    def length(self):
        return norm(self.vector)

@dataclass
class Sphere:
    x: int
    y: int
    z: int
    r: int

    @property
    def origin(self):
        return np.array([self.x, self.y, self.z])

    def ray_intersection(self, ray: Ray):
        vector_w = ray.direction - self.origin
        vector_t_norm = np.dot(ray.direction, vector_w)
        point_t = ray.direction * vector_t_norm + ray.origin
        length_y = norm(point_t - self.origin)
        if length_y < self.r:
            x = np.sqrt(self.r**2 - length_y**2)
            intersection1 = (vector_t_norm - x) * ray.direction + ray.origin
            return  intersection1
        else:
            return False

@dataclass
class Screen:
    x: int
    y: int
    z: int
    width: int
    height: int

    @property
    def xspan(self):
        return np.arange(self.x, self.x + self.width, 1)

    @property
    def yspan(self):
        return np.arange(self.y, self.y + self.height, 1)

max_stars_repo_name: KKastberg/Raytracer

max_issues_repo_name: KKastberg/Raytracer

max_forks_repo_name: KKastberg/Raytracer

reason: import numpy,from numpy


text:
# Script to make a comparison plot from the RJ-MCMC output
# of the AH and IL methods.

# The script assumes that the current directory contains the AH-output, and the directory with the IL-output is specified.
# We also assume that the comparison is direct - i.e. that all parameters and data are shared between the two models.

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
import os
import sys

if not os.path.exists('input_file'):
    print('*'*50)
    print('Cannot find file: input_file')
    print('Check that you are running this Python code in the AH outputs directory \n E.g. cd Outputs \n python ../make_plots.py')
    print('*'*50)
    sys.exit(0)

#
if len(sys.argv) != 2:
    print('*'*50)
    print("Syntax: python make_AH_IL_comparison.py <IL_directory>")
    print("where IL_directory is the relative path to the IL-output")
    print('*'*50)
    sys.exit(0)

# Read some basic data that was saved from the input file to "model_data"
# This can be overwritten by either altering this file, or simply hardwiring the various parameters: e.g.
# age_min, age_max = 0, 100

for line in open('input_file','r'):
    if line.split()[0].upper() == 'Intensity_prior'.upper():
        I_min, I_max =  float(line.split()[1]),float(line.split()[2])
    if line.split()[0].upper() == 'Age_bounds'.upper():
        age_min, age_max =  float(line.split()[1]),float(line.split()[2])
    if line.split()[0].upper() == 'Num_change_points'.upper():
        K_min, K_max =  int(line.split()[1]), int(line.split()[2])
    if line.split()[0].upper() == 'Credible'.upper():
        credible = float(line.split()[1])
    if line.split()[0].upper() == 'output_model'.upper():
        output_model_filename = line.split()[1]
    if line.split()[0].upper() == 'True_data'.upper():
        true_behaviour_file = line.split()[2]
        x_cts_true,y_cts_true=np.loadtxt(os.path.join(os.pardir,true_behaviour_file),unpack=True)
    if line.split()[0].upper() == 'Plotting_intensity_range'.upper():
        I_min,I_max =  float(line.split()[1]),float(line.split()[2])

# read in the various data files that were output by the RJ-MCMC script
x, x_err, y, y_err, strat = np.loadtxt('data.dat', unpack=True)
strat = [int(a) for a in strat]
lx, ly = np.loadtxt('credible_lower.dat', unpack=True)
ux, uy = np.loadtxt('credible_upper.dat', unpack=True)
mode_x, mode_y = np.loadtxt('mode.dat', unpack=True)
median_x, median_y = np.loadtxt('median.dat', unpack=True)
av_x, av_y = np.loadtxt('average.dat', unpack=True)
best_x, best_y = np.loadtxt('best_fit.dat', unpack=True)

if not os.path.exists(sys.argv[1]):
    print('Path ' + sys.argv[1] + ' does not exist')
    sys.exit(0)

lx_IL, ly_IL = np.loadtxt(os.path.join(sys.argv[1],'credible_lower.dat'), unpack=True)
ux_IL, uy_IL = np.loadtxt(os.path.join(sys.argv[1],'credible_upper.dat'), unpack=True)
mode_x_IL, mode_y_IL = np.loadtxt(os.path.join(sys.argv[1],'mode.dat'), unpack=True)
median_x_IL, median_y_IL = np.loadtxt(os.path.join(sys.argv[1],'median.dat'), unpack=True)
av_x_IL, av_y_IL = np.loadtxt(os.path.join(sys.argv[1],'average.dat'), unpack=True)
best_x_IL, best_y_IL = np.loadtxt(os.path.join(sys.argv[1],'best_fit.dat'), unpack=True)

print('Building comparative figure...')

fig1, ax1 = plt.subplots(figsize=(14,5))

ax1.fill_between(lx, ly, uy, facecolor='orange', alpha=0.5, label = 'AH %i%% Credible interval' % credible)

ax1.plot(av_x_IL, av_y_IL, 'r', label = 'Average: IL', linewidth=2)
ax1.plot(av_x, av_y, 'b', label = 'Average: AH', linewidth=2)

if 'x_cts_true' in locals():  #see if "true" data are available to plot --- only for synthetic cases.
    plt.plot(x_cts_true,y_cts_true,'k', linewidth=2, label='Real')

ax1.plot(lx_IL, ly_IL, 'g-', label = 'IL %i%% Credible interval' % credible,linewidth=2)
ax1.plot(ux_IL, uy_IL, 'g-',linewidth=2)

ax1.set_xlabel('Age/yr',fontsize=16)
ax1.set_ylabel('Intensity/$\mu$T',fontsize=16)
ax1.xaxis.set_tick_params(labelsize=16)
ax1.yaxis.set_tick_params(labelsize=16)

ax1.set_xlim(age_min, age_max)
ax1.set_ylim(I_min, I_max)
ax1.legend(loc = 'upper right',fontsize=12,labelspacing=0.2)
plt.savefig('AH_IL_comparison.pdf', bbox_inches='tight',pad_inches=0.0)
plt.close(fig1)

max_stars_repo_name: mingzhaochina/AH-RJMCMC

max_issues_repo_name: mingzhaochina/AH-RJMCMC

max_forks_repo_name: mingzhaochina/AH-RJMCMC


text:
import numpy as np
import pandas as pd

from typing import Dict, Any

class RawDataChannel:

    def __init__(self, signal_header: Dict[str, Any] = None, signal: np.ndarray = None):
        self._signal_header: Dict[str, Any] = signal_header
        self._signal: np.ndarray = signal

    def get_signal(self):
        return self._signal

    def get_signal_as_pd_series(self):
        return pd.Series(self._signal)

    def get_sample_rate(self):
        return self._signal_header['sample_rate']

max_stars_repo_name: aroethen/RBDtector

max_issues_repo_name: aroethen/RBDtector

max_forks_repo_name: aroethen/RBDtector


text:
"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import oneflow as flow
from oneflow.framework.tensor import Tensor
from oneflow.nn.module import Module

class Gather_nd(Module):
    def __init__(self) -> None:
        super().__init__()

    def forward(self, input, index):
        return flow.F.gather_nd(input, index)

def gather_nd_op(input, index):
    """This operator is a high-dimensional extension of `gather`, `index` is a K-dimensional
    tensor, which is regarded as a index of input Tensor `input`.

    Each element defines a slice of `input`:

    .. math::

        output[i_{0},i_{1},...,i_{K-2}] = input[index(i_{0},i_{1},...,i_{K-2})]

    Args:
        input: The input Tensor.
        index: The slice indices.

    For example:

    .. code-block:: python

        >>> import oneflow as flow
        >>> import numpy as np
        >>> input = flow.Tensor(np.array([[1, 2,3], [4, 5,6],[7,8,9]]), dtype=flow.float)
        >>> index_1 = flow.Tensor(np.array([[0], [2]]), dtype=flow.int)
        >>> out_1 = flow.gather_nd(input,index_1)
        >>> print(out_1.shape)
        flow.Size([2, 3])
        >>> out_1
        tensor([[1., 2., 3.],
                [7., 8., 9.]], dtype=oneflow.float32)
        >>> index_2 = flow.Tensor(np.array([[0,2], [2,1]]), dtype=flow.int)
        >>> out_2 = flow.gather_nd(input,index_2)
        >>> out_2
        tensor([3., 8.], dtype=oneflow.float32)

    """
    return Gather_nd()(input, index)

if __name__ == "__main__":
    import doctest

    doctest.testmod(raise_on_error=True)

max_stars_repo_name: Zhangchangh/oneflow

max_issues_repo_name: Zhangchangh/oneflow

max_forks_repo_name: Zhangchangh/oneflow


text:
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from matplotlib.patches import Ellipse
import matplotlib.transforms as transforms
from cycler import cycler

import corridor
from base_data import Points

matplotlib.rcParams.update({
    "pgf.texsystem": "pdflatex",
    'font.family': 'serif',
    'font.size': '10',
    'text.usetex': True,
    'pgf.rcfonts': False,
    'figure.autolayout': True,
    # 'figure.figsize': [7, 4],
    'axes.titlesize': 'medium',
    'xtick.labelsize': 'small',
    'ytick.labelsize': 'small',
    'legend.fontsize': 'x-small',
    'legend.title_fontsize': 'small',
    # 'axes.labelsize': 'small',
})

def confidence_ellipse(ax, mean, cov_mat, n_std=3, facecolor='none', **kwargs):
    pearson = cov_mat[0, 1]/np.sqrt(cov_mat[0, 0] * cov_mat[1, 1])
    # Using a special case to obtain the eigenvalues of this
    # two-dimensional dataset.
    ell_radius_x = np.sqrt(1 + pearson)
    ell_radius_y = np.sqrt(1 - pearson)
    ellipse = Ellipse((0, 0), width=ell_radius_x * 2, height=ell_radius_y * 2,
                      facecolor=facecolor, **kwargs)
    # Calculating the stdandard deviation of x from
    # the squareroot of the variance and multiplying
    # with the given number of standard deviations.
    scale_x = np.sqrt(cov_mat[0, 0]) * n_std
    mean_x = mean[0]

    # calculating the stdandard deviation of y ...
    scale_y = np.sqrt(cov_mat[1, 1]) * n_std
    mean_y = mean[1]

    transf = transforms.Affine2D() \
        .rotate_deg(45) \
        .scale(scale_x, scale_y) \
        .translate(mean_x, mean_y)

    ellipse.set_transform(transf + ax.transData)
    return ax.add_patch(ellipse)

def plot_cartesian_state(ax, cartesian_state, **kwargs):
    # ax.plot(cartesian_state["data"][0],
    #         cartesian_state["data"][1], 'g.', alpha=0.3)

    ax.plot(cartesian_state["mean"][0], cartesian_state["mean"][1], 'r.')
    confidence_ellipse(ax, cartesian_state["mean"], cartesian_state["cov_mat"],
                       3, facecolor='none', zorder=10, edgecolor='r', **kwargs)

    velocity_mean = cartesian_state["mean"][:2] + cartesian_state["mean"][-2:]
    velocity_cov_mat = cartesian_state["cov_mat"][-2:, -2:]
    ax.arrow(cartesian_state["mean"][0], cartesian_state["mean"][1],
             cartesian_state["mean"][2], cartesian_state["mean"][3],
             head_width=.3, head_length=.5, fc='k', hatch='o',
             length_includes_head=True)

    confidence_ellipse(ax, velocity_mean, velocity_cov_mat,
                       3, facecolor='none', zorder=10, edgecolor='b', **kwargs)

def plot_frenet_state(ax, frenet_state, **kwargs):
    # if len(frenet_state["data"]) > 0:
    #     ax.plot(frenet_state["data"][0],
    #             frenet_state["data"][1], 'g.', alpha=0.3)

    ax.plot(frenet_state["mean"][0], frenet_state["mean"][1], 'r.')
    confidence_ellipse(ax, frenet_state["mean"], frenet_state["cov_mat"],
                       3, facecolor='none', zorder=10, edgecolor='r', **kwargs)

    ax.arrow(frenet_state["mean"][0], frenet_state["mean"][1],
             frenet_state["mean"][2], frenet_state["mean"][3],
             head_width=.3, head_length=.5, fc='k', hatch='o',
             length_includes_head=True)

    velocity_mean = frenet_state["mean"][:2] + frenet_state["mean"][-2:]
    velocity_cov_mat = frenet_state["cov_mat"][-2:, -2:]
    confidence_ellipse(ax, velocity_mean, velocity_cov_mat,
                       3, facecolor='none', zorder=10, edgecolor='b', **kwargs)

def lemniscate(alpha, num_nodes):
    t_lemniscate = np.linspace(0, 2*np.pi, num=num_nodes)
    lemniscate_nodes = Points()

    lemniscate_nodes.x = alpha * np.sqrt(2) * np.cos(t_lemniscate) / \
        (np.sin(t_lemniscate)**2 + 1)

    lemniscate_nodes.y = alpha * np.sqrt(2) * np.cos(t_lemniscate) * \
        np.sin(t_lemniscate) / (np.sin(t_lemniscate)**2 + 1)

    return lemniscate_nodes

def state_sample(mean, cov_mat, num_samples):
    x, y, vx, vy = np.random.multivariate_normal(mean, cov_mat, num_samples).T

    # Calculate mean and covariance matrix of the sample data
    cart_data = np.stack((x, y, vx, vy), axis=0)
    cart_mean = np.mean(cart_data, axis=1)
    cart_cov = np.cov(cart_data)

    state = dict()
    state["data"] = cart_data
    state["mean"] = cart_mean
    state["cov_mat"] = cart_cov

    return state

def to_flat_cartesian_state(cartesian_state):
    mean = cartesian_state["mean"]
    cov_mat = cartesian_state["cov_mat"]

    state = corridor.FlatCartesianStateAndCovMat2D()
    state.x = mean[0]
    state.y = mean[1]
    state.vx = mean[2]
    state.vy = mean[3]

    state.var_x = cov_mat[0, 0]
    state.var_y = cov_mat[1, 1]
    state.var_vx = cov_mat[2, 2]
    state.var_vy = cov_mat[3, 3]
    state.cov_xy = cov_mat[0, 1]
    state.cov_xvx = cov_mat[0, 2]
    state.cov_xvy = cov_mat[0, 3]
    state.cov_yvx = cov_mat[1, 2]
    state.cov_yvy = cov_mat[1, 3]
    state.cov_vxvy = cov_mat[2, 3]

    return state

def to_frenet_state(flat_frenet_state):
    mean = np.array([flat_frenet_state.l,
                     flat_frenet_state.d,
                     flat_frenet_state.vl,
                     flat_frenet_state.vd])

    cov_mat = np.array([[flat_frenet_state.var_l, flat_frenet_state.cov_ld,
                         flat_frenet_state.cov_lvl,  flat_frenet_state.cov_lvd],
                        [flat_frenet_state.cov_ld, flat_frenet_state.var_d,
                         flat_frenet_state.cov_dvl,  flat_frenet_state.cov_dvd],
                        [flat_frenet_state.cov_lvl, flat_frenet_state.cov_dvl,
                         flat_frenet_state.var_vl,  flat_frenet_state.cov_vlvd],
                        [flat_frenet_state.cov_lvd, flat_frenet_state.cov_dvd,
                         flat_frenet_state.cov_vlvd,  flat_frenet_state.var_vd]])

    frenet_state = dict()
    frenet_state["data"] = []
    frenet_state["mean"] = mean
    frenet_state["cov_mat"] = cov_mat

    # print("UT Frenet state")
    # print(frenet_state["mean"])
    # print(frenet_state["cov_mat"])

    return frenet_state

def monte_carlo_transformation(moving_frenet_frame_assumption,
                               corridor_wrapper, cartesian_state):
    frenet_state_vector_list = list()
    # Perform transformation
    for column in cartesian_state["data"].T:
        frenet_state_vector_list.append(corridor_wrapper.to_frenet_state_vector(
            column.tolist(), moving_frenet_frame_assumption))

    # Calculate mean and cov mat
    mc_frenet_states = np.array([np.array(x)
                                 for x in frenet_state_vector_list]).T
    # mc_mean = np.mean(mc_frenet_states, axis=1)
    # mc_cov = np.cov(mc_frenet_states)

    frenet_state = dict()
    frenet_state["data"] = mc_frenet_states
    frenet_state["mean"] = np.mean(mc_frenet_states, axis=1)
    frenet_state["cov_mat"] = np.cov(mc_frenet_states)

    # print("MC Frenet state")
    # print(frenet_state["mean"])
    # print(frenet_state["cov_mat"])

    return frenet_state

def unscented_transformation(moving_frenet_frame_assumption,
                             corridor_wrapper, cartesian_state):
    flat_cartesian_state = to_flat_cartesian_state(cartesian_state)
    flat_frenet_state = corridor.ut_cartesian_frenet_transformation(
        corridor_wrapper, flat_cartesian_state, moving_frenet_frame_assumption)
    return to_frenet_state(flat_frenet_state)

def linearized_tranformation(moving_frenet_frame_assumption, corridor_wrapper, cartesian_state):
    flat_cartesian_state = to_flat_cartesian_state(cartesian_state)
    flat_frenet_state = corridor_wrapper.to_frenet_state(
        flat_cartesian_state, moving_frenet_frame_assumption)
    return to_frenet_state(flat_frenet_state)

def z_test(ground_truth, test):
    delta_mean = test["mean"] - ground_truth["mean"]
    inv_delta_cov = np.linalg.inv(
        np.linalg.inv(test["cov_mat"]) + np.linalg.inv(test["cov_mat"]))
    t1 = np.dot(delta_mean.T, inv_delta_cov)
    return np.dot(t1, delta_mean)

def generate_transformations(moving_frenet_frame_assumption,
                             corridor_wrapper, cartesian_states):
    mc_frenet_states = list()
    ut_frenet_states = list()
    ln_frenet_states = list()

    for state in cartesian_states:
        # Monte Carlo transformation
        mc_frenet_states.append(monte_carlo_transformation(
            moving_frenet_frame_assumption, corridor_wrapper, state))
        # Unscented transformation
        ut_frenet_states.append(unscented_transformation(
            moving_frenet_frame_assumption, corridor_wrapper, state))
        # Linearized transformation
        ln_frenet_states.append(linearized_tranformation(
            moving_frenet_frame_assumption, corridor_wrapper, state))

    return mc_frenet_states, ut_frenet_states, ln_frenet_states

def main():

    moving_frenet_frame_assumption = True

    # Initialize corridor from lemniscate
    lemniscate_nodes = lemniscate(30, 6)
    corridor_wrapper = corridor.CorridorWrapper(
        456, lemniscate_nodes.x.tolist(), lemniscate_nodes.y.tolist())

    corridor.TestCorridorHandle(corridor_wrapper)

    polyline_dict = corridor_wrapper.get_polylines(.1)

    # Define states
    cartesian_states = list()

    mean_1 = [12., 10., -9., -5.]
    cov_mat_1 = [[2, .5,   0,  0],
                 [.5, 1.5, 0, 0],
                 [0, 0, 0.9, 0.2],
                 [0, 0, 0.2, 1.1]]
    cartesian_states.append(state_sample(mean_1, cov_mat_1, 5000))

    mean_2 = [-28., 1., -2., 10.]
    cov_mat_2 = [[2, .5,   0,  0],
                 [.5, 1.5, 0, 0],
                 [0, 0, 1.1, 0.2],
                 [0, 0, 0.2, 1.3]]
    cartesian_states.append(state_sample(mean_2, cov_mat_2, 5000))

    mean_3 = [-24, 17, 8, -1]
    cov_mat_3 = [[2, -.5,   0,  0],
                 [-.5, 1.5, 0, 0],
                 [0, 0, 1.1, 0.2],
                 [0, 0, 0.2, 1.3]]
    cartesian_states.append(state_sample(mean_3, cov_mat_3, 5000))

    # transformations under assumption A-1
    mc_frenet_states_A1, ut_frenet_states_A1, ln_frenet_states_A1 = generate_transformations(
        False, corridor_wrapper, cartesian_states)

    # transformations under assumption A-2
    mc_frenet_states_A2, ut_frenet_states_A2, ln_frenet_states_A2 = generate_transformations(
        True, corridor_wrapper, cartesian_states)

    # State transformation evaluation
    a1_ln = []
    a1_ut = []
    a2_ln = []
    a2_ut = []
    for i in range(len(mc_frenet_states_A1)):
        # euclidean distance
        a1_ln.append(np.linalg.norm(ln_frenet_states_A1[i]['mean'] -
                                    mc_frenet_states_A1[i]['mean']))
        a1_ut.append(np.linalg.norm(ut_frenet_states_A1[i]['mean'] -
                                    mc_frenet_states_A1[i]['mean']))

        a2_ln.append(np.linalg.norm(ln_frenet_states_A2[i]['mean'] -
                                    mc_frenet_states_A2[i]['mean']))
        a2_ut.append(np.linalg.norm(ut_frenet_states_A2[i]['mean'] -
                                    mc_frenet_states_A2[i]['mean']))
        # z-test value
        a1_ln.append(z_test(mc_frenet_states_A1[i], ln_frenet_states_A1[i]))
        a1_ut.append(z_test(mc_frenet_states_A1[i], ut_frenet_states_A1[i]))

        a2_ln.append(z_test(mc_frenet_states_A2[i], ln_frenet_states_A2[i]))
        a2_ut.append(z_test(mc_frenet_states_A2[i], ut_frenet_states_A2[i]))

    table_data = [a1_ln, a1_ut, a2_ln, a2_ut]
    print(table_data)

    # PLOTTING #################################################################
    subplot_width = [0.5, 0.5]
    subplot_height = [1]
    # plt.rc('axes', prop_cycle=(cycler('color', ['m', 'g', 'b', 'c'])))
    fig = plt.figure()
    gs = gridspec.GridSpec(2, 2, figure=fig)
    # , width_ratios=subplot_width,
    #  height_ratios=subplot_height)

    ax_cart = fig.add_subplot(gs[:1, :1], aspect='equal')
    ax_frenet_A1 = fig.add_subplot(gs[:1, -1], aspect='equal')
    ax_frenet_A2 = fig.add_subplot(gs[1:2, :1], aspect='equal')
    ax_table = fig.add_subplot(gs[1:2, -1])
    ax_table.axis('off')

    ax_cart.set_title('Cartesian coordinates')
    ax_frenet_A1.set_title('Frenet coordinates,\nstatic Frenet frame (A-1)')
    ax_frenet_A2.set_title('Frenet coordinates,\nmoving Frenet frame (A-2)')
    # cartesian axis labels
    ax_cart.set(xlabel='$x$ [m]')
    ax_cart.set(ylabel='$y$ [m]')

    # frenet A1 axis labels
    ax_frenet_A1.set(xlabel='$l$ [m]')
    ax_frenet_A1.set(ylabel='$d$ [m]')

    ax_cart.plot(lemniscate_nodes.x, lemniscate_nodes.y, 'k.')

    # Cartesian plot

    ax_cart.plot(polyline_dict["reference_line_x"],
                 polyline_dict["reference_line_y"], 'k-.', linewidth=1)
    ax_cart.plot(polyline_dict["left_boundary_x"],
                 polyline_dict["left_boundary_y"], color='tab:gray', linewidth=1)
    ax_cart.plot(polyline_dict["right_boundary_x"],
                 polyline_dict["right_boundary_y"], color='tab:gray', linewidth=1)

    plot_cartesian_state(ax_cart, cartesian_states[0])
    plot_cartesian_state(ax_cart, cartesian_states[1])
    plot_cartesian_state(ax_cart, cartesian_states[2])

    # Frenet plot
    ax_frenet_A1.plot([20, 130], [0, 0], 'k-.', linewidth=1)
    ax_frenet_A1.plot([20, 130], [2, 2], color='tab:gray', linewidth=1)
    ax_frenet_A1.plot([20, 130], [-2, -2], color='tab:gray', linewidth=1)

    ax_frenet_A2.plot([20, 130], [0, 0], 'k-.', linewidth=1)
    ax_frenet_A2.plot([20, 130], [2, 2], color='tab:gray', linewidth=1)
    ax_frenet_A2.plot([20, 130], [-2, -2], color='tab:gray', linewidth=1)

    # Fill A-1 plot
    for state in mc_frenet_states_A1:
        plot_frenet_state(ax_frenet_A1, state, alpha=0.5)

    for state in ut_frenet_states_A1:
        plot_frenet_state(ax_frenet_A1, state, linestyle='--', alpha=0.5)

    for state in ln_frenet_states_A1:
        plot_frenet_state(ax_frenet_A1, state, linestyle='-.', alpha=0.5)

    # Fill A-2 plot
    for state in mc_frenet_states_A2:
        plot_frenet_state(ax_frenet_A2, state, alpha=0.5)

    for state in ut_frenet_states_A2:
        plot_frenet_state(ax_frenet_A2, state, linestyle='--', alpha=0.5)

    for state in ln_frenet_states_A2:
        plot_frenet_state(ax_frenet_A2, state, linestyle='-.', alpha=0.5)

    ax_cart.set_xlim([-50, 60])
    ax_cart.set_ylim([-25, 25])

    ax_frenet_A1.set_xlim([20, 130])
    ax_frenet_A1.set_ylim([-25, 25])
    ax_frenet_A2.set_xlim([20, 130])
    ax_frenet_A2.set_ylim([-25, 25])

    # Fill table
    column_labels = ['Euclidean distance', 'Z-test values']
    row_labels = ['A-1 LT', 'A-1 UT',
                  'A-2 LT', 'A-2 UT']
    # table = ax_table.table(cellText=table_data, colLabels=column_labels,
    #                        rowLabels=row_labels, loc="center")
    # table.set_fontsize(14)

    # ax_cart.set_aspect('equal', adjustable='box')
    # ax_frenet_A1_ut.set_aspect('equal', adjustable='box')
    fig.tight_layout()
    # plt.savefig(
    #     '/home/dsp/Pictures/Matplotlib_PGFs/StateTransformation.pgf', bbox_inches='tight')

    plt.show()

if __name__ == "__main__":
    main()

max_stars_repo_name: m-naumann/corridor

max_issues_repo_name: m-naumann/corridor

max_forks_repo_name: m-naumann/corridor


text:
from __future__ import print_function
import csv, os, cPickle

import numpy

from sklearn import model_selection as cv, metrics
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier

import validation, config as cfg, utils

def get_k_stratified_sets(input_set, k, y_index=2):
    input_set_size = len(input_set)
    input_set.sort(key = lambda x: x[y_index])
    folds = [[] for x in xrange(k)]

    index = 0

    for item in input_set:
        folds[index].append(item)
        index = index + 1
        if index%k == 0:
            index = 0

    return folds

def cross_validation(train, k):
    print("\nCross-validation: {}-fold".format(k))
    folds = get_k_stratified_sets(train, k)
    fold2data = {}

    for fold_index, fold in enumerate(folds):
        fold2data[fold_index] = {"y_true": [], "y_pred": [], "residuals": []}
        train_folds = [f for x, f in enumerate(folds) if x != fold_index]
        test_fold = fold

        train_set = []
        for f in train_folds:
            train_set.extend(f)

        train_x = [t[1] for t in train_set]
        train_y = [t[2] for t in train_set]
        test_x = [t[1] for t in test_fold]
        test_y = [t[2] for t in test_fold]

        rf = RandomForestRegressor(n_estimators=cfg.N_ESTIMATORS, n_jobs=cfg.N_JOBS)
        rf.fit(train_x, train_y)
        pred_y = rf.predict(test_x)

        fold2data[fold_index]["y_true"] = test_y
        fold2data[fold_index]["y_pred"] = pred_y
        fold2data[fold_index]["residuals"] = abs(pred_y-test_y)

        val_metrics = validation.ValidationMetrics(pred_y, test_y)

        fold2data[fold_index]["rmse"] = val_metrics.RMSE
        fold2data[fold_index]["r20"] = val_metrics.Rsquared0
        fold2data[fold_index]["q2"] = val_metrics.Qsquared
        fold2data[fold_index]["ids"] = [t[0] for t in test_fold]

    for fold_index, fold in enumerate(folds):
        train_folds = [f for x, f in enumerate(folds) if x != fold_index]
        train_residuals = [f["residuals"] for x, f in fold2data.items() if x != fold_index]
        test_fold = fold

        train_set = []
        train_y = []
        for i, f in enumerate(train_folds):
            train_set.extend(f)
            train_y.extend(train_residuals[i])

        train_x = [t[1] for t in train_set]
        test_x = [t[1] for t in test_fold]
        test_y = fold2data[fold_index]["residuals"]

        cp = RandomForestRegressor(n_estimators=cfg.N_ESTIMATORS, n_jobs=cfg.N_JOBS)
        cp.fit(train_x, train_y)
        pred_y = cp.predict(test_x)
        alphas = abs(test_y/pred_y)
        fold2data[fold_index]["residuals_pred"] = pred_y
        fold2data[fold_index]["alphas"] = abs(alphas)

    for metric in ["RMSE", "R20", "q2"]:
        print("{}: {}".format(metric, numpy.mean([fold2data[i][metric.lower()] for i, fold in fold2data.items()])))

    return fold2data

def random_forest(train, test, repeat_count=10):
    print("RANDOM FOREST: {}(train), {}(test), {} repeats\n".format(len(train), len(test), repeat_count))
    train_x = [t[1] for t in train]
    train_y = [t[2] for t in train]
    test_x = [t[1] for t in test]
    test_y = [t[2] for t in test]
    max_r20 = -10
    repeat2data = {}

    for x in range(repeat_count):
        rf = RandomForestRegressor(n_estimators=cfg.N_ESTIMATORS, n_jobs=cfg.N_JOBS)
        rf.fit(train_x, train_y)

        if len(test):
            pred_y = rf.predict(test_x)

            val_metrics = validation.ValidationMetrics(pred_y, test_y)
            repeat2data[x] = {"rmse": val_metrics.RMSE, "r20": val_metrics.Rsquared0, "q2": val_metrics.Qsquared}
            repeat2data[x]["ids"] = [t[0] for t in test]
            repeat2data[x]["y_pred"] = pred_y

            if val_metrics.Rsquared0 > max_r20:
                max_r20 = val_metrics.Rsquared0
                final_rf = rf

        else:
            final_rf = rf

    return final_rf, repeat2data

def build_qsar_models(target_set_id):
    print("QSAR models for {}".format(target_set_id))
    train_ids = get_target_train_set(target_set_id)
    test_ids = get_target_test_set(target_set_id)
    fps = get_fps_for_target_set(target_set_id)
    print("Number of compounds: {}".format(len(fps)))

    train_set = [m for m in fps if m["id"] in train_ids]
    test_set = [m for m in fps if m["id"] in test_ids]
    train_list = [[m["id"], [int(bit) for bit in m["fp"]], m["value"]] for m in train_set]
    test_list = [[m["id"], [int(bit) for bit in m["fp"]], m["value"]] for m in test_set]

    fold2data = cross_validation(train_list, 10)

    print("\nCalculating QSAR model...")
    qsar_model, repeat2data = random_forest(train_list, test_list, 10)
    store_qsar_model(qsar_model, target_set_id, cfg.DIRS["QSAR_MODELS"])
    store_modeling_stats(fold2data, repeat2data, target_set_id)

    molid2residual = {}
    for i, fold in fold2data.items():
        for x, molid in enumerate(fold["ids"]):
            molid2residual[molid] = fold["residuals"][x]

    train_list_cp = []
    for item in train_list:
        train_list_cp.append([item[0], item[1], molid2residual[item[0]]])

    print("Calculating ERROR model...")
    conformal_model, repeat2data = random_forest(train_list_cp, [], 1)
    store_qsar_model(conformal_model, target_set_id, cfg.DIRS["ERROR_MODELS"])
    store_alphas(fold2data, target_set_id)

def get_target_test_set(target_set_id):
    with open(os.path.join(cfg.DIRS["QSAR_SETS"], "{}_test.csv".format(target_set_id)), "r") as input_file:
        rows = [r[0] for r in csv.reader(input_file)]
    return rows

def get_target_train_set(target_set_id):
    with open(os.path.join(cfg.DIRS["QSAR_SETS"], "{}_train.csv".format(target_set_id)), "r") as input_file:
        rows = [r[0] for r in csv.reader(input_file)]
    return rows

def get_fps_for_target_set(target_set_id):
    with open(os.path.join(cfg.DIRS["FPS"], "{}.csv".format(target_set_id)), "r") as input_file:
        rows = [{"id":r["id"], "fp":r["fp"], "value": float(r["value"])} for r in csv.DictReader(input_file)]
    return rows

def get_compound_set(target_set_id):
    with open(os.path.join(cfg.DIRS["TARGET_SETS"], "{}.csv".format(target_set_id))) as input_file:
        rows = [r for r in csv.DictReader(input_file)]
    return rows

def get_fingerprint_set(target_set_id):
    with open(os.path.join(cfg.DIRS["FPS"], "{}.csv".format(target_set_id))) as input_file:
        rows = [r for r in csv.DictReader(input_file)]
    return rows

def store_qsar_model(qsar_model, filename, outdir):
    model_file = open(os.path.join(outdir, "{}.cpickle".format(filename)), "w")
    cPickle.dump(qsar_model, model_file)

def store_alphas(fold2data, target_set_id):
    header = ["id", "res", "pred_res", "alpha"]
    output = csv.writer(open(os.path.join(cfg.DIRS["ERROR_MODELS"], "{}.csv".format(target_set_id)), "w"), delimiter=",")
    output.writerow(header)

    for i, fold in fold2data.items():
        for j, molid in enumerate(fold["ids"]):
            row = [molid, round(fold["residuals"][j], 3), round(fold["residuals_pred"][j], 3), round(fold["alphas"][j], 3)]
            output.writerow(row)

def store_modeling_stats(fold2data, repeat2data, filename):
    header = ["set", "rmse", "r20", "q2"]
    output = csv.writer(open(os.path.join(cfg.DIRS["QSAR_MODELS"], "{}.csv".format(filename)), "w"), delimiter=",")
    output.writerow(header)

    for i, fold in fold2data.items():
        row = []
        row.append("cv")
        row.extend([round(fold[h], 3) for h in header[1:]])
        output.writerow(row)

    for i, repeat in repeat2data.items():
        row = []
        row.append("test")
        row.extend([round(repeat[h], 3) for h in header[1:]])
        output.writerow(row)

def get_model(target_set_id):
    with open(os.path.join(cfg.DIRS["QSAR_MODELS"], "{}.cpickle".format(target_set_id))) as input_file:
        model = cPickle.load(input_file)
    return model

def get_error_model(target_set_id):
    with open(os.path.join(cfg.DIRS["ERROR_MODELS"], "{}.cpickle".format(target_set_id))) as input_file:
        model = cPickle.load(input_file)
    return model

def predict_ligands_on_all_models(ligands_file, r20_cutoff=0.6, q2_cutoff=0.5, force=True):
    with open(ligands_file, "r") as input_file:
        ligands = [r for r in csv.DictReader(input_file)]

    ligands_dir = ligands_file.split("/")[-1].split(".")[0]
    models = ["_".join([t["target_id"], t["activity_type"]]) for t in utils.get_models_by_parameters(r20_cutoff=r20_cutoff, q2_cutoff=q2_cutoff)]

    if not force:
        utils.check_dir(os.path.join(cfg.DIRS["PREDICTIONS_DIR"], ligands_dir))
        done = {x.split(".")[0] for x in os.listdir(os.path.join(cfg.DIRS["PREDICTIONS_DIR"], ligands_dir))}
        models = list(set(models).difference(done))

    count = len(models)
    print("Number of selected models: {}".format(count))

    fps = [[bit for bit in r["fp"]] for r in ligands]

    for m in models:
        pred_y = predict_fingerprints_on_model(fps, m)
        pred_error = predict_fingerprints_on_error_model(fps, m)
        data = [{"id": l["id"], "value": round(pred_y[i], 3), "error": round(pred_error[i], 3)} for i, l in enumerate(ligands)]
        utils.store_data_as_csv(os.path.join(cfg.DIRS["PREDICTIONS"], ligands_dir, "{}.csv".format(m)), data, ["id", "value", "error"])

def predict_fingerprints_on_model(fps, model_name):
    print("Predict {} fingerprints on model {}".format(len(fps), model_name))
    model = get_model(model_name)
    return model.predict(fps)

def predict_fingerprints_on_error_model(fps, model_name):
    print("Predict {} fingerprints on error model {}".format(len(fps), model_name))
    error_model = get_error_model(model_name)
    return error_model.predict(fps)

max_stars_repo_name: skutac/QAFFPs

max_issues_repo_name: skutac/QAFFPs

max_forks_repo_name: skutac/QAFFPs


text:
import os
import sys
import shutil
import subprocess
import argparse
import json
import itertools
import re
from collections import defaultdict
from ast import literal_eval

import numpy as np
import pandas as pd
from scipy.stats import hmean
from nexus import NexusReader, NexusWriter

from config import CONFIGS

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('output', nargs='+',
                        help='output directories of results to compare')
    parser.add_argument('--results', default='results',
                        help='results directory to save comparison')
    parser.add_argument('--different-nex', action='store_true')
    parser.add_argument('--compare-to-first', action='store_true')
    parser.add_argument('-B', type=int, default=0,
                        help='if positive, enforce # of bootstrap samples')
    parser.add_argument('--bootstrap', type=int, default=1,
                        help='bootstrap to get confidence intervals on overlaps')
    return parser.parse_args()

def get_name(config, different_nex):
    if different_nex:
        s = config['baseoutfile'] + '-'
    else:
        s = ''
    s += config['model']
    if 'B' in config:
        s += '-c-{c}-a-{a}'.format(**config)
    return s

def get_base_filepath(odir, config, B):
    filename = config['baseoutfile']
    if 'B' in config:
        if B > 0:
            config['B'] = B
        filename += '.{}bootstraps'.format(config['B'])
    return os.path.join(odir, filename)

def get_base_bootstrap_filepaths(odir, config, B):
    filename = config['baseoutfile']
    if B > 0:
        config['B'] = B
    filename += '.bootstrap{}'
    return [os.path.join(odir, filename.format(b+1)) for b in range(config['B'])]

def load_bipartitions(base_filename):
    with open(base_filename+'.parts', 'r') as f:
        # discard name line
        f.readline()
        id_dict = dict()
        for line in f:
            id_str, bipart_str = line.split()
            id_dict[id_str] = bipart_str
    bipart_probs = dict()
    with open(base_filename+'.tstat', 'r') as f:
        # discard ID and column name lines
        f.readline()
        f.readline()
        for line in f:
            id_str, _, prob_str, _ = line.split(maxsplit=3)
            bipart_probs[id_dict[id_str]] = float(prob_str)
    return bipart_probs

def min_element(t):
    if isinstance(t, tuple):
        return np.min([min_element(st) for st in t])
    else:
        return t

def sort_tree(t):
    if isinstance(t, tuple):
        mins = [min_element(st) for st in t]
        sorted_inds = np.argsort(mins)
        return tuple(sort_tree(t[i]) for i in sorted_inds)
    else:
        return t

def load_tree_probs(base_filename, hdp=0.95):
    tree_probs = dict()
    total_prob = 0.0
    line_re = re.compile(r'.* \[&W (?P<prob>.*?)\] (?P<tree>\(.*\));')
    # tree_re = re.compile('^[()\d]*$')

    with open(base_filename+'.trprobs', 'r') as f:
        for line in f:
            line = line.strip()
            if line.startswith('tree'):
                m = line_re.search(line)
                prob = float(m.group('prob'))
                tree_str = m.group('tree')
                tree = str(sort_tree(literal_eval(tree_str)))
                if total_prob + prob > hdp:
                    tree_probs[tree] = hdp - total_prob
                    break
                else:
                    tree_probs[tree] = prob
                    total_prob += prob
    return tree_probs

def disagreement(x, y, eps=1e-3):
    return (x-y) * np.log((x+eps)/(y+eps))

def analyze_bipartitions(biparts):
    df = pd.DataFrame.from_dict(dict(biparts))
    df = df.fillna(0.0)

    max_possible_disag = disagreement(0,1)
    disagreements = []
    for n1, n2 in itertools.combinations(df, 2):
        disag = disagreement(df[n1].values, df[n2].values) / max_possible_disag
        top = np.sort(disag)[-1:-7:-1]
        res = np.mean(top)
        disagreements.append((n1,n2,res,top))
    disagreements.sort(key=lambda t: t[2], reverse=True)
    print('***significant disagreement if > {:.3f}***'.format(1/max_possible_disag))
    print('Disagreement:')
    for n1, n2, res, top in disagreements:
        print('{:22} vs. {:22} {:.3f}   {}'.format(
            n1, n2, res, '(' + ', '.join(['{:.3f}'.format(t) for t in top])) + ')')

def analyze_tree_probs(tree_probs, start_ind, compare_to_first):
    df = pd.DataFrame.from_dict(dict(tree_probs))
    df = df.fillna(0)
    overlaps = []
    overlap_lists = defaultdict(list)
    if compare_to_first:
        names = list(zip(*tree_probs))[0]
        pairs = zip([names[0]]*(len(names)-1), names[1:])
    else:
        pairs = itertools.combinations(df, 2)
    for n1, n2 in pairs:
        min_probs = df.loc[:,(n1, n2)].min(axis=1)
        overlap = min_probs.sum() + 1e-10
        n_overlap = np.count_nonzero(min_probs.values)
        overlaps.append((n1, n2, overlap, n_overlap))
        overlap_lists[n1].append(overlap)
        overlap_lists[n2].append(overlap)

    print('99% HPD tree region overlap:')
    for n1, n2, overlap, n_overlap in overlaps:
        print('{:22} vs. {:22} {:.3f}   {:d}'.format(n1[start_ind:], n2[start_ind:], overlap, n_overlap))

def find_start_ind(args, names):
    start_ind = 0
    if args.different_nex:
        subname = names[0][0]
        while np.all([n.startswith(subname) for n in names]):
            start_ind += 1
            subname = names[0][:start_ind+1]
    return start_ind

def standard_analysis(args, config_info):
    biparts = []
    tree_probs = []
    names = []
    for i, (odir, config) in enumerate(zip(args.output, config_info)):
        name = get_name(config, args.different_nex)
        names.append(name)
        base_filename = get_base_filepath(odir, config, args.B)
        tree_probs.append((name, load_tree_probs(base_filename)))

    start_ind = find_start_ind(args, names)
    analyze_tree_probs(tree_probs, start_ind, args.compare_to_first)

def bootstrap_analysis(args, config_info, hdp=.99):
    df = pd.DataFrame()
    names = []
    print('loading data...')
    for odir, config in zip(args.output, config_info):
        name = get_name(config, args.different_nex)
        print(name)
        names.append(name)
        if 'B' in config:
            base_paths = get_base_bootstrap_filepaths(odir, config, args.B)
        else:
            base_paths = [get_base_filepath(odir, config, args.B)]
        for base_path in base_paths:
            tree_probs = load_tree_probs(base_path, hdp=1.0)
            tree_probs['name'] = name
            df = df.append(tree_probs, ignore_index=True)
    df = df.fillna(0)
    # print(df)
    p_overlaps = defaultdict(list)
    n_overlaps = defaultdict(list)
    print('bootstrapping...')
    for i in range(args.bootstrap):
        if args.bootstrap == 1:
            boot_df = df.groupby('name').mean()
        else:
            boot_df = df.groupby('name').apply(lambda g: g.sample(frac=1., replace=True)).drop('name', axis=1).groupby('name').mean()
        for _, s in boot_df.iterrows():
            total_prob = 0
            all_mass = False
            for i in np.argsort(s.values)[::-1]:
                prob = s[i]
                if all_mass:
                    s[i] = 0
                elif total_prob + prob >= hdp:
                    s[i] = hdp - total_prob
                    all_mass = True
                else:
                    total_prob += prob
        if args.compare_to_first:
            pairs = zip([names[0]]*(len(names)-1), names[1:])
        else:
            pairs = itertools.combinations(names, 2)
        for n1, n2 in pairs:
            min_probs = boot_df.loc[(n1, n2),:].min(axis=0)
            p_overlap = min_probs.sum() + 1e-10
            n_overlap = np.count_nonzero(min_probs.values)
            p_overlaps[(n1,n2)].append(p_overlap)
            n_overlaps[(n1,n2)].append(n_overlap)

    start_ind = find_start_ind(args, names)

    for (n1, n2), p_overlap_list in p_overlaps.items():
        p_ci = np.percentile(p_overlap_list, (10, 90))
        median_n = int(np.median(n_overlaps[(n1,n2)]))
        print('{:22} vs. {:22}  ({:.3f}, {:.3f}) ({})'.format(
            n1[start_ind:], n2[start_ind:], p_ci[0], p_ci[1], median_n))

def main():
    args = parse_args()
    if len(args.output) < 2:
        sys.exit('need at least two outputs to compare')
    print(args.output)
    config_info = []
    for odir in args.output:
        with open(os.path.join(odir, 'experiment.config'), 'r') as f:
            config_info.append(json.load(f))
    if not args.different_nex:
        if np.any([c['nexfile'] != config_info[0]['nexfile'] for c in config_info]):
            sys.exit('all outputs must use the same nexus file')

    if args.bootstrap <= 0:
        standard_analysis(args, config_info)
    else:
        bootstrap_analysis(args, config_info)

if __name__ == '__main__':
    main()


text:
# -*- coding: utf-8 -*-
import math
from collections import defaultdict

import numpy as np

from torch.utils.data.sampler import Sampler

class BucketBatchSampler(Sampler):
    r"""Samples batch indices from sequence-length buckets efficiently
    with very little memory overhead.

    Epoch overhead for 5M dataset with batch_size=32 is around 400ms.

    Arguments:
        lengths (list): List of integer lengths corresponding to each
            item in the dataset.
        batch_size (int): Size of mini-batch.
        max_len (int, optional): A maximum sequence length that will be used
            to filter out very long sequences. A default of `10000` is
            assumed if ``None`` given.

    Example:
        # Generate dummy length information
        >> lengths = np.random.randint(1, 20, size=10000)
        >> sampler = BucketBatchSampler(lens, batch_size=10)
        >> batch = list(sampler)[0]
        >> batch
        [7526, 8473, 9194, 1030, 1568, 4182, 3082, 827, 3688, 9336]
        >> [lengths[i] for i in batch]
        # All samples in the batch have same length
        [4, 4, 4, 4, 4, 4, 4, 4, 4, 4]

    """

    def __init__(self, lengths, batch_size, max_len=None):
        self.batch_size = batch_size
        self.max_len = 10000 if max_len is None else max_len

        # Buckets: lengths -> list of sample indices
        self.buckets = defaultdict(list)

        # Fill the buckets while optionally filtering out long sequences
        for idx, len_ in enumerate(lengths):
            if len_ <= self.max_len:
                self.buckets[len_].append(idx)

        self.bucket_names = list(self.buckets.keys())

        # Pre-compute how many times a bucket will be sampled
        self.bucket_idxs = []

        for len_ in self.buckets:
            # Convery bucket to numpy array
            np_bucket = np.array(self.buckets[len_])

            # How many batches will be done for this bucket?
            bucket_bs = np_bucket.size / self.batch_size
            idxs = [len_] * math.ceil(bucket_bs)

            self.buckets[len_] = np_bucket
            self.bucket_idxs.extend(idxs)

        # Convert to numpy array
        self.bucket_idxs = np.array(self.bucket_idxs)

        # Set number of batches
        self.n_batches = len(self.bucket_idxs)

    def __iter__(self):
        # Keep offsets for each bucket for efficiency
        bucket_offsets = {}

        # Random access indices
        bucket_views = {}

        # Create permuted access indices for each bucket
        # to avoid shuffling the lists
        for len_, elems in self.buckets.items():
            bucket_offsets[len_] = 0
            perms = np.random.permutation(len(elems))
            bucket_views[len_] = perms

        # Shuffle bucket order
        # For each bucket, slide the window to yield the next batch
        shuf_idxs = np.random.permutation(self.bucket_idxs)
        for bidx in shuf_idxs:
            # Get offset pointer for this bucket: 0 initially
            offset = bucket_offsets[bidx]

            # Convert them to permuted view
            idxs = bucket_views[bidx][offset: offset + self.batch_size]

            # Increment offset
            bucket_offsets[bidx] += len(idxs)

            # Return sample indices
            yield self.buckets[bidx][idxs]

    def __len__(self):
        """Returns how many batches are inside."""
        return self.n_batches

max_stars_repo_name: Eurus-Holmes/VAG-NMT

max_issues_repo_name: Eurus-Holmes/VAG-NMT

max_forks_repo_name: wonderseen/OVC-MMT