text:
from keras.models import load_model
import numpy as np
import pandas as pd

model = load_model("network.h5")
df=pd.read_csv("./static/the_ANN_output.csv")
def creation_of_map(x):
    dicr={}
    count=0
    for a in x:
        dicr[count]=a
        count+=1
    return dicr
map_data=creation_of_map(df["Disease"])

df2=pd.read_csv("./static/detail_and_precation.csv")
mapvalue={}
for x,y,p1,p2,p3,p4 in zip(df2["Disease"],df2["Description"],df2["Precaution_1"],df2["Precaution_2"],df2["Precaution_3"],df2["Precaution_4"]):
    mapvalue[x]=[x,y,p1,p2,p3,p4]

def mapdata():
    df=pd.read_csv("./static/symptoms.csv")
    map_value={}
    count=0
    for x in df["Symptoms"]:
        map_value[x]=count
        count=count+1
    return map_value

map_data2=mapdata()

def map_weight():
    df=pd.read_csv("./static/sever.csv")
    map_weight={}
    for x,y in zip(df["Symptom"],df["weight"]):
        x=" "+x
        map_weight[x]=y
    return map_weight
map_weight=map_weight()

def predict_disease(l):
    count=0
    sumvalue=0
    slist=[0]*131
    for x in l:
        if x != "Select Symptoms":
            slist[map_data2[x]]=1
            if x in map_weight:
                sumvalue=sumvalue+map_weight[x]
                count=count+1

    disease=map_data[(np.argmax(model.predict([slist]), axis=-1))[0]]
    out=mapvalue[disease]
    if(count!=0):
        level = sumvalue/count
    if level>0 and level<0.21:
        out.append("Common Symptoms take general medicine and contact the near by Doctor.")
        out.append("green")
    elif level>=0.21 and level<0.49:
        out.append("Moderate Symptoms, the state can be critical, contact near by Doctor.")
        out.append("blue")
    elif level>=0.49 and level<1:
        out.append("Cretical Symptoms, the worst state, immediatly reach a nearby Hospital.")
        out.append("red")
    return out


text:
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import UnivariateSpline
from fluids.optional.pychebfun import chebfun
from fluids.fittings import bend_rounded_Crane_ratios, bend_rounded_Crane_fds, bend_rounded_Crane_coeffs

ratios = np.linspace(1, 20, 1000)

bend_rounded_Crane_obj = UnivariateSpline(bend_rounded_Crane_ratios, bend_rounded_Crane_fds, s=0)
fun = chebfun(f=bend_rounded_Crane_obj, domain=[1,20], N=10)

plt.plot(bend_rounded_Crane_ratios, bend_rounded_Crane_fds, 'x', label='Crane data')
plt.plot(ratios, bend_rounded_Crane_obj(ratios), label='Cubic spline')
plt.plot(ratios, fun(ratios), label='Chebyshev approximation')

plt.legend()
plt.title("Interpolation of Crane ft multipliers for pipe bend losses")
plt.xlabel('Bend radius/pipe diameter ratio')
plt.ylabel('Friction factor multiplier')
#plt.show()

ext: py

max_stars_repo_name: yoelcortes/fluids

max_issues_repo_name: yoelcortes/fluids

max_forks_repo_name: yoelcortes/fluids

reason: import numpy,from scipy


text:
"""
===================================================================
Statistical functions for masked arrays (:mod:`scipy.stats.mstats`)
===================================================================

.. currentmodule:: scipy.stats.mstats

This module contains a large number of statistical functions that can
be used with masked arrays.

Most of these functions are similar to those in `scipy.stats` but might
have small differences in the API or in the algorithm used. Since this
is a relatively new package, some API changes are still possible.

Summary statistics
==================

.. autosummary::
   :toctree: generated/

   describe
   gmean
   hmean
   kurtosis
   mode
   mquantiles
   hdmedian
   hdquantiles
   hdquantiles_sd
   idealfourths
   plotting_positions
   meppf
   moment
   skew
   tmean
   tvar
   tmin
   tmax
   tsem
   variation
   find_repeats
   sem
   trimmed_mean
   trimmed_mean_ci
   trimmed_std
   trimmed_var

Frequency statistics
====================

.. autosummary::
   :toctree: generated/

   scoreatpercentile

Correlation functions
=====================

.. autosummary::
   :toctree: generated/

   f_oneway
   pearsonr
   spearmanr
   pointbiserialr
   kendalltau
   kendalltau_seasonal
   linregress
   siegelslopes
   theilslopes
   sen_seasonal_slopes

Statistical tests
=================

.. autosummary::
   :toctree: generated/

   ttest_1samp
   ttest_onesamp
   ttest_ind
   ttest_rel
   chisquare
   kstest
   ks_2samp
   ks_1samp
   ks_twosamp
   mannwhitneyu
   rankdata
   kruskal
   kruskalwallis
   friedmanchisquare
   brunnermunzel
   skewtest
   kurtosistest
   normaltest

Transformations
===============

.. autosummary::
   :toctree: generated/

   obrientransform
   trim
   trima
   trimmed_stde
   trimr
   trimtail
   trimboth
   winsorize
   zmap
   zscore

Other
=====

.. autosummary::
   :toctree: generated/

   argstoarray
   count_tied_groups
   msign
   compare_medians_ms
   median_cihs
   mjci
   mquantiles_cimj
   rsh

"""
from ._mstats_basic import *
from ._mstats_extras import *
# Functions that support masked array input in stats but need to be kept in the
# mstats namespace for backwards compatibility:
from scipy.stats import gmean, hmean, zmap, zscore, chisquare

max_stars_repo_name: lorentzenchr/scipy

max_issues_repo_name: lorentzenchr/scipy

max_forks_repo_name: lorentzenchr/scipy

reason: from scipy


text:
import numpy as np
import tensorflow as tf

from deephyper.search.nas.baselines.a2c.utils import ortho_init, lstm, lnlstm
from deephyper.search.nas.baselines.common.models import register, nature_cnn

class RNN(object):
    def __init__(self, func, memory_size=None):
        self._func = func
        self.memory_size = memory_size

    def __call__(self, *args, **kwargs):
        return self._func(*args, **kwargs)

@register("ppo_lstm")
def ppo_lstm(num_units=128, layer_norm=False):
    def network_fn(input, mask, state):
        input = tf.layers.flatten(input)
        mask = tf.to_float(mask)

        if layer_norm:
            h, next_state = lnlstm(
                [input], [mask[:, None]], state, scope='lnlstm', nh=num_units)
        else:
            h, next_state = lstm([input], [mask[:, None]],
                                 state, scope='lstm', nh=num_units)
        h = h[0]
        return h, next_state

    return RNN(network_fn, memory_size=num_units * 2)

@register("ppo_lstm_128")
def ppo_lstm_128():
    return ppo_lstm(num_units=128)

@register("ppo_lnlstm_128")
def ppo_lnlstm_128():
    return ppo_lstm(num_units=128, layer_norm=True)

@register("ppo_lstm_64")
def ppo_lstm_64():
    return ppo_lstm(num_units=64)

@register("ppo_lnlstm_64")
def ppo_lnlstm_64():
    return ppo_lstm(num_units=64, layer_norm=True)

@register("ppo_lstm_32")
def ppo_lstm_32():
    return ppo_lstm(num_units=32)

@register("ppo_lnlstm_32")
def ppo_lnlstm_32():
    return ppo_lstm(num_units=32, layer_norm=True)

@register("ppo_cnn_lstm")
def ppo_cnn_lstm(num_units=128, layer_norm=False, **conv_kwargs):
    def network_fn(input, mask, state):
        mask = tf.to_float(mask)
        initializer = ortho_init(np.sqrt(2))

        h = nature_cnn(input, **conv_kwargs)
        h = tf.layers.flatten(h)
        h = tf.layers.dense(h, units=512, activation=tf.nn.relu,
                            kernel_initializer=initializer)

        if layer_norm:
            h, next_state = lnlstm(
                [h], [mask[:, None]], state, scope='lnlstm', nh=num_units)
        else:
            h, next_state = lstm([h], [mask[:, None]],
                                 state, scope='lstm', nh=num_units)
        h = h[0]
        return h, next_state

    return RNN(network_fn, memory_size=num_units * 2)

@register("ppo_cnn_lnlstm")
def ppo_cnn_lnlstm(num_units=128, **conv_kwargs):
    return ppo_cnn_lstm(num_units, layer_norm=True, **conv_kwargs)

max_stars_repo_name: jtchilders/deephyper

max_issues_repo_name: jtchilders/deephyper

max_forks_repo_name: jtchilders/deephyper

reason: import numpy


text:
import streamlit as st
import pandas as pd
import numpy as np
import psycopg2
import datetime
import altair as alt
import sys
import connection.queries as queries
import connection.connection as connection

#Page configuration
st.set_page_config(layout="wide")

st.title("""
GetHired Community Dashboard
""")

#Sidebar configuration

date_min = st.sidebar.date_input(
     "Select initial date",
     datetime.date(2021, 12, 1))
date_max = st.sidebar.date_input(
     "Select End date",
     datetime.date(2023, 1, 1))

st.write('Range from: ' + str(date_min) + ' to ' + str(date_max) )

#SQL connection data

#Parameters
params_dict = {
    "database" : st.secrets.postgres.database,
    "user" : st.secrets.postgres.user,
    "password" : st.secrets.postgres.password,
    "host" : st.secrets.postgres.host,
    "port" : st.secrets.postgres.port

}

#Connection to the DB
connection1 = connection.connect(params_dict)

#Info about users
users_df = connection.postgresql_to_dataframe(connection1, queries.queries_dict['users'], queries.queries_dict['users_columns'])
users_df['sign_up_date'] = pd.to_datetime(users_df['sign_up_date'])

#Get the data from a CSV
df_users  = pd.read_csv('data/USERS_MOCK_DATA_1.csv')

#Users per month analys
created_by_2 = users_df.groupby(pd.Grouper(key='sign_up_date',freq='M'))['id'].count()
created_by_indexes_2 = created_by_2.index

st.subheader('Users analysis')

st.subheader('New Users per month')
st.line_chart(data=created_by_2)

#Column adjustment
col1, col2 = st.columns([3, 1])
#col3, col4  = st.columns(2)

# CSS to inject contained in a string
hide_table_row_index = """
            <style>
            tbody th {display:none}
            .blank {display:none}
            </style>
            """

# Inject CSS with Markdown
st.markdown(hide_table_row_index, unsafe_allow_html=True)

#Analysis per country

countries_count = df_users.groupby('pais')['id'].count()
countries_count_sorted = countries_count.sort_values( ascending=False)
countries_count_countries_name = countries_count_sorted.index

col1.subheader('Countries with most users')
col1.bar_chart(countries_count_sorted.head(6))

#Gender analysis

df_gender = pd.read_csv('data/gender_mock_data.csv')
gender_analysis = (df_gender.groupby('genero').count() / df_gender.groupby('genero').count().sum()) * 100
col2.subheader('Gender analysis')
col2.bar_chart(data=gender_analysis)

#Challenges and community analysis

st.subheader('Community analysis')
col3, col4  = st.columns(2)

#popular_post_count = challenges_df.groupby('discussion-post')['likes'].sum()
popular_post_count = connection.postgresql_to_dataframe(connection1, queries.queries_dict['networking_posts'], queries.queries_dict['networking_posts_columns'])
col3.subheader('Popular posts')
col3.table(popular_post_count)

#Networking categories analysis

categories_count = connection.postgresql_to_dataframe(connection1, queries.queries_dict['networking_categories'], queries.queries_dict['networking_categories_columns'])
col4.subheader('Popular categories')
col4.table(categories_count)

#Challenges analysis

st.subheader('Challenges analysis')

challenges_df = connection.postgresql_to_dataframe(connection1, queries.queries_dict['challenges'], queries.queries_dict['challenges_columns'])
st.subheader('Popular challenges')
st.table(challenges_df)

max_stars_repo_name: joseamaltes/gethired-community-tracking

max_issues_repo_name: joseamaltes/gethired-community-tracking

max_forks_repo_name: joseamaltes/gethired-community-tracking


text:
# Copyright 2019, The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized
import mock
import numpy as np
import tensorflow as tf

from tensorflow_model_optimization.python.core.internal.tensor_encoding.core import encoding_stage
from tensorflow_model_optimization.python.core.internal.tensor_encoding.testing import test_utils

if tf.executing_eagerly():
  tf.compat.v1.disable_eager_execution()

class TFStyleEncodeDecodeTest(tf.test.TestCase, parameterized.TestCase):
  """Tests for `_tf_style_*` decorators.

  The class decorators, `tf_style_encoding_stage` and
  `tf_style_adaptive_encoding_stage` are tested by the example implementations
  of `EncodingStageInterface` and `AdaptiveEncodingStageInterface`, in
  `test_utils.py`.
  """

  _DEFAULT_NAME = 'test_stage_name'

  def _get_mock_stage(self):
    """Returns a mock stage with a name property."""
    stage = mock.Mock()
    type(stage).name = mock.PropertyMock(return_value=self._DEFAULT_NAME)
    return stage

  def _test_initial_state_fn(self):
    """Returns decorated test `initial_state` method."""
    return encoding_stage._tf_style_initial_state(
        lambda _, name: {'state': tf.constant(1.0)})

  def _test_update_state_fn(self):
    """Returns decorated test `update_state` method."""
    return encoding_stage._tf_style_update_state(
        lambda _, s, sut, name: {'state': s['state'] + sut['tensor']})

  def _test_get_params_fn(self):
    """Returns decorated test `get_params` method."""
    return encoding_stage._tf_style_get_params(
        lambda _, name: ({'param': tf.constant(1.0)},) * 2)

  def _test_adaptive_get_params_fn(self):
    """Returns decorated test `get_params` method."""
    return encoding_stage._tf_style_adaptive_get_params(
        lambda _, state, name: ({'param': state['state'] * 2.0},) * 2)

  def _test_encode_fn(self):
    """Returns decorated test `encode` method."""
    return encoding_stage._tf_style_encode(lambda _, x, p, name: x + p['param'])

  def _test_decode_fn(self):
    """Returns decorated test `decode` method."""
    reshape_fn = lambda x, p, shape: tf.reshape(x['val'] + p['param'], shape)
    return encoding_stage._tf_style_decode(
        lambda _, x, p, num_summands, shape, name: reshape_fn(x, p, shape))

  def _assert_all_graph_nodes_in_name_scope(self, graph, name):
    """Asserts that names of all nodes in `graph` contain `name`."""
    for node in graph.as_graph_def().node:
      self.assertIn(name, node.name)

  @parameterized.parameters(1.0, 'string', object)
  def test_tf_style_encoding_stage_raises(self, bad_stage):
    with self.assertRaises(TypeError):
      encoding_stage.tf_style_encoding_stage(bad_stage)

  @parameterized.parameters(1.0, 'string', object)
  def test_tf_style_adaptive_encoding_stage_raises(self, bad_stage):
    with self.assertRaises(TypeError):
      encoding_stage.tf_style_adaptive_encoding_stage(bad_stage)

  @parameterized.parameters(None, 'different_test_name_scope')
  def test_initial_state_decorator(self, name):
    """Test initial_state decorator works as expected."""
    test_initial_state_fn = self._test_initial_state_fn()
    stage = self._get_mock_stage()
    initial_state = self.evaluate(test_initial_state_fn(stage, name))

    # The graph should contain a single node.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 1)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_initial_state')
    # The functionality is not modified.
    self.assertEqual(1.0, initial_state['state'])

  @parameterized.parameters(None, 'different_test_name_scope')
  def test_update_state_decorator(self, name):
    """Test initial_state decorator works as expected."""
    test_update_state_fn = self._test_update_state_fn()
    stage = self._get_mock_stage()
    state = {'state': 1.0}
    state_update_tensors = {'tensor': 2.0}
    updated_state = self.evaluate(
        test_update_state_fn(stage, state, state_update_tensors, name))

    # The graph should contain three nodes. Two for the constants created, and
    # one for their addition.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 3)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_update_state')
    # The functionality is not modified.
    self.assertEqual(3.0, updated_state['state'])

  @parameterized.parameters(None, 'different_test_name_scope')
  def test_get_params_decorator(self, name):
    """Test get_params decorator works as expected."""
    test_get_params_fn = self._test_get_params_fn()
    stage = self._get_mock_stage()
    encode_params, decode_params = self.evaluate(
        test_get_params_fn(stage, name))

    # The graph should contain a single node.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 1)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_get_params')
    # The functionality is not modified.
    self.assertEqual(1.0, encode_params['param'])
    self.assertEqual(1.0, decode_params['param'])

  @parameterized.parameters(None, 'different_test_name_scope')
  def test_adaptive_get_params_decorator(self, name):
    """Test adaptive_get_params decorator works as expected."""
    test_adaptive_get_params_fn = self._test_adaptive_get_params_fn()
    stage = self._get_mock_stage()
    state = {'state': 3.0}
    encode_params, decode_params = self.evaluate(
        test_adaptive_get_params_fn(stage, state, name))

    # The graph should contain three nodes. Two for the constants created, and
    # one for the multiplication to create the params.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 3)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_get_params')
    # The functionality is not modified.
    self.assertEqual(6.0, encode_params['param'])
    self.assertEqual(6.0, decode_params['param'])

  @parameterized.parameters(None, 'different_test_variable_scope')
  def test_encode_decorator(self, name):
    """Test encode decorator works as expected."""
    test_encode_fn = self._test_encode_fn()
    stage = self._get_mock_stage()
    encoded_x = self.evaluate(test_encode_fn(stage, 2.5, {'param': 10.0}, name))

    # The graph should contain three nodes. The two above Python constants
    # converted to a Tensor object, and the resulting sum.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 3)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_encode')
    # The functionality (sum) is not modified.
    self.assertEqual(12.5, encoded_x)

  def test_encode_decorator_different_graphs(self):
    """Input Tensors from different tf.Graph instances should raise an error."""
    # The test method should not actually use the input values, to ensure the
    # error is not raised in a different way.
    test_encode_fn = encoding_stage._tf_style_encode(
        lambda _, x, p, name: tf.constant(0.0))
    stage = self._get_mock_stage()
    graph_1, graph_2 = tf.Graph(), tf.Graph()
    with graph_1.as_default():
      x = tf.constant(2.5)
    with graph_2.as_default():
      params = {'param': tf.constant(10.0)}
    with self.assertRaises(ValueError):
      test_encode_fn(stage, x, params, None)

  @parameterized.parameters(None, 'different_test_variable_scope')
  def test_decode_decorator(self, name):
    """Test decode decorator works as expected."""
    test_decode_fn = self._test_decode_fn()
    stage = self._get_mock_stage()
    decoded_x = self.evaluate(
        test_decode_fn(stage,
                       {'val': np.array([[1.0, 2.0], [3.0, 4.0]], np.float32)},
                       {'param': 1.0}, 1, [4], name))

    # The graph should contain six nodes. The four above Python constants
    # converted to a Tensor object, the subtraction, and the final reshape.
    graph = tf.compat.v1.get_default_graph()
    self.assertLen(graph.as_graph_def().node, 6)
    if name is not None:
      self._assert_all_graph_nodes_in_name_scope(graph, name)
    else:
      self._assert_all_graph_nodes_in_name_scope(
          graph, self._DEFAULT_NAME + '_decode')
    # The functionality (sum + reshape) is not modified.
    self.assertAllEqual(np.array([2.0, 3.0, 4.0, 5.0]), decoded_x)

  def test_decode_decorator_different_graphs(self):
    """Input Tensors from different tf.Graph instances should raise an error."""
    # The test method should not actually use the input values, to ensure the
    # error is not raised in a different way.
    test_decode_fn = encoding_stage._tf_style_decode(
        lambda _, x, p, shape, name: tf.constant(0.0))
    stage = self._get_mock_stage()
    graph_1, graph_2 = tf.Graph(), tf.Graph()
    with graph_1.as_default():
      x = {'val': tf.constant(2.5)}
    with graph_2.as_default():
      params = {'param': tf.constant(10.0)}
    with self.assertRaises(ValueError):
      test_decode_fn(stage, x, params, [], None)

class NoneStateAdaptiveEncodingStageTest(tf.test.TestCase,
                                         parameterized.TestCase):

  def test_as_adaptive_encoding_stage(self):
    """Tests correctness of the wrapped encoding stage."""
    a_var = tf.compat.v1.get_variable('a', initializer=2.0)
    b_var = tf.compat.v1.get_variable('b', initializer=3.0)
    stage = test_utils.SimpleLinearEncodingStage(a_var, b_var)
    wrapped_stage = encoding_stage.as_adaptive_encoding_stage(stage)
    self.assertIsInstance(wrapped_stage,
                          encoding_stage.AdaptiveEncodingStageInterface)

    x = tf.constant(2.0)
    state = wrapped_stage.initial_state()
    encode_params, decode_params = wrapped_stage.get_params(state)
    encoded_x, state_update_tensors = wrapped_stage.encode(x, encode_params)
    updated_state = wrapped_stage.update_state(state, state_update_tensors)
    decoded_x = wrapped_stage.decode(encoded_x, decode_params)

    # Test that the added state functionality is empty.
    self.assertDictEqual({}, state)
    self.assertDictEqual({}, state_update_tensors)
    self.assertDictEqual({}, updated_state)
    self.assertDictEqual({}, wrapped_stage.state_update_aggregation_modes)
    # Test that __getattr__ retrieves attributes of the wrapped stage.
    self.assertIsInstance(wrapped_stage._a, tf.Variable)
    self.assertIs(wrapped_stage._a, a_var)
    self.assertIsInstance(wrapped_stage._b, tf.Variable)
    self.assertIs(wrapped_stage._b, b_var)

    # Test the functionality remain unchanged.
    self.assertEqual(stage.name, wrapped_stage.name)
    self.assertEqual(stage.compressible_tensors_keys,
                     wrapped_stage.compressible_tensors_keys)
    self.assertEqual(stage.commutes_with_sum, wrapped_stage.commutes_with_sum)
    self.assertEqual(stage.decode_needs_input_shape,
                     wrapped_stage.decode_needs_input_shape)

    self.evaluate(tf.compat.v1.global_variables_initializer())
    test_data = test_utils.TestData(*self.evaluate([x, encoded_x, decoded_x]))
    self.assertEqual(2.0, test_data.x)
    self.assertEqual(
        7.0, test_data.encoded_x[
            test_utils.SimpleLinearEncodingStage.ENCODED_VALUES_KEY])
    self.assertEqual(2.0, test_data.decoded_x)

  def test_as_adaptive_encoding_stage_identity(self):
    """Tests that this acts as identity for an adaptive encoding stage."""
    adaptive_stage = encoding_stage.NoneStateAdaptiveEncodingStage(
        test_utils.PlusOneEncodingStage())
    wrapped_stage = encoding_stage.as_adaptive_encoding_stage(adaptive_stage)
    self.assertIs(adaptive_stage, wrapped_stage)

  @parameterized.parameters(1.0, 'string', object)
  def test_as_adaptive_encoding_stage_raises(self, not_a_stage):
    with self.assertRaises(TypeError):
      encoding_stage.as_adaptive_encoding_stage(not_a_stage)

if __name__ == '__main__':
  tf.test.main()

max_stars_repo_name: jessicayywu/model-optimization


text:
#!/usr/bin/env python3
#
# Accepts subsampled fastq files, bam alignment of whole fastq and reference genome.
# How to subsample fastq files?
#  Use the bam file to get reads aligned to reference.
#  Subsample this set of reads to the desired coverage.

import gzip
import logging
import os
import sys
from collections import defaultdict
from functools import partial

import numpy as np
import pysam
from Bio import SeqIO
from scipy import stats

class Contigs:
    genome_size = int(0)
    num_contigs = int(0)

    def __init__(self, record):
        Contigs.num_contigs += 1
        self.name = record.id
        self.sequence = str(record.seq)
        self.length = int(len(self.sequence))
        Contigs.genome_size += self.length

        self.ref_prob_array = np.zeros(self.length)
        self.ref_depth_array = np.zeros(self.length)
        self.ref_covered_bases = float(0)
        self.total_depth = float(0)
        self.ref_depth = float(0)

    def __hash__(self):
        return hash(str(self.sequence))

    def __eq__(self, other):
        return self.name == other.name

    def __ne__(self, other):
        # Not strictly necessary, but to avoid having both
        # x==y and x!=y being True at the same time
        return not (self == other)

    # def weighted_ref_depth(self):
    #     if self.ref_covered_bases == 0:
    #         return 0

    #     return Contigs.weighted_cal(self.ref_depth, self.ref_covered_bases, self.length)

    @staticmethod
    def weighted_cal(success, sample_size, total):
        return (success / float(sample_size)) * float(total)

    @classmethod
    def read_fasta(cls, fasta_file):
        return [
            Contigs(fasta_record) for fasta_record in SeqIO.parse(fasta_file, "fasta")
        ]

# def parallel_calculate_coverage(bamfile_name, contigs, min_qual, min_pid, min_paln, cores):
#     with concurrent.futures.ThreadPoolExecutor(max_workers=cores) as executor:
#         future = [executor.submit(calculate_coverage_per_contig, bamfile_name, min_qual, min_pid, min_paln, contig) for contig in contigs]
#         for f in concurrent.futures.as_completed(future):
#             result = f.result()
#         ## When you don't wish to return anything, but monitor process for exceptions.
#         jobs = concurrent.futures.wait(future, return_when='FIRST_EXCEPTION')
#         if len(jobs.not_done) > 0:
#             print(f'Some ({len(jobs.not_done)}) exceptions occurred')
#             for job in jobs.not_done:
#                 print(f'{job.exception()}')
#             sys.exit(1)

def calculate_coverage(
    bamfile_name, contigs, read_set, min_qual, min_pid, min_paln, fh
):
    covered_contig_lengths = [
        calculate_coverage_per_contig(
            bamfile_name, read_set, min_qual, min_pid, min_paln, contig, fh
        )
        for contig in contigs
    ]

    return sum(covered_contig_lengths)

def calculate_coverage_per_contig(
    bamfile_name, read_set, minQual, thresh_pid, thresh_paln, contig, fh
):
    if len(read_set) == 0:
        check_read = partial(keep_read, min_pid=thresh_pid, min_paln=thresh_paln)
    else:
        check_read = partial(
            keep_read, min_pid=thresh_pid, min_paln=thresh_paln, read_set=read_set
        )
    with pysam.AlignmentFile(bamfile_name, mode="rb") as bamfile:
        # https://pysam.readthedocs.io/en/latest/api.html#pysam.AlignmentFile.count_coverage
        cov_counts = bamfile.count_coverage(
            contig.name,
            start=0,
            end=contig.length,
            read_callback=check_read,
            # quality_threshold is the minimum quality score (in phred) a base has to reach to be counted.
            quality_threshold=minQual,
        )
        contig_len = 0
        for i, ref_allele in enumerate(contig.sequence):
            base_cov = defaultdict(int)
            ref_prob = 0
            ref_depth = 0
            total_depth = 0

            ref_allele = str(ref_allele).upper()
            # if refseq has N, the prob is 'NA'
            if ref_allele in ["A", "T", "C", "G"]:
                base_cov["A"] = cov_counts[0][i]
                base_cov["C"] = cov_counts[1][i]
                base_cov["G"] = cov_counts[2][i]
                base_cov["T"] = cov_counts[3][i]
                total_depth = (
                    base_cov["A"] + base_cov["C"] + base_cov["G"] + base_cov["T"]
                )
                # prob 0 if no depth
                if total_depth > 0:
                    contig.total_depth += total_depth
                    ref_depth = base_cov[ref_allele]
                    ref_prob = float(ref_depth) / float(total_depth)
                    # if ref_depth > 0:
                    #     contig.ref_covered_bases += 1
                    #     contig.ref_depth += ref_depth
            else:
                ref_prob = np.nan
                ref_depth = np.nan
                # if ref_prob < 1:
                # Write output to a file, with columns as:
                # CHROM,POS,REF,A,C,G,T,ref_prob,ref_depth,total_depth
            fh.write(
                f"{contig.name},"
                f"{i},"
                f"{ref_allele},"
                f"{base_cov['A']},"
                f"{base_cov['C']},"
                f"{base_cov['G']},"
                f"{base_cov['T']},"
                f"{ref_prob},"
                f"{ref_depth},"
                f"{total_depth}"
                f"\n"
            )
            contig_len = i + 1
            # contig.ref_prob_array[i] = ref_prob
            # contig.ref_depth_array[i] = ref_depth
    return contig_len

def keep_read(aln, min_pid, min_paln, read_set=None):
    if (read_set is not None) and (aln.query_name not in read_set):
        return False

    edit_dist = dict(aln.tags)["NM"]
    aln_len = aln.query_alignment_length
    read_length = aln.infer_read_length()

    # MetaBAT: %ID is calculated from the CIGAR string and/or NM/MD fields
    # and == 100 * MatchedBases / (MatchedBases + Substituions + Insertions + Deletions)

    pid = 100 * (aln_len - edit_dist) / float(aln_len)
    p_aln_len = aln_len * 100 / float(read_length)

    return (min_pid <= pid <= 100) and (min_paln <= p_aln_len <= 100)

def distance_from_reference(prob_array):
    # remove NANs from array
    prob_array = remove_nan(prob_array)
    # create a unit array of same length representing reference.
    ref_array = np.ones_like(prob_array)

    return (
        root_mean_squared_error(ref_array, prob_array),
        cos_sim(ref_array, prob_array),
    )

def root_mean_squared_error(ref_array, prob_array):
    """ Calculate mean squared error given two arrays of same length """
    if len(ref_array) == len(prob_array):
        return np.sqrt(np.square(np.subtract(ref_array, prob_array)).mean())

def cos_sim(ref_array, prob_array):
    """calculate cosine distance between reference and given."""
    if len(ref_array) == len(prob_array):
        return np.dot(ref_array, prob_array) / (
            np.linalg.norm(ref_array) * np.linalg.norm(prob_array)
        )

def remove_nan(prob_array):
    return prob_array[~np.isnan(prob_array)]

def get_coverage_distribution(
    bam_file, genome, min_qual=20, min_pid=99, min_paln=100, subset_list=None
):
    genome_name = os.path.splitext(os.path.basename(genome))[0]
    output_dir = f"{genome_name}_q{min_qual}_id{min_pid}_aln{min_paln}_vectors"
    # os.makedirs(output_dir, exist_ok=True)

    file_name = os.path.basename(bam_file).split("_vs_")[0].split(".")[0]
    prefix = f"{output_dir}/{file_name}.q{min_qual}_id{min_pid}_aln{min_paln}"

    # ref_prob_array_name = f"{prefix}.ref_prob.npy"
    # ref_depth_array_name = f"{prefix}.ref_depth.npy"
    depth_file = f"{prefix}.ref_depth.csv.gz"
    if os.path.exists(depth_file):
        return {
            "Genome_Name": genome_name,
            "Sample_Name": file_name,
            "Depth_Vector": depth_file,
        }

    depth_file_handle = gzip.open(depth_file, "wt")
    depth_file_handle.write(
        f"contig_name,"
        f"pos,"
        f"ref_allele,"
        f"A,"
        f"C,"
        f"G,"
        f"T,"
        f"ref_prob,"
        f"ref_depth,"
        f"total_depth"
        f"\n"
    )

    contigs = Contigs.read_fasta(genome)

    read_set = set()
    if subset_list is not None:
        with open(subset_list, "r") as subset:
            for line in subset:
                read_set.add(line.strip())

    _ = calculate_coverage(
        bam_file, contigs, read_set, min_qual, min_pid, min_paln, depth_file_handle,
    )
    depth_file_handle.close()

    return {
        "Genome_Name": genome_name,
        "Sample_Name": file_name,
        "Depth_Vector": depth_file,
    }

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO, format="%(asctime)s\t[%(levelname)s]:\t%(message)s",
    )

    bam_file = sys.argv[1]
    genome = sys.argv[2]
    subset_list = None
    if len(sys.argv) > 3:
        subset_list = sys.argv[3]

    min_qual = 20  # quality_threshold is the minimum quality score (in phred) a base has to reach to be counted.
    min_pid = 99
    min_paln = 100

    get_coverage_distribution(
        bam_file, genome, min_qual=20, min_pid=99, min_paln=100, subset_list=None
    )
    sys.exit(0)

    # depth = (
    #     np.sum([contig.weighted_ref_depth() for contig in contigs])
    #     / Contigs.genome_size
    # )
    # coverage = (
    #     100
    #     * np.sum([contig.ref_covered_bases for contig in contigs])
    #     / Contigs.genome_size
    # )

    # # combine all prob arrays in order of contigs
    # genome_ref_prob = np.concatenate(
    #     [remove_nan(contig.ref_prob_array) for contig in contigs]
    # )
    # # np.save(ref_prob_array_name, genome_ref_prob)

    # genome_depth_prob = np.concatenate(
    #     [remove_nan(contig.ref_depth_array) for contig in contigs]
    # )
    # # np.save(ref_depth_array_name, genome_depth_prob)

    # depth_ent = stats.entropy(genome_depth_prob)
    # rmse, sim = distance_from_reference(genome_ref_prob)
    # print(f"{file_name}\t{depth}\t{coverage}\t{depth_ent}\t{rmse}\t{sim}")

max_stars_repo_name: FischbachLab/invaderCheck

max_issues_repo_name: FischbachLab/invaderCheck

max_forks_repo_name: FischbachLab/invaderCheck


text:
import numpy as np
from math import gcd
from functools import reduce

from shift_cipher import break_shift_cipher, shift, shift_char

ENG_ALPH = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
SPA_ALPH = "ABCDEFGHIJKLMNÑOPQRSTUVWXYZ"

def kasiski_decipher(s):
    for keys in calc_keys(s):
        msg = [shift_char(s[i], -keys[i % len(keys)]) for i in range(len(s))]
        print('\n' + str(keys) + ': ' + ''.join(msg))

def calc_keys(s):
    keys = []
    for key_length in calc_key_lengths(s):
        k_cols = calc_k_cols_for_key_length(s, key_length)
        possible_keys = [break_shift_cipher(col.rstrip()) for col in k_cols]
        keys.append(possible_keys)
    return keys

def calc_k_cols_for_key_length(s, key_length):
    n_grams = calc_n_grams(s, key_length)
    k_cols = [[row[i] for row in n_grams] for i in range(key_length)]
    return [''.join(s) for s in k_cols]

def calc_n_grams(s, n):
    rmdr = len(s) % n
    mult = len(s)-rmdr    # Multiple of n
    n_grams = [s[i:i+n] for i in range(0, mult, n)]
    add_padding(s, rmdr, n_grams, n)
    return n_grams

def add_padding(s, rmdr, n_grams, n):
    tail = s[:rmdr]
    if tail:
        n_grams.append((tail).ljust(len(tail)+n-rmdr))
    return

def calc_key_lengths(s):
    lengths = []
    for n in np.arange(3, int(len(s)/2)):
        all_offsets = offsets_for_n_grams(s, n)
        if all_offsets:
            mcd = reduce(gcd, all_offsets)
            if mcd > 1:
                lengths.append(mcd)
    return lengths

def offsets_for_n_grams(s, n):
    seen_n_grams = []
    all_offsets = []
    for i in np.arange(0, len(s)-n):
        n_gram = s[i:i+n]
        if n_gram not in seen_n_grams:
            all_offsets.append(offsets_between_repetitions(s, n_gram))
        seen_n_grams.append(n_gram)
    return flatten([offsets for offsets in all_offsets if offsets])

# Example:  aasdfkjXXXksajkljXXXooieruXXXppperXXX  should return 3 ints for XXX
# And something such as:   wiulnsXXXuioop   should return an empty list
def offsets_between_repetitions(s, subs):
    if s.count(subs) <= 1:
        return []
    inter_strings = (s.split(subs))[1:]
    offsets = [len(inter_s) + len(subs) for inter_s in inter_strings]
    if not s.endswith(subs):
        offsets.pop()
    return offsets

def flatten(list):
    return [y for x in list for y in x]

C1 = "PBVRQVICADSKAÑSDETSJPSIEDBGGMPSLRPWRÑPWYEDSDEÑDRDPCRCPQMNPWKUBZVSFNVRDMTIPWUEQVVCBOVNUEDIFQLONMWNUVRSEIKAZYEACEYEDSETFPHLBHGUÑESOMEHLBXVAEEPUÑELISEVEFWHUNMCLPQPMBRRNBPVIÑMTIBVVEÑIDANSJAMTJOKMDODSELPWIUFOZMQMVNFOHASESRJWRSFQCOTWVMBJGRPWVSUEXINQRSJEUEMGGRBDGNNILAGSJIDSVSUEEINTGRUEETFGGMPORDFOGTSSTOSEQOÑTGRRYVLPWJIFWXOTGGRPQRRJSKETXRNBLZETGGNEMUOTXJATORVJHRSFHVNUEJIBCHASEHEUEUOTIEFFGYATGGMPIKTBWUEÑENIEEU"
C2 = "JGAZNWINHYLZDYVBBJLCQHTNKUDQXMOXJNOZMUSPNONYJMTEJHQHQFOOPUPBCYAÑJONCNNQHNMONDHKUTJMQCMOPNFAOXNTNLOAZMJDQYMOZCJRNBAOQTUIENFAIXTLXJGAZMJAXJVAZMUDNMYLNLJMUMUYHVUMHTÑIGDXDQUCLSJPIBCUSFNUGXXGEEXKAEJMESJÑENASLHLBAEYJROJXACQTCNMYPUCUNMJWOYNHZNKUOGAJDUJXENRYTENJSCNMONTYJNMJYFXFIGJMIBUUSNTFAPNFAFKUROJNYCTUYNBYSGJVACAUCGQWAZMJJHJHSNTPAPXMGNECOGJUTENCNGJGEGAJSPNULGDMAÑJDOFDNPUNNPNTGENMJSNTTOFDKIOXSSQNNFBATOCXMMNVÑEZNMEZBOSNTUSQBUDBTJRBBUYPQZIOQFTBANIBVMEDDYRUMUPNAULBOMAEDHVHNFOCJOSNMJ"
kasiski_decipher(C1)

max_stars_repo_name: lucas-sg/CryptoAlgorithms

max_issues_repo_name: lucas-sg/CryptoAlgorithms

max_forks_repo_name: lucas-sg/CryptoAlgorithms


text:
import numpy as np
from gym import spaces
from gym import Env

class PointEnv(Env):
    """
    point mass on a 2-D plane
    goals are sampled randomly from a square
    """

    def __init__(self, num_tasks=1, checkerboard_size=0.1):
        self.checkerboard_size = checkerboard_size
        self.reset_task()
        self.reset()
        self.observation_space = spaces.Box(low=-np.inf,
                                            high=np.inf,
                                            shape=(2,),
                                            dtype=np.float32)
        self.action_space = spaces.Box(low=-0.1,
                                       high=0.1,
                                       shape=(2,),
                                       dtype=np.float32)
        self._state = None

    def reset_task(self, is_evaluation=False):
        '''
        sample a new task randomly

        Problem 3: make training and evaluation goals disjoint sets
        if `is_evaluation` is true, sample from the evaluation set,
        otherwise sample from the training set
        '''
        # ==================================================================== #
        #                     ----------PROBLEM 3----------
        # ==================================================================== #
        STATIC_SIZE = 10  # final normalized board range: -10..10

        x_cell = np.random.uniform(0, 1)
        y_cell = np.random.uniform(0, 1)
        if self.checkerboard_size == 0:
            # normalize to -10..10
            self._goal = np.array([x_cell, y_cell]) * STATIC_SIZE * 2 - STATIC_SIZE
        else:
            # NB, size of the checkerboard should be quadratic and even to stay fair
            WIDTH = HEIGHT = int(1. / self.checkerboard_size)
            BLACK = int(np.ceil(WIDTH * HEIGHT / 2.))  # evaluation
            WHITE = int(np.floor(WIDTH * HEIGHT / 2.))  # train
            ROW_WIDTH = int(np.ceil(WIDTH / 2))

            if is_evaluation:
                pos = np.random.randint(0, BLACK)
                y_pos = 2 * int(pos / WIDTH) + int((pos % WIDTH) / ROW_WIDTH)
                x_pos = 2 * ((pos % WIDTH) % ROW_WIDTH) + (y_pos % 2)
            else:
                pos = np.random.randint(0, WHITE)
                y_pos = 2 * int(pos / WIDTH) + int((pos % WIDTH) / ROW_WIDTH)
                x_pos = 2 * ((pos % WIDTH) % ROW_WIDTH) + (1 - (y_pos % 2))
            y = y_cell + y_pos
            x = x_cell + x_pos
            # normalize to -10..10
            self._goal = np.array([x, y]) * STATIC_SIZE * 2 / WIDTH - STATIC_SIZE

    def reset(self):
        self._state = np.array([0, 0], dtype=np.float32)
        return self._get_obs()

    def _get_obs(self):
        return np.copy(self._state)

    def reward_function(self, x, y):
        return - (x ** 2 + y ** 2 + 1e-8) ** 0.5

    def step(self, action):
        x, y = self._state
        # compute reward, add penalty for large actions instead of clipping them
        x -= self._goal[0]
        y -= self._goal[1]
        # check if task is complete
        done = abs(x) < .01 and abs(y) < .01
        reward = self.reward_function(x, y)
        # move to next state
        self._state = self._state + action
        ob = self._get_obs()
        return ob, reward, done, dict()

    def viewer_setup(self):
        print('no viewer')
        pass

    def render(self, mode='ansi'):
        assert mode == 'ansi', 'Error: Human and rgb_array rendering is not supported yet.'
        print('current state:', self._state)

    def seed(self, seed=None):
        assert seed is not None, 'Error: A seed has to be provided.'
        np.random.seed = seed
        return [seed]


text:
#!/usr/bin/env python

import argparse
import numpy as np
import mdtraj

parser = argparse.ArgumentParser(description='split the dimer into two monomer pdbs')
parser.add_argument('--pdb', type=str, help='system pdb')
args = parser.parse_args()

def main():
    name = args.pdb
    system = mdtraj.load(name)
    pi = system.topology.select('protein')
    p = system.atom_slice(pi)

    na = p.n_atoms
    first = np.arange(na/2)
    second = np.arange(na/2, na)

    p.atom_slice(first).save_pdb('p0/' + name)
    p.atom_slice(second).save_pdb('p1/' + name)

if __name__ == '__main__':
    main()

max_stars_repo_name: kmckiern/scripts

max_issues_repo_name: kmckiern/scripts

max_forks_repo_name: kmckiern/scripts


text:
from warnings import catch_warnings, simplefilter

import numpy as np
from numpy.random import randn
import pytest

import pandas as pd
from pandas import (
    DataFrame, MultiIndex, Series, Timestamp, date_range, isna, notna)
from pandas.util import testing as tm

@pytest.mark.filterwarnings("ignore:\\n.ix:DeprecationWarning")
class TestMultiIndexSetItem(object):

    def test_setitem_multiindex(self):
        with catch_warnings(record=True):

            for index_fn in ('ix', 'loc'):

                def assert_equal(a, b):
                    assert a == b

                def check(target, indexers, value, compare_fn, expected=None):
                    fn = getattr(target, index_fn)
                    fn.__setitem__(indexers, value)
                    result = fn.__getitem__(indexers)
                    if expected is None:
                        expected = value
                    compare_fn(result, expected)
                # GH7190
                index = MultiIndex.from_product([np.arange(0, 100),
                                                 np.arange(0, 80)],
                                                names=['time', 'firm'])
                t, n = 0, 2
                df = DataFrame(np.nan, columns=['A', 'w', 'l', 'a', 'x',
                                                'X', 'd', 'profit'],
                               index=index)
                check(target=df, indexers=((t, n), 'X'), value=0,
                      compare_fn=assert_equal)

                df = DataFrame(-999, columns=['A', 'w', 'l', 'a', 'x',
                                              'X', 'd', 'profit'],
                               index=index)
                check(target=df, indexers=((t, n), 'X'), value=1,
                      compare_fn=assert_equal)

                df = DataFrame(columns=['A', 'w', 'l', 'a', 'x',
                                        'X', 'd', 'profit'],
                               index=index)
                check(target=df, indexers=((t, n), 'X'), value=2,
                      compare_fn=assert_equal)

                # gh-7218: assigning with 0-dim arrays
                df = DataFrame(-999, columns=['A', 'w', 'l', 'a', 'x',
                                              'X', 'd', 'profit'],
                               index=index)
                check(target=df,
                      indexers=((t, n), 'X'),
                      value=np.array(3),
                      compare_fn=assert_equal,
                      expected=3, )

                # GH5206
                df = DataFrame(np.arange(25).reshape(5, 5),
                               columns='A,B,C,D,E'.split(','), dtype=float)
                df['F'] = 99
                row_selection = df['A'] % 2 == 0
                col_selection = ['B', 'C']
                with catch_warnings(record=True):
                    df.ix[row_selection, col_selection] = df['F']
                output = DataFrame(99., index=[0, 2, 4], columns=['B', 'C'])
                with catch_warnings(record=True):
                    tm.assert_frame_equal(df.ix[row_selection, col_selection],
                                          output)
                check(target=df,
                      indexers=(row_selection, col_selection),
                      value=df['F'],
                      compare_fn=tm.assert_frame_equal,
                      expected=output, )

                # GH11372
                idx = MultiIndex.from_product([
                    ['A', 'B', 'C'],
                    date_range('2015-01-01', '2015-04-01', freq='MS')])
                cols = MultiIndex.from_product([
                    ['foo', 'bar'],
                    date_range('2016-01-01', '2016-02-01', freq='MS')])

                df = DataFrame(np.random.random((12, 4)),
                               index=idx, columns=cols)

                subidx = MultiIndex.from_tuples(
                    [('A', Timestamp('2015-01-01')),
                     ('A', Timestamp('2015-02-01'))])
                subcols = MultiIndex.from_tuples(
                    [('foo', Timestamp('2016-01-01')),
                     ('foo', Timestamp('2016-02-01'))])

                vals = DataFrame(np.random.random((2, 2)),
                                 index=subidx, columns=subcols)
                check(target=df,
                      indexers=(subidx, subcols),
                      value=vals,
                      compare_fn=tm.assert_frame_equal, )
                # set all columns
                vals = DataFrame(
                    np.random.random((2, 4)), index=subidx, columns=cols)
                check(target=df,
                      indexers=(subidx, slice(None, None, None)),
                      value=vals,
                      compare_fn=tm.assert_frame_equal, )
                # identity
                copy = df.copy()
                check(target=df, indexers=(df.index, df.columns), value=df,
                      compare_fn=tm.assert_frame_equal, expected=copy)

    def test_multiindex_setitem(self):

        # GH 3738
        # setting with a multi-index right hand side
        arrays = [np.array(['bar', 'bar', 'baz', 'qux', 'qux', 'bar']),
                  np.array(['one', 'two', 'one', 'one', 'two', 'one']),
                  np.arange(0, 6, 1)]

        df_orig = DataFrame(np.random.randn(6, 3), index=arrays,
                            columns=['A', 'B', 'C']).sort_index()

        expected = df_orig.loc[['bar']] * 2
        df = df_orig.copy()
        df.loc[['bar']] *= 2
        tm.assert_frame_equal(df.loc[['bar']], expected)

        # raise because these have differing levels
        with pytest.raises(TypeError):
            df.loc['bar'] *= 2

        # from SO
        # http://stackoverflow.com/questions/24572040/pandas-access-the-level-of-multiindex-for-inplace-operation
        df_orig = DataFrame.from_dict({'price': {
            ('DE', 'Coal', 'Stock'): 2,
            ('DE', 'Gas', 'Stock'): 4,
            ('DE', 'Elec', 'Demand'): 1,
            ('FR', 'Gas', 'Stock'): 5,
            ('FR', 'Solar', 'SupIm'): 0,
            ('FR', 'Wind', 'SupIm'): 0
        }})
        df_orig.index = MultiIndex.from_tuples(df_orig.index,
                                               names=['Sit', 'Com', 'Type'])

        expected = df_orig.copy()
        expected.iloc[[0, 2, 3]] *= 2

        idx = pd.IndexSlice
        df = df_orig.copy()
        df.loc[idx[:, :, 'Stock'], :] *= 2
        tm.assert_frame_equal(df, expected)

        df = df_orig.copy()
        df.loc[idx[:, :, 'Stock'], 'price'] *= 2
        tm.assert_frame_equal(df, expected)

    def test_multiindex_assignment(self):

        # GH3777 part 2

        # mixed dtype
        df = DataFrame(np.random.randint(5, 10, size=9).reshape(3, 3),
                       columns=list('abc'),
                       index=[[4, 4, 8], [8, 10, 12]])
        df['d'] = np.nan
        arr = np.array([0., 1.])

        with catch_warnings(record=True):
            df.ix[4, 'd'] = arr
            tm.assert_series_equal(df.ix[4, 'd'],
                                   Series(arr, index=[8, 10], name='d'))

        # single dtype
        df = DataFrame(np.random.randint(5, 10, size=9).reshape(3, 3),
                       columns=list('abc'),
                       index=[[4, 4, 8], [8, 10, 12]])

        with catch_warnings(record=True):
            df.ix[4, 'c'] = arr
            exp = Series(arr, index=[8, 10], name='c', dtype='float64')
            tm.assert_series_equal(df.ix[4, 'c'], exp)

        # scalar ok
        with catch_warnings(record=True):
            df.ix[4, 'c'] = 10
            exp = Series(10, index=[8, 10], name='c', dtype='float64')
            tm.assert_series_equal(df.ix[4, 'c'], exp)

        # invalid assignments
        with pytest.raises(ValueError):
            with catch_warnings(record=True):
                df.ix[4, 'c'] = [0, 1, 2, 3]

        with pytest.raises(ValueError):
            with catch_warnings(record=True):
                df.ix[4, 'c'] = [0]

        # groupby example
        NUM_ROWS = 100
        NUM_COLS = 10
        col_names = ['A' + num for num in
                     map(str, np.arange(NUM_COLS).tolist())]
        index_cols = col_names[:5]

        df = DataFrame(np.random.randint(5, size=(NUM_ROWS, NUM_COLS)),
                       dtype=np.int64, columns=col_names)
        df = df.set_index(index_cols).sort_index()
        grp = df.groupby(level=index_cols[:4])
        df['new_col'] = np.nan

        f_index = np.arange(5)

        def f(name, df2):
            return Series(np.arange(df2.shape[0]),
                          name=df2.index.values[0]).reindex(f_index)

        # TODO(wesm): unused?
        # new_df = pd.concat([f(name, df2) for name, df2 in grp], axis=1).T

        # we are actually operating on a copy here
        # but in this case, that's ok
        for name, df2 in grp:
            new_vals = np.arange(df2.shape[0])
            with catch_warnings(record=True):
                df.ix[name, 'new_col'] = new_vals

    def test_series_setitem(
            self, multiindex_year_month_day_dataframe_random_data):
        ymd = multiindex_year_month_day_dataframe_random_data
        s = ymd['A']

        s[2000, 3] = np.nan
        assert isna(s.values[42:65]).all()
        assert notna(s.values[:42]).all()
        assert notna(s.values[65:]).all()

        s[2000, 3, 10] = np.nan
        assert isna(s[49])

    def test_frame_getitem_setitem_boolean(
            self, multiindex_dataframe_random_data):
        frame = multiindex_dataframe_random_data
        df = frame.T.copy()
        values = df.values

        result = df[df > 0]
        expected = df.where(df > 0)
        tm.assert_frame_equal(result, expected)

        df[df > 0] = 5
        values[values > 0] = 5
        tm.assert_almost_equal(df.values, values)

        df[df == 5] = 0
        values[values == 5] = 0
        tm.assert_almost_equal(df.values, values)

        # a df that needs alignment first
        df[df[:-1] < 0] = 2
        np.putmask(values[:-1], values[:-1] < 0, 2)
        tm.assert_almost_equal(df.values, values)

        with pytest.raises(TypeError, match='boolean values only'):
            df[df * 0] = 2

    def test_frame_getitem_setitem_multislice(self):
        levels = [['t1', 't2'], ['a', 'b', 'c']]
        codes = [[0, 0, 0, 1, 1], [0, 1, 2, 0, 1]]
        midx = MultiIndex(codes=codes, levels=levels, names=[None, 'id'])
        df = DataFrame({'value': [1, 2, 3, 7, 8]}, index=midx)

        result = df.loc[:, 'value']
        tm.assert_series_equal(df['value'], result)

        with catch_warnings(record=True):
            simplefilter("ignore", DeprecationWarning)
            result = df.ix[:, 'value']
        tm.assert_series_equal(df['value'], result)

        result = df.loc[df.index[1:3], 'value']
        tm.assert_series_equal(df['value'][1:3], result)

        result = df.loc[:, :]
        tm.assert_frame_equal(df, result)

        result = df
        df.loc[:, 'value'] = 10
        result['value'] = 10
        tm.assert_frame_equal(df, result)

        df.loc[:, :] = 10
        tm.assert_frame_equal(df, result)

    def test_frame_setitem_multi_column(self):
        df = DataFrame(randn(10, 4), columns=[['a', 'a', 'b', 'b'],
                                              [0, 1, 0, 1]])

        cp = df.copy()
        cp['a'] = cp['b']
        tm.assert_frame_equal(cp['a'], cp['b'])

        # set with ndarray
        cp = df.copy()
        cp['a'] = cp['b'].values
        tm.assert_frame_equal(cp['a'], cp['b'])

        # ---------------------------------------
        # #1803
        columns = MultiIndex.from_tuples([('A', '1'), ('A', '2'), ('B', '1')])
        df = DataFrame(index=[1, 3, 5], columns=columns)

        # Works, but adds a column instead of updating the two existing ones
        df['A'] = 0.0  # Doesn't work
        assert (df['A'].values == 0).all()

        # it broadcasts
        df['B', '1'] = [1, 2, 3]
        df['A'] = df['B', '1']

        sliced_a1 = df['A', '1']
        sliced_a2 = df['A', '2']
        sliced_b1 = df['B', '1']
        tm.assert_series_equal(sliced_a1, sliced_b1, check_names=False)
        tm.assert_series_equal(sliced_a2, sliced_b1, check_names=False)
        assert sliced_a1.name == ('A', '1')
        assert sliced_a2.name == ('A', '2')
        assert sliced_b1.name == ('B', '1')

    def test_getitem_setitem_tuple_plus_columns(
            self, multiindex_year_month_day_dataframe_random_data):
        # GH #1013
        ymd = multiindex_year_month_day_dataframe_random_data
        df = ymd[:5]

        result = df.loc[(2000, 1, 6), ['A', 'B', 'C']]
        expected = df.loc[2000, 1, 6][['A', 'B', 'C']]
        tm.assert_series_equal(result, expected)

    def test_getitem_setitem_slice_integers(self):
        index = MultiIndex(levels=[[0, 1, 2], [0, 2]],
                           codes=[[0, 0, 1, 1, 2, 2], [0, 1, 0, 1, 0, 1]])

        frame = DataFrame(np.random.randn(len(index), 4), index=index,
                          columns=['a', 'b', 'c', 'd'])
        res = frame.loc[1:2]
        exp = frame.reindex(frame.index[2:])
        tm.assert_frame_equal(res, exp)

        frame.loc[1:2] = 7
        assert (frame.loc[1:2] == 7).values.all()

        series = Series(np.random.randn(len(index)), index=index)

        res = series.loc[1:2]
        exp = series.reindex(series.index[2:])
        tm.assert_series_equal(res, exp)

        series.loc[1:2] = 7
        assert (series.loc[1:2] == 7).values.all()

    def test_setitem_change_dtype(self, multiindex_dataframe_random_data):
        frame = multiindex_dataframe_random_data
        dft = frame.T
        s = dft['foo', 'two']
        dft['foo', 'two'] = s > s.median()
        tm.assert_series_equal(dft['foo', 'two'], s > s.median())
        # assert isinstance(dft._data.blocks[1].items, MultiIndex)

        reindexed = dft.reindex(columns=[('foo', 'two')])
        tm.assert_series_equal(reindexed['foo', 'two'], s > s.median())

    def test_set_column_scalar_with_ix(self, multiindex_dataframe_random_data):
        frame = multiindex_dataframe_random_data
        subset = frame.index[[1, 4, 5]]

        frame.loc[subset] = 99
        assert (frame.loc[subset].values == 99).all()

        col = frame['B']
        col[subset] = 97
        assert (frame.loc[subset, 'B'] == 97).all()

    def test_nonunique_assignment_1750(self):
        df = DataFrame([[1, 1, "x", "X"], [1, 1, "y", "Y"], [1, 2, "z", "Z"]],
                       columns=list("ABCD"))

        df = df.set_index(['A', 'B'])
        ix = MultiIndex.from_tuples([(1, 1)])

        df.loc[ix, "C"] = '_'

        assert (df.xs((1, 1))['C'] == '_').all()

    def test_astype_assignment_with_dups(self):

        # GH 4686
        # assignment with dups that has a dtype change
        cols = MultiIndex.from_tuples([('A', '1'), ('B', '1'), ('A', '2')])
        df = DataFrame(np.arange(3).reshape((1, 3)),
                       columns=cols, dtype=object)
        index = df.index.copy()

        df['A'] = df['A'].astype(np.float64)
        tm.assert_index_equal(df.index, index)

reason: import numpy,from numpy


text:
import json
import os
from collections import OrderedDict

import numpy as np

dist_path = [
    # '/home/dgy/project/pytorch/fast-reid/logs/NAIC/R101-ibn-arcF-cj-st-amp-gem-all-ranger2/R101-ibn-arcF-cj-st-amp-gem-all-ranger2_it0225599_6558_aqe_cos_rerank.npy',
    # '/home/dgy/project/pytorch/fast-reid/logs/NAIC/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-iebn2/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-iebn2_it0225599_6643_cos_rerank.npy',
    # '/home/dgy/project/pytorch/fast-reid/logs/NAIC/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-ntl/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-ntl_it0115199_0119999_0110399_6512_aqe_cos_rerank.npy'
    # "/root/reid2020/logs/HIT/mgn--R101-plain/mgn--R101-plain_it0105599_6903_cos_rerank.npy",
    "./logs/HIT/mgn--R101-plain/mgn--R101-plain_it0158399_7004_aqe_cos_rerank.npy",
    # "./logs/HIT/mgn--R101-arcFace/mgn--R101-arcFace_it0112799_6668_cos_rerank.npy",
    # "./logs/HIT/bt-arcF-cj-19/bt-arcF-cj-19_it0069599_5392_cos_rerank.npy",
    # "./logs/HIT/bt-EFN0-arcF-cj-19/bt-EFN0-arcF-cj-19_it0167999_5283_aqe_cos_rerank.npy",
    "/root/reid2020/logs/models/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-iebn2/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-iebn2_it0206399_6657_cos_rerank.npy",
    "/root/reid2020/logs/models/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-lm/R101-ibn-arcF-cj-st-amp-gem-all-ranger2-lm_it0244799_6654_aqe_cos_rerank.npy"

]
output_path = './logs/HIT/ensemble/'

img_order_path = [path[:-4] + '_order.json' for path in dist_path]
img_order = []
for path in img_order_path[1:]:
    with open(path, 'r') as f:
        order_dict = json.load(f)
        img_order.append(order_dict)

query_paths = img_order[0]['query_paths']
gallery_paths = img_order[0]['gallery_paths']
for order_dict in img_order[1:]:
    assert order_dict['query_paths'] == query_paths and order_dict['gallery_paths'] == gallery_paths

dist_matrices = [np.load(path) for path in dist_path]
dist_matrices = sum(dist_matrices)
indices = np.argsort(dist_matrices, axis=1)

gallery_paths = np.asarray(gallery_paths)
submit = OrderedDict()
for i, key in enumerate(query_paths):
    submit[key] = gallery_paths[indices[i, :200]].tolist()

saved_name = '_'.join([os.path.split(path)[-1][:-4] for path in dist_path])
# if len(saved_name) > 10:
#     saved_name = saved_name[:10]
with open(os.path.join(output_path, saved_name + '.json'), 'w') as f:
    json.dump(submit, f)


text:
import math
import numpy as np
import torch
import torch.nn as nn

import os
import time
import collections
import matplotlib.pyplot as plt
import logging

flog = None

################################################################################
# Sampling
################################################################################
# -- temperature -- #
def softmax_with_temperature(logits, temperature):
    probs = np.exp(logits / temperature) / np.sum(np.exp(logits / temperature))
    return probs

def weighted_sampling(probs):
    probs /= sum(probs)
    sorted_probs = np.sort(probs)[::-1]
    sorted_index = np.argsort(probs)[::-1]
    word = np.random.choice(sorted_index, size=1, p=sorted_probs)[0]
    return word

# -- nucleus -- #
def nucleus(probs, p):
    probs /= (sum(probs) + 1e-5)
    sorted_probs = np.sort(probs)[::-1]
    sorted_index = np.argsort(probs)[::-1]
    cusum_sorted_probs = np.cumsum(sorted_probs)
    after_threshold = cusum_sorted_probs > p
    if sum(after_threshold) > 0:
        last_index = np.where(after_threshold)[0][0] + 1
        candi_index = sorted_index[:last_index]
    else:
        candi_index = sorted_index[:]
    candi_probs = [probs[i] for i in candi_index]
    candi_probs /= sum(candi_probs)
    word = np.random.choice(candi_index, size=1, p=candi_probs)[0]
    return word

def sampling(logit, p=None, t=1.0):
    logit = logit.squeeze().cpu().numpy()
    probs = softmax_with_temperature(logits=logit, temperature=t)

    if p is not None:
        cur_word = nucleus(probs, p=p)
    else:
        cur_word = weighted_sampling(probs)
    return cur_word

################################################################################
# Model
################################################################################

def network_paras(model):
    # compute only trainable params
    model_parameters = filter(lambda p: p.requires_grad, model.parameters())
    params = sum([np.prod(p.size()) for p in model_parameters])
    return params

class Embeddings(nn.Module):
    def __init__(self, n_token, d_model):
        super(Embeddings, self).__init__()
        self.lut = nn.Embedding(n_token, d_model)
        self.d_model = d_model

    def forward(self, x):
        return self.lut(x) * math.sqrt(self.d_model)

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, dropout=0.1, max_len=20000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        # print(x.shape, self.pe[:, :x.size(1), :].shape)
        x = x + self.pe[:, :x.size(1), :]
        return self.dropout(x)

class BeatPositionalEncoding(nn.Module):
    def __init__(self, d_model, dropout=0.1, max_len=20000):
        super(BeatPositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x, index):
        x = x + self.pe[:, index, :]
        return self.dropout(x)[0]

class Saver(object):
    def __init__(
            self,
            exp_dir,
            mode='w',
            debug=False):

        self.exp_dir = exp_dir
        self.init_time = time.time()
        self.global_step = 0
        self.debug = debug

        # makedirs
        os.makedirs(exp_dir, exist_ok=True)

        # logging config
        path_logger = os.path.join(exp_dir, 'log.txt')
        logging.basicConfig(
            level=logging.DEBUG,
            format='%(message)s',
            filename=path_logger,
            filemode=mode)
        self.logger = logging.getLogger('training monitor')

    def add_summary_msg(self, msg):
        if self.debug:
            return
        self.logger.debug(msg)

    def add_summary(
            self,
            key,
            val,
            step=None,
            cur_time=None):
        if self.debug:
            return
        if cur_time is None:
            cur_time = time.time() - self.init_time
        if step is None:
            step = self.global_step

        # write msg (key, val, step, time)
        if isinstance(val, float):
            msg_str = '{:10s} | {:.10f} | {:10d} | {}'.format(
                key,
                val,
                step,
                cur_time
            )
        else:
            msg_str = '{:10s} | {} | {:10d} | {}'.format(
                key,
                val,
                step,
                cur_time
            )

        self.logger.debug(msg_str)

    def save_model(
            self,
            model,
            optimizer=None,
            outdir=None,
            name='model'):
        if self.debug:
            return
        if outdir is None:
            outdir = self.exp_dir
        print(' [*] saving model to {}, name: {}'.format(outdir, name))
        # torch.save(model, os.path.join(outdir, name+'.pt'))
        torch.save(model.state_dict(), os.path.join(outdir, name + '_params.pt'))

        if optimizer is not None:
            torch.save(optimizer.state_dict(), os.path.join(outdir, name + '_opt.pt'))

    def load_model(
            self,
            path_exp,
            device='cpu',
            name='model.pt'):

        path_pt = os.path.join(path_exp, name)
        print(' [*] restoring model from', path_pt)
        model = torch.load(path_pt, map_location=torch.device(device))
        return model

    def global_step_increment(self):
        self.global_step += 1

def make_loss_report(
        path_log,
        path_figure='loss.png',
        dpi=100):
    # load logfile
    monitor_vals = collections.defaultdict(list)
    with open(path_log, 'r') as f:
        for line in f:
            try:
                line = line.strip()
                key, val, step, acc_time = line.split(' | ')
                monitor_vals[key].append((float(val), int(step), acc_time))
            except:
                continue

    # collect
    step_train = [item[1] for item in monitor_vals['train loss']]
    vals_train = [item[0] for item in monitor_vals['train loss']]

    step_valid = [item[1] for item in monitor_vals['valid loss']]
    vals_valid = [item[0] for item in monitor_vals['valid loss']]

    x_min = step_valid[np.argmin(vals_valid)]
    y_min = min(vals_valid)

    # plot
    fig = plt.figure(dpi=dpi)
    plt.title('training process')
    plt.plot(step_train, vals_train, label='train')
    plt.plot(step_valid, vals_valid, label='valid')
    plt.yscale('log')
    plt.plot([x_min], [y_min], 'ro')
    plt.legend(loc='upper right')
    plt.tight_layout()
    plt.savefig(path_figure)

def log(*args, **kwargs):
    print(*args, **kwargs)
    if flog is not None:
        print(*args, file=flog, flush=True)

max_issues_repo_name: WuKaiYi/video-bgm-generation

max_forks_repo_name: WuKaiYi/video-bgm-generation


text:
import sim
import time
import math
import numpy as np
import cv2
import matplotlib.pyplot as mpl

sim.simxFinish(-1)    # just in case, close all opened connections
clientID = sim.simxStart('127.0.0.1', 19999, True, True, 5000, 5)    # Connect to CoppeliaSim

if clientID != -1:
    print ('Connected to remote API server')

    #-------------------------------- start--初始化部分 --------------------------------#
    ret, URhandle1 = sim.simxGetObjectHandle(clientID, 'UR5_joint1',
                     sim.simx_opmode_blocking)
    ret, URhandle2 = sim.simxGetObjectHandle(clientID, 'UR5_joint2',
                     sim.simx_opmode_blocking)
    ret, URhandle3 = sim.simxGetObjectHandle(clientID, 'UR5_joint3',
                     sim.simx_opmode_blocking)
    ret, URhandle4 = sim.simxGetObjectHandle(clientID, 'UR5_joint4',
                     sim.simx_opmode_blocking)
    ret, URhandle5 = sim.simxGetObjectHandle(clientID, 'UR5_joint5',
                     sim.simx_opmode_blocking)
    ret, URhandle6 = sim.simxGetObjectHandle(clientID, 'UR5_joint6',
                     sim.simx_opmode_blocking)
    ret, KinectRGBAbovehandle = sim.simxGetObjectHandle(clientID, 'kinect_rgb_above',
                     sim.simx_opmode_blocking)
    ret, KinectDepthAbovehandle = sim.simxGetObjectHandle(clientID, 'kinect_depth_above',
                     sim.simx_opmode_blocking)
    ret, KinectRGBhandle = sim.simxGetObjectHandle(clientID, 'kinect_rgb',
                     sim.simx_opmode_blocking)
    ret, KinectDepthhandle = sim.simxGetObjectHandle(clientID, 'kinect_depth',
                     sim.simx_opmode_blocking)

    #-------------------------------- end--初始化部分 --------------------------------#

    #-------------------------------- start--主程序部分 --------------------------------#
    # 测试控制机械臂关节角度，并读取角度值
    ret = sim.simxSetJointTargetPosition(clientID, URhandle1, np.pi, sim.simx_opmode_blocking)
    ret,  position1 = sim.simxGetJointPosition(clientID, URhandle1, sim.simx_opmode_blocking)
    print(position1)

    targetPos = (-0.022, -0.340, 0.75)
    sim_ret, UR5_target_handle = sim.simxGetObjectHandle(clientID, 'UR5_Target', sim.simx_opmode_blocking)
    sim.simxSetObjectPosition(clientID, UR5_target_handle, -1, targetPos, sim.simx_opmode_blocking)

    #-------------------------------- end--主程序部分 --------------------------------#
else:
    print ('Failed connecting to remote API server')

max_stars_repo_name: RrQqY/TableRearrangement

max_issues_repo_name: RrQqY/TableRearrangement

max_forks_repo_name: RrQqY/TableRearrangement


text:
"""Module containing classes for regularization of linear models."""
import numpy as np

class l1_regularization():
    """
    Add l1 regularization penalty to linear models.

    Regularization term:

        alpha * ||w||

    where w is the vector of feature weights and alpha is the hyperparameter
    controlling how much regularization is done to the model.

    Parameters
    ----------
    alpha : float, default=1.0
        Factor determining the amount of regularization to be performed on
        the model.

    Notes
    -----
    The bias term is not regularized and therefore should be omitted from the
    feature weights as input.
    """
    def __init__(self, alpha=1.0):
        self.alpha = alpha

    def __call__(self, w):
        "Calculate l1 regularization term."
        return self.alpha * np.linalg.norm(w, 1)

    def grad(self, w):
        """Calculate subgradient vector of l1 regularization penalty.

                      -1 if w_i < 0
            sign(w) =  0 if w_i = 0
                       1 if w_i > 0

        where w is the vector of feature weights.
        """
        subgradient = self.alpha * np.sign(w)

        # Insert 0 for bias term.
        return np.insert(subgradient, 0, 0, axis=0)

class l2_regularization():
    """
    Add l2 regularization penalty to linear models.

    Regularization term:

        alpha * 1/2 * ||w||^2

    Where w is the vector of feature weights and alpha is the hyperparameter
    controlling how much regularization is done to the model.

    Parameters
    ----------
    alpha : float, default=1.0
        Factor determining the amount of regularization to be performed on
        the model.

    Notes
    -----
    The bias term is not regularized and therefore should be omitted from the
    feature weights as input.
    """
    def __init__(self, alpha=1.0):
        self.alpha = alpha

    def __call__(self, w):
        """Calculate regularization term."""
        return self.alpha * 0.5 * np.linalg.norm(w, 2)

    def grad(self, w):
        """
        Calculate gradient descent regularization term.

            alpha * w

        where alpha is the factor determining the amount of regularization and
        w is the vector of feature weights.
        """
        gradient_penalty = np.asarray(self.alpha) * w

        # Insert 0 for bias term.
        return np.insert(gradient_penalty, 0, 0, axis=0)

class l1_l2_regularization():
    """
    Add a mix of l1 and l2 regularization penalty to linear models.

    Regularization term:

        r * alpha * ||w|| + (1 - r) / 2 * alpha * ||w||^2

    where r is the mix ratio, alpha is the factor determining the amount
    of regularization and w is the vector of feature weights.

    Parameters
    ----------
    alpha : float, default=1.0
        Factor determining the amount of regularization to be performed on
        the model.
    r : float, default=0.5
        Mix ratio determining the amount of l1 vs l2 regularization to add.
        A value of 0 is equivalent to l2 regularization and a value of 1 is
        equivalent to l1 regularization.

    Notes
    -----
    The bias term is not regularized and therefore should be omitted from the
    feature weights as input.
    """
    def __init__(self, alpha=1.0, r=0.5):
        self.alpha = alpha
        self.r = r

    def __call__(self, w):
        """Calculate elastic net regularization penalty."""
        l1_term = self.alpha * np.linalg.norm(w, 1)
        l2_term = self.alpha * 0.5 * np.linalg.norm(w, 2)

        return self.r * l1_term + (1 - self.r) * l2_term

    def grad(self, w):
        """
        Calculate gradient descent regularization penalty.

            alpha * (r * sign(w) + (1 - r) * w)

        where r is the mix ratio, alpha is the factor determining the amount
        of regularization and w is the vector of feature weights.
        """
        l1_grad = self.r * np.sign(w)
        l2_grad = np.asarray(1 - self.r) * w

        gradient_penalty = self.alpha * (l1_grad + l2_grad)

        # Insert 0 for bias term.
        return np.insert(gradient_penalty, 0, 0, axis=0)


text:
"""
Neural Perceptron Network

@author Caio Tomich (caiotomich@gmail.com)
@version 1.0.0
@since 2019-08-27;
"""

import numpy as np
import matplotlib.pyplot as plt

def activation_func(val):
    if val < 0:
        return -1
    return 1

training_data = np.genfromtxt('../docs/training_data.txt', skip_header=False)

t = np.full(30, -1)
w = np.random.uniform(low=-1, high=1, size=4)
x = np.hstack((t[:, np.newaxis], training_data[:, 0:3]))
d = training_data[:, 3]

print("\n --- Training Script --- \n")
print("Initial Weight:", w)

epoch = 0
error = True
weight = []
learn_rate = 0.01

while error:
    error = False

    for i in range(len(x)):
        u = np.dot(weight, x[i])
        y = activation_func(u)

        if y != d[i]:
            w = w + (learn_rate * (d[i] - y) * x[i])
            error = True

    weight.append(w)
    epoch += 1

print("End Weight:", w)
print("Epoch:", epoch)

plt.plot(np.arange(epoch), np.array(weight))
plt.title('Weight x Epoch')
plt.xlabel('Epoch')
plt.ylabel('Weight')
plt.show()

print("\n --- Test Script --- \n")

validation_data = np.genfromtxt('../docs/validation_data.txt', skip_header=False)

w_after_training = weight
t = np.full(len(validation_data), -1)
x = np.hstack((t[:, np.newaxis], validation_data))

y_expected = [-1, 1, 1, 1, 1, 1, -1, 1, -1, -1]
y_response = []

for i in range(len(x)):
    u = np.dot(w_after_training, x[i])
    y = activation_func(u)
    y_response.append(y)

    if y == -1:
        print("Sample: {} => Class P1".format(x[i]))
    else:
        print("Sample: {} => Class P2".format(x[i]))

print("\nExpected output after training")
print("y_expected == y_response: {}".format(np.equal(y_expected, y_response)))

max_stars_repo_name: caiotomich/rede-perceptron-simples

max_issues_repo_name: caiotomich/rede-perceptron-simples

max_forks_repo_name: caiotomich/rede-perceptron-simples


text:
#!/usr/bin/env python

""" I/O utilities for dealing with SIU files """

import os
import struct
import numpy as np

__all__ = ["read_line_list", "write_line_list"]

def chunk(l, n):
    return [l[i:i + n] for i in range(0, len(l), n)]

def guess_mode(filename):
    textchars = ''.join(map(chr, [7,8,9,10,12,13,27] + range(0x20, 0x100)))
    is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
    with open(filename) as fp:
        is_binary = is_binary_string(fp.read(1024))
    return ["w", "b"][is_binary]

def _tuple_to_recarray(transition):
    columns = ("wavelength", "excitation_potential", "J_low", "J_up", "xx1",
        "log(gf)", "xx2", "log(c4)?", "log(c6)", "xx3", "xx4", "xx5", "xx6",
        "xx7", "xx8", "xx9", "xx10", "atomic_number", "xx11", "ionised")
    return np.core.records.fromrecords([transition], names=columns, formats=None)

def read_line_list(filename, mode=None, ignore_blanks=True):
    """
    Read a line list and return a record array.

    :param filename:
        The path of the line list.

    :type filename:
        str

    :param mode: [optional]
        The mode to open the file in. Specifying mode "w" indicates ASCII format.
        Specifying the mode as "b" indicates binary format. If no mode is given,
        it will be guessed from the file contents.

    :type mode:
        str

    :returns:
        A record array of the line list.
    """

    # Read it into tuples, regardless of binary/ascii
    if mode is None:
        mode = guess_mode(filename)

    if "b" in mode.lower():
        records = _read_binary_line_list(filename)
    else:
        records = _read_ascii_line_list(filename)

    # Convert to a record array
    columns = ("wavelength", "excitation_potential", "J_low", "J_up", "xx1",
        "log(gf)", "xx2", "log(c4)?", "log(c6)", "xx3", "xx4", "xx5", "xx6",
        "xx7", "xx8", "xx9", "xx10", "atomic_number", "xx11", "ionised")
    data = np.core.records.fromrecords(records, names=columns, formats=None)
    return data[data["wavelength"] > 0] if ignore_blanks else data

def write_line_list(filename, data, mode="w", clobber=False):
    """
    Write the line list to disk.

    :param filename:
        The path of the line list to write to.

    :type filename:
        str

    :param data:
        The line list information as a record array.

    :type data:
        :class:`np.core.records.recarray`

    :param mode: [optional]
        The mode to open the file in. The default (w) indicates ASCII format.
        Specifying the mode as "b" indicates binary format.

    :type mode:
        str

    :param clobber: [optional]
        Clobber the file if it already exists.

    :type clobber:
        bool
    """

    if os.path.exists(filename) and not clobber:
        raise IOError("filename {0} exists and not clobbering".format(filename))

    # Data should be in record array. Can pass to either binary or ascii writer.
    if "b" in mode.lower():
        return _write_binary_line_list(filename, data)
    else:
        return _write_ascii_line_list(filename, data)

def _write_ascii_line_list(filename, data):
    """
    Write line list to ASCII format.
    """

    #  2010.974   0.0300  2.0  3.0   0.1  -3.3800   0.0000   0.0000 -31.1250   0
    #.0000   0.0000         2    uv6   3P   1G  KP   STD   14  0  1
    record = "{0:10.3f}{1:9.4f}{2:5.1f}{3:5.1f}{4:6.1f}{5:9.4f}{6:9.4f}{7:9.4f}"\
        "{8:9.4f}{9:9.4f}{10:9.4f}{11:10.0f}  {12:>5s}  {13:>3s}  {14:>3s} {15"\
        ":>3s}   {16:>3s}  {17:3.0f}{18:3.0f}{19:3.0f}\n"

    scales = np.ones(len(data.dtype.names))
    scales[6] = 1e+8

    with open(filename, "w+") as fp:
        for row in data:
            row_data = row * scales
            fp.write(record.format(*row_data))

    return True

def _write_binary_line_list(filename, data, num=200):
    """
    Write line list to binary format.
    """

    assert 3.8e5 >= len(data), "SI line list format supports at most 380000 lines"

    num_structures = int(np.ceil(len(data)/float(num)))

    # Create the header
    header_values = np.zeros((1900))
    header_values[:num_structures] = data["wavelength"][::num]
    header = struct.pack("<1900d", *header_values)

    # Create the structures
    contents = header
    for i in xrange(num_structures):
        contents += _pack_binary_structure(data[i*num:(i+1)*num], num)

    with open(filename, "wb+") as fp:
        fp.write(contents)
    return True

def _pack_binary_structure(data, num=200):

    assert num >= len(data)

    columns = ["wavelength", "excitation_potential", "J_low", "J_up", "xx1",
        "log(gf)", "xx2", "log(c4)?", "log(c6)", "xx3", "xx4", "xx5", "xx6",
        "xx7", "xx8", "xx9", "xx10", "atomic_number", "xx11", "ionised"]

    data = data[columns]

    if num > len(data):
        # We will need to copy the data and pad it.
        data = data.copy()
        padded_row = np.array([(0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0,
            ' ', ' ', ' ', ' ', ' ', 0, 0, 0)], dtype=data.dtype)
        for i in xrange(num - len(data)):
            data = np.append(data, padded_row)

    colnames = data.dtype.names[:12]
    scales = np.ones(len(colnames))
    scales[6] = 1e+8
    formatted_data = struct.pack(
        "<{0}d{0}d{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}l".format(num),
        *np.array([data[c]*s for c, s in zip(colnames, scales)]).flatten())

    sizes = (5, 3, 3, 3, 3)
    colnames = data.dtype.names[12:17]
    formatted_byte_data = struct.pack("<{0}s{1}s{2}s{3}s{4}s".format(
        *[num*s for s in sizes]), *["".join(list(data[c])) for c in colnames])

    colnames = data.dtype.names[17:]
    formatted_element_data = struct.pack("<{0}b{0}b{0}b".format(num),
        *np.array([data[c] for c in colnames]).flatten())

    return "".join([formatted_data, formatted_byte_data, formatted_element_data])

def _read_ascii_line_list(filename):
    """
    Read an ASCII-formatted line list and return a list containing tuples for
    each record.
    """

    with open(filename, "r") as fp:
        lines = fp.readlines()
    return map(_read_ascii_line, lines)

def _read_ascii_line(line):
    """
    Read the structure containined in an ASCII line and return the record as a
    tuple.
    """

    #  2010.974   0.0300  2.0  3.0   0.1  -3.3800   0.0000   0.0000 -31.1250   \
    #0.0000   0.0000         2    uv6   3P   1G  KP   STD   14  0  1
    record = []
    record.extend(map(float, line.split()[:12]))
    record[6] = record[6] * 1e-8
    record[-1] = int(record[-1])

    # Split the text:
    record.extend(map(str.strip, chunk(line[101:126], 5)))

    # And the final element information
    record.extend(map(int, line[127:].split()))
    return tuple(record)

def _read_binary_line_list(filename):
    """
    Read a binary-formatted line list and return a list containing tuples for
    each record.
    """

    with open(filename, "rb") as fp:
        # Skip the header (this is also length 15200)
        fp.seek(0x3b60)
        contents = fp.read()

    size = 15200
    return sum([_read_binary_structure(contents[i*size:]) \
        for i in xrange(len(contents)/size)], [])

def _read_binary_structure(contents, num=200):
    """
    Read the structure contained in a binary line list and return the records
    of the structure.
    """

    data_fmt = "<{0}d{0}d{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}f{0}l".format(num)
    data = struct.unpack_from(data_fmt, contents)

    sizes = (5, 3, 3, 3, 3)
    offset = struct.calcsize(data_fmt)
    byte_fmt = "<{0}s{1}s{2}s{3}s{4}s".format(*[num*s for s in sizes])
    str_data = [chunk(d, s) for d, s in \
        zip(struct.unpack_from(byte_fmt, contents[offset:]), sizes)]

    offset += struct.calcsize(byte_fmt)
    element_fmt = "<{0}b{0}b{0}b".format(num)
    element_data = struct.unpack_from(element_fmt, contents[offset:])

    # Join records together
    records = []
    for i in xrange(num):
        record = list(data[i::num])
        record[6] *= 1e-8
        record.extend([each[i] for each in str_data])
        record.extend(element_data[i::num])
        records.append(tuple(record))
    return records

max_stars_repo_name: andycasey/original-oracle

max_issues_repo_name: andycasey/original-oracle

max_forks_repo_name: andycasey/original-oracle


text:
from gym import Wrapper
import numpy as np
from gym import spaces
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
import seaborn as sb

sb.set_style("darkgrid")
current_palette = list(sb.color_palette())

class ActionFrequencyWrapper(Wrapper):
    """
    Wrapper to action frequency.
    Includes interval mode that returns frequencies in lists of len(interval) instead of one long list.
    """

    def __init__(self, env, action_interval=None):
        """
        Initialize wrapper

        Parameters
        -------
        env : gym.Env
            Environment to wrap
        action_interval : int
            If not none, mean in given intervals is tracked, too
        """
        super(ActionFrequencyWrapper, self).__init__(env)
        self.action_interval = action_interval
        self.overall_actions = []
        if self.action_interval:
            self.action_intervals = []
            self.current_actions = []
        self.action_space_type = type(self.env.action_space)

    def __setattr__(self, name, value):
        """
        Set attribute in wrapper if available and in env if not

        Parameters
        ----------
        name : str
            Attribute to set
        value
            Value to set attribute to
        """
        if name in [
            "action_interval",
            "overall_actions",
            "action_intervals",
            "current_actions",
            "env",
            "get_actions",
            "step",
            "render_action_tracking",
        ]:
            object.__setattr__(self, name, value)
        else:
            setattr(self.env, name, value)

    def __getattribute__(self, name):
        """
        Get attribute value of wrapper if available and of env if not

        Parameters
        ----------
        name : str
            Attribute to get

        Returns
        -------
        value
            Value of given name
        """
        if name in [
            "action_interval",
            "overall_actions",
            "action_intervals",
            "current_actions",
            "env",
            "get_actions",
            "step",
            "render_action_tracking",
        ]:
            return object.__getattribute__(self, name)

        else:
            return getattr(self.env, name)

    def step(self, action):
        """
        Execute environment step and record state

        Parameters
        ----------
        action : int
            action to execute

        Returns
        -------
        np.array, float, bool, dict
            state, reward, done, metainfo
        """
        state, reward, done, info = self.env.step(action)
        self.overall_actions.append(action)
        if self.action_interval:
            if len(self.current_actions) < self.action_interval:
                self.current_actions.append(action)
            else:
                self.action_intervals.append(self.current_actions)
                self.current_actions = [action]
        return state, reward, done, info

    def get_actions(self):
        """
        Get state progression

        Returns
        -------
        np.array or np.array, np.array
            all states or all states and interval sorted states

        """
        if self.action_interval:
            complete_intervals = self.action_intervals + [self.current_actions]
            return self.overall_actions, complete_intervals

        else:
            return self.overall_actions

    def render_action_tracking(self):
        """
        Render action progression

        Returns
        -------
        np.array
            RBG data of action tracking

        """

        def plot_single(ax=None, index=None, title=None, x=False, y=False):
            if ax is None:
                plt.xlabel("Step")
                plt.ylabel("Action value")
            elif x and y:
                ax.set_ylabel("Action value")
                ax.set_xlabel("Step")
            elif x:
                ax.set_xlabel("Step")
            elif y:
                ax.set_ylabel("Action value")

            if index is not None:
                ys = [state[index] for state in self.overall_actions]
            else:
                ys = self.overall_actions

            if ax is None:
                p = plt.plot(
                    np.arange(len(self.overall_actions)),
                    ys,
                    label="Step actions",
                    color="g",
                )
            else:
                p = ax.plot(
                    np.arange(len(self.overall_actions)),
                    ys,
                    label="Step actions",
                    color="g",
                )
            p2 = None
            if self.action_interval:
                if index is not None:
                    y_ints = []
                    for interval in self.action_intervals:
                        y_ints.append([state[index] for state in interval])
                else:
                    y_ints = self.action_intervals
                if ax is None:
                    p2 = plt.plot(
                        np.arange(len(self.action_intervals)) * self.action_interval,
                        [np.mean(interval) for interval in y_ints],
                        label="Mean interval action",
                        color="orange",
                    )
                    plt.legend(loc="upper left")
                else:
                    p2 = ax.plot(
                        np.arange(len(self.action_intervals)) * self.action_interval,
                        [np.mean(interval) for interval in y_ints],
                        label="Mean interval action",
                        color="orange",
                    )
                    ax.legend(loc="upper left")
            return p, p2

        if self.action_space_type == spaces.Discrete:
            figure = plt.figure(figsize=(12, 6))
            canvas = FigureCanvas(figure)
            p, p2 = plot_single()
            canvas.draw()
        elif self.action_space_type == spaces.Dict:
            raise NotImplementedError

        elif self.action_space_type == spaces.Tuple:
            raise NotImplementedError

        elif (
            self.action_space_type == spaces.MultiDiscrete
            or self.action_space_type == spaces.MultiBinary
            or self.action_space_type == spaces.Box
        ):
            if self.action_space_type == spaces.MultiDiscrete:
                action_size = len(self.env.action_space.nvec)
            elif self.action_space_type == spaces.MultiBinary:
                action_size = self.env.action_space.n
            else:
                action_size = len(self.env.action_space.high)

            if action_size == 1:
                figure = plt.figure(figsize=(12, 6))
                canvas = FigureCanvas(figure)
                p, p2 = plot_single()
            elif action_size < 5:
                dim = 1
                figure, axarr = plt.subplots(action_size)
            else:
                dim = action_size % 4
                figure, axarr = plt.subplots(action_size % 4, action_size // dim)
            figure.suptitle("State over time")
            canvas = FigureCanvas(figure)
            for i in range(action_size):
                if action_size == 1:
                    continue

                x = False
                if i % dim == dim - 1:
                    x = True
                if action_size < 5:
                    p, p2 = plot_single(axarr[i], i, y=True, x=x)
                else:
                    y = i % action_size // dim == 0
                    p, p2 = plot_single(axarr[i % dim, i // dim], i, x=x, y=y)
            canvas.draw()
        width, height = figure.get_size_inches() * figure.get_dpi()
        img = np.fromstring(canvas.tostring_rgb(), dtype="uint8").reshape(
            int(height), int(width), 3
        )
        return img