text:
import pandas as pd
import numpy as np

from scipy import stats

def columns_views(player_1_df, player_2_df):

    columns = list(player_1_df.columns)
    if list(player_1_df.columns) == list(player_2_df.columns):
        columns = list(player_1_df.columns)

        player_1 = list(player_1_df.values[0])
        player_2 = list(player_2_df.values[0])

        views = []
        for column, player1, player2 in zip(columns, player_1, player_2):
            print('column : {} _ player1-{} , player2-{} < diff : {} >'.format(
                column, player1, player2, abs(player1 - player2)
            ))
            views.append(abs(player1 - player2))

        print(views)

def convert_preferred_foot(df):

    df['preferred_foot'] = df['preferred_foot'].replace('Right', 1)
    df['preferred_foot'] = df['preferred_foot'].replace('Left', 2)

    return df

def convert_work_rate(df):

    convert = {
        'High': 3,
        'Medium': 2,
        'Low': 1
    }

    work_rate = df['work_rate'].values[0].split('/')

    attack = work_rate[0]
    defense = work_rate[1]

    df['attack'] = convert[attack]
    df['defense'] = convert[defense]

    # work_rateの削除処理
    df = df.drop(columns='work_rate')

    return df

def euclidean_distance(v1, v2):
    # ユーグリッド距離を算出
    # https://qiita.com/shim0mura/items/64918dad83d162ef2ac2#ユークリッド距離

    # どちらも同じ値を返す
    # distance = np.linalg.norm(v1 - v2)
    distance = np.sqrt(np.power(v1 - v2, 2).sum())

    # 0から1までの値で似ていれば似ているほど1に近くなる、みたいな類似度として分かりやすい値が欲しい。
    # 0での除算エラーを防ぐためにこのdに1を足して逆数をとるとそのような値を取ることが出来る。
    # 1/(1+d)

    # print('distance', distance)

    return 1 / (1 + distance)

def cos_similarity(v1, v2):
    # Scipyを使ってコサイン類似度を求める方法
    # import scipy.spatial.distance as dis

    # print(np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2)))
    # print(dis.cosine(v1, v2))

    # return dis.cosine(v1, v2)

    # cos類似度を算出
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))

# ピアソンの積率相関係数
def pearson_product_moment_correlation_coefficien(v1, v2):
    # corr = np.corrcoef(v1, v2)[0, 1]
    corr = stats.pearsonr(v1, v2)

    return corr

# スピアマンの順位相関係数
def spearman_rank_correlation_coefficient(v1, v2):
    corr = stats.spearmanr(v1, v2)

    return corr

# ケンドールの順位相関係数
def kendalltau_rank_correlation_coefficient(v1, v2):
    corr = stats.kendalltau(v1, v2)

    return corr

def similarity(v1_df, v2_df):
    v1_value = v1_df.values[0]
    v2_value = v2_df.values[0]

    print('v1_value', v1_value)
    print('v2_value', v2_value)

    # リストをps.Seriesに変換
    s1 = pd.Series(list(v1_value))
    s2 = pd.Series(list(v2_value))

    # 相関係数を計算
    res = s1.corr(s2)
    print(res)

    corr = pearson_product_moment_correlation_coefficien(
        v1_value, v2_value
    )
    print('pearson_product_moment_correlation_coefficien', corr)

    corr = spearman_rank_correlation_coefficient(
        v1_value, v2_value
    )
    print('spearman_rank_correlation_coefficient', corr)

    corr = kendalltau_rank_correlation_coefficient(
        v1_value, v2_value
    )
    print('kendalltau_rank_correlation_coefficient', corr)

    e_distance = euclidean_distance(v1_value, v2_value)
    print('e_distance', e_distance)

    # return euclidean_distance(v1_value, v2_value)
    # return res
    return cos_similarity(v1_value, v2_value)

# 数値型の整形
def shaping_num(value):

    if '+' in value:
        value = str(value).split('+')
        value = int(value[0]) + int(value[1])
        return value

    if '-' in value:
        value = str(value).split('-')
        value = int(value[0]) - int(value[1])
        return value

    return value

def need_columns(df):
    columns = [
        'height_cm',
        'weight_kg',
        'preferred_foot',
        'weak_foot',
        'skill_moves',
        'work_rate',
        'player_tags',

        'pace',
        'shooting',
        'passing',
        'dribbling',
        'defending',
        'physic',
        'player_traits',

        'attacking_crossing',
        'attacking_finishing',
        'attacking_heading_accuracy',
        'attacking_short_passing',
        'attacking_volleys',

        'skill_dribbling',
        'skill_curve',
        'skill_fk_accuracy',
        'skill_long_passing',
        'skill_ball_control',

        'movement_acceleration',
        'movement_sprint_speed',
        'movement_agility',
        'movement_reactions',
        'movement_balance',

        'power_shot_power',
        'power_jumping',
        'power_stamina',
        'power_strength',
        'power_long_shots',

        'mentality_aggression',
        'mentality_interceptions',
        'mentality_positioning',
        'mentality_vision',
        'mentality_penalties',
        'mentality_composure',

        'defending_marking',
        'defending_standing_tackle',
        'defending_sliding_tackle'
    ]

    columns += [
        'ls', 'st', 'rs',
        'lw', 'lf', 'cf', 'rf', 'rw',
        'lam', 'cam', 'ram',
        'lm', 'lcm', 'cm', 'rcm', 'rm',
        'lwb', 'ldm', 'cdm', 'rdm', 'rwb',
        'lb', 'lcb', 'cb', 'rcb', 'rb'
    ]
    # ls,st,rs,lw,lf,cf,rf,rw,
    # lam,cam,ram,lm,lcm,cm,rcm,rm,
    # lwb,ldm,cdm,rdm,rwb,lb,lcb,cb,rcb,rb

    return df[columns]

def convert_num_values(player_1_df, player_2_df):
    num_values = [
        'pace',
        'shooting',
        'passing',
        'dribbling',
        'defending',
        'physic',
        'attacking_crossing',
        'attacking_finishing',
        'attacking_heading_accuracy',
        'attacking_short_passing',
        'attacking_volleys',
        'skill_dribbling',
        'skill_curve',
        'skill_fk_accuracy',
        'skill_long_passing',
        'skill_ball_control',
        'movement_acceleration',
        'movement_sprint_speed',
        'movement_agility',
        'movement_reactions',
        'movement_balance',
        'power_shot_power',
        'power_jumping',
        'power_stamina',
        'power_strength',
        'power_long_shots',
        'mentality_aggression',
        'mentality_interceptions',
        'mentality_positioning',
        'mentality_vision',
        'mentality_penalties',
        'mentality_composure',
        'defending_marking',
        'defending_standing_tackle',
        'defending_sliding_tackle'
    ]

    num_values += [
        'ls', 'st', 'rs',
        'lw', 'lf', 'cf', 'rf', 'rw',
        'lam', 'cam', 'ram',
        'lm', 'lcm', 'cm', 'rcm', 'rm',
        'lwb', 'ldm', 'cdm', 'rdm', 'rwb',
        'lb', 'lcb', 'cb', 'rcb', 'rb'
    ]

    for v in num_values:
        # player1のデータの数値の整形
        value = player_1_df[v].values.astype(str)[0]
        value = shaping_num(str(value))

        # player_1_df[v] = float(value) * 0.01
        player_1_df[v] = float(value)

        # player2のデータの数値の整形
        value = player_2_df[v].values.astype(str)[0]
        value = shaping_num(str(value))

        # player_2_df[v] = float(value) * 0.01
        player_2_df[v] = float(value)

    return player_1_df, player_2_df

def convert_traits(player_1_df, player_2_df):

    # 選手特性関連の処理
    traits_list = [
        'Backs Into Player',  # FIFA 18だけの項目
        'Bicycle Kicks',
        'Chip Shot',
        'Dives Into Tackles',
        'Early Crosser',
        'Fancy Passes',
        'Finesse Shot',
        'Flair',
        'Giant Throw-In',
        'GK Cautious With Crosses',
        'GK Comes For Crosses',
        'GK Flat Kick',
        'GK Long Thrower',
        'GK Save With Foot',
        'Injury Prone',
        'Leadership',
        'Long Passer',
        'Long Shot Taker',
        'Long Throw-In',
        'One Club Player',
        'Outside Foot Shot',
        'Play Maker',
        'Power Header',
        'Rushes Out Of Goal',
        'Second Wind',
        'Set Play Specialist',
        'Solid Player',
        'Speed Dribbler',
        'Swerve',
        'Takes Powerful Driven Free Kicks',
        'Team Player',
        'Technical Dribbler'
    ]

    player_1_df_player_traits = player_1_df['player_traits']
    player_2_df_player_traits = player_2_df['player_traits']

    player_1_df = player_1_df.drop(columns='player_traits')
    player_2_df = player_2_df.drop(columns='player_traits')

    for trait in traits_list:
        trait_value = 0
        for p_trait in player_1_df_player_traits.values[0].split(','):
            if trait in p_trait:
                trait_value = 1
                break
        player_1_df[trait] = trait_value

        trait_value = 0
        for p_trait in player_2_df_player_traits.values[0].split(','):
            if trait in p_trait:
                trait_value = 1
                break
        player_2_df[trait] = trait_value

    return player_1_df, player_2_df

def players_comparison(player_1, player_2):
    df = pd.read_csv('data/players_18.csv')

    player_1_df = df.query('sofifa_id == {}'.format(player_1))
    player_2_df = df.query('sofifa_id == {}'.format(player_2))
    # david_silva = df.query('sofifa_id == 189881')

    player_1_df = need_columns(player_1_df)
    player_2_df = need_columns(player_2_df)

    # num_valuesの変換処理
    player_1_df, player_2_df = convert_num_values(player_1_df, player_2_df)

    # 選手特性関連の処理
    player_1_df, player_2_df = convert_traits(player_1_df, player_2_df)

    # 選手タグ関連の処理
    player_1_df = player_1_df.drop(columns='player_tags')
    player_2_df = player_2_df.drop(columns='player_tags')

    # 利き足の変換
    player_1_df = convert_preferred_foot(player_1_df)
    player_2_df = convert_preferred_foot(player_2_df)

    # 攻撃/守備の優先度の変換
    player_1_df = convert_work_rate(player_1_df)
    player_2_df = convert_work_rate(player_2_df)

    # print(player_1_df.values[0])
    # print(player_2_df.values)
    cos = similarity(player_1_df, player_2_df)
    print('cos', cos)

    # カラムの表示
    # columns_views(player_1_df, player_2_df)

shinji_kagawa = 189358
david_silva = 178088
# david_silva = 41
# 香川真司 : 189358
# 本田圭佑 : 186581
# 清武弘嗣 : 210126
# イニエスタ: 41
# スモーリング : 189881
# セルヒオ・ラモス : 155862
# マリオ・ゲッツェ : 192318
# ユリアン・ヴァイグル : 222028
# ファン・マタ : 178088
# イスコ : 197781
# ダビド・シルバ : 168542
# マルク・バルトラ : 198141
# ロメル・ルカク : 192505
# デブルイネ : 192985
# モドリッチ : 177003
# クロース : 182521
# ラキティッチ : 168651
# ウサマ・デンベレ : 231443
# リオネル・メッシ : 158023
# フンメルス : 178603
# ピケ: 152729
# ボアテング : 183907
# メスト・エジル : 176635
# マルコ・ロイス : 188350
# イヴァン・ペリシッチ : 181458
# トーマス・ミュラー : 189596
# オスカル : 188152
# ヤルモレンコ : 194794
# エデン・アザール : 183277
# ネイマール : 190871
# ロッベン : 9014
# サラー : 209331
# ハリー・ケイン : 202126
# ムバッペ : 231747
# グリーズマン : 194765
# ジェラール・ピケ : 152729

players_comparison(shinji_kagawa, david_silva)

# columns_views(shinji_kagawa, david_silva)

# Weak Foot(逆足)
# https://www.fifplay.com/encyclopedia/weak-foot/

# Work Rate(作業率)
# https://www.fifplay.com/encyclopedia/work-rate/

# ユークリッド距離 vs コサイン類似度
# https://enjoyworks.jp/tech-blog/2242

max_stars_repo_name: shimakaze-git/football-data-analysis

max_issues_repo_name: shimakaze-git/football-data-analysis

max_forks_repo_name: shimakaze-git/football-data-analysis

reason: import numpy,import scipy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Find optimal ICDL parameters for each test case by optimizing over 768 different parameter combinations.
"""

import os
import sys
import glob
import copy
from socket import gethostname
from functools import partial

import numpy as np
from psutil import cpu_count

import sporco.fft
sporco.fft.pyfftw_threads = 1
from sporco.signal import gaussian
from sporco.util import idle_cpu_count, grid_search

from cdlpsf.cdl import PSFEstimator
from cdlpsf.util import sm_snr

# Define standard integer sampling grid -wp ... wp
wp = 7

# Subpixel estimation factor (common for all runs)
M = 5

# Common noise level
slct_noise = 1.0

# Evaluate PSF estimation performance for specified parameters
def psf_est_performance(prm, K, img, psfref):
    sigma0, lmbdaX, lmbdaG, rhoX, LG = prm
    g0 = gaussian((2*wp+1,) * 2, sigma0)
    g0 /= np.linalg.norm(g0)
    opt = PSFEstimator.Options(
            {'Verbose': False, 'MaxMainIter': 100,
             'XslvIter0': 10, 'GslvIter0': 10,
             'Xslv': {'NonNegCoef': False, 'rho': rhoX},
             'Gslv': {'L': LG}})
    pe = PSFEstimator(img, g0, lmbdaX, lmbdaG, M, K, opt)
    psfgrd = pe.solve()
    return sm_snr(psfref, psfgrd)

# Algorithm parameter ranges
paramranges = {
    'sigma0': np.array([0.5, 1.0, 1.5, 2.0]),
    'lmbdaX': np.array([1e-3, 1e-2, 1e-1, 5e-1]),
    'lmbdaG': np.array([1e-2, 5e-2, 1e-1, 5e-1]),
    'rhoX': np.array([1e0, 1e1, 1e2]),
    'LG': np.array([5e1, 1e2, 5e2, 1e3])
}

prmcmb = np.product([paramranges[v].size for v in paramranges])
print('Evaluating %d parameter combinations per input file' % prmcmb)

# Paths to data files
imgpath = 'data/simulated_images'
psfpath = 'data/reference_psfs'
rsltpath = 'data/icdl_results'
if not os.path.isdir(rsltpath):
    os.makedirs(rsltpath)
rsltfile = os.path.join(rsltpath, 'icdl_opt_param_pps.npz')

# Number of processors to use for parameter evaluation
nproc = cpu_count(logical=False)

results = {}
imgfiles = sorted(glob.glob(os.path.join(imgpath, '*.npz')))
# Iterate over input image files
for imgfile in imgfiles:
    # Load data from simulated image file
    npz = np.load(imgfile)
    img = npz['imgn']
    noise = float(npz['noise'])
    shape = str(npz['shape'])
    pps = int(npz['pixperstar'])

    # Skip input images for different noise levels
    if noise != slct_noise:
        continue

    basename = os.path.basename(imgfile)[0:-4]
    print('Computing parameters for %s' % imgfile)

    # Rescale image range
    imx = img.max()
    img /= imx

    # Select Lanczos kernel order depending on PSF shape
    if shape == 'complex' or shape == 'narrow':
        K = 5
    else:
        K = 10

    # Load reference PSF
    psffile = os.path.join(psfpath, '%s.npz' % shape)
    npz = np.load(psffile, allow_pickle=True)
    psfref = npz['refpsf'].item()[M]

    # Evaluate performance over selected parameter ranges
    perf = partial(psf_est_performance, K=K, img=img, psfref=psfref)
    sprm, sfvl, fvmx, sidx = grid_search(
        perf, (paramranges['sigma0'], paramranges['lmbdaX'],
               paramranges['lmbdaG'], paramranges['rhoX'],
               paramranges['LG']),
        fmin=False, nproc=nproc)

    # Add results for this image to results dict
    results[basename] = {'shape': shape, 'pps': pps, 'noise': noise,
                         'K': K, 'sprm': sprm, 'sfvl': sfvl, 'fvmx': fvmx,
                         'sidx': sidx}

    # Save data to NPZ file
    np.savez(rsltfile, paramranges=paramranges, results=results)

ext: py

max_stars_repo_name: bwohlberg/sporco-extra

max_issues_repo_name: bwohlberg/sporco-extra

max_forks_repo_name: bwohlberg/sporco-extra

reason: import numpy


text:
import tensorflow as tf
import numpy as np
from tommy2tommy.layers import embeddings

class TestPositionalEncoding(tf.test.TestCase):
    def test_output_shape(self):
        batch_size = 2
        length = 10
        d_model = 4
        x = tf.ones([batch_size, length, d_model])

        positional_encoding = embeddings.PositionalEncoding(max_length=10)
        output = positional_encoding(x)
        self.assertShapeEqual(np.zeros((batch_size, length, d_model)), output)

if __name__ == "__main__":
    tf.test.main()


text:
#------------------------------------------------------------------------------
# Libraries
#------------------------------------------------------------------------------
# Standard
import os
import pandas as pd
import numpy as np
import bodyguard as bg

# Using SentenceTransformers NLP library
from sentence_transformers import SentenceTransformer

# Using HuggingFace NLP library
from transformers import AutoTokenizer, AutoModel
import torch
#------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------
class DocumentEmbedder(object):
    """
    Embedding of documents

    We rely on two major libraries to embed documents, namely https://github.com/UKPLab/sentence-transformers and https://huggingface.co/transformers/
    """
    # -------------------------------------------------------------------------
    # Constructor function
    # -------------------------------------------------------------------------
    def __init__(self,
                 normalize=True,
                 verbose=True,
                 model_name_or_path="all-roberta-large-v1",
                 ):
        self.normalize = normalize
        self.verbose = verbose
        self.model_name_or_path = model_name_or_path

    # -------------------------------------------------------------------------
    # Class variables
    # -------------------------------------------------------------------------
    MAX_SQN_LENGTH = 256
    RETURN_TYPE_OPT = ["dict", "df"]
    STR_USING_SAVED_MODEL = "Using saved model: '{0}'"
    STR_DOWNLOADING_MODEL = "Downloading model: '{0}'"
    STR_DOCUMENTS_NEED_EMBEDDING_INPUT = "Embeddings from model '{0}' needed"

    # -------------------------------------------------------------------------
    # Private functions
    # -------------------------------------------------------------------------
    #Mean Pooling - Take attention mask into account for correct averaging
    def _mean_pooling(self, model_output, attention_mask):
        """
        See example here: https://huggingface.co/sentence-transformers/all-roberta-large-v1
        """
        token_embeddings = model_output[0] #First element of model_output contains all token embeddings
        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
        sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
        mean_embeddings = sum_embeddings / sum_mask

        return mean_embeddings

    def _check_embeddings_existence(self, documents, model_name_or_path):
        """Check which documents that need to be embeddings versus which documents that have already been embedded"""

        # Check if embeddings exist using this particular method
        if hasattr(self, model_name_or_path+"-embeddings"):
            if self.verbose:
                print(f"Embeddings from model '{model_name_or_path}' found")
            # Load existing embeddings
            documents_embeddings = getattr(self, model_name_or_path+"-embeddings")

            # Check which documents have not already been embedded
            documents_not_embedded = []
            for doc in documents:
                if documents_embeddings.get(doc) is None:
                    documents_not_embedded.append(doc)
        else:
            setattr(self, model_name_or_path+"-embeddings", {})
            documents_not_embedded = documents

        return documents_not_embedded

    def _extract_existing_embeddings(self, documents, model_name_or_path):
        """Extract those embeddings that have already been embedded"""
        # Extract relevant document embeddings
        embeddings = {k:getattr(self, model_name_or_path+"-embeddings")[k] for k in documents}

        return embeddings

    def _extract_embedding_by_str(self, embeddings, which_embedding):
        """
        Extract specific embeddings by str (key in dict of embeddings)
        """
        # Sanity check inputs
        bg.sanity_check.check_type(x=embeddings,
                                   allowed=dict,
                                   name="embeddings")

        bg.sanity_check.check_type(x=which_embedding,
                                   allowed=str,
                                   name="which_embedding")

        # Looking for a specific embedding
        if which_embedding in embeddings:
            embeddings = embeddings[which_embedding]
        else:
            raise Exception(f"Embedding '{which_embedding}' not found in existing embeddings, which include: \n{list(embeddings.keys())}. \nPlease embed!")

        return embeddings

    def _embed_documents(self,
                         documents,
                         model_name_or_path="all-roberta-large-v1"):

        # ---------------------------------------------------------------------
        # Check documents have been embedded before
        # ---------------------------------------------------------------------
        documents_not_embedded = self._check_embeddings_existence(documents=documents,
                                                                  model_name_or_path=model_name_or_path)

        if bool(documents_not_embedded):
            # We have assessed that some documents have not been embedded, hence we construct embeddings and save them!
            if self.verbose:
                print(self.STR_DOCUMENTS_NEED_EMBEDDING_INPUT.format(model_name_or_path))

            # -----------------------------------------------------------------
            # Check if model is already loaded
            # -----------------------------------------------------------------
            if hasattr(self, model_name_or_path+"-model"):
                if self.verbose:
                    print(self.STR_USING_SAVED_MODEL.format(model_name_or_path))

                pretrained_model = getattr(self, model_name_or_path+"-model")
                tokenizer = getattr(self, model_name_or_path+"-tokenizer")

            else:
                if self.verbose:
                    print(self.STR_DOWNLOADING_MODEL.format(model_name_or_path))

                try:
                    # Load model via SentenceTransformer
                    pretrained_model = SentenceTransformer(model_name_or_path=model_name_or_path)
                    tokenizer = None
                    self.loaded_via = "SentenceTransformer"
                except:
                    try:
                        # Load model via HuggingFace
                        pretrained_model = AutoModel.from_pretrained(model_name_or_path)
                        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
                        self.loaded_via = "HuggingFace"
                    except OSError as err:
                        raise Exception(f"Could not load '{model_name_or_path}'. \nWhile trying, this error occured: \n\n{err}")

                # Set tokenizer and pretrained model as attributes under the model name.
                setattr(self, model_name_or_path+"-model", pretrained_model)
                setattr(self, model_name_or_path+"-tokenizer", tokenizer)

            # -----------------------------------------------------------------
            # Embed
            # -----------------------------------------------------------------
            if self.loaded_via=="SentenceTransformer":

                # Update maximum sequence length
                pretrained_model.max_seq_length = self.MAX_SQN_LENGTH

                # Construct embedding
                raw_embeddings = pretrained_model.encode(documents_not_embedded,
                                                         batch_size=32,
                                                         show_progress_bar=False,
                                                         output_value="sentence_embedding",
                                                         convert_to_numpy=True,
                                                         convert_to_tensor=False,
                                                         device=None,
                                                         normalize_embeddings=True)

            elif self.loaded_via=="HuggingFace":

                # Tokenize sentences
                encoded_input = tokenizer(text=documents_not_embedded,
                                          padding=True,
                                          truncation=True,
                                          max_length=self.MAX_SQN_LENGTH,
                                          return_tensors='pt')

                # Compute token embeddings
                with torch.no_grad():
                    model_output = pretrained_model(**encoded_input)

                # Perform pooling (here, mean pooling)
                raw_embeddings = self._mean_pooling(model_output=model_output,
                                                    attention_mask=encoded_input['attention_mask'])

                # Convert tensor to numpy array
                raw_embeddings = raw_embeddings.numpy()

            # -----------------------------------------------------------------
            # Finalize
            # -----------------------------------------------------------------
            # Pre-allocate
            embeddings = {}

            for i,doc in enumerate(documents_not_embedded):
                embeddings[doc] = raw_embeddings[i,:].reshape(-1,)

            # Update existing document embeddings
            setattr(self, model_name_or_path+"-embeddings", {**getattr(self, model_name_or_path+"-embeddings"),
                                                             **embeddings}
                    )

        # Extract embeddings
        embeddings_out = self._extract_existing_embeddings(documents=documents,
                                                           model_name_or_path=model_name_or_path)

        return embeddings_out
    # -------------------------------------------------------------------------
    # Public functions
    # -------------------------------------------------------------------------
    def show_available_models(self):
        """Print available models via SentenceTransformers"""

        # Last last updated
        DATE_LAST_UPDATED = "February 3, 2022"

        # Models
        AVAILABLE_MODELS_VIA_SENTENCETRANSFORMERS = ["all-roberta-large-v1",
                                                     "all-mpnet-base-v1",
                                                     "all-mpnet-base-v2"
                                                     "all-MiniLM-L12-v1",
                                                     "all-distilroberta-v1",
                                                     "all-MiniLM-L12-v2",
                                                     "all-MiniLM-L6-v2",
                                                     "all-MiniLM-L6-v1",
                                                     "paraphrase-mpnet-base-v2",
                                                     "multi-qa-mpnet-base-dot-v1",
                                                     "multi-qa-distilbert-dot-v1",
                                                     "multi-qa-mpnet-base-cos-v1",
                                                     "paraphrase-distilroberta-base-v2",
                                                     "paraphrase-TinyBERT-L6-v2",
                                                     "paraphrase-MiniLM-L12-v2",
                                                     "multi-qa-distilbert-cos-v1",
                                                     "paraphrase-multilingual-mpnet-base-v2",
                                                     "paraphrase-MiniLM-L6-v2",
                                                     "paraphrase-albert-small-v2",
                                                     "multi-qa-MiniLM-L6-cos-v1",
                                                     "paraphrase-multilingual-MiniLM-L12-v2",
                                                     "multi-qa-MiniLM-L6-dot-v1",
                                                     "msmarco-bert-base-dot-v5",
                                                     "msmarco-distilbert-base-tas-b",
                                                     "paraphrase-MiniLM-L3-v2",
                                                     "msmarco-distilbert-dot-v5",
                                                     "distiluse-base-multilingual-cased-v1",
                                                     "distiluse-base-multilingual-cased-v2",
                                                     "average_word_embeddings_komninos",
                                                     "average_word_embeddings_glove.6B.300d",
                                                     ]

        bg.tools.print2(f"""As of out {DATE_LAST_UPDATED}, these models are available through SentenceTransformers: \n\n {AVAILABLE_MODELS_VIA_SENTENCETRANSFORMERS}""")

    def embed_documents(self,
                        documents,
                        return_embeddings=True,
                        return_type="df"):
        """
        Embed documents as vectors
        """
        # Check types
        bg.sanity_check.check_type(x=documents,
                                   allowed=(list,str),
                                   name="documents")

        # Convert to list if str
        if isinstance(documents, str):
            documents = [documents]

        # Check inputs
        bg.sanity_check.check_str(x=return_type,
                                  allowed=self.RETURN_TYPE_OPT,
                                  name="return_type")

        # Embed documents
        embeddings = self._embed_documents(documents=documents,
                                           model_name_or_path=self.model_name_or_path)

        # Normalize
        if self.normalize:
            if isinstance(embeddings, dict):
                embeddings = {k: bg.distance.normalize_by_norm(x=v,norm="L2") for k,v in embeddings.items()}

        # Convert to df to change column type
        embeddings = bg.convert.convert_dict_to_df(x=embeddings)

        # Enforce columns to be strings
        embeddings.columns = embeddings.columns.astype(str)

        if return_type=="dict":
            embeddings = bg.convert.convert_df_to_dict(x=embeddings)

        if return_embeddings:
            return embeddings

    def extract_embeddings(self,embeddings=None,which_embeddings="all",return_type="df"):

        if embeddings is None:
            # Extract all existing embeddings
            embeddings = getattr(self, self.model_name_or_path+"-embeddings")

        bg.sanity_check.check_type(x=embeddings,
                                   allowed=dict,
                                   name="embeddings")

        bg.sanity_check.check_type(x=which_embeddings,
                                   allowed=(str,list),
                                   name="which_embeddings")

        if which_embeddings=="all":
            # Return all embeddings
            pass
        elif isinstance(which_embeddings, str):

            embeddings = {which_embeddings: self._extract_embedding_by_str(embeddings=embeddings,
                                                                           which_embedding=which_embeddings)
                          }

        elif isinstance(which_embeddings, list):

            # Pre-allocate
            embeddings_temp = {}

            # Exact all individual embeddings
            for s in which_embeddings:
                embeddings_temp[s] = self._extract_embedding_by_str(embeddings=embeddings,
                                                                    which_embedding=s)

            # Overwrite
            embeddings = embeddings_temp

        if return_type=="df":
            embeddings = bg.convert.convert_dict_to_df(x=embeddings)

        return embeddings

    def compute_similarity_between_two_embeddings(self,a,b,metric="Lknorm",**kwargs):

        # Sanity check
        bg.sanity_check.check_type(x=a, allowed=str, name="a")
        bg.sanity_check.check_type(x=b, allowed=str, name="b")

        # Join as list
        ab = [a,b]

        # Get embeddings
        embeddings = self.embed_documents(documents=ab,
                                          return_embeddings=True,
                                          return_type="df")

        # Compute similarity
        similarity = bg.distance.compute_similarity(a=embeddings.loc[a],
                                                    b=embeddings.loc[b],
                                                    metric=metric,
                                                    **kwargs)

        # Extract
        similarity = similarity.iloc[0,0]

        return similarity

    def compute_similarity(self,a,b,metric="Lknorm",**kwargs):

        # Sanity check
        bg.sanity_check.check_type(x=a, allowed=(str,list), name="a")
        bg.sanity_check.check_type(x=b, allowed=(str,list), name="b")

        # Convert to list if str
        if isinstance(a,str):
            a = [a]
        if isinstance(b,str):
            b = [b]

        # Join as list
        ab = a+b

        # Get embeddings
        embeddings = self.embed_documents(documents=ab,
                                          return_embeddings=True,
                                          return_type="dict")

        a_embeddings = self.extract_embeddings(embeddings=embeddings,
                                               which_embeddings=a,
                                               return_type="df")

        b_embeddings = self.extract_embeddings(embeddings=embeddings,
                                               which_embeddings=b,
                                               return_type="df")

        # Compute similarity
        similarity = bg.distance.compute_similarity(a=a_embeddings,
                                                    b=b_embeddings,
                                                    metric=metric,
                                                    **kwargs)

        return similarity

max_stars_repo_name: muhlbach/econnlp

max_issues_repo_name: muhlbach/econnlp

max_forks_repo_name: muhlbach/econnlp


text:
from numpy import square
import pandas as pd
import plotly.express as px
import math
import csv

with open("data.csv",newline="") as f:
    reader=csv.reader(f)
    file_data=list(reader)

data=file_data.pop(0)

def mean(data):
    n=len(data)
    total=0
    for x in data:
        total+=int(x)
    mean=total/n
    return mean

#fig.show()
squared_list=[]
for number in data:
    a=int(number)-mean(data)
    a=a**2
    squared_list.append(a)

sum=0
for i in squared_list:
    sum=sum+1
result=sum/(len(data)-1)
sd=math.sqrt(result)
print(sd)

reason: from numpy


text:
#!/usr/bin/python3

import sys
import matplotlib
import pandas
import matplotlib.pyplot as plt
import numpy as np
from toopazo_tools.file_folder import FileFolderTools as FFTools

class PlotTelemetryLog:
    """
    Class to plot data from log
    """

    def __init__(self, filepath, index_col):
        self.file_path = filepath
        [head, tail] = FFTools.get_file_split(filepath)
        self.file_folder = head
        self.file_name = tail
        [root, ext] = FFTools.get_file_splitext(filepath)
        _ = root
        self.file_extension = ext

        if index_col == "None":
            print('index_col == "None"')
            index_col = None

        self.dataframe = pandas.read_csv(
            self.file_path, index_col=index_col, delimiter=',',
            parse_dates=False, skipinitialspace=True)
        # self.dataframe.set_index(
        #     index_col, inplace=True, verify_integrity=True)
        # print(self.dataframe)
        # print(self.dataframe.shape)
        # print(self.dataframe.columns)
        # print(self.dataframe.iterrows())

        matplotlib.use('Qt5Agg')

    def save_current_plot(self, tag_arr, sep, ext):
        assert isinstance(self.file_name, str)
        name = self.file_name.replace(self.file_extension, "")
        for tag in tag_arr:
            name = name + sep + str(tag)
        file_path = self.file_folder + '/' + name + ext

        # plt.show()
        plt.savefig(file_path)
        # return file_path

    def plot_dataframe(self, dataframe):
        tfigsize = (12, 6)
        # fig, axs = plt.subplots(figsize=tfigsize)
        # df.plot()
        # df.plot.area(ax=axs, subplots=True)
        dataframe.plot(figsize=tfigsize, subplots=True)

        # cmap = ListedColormap(['#0343df', '#e50000', '#ffff14', '#929591'])
        # ax = df.plot.bar(x='year', colormap=cmap)
        # ax.set_xlabel(None)
        # ax.set_ylabel('Seats')
        # ax.set_title('UK election results')

def calibrate_dataframe(dataframe, col_curi, col_rpmi):
    assert isinstance(dataframe, pandas.DataFrame)

    cur_arr = dataframe[col_curi].values
    frq_arr = dataframe[col_rpmi].values
    cur_arr = dataframe[col_curi].values
    print(frq_arr)
    print(cur_arr)

    # 1) Divide samples into 0rpm and non0rpm
    for column in dataframe:
        print(column)

        # cur_arr_0rpm = dataframe[columns]

    exit(0)

    # for sample in range(0, nsamples):
    for index, row in dataframe.iterrows():
        # print(f"index: {index}, row: {row}")

        time = sample
        rpm = frq_arr[sample] * 60      # Hz to RPM
        rpm = ArsParser.apply_linreg(rpm)
        cur = cur_arr[sample]

        # Add entire (time, rpm, cur) to caldata
        if ktime in caldata:
            caldata[ktime].append(time)
            caldata[krpm].append(rpm)
            caldata[kcur].append(cur)
        elif ktime not in caldata:
            caldata[ktime] = [time]
            caldata[krpm] = [rpm]
            caldata[kcur] = [cur]

        # Add 0rpm (time, rpm, cur) to caldata
        if (rpm == 0) and (ktime_0rpm in caldata):
            caldata[ktime_0rpm].append(time)
            caldata[krpm_0rpm].append(rpm)
            caldata[kcur_0rpm].append(cur)
        elif (rpm == 0) and (ktime_0rpm not in caldata):
            caldata[ktime_0rpm] = [time]
            caldata[krpm_0rpm] = [rpm]
            caldata[kcur_0rpm] = [cur]

        # Add non0rpm (time, rpm, cur) to caldata
        if (rpm != 0) and (ktime_non0rpm in caldata):
            caldata[ktime_non0rpm].append(time)
            caldata[krpm_non0rpm].append(rpm)
            caldata[kcur_non0rpm].append(cur)
        elif (rpm != 0) and (ktime_non0rpm not in caldata):
            caldata[ktime_non0rpm] = [time]
            caldata[krpm_non0rpm] = [rpm]
            caldata[kcur_non0rpm] = [cur]

        # 2) Change the current scale
        # According to
        # https://www.sparkfun.com/datasheets/BreakoutBoards/0712.pdf
        # The ACS712ELCTR-20A sensor maps current and voltage as
        # [-20A +20A] <=> [0.5V 4.5V]
        # This gets mapped by the Arduino ADC as
        # [0V 5V] <=> [0 1023]
        # 5 volts / 1024 units = 0.0049 volts (4.9 mV) per unit
        try:
            arduino_sensitivity = 5 / 1024      # V/count
            acs712_sensitivity = 100 / 1000     # V/A
            volt_zero_cur = 2.5

            counts = np.array(caldata[kcur])
            voltage = counts * arduino_sensitivity
            current = (voltage - volt_zero_cur) / acs712_sensitivity
            caldata[kcur] = current

            counts = np.array(caldata[kcur_0rpm])
            voltage = counts * arduino_sensitivity
            current = (voltage - volt_zero_cur) / acs712_sensitivity
            caldata[kcur_0rpm] = current

            counts = np.array(caldata[kcur_non0rpm])
            voltage = counts * arduino_sensitivity
            current = (voltage - volt_zero_cur) / acs712_sensitivity
            caldata[kcur_non0rpm] = current
        except KeyError:
            # This channel was always at 0rpm or always non0rpm
            pass

        # 3) Subtract from kcur_non0rpm the average current of 0rpm
        try:
            cur_0rpm_avg = np.mean(caldata[kcur_0rpm])
            cur_non0rpm = caldata[kcur_non0rpm]
            caldata[kcur_non0rpm] = cur_non0rpm - cur_0rpm_avg
            cur_0rpm = caldata[kcur_0rpm]
            caldata[kcur_0rpm] = cur_0rpm - cur_0rpm_avg
        except KeyError:
            # This channel was always at 0rpm or always non0rpm
            pass

        try:
            print('[calibrate_rawdata] channel %s has %s 0rpm samples'
                  % (nchan, len(caldata[ktime_0rpm])))
            print('[calibrate_rawdata] channel %s has %s non0rpm samples'
                  % (nchan, len(caldata[ktime_non0rpm])))
        except KeyError:
            # This channel was always at 0rpm or always non0rpm
            pass
    # # debug
    # caldata = rawdata
    # print('[calibrate_rawdata] caldata.keys() %s'
    #       % caldata.keys())
    return dataframe

def parse_user_arg(filename):
    filename = FFTools.full_path(filename)
    print('target file {}'.format(filename))
    if FFTools.is_file(filename):
        cwd = FFTools.get_cwd()
        print('current folder {}'.format(cwd))
    else:
        arg = '{} is not a file'.format(filename)
        raise RuntimeError(arg)
    return filename

if __name__ == '__main__':
    ufilename = sys.argv[1]
    ufilename = parse_user_arg(ufilename)
    uindexcol = sys.argv[2]

    # Header of kdecan log file log_1_2021-09-16-19-05-59.kdecan
    #     time s, escid, voltage V, current A, angVel rpm, temp degC,
    #     warning, inthtl us, outthtl perc

    col_sps = "sps"
    col_mills = "mills"
    col_sesc = "secs"
    col_dtmills = "dtmills"
    col_cur1 = "cur1"
    col_cur2 = "cur2"
    col_cur3 = "cur3"
    col_cur4 = "cur4"
    col_cur5 = "cur5"
    col_cur6 = "cur6"
    col_cur7 = "cur7"
    col_cur8 = "cur8"
    col_rpm1 = "rpm1"
    col_rpm2 = "rpm2"
    col_rpm3 = "rpm3"
    col_rpm4 = "rpm4"
    col_rpm5 = "rpm5"
    col_rpm6 = "rpm6"
    col_rpm7 = "rpm7"
    col_rpm8 = "rpm8"

    plotlog = PlotTelemetryLog(ufilename, uindexcol)
    udataframe = plotlog.dataframe
    print(udataframe)
    udataframe = calibrate_dataframe(udataframe, col_cur8, col_rpm8)

    # plotlog = PlotTelemetryLog(ufilename, uindexcol)
    # for escid in range(11, 19):
    #     udataframe = plotlog.dataframe
    #     # udataframe = udataframe[udataframe[col_escid] == escid]
    #     col_curi = " cur{}".format(escid-10)
    #     col_rpmi = " rpm{}".format(escid - 10)
    #     col_arr = [col_sps, col_curi, col_rpmi]
    #     udataframe = udataframe[col_arr]
    #     udataframe = calibrate_dataframe(udataframe, col_curi, col_rpmi)
    #     plotlog.plot_dataframe(udataframe)
    #     plotlog.save_current_plot(
    #         tag_arr=["escid{}".format(escid)], sep="_", ext='.png')


text:
# -*- coding: utf-8 -*-

import numpy

import matplotlib.pyplot as plt

from ..core.frequency import FrequencyAxis
from ..core.dfunction import DFunction
from ..core.datasaveable import DataSaveable
from ..core.managers import EnergyUnitsManaged
from ..core.units import cm2int

class AbsSpectrumBase(DFunction, EnergyUnitsManaged, DataSaveable):
    """Provides basic container for absorption spectrum

    """

    def __init__(self, axis=None, data=None):
        super().__init__()
        self.axis = axis
        self.data = data

    def set_axis(self, axis):
        """Sets axis atribute

        Parameters
        ----------

        axis : FrequencyAxis object
            Frequency axis object. This object has managed energy units

        """
        self.axis = axis

    def set_data(self, data):
        """Sets data atribute

        Parameters
        ----------

        data : array like object (numpy array)
            Sets the data of the absorption spectrum

        """
        self.data = data

    def add_data(self, data):
        self.data += data

    def set_by_interpolation(self, x, y, xaxis="frequency"):

        from scipy import interpolate

        if xaxis == "frequency":

            om = self.convert_2_internal_u(x)

        elif xaxis == "wavelength":
            # convert to internal (nano meters) units of wavelength

            # convert to energy (internal units)
            # to cm
            om = 1.0e-7*x
            # to 1/cm
            om = 1.0/om
            # to 1/fs
            om = om*cm2int

        if om[1] > om[2]:
            # reverse order
            om = numpy.flip(om,0)
            y = numpy.flip(y,0)

        # equidistant points on the x-axis
        omin = numpy.amin(om)
        omax = numpy.amax(om)
        length = om.shape[0]
        step = (omax-omin)/length

        # new frequency axis
        waxis = FrequencyAxis(omin, length, step)

        # spline interpolation
        tck = interpolate.splrep(om, y, s=0)
        ynew = interpolate.splev(waxis.data, tck, der=0)

        # setting the axis and data
        self.axis = waxis
        self.data = ynew

    def clear_data(self):
        """Sets spectrum data to zero

        """
        shp = self.data.shape
        self.data = numpy.zeros(shp, dtype=numpy.float64)

    def normalize2(self,norm=1.0):
        """Normalizes spectrum to a given value

        """
        mx = numpy.max(self.data)
        self.data = norm*self.data/mx

    def normalize(self):
        """Normalization to one

        """
        self.normalize2(norm=1.0)

    def subtract(self, val):
        """Subtracts a value from the spectrum to shift its base line

        """
        self.data -= val

    def add_to_data(self, spect):
        """Performs addition on the data.

        Expects a compatible object holding absorption spectrum
        and adds its data to the present absorption spectrum.

        Parameters
        ----------

        spect : spectrum containing object
            This object should have a compatible axis and some data

        """

        if self.axis is None:
            self.axis = spect.axis.copy()

        if not numpy.allclose(spect.axis.data, self.axis.data):
            numpy.savetxt("spect_data_wrong.dat", spect.axis.data)
            numpy.savetxt("self_data_wrong.dat", self.axis.data)
            raise Exception("Incompatible axis")

        if self.data is None:
            self.data = numpy.zeros(len(spect.data),
                                    dtype=spect.axis.data.dtype)

        self.data += spect.data

#    #save method is inherited from DFunction

    def save_data(self, filename):
        """Saves the data of this absorption spectrum

        """
        super().save_data(filename, with_axis=self.axis)

    def load_data(self, filename):
        """Loads data from file into this absorption spectrum

        """
        if self.axis is None:
            raise Exception("The property `axis` has to be defined")
        super().load_data(filename, with_axis=self.axis)

    def plot(self, **kwargs):
        """ Plotting absorption spectrum using the DFunction plot method

        """
        if "ylabel" not in kwargs:
            ylabel = r'$\alpha(\omega)$ [a.u.]'
            kwargs["ylabel"] = ylabel

        fig = super().plot(**kwargs)
        if fig is not None:
            return fig

max_stars_repo_name: slamavl/quantarhei

max_issues_repo_name: slamavl/quantarhei

max_forks_repo_name: slamavl/quantarhei

reason: import numpy,from scipy


text:
import numpy as np
from nose.tools import (assert_equal, assert_less, assert_greater, assert_true,
                        assert_raises_regexp)
from bolero.controller import Controller
from bolero.environment import ObjectiveFunction
from bolero.behavior_search import JustOptimizer
from bolero.representation import DummyBehavior
from bolero.optimizer import CMAESOptimizer, REPSOptimizer
from bolero.utils.testing import assert_pickle
from numpy.testing import assert_array_equal

def test_missing_environment():
    assert_raises_regexp(ValueError, "Environment specification is missing",
                        Controller)

def test_no_environment_subclass():
    class NoEnvironment(object):
        pass

    assert_raises_regexp(
        TypeError, "requires subclass of 'Environment'",
        Controller, environment=NoEnvironment())

def test_no_behavior_search_subclass():
    class NoBehaviorSearch(object):
        pass

    assert_raises_regexp(
        TypeError, "requires subclass of 'BehaviorSearch'",
        Controller, environment=ObjectiveFunction(),
        behavior_search=NoBehaviorSearch())

def test_missing_behavior_search():
    ctrl = Controller(environment=ObjectiveFunction())
    beh = DummyBehavior(initial_params=np.array([0.0, 0.0]))
    beh.init(0, 2)
    feedback = ctrl.episode_with(beh)
    assert_equal(len(feedback), 1)
    assert_less(feedback[0], ctrl.environment.get_maximum_feedback())

def test_learning_fails_with_missing_behavior_search():
    controller = Controller(environment=ObjectiveFunction())
    assert_raises_regexp(ValueError, "BehaviorSearch is required",
                        controller.learn)

def test_controller_cmaes_sphere():
    opt = CMAESOptimizer(initial_params=np.zeros(2))
    ctrl = Controller(environment=ObjectiveFunction(),
                      behavior_search=JustOptimizer(opt))
    returns = ctrl.learn()
    assert_equal(len(returns), 10)

def test_controller_cmaes_sphere_via_config():
    config = {
        "Environment": {"type": "bolero.environment.ObjectiveFunction"},
        "BehaviorSearch": {
            "type": "bolero.behavior_search.JustOptimizer",
            "optimizer": {"type": "bolero.optimizer.CMAESOptimizer",
                          "initial_params": np.zeros(2)}}
    }
    ctrl = Controller(config)
    returns = ctrl.learn()
    assert_equal(len(returns), 10)

def test_record_test_results():
    opt = CMAESOptimizer(initial_params=np.zeros(2))
    ctrl = Controller(environment=ObjectiveFunction(),
                      behavior_search=JustOptimizer(opt),
                      n_episodes_before_test=10, n_episodes=100)
    ctrl.learn()
    results = np.array(ctrl.test_results_)
    assert_equal(results.shape[0], 10)
    assert_true(np.all(results[:-1] <= results[1:]))

def test_record_inputs():
    opt = CMAESOptimizer(initial_params=np.zeros(2))
    ctrl = Controller(environment=ObjectiveFunction(),
                      behavior_search=JustOptimizer(opt),
                      record_inputs=True)
    returns = ctrl.learn()
    assert_equal(len(returns), 10)
    assert_equal(np.array(ctrl.inputs_).shape, (10, 1, 2))

def test_record_outputs():
    opt = CMAESOptimizer(initial_params=np.zeros(2))
    ctrl = Controller(environment=ObjectiveFunction(),
                      behavior_search=JustOptimizer(opt),
                      record_outputs=True)
    returns = ctrl.learn()
    assert_equal(len(returns), 10)
    assert_equal(np.array(ctrl.outputs_).shape, (10, 1, 0))

def test_record_feedbacks():
    opt = CMAESOptimizer(initial_params=np.zeros(2))
    ctrl = Controller(environment=ObjectiveFunction(),
                      behavior_search=JustOptimizer(opt),
                      record_feedbacks=True, accumulate_feedbacks=False)
    returns = ctrl.learn()
    assert_array_equal(returns, ctrl.feedbacks_)

def test_learn_controller_cmaes_sphere():
    opt = CMAESOptimizer(initial_params=np.zeros(2), random_state=0)
    ctrl = Controller(environment=ObjectiveFunction(random_state=0),
                      behavior_search=JustOptimizer(opt),
                      n_episodes=200)
    returns = ctrl.learn()
    dist_to_maximum = returns.max() - ctrl.environment.get_maximum_feedback()
    assert_greater(dist_to_maximum, -1e-5)

def test_pickle_controller():
    names = ["CMAESOptimizer", "REPSOptimizer"]
    optimizers = [CMAESOptimizer, REPSOptimizer]
    for name, Optimizer in zip(names, optimizers):
        opt = Optimizer(initial_params=np.zeros(2))
        ctrl = Controller(environment=ObjectiveFunction(),
                          behavior_search=JustOptimizer(opt))
        assert_pickle(name, ctrl)

max_stars_repo_name: dettmann/bolero

max_issues_repo_name: dettmann/bolero

max_forks_repo_name: dettmann/bolero

reason: import numpy,from numpy


text:
import numpy as np
import pandas as pd
from os import makedirs
from os.path import join, exists
from nilearn.input_data import NiftiLabelsMasker
from nilearn.connectome import ConnectivityMeasure
from nilearn.plotting import plot_anat, plot_roi
import bct
from nipype.interfaces.fsl import InvWarp, ApplyWarp
import datetime

subjects = [
    "101",
    "102",
    "103",
    "104",
    "106",
    "107",
    "108",
    "110",
    "212",
    "213",
    "214",
    "215",
    "216",
    "217",
    "218",
    "219",
    "320",
    "321",
    "322",
    "323",
    "324",
    "325",
    "327",
    "328",
    "329",
    "330",
    "331",
    "332",
    "333",
    "334",
    "335",
    "336",
    "337",
    "338",
    "339",
    "340",
    "341",
    "342",
    "343",
    "344",
    "345",
    "346",
    "347",
    "348",
    "349",
    "350",
    "451",
    "452",
    "453",
    "455",
    "456",
    "457",
    "458",
    "459",
    "460",
    "462",
    "463",
    "464",
    "465",
    "467",
    "468",
    "469",
    "470",
    "502",
    "503",
    "571",
    "572",
    "573",
    "574",
    "575",
    "577",
    "578",
    "579",
    "580",
    "581",
    "582",
    "584",
    "585",
    "586",
    "587",
    "588",
    "589",
    "590",
    "591",
    "592",
    "593",
    "594",
    "595",
    "596",
    "597",
    "598",
    "604",
    "605",
    "606",
    "607",
    "608",
    "609",
    "610",
    "611",
    "612",
    "613",
    "614",
    "615",
    "616",
    "617",
    "618",
    "619",
    "620",
    "621",
    "622",
    "623",
    "624",
    "625",
    "626",
    "627",
    "628",
    "629",
    "630",
    "631",
    "633",
    "634",
]
# subjects = ['101', '102']

data_dir = "/home/data/nbc/physics-learning/data/pre-processed/"
sink_dir = "/home/data/nbc/physics-learning/retrieval-graphtheory/output"

shen = "/home/kbott006/physics-retrieval/shen2015_2mm_268_parcellation.nii.gz"
craddock = "/home/kbott006/physics-retrieval/craddock2012_tcorr05_2level_270_2mm.nii.gz"
masks = {"shen2015": shen, "craddock2012": craddock}

sessions = [0, 1]
sesh = ["pre", "post"]
tasks = ["rest"]
condition = ["rest"]
kappa_upper = 0.21
kappa_lower = 0.31

lab_notebook_dir = "/home/kbott006/lab_notebook/"
index = pd.MultiIndex.from_product([subjects, sessions], names=["subject", "session"])
lab_notebook = pd.DataFrame(index=index, columns=["start", "end", "errors"])

correlation_measure = ConnectivityMeasure(kind="correlation")

invert = InvWarp()
warpspeed = ApplyWarp(interp="nn")

index = pd.MultiIndex.from_product(
    [subjects, sessions, tasks, condition, masks.keys()],
    names=["subject", "session", "task", "condition", "mask"],
)
df = pd.DataFrame(
    index=index, columns=["efficiency", "charpath", "modularity"], dtype=np.float64
)

for subject in subjects:
    for session in sessions:
        lab_notebook.at[(subject, session), "start"] = str(datetime.datetime.now())
        if not exists(
            join(
                sink_dir,
                sesh[session],
                subject,
                "{0}-session-{1}_rest_mni-fnirt-epi-warp.nii.gz".format(
                    subject, session
                ),
            )
        ):
            try:
                mni2epiwarp = join(
                    sink_dir,
                    sesh[session],
                    subject,
                    "{0}-session-{1}_rest_mni-fnirt-epi-warp.nii.gz".format(
                        subject, session
                    ),
                )

                # invert the epi-to-mni warpfield so you can run these analyses in native space
                invert.inputs.warp = join(
                    data_dir,
                    subject,
                    "session-{0}".format(session),
                    "resting-state/resting-state-0/endor1.feat/reg/example_func2standard_warp.nii.gz",
                )
                invert.inputs.inverse_warp = mni2epiwarp
                invert.inputs.reference = join(
                    data_dir,
                    subject,
                    "session-{0}".format(session),
                    "resting-state/resting-state-0/endor1.feat/reg/example_func.nii.gz",
                )
                inverted = invert.run()

                warpspeed.inputs.ref_file = join(
                    data_dir,
                    subject,
                    "session-{0}".format(session),
                    "resting-state/resting-state-0/endor1.feat/reg/example_func.nii.gz",
                )
                warpspeed.inputs.field_file = inverted.outputs.inverse_warp

                xfmd_masks = {}
                for mask in masks.keys():
                    mask_nativespace = join(
                        sink_dir,
                        sesh[session],
                        subject,
                        "{0}-session-{1}_rest_{2}.nii.gz".format(
                            subject, session, mask
                        ),
                    )
                    warpspeed.inputs.in_file = masks[mask]
                    warpspeed.inputs.out_file = mask_nativespace
                    warped = warpspeed.run()
                    xfmd_masks[mask] = mask_nativespace

                    display = plot_roi(
                        mask_nativespace,
                        bg_img=join(
                            data_dir,
                            subject,
                            "session-{0}".format(session),
                            "resting-state/resting-state-0/endor1.feat/reg/example_func.nii.gz",
                        ),
                        colorbar=True,
                    )
                    display.savefig(
                        join(
                            sink_dir,
                            "qa",
                            "{0}-session-{1}_rest_qa_{2}.png".format(
                                subject, session, mask
                            ),
                        ),
                        dpi=300,
                    )
                    display.close()
            except Exception as e:
                print(e)
        try:
            # shen_masker = NiftiLabelsMasker(xfmd_masks['shen2015'], background_label=0, standardize=True, detrend=True,t_r=3.)
            # craddock_masker = NiftiLabelsMasker(xfmd_masks['craddock2012'], background_label=0, standardize=True, detrend=True,t_r=3.)

            # confounds = '/home/data/nbc/physics-learning/anxiety-physics/output/{1}/{0}/{0}_confounds.txt'.format(subject, sesh[session])
            # epi_data = join(data_dir, subject, 'session-{0}'.format(session), 'resting-state/resting-state-0/endor1.feat', 'filtered_func_data.nii.gz')

            # shen_ts = shen_masker.fit_transform(epi_data, confounds)
            # shen_corrmat = correlation_measure.fit_transform([shen_ts])[0]
            # np.savetxt(join(sink_dir, sesh[session], subject, '{0}-session-{1}-rest_network_corrmat_shen2015.csv'.format(subject, session)), shen_corrmat, delimiter=",")
            shen_corrmat = np.genfromtxt(
                join(
                    sink_dir,
                    sesh[session],
                    subject,
                    "{0}-session-{1}-rest_network_corrmat_shen2015.csv".format(
                        subject, session
                    ),
                ),
                delimiter=",",
            )

            # craddock_ts = craddock_masker.fit_transform(epi_data, confounds)
            # craddock_corrmat = correlation_measure.fit_transform([craddock_ts])[0]
            # np.savetxt(join(sink_dir, sesh[session], subject, '{0}-session-{1}-rest_network_corrmat_craddock2012.csv'.format(subject, session)), craddock_corrmat, delimiter=",")
            craddock_corrmat = np.genfromtxt(
                join(
                    sink_dir,
                    sesh[session],
                    subject,
                    "{0}-session-{1}-rest_network_corrmat_craddock2012.csv".format(
                        subject, session
                    ),
                ),
                delimiter=",",
            )

            ge_s = []
            ge_c = []
            cp_s = []
            cp_c = []
            md_s = []
            md_c = []
            for p in np.arange(kappa_upper, kappa_lower, 0.01):
                ntwk = []
                shen_thresh = bct.threshold_proportional(shen_corrmat, p, copy=True)
                craddock_thresh = bct.threshold_proportional(
                    craddock_corrmat, p, copy=True
                )
                # network measures of interest here
                # global efficiency
                ge = bct.efficiency_wei(shen_thresh)
                ge_s.append(ge)
                ge = bct.efficiency_wei(craddock_thresh)
                ge_c.append(ge)

                # characteristic path length
                dist = bct.distance_wei(shen_thresh)
                cp = bct.charpath(dist)
                cp_s.append(cp[0])
                dist = bct.distance_wei(craddock_thresh)
                cp = bct.charpath(dist)
                cp_c.append(cp[0])

                # modularity
                md = bct.modularity_louvain_und(shen_thresh)
                md_s.append(md[1])
                md = bct.modularity_louvain_und(craddock_thresh)
                md_c.append(md[1])
            df.at[
                (subject, session, "rest", "rest", "shen2015"), "efficiency"
            ] = np.trapz(ge_s, dx=0.1)
            df.at[
                (subject, session, "rest", "rest", "shen2015"), "charpath"
            ] = np.trapz(cp_s, dx=0.1)
            df.at[
                (subject, session, "rest", "rest", "shen2015"), "modularity"
            ] = np.trapz(md_s, dx=0.1)
            df.at[
                (subject, session, "rest", "rest", "craddock2012"), "efficiency"
            ] = np.trapz(ge_c, dx=0.1)
            df.at[
                (subject, session, "rest", "rest", "craddock2012"), "charpath"
            ] = np.trapz(cp_c, dx=0.1)
            df.at[
                (subject, session, "rest", "rest", "craddock2012"), "modularity"
            ] = np.trapz(md_c, dx=0.1)
            # df.to_csv(join(sink_dir, 'resting-state_graphtheory_shen+craddock.csv'), sep=',')
            lab_notebook.at[(subject, session), "end"] = str(datetime.datetime.now())
        except Exception as e:
            print(e, subject, session)
            lab_notebook.at[(subject, session), "errors"] = [
                e,
                str(datetime.datetime.now()),
            ]
        df.to_csv(
            join(sink_dir, "resting-state_graphtheory_shen+craddock.csv"), sep=","
        )

df.to_csv(
    join(
        sink_dir,
        "resting-state_graphtheory_shen+craddock_{0}.csv".format(
            str(datetime.datetime.today())
        ),
    ),
    sep=",",
)
lab_notebook.to_csv(
    join(
        lab_notebook_dir,
        "resting-state-graph-theory_{0}.csv".format(str(datetime.datetime.now())),
    )
)

max_stars_repo_name: NBCLab/IDConn

max_issues_repo_name: NBCLab/IDConn

max_forks_repo_name: NBCLab/IDConn


text:
import os
import json
import pickle
import datetime
import numpy as np
import pandas as pd
import seaborn as sns
import yfinance as yf
from tqdm import tqdm
from src.db import DataBase
import matplotlib.pyplot as plt
from configparser import ConfigParser
from src.utils_beeps import beeps
from src.utils_stocks import get_df_c
from src.utils_general import get_df_sym
from src.utils_general import plot_divergences
from src.utils_date import get_ls_date_str_from_db
# directories
DIR_DB = os.path.join(os.getcwd(), 'data', 'db')
DIR_TRAIN = os.path.join(os.getcwd(), 'data', 'train')
F_CFG = os.path.join(os.getcwd(), 'config.ini')
# objects
db = DataBase([], DIR_DB)
cfg = ConfigParser()
cfg.read(F_CFG)
# constants
ERROR_EXCEPTION = 'Error: Exception found ({}: {})'
ERROR_SUMMARY = '{} - {}'
ERROR_PCT = 'Errors: {}/{} {:.3f}'
MSG_DATE_RANGE = 'Creating df_train for date range: {} to {}'
MSG_SAVED = '{} saved!'
FILENAME_TRAIN = 'df_train_{}.parquet'
CFG_SECTION = 'get_train_data'
LIVE_DATA = cfg.getint(CFG_SECTION, 'LIVE_DATA')
TARGET_PROFIT = cfg.getfloat(CFG_SECTION, 'TARGET_PROFIT')
TARGET_LOSS = cfg.getfloat(CFG_SECTION, 'TARGET_LOSS')
DATE_START = cfg.get(CFG_SECTION, 'DATE_START')
DATE_END = cfg.get(CFG_SECTION, 'DATE_END')
LS_DATE_STR = get_ls_date_str_from_db(DATE_START, DATE_END, db)
LS_COL = [
    'is_profit',
    'rsi14',
    'sma9_var',
    'sma180_var',
    'vwap_var',
    'spread14_e',
    'volume14_34_var',
    'prev_close_var',
    'prev_floor_var',
    'prev_ceil_var',
    'prev1_candle_score',
    'prev2_candle_score',
    'prev3_candle_score',
    'mins_from_start',
    'valley_interval_mins',
    'valley_close_score',
    'valley_rsi_score',
    'day_open_var',
    'open_from_prev_close_var',
    'ceil_var',
    'floor_var',
    'day_sma9_var',
    'day_sma180_var',
    'day_rsi14',
    ###
    'sym',
    'datetime',
    'profit',
    'prev_close',
    'divergence',
]
# extract and transform
print(MSG_DATE_RANGE.format(LS_DATE_STR[0], LS_DATE_STR[-1]))
ls_df_t = []
for date_str in LS_DATE_STR:
    dt_errors = {}
    num_runs = 0
    print(date_str)
    df_sym = get_df_sym(db, date_str)
    for i, tup in tqdm(df_sym.iterrows(), total=df_sym.shape[0]):
        sym = tup['sym']
        num_runs += 1
        try:
            df_c = get_df_c(sym, date_str, LIVE_DATA, db, TARGET_PROFIT, TARGET_LOSS)
            ls_df_t.append(df_c[df_c['divergence']!=''][LS_COL])
        except Exception as e:
            dt_errors[sym] = ERROR_EXCEPTION.format(type(e).__name__, e)
    if dt_errors:
        [print(ERROR_SUMMARY.format(sym, dt_errors[sym])) for sym in dt_errors]
        print(ERROR_PCT.format(len(dt_errors), num_runs, len(dt_errors)/num_runs))
if ls_df_t:
    # save df_train
    df_t = pd.concat(ls_df_t)
    df_t = df_t.dropna()
    print(df_t.info())
    # df_train - Export
    timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M')
    f = os.path.join(DIR_TRAIN, FILENAME_TRAIN.format(timestamp))
    df_t.to_parquet(f)
print(MSG_SAVED.format(f))
beeps()

max_stars_repo_name: flowtide/five-minute-midas

max_issues_repo_name: flowtide/five-minute-midas

max_forks_repo_name: flowtide/five-minute-midas


text:
def compute_square_area(side_length):
    return pow(side_length, 2)

class Square:
    def __init__(self, side_length):
        self.side_length = side_length

    def compute_area(self):
        return pow(self, side_length, 2)

if __name__ == "__main__"
    example_square = Square(15)
    print(example_square.compute_area())

class Email:
    def __init__(self, to, subject, body):
        self.to = to
        self.subject = subject
        self.body = body

    def send(self):
        print("Sending email with subject: {self.subject} and body: {self.body} to: {self.to}")

if __name__ == "__main__":
    greeting_email = Email("bob@nasa.gov", "Welcome!", "Welcome to epicpython.io!")
    greeting_email.send()

#----------------------------------------------------------------------------------------------------------------------------------------
import pandas as pd
import numpy as np
import textract
import os
import re
#import zipfile
import shutil

global key_words_and_symbols
key_words_and_symbols = ['as','distinct','cast','when','then','end','case','from','join','on','and','where','like','or','between','order','by','select','from','left', 'join', 'and','is', 'null', '=']

def tables_in_query(sql_str):

    # remove the /* */ comments
    q = re.sub(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", "", sql_str)

    # remove whole line -- and # comments
    lines = [line for line in q.splitlines() if not re.match("^\s*(--|#)", line)]

    # remove trailing -- and # comments
    q = " ".join([re.split("--|#", line)[0] for line in lines])
    #print(q)

    # split on blanks, parens and semicolons
    tokens = re.split(r"[\s)(;]+", q)
    for i in range(len(tokens)):
        tokens[i] = tokens[i].lower()
    #print('TOKENS',tokens)

    # scan the tokens. if we see a FROM or JOIN, we set the get_next
    # flag, and grab the next one (unless it's SELECT).

    tables = list()
    get_next = False
    for tok in tokens:
        if get_next:
            if tok.lower() not in ["", "select"]:
                tables.append(tok)
                pos = tokens.index(tok)
                if tokens[pos +1] not in ["from", "join","where",""]:
                    tables.append(tokens[pos +1])
            get_next = False
        get_next = tok.lower() in ["from", "join"]
    print('TABLES', tables)
        #Extract aliases
    aliases = []
    table = []
    for index, element in enumerate(tables):
    	if index % 2 != 0:
            aliases.append(element)
    for index, element in enumerate(tables):
        if index % 2 == 0:
            table.append(element)

    db_name = []
    schema_name = []
    final_tables = []
    for ele in table:
        lst1 = ele.split('.')
        if len(lst1) == 3:
            db_name.append(lst1[0])
            schema_name.append(lst1[1])
            final_tables.append(lst1[2])
        elif len(lst1) == 2:
            db_name.append(lst1[0])
            final_tables.append(lst1[1])

    return aliases,  table, db_name, schema_name

#FIELDS in query -- DO NOT USE

def fields_in_query(sql_str,tables):
    # remove the /* */ comments
    q = re.sub(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", "", sql_str)

    # remove whole line -- and # comments
    lines = [line for line in q.splitlines() if not re.match("^\s*(--|#)", line)]

    # remove trailing -- and # comments
    q = " ".join([re.split("--|#", line)[0] for line in lines])
    #print(q)

    # split on blanks, parens and semicolons
    tokens = re.split(r"[\s)(;]+", q)
    for i in range(len(tokens)):
        tokens[i] = tokens[i].lower()

            #get the list of values after 'As'
    value_after_as = list()
    get_next = False
    for tok in tokens:
        if get_next:
            if tok.lower() not in ["", "select"]:
                value_after_as.append(tok)
            get_next = False
        get_next = tok.lower() in ["as", "like" ]
    #print(value_after_as)

    fields_1 = list(set(tokens) - set(key_words_and_symbols))
    fields_2 = list(set(fields_1) - set(tables))
    final_fields_list = list(set(fields_2) - set(value_after_as))
    return(final_fields_list)

    #extract fields #2
def get_field_name(qry):
    print('QUERY',qry.lower())
    qry2 = qry.lower()
    name_list = []
    start = qry2.find('select') + 7
    end = qry2.find('from', start)
    skills = qry2[start:end]
    skills = skills.lower()
    #field_name = re.sub(r'/\*[^*]*\*+(?:[^*/][^*]*\*+)*/',"", skills)
    #tokens = re.split(r"[\t\n:]+", field_name)
    #tokens = re.split(r"[\t\n:]+", skills)
    name_list.append(skills)
    fields = ([s.replace('\t', '') for s in name_list])
    fields = ([s.replace('=', '') for s in fields])
    fields = ([s.replace('when', '') for s in fields])
    fields = ([s.replace('case', '') for s in fields])
    fields = ([s.replace('distinct', '') for s in fields])
    fields = ([s.replace('isnull', '') for s in fields])

    # names = pd.DataFrame(columns=['fields'])
    # for name in name_list:
    #     names = names.append(pd.DataFrame({'fields': name}, index=[0]), ignore_index=True)
    return fields

files_path_list = []
files_list = []

for filename in os.listdir(os.getcwd()):
        if filename.endswith(".txt"):
            file_path = os.path.join(os.getcwd(), filename)
            files_path_list.append(file_path)
            files_list.append(filename)

#print(files_path_list)
print(files_list)
#files_list = ['5ED25AF7.txt']

final_dict = {}
final_dict['Server'] = []
final_dict['DataBase'] = []
final_dict['Schema'] = []
final_dict['Tables'] = []
final_dict['Aliases'] = []
final_dict['Fields'] = []

for file in files_list:
    file = open(file)
    qry = file.read().replace("\n", " ")
    file.close()
    print(qry)
    print('\n')
    aliases,tables, db, schemas = tables_in_query(qry)
    #fields = fields_in_query(qry, tables)
    fields = get_field_name(qry)
    #tables_in_query(qry)
    final_dict['Server'].append(file.name)
    final_dict['DataBase'].append(db)
    final_dict['Schema'].append(schemas)
    final_dict['Tables'].append(tables)
    final_dict['Aliases'].append(aliases)
    final_dict['Fields'].append(fields)
    print(file.name)
    print(tables)
    #print(fields)
    #print(list(table))
    print('\n')
    print('\n')

print(final_dict)
final_df=pd.DataFrame(final_dict)
print(final_df)
final_df.to_csv('extracted_text.csv')

#-----------------------------------------------------------------------------------------------------------------------------------------------------------------

import pandas as pd
import numpy as np
import textract
import os
import re
#import zipfile
import shutil

global key_words_and_symbols
key_words_and_symbols = ['as','distinct','cast','when','then','end','case','from','join','on','and','where','like','or','between','order','by','select','from','left', 'join', 'and','is', 'null', '=']

def tables_in_query(sql_str):

    # remove the /* */ comments
    q = re.sub(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", "", sql_str)

    # remove whole line -- and # comments
    lines = [line for line in q.splitlines() if not re.match("^\s*(--|#)", line)]

    # remove trailing -- and # comments
    q = " ".join([re.split("--|#", line)[0] for line in lines])
    #print(q)

    # split on blanks, parens and semicolons
    tokens = re.split(r"[\s)(;]+", q)
    for i in range(len(tokens)):
        tokens[i] = tokens[i].lower()
    #print('TOKENS',tokens)

    # scan the tokens. if we see a FROM or JOIN, we set the get_next
    # flag, and grab the next one (unless it's SELECT).

    tables = list()
    get_next = False
    for tok in tokens:
        if get_next:
            if tok.lower() not in ["", "select"]:
                tables.append(tok)
                pos = tokens.index(tok)
                if tokens[pos +1] not in ["from", "join","where",""]:
                    tables.append(tokens[pos +1])
            get_next = False
        get_next = tok.lower() in ["from", "join"]
    print('TABLES', tables)
        #Extract aliases
    aliases = []
    table = []
    for index, element in enumerate(tables):
    	if index % 2 != 0:
            aliases.append(element)
    for index, element in enumerate(tables):
        if index % 2 == 0:
            table.append(element)

    db_name = []
    schema_name = []
    final_tables = []
    for ele in table:
        lst1 = ele.split('.')
        if len(lst1) == 3:
            db_name.append(lst1[0])
            schema_name.append(lst1[1])
            final_tables.append(lst1[2])
        elif len(lst1) == 2:
            db_name.append(lst1[0])
            final_tables.append(lst1[1])

    return aliases,  table, db_name, schema_name

def get_field_name_3(qry): #qry is a string
    print('QUERY',qry.lower())
    qry = qry.lower()
    fields = re.findall('select(.+?)from', qry)

    # for each_item in fields:
    #     print('EACH_ITEM_TYPE',type(each_item))

    return fields

files_path_list = []
files_list = []

for filename in os.listdir(os.getcwd()):
        if filename.endswith(".txt"):
            file_path = os.path.join(os.getcwd(), filename)
            files_path_list.append(file_path)
            files_list.append(filename)

#print(files_path_list)
print(files_list)
#files_list = ['5ED25AF7.txt']

final_dict = {}
final_dict['Server'] = []
final_dict['DataBase'] = []
final_dict['Schema'] = []
final_dict['Tables'] = []
final_dict['Aliases'] = []
final_dict['Fields'] = []

for file in files_list:
    file = open(file)
    qry = file.read().replace("\n", " ")
    file.close()
    print(qry)
    print('\n')
    aliases,tables, db, schemas = tables_in_query(qry)
    #fields = fields_in_query(qry, tables)#1
    #fields = get_field_name(qry) #2
    fields = get_field_name_3(qry) #3
    # get_field_name_3(qry) #3
        #tables_in_query(qry)
    final_dict['Server'].append(file.name)
    final_dict['DataBase'].append(db)
    final_dict['Schema'].append(schemas)
    final_dict['Tables'].append(tables)
    final_dict['Aliases'].append(aliases)
    final_dict['Fields'].append(fields)
    #print(file.name)
    # print(tables)
    # #print(fields)
    # #print(list(table))
    # print('\n')
    # print('\n')

#print(final_dict)
final_df=pd.DataFrame(final_dict)
for col_name,col_data in final_df.iteritems():
    print(col_name,col_data.values )
final_df.to_csv('extracted_text.csv')