text:
import numpy as np
import matplotlib
import scipy.interpolate

matplotlib.use('TkAgg') # Can change to 'Agg' for non-interactive mode
import pandas as pd
import matplotlib.pyplot as plt
plt.rcParams['svg.fonttype'] = 'none'
import math
from baselines.bench.monitor import load_results

X_TIMESTEPS = 'timesteps'
X_EPISODES = 'episodes'
X_WALLTIME = 'walltime_hrs'
POSSIBLE_X_AXES = [X_TIMESTEPS, X_EPISODES, X_WALLTIME]
EPISODES_WINDOW = 100 #average over the last EPISODE_WINDOWS episodes
COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow', 'black', 'purple', 'pink',
        'brown', 'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise',
        'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue']

def rolling_window(a, window):
    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
    strides = a.strides + (a.strides[-1],)
    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)

def window_func(x, y, window, func):
    yw = rolling_window(y, window)
    yw_func = func(yw, axis=-1)
    return x[window-1:], yw_func

def ts2xy(ts, xaxis):
    if xaxis == X_TIMESTEPS:
        x = np.cumsum(ts.l.values)
        y = ts.r.values
    elif xaxis == X_EPISODES:
        x = np.arange(len(ts))
        y = ts.r.values
    elif xaxis == X_WALLTIME:
        x = ts.t.values / 3600.
        y = ts.r.values
    else:
        raise NotImplementedError
    return x, y
def roundup(x):
    return int(math.ceil(x / 10.0)) * 10
def plot_curves(xy_list, xaxis, title):
    fig = plt.figure(figsize=(8,8))
    maxx = max(xy[0][-1] for xy in xy_list)
    minx = 0
    runs = 10 #trials

    for (i, (x, y)) in enumerate(xy_list):
        # x: time
        # y: reward
        color = COLORS[i//runs]

        if i % runs ==0:
            y_list = []
            x_list = []
            if i//runs == 0:
                label = "Original Loss"

            if i//runs == 1:

                label = "log(2-gratio)"

        # print(x.shape)
        # print(y.shape)
        # input("wait 1 ")
        # plt.scatter(x, y, s=2)
        x, y_mean = window_func(x, y, EPISODES_WINDOW, np.mean) #So returns average of last EPISODE_WINDOW episodes
        # plt.plot(x, y_mean, color=color)
        # print(x.shape)
        y_list.append(y_mean)
        x_list.append(x)
        # input("wait 2 ")

        if len(y_list)==runs:
            shortest_length = min(map(len, y_list))
            print(list(map(len, y_list)))

            lst2 = [item[0] for item in x_list]
            m0 = max(lst2)
            lst2 = [item[-1] for item in x_list]
            m1 = min(lst2)

            xx = np.linspace(roundup(m0), roundup(m1-10), (m1 - m0)//10)
            y_list2 = []
            for e1,e2 in zip(x_list,y_list):
                # interpolate for each run
                # TODO: check
                y_interp = scipy.interpolate.interp1d(e1, e2)
                y_list2.append(y_interp(xx))

            y_list = y_list2
            y_all = np.array(y_list)
            y_min = np.min(y_all, axis=0)
            y_max = np.max(y_all, axis=0)
            y_mean = np.mean(y_all, axis=0)
            # plt.plot(x, y_mean, color=color, label=label)
            plt.plot(xx, y_mean, color=color, label=label)
            # print(y_interp(1839.0))
            # print(y_interp(1840.0))
            # print(y_interp(1841.0))
            # print(y_interp(1842.0))
            plt.legend(loc=4)
            plt.fill_between(xx, y_min,y_max , color=color,alpha=.3,lw=0.0, edgecolor="None")

    plt.grid()
    plt.xlim(minx, maxx)
    plt.title(title)
    plt.xlabel(xaxis)
    plt.ylabel("Episode Rewards")
    plt.tight_layout()
    fig.savefig('./Graphs/'+ title +'.pdf')  # save the figure to file
    plt.close(fig)
def plot_results(dirs, num_timesteps, xaxis, task_name):
    tslist = []
    for dir in dirs:
        ts = load_results(dir)

        ts = ts[ts.l.cumsum() <= num_timesteps]
        tslist.append(ts)
    xy_list = [ts2xy(ts, xaxis) for ts in tslist]
    plot_curves(xy_list, xaxis, task_name)

# Example usage in jupyter-notebook
# from baselines import log_viewer
# %matplotlib inline
# log_viewer.plot_results(["./log"], 10e6, log_viewer.X_TIMESTEPS, "Breakout")
# Here ./log is a directory containing the monitor.csv files

def main():
    import argparse
    import os

    env = "HalfCheetah-v2"
    alg = "trpo_mpi"
    res_list = []
    for loss in range(2):
        for trial in range(10):

                res_list.append("./"+alg+"/log/"+env+"/"+"Loss_"+str(loss)+"_Run_"+str(trial))

    print(res_list)
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--dirs', help='List of log directories', nargs = '*', default=res_list)

    parser.add_argument('--num_timesteps', type=int, default=int(40e6))
    parser.add_argument('--xaxis', help = 'Varible on X-axis', default = X_TIMESTEPS)
    parser.add_argument('--task_name', help = 'Title of plot', default = env + "__" + alg)
    args = parser.parse_args()
    args.dirs = [os.path.abspath(dir) for dir in args.dirs]
    plot_results(args.dirs, args.num_timesteps, args.xaxis, args.task_name)
    plt.show()

if __name__ == '__main__':
    main()

max_stars_repo_name: erincmer/BPO

max_issues_repo_name: erincmer/BPO

max_forks_repo_name: erincmer/BPO


text:
import spacy
import pickle
import warnings
import numpy
import os
import sys
import colorama

warnings.filterwarnings("ignore")
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('bert-base-nli-mean-tokens')

botName = sys.argv[1]

botPath = 'C:\\Users\\Alice Barthe\\PycharmProjects\\mdrs_bots\\bots\\' + botName + '\\'
sents = pickle.load(open(botPath + 'sents.p', 'rb'))
files = pickle.load(open(botPath + 'files.p', 'rb'))
start = pickle.load(open(botPath + 'start.p', 'rb'))
embds = pickle.load(open(botPath + 'embds.p', 'rb'))

colorama.init()

# print("trained on :")
# for f in files:
#     head_tail = os.path.split(f)
#     print(head_tail[1])

def cosine_similarity(a, b):
    res = numpy.dot(a, b)/(numpy.linalg.norm(a)*numpy.linalg.norm(b))
    return res

def find_similar(user_input_fun):
    print('analysing : ' + user_input_fun)
    # user = nlp(user_input_fun)
    # similarity = [user.similarity(nlp(sent)) for sent in sents]
    input_embd = model.encode(user_input_fun)
    input_embd = input_embd[0, :]
    similarity = [cosine_similarity(embd, input_embd) for embd in embds]
    index = numpy.argsort(similarity)
    return index, similarity

def extract_whole(id_doc_fun, id_sen_fun):
    string_whole = ""
    for i in range(start[id_doc_fun], id_sen_fun):
        string_whole = string_whole + " " + sents[i]
    string_whole = string_whole + " " + colorama.Fore.LIGHTYELLOW_EX + sents[id_sen_fun] + colorama.Fore.RESET
    for i in range(id_sen_fun+1, start[id_doc_fun + 1]):
        string_whole = string_whole + " " + sents[i]
    return string_whole

continue_dialogue = True
index_cur = None
similarity_cur = None
id_doc = 0
my_index = 0
answ_cur = 0
index_cur = []

print(colorama.Fore.LIGHTGREEN_EX + "Hello, I am your friend " + botName + ". You can ask me any question :)" + colorama.Fore.RESET)
while continue_dialogue:
    print(colorama.Fore.BLUE + 'tell me' + colorama.Fore.RESET)
    human_text = input()
    if human_text != 'bye':
        if human_text.startswith('ask '):
            answ_cur = 0
            index_cur, similarity_cur = find_similar(human_text[4:])
            my_index = index_cur[-1-answ_cur]
            print(colorama.Fore.LIGHTYELLOW_EX + str(similarity_cur[my_index]) + colorama.Fore.RESET)
            print(colorama.Fore.LIGHTYELLOW_EX + sents[my_index] + colorama.Fore.RESET)
        else:
            if human_text.startswith('next'):
                if len(index_cur) != 0 and answ_cur+1 < len(index_cur):
                    answ_cur += 1
                    my_index = index_cur[-1 - answ_cur]
                    print(colorama.Fore.LIGHTYELLOW_EX + str(similarity_cur[my_index]) + colorama.Fore.RESET)
                    print(colorama.Fore.LIGHTYELLOW_EX + sents[my_index] + colorama.Fore.RESET)
                else:
                    print(colorama.Fore.RED + 'nothing else corresponds' + colorama.Fore.RESET)
                    answ_cur += 1
            else:
                if human_text.startswith('where'):
                    if len(index_cur) != 0:
                        id_doc = 0
                        boolContinue = start[id_doc+1] <= my_index
                        while boolContinue:
                            id_doc += 1
                            if id_doc+1 >= len(start):
                                boolContinue = False
                            else:
                                boolContinue = start[id_doc+1] <= my_index
                        print(colorama.Fore.LIGHTYELLOW_EX + files[id_doc] + colorama.Fore.RESET)
                else:
                    if human_text.startswith('search '):
                        answ_cur = 0
                        user_input = human_text[7:]
                        user_input = user_input.lower()
                        print(colorama.Fore.LIGHTCYAN_EX + 'searching for : ' + user_input + colorama.Fore.RESET)
                        index_cur = [i for i in range(0, len(sents)) if user_input in sents[i]]
                        similarity_cur = [1] * len(sents)
                        if len(index_cur) != 0:
                            my_index = index_cur[-1 - answ_cur]
                            print(colorama.Fore.LIGHTCYAN_EX + "answer number " + str(answ_cur+1) + " out of " + str(len(index_cur)) + colorama.Fore.RESET)
                            print(colorama.Fore.LIGHTYELLOW_EX + sents[my_index] + colorama.Fore.RESET)
                        else:
                            print(colorama.Fore.RED + 'I did not find anything corresponding' + colorama.Fore.RESET)
                    else:
                        if human_text.startswith('whole'):
                            print(colorama.Fore.LIGHTCYAN_EX + "printing the whole document " + files[id_doc] + colorama.Fore.RESET)
                            print(extract_whole(id_doc, my_index))
                        else:
                            print(colorama.Fore.RED + 'I did not get that' + colorama.Fore.RESET)
    else:
        continue_dialogue = False
        print(colorama.Fore.LIGHTGREEN_EX + "Ciao :) " + colorama.Fore.RESET)

ext: py

reason: import numpy


text:
import torch
from torch.autograd import Variable
import torch.nn as nn
import pandas as pd
from pandas import DataFrame
import matplotlib.pyplot as plt
import numpy as np

# RNN 模型
class RNN(nn.Module):
    def __init__(self, look_back):
        super(RNN, self).__init__()  # 面向对象中的继承
        self.lstm = nn.LSTM(look_back, 6, 1)  # 输入数据2个特征维度，6个隐藏层维度，2个LSTM串联，第二个LSTM接收第一个的计算结果
        self.out = nn.Linear(6, 1)  # 线性拟合，接收数据的维度为6，输出数据的维度为2

    def forward(self, x):
        x1, _ = self.lstm(x)
        a, b, c = x1.shape
        out = self.out(x1.view(-1, c))  # 因为线性层输入的是个二维数据，所以此处应该将lstm输出的三维数据x1调整成二维数据，最后的特征维度不能变
        out1 = out.view(a, b, -1)  # 因为是循环神经网络，最后的时候要把二维的out调整成三维数据，下一次循环使用
        return out1

# 数据集和目标值赋值，data_set为数据，look_back为以几行数据为特征维度数量
def create_data_set(data_set, look_back):
    data_x = []
    data_y = []
    for i in range(len(data_set) - look_back):
        data_x.append(data_set[i:i + look_back])
        data_y.append(data_set[i + look_back])
    return np.asarray(data_x), np.asarray(data_y)  # 转为ndarray数据

# 一、数据准备
# 参考长度
look_back = 3

# 本地数据
df1 = pd.read_excel(r'simple2.xlsx')
# df1 = pd.read_excel(r'Stock_Open_Data.xlsx')
datas = df1.values
datas = datas[:]

print('read data ok')
# 归一化处理，这一步必不可少，不然后面训练数据误差会很大，模型没法用
max_value = np.max(datas)
min_value = np.min(datas)
scalar = max_value - min_value
datas = list(map(lambda x: (x - min_value) / scalar, datas))

# 以look_back为特征维度，得到数据集
dataX, dataY = create_data_set(datas, look_back)
dataX = dataX.transpose((0, 2, 1))

train_size = int(len(dataX) * 0.7)
x_train = dataX[:train_size]  # 训练数据
y_train = dataY[:train_size]  # 训练数据目标值
x_train = x_train.reshape(-1, 2, look_back)  # 将训练数据调整成pytorch中lstm算法的输入维度
y_train = y_train.reshape(-1, 2, 1)  # 将目标值调整成pytorch中lstm算法的输出维度

# 将ndarray数据转换为张量，因为pytorch用的数据类型是张量
x_train = torch.from_numpy(x_train)
y_train = torch.from_numpy(y_train)
print('read data ok')

# 二、创建LSTM模型
rnn = RNN(look_back)

# 参数寻优，计算损失函数
optimizer = torch.optim.Adam(rnn.parameters(), lr=0.02)
loss_func = nn.MSELoss()
print('read data ok')

# 三、训练模型
for i in range(1000):
    var_x = Variable(x_train).type(torch.FloatTensor)
    var_y = Variable(y_train).type(torch.FloatTensor)
    out = rnn(var_x)
    loss = loss_func(out, var_y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    if (i + 1) % 100 == 0:
        print('Epoch:{}, Loss:{:.5f}'.format(i + 1, loss.item()))

# 四、模型测试
# 准备测试数据
dataX1 = dataX.reshape(-1, 2, look_back)
dataX2 = torch.from_numpy(dataX1)
var_dataX = Variable(dataX2).type(torch.FloatTensor)

pred = rnn(var_dataX)
pred_data = pred.view(-1, 2).data.numpy()  # 转换成一维的ndarray数据，这是预测值  dataY为真实值

# 五、画图检验
# 把y轴的主刻度设置为10的倍数
# plt.xlim(-0.1, 1.1)
# 把x轴的刻度范围设置为-0.5到11，因为0.5不满一个刻度间隔，所以数字不会显示出来，但是能看到一点空白
# plt.ylim(-0.1, 1.1)
plt.plot(pred_data[:, 0], pred_data[:, 1], 'r', label='prediction')
plt.plot(pred_data[train_size+1, 0], pred_data[train_size+1, 1], 'o', label='start point')
plt.plot(dataY[:, 0], dataY[:, 1], 'b', label='real')
plt.legend(loc='best')
plt.show()


text:
import os
from pathlib import Path

import numpy as np
import rasterio
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
from pl_bolts.models.self_supervised.moco.transforms import GaussianBlur, imagenet_normalization

ALL_BANDS = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B9', 'B11', 'B12']
RGB_BANDS = ['B4', 'B3', 'B2']

QUANTILES = {
    'min_q': {
        'B2': 3.0,
        'B3': 2.0,
        'B4': 0.0
    },
    'max_q': {
        'B2': 88.0,
        'B3': 103.0,
        'B4': 129.0
    }
}

class SeasonalContrastBase(Dataset):

    def __init__(self, root, bands=None, transform=None):
        super().__init__()
        self.root = Path(root)
        self.bands = bands if bands is not None else RGB_BANDS
        self.transform = transform

        self._samples = None

    @property
    def samples(self):
        if self._samples is None:
            self._samples = self.get_samples()
        return self._samples

    def get_samples(self):
        raise NotImplementedError

    def __len__(self):
        return len(self.samples)

def normalize(img, min_q, max_q):
    img = (img - min_q) / (max_q - min_q)
    img = np.clip(img * 255.0, 0, 255).astype(np.uint8)
    return img

def read_image(path, bands, quantiles=None):
    channels = []
    for b in bands:
        ch = rasterio.open(path / f'{b}.tif').read(1)
        if quantiles is not None:
            ch = normalize(ch, min_q=quantiles['min_q'][b], max_q=quantiles['max_q'][b])
        channels.append(ch)
    img = np.dstack(channels)
    img = Image.fromarray(img)
    return img

class SeasonalContrastBasic(SeasonalContrastBase):

    def get_samples(self):
        # return [path for path in self.root.glob('*/*') if path.is_dir()]
        samples = []
        for entry in os.scandir(self.root):
            for subentry in os.scandir(entry.path):
                if subentry.is_dir():
                    samples.append(Path(subentry.path))
        return samples

    def __getitem__(self, index):
        path = self.samples[index]
        img = read_image(path, self.bands, QUANTILES)
        if self.transform is not None:
            img = self.transform(img)
        return img

class SeasonalContrastTemporal(SeasonalContrastBase):

    def get_samples(self):
        return [path for path in self.root.glob('*') if path.is_dir()]

    def __getitem__(self, index):
        root = self.samples[index]
        paths = np.random.choice([path for path in root.glob('*') if path.is_dir()], 2)
        images = []
        for path in paths:
            img = read_image(path, self.bands, QUANTILES)
            if self.transform is not None:
                img = self.transform(img)
            images.append(img)
        return images[0], images[1]

class SeasonalContrastMultiAug(SeasonalContrastBase):

    augment = transforms.Compose([
        transforms.RandomApply([
            transforms.ColorJitter(0.4, 0.4, 0.4, 0.1)  # not strengthened
        ], p=0.8),
        transforms.RandomGrayscale(p=0.2),
        transforms.RandomApply([GaussianBlur([.1, 2.])], p=0.5),
        transforms.RandomHorizontalFlip(),
    ])
    preprocess = transforms.Compose([
        transforms.RandomResizedCrop(224, scale=(0.2, 1.)),
        transforms.ToTensor(),
        imagenet_normalization()
    ])

    def get_samples(self):
        return [path for path in self.root.glob('*') if path.is_dir()]

    def __getitem__(self, index):
        root = self.samples[index]
        sorted_paths = sorted([path for path in root.glob('*') if path.is_dir()], reverse=True)
        t0, t1, t2 = [read_image(path, self.bands, QUANTILES) for path in np.random.choice(sorted_paths, 3)]

        q = t0
        k0 = self.augment(t1)
        k1 = t2
        k2 = self.augment(t0)

        q = self.preprocess(q)
        k0 = self.preprocess(k0)
        k1 = self.preprocess(k1)
        k2 = self.preprocess(k2)

        return q, [k0, k1, k2]

max_stars_repo_name: SaeidAbdolian/seasonal-contrast

max_issues_repo_name: SaeidAbdolian/seasonal-contrast

max_forks_repo_name: SaeidAbdolian/seasonal-contrast


text:
# Primary Packages
import pandas as pd
import numpy as np
import streamlit as st

# Package to Load Model
from sklearn.externals import joblib

@st.cache
def load_data():
    return pd.read_csv('data/Sell-Out Data.csv')

data = load_data()
data = data[data['COMPANY']!='Competitors']

### Set Title
st.title("Cortex Sales Predictor")
st.write("""From the AC metadata, we built a machine learning-based forecasting model
to predict ac unit sales.""")

# Show data
st.subheader('AC Metadata')
if st.checkbox('Show Raw Data'):
    st.write(data.head(20))

st.sidebar.title('Parameters')

# Unit Level
unit_values = pd.Series(data['UNIT'].unique()).str.strip()
unit_dummies = pd.get_dummies(unit_values)

unit_sample = st.sidebar.selectbox("AC Unit", unit_values.values.tolist())

unit_sample_dummies = (unit_dummies.loc[np.where(unit_values.values == unit_sample)[0]]
                                  .values.tolist()[0])

# Dealer Level
dealer_values = pd.Series(data['DEALER'].unique()).str.strip()
dealer_dummies = pd.get_dummies(dealer_values)

dealer_sample = st.sidebar.selectbox("Dealer", dealer_values.values.tolist())

dealer_sample_dummies = (dealer_dummies.loc[np.where(dealer_values.values == dealer_sample)[0]]
                                  .values.tolist()[0])

# Channel Level
channel_values = pd.Series(data['CHANNEL'].unique()).str.strip()
channel_dummies = pd.get_dummies(channel_values)

channel_sample = st.sidebar.selectbox("Channel", channel_values.values.tolist())

channel_sample_dummies = (channel_dummies.loc[np.where(channel_values.values == channel_sample)[0]]
                                  .values.tolist()[0])

# Sales Channel Level
sales_channel_values = pd.Series(data['SALES CHANNEL'].unique()).str.strip()
sales_channel_dummies = pd.get_dummies(sales_channel_values)

sales_channel_sample = st.sidebar.selectbox("Sales Channel", sales_channel_values.values.tolist())

sales_channel_sample_dummies = (sales_channel_dummies.loc[np.where(sales_channel_values.values == sales_channel_sample)[0]]
                                  .values.tolist()[0])

# Region Level
region_values = pd.Series(data['REGION'].unique()).str.strip()
region_dummies = pd.get_dummies(region_values)

region_sample = st.sidebar.selectbox("Region", region_values.values.tolist())

region_sample_dummies = (region_dummies.loc[np.where(region_values.values == region_sample)[0]]
                                  .values.tolist()[0])

# Brand Level
brand_values = pd.Series(data['BRAND'].unique()).str.strip()
brand_dummies = pd.get_dummies(brand_values)

brand_sample = st.sidebar.selectbox("Brand", brand_values.values.tolist())

brand_sample_dummies = (brand_dummies.loc[np.where(brand_values.values == brand_sample)[0]]
                                  .values.tolist()[0])

# Capacity Level
cap_values = pd.Series(data['CAPACITY'].unique()).str.strip()
cap_dummies = pd.get_dummies(cap_values)

cap_sample = st.sidebar.selectbox("Capacity", cap_values.values.tolist())

cap_sample_dummies = (cap_dummies.loc[np.where(cap_values.values == cap_sample)[0]]
                                  .values.tolist()[0])

# Compressor Level
comp_values = pd.Series(data['COMPRESSOR'].unique()).str.strip()
comp_dummies = pd.get_dummies(comp_values)

comp_sample = st.sidebar.selectbox("Compressor", comp_values.values.tolist())

comp_sample_dummies = (comp_dummies.loc[np.where(comp_values.values == comp_sample)[0]]
                                  .values.tolist()[0])

# Prediction
st.title("Predicted Sales")

# Load Model
model = joblib.load('model/cortex_model.sav')

# Input
sample_features = (unit_sample_dummies + dealer_sample_dummies + channel_sample_dummies + sales_channel_sample_dummies +
                   region_sample_dummies + brand_sample_dummies + cap_sample_dummies + comp_sample_dummies + [1])

# Make Prediction
prediction = model.predict([sample_features])[0]

# Write Prediction
st.write('Predicted sales is %s units.' % int(prediction))


text:
import os
import re
import cv2
import numpy as np
import math

PATH_IMAGE = './real_frames/'

class LaneFollowing:
	def __init__(self, W = 680, H = 480, C = 3, polygon = np.array([[0, 480], [0,400], [680,400], [680,480]]), thresh = 150, waypoint_y = 400, bottom = 40):
		self.W, self.H, self.C = W, H, C
		self.bottom = bottom
		self.polygon = polygon
		self.thresh = thresh
		self.waypoint_y = waypoint_y
		# self.initSize()

	# def drawLine(self):
	# 	pass

	# def getFrames(self):
	# 	# get file names of frames
	# 	col_frames = os.listdir(PATH_IMAGE)

	# 	# load frames
	# 	col_images=[]
	# 	for i in col_frames:
	# 			img = cv2.imread(PATH_IMAGE+i)
	# 			col_images.append(img)

	# 	self.NUM, self.H, self.W, self.C = np.array(col_images).shape
	# 	print(np.array(col_images).shape)
	# 	self.col_images = col_images

	# def run(self):
	# 	for img in self.col_images:
	# 		angle = self.getAngle(img)

	def run(self, img):
		dim = (self.W, self.H)
		img = cv2.resize(img, dim, interpolation = cv2.INTER_AREA)
		#cv2.imshow('original', img)

		# create a zero array
		stencil = np.zeros_like(img[:,:,0])

		# fill polygon with ones
		cv2.fillConvexPoly(stencil, self.polygon, 1)

		# apply polygon as a mask on the frame
		img = cv2.bitwise_and(img[:,:,1], img[:,:,1], mask=stencil)
		#cv2.imshow('polygon', img)

		def binary_threshold(img, thresh):
			return  np.array(np.where(img <= thresh, 0, 255), dtype="uint8")

		img_inv = cv2.bitwise_not(img)
		img_inv = cv2.bitwise_and(img_inv, img_inv, mask=stencil)

		thresh2 = binary_threshold(img_inv, self.thresh)
		#print(thresh2)
		#cv2.imshow('thresh', thresh2)

		def calculateWhiteMean(arr):
			max_arr = 1
			len_arr = 1
			start = arr[0]
			cur = arr[0]
			ret = [0,0]
			for i in range(len(arr)):
				if i == 0:
					continue
				if i == len(arr)-1:
					if max_arr < len_arr:
						max_arr = len_arr
						ret[0] = start
						ret[1] = cur-1

				if arr[i] == arr[i-1] + 1:
					len_arr += 1
					cur = arr[i]
				else:
					if max_arr < len_arr:
						max_arr = len_arr
						len_arr = 1
						ret[0] = start
						ret[1] = cur-1
					start = arr[i]

			return ret[0] + int((ret[1]-ret[0])/2)

		def getBlackList(black):
			start = black[0]
			end = black[0]
			ret = list()
			length = 1
			for i in range(len(black)):
				if i == 0:
					continue

				if i == len(black)-1:
					ret.append([length,start,end])

				if black[i] == black[i-1] + 1:
					length += 1
					end = black[i]

				else:
					ret.append([length,start,end])
					length = 1
					start = black[i]

			return list(filter(lambda x: x[0] >= 25 and x[0] <= 60, ret))

		def mean_horizontal_slide(vert_point):
			arr = thresh2[vert_point]
			white = np.where(arr == 255)[0]
			black = np.where(arr == 0)[0]

			if len(white) == 0 or len(black) ==0:
				white_mean = self.W//2

				return white_mean, 2
			else:
				white_mean = calculateWhiteMean(white)
				black_list =  getBlackList(black)
				if len(black_list) == 1 and white[0] == 0:
					return white_mean, 0
				elif len(black_list) == 1 and white[-1] == self.W-1:
					return white_mean, 1
				else:
					return white_mean, 2

		# 0: 1-left
		# 1: 1- right
		# 2: 2
		top_mean, top_black= mean_horizontal_slide(self.waypoint_y)
		center_mean, center_black= mean_horizontal_slide(self.H - self.bottom)
		bot_mean, bot_black= mean_horizontal_slide(self.H-1)

		if top_black == 0 or top_black == 1:
			return thresh2, self.getActionblack(top_black)
		else:
			angle = 180 / math.pi * math.atan2(self.H-self.waypoint_y, top_mean - bot_mean) - 90

			print(angle)
			dmy = thresh2.copy()

			cv2.line(dmy,  (top_mean, self.waypoint_y),  (bot_mean, self.H - 1),  (0,0,0),  3 )

			return dmy, self.getActionWhite(angle, top_mean)

	def getActionWhite(self, angle, top_mean):
		# if top_mean<= 250:
		# 	#return 'right-out'
		# 	return "left"
		# elif top_mean >= 430:
		# 	#return 'left-out'
		# 	return "right"
		# else:

		if abs(angle) < 10:
			return 'forward'
		if angle > 0 :
			#return 'left-2'
			return "left"
		#return 'right-2'
		return "right"

	def getActionblack(self, top):
		if top == 0:
			#return 'left-1'
			return "left-90"
		else:
			#return 'right-1'
			return "right-90"

def main():
	lf = LaneFollowing()
	lf.run()

if __name__ == "__main__":
	cam = cv2.VideoCapture(1)
	cam.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
	cam.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
	lf = LaneFollowing()
	while True:
		ret, frame = cam.read()
		frame, text = lf.run(frame)
		print(text)
		cv2.imshow("", frame)
		cv2.waitKey(1)


text:
"""
@file
@brief Plotting helpers
"""
import numpy
from ..benchmark import enumerate_options

def list_col_options(df, cols):
    """
    @param      df      dataframe
    @param      cols    cols to look for options
    @return             options

    .. exref::
        :title: Enumerate all coordinates

        .. runpython::
            :showcode:

            from pymlbenchmark.plotting.plot_helper import list_col_options
            from pandas import DataFrame
            df = DataFrame([
                dict(i=1, t='aa', x=0.5),
                dict(i=2, t='bb', x=0.5),
                dict(i=2, t='aa', x=0.5),
            ])
            for opt in list_col_options(df, ['i', 't']):
                print(opt)

            # if None...
            print(list_col_options(df, None))
    """
    if cols is None:
        return [None]

    if not isinstance(cols, (tuple, list)):
        cols = [cols]
    elif cols[0] is None:
        return [None]

    options = {k: list(sorted(set(df[k]))) for k in cols}
    return list(enumerate_options(options))

def filter_df_options(df, options):
    """
    Filters out rows from a dataframe.

    @param      df          dataframe
    @param      options     options
    @return                 filtered dataframe

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import filter_df_options
        from pandas import DataFrame
        df = DataFrame([
            dict(i=1, t='aa', x=0.5),
            dict(i=2, t='bb', x=0.5),
            dict(i=2, t='aa', x=0.5),
        ])

        sub = filter_df_options(df, {'i': 1, 't': 'aa'})
        print(sub)

        sub = filter_df_options(df, [None])
        print(sub)
    """
    if options is None:
        return df
    elif isinstance(options, list):
        if options[0] is None:
            return df
        else:
            raise RuntimeError(  # pragma: no cover
                "options must be dictionary or [None] not {}".format(options))
    for k, v in options.items():
        df = df[df[k] == v]
    return df

def options2label(opt, sep=" ", format_number="{:.3g}"):
    """
    Converts a list of options into a label.

    @param      opt             dictionary
    @param      sep             separator
    @param      format_number   how to format real numbers
    @return                     string

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import options2label
        res = options2label({'i': 1, 't': 'aa', 'x': 3.145667e10})
        print(res)
    """
    if opt is None:
        return "-"
    rows = []
    for k, v in sorted(opt.items()):
        if isinstance(v, (float, numpy.float64, numpy.float32)):
            v = format_number.format(v)
        rows.append("{}={}".format(k, v))
    return sep.join(rows)

def ax_position(shape, pos):
    """
    :epkg:`matplotlib` uses a one dimension
    array if the number of columns or rows is 1.
    This function makes a correction.

    @param      shape       graph shape
    @param      pos         graph current position
    @return                 corrected position

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import ax_position
        print(ax_position((2, 2), (0, 0)))
        print(ax_position((1, 2), (0, 0)))
    """
    res = []
    for a, b in zip(shape, pos):
        if a > 1:
            res.append(b)
    return tuple(res)

def plt_colors():
    """
    Returns :epkg:`matplotlib` colors.

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import plt_colors
        from pprint import pprint
        pprint(plt_colors())
    """
    import matplotlib.colors as mcolors  # pylint: disable=C0415
    colors = [k.split(':')[-1] for k in mcolors.TABLEAU_COLORS]
    for k in sorted(mcolors.CSS4_COLORS):
        colors.append(k)
    return colors

def plt_styles():
    """
    Returns :epkg:`matplotlib` styles.

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import plt_styles
        from pprint import pprint
        pprint(plt_styles())
    """
    return [('o', '-'), ('x', '-'), ('*', '-'), ('^', '-'),
            ('o', '--'), ('x', '--'), ('*', '--'), ('^', '--')]

def move_color_add(style):
    """
    Makes color lighter or darker
    based on a style.
    """
    return {'o': 0, 'x': 80, '*': -80, '^': 120}[style]

def move_color(color, add=2):
    """
    Returns a different colors, lighter or darker.

    @param      color       name of something starting with ``#``
    @param      add         what to add to each color,
                            positive to make it lighter
    @return                 lighter column
    """
    if not color.startswith("#"):
        import matplotlib.colors as mcolors  # pylint: disable=C0415
        color = mcolors.CSS4_COLORS[color]
    rgb = tuple(int(color[1 + i * 2: 3 + i * 2], base=16) for i in range(0, 3))
    if add > 0:
        rgb = tuple(min(255, i + add) for i in rgb)
    else:
        rgb = tuple(max(0, i + add) for i in rgb)
    return "#%02X%02X%02X" % rgb

def remove_common_prefix(labels, drop_rename=None):
    """
    Removes the common prefix of a series of labels.

    @param  labels          labels
    @param  drop_rename     skips those labels
    @return                 stripped labels

    .. runpython::
        :showcode:

        from pymlbenchmark.plotting.plot_helper import remove_common_prefix
        labels = ["x=a", "x=b"]
        print(remove_common_prefix(labels))
    """
    drop_rename = set(drop_rename) if drop_rename else []
    prefix = None
    for label in labels:
        if label in drop_rename:
            continue
        if not isinstance(label, str):
            if isinstance(label, float) and numpy.isnan(label):
                continue
            return labels
        if prefix is None:
            prefix = label
        elif not label.startswith(prefix):
            begin = min(len(prefix), len(label))
            for i in range(begin, -1, -1):
                prefix = prefix[:i]  # pylint: disable=E1136
                if label.startswith(prefix):
                    break
            if prefix == "":
                return labels
    last = len(prefix)
    res = [(label[last:] if (isinstance(label, str) and
                             label not in drop_rename) else label)
           for label in labels]
    return res

max_stars_repo_name: googol-lab/pymlbenchmark

max_issues_repo_name: googol-lab/pymlbenchmark

max_forks_repo_name: googol-lab/pymlbenchmark


text:
from utils import evaluate, create_input
from model import Model
from loader import augment_with_pretrained, load_sentences
import numpy as np
import itertools
import gensim, re
import json

class Predictor:

	def __init__(self):
		model_path = "final_model"
		self.model = Model(model_path=model_path)
		self.f = self.model.build(training=False, **self.model.parameters)	#needs a better name
		self.model.reload()

		self.model.parameters['pre_emb'] = 'vectors.bin'

		pretrained = gensim.models.word2vec.Word2Vec.load_word2vec_format(self.model.parameters['pre_emb'], binary=True)
		new_weights = self.model.components['word_layer'].embeddings.get_value()

		n_words = len(self.model.id_to_word)
		freq = json.load(open('/home/wenqiang/tagger-master/mydockerbuild2/mydockerbuild/freq', 'r'))
		words = [item[0] for item in freq]

		self.model.id_to_word = {}

		for i in xrange((n_words/2), n_words):
			word = words[i]
			if word in pretrained:
				self.model.id_to_word[i-640780] = word
				new_weights[i-640780] = pretrained[word]
		#        c_found += 1
			elif word.lower() in pretrained:
				self.model.id_to_word[i-640780] = word.lower()
				new_weights[i-640780] = pretrained[word.lower()]
		#        c_lower += 1
			elif re.sub('\d', '0', word.lower()) in pretrained:
				self.model.id_to_word[i-640780] = re.sub('\d', '0', word.lower())
				new_weights[i-640780] = pretrained[
					re.sub('\d', '0', word.lower())
				]
		 #       c_zeros += 1

		self.model.id_to_word[0] = '<UNK>'
		self.model.components['word_layer'].embeddings.set_value(new_weights)

		del pretrained
		del new_weights

		self.lower = self.model.parameters['lower']
		self.zeros = self.model.parameters['zeros']

		#Create new mapping because model.id_to_word only is an Ordered dict of only training and testing data

		self.word_to_id = {v:i for i,v in self.model.id_to_word.items()}
		self.char_to_id = {v:i for i,v in self.model.id_to_char.items()}

	def prepare_dataset(self, sentences):
		"""
		Prepare the dataset. Return a list of lists of dictionaries containing:
			- word indexes
			- word char indexes
			- tag indexes
		"""
		def f(x):
			#if zeros:
			return re.sub('\d', '0', x)
			#def f(x): return x.lower() if lower else x
		data = []
		for s in sentences:
			str_words = [w[0] for w in s]
			words = [self.word_to_id[f(w) if f(w) in self.word_to_id else '<UNK>']
					 for w in str_words]
			# Skip characters that are not in the training set
			#for num, word in enumerate(words):
			#        if word < 0:
			#               	words[num]=word_to_id['<UNK>']
			chars = [[self.char_to_id[c] for c in w if c in self.char_to_id]
					 for w in str_words]
			caps = [self.cap_feature(w) for w in str_words]
			data.append({
				'str_words': str_words,
				'words': words,
				'chars': chars,
				'caps': caps,
			})
		return data

	def cap_feature(self, s):
		"""
		Capitalization feature:
		0 = low caps
		1 = all caps
		2 = first letter caps
		3 = one capital (not first letter)
		"""
		if s.lower() == s:
			return 0
		elif s.upper() == s:
			return 1
		elif s[0].upper() == s[0]:
			return 2
		else:
			return 3

	#synchronized block: TO DO
	def parseString(self, string):
		#TO DO
		#To be consumed by web-service
		test_file = "test_file"
		file = open(test_file, 'w')
		file.write('\n'.join(string.encode('utf-8').split()))
		file.close()
		test_sentences = load_sentences(test_file, self.lower, self.zeros)
		data = self.prepare_dataset(test_sentences)
		result = ''
		for citation in data:
			input = create_input(citation, self.model.parameters, False)
			y_pred = np.array(self.f[1](*input))[1:-1]
			tags = []
			for i in xrange(len(y_pred)):
				tags.append(self.model.id_to_tag[y_pred[i]])
			for num, word in enumerate(string.encode('utf-8').split()):
				#print word.decode('utf-8')+'\t'+tags[num]
				result += word.decode('utf-8')+'\t'+tags[num]+'\n'
		return result

if __name__ == '__main__':
	p = Predictor()

	while True:
		string = raw_input("Enter the citation string: ").decode('utf-8')
		r = p.parseString(string)
		print(str(r))

max_stars_repo_name: SivamPillai/Neural-ParsCit

max_issues_repo_name: SivamPillai/Neural-ParsCit

max_forks_repo_name: SivamPillai/Neural-ParsCit


text:
#In[]:
# coding=utf-8
import os
os.chdir(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
import numpy as np
import Putil.base.logger as plog

plog.PutilLogConfig.config_handler(plog.stream_method)
plog.PutilLogConfig.config_format(plog.FormatRecommend)
plog.PutilLogConfig.config_log_level(stream=plog.INFO)
root_logger = plog.PutilLogConfig('test_coco').logger()
root_logger.setLevel(plog.DEBUG)
TestCocoLogger = root_logger.getChild('TestCoco')
TestCocoLogger.setLevel(plog.DEBUG)

import Putil.data.coco as COCO
import Putil.data.aug as pAug
import Putil.data.aug as pAug
from Putil.data.data_type_adapter import DataTypeAdapterNoOp as data_type_adapter
from Putil.data.convert_to_input import ConvertToInputNoOp as convert_to_input
import Putil.base.base_setting as pbs
pbs.deterministic_setting(42, torch=False, tf=False, numpy=False)

seed = 64
image_height = 512
image_width = 512
root_dir = '/data2/Public_Data/COCO/unzip_data/2017'

#In[]
dataset_evaluate = COCO.COCOData(root_dir, COCO.COCOData.Stage.Evaluate, './test/data/result/test_coco_detection_result', detection=True,
image_height=image_height, image_width=image_width)
root_node = pAug.AugNode(pAug.AugFuncNoOp())
root_node.freeze_node()
dataset_evaluate.set_aug_node_root(root_node)
class_amount = 80
dataset_evaluate.set_data_type_adapter(data_type_adapter())
dataset_evaluate.set_convert_to_input_method(convert_to_input())
TestCocoLogger.info('evaluate data amount: {0}'.format(len(dataset_evaluate)))

#In[]:
use_amount = 10
image_ids = list()
image_bbox = dict()
for i in range(0, use_amount):
    data = dataset_evaluate[i]
    image = data[0]
    base_informations = data[2]
    bboxes = (np.array(data[1]) * np.array([[base_informations[1] / image.shape[1], base_informations[0] / image.shape[0]] * 2])).tolist()
    classes = data[-1]
    category_ids = [dataset_evaluate.represent_to_cat_id[_class] for _class in classes]
    image_ids.append(base_informations[-1])
    image_bbox[base_informations[-1]] = data[1]
    dataset_evaluate.add_detection_result(image=image, image_id=base_informations[-1], \
        category_ids=category_ids, bboxes=bboxes, scores=[1.0 for bbox in bboxes], \
             save=True if i == use_amount - 1 else False, prefix='test')
    pass
#for i in range(10, len(dataset_evaluate)):
#    dataset_evaluate.add_detection_result(*get_detection_result(*dataset_evaluate[i]), save=True if i == len(dataset_evaluate) else False)
#    pass
#In[]
#print(dataset_evaluate._detection_result)
#len(dataset_evaluate)
dataset_evaluate.evaluate_detection(image_ids=image_ids, prefix='test', use_visual=True)

max_stars_repo_name: jihuacao/Putil

max_issues_repo_name: jihuacao/Putil

max_forks_repo_name: jihuacao/Putil


text:
# -*- coding: latin-1 -*-
# Copyright (c) 2008 Pycircuit Development Team
# See LICENSE for details.

from numpy.testing import assert_equal, assert_array_almost_equal, assert_array_equal
import unittest

from pycircuit.utilities.misc import *

def test_create_index_vectors():
    indices = [0, 1, 1, 2, 2, 2]
    src = N.array([1,2,3,4,5,6])

    refdest = N.zeros(3)
    inplace_add_selected_ref(refdest, indices, src)

    dest = N.zeros(3)

    for dst_i, src_i in create_index_vectors(indices):
        dest[dst_i] += src[src_i]

    assert_array_equal(refdest, dest)

def test_inplace_add_selected():
    src = N.array([1,2,3,9,-1,1])
    indices = [0, 1, 1, 2, 2, 2]

    indices_list = create_index_vectors(indices)

    a = N.zeros(3)
    inplace_add_selected(a, indices_list, src)

    aref = N.zeros(3)
    inplace_add_selected_ref(aref, indices, src)

    assert_array_equal(a, aref)

@unittest.skip("Skip failing test")
def test_ObserverSubject_init():
    a = ObserverSubject()
    assert_equal(a._observers,[])

@unittest.skip("Skip failing test")
def test_ObserverSubject_notify():

    class MySubject(ObserverSubject):
        def hitme(self):
            self.notify(args=10)

    class objectWithUpdateMethod(object):
        def __init__(self):
            self.reset()

        def reset(self):
            self.value = None
            self.value1 = None

        def update(self, subject, args):
            self.value = ('update', subject, args)

        def update1(self, subject, args):
            self.value1 = ('update1', subject, args)

    a = MySubject()
    b = objectWithUpdateMethod()

    a.attach(b)
    a.attach(b, 'update1')

    a.hitme()

    assert_equal(b.value, ('update', a, 10))
    assert_equal(b.value1, ('update1', a, 10))

    b.reset()

    # Test detach
    a.detach(b, 'update')
    a.hitme()

    assert_equal(b.value, None)
    assert_equal(b.value1, ('update1', a, 10))

    a.detach(b, 'update1')
    a.hitme()

    assert_equal(b.value, None)
    assert_equal(b.value1, ('update1', a, 10))

max_stars_repo_name: michaelnt/pycircuit

max_issues_repo_name: michaelnt/pycircuit

max_forks_repo_name: michaelnt/pycircuit

reason: from numpy


text:
from os import path
import subprocess
import anndata as ad
import scipy

## VIASH START
# This code block will be replaced by viash at runtime.
meta = { 'functionality_name': 'foo' }
## VIASH END

method_id = meta['functionality_name']
command = "./" + method_id

# define some filenames
testpar = {
    "input_mod1": "resources_test/joint_embedding/openproblems_bmmc_multiome_starter/openproblems_bmmc_multiome_starter.mod1.h5ad",
    "input_mod2": "resources_test/joint_embedding/openproblems_bmmc_multiome_starter/openproblems_bmmc_multiome_starter.mod2.h5ad",
    "output": "output.h5ad",
}

print("> Running method")
out = subprocess.check_output([
  command,
  "--input_mod1", testpar['input_mod1'],
  "--input_mod2", testpar['input_mod2'],
  "--output", testpar['output']
]).decode("utf-8")

print("> Checking whether output files were created")
assert path.exists(testpar['output'])

print("> Reading h5ad files")
input_mod1 = ad.read_h5ad(testpar['input_mod1'])
output = ad.read_h5ad(testpar['output'])

print("> Checking contents of output.h5ad")
assert output.uns['dataset_id'] == input_mod1.uns['dataset_id']
assert output.uns['method_id'] == method_id
assert output.n_obs == input_mod1.n_obs
assert output.n_vars >= 1
assert output.n_vars <= 100
assert all(output.obs_names == input_mod1.obs_names)
assert not scipy.sparse.issparse(output.X)

print("> Test succeeded!")

reason: import scipy


text:
### Dependencies are:
# numpy - for matrix manipulation
# sklearn - for implementing spectral clustering and standard PCA
# pandas - to check type of dataframes and convert to numpy arrays in method cpca_alpha
###

from __future__ import print_function
import numpy as np
from numpy import linalg as LA
from sklearn import cluster
from sklearn.decomposition import PCA
import pandas as pd

class CPCA(object):
    """
    Contrastive PCA (cPCA)

    Linear dimensionality reduction that uses eigenvalue decomposition
    to identify directions that have increased variance in the primary (foreground)
    dataset relative to a secondary (background) dataset. Then, those directions
    are used to project the data to a lower dimensional space.
    """

    # Getter methods for various attributes
    def get_data(self):
        return self.data
    def get_bg(self):
        return self.bg
    def get_fg(self):
        return self.fg
    def get_active_labels(self):
        return self.active_labels
    def get_pca_directions(self):
        return self.pca_directions
    def get_active_pca_projected(self):
        return self.active_pca
    def get_affinity_matrix(self):
        return self.affinity_matrix

    # A helper method to standardize arrays
    def standardize_array(self, array):
        standardized_array =  (array-np.mean(array,axis=0)) / np.std(array,axis=0)
        return np.nan_to_num(standardized_array)

    #stores
    def __init__(self, n_components=2, standardize=True, verbose=False):
        self.standardize = standardize
        self.n_components = n_components
        self.verbose = verbose
        self.fitted = False

        """
        Finds the covariance matrices of the foreground and background datasets,
        and then transforms the foreground dataset based on the principal contrastive components

        Parameters: see self.fit() and self.transform() for parameter description
        """
    def fit_transform(self, foreground, background, plot=False, gui=False, alpha_selection='auto', n_alphas=40,  max_log_alpha=3, n_alphas_to_return=4, active_labels = None, colors=None, legend=None, alpha_value=None, return_alphas=False):
        self.fit(foreground, background)
        return self.transform(dataset=self.fg, alpha_selection=alpha_selection,  n_alphas=n_alphas, max_log_alpha=max_log_alpha, n_alphas_to_return=n_alphas_to_return, plot=plot, gui=gui, active_labels=active_labels, colors=colors, legend=legend, alpha_value=alpha_value, return_alphas=return_alphas)

        """
        Computes the covariance matrices of the foreground and background datasets

        Parameters
        -----------
        foreground: array, shape (n_data_points, n_features)
            The dataset in which the interesting directions that we would like to discover are present or enriched

        background : array, shape (n_data_points, n_features)
            The dataset in which the interesting directions that we would like to discover are absent or unenriched

        preprocess_with_pca_dim: int
            If this parameter is provided (and it is greater than n_features), then both the foreground and background
            datasets undergo a preliminary round of PCA to reduce their dimension to this number. If it is not provided
            but n_features > 1,000, a preliminary round of PCA is automatically performed to reduce the dimensionality to 1,000.
        """

    def fit(self, foreground, background, preprocess_with_pca_dim=None):
        # Housekeeping
        self.pca_directions = None
        self.bg_eig_vals = None
        self.affinity_matrix = None

        # Datasets and dataset sizes
        self.fg = foreground
        self.bg = background
        self.n_fg, self.features_d = foreground.shape
        self.n_bg, self.features_d_bg = background.shape

        if not(self.features_d==self.features_d_bg):
            raise ValueError('The dimensionality of the foreground and background datasets must be the same')

        #if the dimension is higher than preprocess_with_pca_dim, then PCA is performed to reduce the dimensionality
        if preprocess_with_pca_dim is None:
            preprocess_with_pca_dim = 1000

        #Center the background and foreground data
        self.bg = self.bg - np.mean(self.bg, axis=0)
        if self.standardize: #Standardize if specified
            self.bg = self.standardize_array(self.bg)

        self.fg = self.fg - np.mean(self.fg, axis=0)
        if self.standardize: #Standardize if specified
            self.fg = self.standardize_array(self.fg)

        if (self.features_d>preprocess_with_pca_dim):
            data = np.concatenate((self.fg, self.bg),axis=0)
            pca = PCA(n_components = preprocess_with_pca_dim)
            data = pca.fit_transform(data)
            self.fg = data[:self.n_fg,:]
            self.bg = data[self.n_fg:,:]
            self.features_d = preprocess_with_pca_dim

            if (self.verbose):
                print("Data dimensionality reduced to "+str(preprocess_with_pca_dim)+". Percent variation retained: ~"+str(int(100*np.sum(pca.explained_variance_ratio_)))+'%')

        if (self.verbose):
            print("Data loaded and preprocessed")

        #Calculate the covariance matrices
        self.bg_cov = self.bg.T.dot(self.bg)/(self.bg.shape[0]-1)
        self.fg_cov = self.fg.T.dot(self.fg)/(self.n_fg-1)

        if (self.verbose):
            print("Covariance matrices computed")

        self.fitted = True

    def transform(self, dataset, alpha_selection='auto', n_alphas=40, max_log_alpha=3, n_alphas_to_return=4, plot=False, gui=False, active_labels = None, colors=None, legend=None, alpha_value=None, return_alphas=False):
        if (self.fitted==False):
            raise ValueError("This model has not been fit to a foreground/background dataset yet. Please run the fit() or fit_transform() functions first.")
        if not(alpha_selection=='auto' or alpha_selection=='manual' or alpha_selection=='all'):
            raise ValueError("Invalid argument for parameter alpha_selection: must be 'auto' or 'manual' or 'all'")
        if (alpha_selection=='all' and plot==True):
            raise ValueError('The plot parameter cannot be set to True if alpha_selection is set to "all"')
        if ((alpha_selection=='all' or alpha_selection=='manual') and gui==True):
            raise ValueError('The gui parameter cannot be set to True if alpha_selection is set to "all" or "manual"')
        if ((gui==True or plot==True) and not(self.n_components==2)):
            raise ValueError('The gui and plot parameters modes cannot be used if the number of components is not 2')
        if (not(alpha_value) and alpha_selection=='manual'):
            raise ValueError('The the alpha_selection parameter is set to "manual", the alpha_value parameter must be provided')
        #you can't be plot or gui with non-2 components
        # Handle the plotting variables
        if (plot or gui):
            if active_labels is None:
                active_labels = np.ones(dataset.shape[0])
            self.active_labels = active_labels
            if colors is None:
                 self.colors = ['k','r','b','g','c']

        if gui:
            try:
                import matplotlib.pyplot as plt
                from matplotlib.gridspec import GridSpec
            except ImportError:
                raise ImportError("Something wrong while loading matplotlib.pyplot! You probably don't have plotting libraries installed.")
            try:
                from ipywidgets import widgets, interact, Layout
                from IPython.display import display
            except ImportError:
                raise ImportError("To use the GUI, you must be running this code in a jupyter notebook that supports ipywidgets")

            transformed_data_auto, alphas_auto = self.automated_cpca(dataset, n_alphas_to_return, n_alphas, max_log_alpha)
            transformed_data_manual, alphas_manual = self.all_cpca(dataset, n_alphas, max_log_alpha)
            if (self.n_fg>1000):
                print("The GUI may be slow to respond with large numbers of data points. Consider using a subset of the original data.")

            """
            Handles the plotting
            """
            def graph_foreground(ax,fg, active_labels, alpha):
                for i, l in enumerate(np.sort(np.unique(active_labels))):
                    ax.scatter(fg[np.where(active_labels==l),0],fg[np.where(active_labels==l),1], color=self.colors[i%len(self.colors)], alpha=0.6)
                if (alpha==0):
                    ax.annotate(r'$\alpha$='+str(np.round(alpha,2))+" (PCA)", (0.05,0.05), xycoords='axes fraction')
                else:
                    ax.annotate(r'$\alpha$='+str(np.round(alpha,2)), (0.05,0.05), xycoords='axes fraction')

            """
            This code gets run whenever the widget slider is moved
            """
            def update(value):
                fig = plt.figure(figsize=[10,4])
                gs=GridSpec(2,4)

                for i in range(4):
                    ax1=fig.add_subplot(gs[int(i//2),i%2]) # First row, first column
                    fg = transformed_data_auto[i]
                    graph_foreground(ax1, fg, self.active_labels, alphas_auto[i])

                    ax5=fig.add_subplot(gs[:,2:]) # Second row, span all columns

                    alpha_idx = np.abs(alphas_manual-10**value).argmin()
                    fg = transformed_data_manual[alpha_idx]
                    graph_foreground(ax5, fg, self.active_labels, alphas_manual[alpha_idx])

                #if len(np.unique(self.active_labels))>1:
                    #plt.legend()

                plt.tight_layout()
                plt.show()

            widg = interact(update, value=widgets.FloatSlider(description=r'\(\log_{10}{\alpha} \)', min=-1, max=3, step=4/40, continuous_update=False, layout=Layout(width='80%')))

            return

        elif plot:
            try:
                import matplotlib.pyplot as plt
            except ImportError:
                raise ImportError("Something wrong while loading matplotlib.pyplot! You probably don't have plotting libraries installed.")
            if (alpha_selection=='auto'):
                transformed_data, best_alphas = self.automated_cpca(dataset, n_alphas_to_return, n_alphas, max_log_alpha)
                plt.figure(figsize=[14,3])
                for j, fg in enumerate(transformed_data):
                    plt.subplot(1,4,j+1)
                    for i, l in enumerate(np.sort(np.unique(self.active_labels))):
                        idx = np.where(self.active_labels==l)
                        plt.scatter(fg[idx,0],fg[idx,1], color=self.colors[i%len(self.colors)], alpha=0.6, label='Class '+str(i))
                    plt.title('Alpha='+str(np.round(best_alphas[j],2)))
                if len(np.unique(self.active_labels))>1:
                    plt.legend()
                plt.show()
            elif (alpha_selection=='manual'):
                transformed_data, best_alphas = self.automated_cpca(dataset, n_alphas_to_return, n_alphas, max_log_alpha)
                plt.figure(figsize=[14,3])
                for j, fg in enumerate(transformed_data):
                    plt.subplot(1,4,j+1)
                    for i, l in enumerate(np.sort(np.unique(self.active_labels))):
                        idx = np.where(self.active_labels==l)
                        plt.scatter(fg[idx,0],fg[idx,1], color=self.colors[i%len(self.colors)], alpha=0.6, label='Class '+str(i))
                    plt.title('Alpha='+str(np.round(best_alphas[j],2)))
                if len(np.unique(self.active_labels))>1:
                    plt.legend()
                plt.show()

            return

        else:
            if (alpha_selection=='auto'):
                transformed_data, best_alphas = self.automated_cpca(dataset, n_alphas_to_return, n_alphas, max_log_alpha)
                alpha_values = best_alphas
            elif (alpha_selection=='all'):
                transformed_data, all_alphas = self.all_cpca(dataset, n_alphas, max_log_alpha)
                alpha_values = all_alphas
            else:
                transformed_data = self.cpca_alpha(dataset, alpha_value)
                alpha_values = alpha_value
        if return_alphas:
            return transformed_data, alpha_values
        else:
            return transformed_data

    """
    This function performs contrastive PCA using the alpha technique on the
    active and background dataset. It automatically determines n_alphas=4 important values
    of alpha up to based to the power of 10^(max_log_alpha=5) on spectral clustering
    of the top subspaces identified by cPCA.
    The final return value is the data projected into the top (n_components = 2)
    subspaces, which can be plotted outside of this function
    """
    def automated_cpca(self, dataset, n_alphas_to_return, n_alphas, max_log_alpha):
        best_alphas, all_alphas, _, _ = self.find_spectral_alphas(n_alphas, max_log_alpha, n_alphas_to_return)
        best_alphas = np.concatenate(([0], best_alphas)) #one of the alphas is always alpha=0
        data_to_plot = []
        for alpha in best_alphas:
            transformed_dataset = self.cpca_alpha(dataset=dataset, alpha=alpha)
            data_to_plot.append(transformed_dataset)
        return data_to_plot, best_alphas

    """
    This function performs contrastive PCA using the alpha technique on the
    active and background dataset. It returns the cPCA-reduced data for all values of alpha specified,
    both the active and background, as well as the list of alphas
    """
    def all_cpca(self, dataset, n_alphas, max_log_alpha):
        alphas = np.concatenate(([0],np.logspace(-1,max_log_alpha,n_alphas)))
        data_to_plot = []
        for alpha in alphas:
            transformed_dataset = self.cpca_alpha(dataset=dataset, alpha=alpha)
            data_to_plot.append(transformed_dataset)
        return data_to_plot, alphas

    """
    Returns active and bg dataset projected in the cpca direction, as well as the top c_cpca eigenvalues indices.
    If specified, it returns the top_cpca directions
    """
    def cpca_alpha(self, dataset, alpha=1):
        n_components = self.n_components
        sigma = self.fg_cov - alpha*self.bg_cov
        w, v = LA.eig(sigma)
        eig_idx = np.argpartition(w, -n_components)[-n_components:]
        eig_idx = eig_idx[np.argsort(-w[eig_idx])]
        v_top = v[:,eig_idx]
        reduced_dataset = dataset.dot(v_top)

        if type(reduced_dataset) == type(pd.DataFrame()):
            reduced_dataset = reduced_dataset.to_numpy()

        reduced_dataset[:,0] = reduced_dataset[:,0]*np.sign(reduced_dataset[0,0])
        reduced_dataset[:,1] = reduced_dataset[:,1]*np.sign(reduced_dataset[0,1])
        return reduced_dataset

    """
    This method performs spectral clustering on the affinity matrix of subspaces
    returned by contrastive pca, and returns (`=3) exemplar values of alpha
    """
    def find_spectral_alphas(self, n_alphas, max_log_alpha, n_alphas_to_return):
        self.create_affinity_matrix(max_log_alpha, n_alphas)
        affinity = self.affinity_matrix
        spectral = cluster.SpectralClustering(n_clusters=n_alphas_to_return, affinity='precomputed')
        alphas = np.concatenate(([0],np.logspace(-1,max_log_alpha,n_alphas)))
        spectral.fit(affinity)
        labels = spectral.labels_
        best_alphas = list()
        for i in range(n_alphas_to_return):
            idx = np.where(labels==i)[0]
            if not(0 in idx): #because we don't want to include the cluster that includes alpha=0
                affinity_submatrix = affinity[idx][:, idx]
                sum_affinities = np.sum(affinity_submatrix, axis=0)
                exemplar_idx = idx[np.argmax(sum_affinities)]
                best_alphas.append(alphas[exemplar_idx])
        return np.sort(best_alphas), alphas, affinity[0,:], labels

    """
    This method creates the affinity matrix of subspaces returned by contrastive pca
    """
    def create_affinity_matrix(self, max_log_alpha, n_alphas):
        from math import pi
        alphas = np.concatenate(([0],np.logspace(-1,max_log_alpha,n_alphas)))
        subspaces = list()
        k = len(alphas)
        affinity = 0.5*np.identity(k) #it gets doubled
        for alpha in alphas:
            space = self.cpca_alpha(dataset=self.fg, alpha=alpha)
            q, r = np.linalg.qr(space)
            subspaces.append(q)
        for i in range(k):
            for j in range(i+1,k):
                q0 = subspaces[i]
                q1 = subspaces[j]
                u, s, v = np.linalg.svd(q0.T.dot(q1))
                affinity[i,j] = s[0]*s[1]
        affinity = affinity + affinity.T
        self.affinity_matrix = np.nan_to_num(affinity)

class Kernel_CPCA(CPCA):
    def __init__(self, n_components=2, standardize=True, verbose=False, kernel="linear", gamma=10):
        self.kernel=kernel
        self.gamma=gamma
        super().__init__(n_components, standardize, verbose)

    def fit_transform(self, foreground, background, plot=False, gui=False, alpha_selection='auto', n_alphas=40,  max_log_alpha=3, n_alphas_to_return=4, active_labels = None, colors=None, legend=None, alpha_value=None, return_alphas=False):
        self.fg = foreground
        self.bg = background
        self.n_fg, self.features_d = foreground.shape
        self.n_bg, self.features_d_bg = background.shape
        if (gui or plot):
            print("The parameters gui and plot cannot be set to True in Kernel PCA. Will return transformed data as an array instead")
        if not(alpha_selection=='manual'):
            print("The alpha parameter must be set manually for Kernel PCA. Will be using value of alpha = 2")
            alpha_value = 2
        return cpca_alpha(alpha_value)

    def fit(self, foreground, background, preprocess_with_pca_dim=None):
        raise ValueError("For Kernel CPCA, the fit() function is not defined. Please use the fit_transform() function directly")

    def transform(self, dataset, alpha_selection='auto', n_alphas=40, max_log_alpha=3, n_alphas_to_return=4, plot=False, gui=False, active_labels = None, colors=None, legend=None, alpha_value=None, return_alphas=False):
        raise ValueError("For Kernel CPCA, the transform() function is not defined. Please use the fit_transform() function directly")

    def cpca_alpha(self, alpha,degree=2,coef0=1):
        N=self.n_fg + self.n_bg
        Z=np.concatenate([self.fg,self.bg],axis=0)

        ## selecting the kernel and computing the kernel matrix
        if self.kernel=='linear':
            K=Z.dot(Z.T)
        elif method=='poly':
            K=(Z.dot(Z.T)+coef0)**degree
        elif method=='rbf':
            K=np.exp(-gamma*squareform(pdist(Z))**2)

        ## Centering the data
        K=centering(K,n)

        ## Using Kernel PCA to do the same
        K_til=np.zeros(K.shape)
        K_til[0:n,:]=K[0:n,:]/n
        K_til[n:,:]=-alpha*K[n:,:]/m
        Sig,A=np.linalg.eig(K_til)
        Sig=np.real(Sig)
        Sig[np.absolute(Sig)<1e-6]=0
        idx_nonzero=Sig!=0
        Sig=Sig[idx_nonzero]
        A=np.real(A[:,idx_nonzero])
        sort_idx=np.argsort(Sig)
        Sig=Sig[sort_idx]
        A=A[:,sort_idx]
        # Normalization
        A_norm=np.zeros(A.shape[1])
        for i in range(A.shape[1]):
            A_norm[i]=np.sqrt(A[:,i].dot(K).dot(A[:,i]).clip(min=0))
            A[:,i]/=A_norm[i]+1e-15

        Z_proj_kernel=K.dot(A[:,-2:])
        X_proj_kernel=Z_proj_kernel[0:n,:]
        Y_proj_kernel=Z_proj_kernel[n:,:]

        return X_proj_kernel#,Y_proj_kernel,Sig[-2:],A[:,-2:]

    ## ancillary functions
    def centering(K,n):
        m=K.shape[0]-n
        Kx=K[0:n,:][:,0:n]
        Ky=K[n:,:][:,n:]
        Kxy=K[0:n,:][:,n:]
        Kyx=K[n:,:][:,0:n]
        K_center=np.copy(K)
        K_center[0:n,:][:,0:n]=Kx - np.ones([n,n]).dot(Kx)/n - Kx.dot(np.ones([n,n]))/n\
        +np.ones([n,n]).dot(Kx).dot(np.ones([n,n]))/n/n
        K_center[n:,:][:,n:]=Ky - np.ones([m,m]).dot(Ky)/m - Ky.dot(np.ones([m,m]))/m\
        +np.ones([m,m]).dot(Ky).dot(np.ones([m,m]))/m/m
        K_center[0:n,:][:,n:]=Kxy - np.ones([n,n]).dot(Kxy)/n - Kxy.dot(np.ones([m,m]))/m\
        +np.ones([n,n]).dot(Kxy).dot(np.ones([m,m]))/n/m
        K_center[n:,:][:,0:n]=Kyx - np.ones([m,m]).dot(Kyx)/m - Kyx.dot(np.ones([n,n]))/n\
        +np.ones([m,m]).dot(Kyx).dot(np.ones([n,n]))/m/n
        return K_center

if __name__ == '__main__':
    N = 401; D = 1001; gap=3
    # In B, all the data pts are from the same distribution, which has different variances in three subspaces.
    B = np.zeros((N, D))
    B[:,0:10] = np.random.normal(0,10,(N,10))
    B[:,10:20] = np.random.normal(0,3,(N,10))
    B[:,20:30] = np.random.normal(0,1,(N,10))

    # In A there are four clusters.
    A = np.zeros((N, D))
    A[:,0:10] = np.random.normal(0,10,(N,10))
    # group 1
    A[0:100, 10:20] = np.random.normal(0,1,(100,10))
    A[0:100, 20:30] = np.random.normal(0,1,(100,10))
    # group 2
    A[100:200, 10:20] = np.random.normal(0,1,(100,10))
    A[100:200, 20:30] = np.random.normal(gap,1,(100,10))
    # group 3
    A[200:300, 10:20] = np.random.normal(2*gap,1,(100,10))
    A[200:300, 20:30] = np.random.normal(0,1,(100,10))
    # group 4
    A[300:400, 10:20] = np.random.normal(2*gap,1,(100,10))
    A[300:400, 20:30] = np.random.normal(gap,1,(100,10))
    A_labels = [0]*100+[1]*100+[2]*100+[3]*100

    cpca = CPCA(standardize=False)
    cpca.fit_transform(A, B, plot=True, active_labels=A_labels)

    print(A.shape)
    print(B.shape)

max_stars_repo_name: gabriel-santanna/contrastive

max_issues_repo_name: gabriel-santanna/contrastive

max_forks_repo_name: gabriel-santanna/contrastive

reason: import numpy,from numpy


text:
import math
import os
import random
from collections import deque

import numpy as np
import scipy.linalg as sp_la

import gym
import torch
import torch.nn as nn
import torch.nn.functional as F
from skimage.util.shape import view_as_windows
from torch import distributions as pyd

class eval_mode(object):
    def __init__(self, *models):
        self.models = models

    def __enter__(self):
        self.prev_states = []
        for model in self.models:
            self.prev_states.append(model.training)
            model.train(False)

    def __exit__(self, *args):
        for model, state in zip(self.models, self.prev_states):
            model.train(state)
        return False

def soft_update_params(net, target_net, tau):
    for param, target_param in zip(net.parameters(), target_net.parameters()):
        target_param.data.copy_(tau * param.data +
                                (1 - tau) * target_param.data)

def set_seed_everywhere(seed):
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)

def make_dir(*path_parts):
    dir_path = os.path.join(*path_parts)
    try:
        os.mkdir(dir_path)
    except OSError:
        pass
    return dir_path

def tie_weights(src, trg):
    assert type(src) == type(trg)
    trg.weight = src.weight
    trg.bias = src.bias

def weight_init(m):
    """Custom weight init for Conv2D and Linear layers."""
    if isinstance(m, nn.Linear):
        nn.init.orthogonal_(m.weight.data)
        if hasattr(m.bias, 'data'):
            m.bias.data.fill_(0.0)
    elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
        gain = nn.init.calculate_gain('relu')
        nn.init.orthogonal_(m.weight.data, gain)
        if hasattr(m.bias, 'data'):
            m.bias.data.fill_(0.0)

def mlp(input_dim, hidden_dim, output_dim, hidden_depth, output_mod=None):
    if hidden_depth == 0:
        mods = [nn.Linear(input_dim, output_dim)]
    else:
        mods = [nn.Linear(input_dim, hidden_dim), nn.ReLU(inplace=True)]
        for i in range(hidden_depth - 1):
            mods += [nn.Linear(hidden_dim, hidden_dim), nn.ReLU(inplace=True)]
        mods.append(nn.Linear(hidden_dim, output_dim))
    if output_mod is not None:
        mods.append(output_mod)
    trunk = nn.Sequential(*mods)
    return trunk

def to_np(t):
    if t is None:
        return None
    elif t.nelement() == 0:
        return np.array([])
    else:
        return t.cpu().detach().numpy()

class FrameStack(gym.Wrapper):
    def __init__(self, env, k):
        gym.Wrapper.__init__(self, env)
        self._k = k
        self._frames = deque([], maxlen=k)
        shp = env.observation_space.shape
        self.observation_space = gym.spaces.Box(
            low=0,
            high=1,
            shape=((shp[0] * k,) + shp[1:]),
            dtype=env.observation_space.dtype)
        self._max_episode_steps = env._max_episode_steps

    def reset(self):
        obs = self.env.reset()
        for _ in range(self._k):
            self._frames.append(obs)
        return self._get_obs()

    def step(self, action):
        obs, reward, done, info = self.env.step(action)
        self._frames.append(obs)
        return self._get_obs(), reward, done, info

    def _get_obs(self):
        assert len(self._frames) == self._k
        return np.concatenate(list(self._frames), axis=0)

class TanhTransform(pyd.transforms.Transform):
    domain = pyd.constraints.real
    codomain = pyd.constraints.interval(-1.0, 1.0)
    bijective = True
    sign = +1

    def __init__(self, cache_size=1):
        super().__init__(cache_size=cache_size)

    @staticmethod
    def atanh(x):
        return 0.5 * (x.log1p() - (-x).log1p())

    def __eq__(self, other):
        return isinstance(other, TanhTransform)

    def _call(self, x):
        return x.tanh()

    def _inverse(self, y):
        # We do not clamp to the boundary here as it may degrade the performance of certain algorithms.
        # one should use `cache_size=1` instead
        return self.atanh(y)

    def log_abs_det_jacobian(self, x, y):
        # We use a formula that is more numerically stable, see details in the following link
        # https://github.com/tensorflow/probability/commit/ef6bb176e0ebd1cf6e25c6b5cecdd2428c22963f#diff-e120f70e92e6741bca649f04fcd907b7
        return 2. * (math.log(2.) - x - F.softplus(-2. * x))

class SquashedNormal(pyd.transformed_distribution.TransformedDistribution):
    def __init__(self, loc, scale):
        self.loc = loc
        self.scale = scale

        self.base_dist = pyd.Normal(loc, scale)
        transforms = [TanhTransform()]
        super().__init__(self.base_dist, transforms)

    @property
    def mean(self):
        mu = self.loc
        for tr in self.transforms:
            mu = tr(mu)
        return mu
class RMS(object):
    """running mean and std """
    def __init__(self, device, epsilon=1e-4, shape=(1,)):
        self.M = torch.zeros(shape).to(device)
        self.S = torch.ones(shape).to(device)
        self.n = epsilon

    def __call__(self, x):
        bs = x.size(0)
        delta = torch.mean(x, dim=0) - self.M
        new_M = self.M + delta * bs / (self.n + bs)
        new_S = (self.S * self.n + torch.var(x, dim=0) * bs +
                 torch.square(delta) * self.n * bs /
                 (self.n + bs)) / (self.n + bs)

        self.M = new_M
        self.S = new_S
        self.n += bs

        return self.M, self.S

class PBE(object):
    """particle-based entropy based on knn normalized by running mean """
    def __init__(self, rms, knn_clip, knn_k, knn_avg, knn_rms, device):
        self.rms = rms
        self.knn_rms = knn_rms
        self.knn_k = knn_k
        self.knn_avg = knn_avg
        self.knn_clip = knn_clip
        self.device = device

    def __call__(self, rep):
        source = target = rep
        b1, b2 = source.size(0), target.size(0)
        # (b1, 1, c) - (1, b2, c) -> (b1, 1, c) - (1, b2, c) -> (b1, b2, c) -> (b1, b2)
        sim_matrix = torch.norm(source[:, None, :].view(b1, 1, -1) -
                                target[None, :, :].view(1, b2, -1),
                                dim=-1,
                                p=2)
        reward, _ = sim_matrix.topk(self.knn_k,
                                    dim=1,
                                    largest=False,
                                    sorted=True)  # (b1, k)
        if not self.knn_avg:  # only keep k-th nearest neighbor
            reward = reward[:, -1]
            reward = reward.reshape(-1, 1)  # (b1, 1)
            reward /= self.rms(reward)[0] if self.knn_rms else 1.0
            reward = torch.maximum(
                reward - self.knn_clip,
                torch.zeros_like(reward).to(self.device)
            ) if self.knn_clip >= 0.0 else reward  # (b1, 1)
        else:  # average over all k nearest neighbors
            reward = reward.reshape(-1, 1)  # (b1 * k, 1)
            reward /= self.rms(reward)[0] if self.knn_rms else 1.0
            reward = torch.maximum(
                reward - self.knn_clip,
                torch.zeros_like(reward).to(
                    self.device)) if self.knn_clip >= 0.0 else reward
            reward = reward.reshape((b1, self.knn_k))  # (b1, k)
            reward = reward.mean(dim=1, keepdim=True)  # (b1, 1)
        reward = torch.log(reward + 1.0)
        return reward

max_stars_repo_name: seolhokim/APT

max_issues_repo_name: seolhokim/APT

max_forks_repo_name: seolhokim/APT

reason: import numpy,import scipy


text:
import glob
import cv2
import numpy as np
import logging

# Input 3*224*224 Output 2 dimension one-hot label
class Preprocess():
    def __init__(self):
        self.data_folder = 'data/'
        self.data_dict = {
            'image' : [],
            'label' : []
        }

    def read_image(self):
        filenames = glob.glob(self.data_folder+'*.jpg')
        for filename in filenames:
            image = cv2.imread(filename, cv2.IMREAD_COLOR)
            image = cv2.resize(image, (224,224))
            self.data_dict['image'].append(np.array(image))
            label = filename.split('\\')[-1].split('_')[0]
            if label == 'sportscar':
                label = 0
            elif label == 'SUV':
                label = 1
            else:
                raise Exception
            self.data_dict['label'].append(label)
        self.data_dict['image'] = np.array(self.data_dict['image']).swapaxes(1, 3)
        self.data_dict['label'] = np.array(self.data_dict['label'])
        assert len(self.data_dict['image']) == len(self.data_dict['label'] )
        return self.data_dict

if __name__ == '__main__':
    pass
    #usage
    #preprocess = Preprocess()
    # data_dict = preprocess.read_image


text:
"""Module containing utilities."""
import math
from pathlib import Path

import click
import numpy as np
import rasterio
import yaml

PATH_TO_CONFIGS = Path(__file__).parent / '..' / 'config'

class Config(click.ParamType):
    """A configuration parameter on the command line.

    Configurations will always be read from the config directory.
    """
    name = "configuration"

    def convert(self, value, param, ctx):
        name_of_file = Path(value).name
        path_to_file = PATH_TO_CONFIGS / name_of_file
        return read_config(path_to_file)

def read_config(path_to_file):
    """Reads a configuration file."""
    path_to_file = Path(path_to_file)
    if not path_to_file.exists():
        raise ValueError("Config {} does not exist.".format(path_to_file))
    with path_to_file.open('r') as stream:
        try:
            return yaml.safe_load(stream)
        except yaml.YAMLError as exc:
            raise IOError(exc)

def determine_pixel_areas(crs, bounds, resolution):
    """Returns a raster in which the value corresponds to the area [km2] of the pixel.

    This assumes the data comprises square pixel in WGS84.

    Parameters:
        crs: the coordinate reference system of the data (must be WGS84)
        bounds: an object with attributes left/right/top/bottom given in degrees
        resolution: the scalar resolution (remember: square pixels) given in degrees
    """
    # the following is based on https://gis.stackexchange.com/a/288034/77760
    # and assumes the data to be in WGS84
    assert crs == rasterio.crs.CRS.from_epsg("4326") # WGS84
    width = int((bounds.right - bounds.left) / resolution)
    height = int((bounds.top - bounds.bottom) / resolution)
    latitudes = np.linspace(
        start=bounds.top,
        stop=bounds.bottom,
        num=height,
        endpoint=True,
        dtype=np.float64
    )
    varea_of_pixel = np.vectorize(lambda lat: _area_of_pixel(resolution, lat))
    pixel_area = varea_of_pixel(latitudes) # vector
    return pixel_area.repeat(width).reshape(height, width).astype(np.float64)

def _area_of_pixel(pixel_size, center_lat):
    """Calculate km^2 area of a wgs84 square pixel.

    Adapted from: https://gis.stackexchange.com/a/127327/2397

    Parameters:
        pixel_size (float): length of side of pixel in degrees.
        center_lat (float): latitude of the center of the pixel. Note this
            value +/- half the `pixel-size` must not exceed 90/-90 degrees
            latitude or an invalid area will be calculated.

    Returns:
        Area of square pixel of side length `pixel_size` centered at
        `center_lat` in km^2.

    """
    a = 6378137  # meters
    b = 6356752.3142  # meters
    e = math.sqrt(1 - (b / a)**2)
    area_list = []
    for f in [center_lat + pixel_size / 2, center_lat - pixel_size / 2]:
        zm = 1 - e * math.sin(math.radians(f))
        zp = 1 + e * math.sin(math.radians(f))
        area_list.append(
            math.pi * b**2 * (
                math.log(zp / zm) / (2 * e) +
                math.sin(math.radians(f)) / (zp * zm)))
    return pixel_size / 360. * (area_list[0] - area_list[1]) / 1e6

max_issues_repo_name: timtroendle/out-of-sight

max_forks_repo_name: timtroendle/out-of-sight


text:
import glob
import os
import pickle
import time

import librosa
import numpy as np
import pandas as pd

# =============================================================================
# CONFIG
# =============================================================================

# directory paths
dir_a_directory = "FILL\\ME\\IN\\"
dir_b_directory = "FILL\\ME\\IN\\"
output_directory = "FILL\\ME\\IN\\"

# hyperparameters and options
duration_tolerance_seconds = 0.5
max_chroma_len = 2**12
output_all = False  # if False, performs greedy matching; if True, outputs all similarity scores

# =============================================================================
# PART 0: GET FILE PATHS AND NAMES
# =============================================================================

# get filepath lists
dir_a_filepaths = glob.glob(dir_a_directory + "*")
dir_b_filepaths = glob.glob(dir_b_directory + "*")

# get filename lists
dir_a_filenames = [os.path.splitext(os.path.split(filepath)[1])[0] for filepath in dir_a_filepaths]
dir_b_filenames = [os.path.splitext(os.path.split(filepath)[1])[0] for filepath in dir_b_filepaths]

print(f"[INFO] Matching {len(dir_a_filepaths)} files in directory A with {len(dir_b_filepaths)} files in directory B")

# =============================================================================
# PART 1: GET DURATIONS AND CHROMAGRAMS
# =============================================================================

print(f"[INFO] Loading and processing audio files - this will take some time")
# get chromagrams and durations for files in dir a
dir_a_chromas = []
dir_a_durations = []
for filepath in dir_a_filepaths:
    y, sr = librosa.load(filepath, mono=True, sr=22050)
    dir_a_chromas.append(librosa.feature.chroma_cens(y=y, sr=sr))
    dir_a_durations.append(librosa.get_duration(y=y, sr=sr))
dir_a_durations_np = np.array(dir_a_durations)
# get chromagrams and durations for files in dir b
dir_b_chromas = []
dir_b_durations = []
for filepath in dir_b_filepaths:
    y, sr = librosa.load(filepath, mono=True, sr=22050)
    dir_b_chromas.append(librosa.feature.chroma_cens(y=y, sr=sr))
    dir_b_durations.append(librosa.get_duration(y=y, sr=sr))
dir_b_durations_np = np.array(dir_b_durations)
# save to file
with open(output_directory + "dir_a_chromas.pickle", "wb") as writefile:
    pickle.dump(dir_a_chromas, writefile)
with open(output_directory + "dir_b_chromas.pickle", "wb") as writefile:
    pickle.dump(dir_b_chromas, writefile)
np.save(output_directory + "dir_a_durations_np.npy", dir_a_durations_np)
np.save(output_directory + "dir_b_durations_np.npy", dir_b_durations_np)

# print("[INFO] Loading previously-computed chromagrams and durations")
# # load chromagrams and durations from file
# with open(output_directory + "dir_a_chromas.pickle", "rb") as readfile:
#     dir_a_chromas = pickle.load(readfile)
# with open(output_directory + "dir_b_chromas.pickle", "rb") as readfile:
#     dir_b_chromas = pickle.load(readfile)
# dir_a_durations_np = np.load(output_directory + "dir_a_durations_np.npy")
# dir_b_durations_np = np.load(output_directory + "dir_b_durations_np.npy")

# =============================================================================
# PART 2: MATCH FILES BASED ON DURATION
# =============================================================================

print("[INFO] Matching based on duration")
duration_diffs_np = np.abs(dir_a_durations_np.reshape((-1, 1)) - dir_b_durations_np.reshape((1, -1)))
duration_matched_indices = list(zip(*np.where(duration_diffs_np <= duration_tolerance_seconds)))

# =============================================================================
# PART 3: COMPUTE SIMILARITY USING DYNAMIC TIME WARPING
# =============================================================================

print("[INFO] Computing similarity using dynamic time warping - this will take some time")
# do dynamic time warp for each pair of duration-matched files
dtw_costs_np = np.full(shape=(len(dir_a_filepaths), len(dir_b_filepaths)), fill_value=np.inf, dtype=float)
start_time = time.time()
for idx_i, (idx_a, idx_b) in enumerate(duration_matched_indices):
    chroma_a = dir_a_chromas[idx_a]
    chroma_b = dir_b_chromas[idx_b]
    chroma_a_subseq_lbound = max(0, int((chroma_a.shape[1] - max_chroma_len) / 2))
    chroma_a_subseq_ubound = chroma_a_subseq_lbound + max_chroma_len
    chroma_b_subseq_lbound = max(0, int((chroma_b.shape[1] - max_chroma_len) / 2))
    chroma_b_subseq_ubound = chroma_b_subseq_lbound + max_chroma_len
    cost_matrix, warping_path = librosa.sequence.dtw(
        chroma_a[:, chroma_a_subseq_lbound:chroma_a_subseq_ubound],
        chroma_b[:, chroma_b_subseq_lbound:chroma_b_subseq_ubound],
        subseq=True,
        backtrack=True)
    dtw_costs_np[idx_a, idx_b] = np.min(cost_matrix[-1, :] / warping_path.shape[0])
    minutes_remaining = ((time.time() - start_time) * (len(duration_matched_indices) - idx_i - 1) / (idx_i + 1)) / 60
    # if (idx_i + 1) % 40 == 0:
    #     print(f"[INFO] Estimated remaining time: {minutes_remaining}")
np.save(output_directory + "dtw_costs_np.npy", dtw_costs_np)

# # load dynamic time warp similarities from file
# dtw_costs_np = np.load(output_directory + "dtw_costs_np.npy")

# =============================================================================
# PART 4: GREEDILY MATCH FILES
# =============================================================================

print("[INFO] Performing greedy matching of files")
duration_matched_indices_np = np.array(duration_matched_indices)
duration_matched_costs_np = dtw_costs_np[duration_matched_indices_np[:, 0], duration_matched_indices_np[:, 1]]
duration_matched_indices_sorted_np = duration_matched_indices_np[np.argsort(duration_matched_costs_np)]
matched_dir_a_indices = []
matched_dir_b_indices = []
for idx_a, idx_b in duration_matched_indices_sorted_np:
    if (idx_a not in matched_dir_a_indices) and (idx_b not in matched_dir_b_indices):
        matched_dir_a_indices.append(idx_a)
        matched_dir_b_indices.append(idx_b)

# =============================================================================
# PART 4: SAVE TO CSV
# =============================================================================

print("[INFO] Saving results to CSV")

# output matches to csv
matched_dir_a_filepaths = []
matched_dir_b_filepaths = []
matched_dir_a_filenames = []
matched_dir_b_filenames = []
matched_diffs = []
for idx_a, idx_b in zip(matched_dir_a_indices, matched_dir_b_indices):
    matched_dir_a_filepaths.append(dir_a_filepaths[idx_a])
    matched_dir_b_filepaths.append(dir_b_filepaths[idx_b])
    matched_dir_a_filenames.append(dir_a_filenames[idx_a])
    matched_dir_b_filenames.append(dir_b_filenames[idx_b])
    matched_diffs.append(dtw_costs_np[idx_a, idx_b])
output_df = pd.DataFrame({
    "matched_dir_a_filepaths": matched_dir_a_filepaths,
    "matched_dir_b_filepaths": matched_dir_b_filepaths,
    "matched_dir_a_filenames": matched_dir_a_filenames,
    "matched_dir_b_filenames": matched_dir_b_filenames,
    "matched_diffs": matched_diffs})
output_df = output_df.sort_values(["matched_diffs"])
output_df.to_csv(output_directory + "matches.csv", index=False)

max_stars_repo_name: morgannewellsun/Music-Matcher

max_issues_repo_name: morgannewellsun/Music-Matcher

max_forks_repo_name: morgannewellsun/Music-Matcher


text:
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import numpy as np

d = 64                           # dimension
nb = 100000                      # database size
nq = 10000                       # nb of queries
np.random.seed(1234)             # make reproducible
xb = np.random.random((nb, d)).astype('float32')
xb[:, 0] += np.arange(nb) / 1000.
xq = np.random.random((nq, d)).astype('float32')
xq[:, 0] += np.arange(nq) / 1000.

import faiss                     # make faiss available

ngpus = faiss.get_num_gpus()

print("number of GPUs:", ngpus)

cpu_index = faiss.IndexFlatL2(d)

gpu_index = faiss.index_cpu_to_all_gpus(  # build the index
    cpu_index
)

gpu_index.add(xb)              # add vectors to the index
print(gpu_index.ntotal)

k = 4                          # we want to see 4 nearest neighbors
D, I = gpu_index.search(xq, k) # actual search
print(I[:5])                   # neighbors of the 5 first queries
print(I[-5:])                  # neighbors of the 5 last queries


text:
import numpy as np
import ot
from Utils.load_data import load_candidate_finger, inchi_to_graph
from Utils.metabolites_utils import gaussian_tani_kernel
from time import time

class IOKREstimator:

    def __init__(self):
        self.n = None
        self.M = None
        self.K = None
        self.k = None
        self.Y = None
        self.Features = None
        self.alpha = 0.5
        self.tau = None
        self.g = None

    def train(self, K, Y, L, g):

        self.n = K.shape[0]
        self.M = np.linalg.inv(K + self.n * L * np.eye(self.n))
        self.K = K
        self.Y = Y
        self.g = g

    def train_candidate_kernel(self, F_c, F_tr):

        K_tr_c = gaussian_tani_kernel(F_tr.toarray(), F_c.toarray(), g=self.g)

        return K_tr_c

    def predict(self, K_tr_te, Y_te, n_c_max=200):

        n_te = K_tr_te.shape[1]
        A = self.M.dot(K_tr_te)

        # Compute n_te barycenters with n_bary points max for each barycenter
        Y_pred = []
        mean_topk = np.array([0., 0., 0.])
        mean_fgw = np.array([0., 0.])
        n_pred = 0
        error_type = []

        for i in range(n_te):

            # predict weights and take greatest weight graphs
            lambdas = A[:, i]
            idxs = np.argsort(lambdas)
            lambdas = [A[j, i] for j in idxs]

            # load candidate
            try:
                In, Fingers = load_candidate_finger(Y_te[3][i])
            except FileNotFoundError:
                print('FILE NOT FOUND')
                error_type.append('filenotfound')
                continue

            if In == -1:
                error_type.append('In')
                continue

            # compute score
            t0 = time()
            n_c = len(In)
            if n_c > n_c_max:
                error_type.append('too big')
                continue
            if n_c < 1:
                error_type.append('too small')
                continue

            Fingers_selected_train = self.Y[0][idxs]
            K_tr_c = self.train_candidate_kernel(Fingers, Fingers_selected_train)
            score = K_tr_c.T.dot(lambdas)

            print(f'{i}: Computation time gaussian-tani kernel n_c x n_tr = {n_c} x {len(lambdas)}: {time() - t0} s')

            # predict
            idx_sorted = np.argsort(score)
            Inc_sorted = [In[idx] for idx in idx_sorted]

            y_pred = inchi_to_graph(Inc_sorted[-1])
            G_pred = [y_pred[0], y_pred[1][:, :13]]
            G_te = [Y_te[0][i], Y_te[1][i][:, :13]]

            # Compute scores

            # compute fgw and gw
            fgw = self.fgw_distance(G_pred, G_te)
            gw = self.gw_distance(G_pred, G_te)
            print(f'{i}: FGW: {(fgw) / (1 - self.alpha)} / {gw}')
            # print(f'{i}: FGW: {(fgw - self.alpha * gw) / (1 - self.alpha)} / {gw}')
            mean_fgw += np.array([(fgw) / (1 - self.alpha), gw])

            # compute top-k
            inchi_true = Y_te[2][i]
            topk = [inchi_true in Inc_sorted[-1:], inchi_true in Inc_sorted[-10:], inchi_true in Inc_sorted[-20:]]
            topk = np.array(topk).astype(int)
            mean_topk += topk
            n_pred += 1
            print(f'{i}: n_candidate x n_tr = {len(Inc_sorted)} x {len(lambdas)},'
                  f' mean topk = {mean_topk / n_pred * 100}', flush=True)

        mean_fgw = mean_fgw / n_pred
        mean_topk = mean_topk / n_pred * 100
        print(f'n prediction: {n_pred}', flush=True)

        print(error_type)

        return mean_fgw, mean_topk, n_pred

    def post_process(self, C_bary, F_bary, N_edges):

        # most probable atom predicted
        F = np.zeros(F_bary.shape)
        for i in range(F_bary.shape[0]):
            u = np.zeros(F_bary.shape[1])
            a = np.argmax(F_bary[i])
            u[a] = 1
            F[i] = u

        # keep N largest edges
        ind = np.unravel_index(np.argsort(C_bary, axis=None), C_bary.shape)
        ind_max = (ind[0][-2 * N_edges:], ind[1][-2 * N_edges:])
        C = np.zeros(C_bary.shape)
        C[ind_max] = 1

        return C, F

    def fgw_distance(self, G1, G2):

        n1 = len(G1[1])
        n2 = len(G2[1])
        p1 = ot.unif(n1)
        p2 = ot.unif(n2)
        loss_fun = 'square_loss'
        Y_norms = np.linalg.norm(G1[1], axis=1) ** 2
        Z_norms = np.linalg.norm(G2[1], axis=1) ** 2
        scalar_products = G1[1].dot(G2[1].T)
        M = Y_norms.reshape(-1, 1) + Z_norms.reshape(1, -1) - 2 * scalar_products
        d = ot.gromov.fused_gromov_wasserstein2(M, G1[0], G2[0], p1, p2,
                                                loss_fun=loss_fun, alpha=self.alpha)

        return d

    def gw_distance(self, G1, G2):

        n1 = len(G1[1])
        n2 = len(G2[1])
        p1 = ot.unif(n1)
        p2 = ot.unif(n2)
        loss_fun = 'square_loss'
        d = ot.gromov.gromov_wasserstein2(G1[0], G2[0], p1, p2, loss_fun=loss_fun)

        return d


text:
# 2019-11-18 23:58:23(JST)
import collections
import sys

# import math
# from string import ascii_lowercase, ascii_uppercase, digits
# from bisect import bisect_left as bi_l, bisect_right as bi_r
# import itertools
# from functools import reduce
# import operator as op
# import re
# import heapq
# import array
# from scipy.misc import comb # (default: exact=False)
# import numpy as np

def main():
    n, *l = [int(x) for x in sys.stdin.read().split()]
    l.sort()

    c = collections.defaultdict(int)
    for i in range(n):
        c[l[i]] += 1

    # cumsum: count of {l[j]|l[j] <= l[i]}
    s = [0 for _ in range(l[-3] + l[-2] + 1)] # a+bの最大値以下まで
    for i in range(l[-3] + l[-2]):
        s[i+1] = s[i] + c[i+1]

    count = 0
    # a <= b <= c
    for i in range(n-2):
        a = l[i]
        for j in range(i+1, n-1):
            b = l[j]
            # cの候補の数: a+b > cを満たし、c = l[k]となるkがjより大きい棒の数
            count += s[a+b-1] - (j + 1) # b は j+1番目

    print(count)

if __name__ == "__main__":
    main()

max_stars_repo_name: kagemeka/atcoder-submissions

max_issues_repo_name: kagemeka/atcoder-submissions

max_forks_repo_name: kagemeka/atcoder-submissions

reason: import numpy,from scipy