text:
import pandas as pd
import numpy as np

# 不可分割子系统的程序

Adj_Matrix = pd.read_csv('Adj.csv', header=0, index_col=0)

# 删掉没有输入物流的节点
def DeleteColumns(Matrix, DNE_in_label):

    # 参数预定义
    Delete_Labels = []
    ColumnLabel = Matrix.columns.values
    NoColumns = Matrix.shape[1]
    Matrix_1 = Matrix

    for i in range(NoColumns):
        # print(i)
        sum_of_ith_column = Matrix_1.iloc[:, i].sum()
        if sum_of_ith_column == 0:
            Delete_Labels.append(ColumnLabel[i])
            DNE_in_label.append(ColumnLabel[i])

    if Delete_Labels:
        for j in range(len(Delete_Labels)):
            # 删掉行
            Matrix_1 = Matrix_1.drop(Delete_Labels[j])
            # 删掉列
            Matrix_1 = Matrix_1.drop(columns=Delete_Labels[j])

    # 求列和
    column_sum_Matrix1 = Matrix_1.apply(lambda x: x.sum())
    repeat = 0
    for h in range(column_sum_Matrix1.shape[0]):
        if column_sum_Matrix1[h] == 0:
            repeat = 1
        else:
            repeat = repeat

    return Matrix_1, DNE_in_label, repeat

# 删掉没有输出物流的节点
def DeleteRows(Matrix, DNE_out_label):

    # 参数预定义
    Delete_Labels = []
    RowLabel = Matrix._stat_axis.values
    NoRows = Matrix.shape[0]
    Matrix_1 = Matrix

    for i in range(NoRows):
        # print(i)
        sum_of_ith_column = Matrix_1.iloc[i, :].sum()
        if sum_of_ith_column == 0:
            Delete_Labels.append(RowLabel[i])
            DNE_out_label.append(RowLabel[i])

    if Delete_Labels:
        for j in range(len(Delete_Labels)):
            # 删掉行
            Matrix_1 = Matrix_1.drop(Delete_Labels[j])
            # 删掉列
            Matrix_1 = Matrix_1.drop(columns=Delete_Labels[j])

    # 求列和
    row_sum_Matrix1 = Matrix_1.apply(lambda x: x.sum(), axis=1)
    repeat = 0
    for h in range(row_sum_Matrix1.shape[0]):
        if row_sum_Matrix1[h] == 0:
            repeat = 1
        else:
            repeat = repeat

    return Matrix_1, DNE_out_label, repeat

# 可及矩阵的操作
def Reach_Matrix_Multi(Matrix):
    Mul_index = Matrix.shape[0]
    Matrix_np = np.array(Matrix)
    columns_label = Matrix.columns.values
    row_label = Matrix._stat_axis.values

    Matrix_np_temp1 = Matrix_np
    Matrix_np_temp2 = Matrix_np

    for i in range(2, Mul_index):
        Matrix_np_temp1 = np.matmul(Matrix_np_temp1, Matrix_np)
        Matrix_np_temp2 = Matrix_np_temp2 + Matrix_np_temp1

    Matrix_np_R = Matrix_np_temp2
    Matrix_np_R_array = np.int64(Matrix_np_R > 0)

    Matrix_np_R_T = Matrix_np_R.transpose(1, 0)
    Matrix_np_R_T_array = np.int64(Matrix_np_R_T > 0)

    Matrix_np_R_sum = Matrix_np_R_array * Matrix_np_R_T_array
    Matrix_np_R_sum_array = np.int64(Matrix_np_R_sum > 0)

    Matrix_np_R_sum_pd = pd.DataFrame(Matrix_np_R_sum_array, index=row_label, columns=columns_label)

    return Matrix_np_R_sum_pd

# 可及矩阵的切割
def FinalCut(Matrix):
    together_list = []
    delete_list = []
    column_label = Matrix.columns.values
    row_label = Matrix._stat_axis.values
    Matrix_size = Matrix.shape[0]
    Matrix_delete = Matrix
    # 按行遍历
    for i in range(Matrix_size):
        together_list.append(row_label[i])
        delete_list.append(row_label[i])
        # 按列遍历
        for j in range(Matrix_size):
            #print(Matrix)

            #print(Matrix.iat[i, j])
            if Matrix.iat[i, j] > 0 and i != j:
                together_list.append(column_label[j])
                delete_list.append(column_label[j])
        if j == Matrix_size-1:
            break

    for i in range(len(delete_list)):
        # 删掉行
        Matrix_delete = Matrix_delete.drop(delete_list[i])
        # 删掉列
        Matrix_delete = Matrix_delete.drop(columns=delete_list[i])

    repeat = 0
    # print(Matrix_delete)

    if Matrix_delete.shape[0] > 0:
        repeat = 1

    return Matrix_delete, together_list, repeat

# 删掉没有进去的
Label = Adj_Matrix.columns.values
Adj_Matrix_Step1 = Adj_Matrix

DNE_in_label = []
repeat = 0
s = Adj_Matrix_Step1.apply(lambda x: x.sum())
for i in range(s.shape[0]):
    if s[i] == 0:
        repeat = 1
    else:
        repeat = repeat
while repeat == 1:
    Adj_Matrix_Step1, DNE_in_label, repeat = DeleteColumns(Matrix=Adj_Matrix_Step1,
                                                           DNE_in_label=DNE_in_label)

# 删掉没有出来的
Adj_Matrix_Step2 = Adj_Matrix_Step1
DNE_out_label = []
repeat = 0
s = Adj_Matrix_Step1.apply(lambda x: x.sum(), axis=1)
for i in range(s.shape[0]):
    if s[i] == 0:
        repeat = 1
    else:
        repeat = repeat
while repeat == 1:
    Adj_Matrix_Step2, DNE_out_label, repeat = DeleteRows(Matrix=Adj_Matrix_Step2,
                                                         DNE_out_label=DNE_out_label)

# 打印结果
print('以下节点为没有输入物流的节点,分别为不可分割子系统:')
print(DNE_in_label)
print('以下节点为没有输出物流的节点,分别为不可分割子系统:')
print(DNE_out_label)

# 进行最后的系统切割
Adj_Matrix_Step3 = Reach_Matrix_Multi(Adj_Matrix_Step2)

repeat = 0

Adj_Matrix_Step4 = Adj_Matrix_Step3
repeat = 1
while repeat == 1:
    Adj_Matrix_Step4, system_list, repeat = FinalCut(Matrix=Adj_Matrix_Step4)
    print('以下节点为存在回路连接的节点,总体属于不可分割子系统:')
    print(system_list)


text:
import torch
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

import seaborn as sns
import sys

import matplotlib

font = {"size": 30}
matplotlib.rc("font", **font)
matplotlib.rcParams["text.usetex"] = True

results_df = pd.read_csv(
    "./out/st_alignment_synthetic_warp_mulitple_types.csv", index_col=0
)
# results_df =
results_df.value = np.log(results_df.value)

plt.figure(figsize=(10, 5))
sns.boxplot(data=results_df, x="Warp type", y="value", hue="variable")
plt.ylabel("log(error)")
plt.legend(loc="center left", bbox_to_anchor=(1, 0.5))
plt.tight_layout()
plt.savefig("./out/st_alignment_synthetic_warp_mulitple_types.png")
plt.show()
plt.close()

import ipdb

ipdb.set_trace()

ext: py

max_stars_repo_name: andrewcharlesjones/spatial-alignment

max_issues_repo_name: andrewcharlesjones/spatial-alignment

max_forks_repo_name: andrewcharlesjones/spatial-alignment

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Aug  5 21:18:51 2021

@author: Wojtek
"""
import os
import re
import warnings

from matplotlib.cbook.deprecation import MatplotlibDeprecationWarning
warnings.filterwarnings("ignore", category=MatplotlibDeprecationWarning)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from scipy.optimize import curve_fit

def process_name(filename):
    # date = filename[:10]
    date, sample_nr, power = re.split('Concentration|Power|Sample',filename[:-4])
    return date, sample_nr, power

class Exp:
    fun = lambda t, A, T, B: A*np.exp(-t/T)+B

    def get_p0(t,y):
        B = y.min()
        A = y.max()-y.min()
        T = (t.max()-t.min())/3
        return A, T, B

class Exp_rise:
    fun = lambda t, A, T, B: A*(1-np.exp(-t/T))+B

    def get_p0(t,y):
        B = y.min()
        A = y.max()-y.min()
        T = (t.max()-t.min())/3
        return A, T, B

class Exp2:
    fun = lambda t, A1, T1, A2, T2, B: A1*np.exp(-t/T1)+ A2*np.exp(-t/T2)+B

    def get_p0(t,y):
        B = y.min()
        A1 = (y.max()-y.min())/2
        A2 = A1
        T1 = (t.max()-t.min())/3
        T2 = T1*2
        return A1, T1, A2, T2, B

class Exp2_rise:
    fun = lambda t, A1, T1, A2, T2, B: A1*(1-np.exp(-t/T1))+ A2*(1-np.exp(-t/T2))+B

    def get_p0(t,y):
        B = y.min()
        A1 = (y.max()-y.min())/2
        A2 = A1
        T1 = (t.max()-t.min())/3
        T2 = T1*2
        return A1, T1, A2, T2, B

class ExpS:
    fun = lambda t, A, B, C, D: A*np.exp(-t**C/B) + D

    def get_p0(t,y):
        D = y.min()
        A = y.max()-y.min()
        C = 1
        B = (t.max()-t.min())/10
        return A, B, C, D

class ExpS_rise:
    fun = lambda t, A, B, C, D: A*(1-np.exp(-t**C/B)) + D

    def get_p0(t,y):
        D = y.min()
        A = y.max()-y.min()
        C = 1
        B = (t.max()-t.min())/10
        return A, B, C, D

def process_data(t,y,t_start,t_end,model):
    di = np.logical_and(t>t_start,t<t_end)
    dt, dy = t[di], y[di]
    plt.plot(dt,dy,label='decay')
    # plt.legend()

    p0 = model.get_p0(dt, dy)
    plt.plot(dt,model.fun(dt-t_start,*p0))

    popt, pcov = curve_fit(model.fun,dt-t_start,dy,p0)
    stderr = np.sqrt(np.diag(pcov))
    r = dy - model.fun(dt-t_start,*popt)
    plt.plot(dt,model.fun(dt-t_start,*popt))
    plt.show()

    return popt, stderr, r

def process_file(filename, t_start, t_end, model):
    try:
        data = pd.read_csv(filename,
                           sep=';',
                           decimal=',',
                           header=None,
                           skiprows=15)
    except pd.errors.EmptyDataError:
        print('ERROR invalid file',filename)
        # continue
        return
    except FileNotFoundError:
        print('ERROR missing file',filename)
        # continue
        return

    t,y = np.array(data[0]),np.array(data[1])
    # return process_data(t,y,0.025,0.030,Exp2)
    # return process_data(t,y,0.019,0.025,ExpS_rise)
    return process_data(t, y, t_start, t_end, model)

if __name__ == '__main__':
    dirpath = 'respotkanieponiedziaek'

    for filename in os.listdir(dirpath):
        print(filename,process_name(filename))
        date, sample_nr, power = process_name(filename)
        ret = process_file(os.path.join(dirpath,filename), 0.019,0.025,ExpS_rise)
        if ret is None: continue
        popt, stderr, r = ret
        print(stderr)
        if filename == '28.06.2021Sample208PowerHigh.txt':
            break

reason: import numpy,from scipy


text:
import cPickle as pickle
from scipy import spatial
from sklearn.cluster import MiniBatchKMeans
from doc_collector.vsmapping import VSMapping
import utils

class DocGenerator(object):

    """DocGenerator

    The DocGenerator used to predict new documents for users based on
    similarity score of previous choices by user.

    Attributes
    ---------------
    users : dict_type
            stores a dictionary of user ids with a KMeans Classifier as the value

    vsm : VSMapping
            Used to load vectors from memory

    """

    def __init__(self, n_clusters=8):
        self.users = {}
        self.vsm = VSMapping()
        self.vsm.load_from_mem()
        self.n_clusters = n_clusters

    def _compute_cosine(self, a, b):
        """Return the cosine similarity of two vectors"""
        return 1 - spatial.distance.cosine(a, b)

    def fit(self, user_id, v_ids):
        """Initial fit for user with vectors. If length of vectors is less
        than cluster size, raise exception

        """
        if len(v_ids) < self.n_clusters:
            raise ValueError('Number of vectors must be greater than or equal\
                             to the n_clusters')

        # Load or Initialize the model
        model = MiniBatchKMeans(n_clusters=8)

        # Fit the model with new data
        vectors = [self.vsm.retreive_val(v_id) for v_id in v_ids]
        model.fit(vectors)
        self.users[user_id] = model

    def batch_fit(self, data):
        """Batch fits all user data and vectors into unique classifiers

        inputs
        -----------
        data : list_type
                A list of dictionaries with {user_id, vectors} pairs

        """
        for d in data:
            self.fit(d[0], d[1])

    def partial_fit(self, user_id, v_id):
        """Partial fits user model"""
        if user_id in self.users:
            self.users[user_id].partial_fit(self.vsm.retreive_val(v_id))
        else:
            raise ValueError('User not in model')

    def predict(self, user_id, v_ids, rand_v_ids):
        """Returns predicted vector ids for top 5 predictions. Otherwise
        returns random

        """
        if user_id not in self.users:
            raise ValueError('User not in model')

        # Load vectors and make predictions
        vectors = [self.vsm.retreive_val(v_id) for v_id in v_ids]
        preds = self.users[user_id].predict(vectors)

        # Score each prediction by measuring the cosine distance
        # between the cluster centroid and the given vector
        # Only those who's similarity is greater than 0.20 are used
        scores = [(self._compute_cosine(self.users[user_id].cluster_centers_[c],
                                       vectors[i]), i) for i, c in enumerate(preds)]
        scores = sorted(scores, reverse=True, key=utils.get_key)
        top_v_ids = [v_ids[i] for s, i in scores if s >= 0.20][:5]

        # If the predicted vectors is less than five, choose randomly for the
        # the remaining
        if len(top_v_ids) < 5:
            top_v_ids += rand_v_ids[:5-len(top_v_ids)]

        return top_v_ids

    def load_from_disk(self):
        pass

    def persist_to_disk(self):
        pass

max_stars_repo_name: alexandercrosson/braise

max_issues_repo_name: alexandercrosson/braise

max_forks_repo_name: alexandercrosson/braise

reason: from scipy


text:
import numpy as np
import logging
import multiprocessing as mp
import os
import sys
import time
import matplotlib.pyplot as plt
import argparse
from functools import partial
import logging

from mskernel import mmd, hsic
from mskernel import kernel
from mskernel import util
from mskernel.featsel import MultiSel, PolySel

### Problems
from problem import *

Benchmark = None #Benchmark_HSIC
mmd_or_hsic = None
os.system("taskset -p 0xff %d" % os.getpid())
nsamples_lin = [500, 1000, 1500, 2000]

def one_trial(i, n_samples, problem, n_select, algo):
    p,r = problem.sample(n_samples, i)

    if 'MMD' in mmd_or_hsic:
        bw = util.meddistance(np.vstack((p, r)), subsample=1000) **2
        metric = mmd.MMD_Inc(kernel.KGauss(bw))
    else:
        p_bw = util.meddistance(p, subsample=1000) **2
        r_bw = util.meddistance(r, subsample=1000) **2
        metric = hsic.HSIC_Inc(kernel.KGauss(p_bw), kernel.KGauss(r_bw), 5)

    feat_select = algo(metric)

    results = feat_select.test(p,r, args=n_select, seed=i,)

    ## True selected features.
    if results['sel_vars'].shape[0] > 1:
        true = problem.is_true(results['sel_vars'])
        n_true = np.sum(true)
        fpr = np.sum(results['h0_rejs'][np.logical_not(true)])/max(n_select-n_true,1)
        tpr = np.sum(results['h0_rejs'][true])/max(n_true,1)
    else:
        tpr, fpr = 0, 0
    logging.debug("TPR is :{0:.3f}  FPR is :{1:.3f}".format(tpr, fpr, ))
    return tpr, fpr

def independent_job(problem, n_samples, n_select, pool, n_repeats,
        algo,):
    result = pool.map_async(partial(one_trial,
        n_samples=n_samples,
        problem=problem,
        n_select=n_select,
        algo=algo,
        ),
        [i for i in range(n_repeats)])
    res = result.get()

    tpr, fpr = zip(*res)
    tpr = { 'mean': np.sum(tpr)/n_repeats,
            'sd' : np.sqrt(np.var(tpr)/n_repeats) }
    fpr = { 'mean': np.sum(fpr)/n_repeats,
            'sd' : np.sqrt(np.var(fpr)/n_repeats)}

    logging.debug("Overall TPR is :{0:.3f} ".format(tpr['mean']) + " " + \
            "Its variance is :{0:.3f} ".format(tpr['sd']))

    logging.debug("Overall FPR is :{0:.3f} ".format(fpr['mean']) + " " + \
            "Its variance is :{0:.3f} ".format(fpr['sd']))

    return tpr, fpr

def runExperiments(problem, n_select, algorithm, n_repeats, threads, benchmark):

    if 'MMD' in benchmark:
        Benchmark = Benchmark_MMD
    else:
        Benchmark = Benchmark_HSIC

    if 'Pulsar' in problem:
        nsamples_lin = [100]
        sampler = Benchmark(n_fakes = 30,
            path = './dataset/pulsar/pulsar_stars.csv',
            target = 'target_class',
            classes = [0,1])
    elif 'Heart' in problem:
        nsamples_lin = [138]
        sampler = Benchmark(n_fakes=30,
                path='./dataset/heart/heart.csv',
                target='target',
                classes=[0,1])
    elif 'Wine' in problem:
        nsamples_lin = [100]
        sampler = Benchmark(n_fakes=30,
                path='./dataset/wine/winequality.csv',
                target='type',
                classes=['white','red'])
    else:
        assert(0==1)

    if 'MultiSel' in algorithm:
        algo = MultiSel
    elif 'PolySel' in algorithm:
        algo = PolySel
    else:
        assert(0==1)

    ## Distributed trials.
    with mp.Pool(threads) as pool:

        results = []

        for n_samples in nsamples_lin:

            results.append(independent_job(sampler,
                n_samples,
                n_select,
                pool,
                n_repeats,
                algo))

    tpr, fpr = zip(*results)

    setting = {
                'nsample_lin' : nsamples_lin,
                'problem': problem,
                'algorithm': algorithm,
                'n_select': n_select}

    results = { 'setting':setting,

                'x_lin' : nsamples_lin,
                'fpr'   : fpr,
                'tpr'   : tpr,}

    logging.info("Setting : {0}".format(setting))
    logging.info("Results : {0}".format(results))

    np.save("temp/Ex5_{0}_{1}_{2}".format(n_repeats,
            problem,
            algorithm,
            n_select,
            ),
        results)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(add_help=False)

    parser.add_argument('-p','--problem',
            default = 'Pulsar',
            type = str,
            help = 'Problem setting: MS, VS.')

    parser.add_argument('-s','--n_select',
            default = 30,
            type = int,
            help = 'Number of features to select')

    parser.add_argument('-r','--n_repeats',
            default = 100,
            type = int,
            help = 'Number of trials or "r"epeats.')

    parser.add_argument('-t','--threads',
            default = 5,
            type = int,
            help = 'Number of threads.')

    parser.add_argument('-a','--algo',
            default = 'PolySel',
            type = str,
            help = 'PolySel or MultiSel')

    parser.add_argument('-v','--verbose',
            default = 0,
            type = int,
            help = 'Verbose level: 0, 1, 2.')

    parser.add_argument('-b','--benchmark',
            default = "MMD",
            type = str,
            help = 'Either MMD or HSIC.')

    args = parser.parse_args()

    if args.verbose == 0:
        logging.getLogger().setLevel(logging.WARNING)
    elif args.verbose == 1:
        logging.getLogger().setLevel(logging.INFO)
    elif args.verbose == 2:
        logging.getLogger().setLevel(logging.DEBUG)

    mmd_or_hsic= args.benchmark

    runExperiments(
            args.problem,
            args.n_select,
            args.algo,
            args.n_repeats,
            args.threads,
            args.benchmark)

max_stars_repo_name: jenninglim/multiscale-features

max_issues_repo_name: jenninglim/multiscale-features

max_forks_repo_name: jenninglim/multiscale-features


text:
import networkx as nx
import numpy as np
import argparse
import json
import matplotlib.pyplot as plt
import os
from gensim.models import Doc2Vec
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.manifold import SpectralEmbedding
from sade.community_detection import load_data, generate_graph

def load_clusters(bunch_file):
    with open(bunch_file) as f:
        lines = f.read().splitlines()

    clusters = {}
    for line in lines:
        name, files = line.split('= ')
        try:
            name = int(name)
        except:
            pass

        files = files.split(', ')

        for filename in files:
            clusters[filename] = name

    return clusters

if __name__ == '__main__':
    argparser = argparse.ArgumentParser()
    argparser.add_argument('-g', help='Input graph')
    argparser.add_argument('-e', help='Doc2vec model')
    argparser.add_argument('-m', help='Modules file')
    argparser.add_argument('-b', help='Bunchfile')
    argparser.add_argument('-s', help='System name')
    argparser.add_argument('--method', default='tsne', help='Method for dim reduction')
    argparser.add_argument('-o', default='.')
    args = argparser.parse_args()

    if args.method == 'pca':
        clf = PCA(n_components=2)
    elif args.method == 'tsne':
        clf = TSNE(n_components=2)
    elif args.method == 'spectral':
        clf = SpectralEmbedding(n_components=2)

    modules = json.load(open(args.m))

    X, y, model = load_data(args.e)

    X_ = clf.fit_transform(X)

    G = generate_graph(args.g, model, modules)

    pos_n = dict(zip(y, map(lambda x: x.tolist(), X_)))

    plt.figure(figsize=(8, 6))
    filename = '{}_{}_{}.png'.format(args.s, os.path.split(args.g)[-1].split('.')[0].strip('._all'), os.path.split(args.b)[-1].split('.')[0])

    if args.b == None:
        nx.draw_networkx_edges(G, pos_n, nodelist=G.nodes(), alpha=0.2, edge_color='b')
        nx.draw_networkx_nodes(G, pos_n, nodelist=G.nodes(),
                           node_size=40,
                           alpha=0.8,
                           linewidths=0,
                           cmap=plt.cm.Reds_r)
    else:
        clusters = load_clusters(args.b)
        # import pdb; pdb.set_trace()
        nx.draw_networkx_edges(G, pos_n, nodelist=list(clusters.keys()), alpha=0.1, edge_color='b', arrows=False)
        nx.draw_networkx_nodes(G, pos_n, nodelist=list(clusters.keys()),
                           node_size=40,
                           node_color=(list(clusters.values())),
                           alpha=0.8,
                           linewidths=0,
                           cmap=plt.cm.Reds_r)

    plt.axis('off')
    plt.tight_layout()
    plt.savefig(os.path.join(args.o, filename), format='png')

max_stars_repo_name: papachristoumarios/sade

max_issues_repo_name: azterizk/sade

max_forks_repo_name: azterizk/sade

reason: import numpy,import networkx


text:
import numpy as np

from descartes import PolygonPatch
import alphashape

import matplotlib.image as mpimg
import matplotlib.pyplot as plt

import os, logging, h5py, pickle, cv2

from PIL import Image, ImageDraw
from tqdm import tqdm
from collections import defaultdict
from scipy import spatial
import pycolmap
import pandas as pd

from sklearn.cluster import MiniBatchKMeans
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from scipy.spatial.transform import Rotation as R
import seaborn as sns

import logging
from datetime import datetime

from scipy.spatial.distance import cdist
import json

def box_preprocess(data, w, h):
#     print(w, h, len(data))
    data = list(data)
    data.sort(key = lambda x: x[1])
    remove_list = list()

    for i in range( len(data) - 1):
#         print(i, data[i])
        if data[i][0] == -1:
            continue
        if data[i+1][1] - data[i][1] < 0.1: # close left edge
            if abs(data[i+1][3] - data[i][3]) > 0.2:# large right edge dist: split
                if data[i+1][3] > data[i][3]:
                    data[i+1][1] = data[i][3]
                else:
                    data[i][1] = data[i+1][3]

            else:# merge
                if data[i][2] - data[i][0] > data[i+1][2] - data[i+1][0]:
                    data[i][0] = -1
                else:
                    data[i+1][0] = -1
        if abs(data[i+1][3] - data[i][3]) < 0.1: # close right edge
            if abs(data[i+1][1] - data[i][1]) > 0.2:# large left edge dist: split
                if data[i+1][1] > data[i][1]:
                    data[i][3] = data[i+1][1]
                else:
                    data[i+1][3] = data[i][1]

            else:# merge
                if data[i][2] - data[i][0] > data[i+1][2] - data[i+1][0]:
                    data[i][0] = -1
                else:
                    data[i+1][0] = -1

#         if (data[i][2] - data[i][0])*h < 0.3*(data[i][3] - data[i][1])*w:
#             data[i][0] = -1

#     if len(data) > 0:
#         if (data[-1][2] - data[-1][0])*h < 0.3*(data[-1][3] - data[-1][1])*w:
#             data[-1][0] = -1

    data_new = [x for x in data if x[0] > -1]
    return np.array(data_new)

class PatchSet:
    def __init__(self):
        self.fa = list()
        pass

    def find(self, x):
        if x == self.fa[x]:
            return x
        else:
            self.fa[x] = self.find(self.fa[x])
            return self.fa[x];

    def merge(self, i, j):
        self.fa[self.find(j)] = self.find(i)

class Patch:
    alphashape_thres = 0.5

    def __init__(self, box, image_id, ind):
        self.box = box
        self.image_id = image_id
        self.p3d_ids = set()
        self.p2d_xys = dict()
        self.validaty = False
        self.area = (self.box[2] - self.box[0])*(self.box[3] - self.box[1])
        pass

    def check_validation(self, xys, p3d_ids):
        whole_xys = list()
        for i in range(len(xys)):
            if (xys[i][1] > self.box[0] and xys[i][1] < self.box[2] and
                xys[i][0] > self.box[1] and xys[i][0] < self.box[3]):
                whole_xys.append(xys[i])
                self.p3d_ids.add(p3d_ids[i])
                self.p2d_xys[p3d_ids[i]] = xys[i]

        AS = alphashape.alphashape(whole_xys,10)

        rate = AS.area/self.area
        if rate >= self.alphashape_thres:
            self.validaty = True
        else:
            self.validaty = False

        return self.validaty

    def check_match(self, patch_B, show = False):
        inters = self.p3d_ids.intersection(patch_B.p3d_ids)
        inters = list(inters)
        if show:
            print(inters)
        if len(inters) < 25:
            return False
        if show:
            print (self.check_coverage(inters, True), patch_B.check_coverage(inters))
        if (self.check_coverage(inters) > self.alphashape_thres*0.6 and
           patch_B.check_coverage(inters) > self.alphashape_thres*0.6):
            return True
        else:
            return False

    def save_crop(self, image_path, save_path):
        im = Image.open(image_path)
        width, height = im.size
        im1 = im.crop((self.box[1]*width, self.box[0]*height, self.box[3]*width, self.box[2]*height))
        save_dir = os.path.dirname(save_path)
        os.makedirs(save_dir, exist_ok=True)
        im1.save(save_path)

    def check_coverage(self, p3d_ids, show = False):
        xys = [self.p2d_xys.get(key) for key in p3d_ids]
        AS = alphashape.alphashape(xys,5)
        if show:
            print(AS.area, self.area, self.image_id)
            fig, ax = plt.subplots()
            ax.scatter(*zip(*xys))
            # Plot alpha shape
            ax.add_patch(PolygonPatch(AS, alpha=.2))
            ax.set_ylim(self.box[1],self.box[3])
            ax.set_xlim(self.box[0],self.box[2])

            plt.savefig("alphashape.png")

        return AS.area/self.area
    def compute_similarity(self, patch_a):
        inters_len = len(self.p3d_ids.intersection(patch_a.p3d_ids))
        unions_len = len(self.p3d_ids.union(patch_a.p3d_ids))
        return inters_len/unions_len

def test_generate_patches(images, box_dir, test_image2patch_path, image_dir, do_croping = False, crop_output_dir = None):
    patch_list = list()
    image2patch = dict()

    # for error reporting
    no_box_detected_images = list()
    no_box_left_images = list()

    patch_cnt = 0

    for image in tqdm(images):
        file_name, param = image
        image_name = file_name.split('/')[-1]
        box_file_name = file_name.split('.')[0] + ".npy"

        image2patch[image_name] = list()

        w = param[1]
        h = param[2]

        if not os.path.exists(os.path.join(box_dir, box_file_name)):
            logging.warning("no bbox found for "+ os.path.join(box_dir, box_file_name))
            continue

        data = np.load(os.path.join(box_dir, box_file_name))
        if data.shape[0] == 0:
            no_box_detected_images.append(file_name)
            logging.warning("no object detected for "+ os.path.join(box_dir, box_file_name))
            continue

        data=box_preprocess(data, w, h)
        if data.shape[0] == 0:
            no_box_left_images.append(file_name)
            continue

        for box_id in range(data.shape[0]):
            temp_patch = Patch(data[box_id], image_name, patch_cnt)
            patch_list.append(temp_patch)
            image2patch[image_name].append(patch_cnt)

            if do_croping:
                image_path = os.path.join(image_dir, file_name)
                save_path = os.path.join(crop_output_dir, str(patch_cnt)+ ".png")
                temp_patch.save_crop(image_path, save_path)

            patch_cnt += 1

    print("number of patches:", len(patch_list))

    with open(test_image2patch_path, "wb") as f:
        pickle.dump(image2patch,f)

    for i in range(len(patch_list)):
        del patch_list[i].p2d_xys

    with open("./aachen_test_patch_list.pkl", "wb") as f:
        pickle.dump(patch_list,f)

    return patch_list, image2patch

def database_generate_patches(images, cameras, box_dir, image2patch_path):
    patch_list = list()
    image2patch = dict()

    # for error reporting
    no_box_detected_images = list()
    no_box_left_images = list()

    patch_cnt = 0

    for image_id, image in tqdm(images.items()):

        box_file_name = image.name.split('.')[0] + ".npy"

        image2patch[image.name] = list()

        matched = image.point3D_ids != -1
        points3D_covis = image.point3D_ids[matched]
        total_matched_p2d_xys = image.xys[matched]

        w = cameras[image.camera_id].width
        h = cameras[image.camera_id].height

        # p2d xy normalization
        for i in range(len(total_matched_p2d_xys)):
            total_matched_p2d_xys[i][0] /= w
            total_matched_p2d_xys[i][1] /= h

        if not os.path.exists(os.path.join(box_dir, box_file_name)):
#             logging.warning("no bbox found for "+ os.path.join(box_dir, box_file_name))
            continue

        data = np.load(os.path.join(box_dir, box_file_name))
        if data.shape[0] == 0:
            no_box_detected_images.append(image.name)
            logging.warning("no object detected for "+ os.path.join(box_dir, box_file_name))
            continue

        data=box_preprocess(data, w, h)
        if data.shape[0] == 0:
            no_box_left_images.append(image.name)
            continue

        for box_id in range(data.shape[0]):
            temp_patch = Patch(data[box_id], image_id, patch_cnt)
            validity = temp_patch.check_validation(total_matched_p2d_xys, points3D_covis)#, os.path.join(image_dir, image.name))
#             if validity:
            patch_list.append(temp_patch)
            image2patch[image.name].append(patch_cnt)
            patch_cnt += 1

    print("number of patches:", len(patch_list))

    with open(image2patch_path, "wb") as f:
        pickle.dump(image2patch,f)

    for i in range(len(patch_list)):
        del patch_list[i].p2d_xys

    with open("./aachen_patch_list.pkl", "wb") as f:
        pickle.dump(patch_list,f)

    return patch_list

########################

def patch_similarity_computation(patch_list, db_image_pairs_path, image2patch_path, save_path):
    if os.path.exists(save_path):
        affinity_matrix = np.load(save_path)
        return affinity_matrix

    with open(db_image_pairs_path, "r") as f:
        db_image_pairs = f.readlines()
    with open(image2patch_path, "rb") as f:
        image2patch = pickle.load(f)

    patch_num = len(patch_list)
    affinity_matrix = np.zeros((patch_num, patch_num))

    for pair in tqdm(db_image_pairs):
        image1 = pair.split()[0]
        image2 = pair.split()[1]
        if image1 < image2:
            continue
        for patch1_id in image2patch[image1]:
            for patch2_id in image2patch[image2]:
                affinity_matrix[patch1_id, patch2_id] = affinity_matrix[patch1_id, patch2_id] = patch_list[patch1_id].compute_similarity(patch_list[patch2_id])

    np.save(save_path, affinity_matrix)
    return affinity_matrix

def patch_co_vis_grouping_new(patch_list, image2patch_path, db_image_pairs_path, group_result_dir, n_cluster = 2000):
    patch_num = len(patch_list)

    affinity_matrix = patch_similarity_computation(patch_list, db_image_pairs_path, image2patch_path, "group/aachen_patch/a_matrix.npy")
    labels_path = group_result_dir / "cluster_result.pkl"

    distance_matrix = 1-affinity_matrix
    aggclus = AgglomerativeClustering(n_clusters=n_cluster,
                                      memory= str(str(group_result_dir) + "/temp"),
                                      linkage="average",
                                      affinity = "precomputed").fit(distance_matrix)
    labels = aggclus.fit_predict(distance_matrix)

    index2patch_id = dict()
    ind = 0
    for patch in tqdm(patch_list):
        index2patch_id[ind] = ind
        ind += 1

    with open(labels_path, "wb") as f:
        pickle.dump(labels, f)
    id_path = group_result_dir / "index2patch_id.pkl"
    with open(id_path, "wb") as f:
        pickle.dump(index2patch_id, f)
    return labels, index2patch_id

def patch_desc_grouping(patch_global_feature_path, patch_list, group_result_dir, n_cluster = 2000):
    db_desc = h5py.File(patch_global_feature_path, 'r')

    labels_path = group_result_dir / "cluster_result.pkl"
    id_path = group_result_dir / "index2patch_id.pkl"
    group_desc = []
    index2patch_id = dict()
    ind = 0

    for patch in tqdm(patch_list):
        group_desc.append(np.array(db_desc[str(ind) + '.png']['global_descriptor']))
        index2patch_id[ind] = ind
        ind += 1

    print(len(group_desc))
    group_desc=np.array(group_desc)

    aggclus = AgglomerativeClustering(n_clusters=n_cluster, memory=str(str(group_result_dir) + "/temp")).fit(group_desc)
    labels = aggclus.labels_

    with open(labels_path, "wb") as f:
        pickle.dump(labels, f)
    with open(id_path, "wb") as f:
        pickle.dump(index2patch_id, f)
    return labels, index2patch_id

#  patches match and merge
def co_vis_grouping(patch_list, patch_set_path, db_image_pairs_path, image2patch_path, image_dir,
                    do_crop = False, group_output_dir = None, images = None):

    if os.path.exists(patch_set_path):
        with open(patch_set_path, "rb") as f:
            patch_set = pickle.load(f)
    else:
        patch_set = PatchSet()

        for i in range(len(patch_list)):
            patch_set.fa.append(i)

        with open(image2patch_path, "rb") as f:
            image2patch = pickle.load(f)

        with open(db_image_pairs_path, "r") as f:
            db_image_pairs = f.readlines()

        logging.info("grouping...")

        for pair in tqdm(db_image_pairs):
            image1 = pair.split()[0]
            image2 = pair.split()[1]
            if image1 < image2:
                continue
            for patch1_id in image2patch[image1]:
                for patch2_id in image2patch[image2]:
                    if patch_set.find(patch2_id) != patch2_id:
                        continue
                    if (patch_list[patch1_id].check_match(patch_list[patch2_id]) and
                        patch_list[patch_set.find(patch1_id)].check_match(patch_list[patch2_id])):
                        patch_set.merge(patch1_id, patch2_id)

        with open(patch_set_path, "wb") as f:
            pickle.dump(patch_set, f)

#     if do_crop:
#         logging.info("Cropping...")
#         for i in tqdm(range(len(patch_list))):
#             image_path = os.path.join(image_dir, images[patch_list[i].image_id].name)
#             save_path = os.path.join(group_output_dir, str(patch_set.find(i)), str(i)+ ".png")
#             patch_list[i].save_crop(image_path, save_path)

    return patch_set

def points_grouping(patch_list, images, labels, index2patch_id, patch_global_feature_path):
    patch_group_points = defaultdict(set)
    group_descriptor = defaultdict(list)

    db_desc = h5py.File(patch_global_feature_path, 'r')

    for i in tqdm(range(len(labels))):
        patch = patch_list[index2patch_id[i]]
        p3d_id_set = set(images[patch.image_id].point3D_ids)
#         p3d_id_set = set(patch.p3d_ids)

        patch_group_points[labels[i]] = patch_group_points[labels[i]].union(p3d_id_set)
        group_descriptor[labels[i]].append(np.array(db_desc[str(i) + '.png']['global_descriptor']))

    for label, desc in tqdm(group_descriptor.items()):
        if len(desc) == 0:
            group_descriptor.pop(label, None)
            continue
        arr = np.array(desc)
        dist = cdist(arr, np.mean(arr, axis=0).reshape((1,4096)), metric='euclidean')
        dist_idx = np.argmin(dist, axis=0)
        group_descriptor[label] = arr[dist_idx].reshape(4096)

    cluster_point_num = []
    for gid, cps in patch_group_points.items():
        cluster_point_num.append(len(cps))

    somelist_df = pd.DataFrame(cluster_point_num)
    print(somelist_df.describe())

    return patch_group_points, group_descriptor

def build_tree(group_descriptor, nca = None):
    group_desc = []
    index2name = dict()
    ind = 0
    for gid, desc in tqdm(group_descriptor.items()):
        group_desc.append(desc)
        index2name[ind] = gid
        ind += 1

    if nca is not None:
        group_desc = nca.transform(group_desc)

    group_desc=np.array(group_desc)
    tree = spatial.KDTree(group_desc)
    return tree, index2name

def generate_retrieval_file(global_feature_path, tree, index2name, query_images, retrieval_dir,
                            nca = None, label = "whole", top_n = 50):
    desc = h5py.File(global_feature_path, 'r')

    retrieval_path = retrieval_dir / str("retrieval_" + label + "_"+str(top_n)+".txt")
    file = open(retrieval_path, "w+")

    for query_image in tqdm(query_images):
        name, param = query_image

        cur_desc = np.array([desc[name]['global_descriptor']])
        if nca is not None:
            cur_desc = nca.transform(cur_desc)[0]

        _, group_indices = tree.query(cur_desc, top_n)

        if top_n == 1:
            file.write(name + ' ' + str(index2name[group_indices]) + '\n')
        else:
            for i in range(len(group_indices)):
                file.write(name + ' ' + str(index2name[group_indices[i]]) + '\n')

def db_patch_point_bind(groups_patch_points_path, groups_image_points_path, patch_set_path,
                        image2patch_path, new_images, patch_list):
    groups_patch_points=defaultdict(set) # for retrieval result examine
    groups_image_points=defaultdict(set) # for localization

    with open(image2patch_path, "rb") as f:
        image2patch = pickle.load(f)

    with open(patch_set_path, "rb") as f:
        patch_set = pickle.load(f)

    for image_id, image in tqdm(new_images.items()):
        patches = image2patch[image.name]
        p3d_id_set = set(image.point3D_ids)
        for patch_id in patches:
            group_id = patch_set.find(patch_id)
            groups_image_points[group_id] = groups_image_points[group_id].union(p3d_id_set)
            groups_patch_points[group_id] = groups_patch_points[group_id].union(patch_list[patch_id].p3d_ids)

    with open(groups_patch_points_path, "wb") as f:
        pickle.dump(groups_patch_points,f)
    with open(groups_image_points_path, "wb") as f:
        pickle.dump(groups_image_points,f)

def generate_group_descriptor(db_feature_path ):
    db_desc = h5py.File(db_feature_path, 'r')['db']

    class_list =[]
    database = []
    index = 0
    val_desc = dict()

    for group in tqdm(db_desc.keys()):
        group_desc = []
        for ele in db_desc[group].keys():
            patch_id = int(ele.split('.')[0])
            group_desc.append(np.array(db_desc[group][ele]['global_descriptor']))#.flatten())

        if len(group_desc) > 0:
            group_desc = np.array(group_desc)
            database.append(np.mean(group_desc, axis=0))
            class_list.append(group)
    database=np.array(database)
    print(database.shape)
    tree = spatial.KDTree(database)
    return tree, class_list

def group_retrival(global_feature_path, tree, class_list, k = 5, gt = None):

    test_desc = dict()
#     test_desc_file = h5py.File(global_feature_path, 'r')['query']
    test_desc_file = h5py.File(global_feature_path, 'r')

    for ele in tqdm(test_desc_file.keys()):
        patch_id = int(ele.split('.')[0])
        test_desc[patch_id]=np.array(test_desc_file[ele]['global_descriptor'])

    gt_posi = dict()
    ave_posi = 0
    recall = 0
    retrieve_groups = dict()
    for patch_id in tqdm(test_desc.keys()):
        d, group_indices = tree.query(test_desc[patch_id], k)
        if k == 1:
            group_ids = [int(class_list[group_indices])]
        else:
            group_ids = [int(class_list[i]) for i in group_indices]
        retrieve_groups[patch_id] = group_ids
        if not gt == None:
            for i in range(k):
                gt_posi[patch_id] = -1
                if (group_ids[i] == gt[patch_id] or
                    len(patch_list[patch_id].p3d_ids.intersection(groups_patch_points[group_ids[i]])) > 50):
                    gt_posi[patch_id] = i
                    ave_posi += i+1
                    recall += 1
                    break

    if not gt == None:
        # report
        print("recall:", recall/len(gt))
        print("ave. Posi:", ave_posi/recall)

    return retrieve_groups

def localize(query_images, test_retrieve_groups, local_feature_path, point_desc_path, test_image2patch_path,
             groups_image_points, points3D_new,images_new,cameras_new, log_path):

    print("localizing:", log_path)

    with open(test_image2patch_path, "rb") as f:
        test_image2patch = pickle.load(f)

    # read retrieval result
    point_descs = dict()
    with open(point_desc_path, "rb") as file:
        point_descs = pickle.load(file)
#     logging.info("loading point descs2")

#     with open(str(point_desc_path)[:-4] + "_2.pkl", "rb") as file:
#         temp = pickle.load(file)
#     logging.info("merging")

#     point_descs.update(temp)
#     del temp

    f = h5py.File(local_feature_path, 'r')

#     with open(groups_image_points_path, "rb") as f:
#         groups_image_points = pickle.load(f)

    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
    search_params = dict(checks=50)   # or pass empty dictionary
    flann = cv2.FlannBasedMatcher(index_params,search_params)

#     point_desc_file = h5py.File(point_desc_path, 'a')

    result = dict()
    max_inlier_nums = dict()
    infos = dict()
    for query_image in tqdm(query_images):
        file_name, param = query_image
        image_name = file_name.split('/')[-1]
        if image_name in result:
            continue
        candidate_point_set = list()
        patch_ids = test_image2patch[image_name]

        # point set merging
        for patch_id in patch_ids:
            groups = test_retrieve_groups[patch_id]
            for group_id in groups:
                merged_flag = False
                for i in range(len(candidate_point_set)):
                    if len(candidate_point_set[i].intersection(groups_image_points[group_id])) > 100:
                        merged_flag = True
                        candidate_point_set[i] = candidate_point_set[i].union(groups_image_points[group_id])
                        break
                if not merged_flag:
                    candidate_point_set.append(groups_image_points[group_id])

        cluster_infos = []
        max_inlier_num = 0
        final_qvec = np.array([-0.02718695,-0.12122045,0.20194369,0.97148609])
        final_tvec = np.array([723.98874206,76.41164933,187.85558868])

        candidate_point_set = sorted(candidate_point_set, key=len, reverse=True)

        # fusion & matching
        for cps in candidate_point_set:
            cluster_info = {"#points": len(cps)}
            p3d_descs = list()
            p3d_ids = list()
            for p3d_id in cps:
                if p3d_id == -1:
                    continue
                p3d_ids.append(p3d_id)
                sid = str(p3d_id)
                p3d_descs.append(point_descs[p3d_id])
            if len(p3d_descs) < 8:
                continue

            p3d_descs = np.array(p3d_descs)
            p2d_descs = f[file_name]['descriptors'].__array__().transpose()
            matches = flann.knnMatch(p2d_descs, p3d_descs, k=2)

            mp3d = list()
            mkpq = list()
            for i,(m,n) in enumerate(matches):
                if m.distance < 0.8*n.distance:
                    mp3d.append(points3D_new[p3d_ids[m.trainIdx]].xyz)
                    mkpq.append(f[file_name]['keypoints'][m.queryIdx])

            mp3d = np.array(mp3d).reshape(-1, 3)
            mkpq = np.array(mkpq).reshape(-1, 2)
            mkpq += 0.5
            cluster_info["#matches"] = mp3d.shape[0]
            cfg = {
                'model': param[0],
                'width': param[1],
                'height': param[2],
                'params': param[3],
            }
            ret = pycolmap.absolute_pose_estimation(mkpq, mp3d, cfg)
            cluster_info["suc"] = ret['success']

            if ret['success'] == True:
                cluster_info["#inliers"] = ret['num_inliers']
                if ret['num_inliers'] > max_inlier_num:
                    max_inlier_num = ret['num_inliers']
                    final_qvec = ret['qvec']
                    final_tvec = ret['tvec']

            cluster_infos.append(cluster_info)
        result[image_name] = (final_qvec, final_tvec)
        max_inlier_nums[image_name] = max_inlier_num

        infos[image_name] = {"max_inlier_nums":max_inlier_num,
                             "#candidate" : len(candidate_point_set),
                             "candidates" : cluster_infos,
                             "max_inlier_nums": max_inlier_num
                            }

    with open(log_path, 'w') as fp:
        json.dump(infos, fp)
#     f.close()
    del point_descs
#     point_desc_file.close()

    return result, max_inlier_nums

max_stars_repo_name: nizqleo/Hierarchical-Localization

max_issues_repo_name: nizqleo/Hierarchical-Localization

max_forks_repo_name: nizqleo/Hierarchical-Localization


text:
import os.path
import pickle

import click
import numpy as np
import pandas as pd
import torch
import torch.utils.data

from info_nas.datasets.arch2vec_dataset import prepare_labeled_dataset, split_off_valid
from info_nas.datasets.io.semi_dataset import labeled_network_dataset
from info_nas.datasets.io.transforms import get_transforms, get_all_scales
from info_nas.models.losses import losses_dict
from nasbench import api

from info_nas.config import local_dataset_cfg, load_json_cfg

@click.command()
@click.argument('scale_name')
@click.option('--dataset', default='../data/train_labeled.pt')
@click.option('--scale_dir', default='../data/scales/')
@click.option('--scale_cfg', default='../configs/scale_test_config.json')
@click.option('--nasbench_path', default='../data/nasbench.pickle')
@click.option('--batch_size', default=32)
@click.option('--split_ratio', default=None, type=float)
@click.option('--config', default=None)
@click.option('--save_dir', default=None)
@click.option('--use_larger_part/--use_smaller_part', default=False)
def main(scale_name, dataset, scale_dir, scale_cfg, nasbench_path, batch_size, split_ratio, config, save_dir,
         use_larger_part):
    """
    Compute the baseline - difference between batches and the mean of a scaled dataset. Output and save stats.
    """

    dataset_name = dataset

    if nasbench_path.endswith('.pickle'):
        with open(nasbench_path, 'rb') as f:
            nb = pickle.load(f)
    else:
        nb = api.NASBench(nasbench_path)

    if config is None:
        config = local_dataset_cfg

    scale_cfg = load_json_cfg(scale_cfg)

    # load all scaling
    scale_config = scale_cfg["scale"]
    include_bias = scale_config["include_bias"]
    normalize = scale_config["normalize"]
    multiply_by_weights = scale_config["multiply_by_weights"]
    use_scale_whole = scale_config["scale_whole"]

    for k, v in scale_config.items():
        print(f"{k}: {v}")

    scale_train, scale_valid, scale_whole = get_all_scales(scale_dir, scale_config)
    scale_whole = scale_whole if use_scale_whole else None
    print(f"Scale paths: {scale_train}, {scale_valid}, {scale_whole}")

    transforms = get_transforms(scale_train if scale_name == "train" else scale_valid, include_bias, normalize,
                                multiply_by_weights, scale_whole_path=scale_whole)

    key = 'val' if scale_name == 'valid' else scale_name
    dataset, _ = prepare_labeled_dataset(dataset, nb, key=key, remove_labeled=False, config=config)
    if split_ratio is not None:
        larger_part, dataset = split_off_valid(dataset, ratio=split_ratio)
        dataset = larger_part if use_larger_part else dataset

    dataset = labeled_network_dataset(dataset, transforms=transforms)

    data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=scale_name == "train",
                                              num_workers=0)

    print("Loading output dataset...")
    out_data = []
    for i, item in enumerate(dataset):
        if i % 10000 == 0:
            print(i)
        out_data.append(item[3].numpy())

    out_data = np.array(out_data)
    print(out_data.shape)
    print(np.min(out_data))
    print(np.max(out_data))
    print(np.mean(out_data))

    loss_stats = {k: [] for k in losses_dict.keys() if k != 'weighted'}

    mean_stats = out_data.mean(axis=0)
    print(mean_stats.shape)
    mean_stats = np.tile(mean_stats, (batch_size, 1))
    mean_stats = torch.Tensor(mean_stats)

    losses = {k: v() for k, v in losses_dict.items() if k != 'weighted'}

    for batch in data_loader:
        data = batch[3]

        if len(data) < batch_size:
            mean_stats = mean_stats[:len(data)]

        for loss_name, loss in losses.items():
            val = loss(data, mean_stats).item()
            loss_stats[loss_name].append(val)

    df = []

    for loss_name, stats in loss_stats.items():
        print(f"{loss_name}: mean - {np.mean(stats)} | std - {np.std(stats)} | min - {np.min(stats)} | "
              f"median - {np.median(stats)} | max - {np.max(stats)} |")

        df.append({
            'loss_name': loss_name,
            'mean': np.mean(stats),
            'std': np.std(stats),
            'min': np.min(stats),
            'max': np.max(stats),
            'median': np.median(stats),
        })

    if save_dir is not None:
        df = pd.DataFrame(df)
        out_name = os.path.basename(dataset_name).replace('.pt', '')
        df.to_csv(os.path.join(save_dir, f'{out_name}_baseline.csv'))

if __name__ == "__main__":
    main()

max_stars_repo_name: gabrielasuchopar/info-nas

max_issues_repo_name: gabrielasuchopar/info-nas

max_forks_repo_name: gabrielasuchopar/info-nas


text:
import pybullet as p
import time
import numpy as np
import pybullet_data
import math
physicsClient = p.connect(p.GUI)#or p.DIRECT for non-graphical version
p.setAdditionalSearchPath(pybullet_data.getDataPath()) #optionally
p.setGravity(0,0,-10)
planeId = p.loadURDF("plane.urdf")
c= [0,0,0.1]
cubeStartOrientation = p.getQuaternionFromEuler([0,0,0])
#obs1 = p.loadURDF("r2d2.urdf",[2,0,0.5], cubeStartOrientation)
#obs2= p.loadURDF("r2d2.urdf",[0,2,0.5], cubeStartOrientation)
#obs3 = p.loadURDF("r2d2.urdf",[2,2,0.5], cubeStartOrientation)
obs4 = p.loadURDF("r2d2.urdf",[-2,0,0.5], cubeStartOrientation)
#obs5 = p.loadURDF("r2d2.urdf",[2*math.cos(math.pi/6),0+2*math.sin(math.pi/6),0.5], cubeStartOrientation)
car=p.loadURDF("husky/husky.urdf",c,cubeStartOrientation)

numjoints=p.getNumJoints(car)
#for j in range(numjoints):
    #print(p.getJointInfo(car,j))
value = 8 #rad/s
maxForce = 100 #Newton

#print (position)
def fun(pos1,pos2,pos3,thetha):
    r=5
    x=["x1","x2","x3","x4","x5","x6","x7","x8","x9","x10","x11","x12"]
    t=[0 for i in range(12)]
    dist=np.zeros(12)
    for i in range(12):
        x[i]=p.rayTest([pos1,pos2,pos3+0.7],[pos1+r*math.cos(math.pi*i/6+thetha),pos2+r*math.sin(math.pi*i/6+thetha),pos3+0.7])
        #print(x[i])
        x[i]=x[i][0]
        x[i]=list(x[i])
        #print(x[i])
        t[i]=list(x[i][3])
        #print(t[i])
        #p.addUserDebugLine([pos1,pos2,pos3+0.2],[pos1+r*math.cos(0+math.pi*i/6),pos2+r*math.sin(0+math.pi*i/6),pos3],[1,0,1])
        if(x[i][0]==-1 or 0):
            #p.addUserDebugLine([pos1,pos2,pos3+0.7],[pos1+r*math.cos(0+math.pi*i/6+thetha),pos2+r*math.sin(0+math.pi*i/6+thetha),pos3],[1,0,1])
            dist[i]=1000
            #print(dist[i])
        else:
            #p.addUserDebugLine([pos1,pos2,pos3+0.7],t[i],[1,0,0])
            dist[i]=np.sqrt(pow(t[i][0]-pos1,2)+pow(t[i][1]-pos2,2)+pow(t[i][2]-pos3,2))
    return dist

def forward():
    targetVel = value
    for joint in range(2, 6):
        p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL, targetVelocity =targetVel,force = maxForce)
        p.stepSimulation()
def backward():
    targetVel = -value
    for joint in range(2, 6):
        p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL, targetVelocity =targetVel,force = maxForce)
        p.stepSimulation()
def left():
    targetVel=value
    for joint in range(2, 6):
        if joint%2!=0:
            p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
def right():
    targetVel=value
    for joint in range(2, 6):
        if joint%2==0:
            p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
n=1
pos3=[0,0,0]
thetha=0
while(n>0):
    pos=p.getBasePositionAndOrientation(car)
    pos1=pos[0]
    dist=np.zeros(12)
    pos2=p.getEulerFromQuaternion(pos[1])
    thetha=thetha+pos2[2]-pos3[2]
    pos3=pos2
    dist=fun(pos1[0],pos1[1],pos1[2],thetha)
    if(n==2):
        print(thetha)
        print(dist)
    if(n%500==0):
        print(thetha)
        print(dist)

    keys = p.getKeyboardEvents()

    for k, v in keys.items():
        if (k == p.B3G_UP_ARROW and (v & p.KEY_IS_DOWN)):
            targetVel = value
            for joint in range(2, 6):
                p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL, targetVelocity =targetVel,force = maxForce)

            p.stepSimulation()
            #time.sleep(1./240.)
        if (k == p.B3G_UP_ARROW and (v & p.KEY_WAS_RELEASED)):
            targetVel = 0
            for joint in range(2, 6):
                p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)

            p.stepSimulation()
            #time.sleep(1./240.)

        if (k == p.B3G_DOWN_ARROW and (v & p.KEY_IS_DOWN)):
            targetVel = -value
            for joint in range(2, 6):
                p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)

            p.stepSimulation()
            #time.sleep(1./240.)

        if (k == p.B3G_DOWN_ARROW and (v & p.KEY_WAS_RELEASED)):
            targetVel = 0
            for joint in range(2, 6):
                p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)

            p.stepSimulation()
            #time.sleep(1./240.)

        if(k== p.B3G_RIGHT_ARROW and (v & p.KEY_IS_DOWN)):
            targetVel=value
            for joint in range(2, 6):
                if joint%2==0:
                    p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
            #time.sleep(1./240.)

        if(k== p.B3G_RIGHT_ARROW and (v & p.KEY_WAS_RELEASED)):
            targetVel=0
            for joint in range(2, 6):
                if joint%2==0:
                    p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
            #time.sleep(1./240.)

        if(k== p.B3G_LEFT_ARROW and (v & p.KEY_IS_DOWN)):
            targetVel=value
            for joint in range(2, 6):
                if joint%2!=0:
                    p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
            #time.sleep(1./240.)

        if(k== p.B3G_LEFT_ARROW and (v & p.KEY_WAS_RELEASED)):
            targetVel=0
            for joint in range(2, 6):
                if joint%2!=0:
                    p.setJointMotorControl2(car, joint, p.VELOCITY_CONTROL,targetVelocity = targetVel,force = maxForce)
            p.stepSimulation()
            #time.sleep(1./240.)
    p.stepSimulation()
    n=n+1
p.disconnect()


text:
import sympy as sp
from sympy.physics.mechanics import dynamicsymbols
from sympy.physics.mechanics.functions import find_dynamicsymbols
from simupy.utils.symbolic import (lambdify_with_vector_args, grad,
    DEFAULT_LAMBDIFY_MODULES)
from simupy.array import Array

from simupy.systems import DynamicalSystem as DynamicalSystemBase

DEFAULT_CODE_GENERATOR = lambdify_with_vector_args
DEFAULT_CODE_GENERATOR_ARGS = {
    'modules': DEFAULT_LAMBDIFY_MODULES
}

empty_array = lambda: []

class DynamicalSystem(DynamicalSystemBase):
    def __init__(self, state_equation=None, state=None, input_=None,
                 output_equation=None, constants_values={}, dt=0,
                 initial_condition=None, code_generator=None,
                 code_generator_args={}):
        """
        DynamicalSystem constructor, used to create systems from symbolic
        expressions.

        Parameters
        ----------
        state_equation : array_like of sympy Expressions, optional
            Vector valued expression for the derivative of the state.
        state : array_like of sympy symbols, optional
            Vector of symbols representing the components of the state, in the
            desired order, matching state_equation.
        input_ : array_like of sympy symbols, optional
            Vector of symbols representing the components of the input, in the
            desired order. state_equation may depend on the system input. If
            the system has no state, the output_equation may depend on the
            system input.
        output_equation : array_like of sympy Expressions
            Vector valued expression for the output of the system.
        constants_values : dict
            Dictionary of constants substitutions.
        dt : float
            Sampling rate of system. Use 0 for continuous time systems.
        initial_condition : array_like of numerical values, optional
            Array or Matrix used as the initial condition of the system.
            Defaults to zeros of the same dimension as the state.
        code_generator : callable, optional
            Function to be used as code generator.
        code_generator_args : dict, optional
            Dictionary of keyword args to pass to the code generator.

        By default, the code generator uses a wrapper for ``sympy.lambdify``.
        You can change it by passing the system initialization arguments
        ``code_generator`` (the function) and additional keyword arguments to
        the generator in a dictionary ``code_generator_args``. You can change
        the defaults for future systems by changing the module values. See the
        readme or docs for an example.

        """
        self.constants_values = constants_values
        self.state = state
        self.input = input_

        self.code_generator = code_generator or DEFAULT_CODE_GENERATOR

        code_gen_args_to_set = DEFAULT_CODE_GENERATOR_ARGS.copy()
        code_gen_args_to_set.update(code_generator_args)
        self.code_generator_args = code_gen_args_to_set

        self.state_equation = state_equation
        self.output_equation = output_equation

        self.initial_condition = initial_condition
        self.dt = dt

        self.validate()

    @property
    def state(self):
        return self._state

    @state.setter
    def state(self, state):
        if state is None:  # or other checks?
            state = empty_array()
        if isinstance(state, sp.Expr):
            state = Array([state])
        self.dim_state = len(state)
        self._state = state

    @property
    def input(self):
        return self._inputs

    @input.setter
    def input(self, input_):
        if input_ is None:  # or other checks?
            input_ = empty_array()
        if isinstance(input_, sp.Expr):  # check it's a single dynamicsymbol?
            input_ = Array([input_])
        self.dim_input = len(input_)
        self._inputs = input_

    @property
    def state_equation(self):
        return self._state_equation

    @state_equation.setter
    def state_equation(self, state_equation):
        if state_equation is None:  # or other checks?
            state_equation = empty_array()
        else:
            assert len(state_equation) == len(self.state)
            assert find_dynamicsymbols(state_equation) <= (
                    set(self.state) | set(self.input)
                )
            assert state_equation.atoms(sp.Symbol) <= (
                    set(self.constants_values.keys())
                    | set([dynamicsymbols._t])
                )

        self._state_equation = state_equation
        self.update_state_equation_function()

        self.state_jacobian_equation = grad(self.state_equation, self.state)
        self.update_state_jacobian_function()

        self.input_jacobian_equation = grad(self.state_equation, self.input)
        self.update_input_jacobian_function()

    @property
    def output_equation(self):
        return self._output_equation

    @output_equation.setter
    def output_equation(self, output_equation):
        if isinstance(output_equation, sp.Expr):
            output_equation = Array([output_equation])
        if output_equation is None and self.dim_state == 0:
            output_equation = empty_array()
        else:
            if output_equation is None:
                output_equation = self.state

            assert output_equation.atoms(sp.Symbol) <= (
                    set(self.constants_values.keys())
                    | set([dynamicsymbols._t])
                   )

            if self.dim_state:
                assert find_dynamicsymbols(output_equation) <= set(self.state) # or set(self.input) TODO: augment state to allow inputs in output equation if two systems (one stateless and one statefull) are placed in parallel.
            else:
                assert find_dynamicsymbols(output_equation) <= set(self.input)

        self.dim_output = len(output_equation)

        self._output_equation = output_equation
        self.update_output_equation_function()

    def update_state_equation_function(self):
        if not self.dim_state or self.state_equation == empty_array():
            return
        self.state_equation_function = self.code_generator(
            [dynamicsymbols._t] + sp.flatten(self.state) +
            sp.flatten(self.input),
            self.state_equation.subs(self.constants_values),
            **self.code_generator_args
        )

    def update_state_jacobian_function(self):
        if not self.dim_state or self.state_equation == empty_array():
            return
        self.state_jacobian_equation_function = self.code_generator(
            [dynamicsymbols._t] + sp.flatten(self.state) +
            sp.flatten(self.input),
            self.state_jacobian_equation.subs(self.constants_values),
            **self.code_generator_args
        )

    def update_input_jacobian_function(self):
        # TODO: state-less systems should have an input/output jacobian
        if not self.dim_state or self.state_equation == empty_array():
            return
        self.input_jacobian_equation_function = self.code_generator(
            [dynamicsymbols._t] + sp.flatten(self.state) +
            sp.flatten(self.input),
            self.input_jacobian_equation.subs(self.constants_values),
            **self.code_generator_args
        )

    def update_output_equation_function(self):
        if not self.dim_output or self.output_equation == empty_array():
            return
        if self.dim_state:
            self.output_equation_function = self.code_generator(
                [dynamicsymbols._t] + sp.flatten(self.state),
                self.output_equation.subs(self.constants_values),
                **self.code_generator_args
            )
        else:
            self.output_equation_function = self.code_generator(
                [dynamicsymbols._t] + sp.flatten(self.input),
                self.output_equation.subs(self.constants_values),
                **self.code_generator_args
            )

    def prepare_to_integrate(self):
        self.update_output_equation_function()
        self.update_state_equation_function()

    def copy(self):
        copy = self.__class__(
            state_equation=self.state_equation,
            state=self.state,
            input_=self.input,
            output_equation=self.output_equation,
            constants_values=self.constants_values,
            dt=self.dt
        )
        copy.output_equation_function = self.output_equation_function
        copy.state_equation_function = self.state_equation_function
        return copy

    def equilibrium_points(self, input_=None):
        return sp.solve(self.state_equation, self.state, dict=True)

class MemorylessSystem(DynamicalSystem):
    """
    A system with no state.

    With no input, can represent a signal (function of time only). For example,
    a stochastic signal could interpolate points and use prepare_to_integrate
    to re-seed the data.
    """
    def __init__(self, input_=None, output_equation=None, **kwargs):
        """
        DynamicalSystem constructor

        Parameters
        ----------
        input_ : array_like of sympy symbols
            Vector of symbols representing the components of the input, in the
            desired order. The output may depend on the system input.
        output_equation : array_like of sympy Expressions
            Vector valued expression for the output of the system.
        """
        super().__init__(
              input_=input_, output_equation=output_equation, **kwargs)

    @property
    def state(self):
        return self._state

    @state.setter
    def state(self, state):
        if state is None:  # or other checks?
            state = empty_array()
        else:
            raise ValueError("Memoryless system should not have state or " +
                             "state_equation")
        self.dim_state = len(state)
        self._state = state

max_stars_repo_name: jjuch/simupy

max_issues_repo_name: jjuch/simupy

max_forks_repo_name: jjuch/simupy

reason: import sympy,from sympy


text:
"""
Function are subprograms which are used to compute a value or perform a task.

Type of Functions:-

Built in Functions:
       print(), upper()

User define functions

Advantage of Functions

1. Write once and use it as many time as you need. This provides code reusability
2. Function facilitates ease of code maintenance
3. Divide Large task into many small task so it will help you to debug code
4. You can remove or add new feature to a function anytime.

"""

"""
We can define a function using def keyword followed by function name with parentheses. This is also called
as Creating a function, Writing a Function, Defining a FUnction.

Syntax:-
def function_name():
    Local Variable
    block of statement
    return(variable or expression)

def function_name(param1, param2, param3, .....)
    Local Variable
    Block of statement
    return (variable or expression)

Note - Nooed to mainitain a proper indentation
"""

# creating a list

def add():
    list = [8, 2, 3, 0, 7]

    total = 0;

    for i in range(0, len(list)):
        total = total + list[i]
    print('Sum of all elements in given list: ', total)

if __name__ == '__main__':
    add()
print()

# another method

def sum_list():
    mylist = [8, 2, 3, 0, 7]

    # Using inbuilt sum method
    total = sum(mylist)

    print("Sum of all elements in given list1: ", total)

if __name__ == '__main__':
    sum_list()
print()

def multiplylist():
    list_multiply = [8, 2, 3, -1, 7]
    total = 1;

    for x in list_multiply:
        total = total * x

    print(total)

if __name__ == '__main__':
    multiplylist()

# Method 2: Unsing numpy.prid() ^ Install numpy package

import numpy

def product_total():
    list_product = [8, 2, 3, -1, 7]

    total = numpy.prod(list_product)

    print("Another method using numpy method to find product in list: ", total)

product_total()

print()

def findingminmax(num1: int, num2: int, num3: int) -> int:
    max = 0;

    if (num1 > num2 and num1 > num2):
        max = num1
    elif (num2 > num1 and num2 > num3):
        max = num2
    else:
        max = num3

    print("The maximum number in given list is: ", max)

findingminmax(22, 26, 30)

print()
print("Another Method to find maximum")

def findingmaximum(num1: int, num2: int, num3: int) -> int:
    find_max_list = (num1, num2, num3)

    return max(find_max_list)

x = int(input("Enter your first Number: "))

y = int(input("Enter your second Number: "))

z = int(input("Enter your third Number: "))

print("Maximum number is ::>", findingmaximum(x, y, z))

"""Python program to print the even numbers from a given list"""

def find_even():
    sample_list = [1, 2, 3, 4, 5, 6, 7, 8, 9]

    for num in sample_list:

        if num % 2 == 0:
            print(num, end=" ")

find_even()

print()
"""
Pythhon program to find prime numbers in given list
Function should return true if the number is prime; else false
"""

def isPrime(num):

    if (num < 2):
        return True

    for i in range (2, num//2+1):
        if(num%i==0):
            return False
    return True

number =int(input("Enter the number you will like to check whether the number is prime or not: \n"))

if isPrime(number):
    print(number, "is a Prime Number")
else:
    print(number, "is not a Prime number")

"""
Another Method to find prime number
"""


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 27 08:52:36 2018

@author: weety
"""

import numpy as np
import matplotlib.pyplot as plt

PI = 3.1415926

# read data from file
file = open("sensor_data_delta2.txt","r")
list_arr = file.readlines()
l = len(list_arr)
for i in range(l):
    list_arr[i] = list_arr[i].split()
data = np.array(list_arr)
data = data.astype(float)
#print (data)
file.close()

acc = data[:, 0:3]
gyr = data[:, 3:6]

dt = 0.01

A = np.mat([[1, -dt], [0,1]])
B = np.mat([[dt], [0]])
Q = np.mat([[0.00001, 0], [0, 0.00003]])
R = np.mat([0.8])
H = np.mat([1,0])
I = np.mat(np.eye(2,2))

Gyro = gyr[:,0]
zk = np.arctan(acc[:,1]/np.sqrt(acc[:,0]**2 + acc[:,2]**2))

X=np.mat([[0], [0]])
P=np.mat([[1, 0], [0, 1]])
K=np.mat([[0], [0]])

data_len = len(acc)
data_range = range(data_len)

Angle = np.zeros(data_len)
Qbias = np.zeros(data_len)
Gyro_final = np.zeros(data_len)

#kalman filter
for i in data_range:
    #X(k|k-1) = A * X(k-1|k-1) + B*u(k)
    X = A * X + B * (Gyro[i] - X[1])
    #P(k|k-1) = A * P(k-1|k-1) * AT + Q
    P = A * P * A.T + Q
    #K(k) = P(k|k-1) * HT / (H * P(k|k-1) * HT + R)
    K = P * H.T * np.linalg.pinv(H * P * H.T + R)
    #e = Z(k)- H * X(k|k-1)
    #X(k|k) = X(k|k-1) + K * e = X(k|k-1) + K * (Z(k) - H * X(k|k-1))
    X = X + K * (zk[i] - H * X)
    #P(k|k) = P(k|k-1) - K * H * P(k|k-1) = (I - K * H) * P(k|k-1)
    P = (I - K * H) * P
    #Y(k) = H * X(k|k)
    Angle[i] = X[0]
    Qbias[i] = X[1]
    Gyro_final[i] = Gyro[i] - X[1]

#draw wave picture
plt.figure()
plt.plot(data_range, Angle*180/PI, label="Angle",color="red",linewidth=2)
plt.plot(data_range, zk*180/PI, "g--", label="Angle_acc")
plt.xlabel("Time(ms)")
plt.ylabel("Angle(°)")
plt.title("Angle filter")
plt.legend()
plt.show()
plt.figure()
plt.plot(data_range, Gyro_final, "b", label="Gyro_final")
plt.plot(data_range, Gyro, "r--", label="Gyro")
plt.plot(data_range, Qbias, "g--", label="Qbias")
plt.xlabel("Time(ms)")
plt.ylabel("Angular velocity(rad)")
plt.title("Angle velocity filter")
plt.legend()
plt.show()

max_stars_repo_name: weety/gorilla

max_issues_repo_name: weety/gorilla

max_forks_repo_name: weety/gorilla


text:
import numpy as np
import pytest

from forest_benchmarking import qubit_spectroscopy as qs

@pytest.fixture()
def mock_ramsey():
    return {'fixed_rz': 0.03,
            'tunable_rz': 0.51}

def test_sinusoidal_waveform(mock_ramsey):
    thetas = np.linspace(0, 2 * np.pi, 30)

    fixed_data = np.asarray([0.5 * np.sin(theta + mock_ramsey['fixed_rz']) + 0.5
                             for theta in thetas])
    params, params_errs = qs.fit_to_sinusoidal_waveform(thetas, fixed_data)
    assert np.isclose(params[3], mock_ramsey['fixed_rz'])

    tunable_data = np.asarray([0.5 * np.sin(theta + mock_ramsey['tunable_rz']) + 0.5
                               for theta in thetas])
    params, params_errs = qs.fit_to_sinusoidal_waveform(thetas, tunable_data)
    assert np.isclose(params[3], mock_ramsey['tunable_rz'])

max_stars_repo_name: msohaibalam/forest-benchmarking

max_issues_repo_name: msohaibalam/forest-benchmarking

max_forks_repo_name: msohaibalam/forest-benchmarking


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import sys
reload(sys)
sys.setdefaultencoding('utf8')
import numpy as np
import tensorflow as tf
from tensorflow.python.platform import gfile
from annotation import glove
from collections import defaultdict

embedding_dim = 300
path = os.path.abspath(__file__)
dir_path = os.path.dirname(path).replace('utils', 'data')
wiki_path = dir_path + '/DATA/wiki'
save_path = dir_path
'''
0: pad
1: bos
2: eos
'''
_PAD = 0
_GO = 1
_END = 2
_EOF = 3
_UNK = 4
_TOKEN_NUMBER = 5
_TOKEN_MODEL = 6
_EOC = 7
ori_files1 = [ 'train.lon', 'train.qu', 'test.lon', 'test.qu', 'dev.lon', 'dev.qu']
ori_files2 = [ 'train_vocab.lon', 'train_vocab.qu', 'test_vocab.lon', 'test_vocab.qu']
vocab_files = [ os.path.join(wiki_path, x) for x in ori_files1 ]
for subset in ['basketball', 'calendar', ...]:
    overnight_path = dir_path + '/DATA/overnight/%s'%subset
    vocab_files.extend([ os.path.join(overnight_path, x) for x in ori_files2 ])
annotation = ['<f0>','<f1>','<f2>','<f3>','<v0>','<v1>','<v2>','<v3>']
def build_vocab_all(load=True, files=vocab_files):
    if load==False:
        print('\nRebuild vocab...')
        vocab_tokens = ['<pad>','<bos>','<eos>','<eof>','<unk>','<@number>','<@model>','<eoc>']
        vocab_tokens.extend(annotation)
        vocabs = set()

        for fname in files:
            with gfile.GFile(fname, mode='r') as DATA:
                lines = DATA.readlines()
                for line in lines:
                    for word in line.split():
                        if word not in vocabs and word not in vocab_tokens:
                            vocabs.add(word)

        vocab_tokens.extend(list(vocabs))
        np.save(os.path.join(save_path,'vocab_tokens_all.npy'),vocab_tokens)
        print('\nBuild vocab done...')
        print('\nVocab size: %d'%len(vocab_tokens))
    else:
        vocab_tokens=np.load(os.path.join(save_path,'vocab_tokens_all.npy'))
        print('Vocab size:{0}'.format(len(vocab_tokens)))
    return vocab_tokens

def load_vocab_all(load=True):

    if load == False:
        print('\nRebuild vocab embedding...')
        vocab_dict = {}
        reverse_vocab_dict = {}
        embedding = glove.Glove()
        vocabs = build_vocab_all()
        vocab_tokens = []
        for i,word in enumerate(vocabs):
            vocab_dict[word]=i
            reverse_vocab_dict[i]=word.decode('utf-8')
            vocab_tokens.append([word.decode('utf-8')])
        np.save(os.path.join(save_path,'vocab_dict_all.npy'),vocab_dict)
        np.save(os.path.join(save_path,'reverse_vocab_dict_all.npy'),reverse_vocab_dict)
        vocab_emb, unk_idx = embedding.embedding(vocab_tokens, maxlen=1)
        unk_idx = np.asarray(unk_idx)
        vocab_emb = vocab_emb[:,0] #retrieve embedding
        train_idx = unk_idx
        #train_idx = np.concatenate( (np.arange(15), unk_idx) )
        np.save(os.path.join(save_path,'train_idx.npy'),train_idx)
        i = 0
        emb_dict = {}
        emb_dict['f'] = (np.random.rand(embedding_dim/2)-.5)*2*np.sqrt(3)
        emb_dict['v'] = (np.random.rand(embedding_dim/2)-.5)*2*np.sqrt(3)
        emb_dict['c'] = (np.random.rand(embedding_dim/2)-.5)*2*np.sqrt(3)
        for token,emb in zip(vocab_tokens,vocab_emb):
            token = token[0]
            if len(token)>=4 and token[0]=='<' and token[3]=='>' and token[2].isdigit():
                if token[2] in emb_dict:
                    right = emb_dict[token[2]]
                else:
                    emb_dict[token[2]] = (np.random.rand(embedding_dim/2)-.5)*2*np.sqrt(3)
                    right = emb_dict[token[2]]
                re = np.concatenate((emb_dict[token[1]],right))
                assert re.shape==(300,)
                vocab_emb[i]=re
            elif len(token)>=5 and token[0]=='<' and token[4]=='>' and token[2:4].isdigit():
                if token[2:4] in emb_dict:
                    right = emb_dict[token[2:4]]
                else:
                    emb_dict[token[2:4]] = (np.random.rand(embedding_dim/2)-.5)*2*np.sqrt(3)
                    right = emb_dict[token[2:4]]
                re = np.concatenate((emb_dict[token[1]],right))
                assert re.shape==(300,)
                vocab_emb[i]=re
            i += 1

        np.save(os.path.join(save_path,'vocab_emb_all.npy'),vocab_emb)
    else:
        vocab_emb=np.load(os.path.join(save_path,'vocab_emb_all.npy'))
        vocab_dict=np.load(os.path.join(save_path,'vocab_dict_all.npy')).item()
        reverse_vocab_dict=np.load(os.path.join(save_path,'reverse_vocab_dict_all.npy')).item()
        train_idx = np.load(os.path.join(save_path,'train_idx.npy'))
    return vocab_dict, reverse_vocab_dict, vocab_emb, train_idx

def load_data_wiki(maxlen=30, load=True, s='train'):
    if load:
        emb = np.load(os.path.join(save_path,'vocab_emb_all.npy'))
        all_q_tokens = np.load(os.path.join(save_path,'wiki/%s_qu_idx.npy'%(s)))
        all_logic_ids = np.load(os.path.join(save_path,'wiki/%s_lon_idx.npy'%(s)))
    else:
        all_q_tokens = []
        all_logic_ids = []
        vocab_dict,_,_,_=load_vocab_all()
        vocab_dict = defaultdict(lambda:_UNK, vocab_dict)
        questionFile=os.path.join(wiki_path,'%s.qu'%(s))
        logicFile=os.path.join(wiki_path,'%s.lon'%(s))
        with gfile.GFile(questionFile, mode='r') as questions, gfile.GFile(logicFile, mode='r') as logics:
            q_sentences = questions.readlines()
            logics = logics.readlines()
            assert len(q_sentences)==len(logics)
            i = 0
            length = len(logics)
            for q_sentence,logic in zip(q_sentences,logics):
                i+=1
                print('counting: %d / %d'%(i,length),end='\r')
                sys.stdout.flush()
                token_ids = [_GO]
                token_ids.extend([vocab_dict[x] for x in q_sentence.split()])
                for x in q_sentence.split():
                    if vocab_dict[x]==_UNK:
                        print('ERROR unknow word in question:'+x)
                #token_ids.append(_END)
                logic_ids = [_GO]
                logic_ids.extend([vocab_dict[x] for x in logic.split()])
                for x in logic.split():
                    if vocab_dict[x]==_UNK:
                        print('ERROR unknow word in logic:'+x)
                logic_ids.append(_END)
                if maxlen>len(logic_ids):
                    logic_ids.extend([ _PAD for i in range(len(logic_ids),maxlen)])
                else:
                    logic_ids = logic_ids[:maxlen]
                if maxlen>len(token_ids):
                    token_ids.extend([ _PAD for i in range(len(token_ids),maxlen)])
                else:
                    token_ids = token_ids[:maxlen]
                all_q_tokens.append(token_ids)
                all_logic_ids.append(logic_ids)
            all_logic_ids=np.asarray(all_logic_ids)
            all_q_tokens=np.asarray(all_q_tokens)
            np.save(os.path.join(save_path,'wiki/%s_lon_idx.npy'%s),all_logic_ids)
            np.save(os.path.join(save_path,'wiki/%s_qu_idx.npy'%s),all_q_tokens)

    return all_q_tokens,all_logic_ids

def load_data_overnight(maxlen=30, subset=subset, load=False, s='train'):
    overnight_path = dir_path + '/DATA/overnight/%s'%subset
    all_q_tokens = []
    all_logic_ids = []
    vocab_dict, _, _, _ = load_vocab_all()
    vocab_dict = defaultdict(lambda: _UNK, vocab_dict)
    questionFile = os.path.join(overnight_path, 'new_%s.qu'%(s))
    logicFile = os.path.join(overnight_path, 'new_%s.lon'%(s))
    with gfile.GFile(questionFile, mode='r') as questions, gfile.GFile(logicFile, mode='r') as logics:
        q_sentences = questions.readlines()
        logics = logics.readlines()
        assert len(q_sentences) == len(logics)
        for q_sentence, logic in zip(q_sentences, logics):
            token_ids = [_GO]
            token_ids.extend([vocab_dict[x] for x in q_sentence.split()])
            #token_ids.append(_END)
            for x in token_ids:
                if x == _UNK:
                    print('ERROR')
            logic_ids = [_GO]
            logic_ids.extend([vocab_dict[x] for x in logic.split()])
            logic_ids.append(_END)
            for x in logic_ids:
                if x == _UNK:
                    print('ERROR')
            if maxlen > len(logic_ids):
                logic_ids.extend([ _PAD for i in range(len(logic_ids),maxlen)])
            else:
                logic_ids = logic_ids[:maxlen]
            if maxlen > len(token_ids):
                token_ids.extend([ _PAD for i in range(len(token_ids),maxlen)])
            else:
                token_ids = token_ids[:maxlen]
            all_q_tokens.append(token_ids)
            all_logic_ids.append(logic_ids)
    all_logic_ids = np.asarray(all_logic_ids)
    print('--------overnight ' + s + ' shape---------')
    print(all_logic_ids.shape)
    all_q_tokens=np.asarray(all_q_tokens)
    return all_q_tokens, all_logic_ids

def load_data_one(maxlen=30, filename=''):
    vocab_dict,_,_,_=load_vocab_all()
    with open(filename, 'r') as f:
        qs = f.readlines()

    res = []
    for q_sentence in qs:
        q_sentence = q_sentence.lower()
        q_sentence = q_sentence.strip('\n')
        token_ids = [_GO]
        token_ids.extend([vocab_dict[x] for x in q_sentence.split()])
        if maxlen>len(token_ids):
            token_ids.extend([ _PAD for i in range(len(token_ids),maxlen)])
        else:
            token_ids = token_ids[:maxlen]
        res.append(token_ids)
    return res

def load_data_sent(maxlen=30, sent=''):
    vocab_dict,_,_,_=load_vocab_all()
	res = []
	q_sentence = sent
	q_sentence = q_sentence.lower()
	q_sentence = q_sentence.strip('\n')
	token_ids = [_GO]
	token_ids.extend([vocab_dict[x] for x in q_sentence.split()])
	if maxlen>len(token_ids):
		token_ids.extend([ _PAD for i in range(len(token_ids),maxlen)])
	else:
		token_ids = token_ids[:maxlen]
	res.append(token_ids)
    return res

def load_data(maxlen=30, load=False, s='train'):
    if s == 'test' or s == 'train' or s == 'dev':
        X, y = load_data_wiki(maxlen=maxlen,load=load,s=s)
        return X, y
    elif s=='overnight':
        X_all, y_all = None, None
        for subset in ['basketball','calendar',...]:
            print('======='+str(subset)+'=======')
            X1, y1 = load_data_overnight(maxlen=maxlen, subset=subset, load=load, s='train')
            X2, y2 = load_data_overnight(maxlen=maxlen, subset=subset, load=load, s='test')
            X = np.concatenate([X1,X2], axis=0)
            if X_all is not None:
                X_all = np.concatenate([X_all, X], axis=0)
            else:
                X_all = X

            y = np.concatenate([y1,y2], axis=0)
            if y_all is not None:
                y_all = np.concatenate([y_all,y], axis=0)
            else:
                y_all = y
        X, y = X_all, y_all
        print('========data '+s+' shape=======')
        print(X.shape)
        print(y.shape)
        return X, y
    else:
        lists = []
        for subset in ['basketball', 'calendar', ...]:
            X1, y1 = load_data_overnight(maxlen=maxlen, subset=subset, load=load, s='train')
            X2, y2 = load_data_overnight(maxlen=maxlen, subset=subset, load=load, s='test')
            X = np.concatenate([X1,X2], axis=0)
            y = np.concatenate([y1,y2], axis=0)
            lists.append((X, y))

        return lists

def rebuild(args):
    #build_vocab_all(load=False)
    #load_vocab_all(load=False)

    maxlen = args.maxlen
    load_data(maxlen=maxlen, load=False, s='train')
    load_data(maxlen=maxlen, load=False, s='test')
    load_data(maxlen=maxlen, load=False, s='dev')
    #load_data(maxlen=maxlen, load=False, s='overnight')

if __name__ == "__main__":
    rebuild = True
    if rebuild:
        #build_vocab_all(load=False)
        #load_vocab_all(load=False)
        build_vocab_all(load=True)
        load_vocab_all(load=True)
        maxlen = 60
        load_data(maxlen=maxlen, load=False, s='train')
        load_data(maxlen=maxlen, load=False, s='test')
        load_data(maxlen=maxlen, load=False, s='dev')
        #load_data(maxlen=maxlen, load=False, s='overnight')


text:
import logging

logger = logging.getLogger('root')

from copy import deepcopy
import numpy as np
import torch
from torch.autograd import Variable

from abp.adaptives.common.memory import Memory, Experience
from abp.utils import clear_summary_path
from abp.models import ActorModel
from tensorboardX import SummaryWriter

from baselines.common.schedules import LinearSchedule

# TODO Too many duplicate code. Need to refactor!

class PGAdaptive(object):
    """PGAdaptive using Vanilla Policy Gradient"""

    def __init__(self, name, choices, network_config, reinforce_config):
        super(PGAdaptive, self).__init__()
        self.name = name
        self.choices = choices
        self.network_config = network_config
        self.reinforce_config = reinforce_config
        self.update_frequency = reinforce_config.update_frequency

        self.replay_memory = Memory(self.reinforce_config.batch_size)

        self.steps = 0
        self.total_reward = 0

        self.previous_state = None
        self.previous_action = None
        self.clear_rewards()

        self.model = ActorModel(self.name + "_actor", self.network_config)
        self.summary = SummaryWriter(log_dir = self.reinforce_config.summaries_path + "/" + self.name)

        self.episode = 0
        self.epsilon_schedule = LinearSchedule(10 * 1000, initial_p = self.reinforce_config.starting_epsilon, final_p = 0.1)

    def __del__(self):
        self.summary.close()

    def predict(self, state):
        self.steps += 1

        if self.previous_state is not None and self.previous_action is not None:
            self.replay_memory.add((self.previous_state, self.previous_action, self.current_reward, state, False))

        _state = Variable(torch.Tensor(state)).unsqueeze(0)
        action_probs = self.model.predict(_state)

        #TODO continuous action
        m = Categorical(action_probs)
        action = m.sample()

        choice = self.choices[action]

        self.update()

        self.clear_rewards()

        self.previous_state = state
        self.previous_action = action

        return choice, q_values

    def disable_learning(self):
        logger.info("Disabled Learning for %s agent" % self.name)
        self.model.save_network()
        self.episode = 0

    def end_episode(self, state):
        if not self.learning:
            return

        logger.info("End of Episode %d with total reward %.2f" % (self.episode + 1, self.total_reward))

        self.episode += 1

        self.summary.add_scalar(tag='%s agent reward' % self.name,scalar_value=self.total_reward,
                                global_step=self.episode)

        self.replay_memory.add((self.previous_state, self.previous_action, self.reward_list(), state, True))

        self.clear_rewards()
        self.total_reward = 0

        self.previous_state = None
        self.previous_action = None

        self.update()

    def clear_rewards(self):
        self.current_reward = 0

    def reward(self, value):
        self.current_reward += value
        self.total_reward += value

    def update(self):
        if self.steps <= self.reinforce_config.batch_size:
            return

        states, actions, reward, next_states, is_terminal, weights, batch_idxes = self.replay_memory.sample(self.reinforce_config.batch_size,
                                                                                self.beta_schedule.value(self.steps))
        states = Variable(torch.Tensor(states))
        next_states = Variable(torch.Tensor(next_states))

        is_terminal = [0 if t else 1 for t in is_terminal]

        self.replay_memory.clear()

        self.model.fit(states, q_target, self.steps)

        return td_errors

max_stars_repo_name: LinearZoetrope/abp

max_issues_repo_name: LinearZoetrope/abp

max_forks_repo_name: Zaerei/abp


text:
import streamlit as st
import numpy as np
import pandas as pd
import calendar

import proj_pipeline

def analysis_search_dataframe(df, song_str='', artist_str='', album_str=''):
    if (song_str == '') & (artist_str == '') & (album_str == ''):
        return st.error('Please search for song/artist/album in data')
    elif (song_str != '') and (artist_str != ''):
        return df[(df['title'] == song_str) & (df['artist'] == artist_str)]
    elif (song_str != '') and (album_str != ''):
        return df[(df['title'] == song_str) & (df['album'] == album_str)]
    elif (album_str != '') and (artist_str != ''):
        return df[(df['album'] == album_str) & (df['artist'] == artist_str)]
    elif song_str != '':
        return df[df['title'] == song_str]
    elif artist_str != '':
        return df[df['artist'] == artist_str]
    elif album_str != '':
        return df[df['album'] == album_str]
    else:
        return df

def analysis_filter_dataframe(df, playlist_list, year_range):
    if (playlist_list == []) & (year_range == (df['artist_date'].min().year, df['artist_date'].max().year)):
        return df

    elif (playlist_list != []) & (year_range != (df['artist_date'].min().year, df['artist_date'].max().year)):
        return df[(df['playlist'].isin(playlist_list)) & (df['artist_date'].astype(str).str[:4].astype(int).isin(range(year_range[0], year_range[1]+1)))].reset_index(drop=True)

    elif playlist_list != []:
        return df[df['playlist'].isin(playlist_list)].reset_index(drop=True)

    elif year_range != (df['artist_date'].min().year, df['artist_date'].max().year):
        return df[df['artist_date'].astype(str).str[:4].astype(int).isin(range(year_range[0], year_range[1]+1))].reset_index(drop=True)

    else:
        return "INPUT ERROR: Cannot return desired intersection of input fields."