text:
import pandas as pd
import numpy as np
import librosa
from scipy.io import loadmat
import os

'''
The function `make_ground` creates the ground truth (midi like) file for each audio file.

The functions `RCA` and `RPA` calculate the Raw Chroma Accuracy an the Raw Pitch Accuracy using the output of NMF and the ground truth. These functions are not used in the `make_groud` function, rather are imported separately in a jupyter notebook.
'''

def RCA():
    n=N
    acc = 0
    for i in range(N):
        min = np.max
        pr = np.nonzero(predict[:,i])[0]

        if gl[i] == 0:
            n-=1
            continue
        if pr.size==0:
            continue

        x = np.min(abs(pr - gl[i]))

        if x == 0:
            acc+=1
    return acc/n

def RPA():
    n=N
    acc = 0
    for i in range(N):
        min = np.max
        pr = np.nonzero(predict[:,i])[0]

        if gl[i] == 0:
            n-=1
            continue
        if pr.size==0:
            continue

        x = np.min(abs(pr - gl[i]))

        if x == 0:
            acc+=1
    return acc/n

def find_nearest(array, value):
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return array[idx]

def uf(u,Fmin,Ust):
    return Fmin * np.power(2,(u)/(12*Ust))

def make_ground():

    # ground_path = '/mnt/Stuff/Acads/UGP/mycode/ground_voc'
    # ground_list = [x.split('.')[0].strip() for x in os.listdir(ground_path)]

    #os.chdir("./ground_voc")

    Fmin = librosa.note_to_hz('C2')
    Fmax = librosa.note_to_hz('C7')

    note = uf(np.arange(int(np.log2(Fmax/Fmin) * 12*1 +1)) , Fmin,1)

    # melody2 = "/mnt/Stuff/Acads/UGP/medleydb/medleydb/data/Annotations/Melody/\
    #             Melody2/"

    # mirk = "/mnt/data/datasets/MIR-1K/"

    bach = "/mnt/data/datasets/Bach10_v1.1/"

    # for song in os.listdir(melody2):
    #     print(song.rsplit(".")[0][:-8])
    #     path = melody2 + song
    #     liz = pd.read_csv(path,names = ['time','freq'])
    #     liz = liz.to_numpy()
    #
    #     #print(liz.shape[0])
    #
    #     N = int(liz.shape[0]/2 +1)
    #     #print(N)
    #
    #     i=0
    #     #ground_liz = np.zeros([61,N])
    #     gl = np.zeros(N)
    #     for x in liz:
    #         if i%2==0:
    #             gl[int(i/2)] = np.argwhere(note == find_nearest(note,x[1]))
    #         #ground_liz[int(gl[int(i/2)]) , int(i/2)] = 1
    #         i+=1
    #     save_path = song.rsplit(".")[0][:-8] + ".npy"
    #     np.save(save_path , gl)
    #     print(" Done.")

    # for file in os.listdir(mirk):
    #     if file.split('.')[1] == 'pv':
    #         path = mirk + file
    #         liz = []
    #         ff = open(path , 'r')
    #         freq = [float(x) for x in ff]
    #
    #         N = len(freq)
    #
    #         print(N)
    #
    #         i = 0
    #         gl = np.zeros(N)
    #         for x in freq:
    #             gl[i] = round(x) #np.argwhere(note == find_nearest(note , x))
    #             i+=1
    #         save_path = file.split('.')[0] + '.npy'
    #         np.save(save_path , gl)
    #         print('Done.')

    for song in os.listdir(bach):
        print(song)
        file = bach + song + '/' + song + '-GTF0s.mat'
        f = loadmat(file)
        f = f['GTF0s']
        i=0
        for ch in ['violin', 'clarinet','saxophone','bassoon']:
            fr = f[i]
            i+=1
            N = fr.shape[0]
            gl = np.zeros(N)
            j=0
            for x in fr:
                gl[j] = x - 36 +1
                j+=1
            save_path = 'ground_bach/' + song  + '-' + ch + '.npy'
            np.save(save_path , gl)
            print('Done.')

if __name__ == "__main__":
    make_ground()


text:
import pandas as pd
import numpy as np
import os
from bs4 import BeautifulSoup
from selenium import webdriver

# Pages 2 use: Investing and markets insider
url_mktIns = "https://markets.businessinsider.com/commodities/oil-price?type=wti"

driver = webdriver.Chrome(os.path.abspath('chromedriver.exe'))
driver.get("https://www.investing.com/commodities/crude-oil-news")

art_d = []

page_source = BeautifulSoup(driver.page_source, "lxml")
sec = page_source.find("section")
div = sec.find("div", {'class': 'mediumTitle1'})
for tag in div.contents:
    if str(type(tag)) == "<class 'bs4.element.Tag'>":
        art_d.append(tag)
art_d.pop(-1)
for art in art_d:
    # print(f"Len: {len(art.attrs)}")
    # print("Text:")
    t_s_n = art.text.split('\n')
    # print(t_s_n)
    t_s_n = [s1 for s1 in t_s_n if s1 != '']
    # print(t_s_n)
    # print(f"Len: {len(t_s_n)}")
    if len(t_s_n) == 3:
        t_s_n[1] = t_s_n[1].split('\xa0-\xa0')
        t_s_n[1] = t_s_n[1][0] + ", " + t_s_n[1][1]
        print(f"Title: {t_s_n[0]}")
        print(f"Provider: {t_s_n[1]}")
        print(f"Abstract: {t_s_n[2]}")
    else:
        t_s_n[2] = t_s_n[2].split('\xa0-\xa0')
        t_s_n[2] = t_s_n[2][0] + ", " + t_s_n[2][1]
        print(f"Title: {t_s_n[1]}")
        print(f"Provider: {t_s_n[2]}")
        if len(t_s_n) == 6:
            print(f"Abstract: {t_s_n[5]}")
        else:
            print(f"Abstract: {t_s_n[4]}")

    if len(art.attrs) == 5:
        print(f"Link: {art.attrs['data-link']}")
    else:
        inv = "https://investing.com"
        print(f"Link: {inv + art.contents[1].attrs['href']}")
    print("-------------")
driver.close()
driver.quit()

ext: py

reason: import numpy


text:
import numpy as np

def lsq(X, y):
    """
    Least squares linear regression
    :param X: Input data matrix
    :param y: Target vector
    :return: Estimated coefficient vector for the linear regression
    """

    # add column of ones for the intercept
    ones = np.ones((len(X), 1))
    X = np.concatenate((ones, X), axis=1)

    # calculate the coefficients
    beta = np.dot(np.linalg.inv(np.dot(X.T, X)), np.dot(X.T, y))

    return beta

def predict(X, y, beta):
    """
    Predict target scores using multi linear regression
    :param X: Input data matrix
    :param y: Target vector
    :param beta: Estimated coefficient vector for the linear regression
    :return: MSE
    """

    # add colum of ones for intercept
    # same as in the least squares
    ones = np.ones((len(X), 1))
    X = np.concatenate((ones, X), axis=1)

    # predict target values
    predict = np.dot(X,beta)

    return predict

def MSEevaluation(y, predict):
    """
    Evaluate predictions the mean sqaured error
    :param y: original values
    :param predict: predicted values
    :return: the mean squared error of the predicted values
    """

    n = len(y)                                                                  # number of samples
    square = 0
    for i in range(0, n-1):
        square += (y[i]- predict[i])**2

    MSE = square/n

    return MSE


text:
import basix
import numpy
import pytest

def xtest_create_simple():
    # Creates Lagrange P1 element on triangle

    # Point evaluation of polynomial set
    degree = 1
    points = numpy.array([[0, 0], [1, 0], [0, 1]], dtype=numpy.float64)
    matrix = numpy.identity(points.shape[0])

    # Create element from space and dual
    coeff_space = numpy.identity(points.shape[0])
    fe = basix.create_new_element("Custom element", "triangle", degree, [1], points, matrix, coeff_space,
                                  [[1, 1, 1], [0, 0, 0], [0]], [numpy.identity(3) for i in range(3)],
                                  basix.MappingType.identity)
    numpy.set_printoptions(suppress=True, precision=2)
    points = numpy.array([[.5, 0], [0, .5], [.5, .5]], dtype=numpy.float64)
    print(fe.tabulate(0, points))

def xtest_create_custom():
    # Creates second order element on triangle

    # Point evaluation of polynomial set
    degree = 2
    points = numpy.array([[0, .5], [0.5, 0], [0.5, 0.5], [0.25, 0.25], [0.25, 0.5], [0.5, 0.25]], dtype=numpy.float64)
    matrix = numpy.identity(points.shape[0])

    # Create element from space and dual
    coeff_space = numpy.identity(points.shape[0])
    fe = basix.create_new_element("Custom element", "triangle", degree, [1], points, matrix, coeff_space,
                                  [[0, 0, 0], [1, 1, 1], [3]],
                                  [numpy.identity(5) for i in range(3)],
                                  basix.MappingType.identity)
    numpy.set_printoptions(suppress=True, precision=2)
    points = numpy.array([[.25, 0], [0, .25], [.25, .25]], dtype=numpy.float64)
    print(fe.tabulate(0, points))

def xtest_create_invalid():
    degree = 2
    # Try to create an invalid element of order 2
    points = numpy.array([[0, 0.25], [0, 0.75], [0.25, 0.75], [0.75, 0.25],
                          [0.25, 0.0], [0.75, 0.0]], dtype=numpy.float64)
    matrix = numpy.identity(points.shape[0])

    # Create element from space and dual
    coeff_space = numpy.identity(points.shape[0])
    with pytest.raises(RuntimeError):
        basix.create_new_element("Custom element", "triangle", degree, [1], points, matrix, coeff_space,
                                 [[0, 0, 0], [2, 2, 2], [0]],
                                 [numpy.identity(6) for i in range(3)], basix.MappingType.identity)

max_stars_repo_name: nikhilTkur/basix

max_issues_repo_name: nikhilTkur/basix

max_forks_repo_name: nikhilTkur/basix


text:
import pandas as pd
import numpy as np
from sklearn.preprocessing import PowerTransformer
from sklearn.covariance import MinCovDet
from scipy.stats import chi2

def add_is_outlier_IQR(data, col_name):
    col_values = data[col_name]

    Q1=col_values.quantile(0.25)
    Q3=col_values.quantile(0.75)
    IQR=Q3-Q1

    outliers_col_name = f'is_{col_name.replace(" ", "_")}_outlier'
    data[outliers_col_name] = ((col_values < (Q1 - 1.5 * IQR)) | (col_values > (Q3 + 1.5 * IQR)))

    return data

def yeo_johnson_transf(data):
    pt = PowerTransformer(method='yeo-johnson', standardize=True)
    pt.fit(data)

    lambdas = pt.lambdas_
    df_yeojohnson = pd.DataFrame( pt.transform(dataset[numeric_col_names]), columns=numeric_col_names )

    return df_yeojohnson, lambdas

# Get numeric column names but the quality
numeric_col_names = dataset.drop('quality', axis=1).columns.values

# As you see there are outliers, let's add a boolean
# column to the dataframeindicating which row
# has a sulphate outlier
add_is_outlier_IQR(dataset, 'sulphates')

# Let's plot the boxplot removing the initial outliers
df_no_outliers = dataset.loc[~dataset['is_sulphates_outlier']]

# Let's apply Yeo-Johnson transformations
# in order to remove skewness
df_transf, lambda_arr = yeo_johnson_transf(dataset[numeric_col_names])

# Let's compute the squared Mahalanobis distances using
# the Minimum Covariance Determinant to calculate a
# robust covariance matrix
robust_cov = MinCovDet(support_fraction=0.7).fit(df_transf)
center = robust_cov.location_

D = robust_cov.mahalanobis(df_transf - center)

# %%
# The squared Mahalanobis distance (D) follows a Chi-Square distribution
# (https://markusthill.github.io/mahalanbis-chi-squared/#the-squared-mahalanobis-distance-follows-a-chi-square-distribution-more-formal-derivation)
#
# Given a cutoff value associated with the statistical significance
# with which we want to determine outliers, we obtain the corresponding
# threshold value above which to consider an observation an outlier
cutoff = 0.98
degrees_of_freedom = df_transf.shape[1]  # given by the number of variables (columns)
cut = chi2.ppf(cutoff, degrees_of_freedom) # threshold value

# Calculate the probability that the distance D[5]
# is an outlier
chi2.cdf(D[5], degrees_of_freedom)

# Calulate if the observation is an outlier given the cutoff
is_outlier_arr = (D > cut)

# Calculate the probability that an observation is an outlier not by chance
outliers_stat_proba = np.zeros(len(is_outlier_arr))

for i in range(len(is_outlier_arr)):
    outliers_stat_proba[i] = chi2.cdf(D[i], degrees_of_freedom)

# Adding outliers info to the dataframe according to
# the squared Mahalanobis distance
dataset['is_mahalanobis_outlier'] = is_outlier_arr
dataset['mahalanobis_outlier_proba'] = outliers_stat_proba

reason: import numpy,from scipy


text:
# Copyright 2019 Yan Yan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from spconv_lite import spconv_utils_cpp

def points_to_voxel(points,
                    voxel_size,
                    coors_range,
                    coor_to_voxelidx,
                    max_points=35,
                    max_voxels=20000,
                    full_mean=False,
                    block_filtering=True,
                    block_factor=1,
                    block_size=8,
                    height_threshold=0.2,
                    height_high_threshold=3.0,
                    pad_output=False):
    """convert 3d points(N, >=3) to voxels. This version calculate
    everything in one loop. now it takes only 0.8ms(~6k voxels)
    with c++ and 3.2ghz cpu.

    Args:
        points: [N, ndim] float tensor. points[:, :3] contain xyz points and
            points[:, 3:] contain other information such as reflectivity.
        voxel_size: [3] list/tuple or array, float. xyz, indicate voxel size
        coors_range: [6] list/tuple or array, float. indicate voxel range.
            format: xyzxyz, minmax
        coor_to_voxelidx: int array. used as a dense map.
        max_points: int. indicate maximum points contained in a voxel.
        max_voxels: int. indicate maximum voxels this function create.
            for voxelnet, 20000 is a good choice. you should shuffle points
            before call this function because max_voxels may drop some points.
        full_mean: bool. if true, all empty points in voxel will be filled with mean
            of exist points.
        block_filtering: filter voxels by height. used for lidar point cloud.
            use some visualization tool to see filtered result.
    Returns:
        voxels: [M, max_points, ndim] float tensor. only contain points.
        coordinates: [M, 3] int32 tensor. zyx format.
        num_points_per_voxel: [M] int32 tensor.
    """
    if full_mean:
        assert block_filtering is False
    if not isinstance(voxel_size, np.ndarray):
        voxel_size = np.array(voxel_size, dtype=points.dtype)
    if not isinstance(coors_range, np.ndarray):
        coors_range = np.array(coors_range, dtype=points.dtype)
    voxelmap_shape = (coors_range[3:] - coors_range[:3]) / voxel_size
    voxelmap_shape = tuple(np.round(voxelmap_shape).astype(np.int32).tolist())
    voxelmap_shape = voxelmap_shape[::-1]
    num_points_per_voxel = np.zeros(shape=(max_voxels, ), dtype=np.int32)
    voxels = np.zeros(shape=(max_voxels, max_points, points.shape[-1]),
                      dtype=points.dtype)
    voxel_point_mask = np.zeros(shape=(max_voxels, max_points),
                                dtype=points.dtype)
    coors = np.zeros(shape=(max_voxels, 3), dtype=np.int32)
    res = {
        "voxels": voxels,
        "coordinates": coors,
        "num_points_per_voxel": num_points_per_voxel,
        "voxel_point_mask": voxel_point_mask,
    }
    if full_mean:
        # means = np.zeros(shape=(max_voxels, points.shape[-1]),
        #                  dtype=points.dtype)
        # voxel_num = points_to_voxel_3d_np_mean(points, voxels,
        #                                        voxel_point_mask, means, coors,
        #                                        num_points_per_voxel,
        #                                        coor_to_voxelidx,
        #                                        voxel_size.tolist(),
        #                                        coors_range.tolist(),
        #                                        max_points, max_voxels)
        raise NotImplementedError
    else:
        if block_filtering:
            # block_shape = [*voxelmap_shape[1:]]
            # block_shape = [b // block_factor for b in block_shape]
            # mins = np.full(block_shape, 99999999, dtype=points.dtype)
            # maxs = np.full(block_shape, -99999999, dtype=points.dtype)
            # voxel_mask = np.zeros((max_voxels, ), dtype=np.int32)
            # voxel_num = points_to_voxel_3d_with_filtering(
            #     points, voxels, voxel_point_mask, voxel_mask, mins, maxs,
            #     coors, num_points_per_voxel, coor_to_voxelidx,
            #     voxel_size.tolist(), coors_range.tolist(), max_points,
            #     max_voxels, block_factor, block_size, height_threshold,
            #     height_high_threshold)
            # voxel_mask = voxel_mask.astype(np.bool_)
            # coors_ = coors[voxel_mask]
            # if pad_output:
            #     res["coordinates"][:voxel_num] = coors_
            #     res["voxels"][:voxel_num] = voxels[voxel_mask]
            #     res["voxel_point_mask"][:voxel_num] = voxel_point_mask[
            #         voxel_mask]
            #
            #     res["num_points_per_voxel"][:voxel_num] = num_points_per_voxel[
            #         voxel_mask]
            #     res["coordinates"][voxel_num:] = 0
            #     res["voxels"][voxel_num:] = 0
            #     res["num_points_per_voxel"][voxel_num:] = 0
            #     res["voxel_point_mask"][voxel_num:] = 0
            # else:
            #     res["coordinates"] = coors_
            #     res["voxels"] = voxels[voxel_mask]
            #     res["num_points_per_voxel"] = num_points_per_voxel[voxel_mask]
            #     res["voxel_point_mask"] = voxel_point_mask[voxel_mask]
            # voxel_num = coors_.shape[0]
            raise NotImplementedError
        else:
            voxel_num = spconv_utils_cpp.points_to_voxel_3d_np(points,                #
                                                            voxels,                #
                                                            voxel_point_mask,      #
                                                            coors,                 #
                                                            num_points_per_voxel,  #
                                                            coor_to_voxelidx,      #
                                                            voxel_size.tolist(),   #
                                                            coors_range.tolist(),  #
                                                            max_points,            #
                                                            max_voxels)            #
    res["voxel_num"] = voxel_num
    res["voxel_point_mask"] = res["voxel_point_mask"].reshape(
        -1, max_points, 1)
    return res

class VoxelGenerator:
    def __init__(self,
                 voxel_size,
                 point_cloud_range,
                 max_num_points,
                 max_voxels=20000,
                 full_mean=True):
        point_cloud_range = np.array(point_cloud_range, dtype=np.float32)
        # [0, -40, -3, 70.4, 40, 1]
        voxel_size = np.array(voxel_size, dtype=np.float32)
        grid_size = (point_cloud_range[3:] -
                     point_cloud_range[:3]) / voxel_size
        grid_size = np.round(grid_size).astype(np.int64)
        voxelmap_shape = tuple(np.round(grid_size).astype(np.int32).tolist())
        voxelmap_shape = voxelmap_shape[::-1]

        self._coor_to_voxelidx = np.full(voxelmap_shape, -1, dtype=np.int32)
        self._voxel_size = voxel_size
        self._point_cloud_range = point_cloud_range
        self._max_num_points = max_num_points
        self._max_voxels = max_voxels
        self._grid_size = grid_size
        self._full_mean = full_mean

    def generate(self, points, max_voxels=None):
        res = points_to_voxel(points, self._voxel_size,
                              self._point_cloud_range, self._coor_to_voxelidx,
                              self._max_num_points, max_voxels
                              or self._max_voxels, self._full_mean)
        voxels = res["voxels"]
        coors = res["coordinates"]
        num_points_per_voxel = res["num_points_per_voxel"]
        voxel_num = res["voxel_num"]
        coors = coors[:voxel_num]
        voxels = voxels[:voxel_num]
        num_points_per_voxel = num_points_per_voxel[:voxel_num]

        return voxels, coors, num_points_per_voxel

    def generate_multi_gpu(self, points, max_voxels=None):
        res = points_to_voxel(points, self._voxel_size,
                              self._point_cloud_range, self._coor_to_voxelidx,
                              self._max_num_points, max_voxels
                              or self._max_voxels, self._full_mean)
        voxels = res["voxels"]
        coors = res["coordinates"]
        num_points_per_voxel = res["num_points_per_voxel"]
        voxel_num = res["voxel_num"]
        return voxels, coors, num_points_per_voxel

    @property
    def voxel_size(self):
        return self._voxel_size

    @property
    def max_num_points_per_voxel(self):
        return self._max_num_points

    @property
    def point_cloud_range(self):
        return self._point_cloud_range

    @property
    def grid_size(self):
        return self._grid_size

class VoxelGeneratorV2:
    def __init__(self,
                 voxel_size,
                 point_cloud_range,
                 max_num_points,
                 max_voxels=20000,
                 full_mean=False,
                 block_filtering=False,
                 block_factor=8,
                 block_size=3,
                 height_threshold=0.1,
                 height_high_threshold=2.0):
        assert full_mean is False, "don't use this."
        point_cloud_range = np.array(point_cloud_range, dtype=np.float32)
        # [0, -40, -3, 70.4, 40, 1]
        voxel_size = np.array(voxel_size, dtype=np.float32)
        grid_size = (point_cloud_range[3:] -
                     point_cloud_range[:3]) / voxel_size
        grid_size = np.round(grid_size).astype(np.int64)
        if block_filtering:
            assert block_size > 0
            assert grid_size[0] % block_factor == 0
            assert grid_size[1] % block_factor == 0

        voxelmap_shape = tuple(np.round(grid_size).astype(np.int32).tolist())
        voxelmap_shape = voxelmap_shape[::-1]
        self._coor_to_voxelidx = np.full(voxelmap_shape, -1, dtype=np.int32)
        self._voxel_size = voxel_size
        self._point_cloud_range = point_cloud_range
        self._max_num_points = max_num_points
        self._max_voxels = max_voxels
        self._grid_size = grid_size
        self._full_mean = full_mean
        self._block_filtering = block_filtering
        self._block_factor = block_factor
        self._height_threshold = height_threshold
        self._block_size = block_size
        self._height_high_threshold = height_high_threshold

    def generate(self, points, max_voxels=None):
        res = points_to_voxel(points, self._voxel_size,
                              self._point_cloud_range, self._coor_to_voxelidx,
                              self._max_num_points, max_voxels
                              or self._max_voxels, self._full_mean,
                              self._block_filtering, self._block_factor,
                              self._block_size, self._height_threshold,
                              self._height_high_threshold)
        for k, v in res.items():
            if k != "voxel_num":
                res[k] = v[:res["voxel_num"]]
        return res

    def generate_multi_gpu(self, points, max_voxels=None):
        res = points_to_voxel(points,
                              self._voxel_size,
                              self._point_cloud_range,
                              self._coor_to_voxelidx,
                              self._max_num_points,
                              max_voxels or self._max_voxels,
                              self._full_mean,
                              self._block_filtering,
                              self._block_factor,
                              self._block_size,
                              self._height_threshold,
                              self._height_high_threshold,
                              pad_output=True)
        return res

    @property
    def voxel_size(self):
        return self._voxel_size

    @property
    def max_num_points_per_voxel(self):
        return self._max_num_points

    @property
    def point_cloud_range(self):
        return self._point_cloud_range

    @property
    def grid_size(self):
        return self._grid_size


text:
import ConfigSpace as CS
import numpy as np
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import Matern
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

from pyPDP.surrogate_models import SurrogateModel

class SkLearnPipelineSurrogate(SurrogateModel):
    def __init__(self, pipeline: Pipeline, cs: CS.ConfigurationSpace, seed=None):
        super().__init__(cs, seed=seed)
        self.pipeline = pipeline

    def _fit(self, X: np.ndarray, y: np.ndarray):
        self.pipeline.fit(X, y)

    def predict(self, X: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
        """
        Calculate mean and sigma of predictions

        :param X: input data shaped [NxK] for N samples and K parameters
        :return: means, sigmas
        """

        return self.pipeline.predict(X, return_std=True)

class GaussianProcessSurrogate(SkLearnPipelineSurrogate):
    def __init__(self, cs: CS.ConfigurationSpace, kernel=Matern(nu=1.5), seed=None):
        pipeline = Pipeline([
            ("standardize", StandardScaler()),
            ("GP", GaussianProcessRegressor(kernel=kernel, normalize_y=True,
                                            n_restarts_optimizer=20,
                                            alpha=1e-8,
                                            random_state=seed)),
        ])
        super().__init__(pipeline, cs, seed=seed)

max_stars_repo_name: dwoiwode/py-pdp-partitioner

max_issues_repo_name: dwoiwode/py-pdp-partitioner

max_forks_repo_name: dwoiwode/py-pdp-partitioner


text:
import numpy as np
import torch

def unmask(tensor, mask, cut_length=0):
    """
    Unmask a tensor and convert it back to a list of lists.
    Args:
        tensor (torch.Tensor): tensor with shape (bs, max_len, ...)
        mask (torch.Tensor): tensor with shape (bs, max_len) where 1 (or True)
            indicates a valid position, and 0 (or False) otherwise
        cut_length (int): remove the last `cut_length` elements from the tensor.
            In practice, the lengths calculated from the mask are going to be
            subtracted by `cut_length`. This is useful when you have <bos> and
            <eos> tokens in your words field and the mask was computed with
            words != <pad>. Default is 0, i.e., no cut
    Returns:
         a list of lists with variable length
    """
    lengths = mask.int().sum(dim=-1)
    if cut_length > 0:
        lengths -= cut_length
    lengths = lengths.tolist()
    return [x[:lengths[i]].tolist() for i, x in enumerate(tensor)]

def unroll(list_of_lists, rec=False):
    """
    Unroll a list of lists
    Args:
        list_of_lists (list): a list that contains lists
        rec (bool): unroll recursively
    Returns:
        a single list
    """
    if not isinstance(list_of_lists[0], (np.ndarray, list)):
        return list_of_lists
    new_list = [item for l in list_of_lists for item in l]
    if rec and isinstance(new_list[0], (np.ndarray, list)):
        return unroll(new_list, rec=rec)
    return new_list

def freeze_all_module_params(module):
    """
    Set requires_grad of all params of a given module to False
    Args:
        module (torch.nn.Moduke): a torch.nn's module
    """
    for param in module.parameters():
        param.required_grad = False

max_stars_repo_name: deep-spin/spec-blackboxnlp

max_issues_repo_name: deep-spin/spec-blackboxnlp

max_forks_repo_name: deep-spin/spec-blackboxnlp


text:
# coding=utf-8
import numpy as np
import Putil.base.logger as plog

plog.PutilLogConfig.config_log_level(stream=plog.DEBUG)
plog.PutilLogConfig.config_handler(plog.stream_method)
logger = plog.PutilLogConfig('TesCommonData').logger()
logger.setLevel(plog.DEBUG)
MainLogger = logger.getChild('Main')
MainLogger.setLevel(plog.DEBUG)

import Putil.data.common_data as pcd
import multiprocessing
from Putil.data import aug
AugFuncNoOp = aug.AugFuncNoOp
AugNode = aug.AugNode

class TestCommonData(pcd.CommonDataWithAug):
    def __init__(self):
        pcd.CommonDataWithAug.__init__(self)
        self._data_field = list(range(0, 100))
        pass

    def _restart_process(self, restart_param):
        pass

    def _generate_from_origin_index(self, index):
        data = self._data_field[index]
        return np.array([data]), np.array([data])

    def _data_set_field(self):
        return list(range(0, 100))

    def _inject_operation(self, inject_param):
        pass
    pass

pcd.CommonDataManager.register('TestCommonData', TestCommonData)

import Putil.data.aug as PAug
class Aug(PAug.AugFunc):
    def __init__(self, eps):
        PAug.AugFunc.__init__(self)
        def g(*args):
            return args[0] + 1, args[1]
        self._func = g
        pass

    def _generate_name(self):
        return 'gain'
        pass

    def _generate_name(self):
        return 'gain'

    def _generate_doc(self):
        return 'gain one'
    pass
    pass

if __name__ == '__main__':
    # manager_common_data = pcd.CommonDataManager()
    # manager_common_data.start()
    # data = manager_common_data.TestCommonData()

    data = TestCommonData()

    root_aug = AugNode(AugFuncNoOp())
    root_aug.add_child(AugNode(AugFuncNoOp()))
    root_aug.freeze_node()

    data.set_aug_node_root(root_aug)

    restart_param = dict()

    test_time = 10
    for i in range(0, test_time):
        restart_param['device_batch'] = [1]
        restart_param['critical_process'] = 'random_fill'

        data.restart_data(restart_param)

        assert data.generate_epoch_done() is False
        count = 0
        while data.generate_epoch_done() is False:
            got_data = data.generate_data()
            assert len(got_data) == 1, print(len(data))
            for d in got_data:
                for v in d:
                    get = v
                    assert get.datas().shape == (1, 1)
                    assert get.indexs().shape == (1,)
                    assert get.indexs()[0].index_info().type() == 'normal'
                    assert get.indexs()[0].data_range() == [0, 1]
            count += 1
            pass
        assert count == 100

        restart_param['device_batch'] = [11]
        restart_param['critical_process'] = 'random_fill'

        data.restart_data(restart_param)

        assert data.generate_epoch_done() is False
        count = 0
        while data.generate_epoch_done() is False:
            got_data = data.generate_data()
            assert len(got_data) == 1
            for d in got_data:
                for v in d:
                    get = v
                    assert get.datas().shape == (11, 1)
                    assert get.indexs().shape == (11,)
            count += 1
            pass
        assert count == 10

        restart_param['device_batch'] = [3, 5]
        restart_param['critical_process'] = 'random_fill'

        data.restart_data(restart_param)

        assert data.generate_epoch_done() is False
        count = 0
        while data.generate_epoch_done() is False:
            got_data = data.generate_data()
            assert len(got_data) == 2
            for index, d in enumerate(got_data):
                for v in d:
                    get = v
                    assert get.datas().shape == (restart_param['device_batch'][index], 1), print(get.datas().shape)
                    assert get.indexs().shape == (restart_param['device_batch'][index],)
                    if count == 12 and index == 1:
                        assert get.indexs()[2].index_info().type() == 'random_fill'
                        assert get.indexs()[3].index_info().type() == 'random_fill'
                        assert get.indexs()[4].index_info().type() == 'random_fill'
                        pass
                    elif count == 12 and index == 0:
                        assert get.indexs()[2].index_info().type() == 'random_fill'
                        pass
                    else:
                        assert get.indexs()[-1].index_info().type() == 'normal', print(get.indexs()[-1].index_info().type(), count)
                        assert get.indexs()[-1].index_info().type() == 'normal'
                        pass
            count += 1
            pass
        assert count == 13

        restart_param['device_batch'] = [3]
        restart_param['critical_process'] = 'allow_low'

        data.restart_data(restart_param)

        assert data.generate_epoch_done() is False
        count = 0
        while data.generate_epoch_done() is False:
            got_data = data.generate_data()
            assert len(got_data) == 1
            if count == 33:
                for index, d in enumerate(got_data):
                    for v in d:
                        get = v
                        assert get.indexs()[-1].index_info().type() == 'normal', print(get.indexs()[-1].index_info().type())
                        assert get.datas().shape == (1, 1), print(get[0].datas().shape)
            count += 1
            pass
        assert count == 34

        restart_param['device_batch'] = [1, 2]
        restart_param['critical_process'] = 'allow_low'

        data.restart_data(restart_param)

        assert data.generate_epoch_done() is False
        count = 0
        while data.generate_epoch_done() is False:
            got_data = data.generate_data()
            assert len(got_data) == 2
            if count == 33:
                for index, d in enumerate(got_data):
                    for v in d:
                        get = v
                        assert get.datas().shape == (1, 1), print(get.datas().shape)
                        assert get.datas().shape == (1, 1), print(get.datas().shape)
                        assert get.indexs()[0].index_info().type() == 'normal' if index == 0 else 'allow_low', print(get.indexs()[0].index_info().type())
                        pass
                    pass
                pass
            else:
                for index, d in enumerate(got_data):
                    for v in d:
                        get = v
                        assert get.datas().shape == (1 if index == 0 else 2, 1), print(get.datas().shape)
                        pass
                    pass
                pass
            count += 1
            pass
        assert count == 34

max_stars_repo_name: jihuacao/Putil

max_issues_repo_name: jihuacao/Putil

max_forks_repo_name: jihuacao/Putil


text:
# TODO: use only one (RGB) channel
import numpy as np
from math import ceil
import os
from torch.utils import data
from torch.utils.data.dataloader import DataLoader as DataLoader
import torch
from torchvision import transforms
from natsort import natsorted, ns
import cv2
from PIL import Image
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from tqdm import tqdm
from time import time
import pickle
torch.manual_seed(42)

#CONTROL PANEL -------------
dataset_path = "/home/igor/mlprojects/Csgo-NeuralNetwork/output/"
model_save_path = "/home/igor/mlprojects/Csgo-NeuralNetwork/modelsave"
net_path = None #specify path to load specific network
# train_split and test_split 0.1 > x > 0.9 and must add up to 1
train_split = 0.01
val_split = 0.15
test_split = 0.84
num_epochs = 10
batch_size =  1
save = True
##dataset ---------------
dict_override = False
##optimizer -------------
lr = 0.001
momentum = 0.5

if torch.cuda.is_available():
    device = torch.device("cuda:0")
    print("Running on: %s" % (torch.cuda.get_device_name(device)))
else:
    device = torch.device("cpu")
    print('running on: CPU')

class CsgoPersonDataset(data.Dataset):
    """preety description."""

    length = -1

    def __init__(self, root_dir, transform=None):
        """
        Args:
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.6)
        """
        self.root_dir = root_dir
        self.transform = transform
        self.length = 0
        # dictionary that marks what the last frame of each folder is
        # ie. number of examples in specific folder
        self.folder_system = {}

        for folder in os.listdir(dataset_path):
            folder_len = 0
            for file in os.listdir(dataset_path + folder):
                folder_len += 1
            self.folder_system[folder_len] = '%s' % (folder)

        for folder_len in self.folder_system:
            self.length += folder_len
        self.length = int(self.length / 2)
        print(self.length)

    # returns name of folder that contains specific frame
    def find_folder(self, idx):
        for num_frames in self.folder_system:
            if num_frames >= idx:
                return (str(self.folder_system[num_frames]), str(num_frames))
            else:
                idx = idx - num_frames

    def __len__(self):
        return self.length

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        # sets path and gets txt/jpg files
        img_path, idx = self.find_folder(idx)
        img_name = "%sframe#%s" % (img_path, idx)
        img_path = os.path.join(self.root_dir,
                                img_path, img_name)
        img_path_ext = img_path + '.jpg'
        img = Image.open((img_path_ext))
        # img = np.array(img)
        label_path = str(img_path) + '.txt'
        label = 0
        # loads label from disk, converts csv to tensor
        with open(label_path) as file:
            # if file has important data
            if os.stat(label_path).st_size != 0:
                label = np.genfromtxt(file, delimiter=',')
                if np.shape(label) == (4,):
                    label = np.reshape(label, (1, 4))
                label = label[0:1, 0:4]
                label_shape = np.shape(label)
                label = torch.as_tensor(label, dtype=torch.long)
                label_shape = np.shape(label)
            # if file is blank (no data in the image), create -1 matrix
            else:
                label = torch.zeros([1, 4], dtype=torch.float)
                label[label == 0] = -1

            sample = {'image': img, 'label': label}

        # apply transforms
        # TODO: farofa aqui hein
        if self.transform:
            img = self.transform(sample['image'])
            img = img.reshape(172800)
            sample['image'] = img

        return sample

class CsgoClassificationDataset(data.Dataset):
    """
    Dataset extracts image label pairs from multiple folders and applies transforms to image.
    if labels are empty: creates

    label format ==> [likelyhood there is a person]
                    so either [1] if there is a person
                    or [0] if there is none.
    """

    length = -1

    def __init__(self, root_dir, transform=None, dict_override = False):
        """
        Args:
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.6)
        """
        self.root_dir = root_dir
        self.transform = transform
        self.length = 0
        #dictionary that marks what the last frame of each folder is
        #ie. number of examples in specific folder

        if dict_override:
            self.folder_system = {2072:'CSGOraw1',
                                2096: 'CSGOraw3',
                                3656: 'CSGOraw4',
                                2500: 'CSGOraw7'}

        else:
            self.folder_system = {}
            for folder in os.listdir(dataset_path):
                folder_len  = 0
                for file in os.listdir(dataset_path + folder):
                    folder_len += 1
                self.folder_system[int((folder_len/2)-1)] = '%s'%(folder)

        for folder_len in self.folder_system:
            self.length += folder_len
        self.length = self.length
        print(self.length)

    #returns name of folder that contains specific frame
    def find_folder(self, idx):
        for num_frames in self.folder_system:
            if num_frames >= idx:
                return (str(self.folder_system[num_frames]), str(idx))
            else:
                idx = idx - num_frames

    def __len__(self):
        return self.length

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        #sets path and gets txt/jpg files
        img_path, idx = self.find_folder(idx)
        img_name = "%sframe#%s" % (img_path, idx)
        img_path = os.path.join(self.root_dir,
                                img_path, img_name)
        img_path_ext = img_path + '.jpg'
        img = Image.open((img_path_ext))
        # img = np.array(img)
        label_path = str(img_path) + '.txt'
        label = 0
        #loads label from disk, converts csv to tensor
        with open(label_path) as file:
            #create [0] or [1] tensor
            label = torch.as_tensor((os.stat(label_path).st_size != 0), dtype=torch.long)
            sample = {'image':img, 'label':label}

        #apply transforms
        #TODO: farofa aqui hein
        if self.transform:
            img = self.transform(sample['image'])
            # img = img.reshape(172800)
            sample['image'] = img

        return sample

def file_index(model_save_path):
    n_files = 0
    for file in os.listdir(model_save_path):
        n_files += 1
    n_files = int(np.ceil((n_files/2)))
    return n_files

def binary_acc(y_pred, y_test):
    y_pred_tag = torch.log_softmax(y_pred, dim = 1)
    _, y_pred_tags = torch.max(y_pred_tag, dim = 1)
    correct_results_sum = (y_pred_tags == y_test).sum().float()
    acc = correct_results_sum/y_test.shape[0]
    acc = torch.round(acc * 100)
    return acc

def get_TFNP_classification(outputs, labels):
    TP, TN, FP, FN = 0, 0, 0, 0

    outputs = outputs.detach().cpu()
    labels = labels.detach().cpu().numpy()
    outputs = torch.softmax(outputs, dim=1)
    outputs = np.argmax(outputs, axis=1)
    outputs = outputs.numpy()

    TP = np.logical_and(outputs, labels)
    TN = np.logical_and(np.logical_not(outputs), np.logical_not(labels))
    FP = np.logical_and(outputs, np.logical_not(labels))
    FN = np.logical_and(np.logical_not(outputs), labels)
    return int(TP[0]), int(TN[0]), int(FP[0]), int(FN[0])

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1 or classname.find('Linear') != -1:
        torch.nn.init.xavier_uniform_(m.weight.data)

#defining NN layeres
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.conv1 = nn.Conv2d(in_channels=3, out_channels=256, kernel_size=5, stride=1, padding=2)
        self.bn1 = nn.BatchNorm2d(num_features=256)
        self.mp1 = nn.MaxPool2d(kernel_size=2, stride=2)

        self.conv2 = nn.Conv2d(in_channels=256, out_channels=32, kernel_size=3, stride=1, padding=1)
        self.bn2 = nn.BatchNorm2d(num_features=32)
        self.mp2 = nn.MaxPool2d(kernel_size=2, stride=2)

        self.fc1 = nn.Linear(in_features=32 * 80 * 45, out_features=100)
        self.fc2 = nn.Linear(in_features=100, out_features=100)
        # self.fc3 = nn.Linear(in_features=100, out_features=2)

    def forward(self, x):
        # X comes in with shape(batch_size, 3, 320, 180)
        x = self.conv1(x)
        # After conv1 goes to shape (batch_size, 256, 320, 180)
        x = self.bn1(x)
        # After bn1 goes to the same shape (batch_size, 256, 320, 180)
        x = self.mp1(x)
        # After mp1 goes to the same shape (batch_size, 256, 160, 90)
        x = F.relu(x)
        # After relu it does not change shape (activations do not change shape)

        x = self.conv2(x)
        # After conv2 goes to shape (batch_size, 32, 160, 90)
        x = self.bn2(x)
        # After bn2 goes to shape (batch_size, 32, 160, 90)
        x = self.mp2(x)
        # After mp2 goes to shape (batch_size, 32, 80, 45)
        x = F.relu(x)
        # After relu it does not change shape (activations do not change shape)

        x = x.view(-1, 32 * 80 * 45)
        # After this view, it goes to shape (batch_size, 115200)
        x = F.relu(self.fc1(x))
        # x = F.relu(self.fc2(x))
        x = self.fc2(x)
        # After last_linear, it goes to shape (batch_size, 2)
        return x

# runs NN in training mode
def train_run(criterion, optimizer, device, train_loader, val_loader=None, save = True):

    n_files = file_index(model_save_path)
    if save == False:
        print('ATTENTION: NO PROGRESS WILL BE SAVED\n--------------------------------------')
    tic = time()

    train_inferences = []
    train_losses = []
    train_accs = []

    val_losses = []
    val_accs = []

    train_tpfn = []
    val_tpfn = []

    print(len(train_loader.dataset))
    log_interval = 10

    num_batches_train = len(train_loader)
    num_batches_val = len(val_loader)

    for epoch in range(num_epochs):  # loop over the dataset multiple times

        tpfn = [0, 0, 0, 0] #tp, tn, fp, fn
        running_inference, running_loss, running_acc = 0.0, 0.0, 0.0

        for i, data in enumerate(train_loader):

            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data['image'], data['label']

            #sends batch to gpu
            inputs, labels = inputs.to(device), labels.to(device)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward pass
            inferecence_start = time()
            outputs = net(inputs)
            inference = time() - inferecence_start
            running_inference += inference

            loss = criterion(outputs, labels)
            running_loss += loss.item()

            acc = binary_acc(outputs, labels)
            running_acc += acc

            TP, TN, FP, FN = get_TFNP_classification(outputs, labels)
            tpfn[0] += TP
            tpfn[1] += TN
            tpfn[2] += FP
            tpfn[3] += FN

            #backprop + optimizer
            loss.backward()
            optimizer.step()

            if (i + 1) % log_interval == 0:  # print every 10 mini-batches
                print('training: [%d, %5d] loss: %.5f acc: %.0f'%\
                    (epoch + 1, i + 1, (running_loss / (i+1)), running_acc / (i+1)))

        train_tpfn.append(tpfn)
        train_inferences.append(running_inference / num_batches_train)
        train_losses.append(running_loss / num_batches_train)
        train_accs.append(running_acc / num_batches_train)

        tpfn = [0, 0, 0, 0] #tp, tn, fp, fn
        running_val_loss, running_val_acc = 0.0, 0.0
        #validation run
        for i, data in enumerate(val_loader):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data['image'], data['label']
            #sends batch to gpu
            inputs, labels = inputs.to(device), labels.to(device)
            #run NN
            outputs = net(inputs)

            loss = criterion(outputs, labels)
            running_val_loss += loss.item()

            acc = binary_acc(outputs, labels)
            running_val_acc += acc

            TP, TN, FP, FN = get_TFNP_classification(outputs, labels)
            tpfn[0] += TP
            tpfn[1] += TN
            tpfn[2] += FP
            tpfn[3] += FN

            #backprop + optimizer
            loss.backward()
            optimizer.step()

            if (i + 1) % log_interval == 0:  # print every 10 mini-batches
                print('training: [%d, %5d] loss: %.5f acc: %.0f'%\
                    (epoch + 1, i + 1, (running_loss / (i+1)), running_acc / (i+1)))

        train_tpfn.append(tpfn)
        train_inferences.append(running_inference / num_batches_train)
        train_losses.append(running_loss / num_batches_train)
        train_accs.append(running_acc / num_batches_train)

        tpfn = [0, 0, 0, 0] #tp, tn, fp, fn
        running_val_loss, running_val_acc = 0.0, 0.0
        #validation run
        for i, data in enumerate(val_loader):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data['image'], data['label']
            #sends batch to gpu
            inputs, labels = inputs.to(device), labels.to(device)
            #run NN
            outputs = net(inputs)

            loss = criterion(outputs, labels)
            running_val_loss += loss.item()

            acc = binary_acc(outputs, labels)
            running_val_acc += acc

            TP, TN, FP, FN = get_TFNP_classification(outputs, labels)
            tpfn[0] += TP
            tpfn[1] += TN
            tpfn[2] += FP
            tpfn[3] += FN

            if (i + 1) % log_interval == 0:  # print every 10 mini-batches
                print('validation: [%d, %5d] loss: %.5f acc: %.0f'%\
                    (epoch + 1, i + 1, (running_val_loss / (i+1)), running_val_acc / (i+1)))

        val_tpfn.append(tpfn)
        val_losses.append(running_val_loss / num_batches_val)
        val_accs.append(running_val_acc / num_batches_val)

    toc = time()
    toc = toc - tic
    print('Finished Training, elapsed time: %s seconds'%(int(toc)))

    if save == True:
        print('saving state...')
        fname = 'model#%s'%(n_files)
        torch.save(net, fname)
        os.replace(fname, model_save_path+'/'+fname)

        #saving stats
        with open(model_save_path+'/'+fname+'r', 'wb') as file:
            to_dump = {
                'type' : 'training',
                'lr' : lr,
                'momentum' : momentum,
                'epochs' : num_epochs,
                'inferences' : train_inferences,
                'losses' : train_losses,
                'accuracy' : train_accs,
                'runtime' : toc,
                'train_tpfn' : train_tpfn}
            if val_loader != None:
                to_dump['val_losses'] = val_losses
                to_dump['val_accs'] = val_accs
                to_dump['val_tpfn'] = val_tpfn
            pickle.dump(to_dump, file)

    return train_inferences, train_losses, train_accs, val_losses, val_accs

#runs NN in testing mode
def test_run(criterion, device, test_loader, save = True):

    n_files = file_index(model_save_path)

    if save == False:
        print('ATTENTION: NO PROGRESS WILL BE SAVED\n--------------------------------------')

    net.eval()
    tic = time()

    test_inferences = []
    test_losses = []
    test_accs = []
    test_tp, test_tn, test_fp, test_fn = 0, 0, 0, 0

    print(len(test_loader.dataset))
    log_interval = 10
    num_batches = len(test_loader)

    for epoch in range(num_epochs):  # loop over the dataset multiple times

        running_inference, running_loss, running_acc = 0.0, 0.0, 0.0

        for i, data in enumerate(test_loader):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data['image'], data['label']

            #sends batch to gpu
            inputs, labels = inputs.to(device), labels.to(device)

            # forward pass + inference measure
            inferecence_start = time()
            outputs = net(inputs)
            inference = time() - inferecence_start
            running_inference += inference

            loss = criterion(outputs, labels)
            running_loss += loss.item()

            acc = binary_acc(outputs, labels)
            running_acc += acc

            TP, TN, FP, FN = get_TFNP_classification(outputs, labels)
            test_tp += TP
            test_tn += TN
            test_fp += FP
            test_fn += FN

            if (i + 1) % log_interval == 0:  # print every 10 mini-batches
                print('[%d, %5d] loss: %.5f acc: %.0f' %
                      (epoch + 1, i + 1, (running_loss / (i+1)), running_acc / (i+1)))

        test_inferences.append(running_inference / num_batches)
        test_losses.append(running_loss / num_batches)
        test_accs.append(running_acc / num_batches)

    toc = time()
    toc = toc - tic
    print('Finished Testing, elapsed time: %s seconds'%(int(toc)))

    if save == True:
        print('saving state...')
        fname = 'model#%s'%(n_files)
        torch.save(net, fname)
        os.replace(fname, model_save_path+'/'+fname)
        with open(model_save_path+'/'+fname+'t', 'wb') as file:
            to_dump = {
                'type' : 'testing',
                'lr' : lr,
                'momentum' : momentum,
                'epochs' : num_epochs,
                'inferences' : test_inferences,
                'losses' : test_losses,
                'accuracy' : test_accs,
                'runtime' : toc,
                'tp':test_tp,
                'tn':test_tn,
                'fp':test_fp,
                'fn':test_fn}
            pickle.dump(to_dump, file)

    return test_inferences, test_losses, test_accs

net = Net().to(device)
if net_path == None:
    # net.apply(weights_init)
    pass
else:
    net.load_state_dict(torch.load(net_path))

transform = transforms.Compose([
    transforms.Resize([320, 180]),
    # transforms.Resize([57600, 1]),
    transforms.ToTensor(),
])

dataset = CsgoClassificationDataset(dataset_path, transform, dict_override=dict_override)

dataset_len = len(dataset)

train_split = int(np.floor(dataset_len * train_split))
val_split = int(np.floor(dataset_len * val_split))
test_split = int(np.floor(dataset_len * test_split))
while train_split + val_split + test_split != dataset_len:
    train_split += 1
train_set, val_set, test_set = torch.utils.data.random_split( \
    dataset, [train_split, val_split, test_split])

train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True, num_workers=4)
val_loader =   DataLoader(dataset = val_set, batch_size=batch_size, shuffle=True, num_workers=4)
test_loader =  DataLoader(dataset=test_set,  batch_size=batch_size, shuffle=True, num_workers=4)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters())
# optimizer = optim.SGD(net.parameters(), lr=lr, momentum=momentum)

lr, momentum = 0.1, 0.5
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)
lr, momentum = 1, 0.5
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)
lr, momentum = 0.001, 0.9
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)
lr, momentum = 0.01, 0.9
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)
lr, momentum = 0.1, 0.9
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)
lr, momentum = 1, 0.9
train_inferences, train_losses, train_accs, val_losses, val_accs = train_run(\
    criterion, optimizer, device, train_loader, val_loader,save=save)

# plt.plot(train_inferences)
# plt.plot(train_losses)
# plt.plot(train_accs)
# plt.plot(val_losses)
# plt.plot(val_accs)
# plt.show()

# test_inferences, test_losses, test_accs = test_run(criterion, device, test_loader, save=save)
# plt.plot(test_inferences)
# plt.plot(test_losses)
# plt.plot(test_accs)
# plt.show()

max_stars_repo_name: IgaoGuru/Csgo-NeuralNetwork

max_issues_repo_name: IgaoGuru/Csgo-NeuralNetwork

max_forks_repo_name: IgaoGuru/Csgo-NeuralNetwork


text:
import cv2
import numpy as np

import win32api
global cam
cam = cv2.VideoCapture(0)

def diffImg(t0, t1, t2):
    d1 = cv2.absdiff(t2, t1)
    d2 = cv2.absdiff(t1, t0)
    return cv2.bitwise_and(d1, d2)
def main():
    global cam
    t_minus = cv2.cvtColor(cam.read()[1], cv2.COLOR_RGB2GRAY)
    t = cv2.cvtColor(cam.read()[1], cv2.COLOR_RGB2GRAY)
    t_plus = cv2.cvtColor(cam.read()[1], cv2.COLOR_RGB2GRAY)
    while True:
          ret, frame = cam.read()
          hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
          lower_hsv = np.array([0, 130, 147])
          higher_hsv = np.array([22, 255, 215])
          mask = cv2.inRange(hsv, lower_hsv, higher_hsv)
          mask = cv2.erode(mask, None, iterations=5)
          mask = cv2.dilate(mask, None, iterations=5)
          cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
              cv2.CHAIN_APPROX_SIMPLE)
          cnt=cnts[0]
          if len(cnts)>0:
              x,y,w,h = cv2.boundingRect(cnt)
              xx,yy=(2*x+w)/2 , (2*y+h)/2
              if xx>0 and xx<1366 and yy>0 and yy<768:
                  win32api.SetCursorPos((1366-(xx*3),yy*3))

          (cnts) = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
            cv2.CHAIN_APPROX_SIMPLE)
          t_minus = t
          t = t_plus
          t_plus = cv2.cvtColor(cam.read()[1], cv2.COLOR_RGB2GRAY)
          cv2.imshow("Mouse", frame)
          key = cv2.waitKey(10)
          if key == 27:
            cv2.destroyWindow(winName)
            break
main()

max_stars_repo_path: Code-Sleep-Python/Controll Mouse with a moving object/Mouse.py


text:
from flask import Flask, request, jsonify,render_template,session,redirect,url_for
from PIL import Image
import numpy as np
import base64
import io
import os
import cv2
from backend.yolo_inference import load_model, inference

os.environ['CUDA_VISIBLE_DEVICES'] = '0'

app = Flask(__name__)
app.config['DEBUG']=False
app.config['SECRET_KEY']='secret'
img_arr = np.array([])
path = "backend/model_config/ladder/images/"
path_config = "D:\WM\Project\ladder\yolov4\darknet\images"
threhold = 20
#print("cur dir is",os.getcwd())

@app.route('/')
def index():
    num_f = get_newData_cnt()
    return render_template('index.html',num_f=num_f)

@app.route('/api', methods=["GET","POST"])
def main_interface():
    response = request.get_json()
    print('data is',request.files)
    data_str = response['image']
    point = data_str.find(',')
    base64_str = data_str[point:]  # remove unused part like this: "data:image/jpeg;base64,"

    image = base64.b64decode(base64_str)
    img = Image.open(io.BytesIO(image))
    print('img',img)
    if(img.mode!='RGB'):
        img = img.convert("RGB")

    # convert to numpy array.
    global img_arr
    #img_arr = np.array(img)

    im1 = img.save("test.jpg")
    print("shapre is ",img_arr.shape)
    img_arr = cv2.imread("test.jpg")
    # do object detection in inference function.
    results = inference(img_arr, conf_thresh=0.5, max_suppression_thresh=0.4)

    is_ladder=1
    if not results['results']:
        is_ladder=0
    results['is_ladder']=is_ladder
    print(results)
    return jsonify(results)

@app.route('/save', methods=["POST"])
def save():
    response = request.get_json()
    coordinates = response['coordinates']
    filename = response['filename']
    w = response['width']
    h = response['height']
    print('coordinates',coordinates)
    # save the coor and image
    write_new_data(filename,coordinates,w,h)
    # caculate new data number
    num_f = get_newData_cnt()
    #if num_f >= threhold:
        #write_new_config()
    print('num_f new is ',num_f)
    return jsonify({"num_f":num_f})

def write_new_data(filename,co,w,h):
    im = Image.fromarray(img_arr)
    im.save(os.path.join(path,filename))
    #cv2.imwrite(os.path.join(path,filename), img_arr)
    new_cor = []
    for c in co:
        x,y,wi,he = c
        x = (x+wi/2.)/w
        y = (y+he/2.)/h
        wi = wi/w
        he = he/h
        new_cor.append([0,x,y,wi,he])
    with open(os.path.join(path,filename.split(".")[0]+'.txt'), "w") as result:
        result.write("\n".join([' '.join(map(str, item)) for item in new_cor]))

def get_newData_cnt(): return int(len(next(os.walk("backend/model_config/ladder/images/"))[2])/2)

@app.after_request
def add_headers(response):
    response.headers.add('Access-Control-Allow-Origin', '*')
    response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
    return response

if __name__ == '__main__':
    app.run(host='0.0.0.0',port=5000)


text:
####################################################################
# Interfaces with the GENESIS version of the auditory cortex model of Beeman,
# BMC Neuroscience (Suppl. 1), 2013 (i.e. a slightly modified version
# as used in Metzner et al., Front Comp Neu, 2016)
#
# @author: Christoph Metzner, 02/11/2017
####################################################################
import os
import subprocess
import sys

import matplotlib.mlab as mlab
import numpy as np
import sciunit
from assrunit.capabilities import ProduceXY
from assrunit.constants import ACNET2_GENESIS_PATH

class GenesisModel(object):
    def __init__(self, params):
        # extract the model parameters from the params dictionary
        self.filename = params["Filename"]
        self.random_seed = params["Random Seed"]
        self.ee_weight = params["E-E Weight"]
        self.ie_weight = params["I-E Weight"]
        self.ei_weight = params["E-I Weight"]
        self.ii_weight = params["I-I Weight"]
        self.bg_weight = params["Background Noise Weight"]
        self.edrive_weight = params["E-Drive Weight"]
        self.idrive_weight = params["I-Drive Weight"]
        self.bg_noise_frequency = params["Background Noise Frequency"]

    def genesisModelRun(self, stimfrequency, power_band):
        """
        Runs the simulation, calculates the power spectrum
        and returns the power in the specified the frequency band (Note: so far only 3 frequency
        bands are possible; 20, 30 and 40 Hz).
        Parameters:
        stimfrequency:	the frequency at which the network is driven
        power_band:	the frequency band of interest
        """

        # put the execution string together
        execstring = "genesis {} {} {} {} {} {} {} {} {} {} {} {}".format(
            os.path.join(ACNET2_GENESIS_PATH, "ACnet2-batch-new-standard.g"),
            self.filename,
            str(stimfrequency),
            str(self.random_seed),
            str(self.ee_weight),
            str(self.ie_weight),
            str(self.ei_weight),
            str(self.ii_weight),
            str(self.bg_weight),
            str(self.edrive_weight),
            str(self.idrive_weight),
            str(self.bg_noise_frequency),
        )

        print("running simulation")
        # execute the simulation
        self._runProcess(execstring)
        print("finished simulation")

        # load and analyse the data
        datafile = "EPSC_sum_" + self.filename + ".txt"
        pxx, freqs = self._calculate_psd(datafile)
        os.chdir("../notebooks/")

        # extract power at the frequency band of interest

        lbounds = {"forty": 93, "thirty": 69, "twenty": 44}
        ubounds = {"forty": 103, "thirty": 78, "twenty": 54}

        lb = lbounds[power_band]
        ub = ubounds[power_band]

        power = np.sum(pxx[lb:ub])

        return power

    def _runProcess(self, execstring):
        """
        This function executes the Genesis simulation in a shell.
        Parameters:
        execstring : the command which executes the Genesis model with all its parameters
        """

        return subprocess.call(execstring, shell=True)

    def _calculate_psd(self, datafile):
        """
        Calculates the power spectral density of the simulated EEG/MEG signal
        """
        i = 0
        if not datafile:
            print("No files were specified for plotting!")
            sys.exit()

        with open(datafile, "r") as fp:
            lines = fp.readlines()
            count = len(lines)

            tn = np.zeros(count)
            yn = np.zeros(count)
            i = 0
            for line in lines:
                # Note that tn[i] is replaced, and yn[i] is addded to,
                tn[i] = float(line[0])
                yn[i] = float(line[1])
                i += 1

        # Now do the plotting of the  array data
        dt = tn[1] - tn[0]
        # fourier sample rate
        fs = 1.0 / dt

        npts = len(yn)
        startpt = int(0.2 * fs)

        if (npts - startpt) % 2 != 0:
            startpt = startpt + 1

        yn = yn[startpt:]
        tn = tn[startpt:]
        nfft = len(tn) // 4
        overlap = nfft // 2

        pxx, freqs = mlab.psd(
            yn, NFFT=nfft, Fs=fs, noverlap=overlap, window=mlab.window_none
        )
        pxx[0] = 0.0

        return pxx, freqs

class BeemanGenesisModel(sciunit.Model, ProduceXY):
    """The auditory cortex model from Beeman (2013) [using a slightly modified version of the
    original Genesis model; For more details see Metzner et al. (2016)]"""

    def __init__(self, controlparams, schizparams):
        """
        Constructor method. Both parameter sets, for the control and the
        schizophrenia-like network, have to be a dictionary containing the following
        parmaeters (Filename,Stimulation Frequency,Random Seed,
        E-E Weight,I-E Weight,E-I Weight,I-I Weight,Background Noise Weight,E-Drive Weight,I-Drive
        Weight,Background Noise Frequency)
        Parameters:
        controlparams: Parameters for the control network
        schizparams: Parameters for the schizophrenia-like network
        name: name of the instance
        """
        self.controlparams = controlparams
        self.schizparams = schizparams
        super(BeemanGenesisModel, self).__init__(
            controlparams=controlparams, schizparams=schizparams
        )

    def produce_XY(self, stimfrequency=40.0, powerfrequency=40.0):
        """
        Simulates Y Hz drive to the control and the schizophrenia-like network for all random
        seeds, calculates a Fourier transform of the simulated MEG
        and extracts the power in the X Hz frequency band for each simulation. Returns the mean
        power for the control and the schizophrenia-like network, respectively.
        Note: So far, only three power bands [20,30,40] are possible.
        """
        powerband = "forty"  # default frequency band
        if powerfrequency == 30.0:
            powerband = "thirty"
        elif powerfrequency == 20.0:
            powerband = "twenty"
        # generate the control network and run simulation
        print("Generating control model")
        ctrl_model = GenesisModel(self.controlparams)
        print("Running control model")
        controlXY = ctrl_model.genesisModelRun(stimfrequency, powerband)

        # generate the schizophrenia-like network and run simulation
        print("Generating schizophrenia model")
        schiz_model = BeemanGenesisModel(self.schizparams)
        print("Running schizophrenia model")
        schizXY = schiz_model.genesisModelRun(stimfrequency, powerband)

        return [controlXY, schizXY]

max_stars_repo_name: rfels/ASSRUnit

max_issues_repo_name: rfels/ASSRUnit

max_forks_repo_name: rfels/ASSRUnit


text:
import logging
from typing import Callable, Dict, Final, Literal, Optional, Tuple

import numpy as np
import scipy.optimize as opt  # type: ignore
import tensorflow as tf  # type: ignore
from driver.car import LegacyPlanCar, LegacyRewardCar
from driver.legacy.models import Driver  # type: ignore
from driver.simulation_utils import legacy_car_dynamics_step_tf
from driver.world import ThreeLaneCarWorld

from active import algos

def shape_compat(arr: np.ndarray, shape: Tuple[int, ...]) -> bool:
    if len(arr.shape) != len(shape):
        return False
    for i, j in zip(arr.shape, shape):
        if j != -1 and i != j:
            return False
    return True

def orient_normals(
    normals: np.ndarray,
    preferences: np.ndarray,
    use_equiv: bool = False,
    n_reward_features: int = 4,
) -> np.ndarray:
    """Orients halfplane normal vectors relative to preferences

    Args:
        normals (np.ndarray): (n, 4) array of difference of reward features
        preferences (np.ndarray): 0/1 or +-1 vector of preferences
        use_equiv (bool, optional): If equivalent preferences are allowed. Defaults to False.
        n_reward_features (int, optional): Number of reward features. Defaults to 4.

    Returns:
        np.ndarray: normals oriented so that the prefered trajectory comes first
    """

    """ Orients halfplane normal vectors relative to preferences. """
    assert_normals(normals, use_equiv, n_reward_features)
    assert preferences.shape == (normals.shape[0],)

    if np.all((preferences == 0) | (preferences == 1)):
        preferences = preferences * 2 - 1

    assert np.all((preferences == 1) | (preferences == -1))

    oriented_normals = (normals.T * preferences).T

    assert_normals(oriented_normals, use_equiv, n_reward_features)
    return oriented_normals

def make_normals(inputs: np.ndarray, sim: Driver, use_equiv: bool) -> Tuple[np.ndarray, np.ndarray]:
    """Converts pairs of car inputs to trajectory preference normal vectors.

    Args:
        inputs (np.ndarray): (n, 2, T, 2) array of pairs of 2-dimension actions for T timesteps
        sim (Driver): Driving simulation to get features from
        use_equiv (bool): Allow equivalent preferences?

    Returns:
        Tuple[np.ndarray, np.ndarray]: input features and normal vectors
    """
    if len(inputs.shape) == 3:
        shape_compat(inputs, (-1, 2, -1))
    elif len(inputs.shape) == 4:
        shape_compat(inputs, (-1, 2, -1, 2))

    normals = np.empty(shape=(inputs.shape[0], sim.num_of_features))
    input_features = np.empty(shape=(inputs.shape[0], 2, sim.num_of_features))
    for i, (input_a, input_b) in enumerate(inputs):
        sim.feed(input_a)
        phi_a = np.array(sim.get_features())

        sim.feed(input_b)
        phi_b = np.array(sim.get_features())

        input_features[i] = np.stack((phi_a, phi_b))

        normals[i] = phi_a - phi_b
    assert_normals(normals, use_equiv)
    return input_features, normals

def assert_normals(
    normals: np.ndarray, use_equiv: bool = False, n_reward_features: int = 4
) -> None:
    """ Asserts the given array is an array of normal vectors defining half space constraints."""
    shape = normals.shape
    assert len(shape) == 2, f"shape does not have 2 dimensions:{shape}"
    # Constant offset constraint adds one dimension to normal vectors.
    assert shape[1] == n_reward_features + int(use_equiv)

class TrajOptimizer:
    """ Finds optimal trajectories in the Driver environment. """

    HORIZON: Final[int] = 50

    def __init__(
        self,
        n_planner_iters: int,
        optim: tf.keras.optimizers.Optimizer = tf.keras.optimizers.SGD(0.1),
        init_controls: Optional[np.ndarray] = None,
        log_best_init: bool = False,
    ):
        self.world = ThreeLaneCarWorld()

        self.optim = optim
        self.n_opt_iters: Final[int] = n_planner_iters

        self.init_controls = np.array(
            [
                [[0.0, 0.0]] * self.HORIZON,
                [[-5 * 0.13, 0]] * self.HORIZON,
                [[5 * 0.13, 0]] * self.HORIZON,
            ]
        )
        assert self.init_controls.shape == (3, self.HORIZON, 2)
        if init_controls is not None:
            self.init_controls = np.concatenate((self.init_controls, init_controls))

        self.tf_controls = tf.Variable(np.zeros((self.HORIZON, 2)), dtype=tf.float32)

        self.main_car = LegacyRewardCar(
            env=self.world,
            init_state=np.array([0.0, -0.3, np.pi / 2.0, 0.4], dtype=np.float32),
            weights=np.zeros(4),
        )
        self.other_car = LegacyPlanCar(env=self.world)

        self.log_best_init = log_best_init

        self.cache: Dict[Tuple[bytes, bytes], Tuple[np.ndarray, float]] = {}

    def make_loss(self) -> Callable[[], tf.Tensor]:
        other_actions = tf.constant(self.other_car.plan, dtype=tf.float32)

        main_init = self.main_car.init_state
        other_init = self.other_car.init_state

        @tf.function
        def loss() -> tf.Tensor:
            sum_reward = 0.0

            controls = tf.stack((self.tf_controls, other_actions), axis=1)

            main_car_state = main_init
            other_car_state = other_init

            for control in controls:
                main_control = control[0]
                other_control = control[1]
                # tf.print("state=", main_car_state, other_car_state)
                # tf.print("action=", main_control)
                assert main_control.shape == (2,)
                main_car_state = legacy_car_dynamics_step_tf(main_car_state, main_control)

                other_car_state = legacy_car_dynamics_step_tf(other_car_state, other_control)

                # tf.print("after step state=", main_car_state)

                reward = self.main_car.reward_fn(
                    (main_car_state, other_car_state), None
                )  # Action doesn't matter for reward

                sum_reward += reward
            return -sum_reward

        return loss

    def make_opt_traj(
        self, reward: np.ndarray, start_state: Optional[np.ndarray] = None, memorize: bool = False,
    ) -> Tuple[np.ndarray, float]:
        """ Finds the optimal sequence of actions under a given reward and starting state. """
        if start_state is not None:
            assert start_state.shape == (2, 4)
            self.main_car.init_state = start_state[0]
            self.other_car.set_init_state(start_state[1])
        else:
            start_state = np.stack(
                (self.main_car.init_state.numpy(), self.other_car.init_state.numpy())
            )

        if (reward.tobytes(), start_state.tobytes()) in self.cache.keys():
            return self.cache[reward.tobytes(), start_state.tobytes()]

        self.main_car.weights = reward

        loss = self.make_loss()

        best_loss = float("inf")
        best_init = -1
        for i, init_control in enumerate(self.init_controls):
            assert init_control.shape == (50, 2)
            self.tf_controls.assign(init_control)

            for _ in range(self.n_opt_iters):
                self.optim.minimize(loss, self.tf_controls)

            # TODO(joschnei): Figure out if this recomputation can be avoided. Or maybe the result
            # is cached and this is free.
            current_loss = loss()

            if current_loss < best_loss:
                best_loss = current_loss
                best_plan: np.ndarray = self.tf_controls.numpy()
                best_init = i

        assert i > 0

        if self.log_best_init:
            logging.info(f"Best traj found from init={best_init}")

        if memorize:
            self.cache[reward.tobytes(), start_state.tobytes()] = (best_plan, float(best_loss))

        return best_plan, best_loss

def get_simulated_feedback(
    simulation: Driver,
    input_A: np.ndarray,
    input_B: np.ndarray,
    query_type: str,
    true_reward: np.ndarray,
    delta: Optional[float] = None,
) -> Tuple[np.ndarray, np.ndarray, int]:
    """ Gets preference between trajectories from an agent simulated by true_reward """
    simulation.feed(input_A)
    phi_A = np.array(simulation.get_features())
    simulation.feed(input_B)
    phi_B = np.array(simulation.get_features())
    if query_type == "weak":
        # TODO(joschnei): Implement weak errors using delta. I think there's a model for this but I can't remember off hand.
        raise NotImplementedError("Simulated weak preferences not implemented.")
        if delta is None:
            raise ValueError("Must provide delta when using weak queries.")
    elif query_type == "strict":
        s = 1 if true_reward @ (phi_A - phi_B) > 0 else -1
    else:
        raise ValueError(f'query type {query_type} must be either "strict" or "weak"')
    return phi_A, phi_B, s

def get_feedback(simulation_object, input_A, input_B, query_type):
    """ Gets a preference between trajectories from a human user """
    simulation_object.feed(input_A)
    phi_A = np.array(simulation_object.get_features())
    simulation_object.feed(input_B)
    phi_B = np.array(simulation_object.get_features())
    s = -2
    while s == -2:
        if query_type == "weak":
            selection = input('A/B to watch, 1/2 to vote, 0 for "About Equal": ').lower()
        elif query_type == "strict":
            selection = input("A/B to watch, 1/2 to vote: ").lower()
        else:
            raise ValueError("There is no query type called " + query_type)
        if selection == "a":
            simulation_object.feed(input_A)
            simulation_object.watch(1)
        elif selection == "b":
            simulation_object.feed(input_B)
            simulation_object.watch(1)
        elif selection == "0" and query_type == "weak":
            s = 0
        elif selection == "1":
            s = 1
        elif selection == "2":
            s = -1
    return phi_A, phi_B, s

def run_algo(
    criterion: Literal["information", "volume", "random"],
    env: Driver,
    w_samples: np.ndarray,
    delta_samples: np.ndarray,
    continuous: bool = False,
):
    """ Gets next pair of trajectories to ask for a preference over. """
    shape_compat(w_samples, (-1, 4))
    shape_compat(delta_samples, (-1,))

    if criterion == "information":
        return algos.information(env, w_samples, delta_samples, continuous)
    if criterion == "volume":
        return algos.volume(env, w_samples, delta_samples, continuous)
    elif criterion == "random":
        return algos.random(env)
    else:
        raise ValueError("There is no criterion called " + criterion)

def play(sim: Driver, optimal_ctrl):
    """ Renders trajectory for user. """
    sim.set_ctrl(optimal_ctrl)
    keep_playing = "y"
    while keep_playing == "y":
        keep_playing = "u"
        sim.watch(1)
        while keep_playing != "n" and keep_playing != "y":
            keep_playing = input("Again? [y/n]: ").lower()
    return optimal_ctrl

reason: import numpy,import scipy


text:
#!/usr/bin/env python3
# Copyright 2021 by Julian Nubert, Robotic Systems Lab, ETH Zurich.
# All rights reserved.
# This file is released under the "BSD-3-Clause License".
# Please see the LICENSE file that has been included as part of this package.
import numpy as np
import torch
import yaml

import ros_utils.publish_point_cloud_and_normals

def config():
    f = open('config/config_datasets.yaml')
    config = yaml.load(f, Loader=yaml.FullLoader)
    f = open('config/deployment_options.yaml')
    deployment_options = yaml.load(f, Loader=yaml.FullLoader)
    config.update(deployment_options)

    # Device to be used
    if config["device"] == "cuda":
        config["device"] = torch.device("cuda")
    else:
        config["device"] = torch.device("cpu")

    # Data identifiers
    for dataset in config["datasets"]:
        if config["mode"] == "training":
            config[dataset]["data_identifiers"] = config[dataset]["training_identifiers"]
        elif config["mode"] == "testing":
            config[dataset]["data_identifiers"] = config[dataset]["testing_identifiers"]
        else:
            raise Exception('Only modes "training" and "testing" are valid.')

    # Convert angles to radians
    for dataset in config["datasets"]:
        config[dataset]["vertical_field_of_view"][0] *= (np.pi / 180.0)
        config[dataset]["vertical_field_of_view"][1] *= (np.pi / 180.0)
    config["horizontal_field_of_view"][0] *= (np.pi / 180.0)
    config["horizontal_field_of_view"][1] *= (np.pi / 180.0)

    print("----------------------------------")
    print("Configuration for this run: ")
    print(config)
    print("----------------------------------")

    return config

if __name__ == "__main__":
    config = config()
    publisher = ros_utils.publish_point_cloud_and_normals.ROSPublisher(config=config)
    publisher.publish_dataset()

max_stars_repo_name: leggedrobotics/DeLORA

max_issues_repo_name: leggedrobotics/DeLORA

max_forks_repo_name: leggedrobotics/DeLORA


text:
import sys
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from diffpy.utils.parsers.loaddata import loadData
from matplotlib.ticker import MultipleLocator
try:
    from bg_mpl_stylesheet.bg_mpl_stylesheet import bg_mpl_style
    PLOT_STYLE = "found"
except ModuleNotFoundError:
    PLOT_STYLE = None

DPI = 600
FONTSIZE_LABELS = 20
FONTSIZE_TICKS = 16
FONTSIZE_LEGEND = 20
FONTSIZE_TEXT = 16
RLABEL = r"$r$ $[\mathrm{\AA}]$"
GLABEL = r"$G$ $[\mathrm{\AA}^{-2}]$"
GOBSLABEL = r"$G_{\mathrm{obs}}$"
GCALCLABEL = r"$G_{\mathrm{calc}}$"
GDIFFLABEL = r"$G_{\mathrm{diff}}$"
SAMPLELABELS = ["3 nm", "5 nm", "7 nm", "3 nm (Li)", "5 nm (Li)", "7 nm (Li)"]
LINEWIDTH = 1
LINEWIDTH_LEGEND = 2
SCATTERSIZE = 1
XINDEX_MAJOR = 5
XINDEX_MINOR = XINDEX_MAJOR / 5
YINDEX_MAJOR = 1
YINDEX_MINOR = YINDEX_MAJOR / 5
COLORS = dict(bg_blue='#0B3C5D', bg_red='#B82601', bg_green='#1c6b0a',
              bg_lightblue='#328CC1', bg_darkblue='#062F4F', bg_yellow='#D9B310',
              bg_darkred='#984B43', bg_bordeaux='#76323F', bg_olivegreen='#626E60',
              bg_yellowgrey='#AB987A', bg_brownorange='#C09F80')

def dict_extract(files):
    d = {}
    for i in range(len(files)):
        d[i] = loadData(str(files[i]))

    return d

def dict_rw_extract(files):
    d = {}
    for i in range(len(files)):
        with files[i].open(mode="r") as f:
            lines = f.readlines()
        for l in lines:
            if "Rw" in l:
                d[i] = l.split()[-1]
                break

    return d

def fit_res_plot(d_fit, d_res, output_folders):
    keys = d_fit.keys()
    if not isinstance(PLOT_STYLE, type(None)):
        plt.style.use(bg_mpl_style)
        colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
    else:
        colors = list(COLORS.values())
    fig, axs = plt.subplots(ncols=1,
                            nrows=len(keys),
                            dpi=DPI,
                            figsize=(8, 2*len(keys)),
                            sharex=True,
                            sharey=True,
                            gridspec_kw=dict(hspace=0)
                            )
    for i in range(len(keys)):
        x, ycalc, y = d_fit[i][:,0], d_fit[i][:,1], d_fit[i][:,2]
        xmin, xmax = np.amin(x), np.amax(x)
        xrange = xmax - xmin
        ydiff = y - ycalc
        ymin, ymax = np.amin(y), np.amax(y)
        ydiffmin, ydiffmax = np.amin(ydiff), np.amax(ydiff)
        ydiff_offset = ydiff - 1.1 * (abs(ymin) + abs(ydiffmax))
        if len(keys) > 1:
            axs[i].scatter(x, y, s=SCATTERSIZE, c=colors[0], label=GOBSLABEL)
            axs[i].plot(x, ycalc, lw=LINEWIDTH, c=colors[1], label=GCALCLABEL)
            axs[i].plot(x, ydiff_offset, lw=LINEWIDTH, c=colors[2], label=GDIFFLABEL)
            axs[i].set_xlim(xmin, xmax)
            axs[i].xaxis.set_major_locator(MultipleLocator(XINDEX_MAJOR))
            axs[i].xaxis.set_minor_locator(MultipleLocator(XINDEX_MINOR))
            axs[i].yaxis.set_major_locator(MultipleLocator(YINDEX_MAJOR))
            axs[i].yaxis.set_minor_locator(MultipleLocator(YINDEX_MINOR))
            axs[i].tick_params(axis="both", labelsize=FONTSIZE_TICKS)
            axs[-1].set_ylabel('.', color=(0, 0, 0, 0))
        else:
            axs.scatter(x, y, s=SCATTERSIZE, c=colors[0], label=GOBSLABEL)
            axs.plot(x, ycalc, lw=LINEWIDTH, c=colors[1], label=GCALCLABEL)
            axs.plot(x, ydiff_offset, lw=LINEWIDTH, c=colors[2], label=GDIFFLABEL)
            axs.set_xlim(xmin, xmax)
            axs.xaxis.set_major_locator(MultipleLocator(XINDEX_MAJOR))
            axs.xaxis.set_minor_locator(MultipleLocator(XINDEX_MINOR))
            axs.yaxis.set_major_locator(MultipleLocator(YINDEX_MAJOR))
            axs.yaxis.set_minor_locator(MultipleLocator(YINDEX_MINOR))
            axs.tick_params(axis="both", labelsize=FONTSIZE_TICKS)
            axs.set_ylabel('.', color=(0, 0, 0, 0))
        if i == 0:
            blue_marker = plt.scatter([], [], marker='o', facecolors='none',
                                      edgecolors=colors[0])
            blue_marker.remove()
            red_line = Line2D([], [], color=colors[1], linewidth=LINEWIDTH_LEGEND)
            green_line = Line2D([], [], color=colors[2], linewidth=LINEWIDTH_LEGEND)
            if len(keys) == 1:
                anchor_y = 1.2
            elif len(keys) == 2:
                anchor_y = 1.025
            elif len(keys) == 3:
                anchor_y = 1.0
            elif len(keys) == 4:
                anchor_y = 0.97
            else:
                anchor_y = 0.935
            fig.legend(handles=[(blue_marker), (red_line), (green_line)],
                       labels=['$G_{\mathrm{obs}}$', '$G_{\mathrm{calc}}$',
                               '$G_{\mathrm{diff}}$'],
                       ncol=3,
                       facecolor="white",
                       frameon=False,
                       fontsize=FONTSIZE_LEGEND,
                       loc="upper center",
                       bbox_to_anchor=(0.5, anchor_y)
                       )
    for i in range(len(keys)):
        rw = float(d_res[i])
        s = f"{SAMPLELABELS[i]}\n" + r"$R_{\mathrm{w}}=$" + fr"$\;{rw:.2f}$"
        if len(keys) > 1:
            xlim, ylim = axs[i].get_xlim(), axs[i].get_ylim()
            xrange = float(xlim[1]) - float(xlim[0])
            yrange = float(ylim[1]) - float(ylim[0])
            axs[i].text(1.03 * xrange,
                        0.225 * yrange,
                        s,
                        fontsize=FONTSIZE_TEXT,
                        ha="right")
        else:
            xlim, ylim = axs.get_xlim(), axs.get_ylim()
            xrange = float(xlim[1]) - float(xlim[0])
            yrange = float(ylim[1]) - float(ylim[0])
            axs.text(1.03 * xrange,
                     0.225 * yrange,
                     s,
                     fontsize=FONTSIZE_TEXT,
                     ha="right")
    plt.xlabel(RLABEL, fontsize=FONTSIZE_LABELS)
    fig.text(0.04, 0.5, GLABEL, va='center', ha='center',
             rotation='vertical', fontsize=FONTSIZE_LABELS)
    for e in output_folders:
        plt.savefig(f"{e}/fit_res_plot_stack.{e}", bbox_inches="tight")

    return None

def main():
    fit_path, res_path = Path.cwd() / "fit", Path.cwd() / "res"
    data_paths = [fit_path, res_path]
    exit = False
    for p in data_paths:
        if not p.exists():
            p.mkdir()
            print(f"{80*'-'}\nA folder called '{p.name}' has been created.\n"
                  f"Please place your .{p.name} files here and rerun the "
                  f"program.")
            exit = True
    if exit is True:
        sys.exit()
    fit_files = list(fit_path.glob("*.fit"))
    res_files = list(res_path.glob("*.res"))
    if len(fit_files) == 0:
        print(f"{80*'-'}\nNo .{fit_path.name} files were found in the "
              f"'{fit_path.name}' folder.\nPlease place your .{fit_path.name} "
              f"files here and rerun the program.")
        exit = True
    if len(res_files) == 0:
        print(f"{80*'-'}\nNo .{res_path.name} files were found in the "
              f"'{res_path.name}' folder.\nPlease place your .{fit_path.name} "
              f"files here and rerun the program.")
        exit = True
    if len(fit_files) != len(res_files):
        print(f"{80*'-'}\n{len(fit_files)} .{fit_path.name} files were found "
              f"in the '{fit_path.name}' folder.\n{len(res_files)} "
              f".{res_path.name} files were found  in the '{res_path.name}' "
              f"folder.\n{80*'-'}\nPlease review the '{fit_path.name}' and "
              f"'{res_path.name}' folders to ensure files are properly placed "
              f"\nand rerun the program.")
        exit = True
    if exit is True:
        sys.exit()
    print(f"{80*'-'}\n.{fit_path.name} and .{res_path.name} files will be "
          f"paired the following way:")
    for i in range(len(fit_files)):
        print(f"\t{i}\t{fit_files[i].name}\t{res_files[i].name}")
    output_folders = ["pdf", "png", "svg"]
    for folder in output_folders:
        p = Path.cwd() / folder
        if not p.exists():
            p.mkdir()
    print(f"{80*'-'}\nExtracting data from .{fit_path.name} and "
          f".{res_path.name} files...")
    d_fit = dict_extract(fit_files)
    d_res = dict_rw_extract(res_files)
    print(f"Done extracting data.\n{80*'-'}\nPlotting data...")
    fit_res_plot(d_fit, d_res, output_folders)
    print(f"Done plotting data.\n{80*'-'}\nPlease see the {output_folders} "
          f"folders.")

    return None

if __name__ == "__main__":
    main()

# End of file.

max_stars_repo_name: maak-sdu/pyScattData

max_issues_repo_name: maak-sdu/pyScattData

max_forks_repo_name: maak-sdu/pyScattData


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import rospy
import cv2
import numpy as np
from std_msgs.msg import Int32
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError

class image_converter:
  def __init__(self):
     self.image_pub = rospy.Publisher("image_topic", Image, queue_size=1)
     self.bridge = CvBridge()
     self.image_sub = rospy.Subscriber("/usb_cam/image_raw",Image,self.callback)

  def callback(self,data):
     pub1 = rospy.Publisher("detect_face", Int32, queue_size=1)
     pub2 = rospy.Publisher("number_of_face", Int32, queue_size=1)
     try:
       cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
     except CvBridgeError as e:
       print(e)

     #分類器までのパス
     faceCascade = cv2.CascadeClassifier('/home/dan/catkin_ws/src/detect_human_chime/scripts/haarcascade_frontalface_alt2.xml')

     #グレースケール
     gray = cv2.cvtColor(cv_image, cv2.COLOR_BGR2GRAY)

     #顔判定
     face = faceCascade.detectMultiScale(gray,1.1,3,minSize=(50,50))

     #フラッグ 顔があれば1, なければ0をパブリッシュする
     flag = 1

     #顔周りに四角
     if len(face) > 0:
       for rect in face:
         cv2.rectangle(cv_image, tuple(rect[0:2]), tuple(rect[0:2]+rect[2:4]), (255, 255, 255), thickness=2)
       print("face")
       print('{0}人ご来店'.format(len(face)))
       pub2.publish(len(face))
     else:
       flag = 0

     pub1.publish(flag)

     cv2.imshow('detected_face', cv_image)
     cv2.waitKey(3)

def main(args):
  rospy.init_node('image_converter', anonymous=True)
  ic = image_converter()
  try:
    rospy.spin()
  except KeyboardInterrupt:
    print("Shutting down")
  cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)
    rospy.spin()