text:
from djitellopy import Tello
import cv2
import numpy as np

# Image dimensions
width = 640
height = 480
deadZone = 100

# Drone flight controller (0 = fly, 1 = do not fly)
startCounter = 0

# Connect to tello, ensure no movement
drone = Tello()
drone.connect()
drone.for_back_velocity = 0
drone.left_right_velocity = 0
drone.up_down_velocity = 0
drone.yaw_velocity = 0
drone.speed = 0

print(drone.get_battery())

drone.streamoff()
drone.streamon()

frameWidth = width
frameHeight = height
# cap = cv2.VideoCapture(1)
# cap.set(3, frameWidth)
# cap.set(4, frameHeight)
# cap.set(10,200)

global imgContour
global dir;

def empty(a):
    pass

# Define trackbars (Hue, Saturation, thresholds, area)
cv2.namedWindow("HSV")
cv2.resizeWindow("HSV", 640, 240)
cv2.createTrackbar("HUE Min", "HSV", 20, 179, empty)
cv2.createTrackbar("HUE Max", "HSV", 40, 179, empty)
cv2.createTrackbar("SAT Min", "HSV", 148, 255, empty)
cv2.createTrackbar("SAT Max", "HSV", 255, 255, empty)
cv2.createTrackbar("VALUE Min", "HSV", 89, 255, empty)
cv2.createTrackbar("VALUE Max", "HSV", 255, 255, empty)

cv2.namedWindow("Parameters")
cv2.resizeWindow("Parameters", 640, 240)
cv2.createTrackbar("Threshold1", "Parameters", 166, 255, empty)
cv2.createTrackbar("Threshold2", "Parameters", 171, 255, empty)
cv2.createTrackbar("Area", "Parameters", 1750, 30000, empty)

# Display the true, masked, and edge image within one frame
def stackImages(scale, imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range(0, rows):
            for y in range(0, cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape[:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1],
                                                                 imgArray[0][0].shape[0]), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y] = cv2.cvtColor(imgArray[x][y],
                                                                                cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank] * rows
        hor_con = [imageBlank] * rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
        ver = np.vstack(hor)
    else:
        for x in range(0, rows):
            if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
            else:
                imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]),
                                         None, scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor = np.hstack(imgArray)
        ver = hor
    return ver

# Find contours of object detected given an img (which has Canny edge detection applied) and imgContour (original image)
def getContours(img, imgContour):
    global dir
    # Store all boundary points in contour, as the curvature of the boundary is unknown
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2:]

    # For all contours with contour area greater than the trackbar area, draw all contours,
    for cnt in contours:
        area = cv2.contourArea(cnt)
        areaMin = cv2.getTrackbarPos("Area", "Parameters")
        if area > areaMin:
            cv2.drawContours(imgContour, cnt, -1, (255, 0, 255), 7)
            perimeter = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt, 0.02 * perimeter, True)
            x, y, w, h = cv2.boundingRect(approx)
            cx = int(x + (w / 2))  # center x of the object
            cy = int(y + (h / 2))  # center y of the object

            # Find location of object, move accordingly
            if (cx < int(frameWidth / 2) - deadZone):
                cv2.putText(imgContour, " GO LEFT ", (20, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 3)
                cv2.rectangle(imgContour, (0, int(frameHeight / 2 - deadZone)), (int(frameWidth / 2) - deadZone,
                                                                                int(frameHeight / 2) + deadZone),
                              (0, 0, 255), cv2.FILLED)
                dir = 1
            elif (cx > int(frameWidth / 2) + deadZone):
                cv2.putText(imgContour, " GO RIGHT ", (20, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 3)
                cv2.rectangle(imgContour, (int(frameWidth / 2 + deadZone), int(frameHeight / 2 - deadZone)),
                              (frameWidth, int(frameHeight / 2) + deadZone), (0, 0, 255), cv2.FILLED)
                dir = 2
            elif (cy < int(frameHeight / 2) - deadZone):
                cv2.putText(imgContour, " GO UP ", (20, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 3)
                cv2.rectangle(imgContour, (int(frameWidth / 2 - deadZone), 0), (int(frameWidth / 2 + deadZone),
                                                                                int(frameHeight / 2) - deadZone),
                              (0, 0, 255), cv2.FILLED)
                dir = 3
            elif (cy > int(frameHeight / 2) + deadZone):
                cv2.putText(imgContour, " GO DOWN ", (20, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 3)
                cv2.rectangle(imgContour, (int(frameWidth / 2 - deadZone), int(frameHeight / 2) + deadZone),
                              (int(frameWidth / 2 + deadZone), frameHeight), (0, 0, 255), cv2.FILLED)
                dir = 4
            else:
                dir = 0

            cv2.line(imgContour, (int(frameWidth / 2), int(frameHeight / 2)), (cx, cy), (0, 0, 255), 3)
            cv2.rectangle(imgContour, (x, y), (x + w, y + h), (0, 255, 0), 5)
            cv2.putText(imgContour, "Points: " + str(len(approx)), (x + w + 20, y + 20),
                        cv2.FONT_HERSHEY_COMPLEX, .7, (0, 255, 0), 2)
            cv2.putText(imgContour, "Area: " + str(int(area)), (x + w + 20, y + 45),
                        cv2.FONT_HERSHEY_COMPLEX, 0.7, (0, 255, 0), 2)
            cv2.putText(imgContour, " " + str(int(x)) + " " + str(int(y)), (x - 20, y - 45),
                        cv2.FONT_HERSHEY_COMPLEX, 0.7, (0, 255, 0), 2)
        else:
            dir = 0

def display(img):
    cv2.line(img, (int(frameWidth / 2) - deadZone, 0), (int(frameWidth / 2) - deadZone, frameHeight), (255, 255, 0), 3)
    cv2.line(img, (int(frameWidth / 2) + deadZone, 0), (int(frameWidth / 2) + deadZone, frameHeight), (255, 255, 0), 3)
    cv2.circle(img, (int(frameWidth / 2), int(frameHeight / 2)), 5, (0, 0, 255), 5)
    cv2.line(img, (0, int(frameHeight / 2) - deadZone), (frameWidth, int(frameHeight / 2) - deadZone),
             (255, 255, 0), 3)
    cv2.line(img, (0, int(frameHeight / 2) + deadZone), (frameWidth, int(frameHeight / 2) + deadZone),
             (255, 255, 0), 3)

# Main loop
while True:

    # Get image from drone
    frame_read = drone.get_frame_read()
    myFrame = frame_read.frame
    img = cv2.resize(myFrame, (width, height))
    imgContour = img.copy()
    imgHsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    # Get positions of trackbar at hue, saturation, and value min and max threshold
    h_min = cv2.getTrackbarPos("HUE Min", "HSV")
    h_max = cv2.getTrackbarPos("HUE Max", "HSV")
    s_min = cv2.getTrackbarPos("SAT Min", "HSV")
    s_max = cv2.getTrackbarPos("SAT Max", "HSV")
    v_min = cv2.getTrackbarPos("VALUE Min", "HSV")
    v_max = cv2.getTrackbarPos("VALUE Max", "HSV")

    # Check if HSV values lie between lower and upper hue, saturation, and value bounds. With mask and original image,
    # obtain new image masked over.
    lower = np.array([h_min, s_min, v_min])
    upper = np.array([h_max, s_max, v_max])
    mask = cv2.inRange(imgHsv, lower, upper)
    result = cv2.bitwise_and(img, img, mask=mask)
    mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)

    # Canny edge detection
    imgBlur = cv2.GaussianBlur(result, (7, 7), 1)  # noise reduction
    imgGray = cv2.cvtColor(imgBlur, cv2.COLOR_BGR2GRAY)
    threshold1 = cv2.getTrackbarPos("Threshold1", "Parameters")
    threshold2 = cv2.getTrackbarPos("Threshold2", "Parameters")
    # noise reduction, finding intensity gradient, non-maximum suppression, hysteresis thresholding
    imgCanny = cv2.Canny(imgGray, threshold1, threshold2)
    kernel = np.ones((5, 5))
    # increase area covered by activated pixels, patch up holes between activated pixels
    imgDil = cv2.dilate(imgCanny, kernel, iterations=1)
    getContours(imgDil, imgContour)
    display(imgContour)

    # Initiate flight if drone is not flying
    if startCounter == 0:
        drone.takeoff()
        startCounter = 1

    # Based on location of object, dir is set, so move accordingly
    if dir == 1:
        drone.yaw_velocity = -60
    elif dir == 2:
        drone.yaw_velocity = 60
    elif dir == 3:
        drone.up_down_velocity = 60
    elif dir == 4:
        drone.up_down_velocity = -60
    else:
        drone.left_right_velocity = 0
        drone.for_back_velocity = 0
        drone.up_down_velocity = 0
        drone.yaw_velocity = 0

    # Give velocity info to drone
    if drone.send_rc_control:
        drone.send_rc_control(drone.left_right_velocity,
                              drone.for_back_velocity,
                              drone.up_down_velocity,
                              drone.yaw_velocity)
    print(dir)

    # Stack images
    stack = stackImages(0.9, ([img, result], [imgDil, imgContour]))
    cv2.imshow('Horizontal Stacking', stack)  # Show images

    if cv2.waitKey(1) & 0xFF == ord('q'):
        drone.land()
        break

# cap.release()
cv2.destroyAllWindows()

max_stars_repo_name: UTSA-ACM/tello-drone

max_issues_repo_name: UTSA-ACM/tello-drone

max_forks_repo_name: UTSA-ACM/tello-drone


text:
####################################################### README ####################################################################

# This is the main file which calls all the functions and trains the network by updating weights

#####################################################################################################################################

import numpy as np
from snn.neuron import neuron
import random
from matplotlib import pyplot as plt
from snn.recep_field import rf
import imageio
from snn.spike_train import encode
from snn.rl import rl, update
from snn.reconstruct import reconst_weights
from snn.parameters import param as par
from snn.var_th import threshold
import os
import time as timing

start = timing.time()
print ("shikhar", start)
#potentials of output neurons
pot_arrays = []
for i in range(par.n):
    pot_arrays.append([])

#time series
time  = np.arange(1, par.T+1, 1)

layer2 = []

# creating the hidden layer of neurons
for i in range(par.n):
    a = neuron()
    layer2.append(a)

#synapse matrix    initialization
synapse = np.zeros((par.n,par.m))

for i in range(par.n):
    for j in range(par.m):
        synapse[i][j] = random.uniform(0,par.w_max*0.5)

spike_probe = []
for i in range(par.n):
    spike_probe.append([(0, 0)])

for k in range(par.epoch):
    for i in range(3):
        #print(i,"  ",k)
        img = imageio.imread("training/{}.png".format(i))

        #Convolving image with receptive field
        pot = rf(img)

        #Generating spike train
        train = np.array(encode(pot))
        if k == 0:
            print(train.shape)
        #calculating threshold value for the image
        var_threshold = threshold(train)

        # print var_threshold
        # synapse_act = np.zeros((par.n,par.m))
        # var_threshold = 9
        # print var_threshold
        #var_D = (var_threshold*3)*0.07

        var_D = par.D

        for x in layer2:
            x.initial(par.Pth)

        #flag for lateral inhibition
        f_spike = 0

        img_win = 100

        active_pot = []
        for index1 in range(par.n):
            active_pot.append(0)

        #Leaky integrate and fire neuron dynamics
        for t in time:
            for j, x in enumerate(layer2):
                active = []
                if(x.t_rest<t):
                    x.P = x.P + np.dot(synapse[j], train[:,t])
                    if(x.P>par.Prest):
                        x.P -= var_D
                    active_pot[j] = x.P

                pot_arrays[j].append(x.P)

            # Lateral Inhibition
            if(f_spike==0):
                high_pot = max(active_pot)
                if(high_pot>par.Pth):
                    f_spike = 1
                    winner = np.argmax(active_pot)
                    img_win = winner
                    print("winner is " + str(winner))
                    for s in range(par.n):
                        if(s!=winner):
                            layer2[s].P = -500

            #Check for spikes and update weights
            for j,x in enumerate(layer2):
                s = x.check()
                if(s==1):
                    spike_probe[j].append((len(pot_arrays[j]), 1))
                    x.t_rest = t + x.t_ref
                    x.P = par.Prest
                    for h in range(par.m):
                        for t1 in range(-2,par.t_back-1, -1):
                            if 0<=t+t1<par.T+1:
                                if train[h][t+t1] == 1:
                                    # print "weight change by" + str(update(synapse[j][h], rl(t1)))
                                    synapse[j][h] = update(synapse[j][h], rl(t1))

                        for t1 in range(2,par.t_fore+1, 1):
                            if 0<=t+t1<par.T+1:
                                if train[h][t+t1] == 1:
                                    # print "weight change by" + str(update(synapse[j][h], rl(t1)))
                                    synapse[j][h] = update(synapse[j][h], rl(t1))

        if(img_win!=100):
            for p in range(par.m):
                if sum(train[p])==0:
                    synapse[img_win][p] -= 0.06*par.scale
                    if(synapse[img_win][p]<par.w_min):
                        synapse[img_win][p] = par.w_min

print ("total time taken", (timing.time() - start))
# ttt = np.arange(0,len(pot_arrays[0]),1)
# Pth = []
# for i in range(len(ttt)):
#     Pth.append(layer2[0].Pth)

# #plotting
# plt.figure(0)
# for i in range(par.n):
#     plt.subplot(par.n, 1, i+1)
#     axes = plt.gca()
#     axes.set_ylim([-20,60])
#     plt.plot(ttt,Pth, 'r')
#     plt.plot(ttt,pot_arrays[i])

# plt.figure(1)

# for i in range(par.n):
#     plt.subplot(par.n, 1, i+1)
#     axes = plt.gca()
#     axes.set_ylim([0, 1])
#     vals = np.array(spike_probe[i])
#     plt.stem(vals[:,0],vals[:,1])
# plt.show()

# plt.figure(2)
# for i in range(par.n):
#     plt.subplot(par.n/2, par.n/2, i+1)
#     plt.gca().grid(False)
#     weights = np.array(synapse[i])
#     weights = np.reshape(weights, (par.pixel_x,par.pixel_x))
#     img = np.zeros((par.pixel_x,par.pixel_x))
#     for i in range(par.pixel_x):
#         for j in range(par.pixel_x):
#             img[i][j] = np.interp(weights[i][j], [par.w_min,par.w_max], [-1.0,1.0])
#     plt.imshow(img)
#     plt.colorbar()
# plt.show()

# with open('weights_training.txt', 'w') as weight_file:
#     for i in range(len(synapse)):
#         weights = []
#         for j in synapse[i]:
#             weights.append(str(j))
#         convert = '\t'.join(weights)
#         weight_file.write("%s\n" % convert)

# #Reconstructing weights to analyse training
# for i in range(par.n):
#     reconst_weights(synapse[i],i+1)

ext: py

max_stars_repo_name: Tab-ct/Spiking-Neural-Network

max_issues_repo_name: Tab-ct/Spiking-Neural-Network

max_forks_repo_name: Tab-ct/Spiking-Neural-Network

reason: import numpy


text:
import sys, os

from src.dqn import DQNetwork, FCQNetwork
from src.env import SumoIntersection
from src.memory import DQNBuffer
from src.data_storage import StoreState

import torch
import torch.nn as nn
import numpy as np
from tqdm import tqdm

def weights_init(m):
    if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
        torch.nn.init.xavier_uniform(m.weight.data)
        nn.init.constant(m.bias.data, 0)

def soft_update(local_model, target_model, tau):
        for target_param, local_param in zip(target_model.parameters(),
                                             local_model.parameters()):
            target_param.data.copy_(tau*local_param.data + (1-tau)*target_param.data)

def train_net(q, q_target, memory, optimizer, batch_size, gamma):
    state, a, r, state_prime, done_mask = memory.sample(batch_size)

    q_out = q(state)
    q_a = q_out.gather(1, a)
    max_q_prime = q_target(state_prime).max(1)[0].unsqueeze(1)
    target = r + gamma * max_q_prime * done_mask
    criterion = nn.MSELoss()
    loss = criterion(q_a, target)
    mse = loss.item()

    optimizer.zero_grad()
    loss.backward()
    for param in q.parameters():
       param.grad.data.clamp_(-1, 1)
    optimizer.step()

def training(config):
    learning_rate = config['learning_rate']
    gamma = config['gamma']
    buffer_limit = config['buffer_limit']
    batch_size = config['batch_size']
    sim_len = config['sim_len']
    mem_refill = config['mem_refill']
    epochs = config['epochs']
    n_cars = config['n_cars']
    weights_path = config['weights_path']

    sumoBinary = config['sumoBinary']
    sumoCmd = config['sumoCmd']
    sys.path.append(os.path.join(config['sumoTools']))

    q = DQNetwork()

    try:
        q.load_state_dict(torch.load(weights_path))
    except FileNotFoundError:
        q.apply(weights_init)
        print('No model weights found, initializing xavier_uniform')

    q_target = DQNetwork()
    q_target.load_state_dict(q.state_dict())

    memory = DQNBuffer(buffer_limit, 0.)

    env = SumoIntersection(sumoBinary, sumoCmd, sim_len, n_cars)
    optimizer = torch.optim.RMSprop(q.parameters(), lr=learning_rate)

    state = StoreState()
    min_wait = float('inf')
    total_steps=0
    pbar = tqdm(total=epochs)

    for epoch in np.arange(epochs):
        eps = max(0.01, 0.07 - 0.01*(epoch/200))
        step = 0

        if epoch != 0:
            env.reset()
            pbar.set_description(f'EPOCH: {epoch}, mean reward: {info[6]}, mean waiting time: {info[2]}')

        state, _, _, _ = env.step(0)
        done = False

        done_mask = 1.0
        mse = 0
        while not done:
            with torch.no_grad():
                a = q.predict(state.as_tuple, eps)
            state, r, done, info = env.step(a) # storing prime state

            if done:
                done_mask = 0.0

            if r != 0 or step > 60:
                memory.add((state.position, state.speed,
                    state.tl, state.p_position,
                    state.p_speed, state.p_tl,
                    a, r, done_mask))

            state.swap()  # state = state_prime

            if memory.size > batch_size:
                train_net(q, q_target, memory, optimizer, batch_size, gamma)

            step += 1
            total_steps += 1

        #  clear memory each mem_refill epoch
        if epoch%mem_refill == 0 and epoch != 0:
            memory.refill()
        pbar.update(1)

        # soft update weights at the end of epoch
        soft_update(q, q_target, 0.01)

        if info[4]/info[5] < min_wait and epoch > 50:
            torch.save(q.state_dict(), f'../model/dqn_{epoch}.pt')
            min_wait = info[4]/info[5]
        if epoch == 1599:
            torch.save(q.state_dict(), f'../model/dqn_{epoch}_final.pt')

    print('finished training')

max_stars_repo_name: suessmann/TLC-DQN


text:
import threading
import math
import cv2 as cv
import numpy as np
import imutils
from statistics import mean

class Strategy(threading.Thread):

    # threading properties
    running = True
    lock = None
    placeable_points = None

    def __init__(self):
        #initalize thread from thread class
        threading.Thread.__init__(self)
        #create a lock
        self.lock = threading.Lock()

        self.placeable_points = None

    def get_placement_points(self, target_points, placeable_points):
        for target_point in target_points:
            min_distance = 999999
            for placeable_point in placeable_points:
                distance = int(math.sqrt((target_point[1]-placeable_point[1])**2 + (target_point[0]-placeable_point[0])**2))
                if distance < min_distance:
                    min_distance = distance
                    print(min_distance)

    def find_placeable_points(self, image):
        blur = cv.GaussianBlur(image, (5,5), cv.BORDER_DEFAULT)
        gray = cv.cvtColor(blur, cv.COLOR_BGR2GRAY)
        kernel = np.ones((5,5), np.uint8)
        erode = cv.erode(gray, kernel, iterations=2)
        thresh = cv.threshold(erode, 100, 255, cv.THRESH_BINARY)[1]

        # Gets rid of areas that are smaller than desired
        cnts = cv.findContours(thresh, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)
        cnts = sorted(cnts, key=cv.contourArea, reverse=True)
        rect_areas = []
        # this method of using boundingrect may be problematic since its detecting only rectangles
        for c in cnts:
            (x, y, w, h) = cv.boundingRect(c)
            rect_areas.append(w * h)
        avg_area = mean(rect_areas)
        for c in cnts:
            (x, y, w, h) = cv.boundingRect(c)
            cnt_area = w * h
            if cnt_area < 4 * avg_area:
                thresh[y:y + h, x:x + w] = 0

        # Find placeable edge points for each contour
        placeable_edge_cnts, h = cv.findContours(thresh, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
        approx_edge_cnts = []

        # approximate points so there are less points for each contour
        for contours in placeable_edge_cnts:
            epsilon = 0.00025*cv.arcLength(contours,True)
            approx_edge_points = cv.approxPolyDP(contours,epsilon,False)
            approx_edge_cnts.append(approx_edge_points)

        self.lock.acquire()
        self.placeable_points = approx_edge_cnts
        self.lock.release()

    def stop(self):
        self.running = False

    def run(self):
        # TODO: you can write your own time/iterations calculation to determine how fast this is
        while self.running:
            pass


text:
#%%
from pyAudioAnalysis import audioAnalysis
from pyAudioAnalysis import audioBasicIO
from pyAudioAnalysis import ShortTermFeatures
from pyAudioAnalysis import MidTermFeatures
import matplotlib.pyplot as plt
import subprocess
import numpy as np

#extract some audio

VIDEOFILE = "../data/raw/8/replay.mp4"
AUDIOFILE = "./extracted.wav"
FEATUREFILE = "./extracted.ft"

command = f"ffmpeg -i {VIDEOFILE} -vn {AUDIOFILE} -y"

subprocess.call(command, shell=True)

[Fs, x] = audioBasicIO.read_audio_file(AUDIOFILE)
x = audioBasicIO.stereo_to_mono(x)

midF, shortF, midFNames = MidTermFeatures.mid_feature_extraction(x,Fs, 0.1*Fs,0.05*Fs,0.05*Fs,0.025*Fs)

np.save(FEATUREFILE, midF)
np.savetxt(FEATUREFILE + ".csv", midF.T, delimiter=",", header=",".join(midFNames))
#%%
audioAnalysis.thumbnailWrapper(AUDIOFILE,50)
#explore the audio

audioAnalysis.fileSpectrogramWrapper(AUDIOFILE)

audioAnalysis.fileChromagramWrapper(AUDIOFILE)

audioAnalysis.beatExtractionWrapper(AUDIOFILE, True)
#%%
var = 48
print(f"{var} : {3/199:.5f}")
#%%

max_stars_repo_name: helldragger/MetaWatch

max_issues_repo_name: helldragger/MetaWatch

max_forks_repo_name: helldragger/MetaWatch


text:
# -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import abel
import os
import bz2

import matplotlib.pylab as plt

# This example demonstrates ``linbasex`` inverse Abel transform
# of a velocity-map image of photoelectrons from O2- photodetachment at 454 nm.
# Measured at  The Australian National University
# J. Chem. Phys. 133, 174311 (2010) DOI: 10.1063/1.3493349

# Load image as a numpy array - numpy handles .gz, .bz2
imagefile = bz2.BZ2File('data/O2-ANU1024.txt.bz2')
IM = np.loadtxt(imagefile)

if os.environ.get('READTHEDOCS', None) == 'True':
    IM = IM[::2,::2]
# the [::2, ::2] reduces the image size x1/2, decreasing processing memory load
# for the online readthedocs.org

# Image center should be mid-pixel and the image square,
# `center=convolution` takes care of this

un = [0, 2]  # spherical harmonic orders
proj_angles = np.arange(0, 2*np.pi, np.pi/20) # projection angles
# adjust these parameter to 'improve' the look
smoothing = 0.9  # smoothing Gaussian 1/e width
threshold = 0.01 # exclude small amplitude Newton spheres
# no need to change these
radial_step = 1
clip = 0

# linbasex inverse Abel transform
LIM = abel.Transform(IM, method="linbasex", origin="convolution",
                     center_options=dict(square=True),
                     transform_options=dict(basis_dir=None, return_Beta=True,
                                            legendre_orders=un,
                                            proj_angles=proj_angles,
                                            smoothing=smoothing,
                                            radial_step=radial_step, clip=clip,
                                            threshold=threshold))

# angular, and radial integration - direct from `linbasex` transform
# as class attributes
radial = LIM.radial
speed  = LIM.Beta[0]
anisotropy = LIM.Beta[1]

# normalize to max intensity peak i.e. max peak height = 1
speed /= speed[200:].max()  # exclude transform noise near centerline of image

# plots of the analysis
fig = plt.figure(figsize=(11, 5))
ax1 = plt.subplot2grid((1, 2), (0, 0))
ax2 = plt.subplot2grid((1, 2), (0, 1))

# join 1/2 raw data : 1/2 inversion image
inv_IM = LIM.transform
cols = inv_IM.shape[1]
c2 = cols//2
vmax = IM[:, :c2-100].max()
inv_IM *= vmax/inv_IM[:, c2+100:].max()
JIM = np.concatenate((IM[:, :c2], inv_IM[:, c2:]), axis=1)

# raw data
im1 = ax1.imshow(JIM, origin='upper', aspect='auto', vmin=0, vmax=vmax)
ax1.set_xlabel('column (pixels)')
ax1.set_ylabel('row (pixels)')
ax1.set_title('VMI, inverse Abel: {:d}x{:d}'.format(*inv_IM.shape),
              fontsize='small')

# Plot the 1D speed distribution and anisotropy parameter ("looks" better
# if multiplied by the intensity)
ax2.plot(radial, speed, label='speed')
ax2.plot(radial, speed*anisotropy, label=r'anisotropy $\times$ speed')
ax2.set_xlabel('radial pixel')
row, cols = IM.shape
ax2.axis(xmin=100*cols/1024, xmax=500*cols/1024, ymin=-1.5, ymax=1.8)
ax2.set_title("speed, anisotropy parameter", fontsize='small')
ax2.set_ylabel('intensity')
ax2.set_xlabel('radial coordinate (pixels)')

plt.legend(loc='best', frameon=False, labelspacing=0.1, fontsize='small')
plt.suptitle(
r'linbasex inverse Abel transform of O$_{2}{}^{-}$ electron velocity-map image',
             fontsize='larger')

# Save a image of the plot
plt.savefig("plot_example_linbasex.png", dpi=100)

# Show the plots
plt.show()

max_stars_repo_name: beverlyru/PyAbel

max_issues_repo_name: beverlyru/PyAbel

max_forks_repo_name: beverlyru/PyAbel


text:
#!/bin/env python

import SimpleITK as sitk
import numpy as np
import math
import itertools
from timeit import Timer
import matplotlib.pyplot as plt

import os

num_threads = int(os.environ['ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS'])
sz = [128*3]*3
t_repeat = 3
t_number = 1
factors = [f for f in range(2, 34) if sz[-1]%(f*num_threads)==0]

print "Using factors:", factors

n = np.random.normal(0, scale=1.0, size=sz).astype(np.float32)
img = sitk.GetImageFromArray(n)

def np_shrink1(array,b):
    trim = [ s%b for s in n.shape]
    accum = np.zeros([ s//b for s in array.shape])
    for offset in itertools.product(range(b), repeat=3):
        #print offset
        accum += array[offset[0]::b, offset[1]::b, offset[2]::b]
    return accum

# Bin Shrink
def my_shrink1(img, shrink):
    return sitk.BinShrink(img,[shrink]*3)

def my_shrink2(img, shrink):
    return sitk.BinShrink2(img,[shrink]*3)

def my_smooth(img,shrink):
    sigma = shrink*0.7*img.GetSpacing()[0]
    return sitk.Shrink(sitk.SmoothingRecursiveGaussian(img, sigma), [shrink]*3)

def my_mean(img,shrink):
    r = (shrink+1)//2
    return sitk.Shrink(sitk.Mean(img, [r]*3), [shrink]*3)

#factors = [2,4,8]

med_times0 = []
print "Timing np shrink1..."
for factor in factors:
    timer = Timer( lambda: np_shrink1(n, factor) )
    times = timer.repeat( repeat=t_repeat, number= t_number )
    med_times0.append(np.median(times) )
print med_times0

med_times1 = []
print "Timing shrink1..."
for factor in factors:
    timer = Timer( lambda: my_shrink1(img, factor) )
    times = timer.repeat( repeat=t_repeat, number= t_number )
    med_times1.append(np.median(times) )
print med_times1

print "Timing shrink2..."
med_times2 = []
for factor in factors:
    timer = Timer( lambda: my_shrink2(img, factor) )
    times = timer.repeat( repeat=t_repeat, number= t_number )
    med_times2.append(np.median(times) )

print "Timing smooth..."
med_times3 = []
for factor in factors:
    timer = Timer( lambda: my_smooth(img, factor) )
    times = timer.repeat( repeat=t_repeat, number= t_number )
    med_times3.append(np.median(times) )

print "Timing mean..."
med_times4 = []
for factor in factors:
    print "computing mean with factor", factor
    timer = Timer( lambda: my_mean(img, factor) )
    times = timer.repeat( repeat=t_repeat, number= t_number )
    med_times4.append(np.median(times) )

plt.plot(factors,med_times1, 'ro:', label="BinShrink")
plt.plot(factors,med_times2, 'b^:', label="BinShrink2")
plt.plot(factors,med_times3, 'cs:', label="Gaussian Shrink")
plt.plot(factors,med_times4, 'gx:', label="Mean Shrink")
plt.yscale("log")
plt.legend(loc='upper center')
plt.xlabel("Shrink Size")
plt.ylabel("Time (seconds)")
plt.show()

max_stars_repo_name: blowekamp/itkBinShrink

max_issues_repo_name: blowekamp/itkBinShrink

max_forks_repo_name: blowekamp/itkBinShrink


text:
#!/usr/bin/env python
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
""" OverTake agent for kyber sim."""

import random
import numpy as np

from xt.agent import Agent
from xt.framework.register import Registers
from xt.framework.comm.message import message

VIEW_RANGE = 125.0  # meter
TIME_RANGE = 12.5  # seconds
SPEED_RANGE = 300.0

@Registers.agent.register
class OverTakeKyber(Agent):
    """
    OverTakeKyber class base on AsyncAgent class
    """
    def __init__(self, env, alg, agent_config, **kwargs):
        super(OverTakeKyber, self).__init__(env, alg, agent_config, **kwargs)
        self.epsilon = 1.0
        self.episode_count = agent_config.get("episode_count", 100000)
        self.goal_lane = 1
        self.last_lane_remains = 2500.0
        self.step = 0
        self.track_length = 6000.0
        self.lanes_num = 3.0
        self.total_change_steps = 0
        self.total_change_count = 0
        self.ave_steps = 0
        self.test_info = ["str", "turn", "ab", "ab_1", "rel", "step", "real_step"]

    def infer_action(self, state, use_explore):
        s_t = self._state_convert(state)
        # if explore action
        if use_explore and random.random() < self.epsilon:
            action = np.random.randint(0, 3)
        elif use_explore:
            # Get Q values with deliver for each action.
            send_data = message(s_t, agent_id=self.id)
            self.send_explorer.send(send_data)
            action = self.recv_explorer.recv()
        else:
            action = self.alg.predict(s_t)

        # update episode value
        if use_explore:
            self.epsilon -= 1.0 / self.episode_count
            self.epsilon = max(0.01, self.epsilon)

        # update transition data
        self.transition_data.update({
            "cur_state": s_t,
            "action": action,
        })
        return action

    def handle_env_feedback(self, next_raw_state, reward, done, info, use_explore):
        s_t1 = self._state_convert(next_raw_state)
        if use_explore:
            r_t = self._get_reward(
                self.transition_data["cur_state"],
                s_t1,
                self.transition_data["action"],
                info,
            )
        else:
            r_t = self._get_test_reward(next_raw_state)

        self.transition_data.update({"next_state": s_t1, "reward": r_t, "done": done, "info": info})

        # deliver this transition data to learner, trigger train process.
        if use_explore:
            train_data = {k: [v] for k, v in self.transition_data.items()}
            train_data = message(train_data, agent_id=self.id)
            self.send_explorer.send(train_data)

        return self.transition_data

    def do_one_interaction(self, raw_state, use_explore=True):
        """Use the Agent do once interaction.
        Owing to the env.step is non-normal,
        here, rewrite the `do_one_interaction` function.
        :param raw_state:
        :param use_explore:
        :return:
        """
        action = self.infer_action(raw_state, use_explore)

        next_raw_state, reward, done, info = self._agent_step(raw_state, action)

        # next_raw_state, reward, done, info = self.env.step(action)
        return self.handle_env_feedback(next_raw_state, reward, done, info, use_explore)

    @staticmethod
    def _get_reward(state, new_state, action, info):
        """refer to AgentBase"""
        # state = self.convert(state)
        # new_state = self.convert(new_state)
        speed_cur = state[6] * SPEED_RANGE
        speed_sec = speed_cur / 3.6
        front_car_left = state[0] * TIME_RANGE * speed_sec
        front_car_mid = state[1] * TIME_RANGE * speed_sec + 0.1
        front_car_right = state[2] * TIME_RANGE * speed_sec
        speed = new_state[6]
        speed_ = speed * 300.0

        if info[-1][1] is False:
            reward = -100
        elif action != 0:
            k_l = (np.clip((front_car_left / front_car_mid), 0.5, 1.5) if action == 1 else np.clip(
                (front_car_right / front_car_mid), 0.5, 1.5))
            if k_l > 1.0:
                k_l = 1.5
            elif k_l < 1.0:
                k_l = 0.5
            reward = 0.3 * k_l * speed_
            if k_l == 1 and state[1] == 0.8:
                reward = -30
        else:
            reward = speed_
        return reward

    @staticmethod
    def _get_test_reward(new_raw_state):
        """refer to AgentBase"""
        sp_ = new_raw_state[3]
        reward = sp_
        return reward

    def _state_convert(self, raw_state):
        """convert raw state to training state"""
        variables_dict = dict()
        variables_dict["s_t"] = np.hstack((0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.0, 0.0, 1.0))
        variables_dict["v_t"] = np.hstack((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
        variables_dict["add_s_t"] = np.hstack((0.8, 0.8))
        variables_dict["add_v_t"] = np.hstack((0.0, 0.0))
        variables_dict["flag_t"] = 0.0
        variables_dict["add_dist_min"] = np.hstack((1000.0, 1000.0, 1000.0, 1000.0))
        variables_dict["dist_min"] = np.hstack((1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0))
        variables_dict["ego_lane"] = raw_state[16]
        variables_dict["lane_ids"] = raw_state[18]
        variables_dict["ego_lane"] = variables_dict["lane_ids"].index(variables_dict["ego_lane"])
        if variables_dict["ego_lane"] == 0 or variables_dict["ego_lane"] == 2:
            variables_dict["s_t"][variables_dict["ego_lane"]] = 1.0
            variables_dict["s_t"][variables_dict["ego_lane"] + 3] = 1.0
            variables_dict["flag_t"] = 1 if variables_dict["ego_lane"] == 0 else -1

        variables_dict["ego_raw_speed"] = raw_state[3]
        variables_dict["filter_speed"] = (variables_dict["ego_raw_speed"]
                                          if variables_dict["ego_raw_speed"] >= 10.0 else 10.0)
        variables_dict["s_t"][6] = variables_dict["ego_raw_speed"] / SPEED_RANGE
        objects = raw_state[-1]
        # print("ego_speed",ego_raw_speed,"ego_lane",ego_lane)
        if objects[0] is not None:
            # for i in range(len(objects)):
            for i, _object in enumerate(objects):
                lane_id = objects[i][0]
                dist = abs(objects[i][1]) * np.sign(objects[i][1])
                speed = objects[i][2]
                pre_post = np.sign(dist)
                flag = 0 if pre_post == 1.0 else 1

                if abs(dist) < VIEW_RANGE:
                    for j in range(3):
                        adjacent_lane = variables_dict["ego_lane"] - 1 + j
                        dist_index = j + flag * 3
                        if (lane_id == adjacent_lane and abs(dist) < variables_dict["dist_min"][dist_index]):
                            self.min_dist(
                                variables_dict["v_t"],
                                variables_dict["s_t"],
                                dist_index,
                                speed,
                                dist,
                                variables_dict["filter_speed"],
                            )
                            variables_dict["dist_min"][dist_index] = abs(dist)

                    if abs(dist) < variables_dict["add_dist_min"][flag]:
                        if (variables_dict["ego_lane"] == 0 and lane_id == variables_dict["ego_lane"] + 2
                                or variables_dict["ego_lane"] == len(variables_dict["lane_ids"]) - 1
                                and lane_id == variables_dict["ego_lane"] - 2):
                            self.min_dist(
                                variables_dict["add_v_t"],
                                variables_dict["add_s_t"],
                                flag,
                                speed,
                                dist,
                                variables_dict["filter_speed"],
                            )

        state = np.hstack((
            variables_dict["s_t"],
            variables_dict["v_t"],
            variables_dict["add_s_t"],
            variables_dict["add_v_t"],
            variables_dict["flag_t"],
        ))
        return state

    @staticmethod
    def min_dist(v_t, s_t, flag, speed, dist, f_speed):
        """compute minimum distance"""
        v_t[flag] = np.clip(speed / f_speed, 0.0, 2.0)
        s_t[flag] = np.clip(3.6 * abs(dist) / (f_speed * TIME_RANGE), 0, 0.8)

    @staticmethod
    def collision_detection(objects, target_lane):
        """detect collision"""
        if objects is list:
            return False
        value = False
        goal_front_dist = 1000
        goal_back_dist = -1000
        # for index in range(len(objects)):
        for index, _object in enumerate(objects):
            lane_id = objects[index][0] + 1
            dist = abs(objects[index][1]) * np.sign(objects[index][1])
            if lane_id == target_lane and 0 < dist < goal_front_dist:
                goal_front_dist = dist
            if lane_id == target_lane and 0 > dist > goal_back_dist:
                goal_back_dist = dist

        if goal_front_dist < 6 or goal_back_dist > -6:
            value = True
        return value

    def check_state(self, state, action):
        """check_state"""
        if action == 0:
            return "str"
        s_t = self._state_convert(state)
        front_car_left = s_t[0]
        front_car_mid = s_t[1]
        front_car_right = s_t[2]

        if front_car_mid == 0.8:
            return "ab"
        if (action == 1 and front_car_left == 1.0) or (action == 2 and front_car_right == 1.0):
            return "ab"
        if (action == 1 and front_car_left < front_car_mid) or (action == 2 and front_car_right < front_car_mid):
            if state[16] == 1:
                return "ab_1"
            return "rel"

        return "turn"

    def _action_check(self, raw_state, action):
        """check action"""
        variables_dict = dict()
        variables_dict["state_follow"] = 4
        # state_follow = 4
        variables_dict["state_failed"] = 3
        variables_dict["state_process"] = 2
        sc_ = False
        variables_dict["target_lane"] = None
        danger = False
        step_count = 0
        if action == 0:
            variables_dict["env_action"] = 0
            variables_dict["steps"] = random.randint(10, 20)
        else:
            if action == 1:
                variables_dict["env_action"] = 1
                variables_dict["target_lane"] = raw_state[15]
            else:
                variables_dict["env_action"] = 2
                variables_dict["target_lane"] = raw_state[17]
            variables_dict["steps"] = 120
        if action == 0:
            sc_ = True
            for i in range(variables_dict["steps"]):
                state, reward, done, info = self.env.step(variables_dict["env_action"], 0)
                step_count += 1
                if done:
                    break
        else:
            variables_dict["exec_action"] = variables_dict["env_action"]
            for i in range(variables_dict["steps"]):
                state, reward, done, info = self.env.step(variables_dict["exec_action"], 0)
                step_count += 1
                lane_chang_state = state[20]
                cur_lane = state[16]
                if (lane_chang_state == variables_dict["state_failed"] or variables_dict["target_lane"] == -1 or done):
                    break
                if cur_lane == variables_dict["target_lane"]:
                    if lane_chang_state == variables_dict["state_follow"]:
                        sc_ = True
                        break
                    if lane_chang_state == variables_dict["state_process"]:
                        variables_dict["exec_action"] = 0
                objects = state[-1]
                variables_dict["collision"] = self.collision_detection(objects, variables_dict["target_lane"])
                if variables_dict["collision"] is True:
                    danger = True
        return sc_, danger, state, reward, done, info, step_count

    def _agent_step(self, raw_state, action):
        """refer to AgentBase"""
        sc_, danger, state, reward, done, info, step_count = self._action_check(raw_state, action)
        if danger is True:
            sc_ = False

        if state[19][2] < 20:
            done = True
        info = []
        action_info = self.check_state(raw_state, action)
        info.append((action_info, 1))
        info.append(("step", step_count))
        real_count = state[1] - raw_state[1]
        if real_count < 0:
            real_count = state[1]
        info.append(("real_step", real_count))
        info.append(("danger", danger))
        info.append(("sc", sc_))
        if state[1] < raw_state[1]:
            print("#########error count $$$$$$$$$$", state[1], raw_state[1])
        return state, reward, done, info

max_stars_repo_name: Huawei-Ascend/modelzoo

max_issues_repo_name: Huawei-Ascend/modelzoo

max_forks_repo_name: Huawei-Ascend/modelzoo


text:
# -*- coding: utf-8 -*-

import numpy as np
import neuralnetwork as nn
import userInteractivity as UI

class trainer:
    def __init__(self):
        self.__network = nn.network()
        self.__trainingLoaded = False
        self.__validationLoaded = False
        self.__miniBatchSize = []
        self.__inputSize = []
        self.__learningRate = []
        self.__nEpochs = []
        self.__regularisationConst = []

    def initialiseNetwork(self, structure, *seed):
        self.__network.setStructure(structure)
        if len(seed[0])==0:
            np.random.seed()
        else:
            np.random.seed(seed[0])
        for i in range(len(structure)-1):
            self.__network.setConnectionWeights(i, np.random.random((structure[i+1],structure[i]))*2-1)
            self.__network.setNeuronBias(i+1, range(structure[i+1]), np.random.random(structure[i+1])*2-1)

    def run(self, **kwargs):
        if not self.__trainingLoaded:
            raise NameError('Training data not loaded!')
        if not self.__validationLoaded:
            raise NameError('Validation data not loaded!')
        self.__UIelements = UI.elements(kwargs)
        trainingIndices = list(range(len(self.__trainingSetLabels)))
        validationIndices = list(range(len(self.__validationSetLabels)))
        lossHistoryTrainer = []
        lossHistoryValidation = []
        errorHistory = []
        for epoch in range(int(self.getEpochs())):
            self.__UIelements.writeToLog('Executing training epoch {:n} of {:n}...'.format(epoch+1,self.getEpochs()))

            # Back propagation / training
            np.random.shuffle(trainingIndices)
            lastProgressUpdate = 0
            totalTrainingCost = 0
            for miniBatch in range(int(self.__inputSize / self.__miniBatchSize)):
                deltaSum = [np.array([0])]
                for layerSize in self.__network.getStructure()[1:]:
                    deltaSum.append(np.zeros(layerSize))
                grads = []
                for layer in range(len(self.__network.getStructure())-1):
                    grads.append(np.zeros((self.__network.getStructure()[layer+1], self.__network.getStructure()[layer])))

                for trainingExample in range(self.__miniBatchSize):
                    currentIndex = miniBatch * self.__miniBatchSize + trainingExample
                    percentProgress = currentIndex / (int(self.__inputSize / self.__miniBatchSize) * self.__miniBatchSize)
                    if percentProgress > lastProgressUpdate + 0.005:
                        self.__UIelements.updateProgressBar(percentProgress*100)
                        lastProgressUpdate = percentProgress

                    # forward calculation
                    trainingImage, actualLabel = self.__getTrainingExample(trainingIndices[currentIndex])
                    exampleCost = self.__exampleCost(trainingImage.reshape(784), actualLabel)
                    totalTrainingCost += exampleCost

                    # back propagation of errors
                    deltas = [np.array([0])]
                    for layerSize in self.__network.getStructure()[1:]:
                        deltas.append(np.zeros(layerSize))
                    targetOutputActivations = np.zeros(self.__network.getStructure()[-1])
                    targetOutputActivations[actualLabel] = 1
                    outputActivations = self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))
                    deltas[len(self.__network.getStructure())-1] = (outputActivations - targetOutputActivations) * (outputActivations) * (1 - outputActivations)
                    for layer in range(len(self.__network.getStructure())-1,0,-1):
                        layerActivation = self.__network.getNeuronActivation(layer-1, range(self.__network.getStructure()[layer-1]))
                        deltas[layer-1] = (np.transpose(self.__network.getConnectionWeights(layer-1)) @ deltas[layer]) * (layerActivation) * (1 - layerActivation)

                    # accumulators
                    for layer in range(len(self.__network.getStructure())-1):
                        deltaSum[layer+1] += deltas[layer+1]
                        grads[layer] += deltas[layer+1].reshape(len(deltas[layer+1]),1) * self.__network.getNeuronActivation(layer, range(self.__network.getStructure()[layer]))

                # gradient descent
                for layer in range(len(self.__network.getStructure())-1):
                    weights = self.__network.getConnectionWeights(layer)
                    weights -= (self.__learningRate / self.__miniBatchSize) * grads[layer] + self.__regularisationConst * weights
                    self.__network.setConnectionWeights(layer, weights)

                    biases = self.__network.getNeuronBias(layer+1, range(self.__network.getStructure()[layer+1]))
                    biases -= (self.__learningRate / self.__miniBatchSize) * deltaSum[layer+1]
                    self.__network.setNeuronBias(layer+1, range(self.__network.getStructure()[layer+1]), biases)

            lossHistoryTrainer.append(totalTrainingCost / self.__inputSize)

            # Validation (random subset of test set)
            np.random.shuffle(validationIndices)
            totalValidationCost = 0
            totalErrors = 0
            validationSetSize = max(int(self.__inputSize/60), 500)
            for validationIndex in range(validationSetSize):
                validationImage, actualLabel = self.__getValidationExample(validationIndices[validationIndex])
                testImageCost = self.__exampleCost(validationImage.reshape(784), actualLabel)
                totalValidationCost += testImageCost
                if actualLabel != np.argmax(self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))):
                    totalErrors += 1
            lossHistoryValidation.append(totalValidationCost / validationSetSize)
            errorHistory.append(10 * totalErrors / validationSetSize)

            self.__UIelements.drawGraphs(lossHistoryTrainer, lossHistoryValidation, errorHistory)
            self.__UIelements.writeToLog('done.\n')

            if 'saveIntervals' in kwargs and 'saveFileRoot' in kwargs:
                if kwargs['saveFileRoot'].endswith('.nn'):
                    fileRoot = kwargs['saveFileRoot'][:-len('.nn')]
                else:
                    fileRoot = kwargs['saveFileRoot']
                indexLength = int(np.ceil(np.log10(max(kwargs['saveIntervals'])+1)))
                if epoch+1 in kwargs['saveIntervals']:
                    saveFile = fileRoot + '{:0{}d}.nn'.format(epoch+1, indexLength)
                    nn.saveNetwork(self.__network, saveFile)

    def __exampleCost(self, inputLayer, desiredOutput):
        self.__network.setNeuronActivation(0, range(self.__network.getStructure()[0]), inputLayer)
        self.__network.evaluate()
        target = np.zeros(self.__network.getStructure()[-1])
        target[desiredOutput] = 1
        return np.sum((self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))-target)**2)

    def setTrainingSet(self, trainingSetImages, trainingSetLabels):
        self.__trainingSetImages = trainingSetImages
        self.__trainingSetLabels = trainingSetLabels
        self.__trainingLoaded = True

    def __getTrainingExample(self, indices):
        images = self.__trainingSetImages[indices]
        labels = self.__trainingSetLabels[indices]
        return images, labels

    def setValidationSet(self, validationSetImages, validationSetLabels):
        self.__validationSetImages = validationSetImages
        self.__validationSetLabels = validationSetLabels
        self.__validationLoaded = True

    def __getValidationExample(self, indices):
        images = self.__trainingSetImages[indices]
        labels = self.__trainingSetLabels[indices]
        return images, labels

    def setNetwork(self, network):
        self.__network = network

    def getNetwork(self):
        return self.__network

    def setMiniBatchSize(self, miniBatchSize):
        self.__miniBatchSize = miniBatchSize

    def getMiniBatchSize(self):
        return self.__miniBatchSize

    def setInputSize(self, inputSize):
        self.__inputSize = inputSize

    def getInputSize(self):
        return self.__inputSize

    def setEpochs(self, nEpochs):
        self.__nEpochs = nEpochs

    def getEpochs(self):
        return self.__nEpochs

    def setLearningRate(self, learningRate):
        self.__learningRate = learningRate

    def getLearningRate(self):
        return self.__learningRate

    def setRegularisationConst(self, regularisationConst):
        self.__regularisationConst = regularisationConst

    def getRegularisationConst(self):
        return self.__regularisationConst

max_stars_repo_name: rpcook/mnist

max_issues_repo_name: rpcook/mnist

max_forks_repo_name: rpcook/mnist


text:
import numpy as np
from numpy.testing import assert_allclose, run_module_suite, assert_
import pytest
import multiprocessing

from repeater_mc import repeater_mc
from repeater_algorithm import repeater_sim
from protocol_units import *
from logging_utilities import *
from utility_functions import *
from optimize_cutoff import CutoffOptimizer

def test_opt_adaptive_trunc():
    """
    t_trunc should be increased
    """
    np.random.seed(1)
    parameters = {
        "protocol": (0, ),
        "p_gen": 0.1,
        "p_swap": 0.5,
        "w0": 0.99,
        "t_coh": 30,
        "t_trunc": 300
        }

    logging.info("Full tau optimization\n")
    opt = CutoffOptimizer(adaptive=True)
    best_cutoff_dict = opt.run(parameters)
    assert(best_cutoff_dict["memory_time"] == (6,))

def test_opt_adaptive_search_range():
    """
    The search range should be restricted
    """
    np.random.seed(3)
    parameters = {
        "protocol": (0, 0),
        "p_gen": 0.2,
        "p_swap": 0.6,
        "w0": 0.95,
        "t_coh": 300,
        "t_trunc": 400
        }

    opt = CutoffOptimizer(adaptive=True, popsize=5)
    best_cutoff_dict = opt.run(parameters)
    assert_allclose(best_cutoff_dict["memory_time"], (18, 30))

def test_opt_uniform():
    np.random.seed(0)
    parameters = {
        "protocol": (0, 0),
        "p_gen": 0.1,
        "p_swap": 0.8,
        "t_trunc": 500,
        "w0": 0.99,
        "t_coh": 400,
        }

    opt = CutoffOptimizer(
        opt_kind="uniform_de", adaptive=True)
    best_cutoff_dict = opt.run(parameters)
    assert_allclose(best_cutoff_dict["memory_time"], (45, 45))

max_stars_repo_name: BoxiLi/repeater-cut-off-optimization

max_issues_repo_name: BoxiLi/repeater-cut-off-optimization

max_forks_repo_name: BoxiLi/repeater-cut-off-optimization

reason: import numpy,from numpy


text:
# Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

import pytest
import numpy as np
import numpy.testing as npt
from astropy.utils.misc import NumpyRNGContext

from ..statistics.lm_seg import Lm_Seg

def test_lmseg():
    x = np.linspace(0, 10, 200)

    with NumpyRNGContext(12129):
        y = 2 + 2 * x * (x < 5) + (5 * x - 15) * (x >= 5) + \
            np.random.normal(0, 0.1, 200)

    model = Lm_Seg(x, y, 3)
    model.fit_model(tol=1e-3, verbose=False)

    npt.assert_approx_equal(5.0, model.brk, significant=2)
    npt.assert_allclose([2.0, 5.0], model.slopes, rtol=0.1)

def test_lmseg_weighted():
    x = np.linspace(0, 10, 200)

    with NumpyRNGContext(12129):
        yerr = np.random.normal(0, 0.1, 200)
        y = 2 + 2 * x * (x < 5) + (5 * x - 15) * (x >= 5) + \
            yerr

    model = Lm_Seg(x, y, 3, weights=yerr**-2)
    model.fit_model(tol=1e-3, verbose=True)

    npt.assert_approx_equal(5.0, model.brk, significant=2)
    npt.assert_allclose([2.0, 5.0], model.slopes, rtol=0.1)

max_stars_repo_name: CFD-UTSA/Turbulence-stars

max_issues_repo_name: CFD-UTSA/Turbulence-stars

max_forks_repo_name: CFD-UTSA/Turbulence-stars

reason: import numpy,from astropy


text:
import torch
import torch.nn as nn
import itertools
import numpy as np
import random

zip = getattr(itertools, 'izip', zip)

def get_none_id(type_filename):
    with open(type_filename, encoding='utf-8') as type_file:
        for line in type_file:
            ls = line.strip().split()
            if ls[0] == "None":
                return int(ls[1])

def get_distribution(type_filename):
    with open(type_filename, encoding='utf-8') as type_file:
        sum = 0
        prob_lst = []
        for line in type_file:
            ls = line.strip().split()
            sum += int(ls[2])
            prob_lst.append(int(ls[2]))
        prob_lst = [(c+1e-10) / float(sum) for c in prob_lst]
    return prob_lst

def get_distribution_from_list(labels, max_index=None):
    labels = [labels[i][0] for i in range(len(labels))]
    #print(labels)
    if max_index is None:
        max_index = max(labels) + 1
    count = [0] * (max_index)
    for i, item in enumerate(labels):
        count[item] += 1
    prob = [item / len(labels) for item in count]
    return prob

def to_scalar(var):
    return var.view(-1).data.tolist()[0]

def log_sum_exp(vec):
    # vec: B * L * M
    # output: B * 1 * M
    _, idx = torch.max(vec, 1)  # B * 1 * M
    max_score = torch.gather(vec, 1, idx)  # B * 1 * M
    return max_score + torch.log(torch.sum(torch.exp(vec - max_score.expand_as(vec)), 1))  # B * 1 * M

def soft_max(vec, mask):
    batch_size = vec.size(0)
    _, idx = torch.max(vec, 1)  # B * 1
    idx = idx.view(batch_size, 1)
    max_score = torch.gather(vec, 1, idx)  # B * 1
    max_score = max_score.view(batch_size, 1)
    exp_score = torch.exp(vec - max_score.expand_as(vec))  # B * L
    exp_score = exp_score * mask  # B * L
    exp_score_sum = torch.sum(exp_score, 1).view(batch_size, 1).expand_as(exp_score)
    prob_score = exp_score / exp_score_sum
    return prob_score

def load_embedding(emb_file, randIni=True):
    fin = open(emb_file, 'r')
    line = fin.readline().split(' ')
    size = int(line[0])
    embLen = int(line[1])
    if randIni:
        bias = np.sqrt(3.0 / embLen)
        pos_embedding_tensor = torch.rand(size, embLen) * 2 * bias - bias
        neg_embedding_tensor = torch.FloatTensor(size, embLen).zero_()
    else:
        pos_embedding_array = list()
        neg_embedding_array = list()
        for ind in range(size):
            line = fin.readline()
            if line.isspace():
                break
            line = line.split(' ')
            vector = list(map(lambda t: float(t), filter(lambda n: n and not n.isspace(), line)))
            pos_embedding_array.append(vector)
        fin.readline()
        for ind in range(size):
            line = fin.readline()
            if line.isspace():
                break
            line = line.split(' ')
            vector = list(map(lambda t: float(t), filter(lambda n: n and not n.isspace(), line)))
            neg_embedding_array.append(vector)
        pos_embedding_tensor = torch.FloatTensor(np.asarray(pos_embedding_array))
        neg_embedding_tensor = torch.FloatTensor(np.asarray(neg_embedding_array))
    return size, embLen, pos_embedding_tensor, neg_embedding_tensor

def initialize_embedding(feature_file, embLen):
    fin = open(feature_file, 'r')
    lines = fin.readlines()
    size = len(lines)
    bias = np.sqrt(3.0 / embLen)
    embedding_tensor = torch.rand(size, embLen) * 2 * bias - bias
    return size, embedding_tensor

def load_corpus(corpus):
    fin = open(corpus, 'r')
    line = fin.readline()
    size = int(line)
    feature_list = list()
    label_list = list()
    type_size = -1
    for line in fin:
        if line.isspace():
            break
        line = line.split('\t')
        fv = torch.LongTensor()
        feature_list.append(torch.LongTensor(list(map(lambda t: int(t), line[2].split(' ')))))
        label_vec = list(map(lambda t: int(t), line[4].split(' ')))
        tmp = max(label_vec)
        if type_size < tmp:
            type_size = tmp
        #if if_cuda:
        #    label_list.append(torch.cuda.LongTensor(label_vec))
        #else:
        label_list.append(torch.LongTensor(label_vec))
    type_size = type_size + 1
    type_list = list()
    for label in label_list:
        type_vec = torch.FloatTensor(type_size).zero_()
        type_vec[label] = 1
        type_vec = type_vec.view(1, -1)
        type_list.append(type_vec)
    return size, type_size, feature_list, label_list, type_list

def load_qa_corpus(corpus):
    fin = open(corpus, 'r')
    size = 0
    feature_list = list()
    for line in fin:
        if line.isspace():
            break
        size += 1
        line = line.strip('\r\n').split('\t')
        feature_list.append(torch.LongTensor(list(map(lambda t: int(t), line[1].split(',')))))
    return size, feature_list

def load_question_info(mention_question_file):
    pos_qapairs = []
    pos_qapair_to_question = {}
    question_to_qapairs = {}
    with open(mention_question_file, 'r') as fin:
        for line in fin:
            if line.isspace():
                break
            line = line.strip('\r\n').split('\t')
            mid = int(line[0])
            qid = int(line[1])
            if float(line[2]) == 1.0:
                pos_qapairs.append(mid)
                pos_qapair_to_question[mid] = qid
                if qid in question_to_qapairs:
                    pos_mids = question_to_qapairs[qid][0]
                    pos_mids.append(mid)
                    question_to_qapairs[qid] = (pos_mids, question_to_qapairs[qid][1])
                else:
                    question_to_qapairs[qid] = ([mid], [])
            else:
                if qid in question_to_qapairs:
                    neg_mids = question_to_qapairs[qid][1]
                    neg_mids.append(mid)
                    question_to_qapairs[qid] = (question_to_qapairs[qid][0], neg_mids)
                else:
                    question_to_qapairs[qid] = ([], [mid])
    return pos_qapairs, pos_qapair_to_question, question_to_qapairs

def shuffle_data(ori_labels, ori_corpus):
    assert (len(ori_labels) == len(ori_corpus))
    index_shuf = list(range(len(ori_labels)))
    random.shuffle(index_shuf)
    labels = [ori_labels[i] for i in index_shuf]
    corpus = [ori_corpus[i] for i in index_shuf]
    return labels, corpus

def calcEntropy(batch_scores):
    # input: B * L
    # output: B
    batch_probs = nn.functional.softmax(batch_scores)
    return torch.sum(batch_probs * torch.log(batch_probs), 1).neg()

def calcInd(batch_probs):
    # input: B * L
    # output: B
    _, ind = torch.max(batch_probs, 1)
    return ind

def calcMaxProb(batch_scores):
    batch_probs = nn.functional.softmax(batch_scores)
    # batch_probs = batch_scores / sum(batch_scores)
    prob, _ = torch.max(batch_probs, 1)
    return prob

def dropout(input_tensor, ratio):
    keep_vec = torch.ge(torch.rand(input_tensor.size()), ratio)
    return input_tensor[keep_vec]

def resample(input_tensor, resample_num):
    resample_ind = torch.Tensor(resample_num).uniform_(0, input_tensor.size(0) - 1).long()
    return input_tensor[resample_ind].view(-1)

def clip_grad(parameters, max_value):
    parameters = list(filter(lambda p: p.grad is not None, parameters))
    for p in parameters:
        mask = p.grad.data.gt(max_value)
        p.grad.data[mask] = max_value
        mask = p.grad.data.lt(-max_value)
        p.grad.data[mask] = -max_value

def CrossValidation(pre_ind, pre_entropy, true_ind, noneInd, ratio=0.1, cvnum=100):
    f1score = 0.0
    recall = 0.0
    precision = 0.0
    meanBestF1 = 0.0
    valSize = int(np.floor(ratio * len(pre_ind)))
    data = [[pre_ind[ind], pre_entropy[ind], true_ind[ind]] for ind in range(0, len(pre_ind))]
    for cvind in range(cvnum):
        random.shuffle(data)
        val = data[0:valSize]
        eva = data[valSize:]

        # find best threshold
        max_ent = max(val, key=lambda t: t[1])[1]
        min_ent = min(val, key=lambda t: t[1])[1]
        stepSize = (max_ent - min_ent) / 100
        thresholdList = [min_ent + ind * stepSize for ind in range(0, 100)]
        ofInterest = 0
        for ins in val:
            if ins[2][0] != noneInd:
                ofInterest += 1
        bestThreshold = float('nan')
        bestF1 = float('-inf')
        for threshold in thresholdList:
            corrected = 0
            predicted = 0
            for ins in val:
                if ins[1] < threshold and ins[0] != noneInd:
                    predicted += 1
                    if ins[0] == ins[2][0]:
                        corrected += 1
            curF1 = 2.0 * corrected / (ofInterest + predicted)
            if curF1 > bestF1:
                bestF1 = curF1
                bestThreshold = threshold
        meanBestF1 += bestF1
        ofInterest = 0
        corrected = 0
        predicted = 0
        for ins in eva:
            if ins[2][0] != noneInd:
                ofInterest += 1
            if ins[1] < bestThreshold and ins[0] != noneInd:
                predicted += 1
                if ins[0] == ins[2][0]:
                    corrected += 1
        f1score += (2.0 * corrected / (ofInterest + predicted))
        recall += (1.0 * corrected / ofInterest)
        precision += (1.0 * corrected / (predicted + 0.00001))

    meanBestF1 /= cvnum
    f1score /= cvnum
    recall /= cvnum
    precision /= cvnum
    return f1score, recall, precision, meanBestF1

def eval_score(pre_ind, true_ind_dev, noneInd):
    eva = [[pre_ind[ind], true_ind_dev[ind]] for ind in range(0, len(pre_ind))]

    ofInterest = 0
    corrected = 0
    predicted = 0

    for ins in eva:
        if ins[1][0] != noneInd:
            ofInterest += 1
        if ins[0] != noneInd:
            predicted += 1
            if ins[0] == ins[1][0]:
                corrected += 1
    f1score = (2.0 * corrected / (ofInterest + predicted))
    recall = (1.0 * corrected / ofInterest)
    precision = (1.0 * corrected / (predicted + 1e-10))

    return f1score, recall, precision

def TuneThres(pre_ind, pre_entropy, true_ind, noneInd, thres_type='max', ratio=0.2, cvnum=100):
    # > for 'max' and < for 'entropy'
    if thres_type == 'max':
        SIGN = 1
    else:
        SIGN = -1

    f1score = 0.0
    recall = 0.0
    precision = 0.0
    val_f1 = 0.0

    valSize = int(np.floor(ratio * len(pre_ind)))
    data = [[pre_ind[ind], pre_entropy[ind], true_ind[ind]] for ind in range(0, len(pre_ind))]

    for cvind in range(cvnum):
        random.shuffle(data)
        val = data[0:valSize]
        eva = data[valSize:]

        # find best threshold
        max_ent = max(val, key=lambda t: t[1])[1]
        min_ent = min(val, key=lambda t: t[1])[1]
        stepSize = (max_ent - min_ent) / 100
        thresholdList = [min_ent + ind * stepSize for ind in range(0, 100)]

        ofInterest = 0
        for ins in val:
            if ins[2][0] != noneInd:
                ofInterest += 1

        bestThreshold = float('nan')
        bestF1 = float('-inf')
        for threshold in thresholdList:
            corrected = 0
            predicted = 0
            for ins in val:
                if SIGN * ins[1] > SIGN * threshold and ins[0] != noneInd:
                    predicted += 1
                    if ins[0] == ins[2][0]:
                        corrected += 1
            curF1 = 2.0 * corrected / (ofInterest + predicted)
            if curF1 > bestF1:
                bestF1 = curF1
                bestThreshold = threshold

        val_f1 += bestF1

        ofInterest = 0
        corrected = 0
        predicted = 0
        for ins in eva:
            if ins[2][0] != noneInd:
                ofInterest += 1
            if SIGN * ins[1] > SIGN * bestThreshold and ins[0] != noneInd:
                predicted += 1
                if ins[0] == ins[2][0]:
                    corrected += 1

        f1score += (2.0 * corrected / (ofInterest + predicted))
        recall += (1.0 * corrected / ofInterest)
        precision += (1.0 * corrected / (predicted + 1e-10))

    val_f1 /= cvnum
    f1score /= cvnum
    recall /= cvnum
    precision /= cvnum

    return f1score, recall, precision, val_f1

def eval_score_with_thres(pre_ind, pre_entropy, true_ind, noneInd, threshold, thres_type='max'):
    # print prediction after thresholding to a file
    if thres_type == 'max':
        SIGN = 1
    else:
        SIGN = -1

    # calculate the f1 without threshold on ndev set
    f1score = 0.0
    recall = 0.0
    precision = 0.0
    data = [[pre_ind[ind], pre_entropy[ind], true_ind[ind]] for ind in range(0, len(pre_ind))]

    ofInterest = 0
    for ins in data:
        if ins[2][0] != noneInd:
            ofInterest += 1
    corrected = 0
    predicted = 0
    for ins in data:
        if SIGN * ins[1] > SIGN * threshold and ins[0] != noneInd:
            predicted += 1
            if ins[0] == ins[2][0]:
                corrected += 1
    f1score = (2.0 * corrected / (ofInterest + predicted))
    recall = (1.0 * corrected / ofInterest)
    precision = (1.0 * corrected / (predicted + 1e-8))

    print(f1score, recall, precision)
    print(corrected, ofInterest, predicted)

    pre_ind_cutoff = pre_ind.clone()
    for ind in range(0, len(pre_ind)):
        ins = data[ind]
        if SIGN * ins[1] > SIGN * threshold:
            pre_ind_cutoff[ind] = pre_ind[ind]
        else:
            pre_ind_cutoff[ind] = noneInd

    return pre_ind_cutoff

def SampleBias(model, packer, label_list, feature_list, ratio, type, cvnum=100):
    """Sample a ratio of test set as clean dev, estimate bias with clean dev, and evaluate the adjusted model"""
    assert type=='fix' or type=='set'
    results = []
    none_ind = model.opt['none_ind']
    for i in range(cvnum):
        labels, feature = shuffle_data(label_list, feature_list)
        eva_n = int(ratio * len(label_list))

        eva_label = labels[:eva_n]
        eva_label_distribution = get_distribution_from_list(eva_label, max_index=model.type_size)

        test_label = labels[eva_n:]
        test_feature = feature[eva_n:]

        fl_t, of_t = packer.repack_eva(test_feature)

        model.eval()
        ### fix bias (train: b = p_train, test: b = p_test)
        if type == 'fix':
            delta_bias = eva_label_distribution
        ### set bias (b' = b - p_train + p_test)
        else:
            delta_bias = np.array(eva_label_distribution) / np.array(model.opt['label_distribution'])
        scores = model.test_with_bias(fl_t, of_t, delta_bias)

        ind = calcInd(scores)

        f1score, recall, precision = eval_score(ind.data, test_label, none_ind)
        results.append([f1score, recall, precision])

    return np.average(results, axis=0)

max_stars_repo_name: INK-USC/shifted-label-distribution

max_issues_repo_name: INK-USC/shifted-label-distribution

max_forks_repo_name: INK-USC/shifted-label-distribution


text:
import os
from collections import Counter
import numpy as np
import pandas as pd

COLS_TO_TRANLSATE = "site_id,site_domain,site_category,app_id,app_domain,app_category,device_id,device_ip,device_model,C1,C14,C15,C16,C17,C18,C19,C20,C21".split(",")

COLS = "id,click,day,hour,C1,banner_pos,site_id,site_domain,site_category,app_id,app_domain,app_category,device_id,device_ip,device_model,device_type,device_conn_type,C14,C15,C16,C17,C18,C19,C20,C21".split(",")

TRAIN_DATA = "../train.csv"
TEST_DATA = "../test.csv"
TRAIN_MERGED_DATA = '../merged_train.csv'
TEST_MERGED_DATA = "../merged_test.csv"
COUNTERS_FILE = '../counters.txt'

CHUNK_SIZE = 1000000
THRESH = 100

def create_counters():
    counters = {}
    for col in COLS_TO_TRANLSATE:
        counters[col] = Counter()
    return counters

def update_counter(path,counters):
    df = pd.read_csv(path,chunksize=CHUNK_SIZE,iterator=True)
    for chunk in df:
        for col in COLS_TO_TRANLSATE:
            counters[col].update(chunk.ix[:,col])
        print chunk.id.max()

def convert_counts_to_id(counters):
    ids = {}
    for col in COLS_TO_TRANLSATE:
        ids[col] = {}
        imax = 0
        highest_seen = 0
        for i,(val,count) in enumerate(counters[col].most_common()):
            if imax == 0 and count <= THRESH:
                imax = i
            index = i if count > THRESH else imax
            highest_seen = max(index, highest_seen)
            ids[col][val] = index
        print "Col " + col + " highest seen is " + str(highest_seen)
    return ids

def write_translated(input_path,output_path,ids,mode="w",start_id=0):
    df = pd.read_csv(input_path,chunksize=CHUNK_SIZE,iterator=True)
    for i, chunk in enumerate(df):
        for col in COLS_TO_TRANLSATE:
            chunk.ix[:,col] = chunk.ix[:,col].map(ids[col])
        chunk["id"] = np.arange(chunk.shape[0]) + i*CHUNK_SIZE + 1 + start_id
        chunk["day"] = chunk["hour"].map(lambda v: int(str(v)[-4:-2]))
        chunk["hour"] = chunk["hour"].map(lambda v: int(str(v)[-2:]))

        if "click" not in chunk.columns:
            chunk["click"] = 0
        chunk = chunk.ix[:,COLS]

        if i == 0 and mode == "w":
            chunk.to_csv(output_path,index=False)
        else:
            chunk.to_csv(output_path,index=False,mode="a",header=False)

        print chunk.id.max()

    return chunk.id.max()

def write_counters(file_path, counters):
    with open(file_path, 'w') as outfile:
        for counter in counters.values():
            outfile.write('%s\n' % (counter))

if __name__ == "__main__":
    counters = create_counters()
    update_counter(TRAIN_DATA,counters)
    update_counter(TEST_DATA,counters)
    write_counters(COUNTERS_FILE, counters)
    ids = convert_counts_to_id(counters)
    max_id = write_translated(TRAIN_DATA,TRAIN_MERGED_DATA,ids)
    _ = write_translated(TRAIN_DATA,TEST_MERGED_DATA,ids, start_id=max_id)
    max_id = write_translated(TEST_DATA,TEST_MERGED_DATA,ids)
    _ = write_translated(TEST_DATA,TEST_MERGED_DATA,ids, start_id=max_id)


text:
import os

import numpy as np
import paddle.fluid as fluid

from seq2annotation.input_paddle import read_vocabulary
from tokenizer_tools.tagset.NER.BILUO import BILUOSequenceEncoderDecoder

class Inference(object):
    def __init__(self, model_path):
        # load model
        self.place = fluid.CPUPlace()
        self.exe = fluid.Executor(self.place)
        [self.inference_program, self.feed_target_names, self.fetch_targets] = fluid.io.load_inference_model(dirname=model_path, executor=self.exe)

        # load vocabulary
        self.vocabulary = read_vocabulary(os.path.join(model_path, 'data/vocabulary.txt'))
        self.tag = read_vocabulary(os.path.join(model_path, 'data/tags.txt'))

        # prepare tag set decoder
        self.decoder = BILUOSequenceEncoderDecoder()

    def infer(self, input_text):
        data = [self.vocabulary.lookup(i) for i in input_text]
        print(data)

        word = fluid.create_lod_tensor([data], [[len(data)]], self.place)

        results, = self.exe.run(
            self.inference_program,
            feed={self.feed_target_names[0]: word},
            fetch_list=self.fetch_targets,
            return_numpy=False
        )

        # translate to str list
        result = np.array(results).reshape([-1]).tolist()
        print(result)
        output_tag = [self.tag.id_to_str(i) for i in result]
        print(output_tag)

        # decode to offset
        result = self.decoder.to_offset(output_tag, input_text)

        return result

if __name__ == '__main__':
    inference = Inference('/Users/howl/PyCharmProjects/seq2annotation/results/saved_model/1557904173')

    result = inference.infer('查这几天的天气。')

    print(result)


text:
import torch
import cv2
import numpy as np
from torch.utils.data import DataLoader
from .basenet import MobileNet_GDConv
from .pfld_compressed import PFLDInference
from batch_face.utils import (
    detection_adapter,
    get_default_onnx_file,
    is_image,
    is_box,
    to_numpy,
    load_weights,
)

try:
    import onnx
    import onnxruntime
except:
    onnx = None
    onnxruntime = None

def get_device(gpu_id):
    if gpu_id > -1:
        return torch.device("cuda:%s" % str(gpu_id))
    else:
        return torch.device("cpu")

# landmark of (5L, 2L) from [0,1] to real range
def reproject(bbox, landmark):
    landmark_ = (
        landmark.clone() if isinstance(landmark, torch.Tensor) else landmark.copy()
    )
    x1, y1, x2, y2 = bbox
    w = x2 - x1
    h = y2 - y1
    landmark_[:, 0] *= float(w)
    landmark_[:, 0] += float(x1)
    landmark_[:, 1] *= float(h)
    landmark_[:, 1] += float(y1)
    return landmark_

def prepare_feed(img, face, backbone):
    if backbone == "MobileNet":
        out_size = 224
    else:
        out_size = 112
    mean = np.asarray([0.485, 0.456, 0.406])
    std = np.asarray([0.229, 0.224, 0.225])
    x1 = face[0]
    y1 = face[1]
    x2 = face[2]
    y2 = face[3]
    height, width, _ = img.shape
    w = x2 - x1 + 1
    h = y2 - y1 + 1
    size = int(min([w, h]) * 1.2)
    cx = x1 + w // 2
    cy = y1 + h // 2
    x1 = cx - size // 2
    x2 = x1 + size
    y1 = cy - size // 2
    y2 = y1 + size

    dx = max(0, -x1)
    dy = max(0, -y1)
    x1 = max(0, x1)
    y1 = max(0, y1)

    edx = max(0, x2 - width)
    edy = max(0, y2 - height)
    x2 = min(width, x2)
    y2 = min(height, y2)
    new_bbox = torch.Tensor([x1, y1, x2, y2]).int()
    x1, y1, x2, y2 = new_bbox
    cropped = img[y1:y2, x1:x2]
    if dx > 0 or dy > 0 or edx > 0 or edy > 0:
        cropped = cv2.copyMakeBorder(
            cropped, int(dy), int(edy), int(dx), int(edx), cv2.BORDER_CONSTANT, 0
        )
    cropped_face = cv2.resize(cropped, (out_size, out_size))

    if cropped_face.shape[0] <= 0 or cropped_face.shape[1] <= 0:
        raise NotADirectoryError
    test_face = cropped_face.copy()
    test_face = test_face / 255.0
    if backbone == "MobileNet":
        test_face = (test_face - mean) / std
    test_face = test_face.transpose((2, 0, 1))
    test_face = torch.from_numpy(test_face).float()
    return dict(data=test_face, bbox=new_bbox)

@torch.no_grad()
def single_predict(model, feed, device):
    landmark = model(feed["data"].unsqueeze(0).to(device)).cpu()
    landmark = landmark.reshape(-1, 2)
    landmark = reproject(feed["bbox"], landmark)
    return landmark.numpy()

@torch.no_grad()
def batch_predict(model, feeds, device):
    if not isinstance(feeds, list):
        feeds = [feeds]
    # loader = DataLoader(FeedDataset(feeds), batch_size=50, shuffle=False)
    data = []
    for feed in feeds:
        data.append(feed["data"].unsqueeze(0))
    data = torch.cat(data, 0).to(device)
    results = []

    landmarks = model(data).cpu()
    for landmark, feed in zip(landmarks, feeds):
        landmark = landmark.reshape(-1, 2)
        landmark = reproject(feed["bbox"], landmark)
        results.append(landmark.numpy())
    return results

@torch.no_grad()
def batch_predict_with_loader(model, feeds, device, batch_size=None):
    if not isinstance(feeds, list):
        feeds = [feeds]
    if batch_size is None:
        batch_size = len(feeds)
    loader = DataLoader(feeds, batch_size=len(feeds), shuffle=False)
    results = []
    for feed in loader:
        landmarks = model(feed["data"].to(device)).cpu()
        for landmark, bbox in zip(landmarks, feed["bbox"]):
            landmark = landmark.reshape(-1, 2)
            landmark = reproject(bbox, landmark)
            results.append(landmark.numpy())
    return results

def split_feeds(all_feeds, all_faces):
    counts = [len(faces) for faces in all_faces]
    sum_now = 0
    ends = [0]
    for i in range(len(counts)):
        sum_now += counts[i]
        end = sum_now
        ends.append(end)
    return [all_feeds[ends[i - 1] : ends[i]] for i in range(1, len(ends))]

@torch.no_grad()
def batch_predict_onnx(ort_session, feeds, batch_size=None):
    if not isinstance(feeds, list):
        feeds = [feeds]
    if batch_size is None:
        batch_size = len(feeds)
    results = []
    for feed in feeds:
        ort_inputs = {
            ort_session.get_inputs()[0].name: to_numpy(feed["data"])[
                None,
            ]
        }
        landmark = ort_session.run(None, ort_inputs)[0][0]
        bbox = feed["bbox"]
        landmark = landmark.reshape(-1, 2)
        landmark = reproject(bbox, landmark)
        results.append(landmark)
    return results

class LandmarkPredictor:
    def __init__(self, gpu_id=0, backbone="MobileNet", file=None):
        """
        gpu_id: -1 for cpu, onnx for onnx
        """
        if gpu_id == "onnx":
            self._initialize_onnx(backbone, file)
        else:
            self._initialize_normal(gpu_id, backbone, file)

    def _initialize_onnx(self, backbone, file):
        self.device = "onnx"
        if onnx is None:
            raise "Please install onnx and onnx runtime first"
        self.backbone = backbone
        if file is None:
            file = get_default_onnx_file(backbone)
        onnx_model = onnx.load(file)
        onnx.checker.check_model(onnx_model)
        self.model = onnxruntime.InferenceSession(file)
        self._batch_predict = batch_predict_onnx

    def _initialize_normal(self, gpu_id=0, backbone="MobileNet", file=None):
        self.device = get_device(gpu_id)
        self.backbone = backbone
        if backbone == "MobileNet":
            model = MobileNet_GDConv(136)
        elif backbone == "PFLD":
            model = PFLDInference()
        else:
            raise NotImplementedError(backbone)

        weights = load_weights(file, backbone)

        model.load_state_dict(weights)
        self.model = model.to(self.device).eval()

        self._batch_predict = batch_predict_with_loader

    def __call__(self, all_boxes, all_images, from_fd=False):
        batch = not is_image(all_images)
        if from_fd:
            all_boxes = detection_adapter(all_boxes, batch=batch)
        if not batch:  # 说明是 1 张图
            if is_box(all_boxes):  # 单张图 单个box
                assert is_image(all_images)
                return self._inner_predict(self.prepare_feed(all_images, all_boxes))
            else:
                feeds = [self.prepare_feed(all_images, box) for box in all_boxes]
                return self._inner_predict(feeds)  # 一张图 多个box
        else:
            assert len(all_boxes) == len(all_images)
            assert is_image(all_images[0])
            return self.batch_predict(all_boxes, all_images)  # 多张图 多个box列表

    def _inner_predict(self, feeds):
        results = self._batch_predict(self.model, feeds, self.device)
        if not isinstance(feeds, list):
            results = results[0]
        return results

    def batch_predict(self, all_boxes, all_images):
        all_feeds = []
        for i, (faces, image) in enumerate(zip(all_boxes, all_images)):
            feeds = [self.prepare_feed(image, box) for box in faces]
            all_feeds.extend(feeds)
        all_results = self._inner_predict(all_feeds)
        all_results = split_feeds(all_results, all_boxes)
        return all_results

    def prepare_feed(self, img, face):
        return prepare_feed(img, face, self.backbone)

max_stars_repo_name: mowshon/batch-face

max_issues_repo_name: mowshon/batch-face

max_forks_repo_name: mowshon/batch-face