text:
# Cox Orb Data Analyzers

from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, FigureCanvasAgg
from matplotlib import pyplot as plt
import numpy as np
import PySimpleGUI as sg

from src import layout

def draw_figure(canvas, fig):
    if canvas.children:
        for child in canvas.winfo_children():
            child.destroy()
    figure_canvas_agg = FigureCanvasTkAgg(fig, master=canvas)
    figure_canvas_agg.draw()
    figure_canvas_agg.get_tk_widget().pack(side='right', fill='both', expand=1)

def main():

    sg.theme(layout.THEME)

    # Create the Window
    window = sg.Window('CODA', layout.layout, size=layout.SIZE)
    window.finalize()

    # Event Loop to process "events" and get the "values" of the inputs
    while True:
        event, values = window.read()
        if event == sg.WIN_CLOSED: # if user closes window
            break
        elif event == 'TEST':

            plt.figure(1)
            fig = plt.gcf()
            DPI = fig.get_dpi()
            # ------------------------------- you have to play with this size to reduce the movement error when the mouse hovers over the figure, it's close to canvas size
            fig.set_size_inches(700 / float(DPI), 500 / float(DPI))
            # -------------------------------
            x = np.linspace(0, 2 * np.pi)
            y = np.sin(x)
            plt.plot(x, y)
            plt.title('y=sin(x)')
            plt.xlabel('X')
            plt.ylabel('Y')
            plt.grid('on')

            # ------------------------------- Instead of plt.show()
            draw_figure(window['CANVAS'].TKCanvas, fig)

    window.close()

ext: py

max_stars_repo_name: aaronzlim/coda

max_issues_repo_name: aaronzlim/coda

max_forks_repo_name: aaronzlim/coda

reason: import numpy


text:
from env import CholeskyTaskGraph, DAGEnv
from a2c import A2C
from a2c import *
from model import Net, SimpleNet, SimpleNet2, ResNetG, SimpleNetMax, ModelHeterogene
from log_utils import name_dir_logger, name_dir
from torch.utils.tensorboard import SummaryWriter
import os
import argparse
import numpy as np
parser = argparse.ArgumentParser()
# Training settings

parser.add_argument('--model_path', type=str, default='none', help='path to load model')
parser.add_argument('--num_env_steps', type=int, default=10 ** 4, help='num env steps')
parser.add_argument('--num_processes', type=int, default=1, help='num proc')
parser.add_argument('--lr', type=float, default=10 ** -2, help='learning rate')
parser.add_argument('--eps', type=float, default=10 ** -1, help='Random seed.')
parser.add_argument('--optimizer', type=str, default='rms', help='sgd or adam or rms')
parser.add_argument('--scheduler', type=str, default='lambda', help='lambda or cyclic')
parser.add_argument('--step_up', type=float, default=100, help='step_size_up for cyclic scheduler')
parser.add_argument('--sched_ratio', type=float, default=10, help='lr ratio for cyclic scheduler')
parser.add_argument('--entropy_coef', type=float, default=0.002, help='entropy loss weight')
parser.add_argument('--gamma', type=float, default=1, help='inflation')
parser.add_argument('--loss_ratio', type=float, default=0.5, help='value loss weight')
parser.add_argument('--trajectory_length', type=int, default=40, help='batch size')
parser.add_argument('--log_interval', type=int, default=10, help='evaluate every log_interval steps')
parser.add_argument('--seed', type=int, default=42, help='Random seed.')
parser.add_argument('--agent', type=str, default='A2C', help='A2C')
parser.add_argument("--result_name", type=str, default="results.csv", help="filename where results are stored")

# model settings
parser.add_argument('--input_dim', type=int, default=16, help='input dim')
parser.add_argument('--hidden_dim', type=int, default=128, help='hidden dim')
parser.add_argument('--ngcn', type=int, default=0, help='number of gcn')
parser.add_argument('--nmlp', type=int, default=1, help='number of mlp to compute probs')
parser.add_argument('--nmlp_value', type=int, default=1, help='number of mlp to compute v')
parser.add_argument('--res', action='store_true', default=False, help='with residual connexion')
parser.add_argument('--withbn', action='store_true', default=False, help='with batch norm')

# env settings
parser.add_argument('--n', type=int, default=4, help='number of tiles')
parser.add_argument('--nGPU', type=int, default=1, help='number of GPUs')
parser.add_argument('--nCPU', type=int, default=3, help='number of cores')
parser.add_argument('--window', type=int, default=0, help='window')
parser.add_argument('--noise', type=float, default=0, help='noise')
parser.add_argument('--env_type', type=str, default='QR', help='chol or LU or QR')
parser.add_argument('--seed_env', type=int, default=42, help='Random seed env ')

args = parser.parse_args()
config_enhanced = vars(args)

writer = SummaryWriter('runs')

p_input = np.array([1] * args.nGPU + [0] * args.nCPU)

print("Current config_enhanced is:")
pprint(config_enhanced)

main_path = "HPC"

env = DAGEnv(n=args.n, node_types=p_input, window=args.window, env_type=args.env_type, noise=args.noise)
env.reset()

model = ModelHeterogene(input_dim=args.input_dim,
                        hidden_dim=args.hidden_dim,
                        ngcn=args.ngcn,
                        nmlp=args.nmlp,
                        nmlp_value=args.nmlp_value,
                        res=args.res,
                        withbn=args.withbn)

agent = A2C(config_enhanced, env, model=model, writer=writer)

best_perf, _ = agent.training_batch()


text:
import numpy as np
import scipy.stats as stats
import matplotlib.pyplot as plt

def gauss_evaluate_likelihood(X, x_true):
  if len(X.shape) == 3:
    a,b,c = X.shape
    X = np.reshape(X, newshape=(a,b*c))
    x_true = np.reshape(x_true, newshape=(b*c,))
  X_mean = np.mean(X,0)
  X_sd = np.std(X,0)
  return np.mean(-(x_true - X_mean)**2/(2*X_sd**2) - 0.5*np.log(2*np.pi*X_sd**2))

def evaluate_likelihood(X, x_true, s=0.00001):
  def ker(x,y,bw):
      return np.mean(np.log(np.mean(np.exp(-(x - y)**2/(2*bw**2))/(np.sqrt(2*np.pi)*bw),0) + s))
  if len(X.shape) == 3:
    N,b,c = X.shape
    a = b*c
  else:
    N,a = X.shape
  X = np.reshape(X, newshape=(N,a))
  x_true = np.reshape(x_true, newshape=(1,a))
  bw = 0.9*np.std(X,0)*N**(-1/5)
  return ker(X, x_true, bw)

def evaluate_multi_likelihood(X, x_true, s=0.01):
  try:
    if len(X.shape) == 3:
      a,b,c = X.shape
      N = b*c
      X = np.reshape(X, newshape=(a,b*c))
      x_true = np.reshape(x_true, newshape=(b*c,))
    else:
      N = X.shape[1]
    mu = np.mean(X,0)
    S = np.cov(np.transpose(X)) + s*np.identity(N)
    return stats.multivariate_normal.logpdf(x_true, mean=mu, cov=S)
  except np.linalg.LinAlgError:
    print("Something went wrong in the multivariate evaluation metric")
    return np.nan

def evaluate_model(variational_model, X_true, M, emission_model, emission_distribution, scale, out_data, T_data):
  X, mu, x_pre, log_jacobian, epsilon_loss = variational_model.sample_timeseries(M)
  uni_lk = evaluate_likelihood(X.detach().numpy(), X_true.detach().numpy())
  multi_lk = evaluate_multi_likelihood(X.detach().numpy(), X_true.detach().numpy())
  multi_pred_lk = evaluate_predictive_error(X, emission_model, emission_distribution, scale, out_data, T_data)
  print("Avarage univariate latent likelihood: {}".format(uni_lk))
  print("Multivariate latent likelihood: {}".format(multi_lk))
  print("Predictive observable likelihood: {}".format(multi_pred_lk))
  return uni_lk, multi_lk, multi_pred_lk

def evaluate_predictive_error(X, emission_model, emission_distribution, scale, out_data, T_data):
  Y = np.zeros((X.shape[0], X.shape[2] - T_data))
  for t in range(T_data, X.shape[2]):
    tau = t - T_data
    eps = np.random.normal(0,0.0001,(Y.shape[0],))
    if scale is None:
      Y[:,tau] = emission_distribution.rsample(emission_model(X[:,:,t]), None).detach().numpy() + eps
    else:
      Y[:,tau] = emission_distribution.rsample(emission_model(X[:,:,t]), scale).detach().numpy() + eps
  #plt.plot(np.transpose(Y), alpha=0.01, c="r")
  #plt.plot(np.transpose(X[:20,0,T_data:].detach().numpy()), alpha=0.8, c="k")
  #plt.plot(np.transpose(out_data))
  #plt.show()
  return evaluate_likelihood(Y, out_data.detach().numpy())

max_stars_repo_name: LucaAmbrogioni/CascadingFlow

max_issues_repo_name: LucaAmbrogioni/CascadingFlow

max_forks_repo_name: LucaAmbrogioni/CascadingFlow

reason: import numpy,import scipy


text:
import numpy as np
import cv2
import imutils
import time
from sklearn.metrics import pairwise
from imutils.video import FPS
import copy
import os
import sys
import tensorflow as tf
import pathlib
from collections import defaultdict

from utils import tracking_utils
from utils import signalDetection_utils
from utils import estimate_collide_utils
from utils import estimate_stepping_utils
from utils import lane_detection_utils
from utils import break_light_utils
from utils import functions

from utils import ops as utils_ops
from utils import label_map_util

utils_ops.tf = tf.compat.v1
tf.gfile = tf.io.gfile
PATH_TO_LABELS = '../bigdata/data/mscoco_label_map.pbtxt'
category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)

model_name = 'ssd_inception_v2_coco_2018_01_28'
model_dir =  "../bigdata/models/" + model_name + "/saved_model"
detection_model = tf.saved_model.load(str(model_dir))
detection_model = detection_model.signatures['serving_default']

# print(category_index)
colors = np.random.uniform(0, 255, size=(len(category_index), 3))
font = cv2.FONT_HERSHEY_SIMPLEX
blackLower = (0 , 0 , 0)
blackUpper = (180 , 255 , 35)

print(detection_model.inputs)
print(detection_model.output_dtypes)
print(detection_model.output_shapes)

def click_and_crop(event, x, y, flags, param):
    global refPt
    # if the left mouse button was clicked, record the starting (x, y) coordinates
    if event == cv2.EVENT_LBUTTONDOWN:
        refPt.append([x, y])

def confirm_day_or_night(frame , flag_night_counter):
    blurred = cv2.GaussianBlur(frame, (11, 11), 0)
    hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV)
    mask = cv2.inRange(hsv, blackLower , blackUpper)
    mask = cv2.erode(mask, None, iterations=2)
    mask = cv2.dilate(mask , None, iterations=2)
    # cv2.imshow('black',imutils.resize(mask,width=250))
    # cv2.imshow('frame',frame)
    pixel_ct = 0
    pixel_len = 0
    for i in mask:
      pixel_ct = pixel_ct + np.sum(i==0)
      pixel_len = pixel_len + len(i)
    ratio = pixel_ct / pixel_len
    print("ratio = ",ratio)
    if ratio < 0.5:
        flag_night_counter = flag_night_counter + 1
        return flag_night_counter
    else:
        flag_night_counter = flag_night_counter - 1
        return flag_night_counter

# flagPerson and areaPerson are pedestrian crash
flagPerson = 0
areaPerson = 0
areaDetails = []

# crash_count_frames are vehicle warning variables
crash_count_frames = 0

# signalCounter and flagSignal are traffic signal variables
signalCounter = -99999
flagSignal = [0] * 20

# number and prev_frame are tracking variables
number = 0
prev_frame = []
def show_inference(dashPointer , lanePointer , frame):
    global flagPerson , areaPerson , areaDetails
    global crash_count_frames
    global signalCounter , flagSignal
    global prev_frame , number

    image_np = np.array(frame)
    lane_image = copy.deepcopy(image_np)
    height , width , channel = image_np.shape

    output_dict = functions.get_dict(dashPointer , detection_model , image_np)

    confidencesCars , boxesCars , confidencesLights , boxesLights , confidencesPersons , boxesPersons = functions.findBoxes(image_np , output_dict)

    indexesLights = cv2.dnn.NMSBoxes(boxesLights, confidencesLights, 0.5, 0.4)
    indexesCars = cv2.dnn.NMSBoxes(boxesCars, confidencesCars, 0.5, 0.4)
    indexesPersons = cv2.dnn.NMSBoxes(boxesPersons, confidencesPersons, 0.5, 0.4)

    image_np , signalCounter , flagSignal = signalDetection_utils.signalDetection(indexesLights , boxesLights , image_np , signalCounter , flagSignal)
    image_np , prev_frame , number = tracking_utils.tracking(indexesCars , boxesCars , image_np , prev_frame , number)
    image_np , crash_count_frames = estimate_collide_utils.estimate_collide(indexesCars , boxesCars , image_np , crash_count_frames)
    image_np , flagPerson , areaPerson , areaDetails = estimate_stepping_utils.estimate_stepping(indexesPersons , boxesPersons , image_np, flagPerson , areaPerson , areaDetails)

    cv2.putText(image_np,"DAY",(50 ,90), font, 2, (167,133,0) , 2 , cv2.LINE_AA)

    image_np = lane_detection_utils.draw_lines(lanePointer , lane_image , image_np)
    # lane_detection_utils.all_lines(lanePointer , lane_image , image_np)

    cv2.imshow("finally", image_np)

def selectRegions(image  , text , flag):
    global refPt
    clone = copy.deepcopy(image)
    while True:
        key = cv2.waitKey(1) & 0xFF
        # display the image and wait for a keypress
        if flag==1:
            cv2.putText(image, text ,  (240,30), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'r' key to reset everything.",  (290,70), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'd' key if the region selection is done.",  (180,110), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
        else:
            cv2.putText(image, text ,  (240,30), font , 1.2, [0,255,0], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'r' key to reset everything.",  (290,70), font , 1.2, [0,255,0], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'd' key if the region selection is done.",  (180,110), font , 1.2, [0,255,0], 2,cv2.LINE_AA)

        for pt in range(len(refPt)-1):
            pt1 , pt2 = refPt[pt] , refPt[pt+1]
            cv2.line(image, (pt1[0],pt1[1]), (pt2[0],pt2[1]), [0,255,255], 3)

        cv2.imshow("ROI", image)
        if key == ord("r"):
            image = copy.deepcopy(clone)
            refPt = []
        elif key == ord("d"):
            if flag == 1:
                return 0
            elif flag == 2 and len(refPt) > 2:
                return 0
        elif key == ord('q'):
            return 1

def night():
    global refPt
    _ , image = cap.read()
    image=imutils.resize(image, width=1280)

    ctt = 0

    Quit = selectRegions(image  , "Click points to select your vehicle dash." , 1)
    dashPointer = refPt
    if len(dashPointer) <= 2:
        dashPointer = [[0,0], [0,0], [0,0]]
    refPt = []
    print("For dash: ",dashPointer)
    if Quit == 1:
        return
    cv2.destroyWindow("ROI")

    fps = FPS().start()
    while True:
        _,frame = cap.read()
        frame=imutils.resize(frame, width=1280)

        if _ == False:
            break

        # print(ctt ,fps._numFrames)
        # ctt = ctt + 1
        break_light_utils.break_light(dashPointer , frame)

        # cv2.imshow("original",frame)
        key = cv2.waitKey(1) & 0xFF
        fps.update()
        if key == ord('q'):
            break
    fps.stop()
    print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

def day():
    global refPt
    _ , image = cap.read()
    image=imutils.resize(image, width=1280)

    ctt = 0

    Quit = selectRegions(copy.deepcopy(image)  , "Click points to select your vehicle dash." , 1)
    dashPointer = refPt
    if len(dashPointer) <= 2:
        dashPointer = [[0,0], [0,0], [0,0]]
    refPt = []
    print("For dash: ",dashPointer)
    if Quit == 1:
        return

    Quit = selectRegions(copy.deepcopy(image)  , "Click points to select bird's eye view." , 2)
    lanePointer = refPt
    print("For lanes: ",lanePointer)
    if Quit == 1:
        return

    cv2.destroyWindow("ROI")

    fps = FPS().start()
    while True:
        _,frame = cap.read()
        frame = imutils.resize(frame, width=1280)
        if _ == False:
            break
        # print(ctt ,fps._numFrames)
        # ctt = ctt + 1

        show_inference(dashPointer , lanePointer , frame)

        # cv2.imshow("original",frame)
        key = cv2.waitKey(1) & 0xFF
        fps.update()
        if key == ord('q'):
            break

    fps.stop()
    print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

refPt = []                  # to store refernece pointers
flag_night_counter = 0      # counter to count night frames

cap=cv2.VideoCapture('../videos/c.mp4')
start_frame = 113*24
cap.set(1,start_frame)
_ , image = cap.read()
image=imutils.resize(image, width=1280)
cv2.namedWindow("ROI")
cv2.setMouseCallback("ROI", click_and_crop)

for z in range(10):
    (grabbed, frame) = cap.read()
    frame=imutils.resize(frame, width=1280)
    height,width,channel = frame.shape

    flag_night_counter = confirm_day_or_night(frame , flag_night_counter)
print("flag_night_counter = ",flag_night_counter)
cap.set(1 , start_frame)

if flag_night_counter > 0:
    night()
else:
    day()

cv2.destroyAllWindows()
cap.release()


text:
import argparse
import time
import os

import numpy as np
import pandas as pd
import tqdm
import h5py

from generation.config import DF_DIR, EVENTS_PATH, DETECTORS_PATH, FULL_SIGNALS_DIR, PROCESSING_TIME_NORM_COEF, \
    SIGNAL_DIM, SPACAL_DATA_PATH, FRAC_SIGNALS_DIR, H5_DATASET_NAME, POSTPROCESSED_SIGNALS_DIR

def create_dir(path):
    if not os.path.exists(path):
        os.mkdir(path)

def save_h5(data, dataset_name, path):
    h5f = h5py.File(path, 'w')
    h5f.create_dataset(dataset_name, data=data, compression='gzip')
    h5f.close()

def load_h5(path, dataset_name):
    h5f = h5py.File(path, 'r')
    dataset_h5 = h5f[dataset_name][:]
    h5f.close()
    return dataset_h5

def get_events(path=EVENTS_PATH):
    """
    Given path to events .npy file returns events
    param: path : path to events file
    return: events np array
    """
    events = np.load(path)
    return events

def get_detectors(path=DETECTORS_PATH):
    """
    Given path to detectors .npy file returns detectors
    param: path : path to detectors file
    return: detectors np array
    """
    detectors = np.load(path)
    return detectors

def get_event_dir(base_dir: str, event: int):
    """
    Given event and base directory return event directory
    :param base_dir: base directory
    :param event: event number
    :return: directory with files associated with given event
    """
    return os.path.join(base_dir, 'event_{}').format(event)

def get_event_detector_df_path(event: int, detector: int, df_dir: str = DF_DIR):
    """
    Given detector and event returns path to corresponding df
    :param event: event number
    :param detector: detector number
    :param df_dir: directory with dataframe files
    :return: path to pandas dataframe
    """
    event_dir = get_event_dir(df_dir, event)
    df_path = os.path.join(event_dir, 'detector_{}.parquet').format(detector)
    return df_path

def get_event_detector_df(event: int, detector: int):
    """
    Given detector and event returns corresponding df
    :param event: event number
    :param detector: detector number
    :return: pandas dataframe
    """
    df_path = get_event_detector_df_path(event, detector)
    df = pd.read_parquet(df_path)
    return df

def get_detector_signals_path(detector: int, signals_dir: str = FULL_SIGNALS_DIR):
    """
    Given detector and event returns path to signal
    :param detector: detector number
    :param signal_dir: directory with signal files
    :return: path to np array
    """
    signals_path = os.path.join(signals_dir, 'detector_{}.h5').format(detector)
    return signals_path

def get_detector_signals(detector: int):
    """
    Given detector and event returns corresponding signal
    :param event: event number
    :param detector: detector number
    :return: numpy array with shape SIGNAL_DIM
    """
    signals_path = get_detector_signals_path(detector)
    signals = load_h5(signals_path, H5_DATASET_NAME)
    return signals

def get_detector_postprocessed_signals_path(detector: int, signals_dir: str = POSTPROCESSED_SIGNALS_DIR):
    """
    Given detector and event returns path to signal
    :param detector: detector number
    :param signal_dir: directory with signal files
    :return: path to np array
    """
    signals_path = os.path.join(signals_dir, 'detector_{}.h5').format(detector)
    return signals_path

def get_detector_postprocessed_signals(detector: int):
    """
    Given detector and event returns corresponding signal
    :param event: event number
    :param detector: detector number
    :return: numpy array with shape SIGNAL_DIM
    """
    signals_path = get_detector_postprocessed_signals_path(detector)
    signals = load_h5(signals_path, H5_DATASET_NAME)
    return signals

def get_detector_training_data_path(detector: int):
    return os.path.join(FRAC_SIGNALS_DIR, f'detector_{detector}.h5')

def get_detector_training_data(detector: int):
    data_path = get_detector_training_data_path(detector)
    return load_h5(data_path, H5_DATASET_NAME)

def get_attributes_df(data_path=SPACAL_DATA_PATH):
    """
    :param data_path:
    :return: df, where each column is an attribute
    """
    df = pd.read_parquet(data_path)
    return df

def _calculate_step(df, signal_dim):
    min_timestamp = min(df['timestamp'])  # TODO: (@whiteRa2bit, 2020-09-22) Replace with config params
    max_timestamp = max(df['timestamp'])
    step = (max_timestamp - min_timestamp) / signal_dim
    return step

def _get_step_energies(df, min_timestamp, max_timestamp):
    step_df = df[df['timestamp'] > min_timestamp]
    step_df = step_df[step_df['timestamp'] < max_timestamp]
    step_energies = step_df['PhotonEnergy'].values
    return step_energies

def generate_one_signal(df, signal_dim: int = SIGNAL_DIM, use_postprocessing: bool = False, frac: float = 1.0):
    """
    Generates one output for given df
    :param df: df with info of given detector and event
    :param signal_dim: number of timestamps by which time will be splitted
    :return: np.array [signal_dim] with energies
    """
    if df.empty:
        return np.zeros(signal_dim)

    step = _calculate_step(df, signal_dim)
    steps_energy = []
    for i in range(signal_dim):
        step_energies = _get_step_energies(df, i * step, (i + 1) * step)
        step_energies = np.random.choice(step_energies, int(len(step_energies) * frac))
        step_energy = np.sum(step_energies)
        steps_energy.append(step_energy)
    if use_postprocessing:
        steps_energy = postprocess_signal(steps_energy)

    return np.array(steps_energy)

def generate_signals(df,
                     data_size: int,
                     use_postprocessing: bool = False,
                     signal_dim: int = SIGNAL_DIM,
                     frac: float = 1.0):
    """
    Generates data for a given detector
    :param df_full: pandas df, output of get_events_df()
    :param data_size: number of samples to get
    :param use_postprocessing: whether to use output before or after photodetector
    :param signal_dim: number of timestamps by which time will be splitted
    :param sample_coef: percent of data to take for each step
    :return: np.array with generated events
    """
    if df.empty:
        return np.zeros((data_size, signal_dim))

    step = _calculate_step(df, signal_dim)
    steps_energies = []
    for i in range(signal_dim):
        step_energies = _get_step_energies(df, i * step, (i + 1) * step)
        steps_energies.append(step_energies)

    output_signals = np.zeros((data_size, signal_dim))
    for signal_idx in range(data_size):
        for step_idx, step_energies in enumerate(steps_energies):
            step_energies = np.random.choice(step_energies, int(len(step_energies) * frac))
            step_energy = np.sum(step_energies)
            output_signals[signal_idx][step_idx] = step_energy

    # TODO: (@whiteRa2bit, 2020-09-22) Add postprocessing
    return output_signals

def postprocess_signal(signal):
    """
    Getting result signal after photodetector
    :param signal: Output from generate_one_signal
    :return: processed signal
    """

    def build_kernel(x_cur, energy, x_min, x_max):
        kernel = lambda x: ((x - x_cur) ** 2) / np.exp((x - x_cur) / PROCESSING_TIME_NORM_COEF)
        x_linspace = np.linspace(x_min, x_max, x_max - x_min)
        y_linspace = energy * np.array(list(map(kernel, x_linspace)))
        y_linspace[:x_cur] = np.zeros(x_cur)
        return y_linspace

    result = np.zeros(len(signal))
    for x_cur, energy in enumerate(signal):
        y_cur = build_kernel(x_cur, energy, x_min=0, x_max=len(signal))
        result += y_cur
    return result


text:
import threading
from fastmc.proto import NbtTag
import itertools
import scipy.spatial as ss
import numpy as np

class ThreadSafeCounter:
    def __init__(self, start=0, step=1):
        self.i = start
        self.step = step
        # create a lock
        self.lock = threading.Lock()

    def __iter__(self):
        return self

    def next(self):
        # acquire/release the lock when updating self.i
        with self.lock:
            self.i += self.step
            return self.i

class LocksWrapper(object):
    def __init__(self, locks):
        self.locks = locks

    def __repr__(self):
        return 'LocksWrapper([{}])'.format(
            ', '.join(self.locks.keys()))

    def __enter__(self, *args, **kwargs):
        for lock in self.locks.itervalues():
            lock.acquire()

    def __exit__(self, *args, **kwargs):
        for lock in self.locks.itervalues():
            lock.release()

def iter_spiral(x=0, y=0):
    dx = 0
    dy = -1
    while True:
        yield (x, y)
        if x == y or (x < 0 and x == -y) or (x > 0 and x == 1 - y):
            dx, dy = -dy, dx
        x, y = x + dx, y + dy

def unpack_nbt(tag):
    if not isinstance(tag, NbtTag):
        return None
    if tag.tag_type == NbtTag.LIST:
        return [unpack_nbt(i) for i in tag.value]
    if tag.tag_type == NbtTag.COMPOUND:
        return {k: unpack_nbt(t) for k, t in tag.value.iteritems()}
    else:
        return tag.value

class Cluster(object):
    def __init__(self, cluster_points):
        self.centroid = tuple(int(i) for i in
                              np.mean(cluster_points, axis=0))
        self.points = tuple(tuple(int(i) for i in c) for
                            c in cluster_points)
        self.size = len(cluster_points)

    def __repr__(self):
        return 'Cluster(centroid={}, size={})'.format(self.centroid, self.size)

def get_clusters(points, radius):
    points = np.array(points)
    tree = ss.KDTree(points)
    neighbors = [tree.query_ball_point(p, radius * 2) for
                 i, p in enumerate(points)]
    clusters = set([tuple(set(a).intersection(b)) for
                    a, b in itertools.combinations(neighbors, 2)])
    return [Cluster(points[np.array(cluster)]) for
            cluster in clusters if len(cluster) >= 2]

max_stars_repo_name: Rexkh/mcpython

max_issues_repo_name: Rexkh/mcpython

max_forks_repo_name: Rexkh/mcpython


text:
import morphforge.stdimports as mf

from ..core import StdChlAlphaBeta

import numpy as np
import pylab
import matplotlib as mpl

def set_fontsize(fig,fontsize):
    """
    For each text object of a figure fig, set the font size to fontsize
    """
    def match(artist):
        return artist.__module__ == "matplotlib.text"

    for textobj in fig.findobj(match=match):
        textobj.set_fontsize(fontsize)

def plot_sv(ax, sv_name, sv_coeffs, what, plot_kwargs=None):
    assert what in ['alpha','beta','tau','inf']

    SV = sv_coeffs
    v = np.linspace(-80,50,100)
    alpha = (SV[0][0] + SV[0][1] * v) / ( SV[0][2] + np.exp( (SV[0][3] + v ) / SV[0][4] ))
    beta  = (SV[1][0] + SV[1][1] * v) / ( SV[1][2] + np.exp( (SV[1][3] + v ) / SV[1][4] ))

    inf = alpha/(alpha+beta)
    tau = 1.0/(alpha+beta)

    vals = { 'alpha':alpha,
            'beta':beta,
            'inf':inf,
            'tau':tau,
            }[what]

    plot_kwargs = plot_kwargs or {}
    return ax.plot(v, vals, label=sv_name ,**plot_kwargs)

class AlphaBetaSummariser(object):

    @classmethod
    def build(cls, obj):
        import mredoc as mrd
        param_tbl= mrd.VerticalColTable( ['Parameter','Value'],
                [
                    ('Conductance (gmax)', str(obj.conductance.rescale('mS/cm2')) ),
                    ('Reversal Potential', str(obj.reversalpotential.rescale('mV'))),
                 ] )

        sv_eqns = []
        sv_tbls = []
        for sv in obj.statevars:
            sv_table = mrd.VerticalColTable( ['$%s$'%sv,'A','B','C','D','E'],
                   (  ['Alpha'] +[str(f) for f in  obj.statevars[sv][0]],
                      ['Beta'] + [str(f) for f in  obj.statevars[sv][1]],
                       ) )
            sv_tbls.append(sv_table)

            sv_eqns.append( r"""\frac{d}{dt}%s &= \frac{%s_\infty(V) - %s}{\tau_{%s}(V)}""".replace('%s',sv) )
            sv_eqns.append( r"""%s_\infty(V) &= \frac{\alpha_{%s}(V)}{\alpha_{%s}(V) + \beta_{%s}(V)}""".replace('%s',sv) )
            sv_eqns.append( r"""\tau_{%s}(V) &= \frac{1.0}{\alpha_{%s}(V) + \beta_{%s}(V)}""".replace('%s',sv) )
            sv_eqns.append( r"""\alpha_{%s}(V),\beta_{%s}(V)  &= \frac{A + BV}{C + exp(\dfrac{D+V}{E})} """.replace('%s',sv) )

        sv_fig = pylab.figure(figsize=(6,4))
        axes = {
        'alpha': sv_fig.add_subplot(2,2,1), 'beta':  sv_fig.add_subplot(2,2,2),
        'inf':   sv_fig.add_subplot(2,2,3), 'tau':   sv_fig.add_subplot(2,2,4),
        }

        for sv_name, sv_vars in obj.statevars.items():
            plot_sv(axes['alpha'], sv_name, sv_vars, 'alpha')
            plot_sv(axes['beta'] , sv_name, sv_vars, 'beta')
            plot_sv(axes['inf'] , sv_name, sv_vars, 'inf')
            plot_sv(axes['tau'] , sv_name, sv_vars, 'tau')

        for ax_name, ax in axes.items():
            ax.legend()
            ax.xaxis.set_major_locator(mpl.ticker.MaxNLocator(6))
            ax.yaxis.set_major_locator(mpl.ticker.MaxNLocator(5))
            ax.set_ylabel(ax_name)

        set_fontsize(sv_fig, 8)
        img_caption = 'The rate constants and resulting steady-state activation and time constants for %s' % ','.join( obj.statevars.keys() )
        mpl_fig = mrd.Figure( mrd.Image(sv_fig, fig_size=(6,4), subplots_adjust={'left':0.25, 'right':0.95}, max_font_size=7), caption=img_caption )

        return mrd.Section('Summary of %s (StdChlAlphaBeta)' % obj.name,
                mrd.EquationBlock(
                    mrd.Equation( 'g &= gmax * %s' % obj.eqn),
                    mrd.Equation( 'i &= g * (erev-V)' ),
                    ),
                mrd.EquationBlock(
                    *sv_eqns
                    ),
                param_tbl,
                sv_tbls,
                mpl_fig
                 ),

#assert False
mf.SummariserLibrary.register_summariser( StdChlAlphaBeta, AlphaBetaSummariser)

max_stars_repo_name: mikehulluk/morphforge

max_issues_repo_name: mikehulluk/morphforge

max_forks_repo_name: mikehulluk/morphforge


text:
import codecs
import os.path
import xml.dom.minidom
from xml.dom.minidom import Document

import numpy as np

import utility
from generateTypeSpeakers import classify_speakers
from getImageDescriptors import get_number_of_faces_variation, get_key_frames_inter_intra_segments_intensity_variation
from getInteractionPercentage import percentage_interaction
from getInteractionsInterventions import get_speech_sequence, write_interaction_intervention_list
from getInteventionPercentage import percentage_intervention
from getSpeakerDistributionOverSegments import get_speaker_distribution
from getSpeakerSwitchNumberPerSegment import compute_number_speaker_switch_per_segment
from getSpeakerTypePercentage import percentage_speaker
from getSpeechMusicPercentage import get_speech_music_non_speech_non_music_percentage
from getTextTranscripts import extract_text_data

def create_descriptors_file(output_xml_file, number_of_shots):
    # print "XML file for the descriptors does not exist yet. We will create it"
    dom_tree_out = Document()
    root = dom_tree_out.createElement("Descriptors")
    dom_tree_out.appendChild(root)
    descriptors = dom_tree_out.createElement("Shots")
    descriptors.setAttribute("count", str(number_of_shots))
    root.appendChild(descriptors)
    file_handle = codecs.open(output_xml_file, 'wb', 'utf8')
    dom_tree_out.writexml(file_handle)
    file_handle.close()
    return dom_tree_out

def generate_percentage_descriptors(input_xml_file, output_xml_file, shot_list):
    number_of_shots = len(shot_list)
    if number_of_shots <= 0:
        return 0
    # get the list of interaction and intervention
    speech_sequence = get_speech_sequence(input_xml_file, 2)
    write_interaction_intervention_list(input_xml_file, speech_sequence, 10)

    # classify the type of speakers
    classify_speakers(input_xml_file, 20, 20)
    if not os.path.exists(output_xml_file):
        dom_tree_out = create_descriptors_file(output_xml_file, number_of_shots)
    else:
        utility.remove_file(output_xml_file)
        dom_tree_out = create_descriptors_file(output_xml_file, number_of_shots)

    shot_number = 1
    counter = 0
    shot_descriptor_list = []
    input_dom_tree = xml.dom.minidom.parse(input_xml_file)
    category_element = input_dom_tree.getElementsByTagName("Category")
    category = category_element[0].firstChild.nodeValue
    for shot_item in shot_list:
        start = shot_item["start"]
        end = shot_item["end"]
        duration = shot_item["duration"]
        if duration < 3:
            continue
        shot_descriptors = dom_tree_out.createElement("Shot")
        shot_descriptors.setAttribute("start", str(start))
        shot_descriptors.setAttribute("end", str(end))
        shot_descriptors.setAttribute("duration", str(duration))
        shot_descriptors.setAttribute("duration_percentage", str((duration * 100) / 11367))
        shot_descriptors.setAttribute("shot_number", str(shot_number))
        shot_descriptors.setAttribute("category", category)
        #
        # print "Shot number: " + str(shot_number)
        # print "{Start: " + str(start) + ", End: " + str(end) + " Duration: " \
        #       + str(duration) + "}"
        shot_number += 1
        counter += 1
        # region Interaction (get interaction percentage then add the values in the xml file)
        # print "get interaction percentage then add the values in the xml file:"
        # print 'start:  {}'.format(datetime.datetime.now().time())
        p_interaction = percentage_interaction(input_dom_tree, start, end)

        interaction_list = dom_tree_out.createElement("Interactions")
        inter2sp = dom_tree_out.createElement("Interaction")
        inter2sp.setAttribute("numberSpeakers", "2")
        inter2sp.appendChild(dom_tree_out.createTextNode(str(p_interaction["interaction2Speakers"])))
        interaction_list.appendChild(inter2sp)

        inter3sp = dom_tree_out.createElement("Interaction")
        inter3sp.setAttribute("numberSpeakers", "3")
        inter3sp.appendChild(dom_tree_out.createTextNode(str(p_interaction["interaction3Speakers"])))
        interaction_list.appendChild(inter3sp)

        inter4sp = dom_tree_out.createElement("Interaction")
        inter4sp.setAttribute("numberSpeakers", "4")
        inter4sp.appendChild(dom_tree_out.createTextNode(str(p_interaction["interaction4Speakers"])))
        interaction_list.appendChild(inter4sp)

        inter5sp = dom_tree_out.createElement("Interaction")
        inter5sp.setAttribute("numberSpeakers", "4+")
        inter5sp.appendChild(dom_tree_out.createTextNode(str(p_interaction["interaction4+Speakers"])))
        interaction_list.appendChild(inter5sp)
        shot_descriptors.appendChild(interaction_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region Intervention (get intervention percentage and add them to the file)
        # print "get intervention percentage and add them to the file:"
        # print 'start:  {}'.format(datetime.datetime.now().time())
        p_intervention = percentage_intervention(input_dom_tree, start, end)
        intervention_list = dom_tree_out.createElement("Interventions")
        inter_short = dom_tree_out.createElement("Intervention")
        inter_short.setAttribute("type", "short")
        inter_short.appendChild(dom_tree_out.createTextNode(str(p_intervention["short"])))
        inter_long = dom_tree_out.createElement("Intervention")
        inter_long.setAttribute("type", "long")
        inter_long.appendChild(dom_tree_out.createTextNode(str(p_intervention["long"])))
        intervention_list.appendChild(inter_short)
        intervention_list.appendChild(inter_long)
        shot_descriptors.appendChild(intervention_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region SpeakersTypeList (get speaker distribution and add the values to the xml)
        # print "get speaker distribution and add the values to the xml:"
        # print 'start:  {}'.format(datetime.datetime.now().time())
        p_speakers = percentage_speaker(input_dom_tree, start, end)
        speaker_type_list = dom_tree_out.createElement("SpeakersTypeList")
        sp_type = dom_tree_out.createElement("SpeakersType")

        punctual = dom_tree_out.createElement("Ponctuel")
        punctual.appendChild(dom_tree_out.createTextNode(str(p_speakers["ponctuel"])))
        sp_type.appendChild(punctual)

        localise = dom_tree_out.createElement("Localise")
        localise.appendChild(dom_tree_out.createTextNode(str(p_speakers["localise"])))
        sp_type.appendChild(localise)

        present = dom_tree_out.createElement("Present")
        present.appendChild(dom_tree_out.createTextNode(str(p_speakers["present"])))
        sp_type.appendChild(present)

        regular = dom_tree_out.createElement("Regulier")
        regular.appendChild(dom_tree_out.createTextNode(str(p_speakers["regulier"])))
        sp_type.appendChild(regular)

        important = dom_tree_out.createElement("Important")
        important.appendChild(dom_tree_out.createTextNode(str(p_speakers["important"])))
        sp_type.appendChild(important)

        speaker_type_list.appendChild(sp_type)
        shot_descriptors.appendChild(speaker_type_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region SpeakersDistribution (get speaker distribution over segments (how many speakers from the total
        # number is speaking during the segment)
        # print "get speaker distribution over segments (how many speakers from the total " \
        #       "number is speaking during the segment:"
        # print 'start:  {}'.format(datetime.datetime.now().time())
        d_speakers = get_speaker_distribution(input_dom_tree, start, end)
        d_speakers_list = dom_tree_out.createElement("SpeakersDistribution")
        d_speaker = dom_tree_out.createElement("SpeakerDistribution")
        d_speaker.appendChild(dom_tree_out.createTextNode(str(d_speakers)))
        d_speakers_list.appendChild(d_speaker)
        shot_descriptors.appendChild(d_speakers_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region nbFacesList (get the list of nb of faces during segments)
        # print "get the list of nb of faces during segments "
        # print 'start:  {}'.format(datetime.datetime.now().time())
        nb_faces_result = get_number_of_faces_variation(input_dom_tree, start, end)
        nb_faces_list = dom_tree_out.createElement("nbFacesList")
        nb_faces = dom_tree_out.createElement("nbFaces")
        ls = list(nb_faces_result)
        mean = 0
        sigma = 0
        if len(ls) != 0:
            mean = np.mean(ls)
            sigma = np.std(ls)

        min_node = dom_tree_out.createElement("MeanNbFaces")
        min_node.appendChild(dom_tree_out.createTextNode(str(mean)))
        nb_faces.appendChild(min_node)
        max_node = dom_tree_out.createElement("StdNbFaces")
        max_node.appendChild(dom_tree_out.createTextNode(str(sigma)))
        nb_faces.appendChild(max_node)
        nb_faces_list.appendChild(nb_faces)
        shot_descriptors.appendChild(nb_faces_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region IntensityVariation (get the list of intensity variations)
        # print "get the list of intensity variations "
        # print 'start:  {}'.format(datetime.datetime.now().time())
        list_intensity_variations_inter, \
        list_intensity_variations_intra, \
        nb_transitions = get_key_frames_inter_intra_segments_intensity_variation(input_dom_tree, start, end)

        intensity_var = dom_tree_out.createElement("IntensityVariation")
        shot_descriptors.appendChild(intensity_var)

        inter_intensity_variation_list = dom_tree_out.createElement("InterIntensityVariationList")

        for intensity_variations_inter in list_intensity_variations_inter:
            i_variation = dom_tree_out.createElement("InterIntensityVariation")
            normalized_intensity_variations_inter = (intensity_variations_inter * 100) / 255
            txt_node = dom_tree_out.createTextNode(str(normalized_intensity_variations_inter))
            i_variation.appendChild(txt_node)
            inter_intensity_variation_list.appendChild(i_variation)
        intensity_var.appendChild(inter_intensity_variation_list)

        intra_intensity_variation_list = dom_tree_out.createElement("IntraIntensityVariationList")

        for intra_intensity_variation in list_intensity_variations_intra:
            i_variation = dom_tree_out.createElement("IntraIntensityVariation")
            normalized_intra_intensity_variation = (intra_intensity_variation * 100) / 255
            txt_node = dom_tree_out.createTextNode(str(normalized_intra_intensity_variation))
            i_variation.appendChild(txt_node)
            intra_intensity_variation_list.appendChild(i_variation)
        intensity_var.appendChild(intra_intensity_variation_list)

        nb_shot_transitions_list = dom_tree_out.createElement("NumberShotTransitionList")
        nb_trans = dom_tree_out.createElement("NumberShotTransition")
        txt_node = dom_tree_out.createTextNode(str(nb_transitions * 100))
        nb_trans.appendChild(txt_node)
        nb_shot_transitions_list.appendChild(nb_trans)
        shot_descriptors.appendChild(nb_shot_transitions_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region Number Speaker Transition List
        # print "Number Speaker Transition List "
        # print 'start:  {}'.format(datetime.datetime.now().time())
        nb_speaker_transition = compute_number_speaker_switch_per_segment(input_dom_tree, start, end)
        nb_speaker_transition_list = dom_tree_out.createElement("NumberSpeakerTransitionList")

        nb_trans = dom_tree_out.createElement("NumberSpeakerTransition")
        txt_node = dom_tree_out.createTextNode(str(nb_speaker_transition))
        nb_trans.appendChild(txt_node)
        nb_speaker_transition_list.appendChild(nb_trans)

        shot_descriptors.appendChild(nb_speaker_transition_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region Speech Music Alignment List
        # print "Speech Music Alignment List "
        # print 'start:  {}'.format(datetime.datetime.now().time())
        list_speech_music_combinations = get_speech_music_non_speech_non_music_percentage(input_dom_tree, start, end)
        s_m_ns_nm_list = dom_tree_out.createElement("SpeechMusicAlignmentList")

        alignment = dom_tree_out.createElement("SpeechMusicAlignment")

        s = dom_tree_out.createElement("Speech")
        s.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["P"], 2))))
        alignment.appendChild(s)

        m = dom_tree_out.createElement("Music")
        m.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["M"], 2))))
        alignment.appendChild(m)

        s_m = dom_tree_out.createElement("SpeechWithMusic")
        s_m.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["PM"], 2))))
        alignment.appendChild(s_m)

        s_nm = dom_tree_out.createElement("SpeechWithNonMusic")
        s_nm.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["PNM"], 2))))
        alignment.appendChild(s_nm)

        ns_m = dom_tree_out.createElement("NonSpeechWithMusic")
        ns_m.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["NPM"], 2))))
        alignment.appendChild(ns_m)

        ns_nm = dom_tree_out.createElement("NonSpeechWithNonMusic")
        ns_nm.appendChild(dom_tree_out.createTextNode(str(round(list_speech_music_combinations["NPNM"], 2))))
        alignment.appendChild(ns_nm)

        s_m_ns_nm_list.appendChild(alignment)
        shot_descriptors.appendChild(s_m_ns_nm_list)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion

        # region get the list of words
        # print "get the list of words "
        # print 'start:  {}'.format(datetime.datetime.now().time())
        data = extract_text_data(input_dom_tree, start, end)
        shot_descriptors.appendChild(data)
        # print 'end:  {}'.format(datetime.datetime.now().time())
        # endregion
        shot_descriptor_list.append(shot_descriptors)
        # print "*************************************************************"
        if counter > 5:
            counter = 0
            dom_tree_out = xml.dom.minidom.parse(output_xml_file)
            for sh in shot_descriptor_list:
                dom_tree_out.getElementsByTagName("Shots")[0].appendChild(sh)
            file_handle = codecs.open(output_xml_file, 'wb', 'utf8')
            dom_tree_out.writexml(file_handle)
            file_handle.close()
            shot_descriptor_list = []
            dom_tree_out = Document()

    dom_tree_out = xml.dom.minidom.parse(output_xml_file)
    for sh in shot_descriptor_list:
        dom_tree_out.getElementsByTagName("Shots")[0].appendChild(sh)
    file_handle = codecs.open(output_xml_file, 'wb', 'utf8')
    dom_tree_out.writexml(file_handle)
    file_handle.close()
    # print "Combination of descriptors are generated successfully for file:" + input_xml_file


text:
#! /usr/local/bin/python
# -*- coding:utf-8 -*-

import os
import sys
import datetime

import cv2
import numpy as np

from model_train import Model
from facetime_call import FaceTimeDriver

DATA_FILE_PATH = './data'

def capture(image, label):
    sub_dir = os.path.join(DATA_FILE_PATH, label)
    if not os.path.exists(sub_dir):
        os.mkdir(sub_dir)
    filename = datetime.date.strftime(datetime.datetime.now(), "%Y_%m_%d_%H_%M_%S") + "_" + label + '.png'
    filepath = os.path.join(sub_dir, filename)
    cv2.imwrite(filepath, image)

def process(image, model, face_driver):
    result = model.predict(image)
    if face_driver is not None:
        face_driver.action(result)
    return result

if __name__ == '__main__':
    driver = None
    # load model
    if len(sys.argv) <= 1:
        model = Model()
        model.load()
        driver = FaceTimeDriver()

    cap = cv2.VideoCapture(0)
    cascade_path = "/usr/local/opt/opencv/share/OpenCV/haarcascades/haarcascade_frontalface_default.xml"
    cascade = cv2.CascadeClassifier(cascade_path)

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret: continue
        frame = frame[:, ::-1, :]
        frame = frame.copy()
        frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        face_rect = cascade.detectMultiScale(frame_gray, scaleFactor=1.2, minNeighbors=3, minSize=(10, 10))
        if len(face_rect) > 0:
            # print('face detected')
            color = (255, 255, 255)  # 白
            for rect in face_rect:
                width, height = rect[2:4]
                if width < 150 or height < 150:
                    continue
                x, y = rect[0:2]
                cv2.rectangle(frame, tuple(rect[0:2]), tuple(rect[0:2] + rect[2:4]), color, thickness=2)
                image = frame_gray[y - 10: y + height, x: x + width]
                image = image[:, :, None] * np.array([[[1, 1, 1]]])
                image = image.copy()
                if len(sys.argv) > 1:
                    label = sys.argv[1]
                    capture(image, label)
                else:
                    guess = process(image, model, driver)
                    if guess != 0:
                        cv2.rectangle(frame, tuple(rect[0:2]), tuple(rect[0:2] + rect[2:4]), (0, 255, 0), thickness=2)

        # don't show if you don't need it.
        cv2.imshow('Gesture', frame)
        k = cv2.waitKey(50)
        if k == 27:
            break

    cap.release()
    cv2.destroyAllWindows()

max_stars_repo_name: gavinhub/KissSensor

max_issues_repo_name: gavinhub/KissSensor

max_forks_repo_name: gavinhub/KissSensor


text:
import sys, os
project_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-2])
if project_path not in sys.path:
    sys.path.append(project_path)

import argparse
import numpy as np
from copy import deepcopy
from sklearn.utils import shuffle
from typing import Callable, Optional, Sequence, Iterable
from commonly_used_code import config, helper_fn

class Batch:
    def __init__(self):
        self.init_all()

    def init_all(self):
        self.init_oovs()
        self.init_src_tar()
        self.init_fact()
        self.init_conv()

    def init_oovs(self):
        # each sample has an oov ids and its max oov lens
        # oov dict
        self.oov_ids_dict = []

    def init_src_tar(self):
        # normal ids
        self.src_ids = []
        self.tar_ids = []
        self.tar_loss_ids = []

        # for multi-task, take the first fact as the answer
        self.fact_tar_ids = []
        self.fact_tar_loss_ids = []

        # OOV expand for each data set
        self.src_ids_exp = []
        self.tar_ids_exp = []
        self.tar_loss_ids_exp = []

    def init_fact(self):
        self.fact_ids = []
        self.fact_ids_exp = []

    def init_conv(self):
        self.conv_ids = []
        self.conv_ids_exp = []

    def np_format(self):
        self.src_ids = np.asarray(self.src_ids)
        self.tar_ids = np.asarray(self.tar_ids)
        self.tar_loss_ids = np.asarray(self.tar_loss_ids)
        self.tar_loss_ids = np.reshape(self.tar_loss_ids, (self.tar_loss_ids.shape[0], self.tar_loss_ids.shape[1], 1))
        self.fact_ids = np.asarray(self.fact_ids)
        if len(self.conv_ids) != 0:
            self.conv_ids = np.asarray(self.conv_ids)

        # used for multi-task
        self.fact_tar_ids = np.asarray(self.fact_tar_ids)
        self.fact_tar_loss_ids = np.asarray(self.fact_tar_loss_ids)
        self.fact_tar_loss_ids = np.reshape(self.fact_tar_loss_ids, (self.fact_tar_loss_ids.shape[0], self.fact_tar_loss_ids.shape[1], 1))

        self.src_ids_exp = np.asarray(self.src_ids_exp)
        self.tar_ids_exp = np.asarray(self.tar_ids_exp)
        self.tar_loss_ids_exp = np.asarray(self.tar_loss_ids_exp)
        self.tar_loss_ids_exp = np.reshape(self.tar_loss_ids_exp, (self.tar_loss_ids_exp.shape[0], self.tar_loss_ids_exp.shape[1], 1))
        self.fact_ids_exp = np.asarray(self.fact_ids_exp)

        if len(self.conv_ids_exp) != 0:
            self.conv_ids_exp = np.asarray(self.conv_ids_exp)

    def batch_shuffle(self, have_conv=None):
        if have_conv == None:
            self.src_ids, self.tar_ids, self.tar_loss_ids, self.fact_ids, self.fact_tar_ids, self.fact_tar_loss_ids, \
                self.src_ids_exp, self.tar_ids_exp, self.tar_loss_ids_exp, self.fact_ids_exp = \
                    shuffle( self.src_ids, self.tar_ids, self.tar_loss_ids, self.fact_ids, self.fact_tar_ids, self.fact_tar_loss_ids, \
                        self.src_ids_exp, self.tar_ids_exp, self.tar_loss_ids_exp, self.fact_ids_exp)
        else:
            self.src_ids, self.tar_ids, self.tar_loss_ids, self.conv_ids, self.fact_ids, self.fact_tar_ids, self.fact_tar_loss_ids, \
                self.src_ids_exp, self.tar_ids_exp, self.tar_loss_ids_exp, self.fact_ids_exp = \
                    shuffle( self.src_ids, self.tar_ids, self.tar_loss_ids, self.conv_ids, self.fact_ids, self.fact_tar_ids, self.fact_tar_loss_ids, \
                        self.src_ids_exp, self.tar_ids_exp, self.tar_loss_ids_exp, self.fact_ids_exp)

class DataSet:
    def __init__(self, args):
        self.args = args
        self.__set_file_path()

        # get global token and ids
        self.src_token_ids, self.src_id_tokens, self.src_vocab_size = self.__read_global_ids(self.src_global_token_path)
        self.tar_token_ids, self.tar_id_tokens, self.tar_vocab_size = self.__read_global_ids(self.tar_global_token_path)
        self.train_sample_num = 0
        self.valid_sample_num = 0
        self.test_sample_num = 0

        # just set the max fact and conv length the same as source
        self.max_src_len = args.src_seq_length
        self.max_tar_len = args.tar_seq_length
        if not args.fact_seq_length:
            self.max_fact_len = args.src_seq_length
        else:
            print('This is fact length')
            self.max_fact_len = args.fact_seq_length
        #if not args.conv_seq_length:
        self.max_conv_len = args.conv_seq_length
        #else:
        #    self.max_conv_len = args.conv_seq_length
        self.src_len = self.max_src_len - config.src_reserved_pos
        self.tar_len = self.max_tar_len - config.tar_reserved_pos

        self.pad_fact_seqs = helper_fn.pad_with_pad([self.pad_id], self.max_fact_len, self.pad_id)
        self.pad_conv_seqs = helper_fn.pad_with_pad([self.pad_id], self.max_conv_len, self.pad_id)

        self.__get_sample_numbers()

    def __set_file_path(self):
        if self.args.data_set == 'example_data':
            self.train_set_path = config.example_data_train_qa_path
            self.train_conv_path = config.example_data_train_conv_path
            self.train_sent_fact_path = config.example_data_train_sent_fact_path
            self.valid_set_path = config.example_data_valid_qa_path
            self.valid_conv_path = config.example_data_valid_conv_path
            self.valid_sent_fact_path = config.example_data_valid_sent_fact_path
            self.test_set_path = config.example_data_test_qa_path
            self.test_conv_path = config.example_data_test_conv_path
            self.test_sent_fact_path = config.example_data_test_sent_fact_path
            self.src_global_token_path = config.example_data_global_token_path
            self.tar_global_token_path = config.example_data_global_token_path

    def __get_sample_numbers(self):
        print('Getting total samples numbers...')
        with open(self.train_set_path) as f:
            for line in f:
                self.train_sample_num += 1
        with open(self.valid_set_path) as f:
            for line in f:
                self.valid_sample_num += 1
        with open(self.test_set_path) as f:
            for line in f:
                self.test_sample_num += 1

    def _line2ids(self, line, max_len, oov_token2id=None):
        seq = []
        seq_exp = []
        for token in line.strip().split(' '):
            _id = self.src_token_ids.get(token, self.unk_id)
            seq.append(_id)
            _id_exp = _id
            if _id == self.unk_id:
                if oov_token2id != None:
                    _id_exp = oov_token2id.get(token, self.unk_id)
            seq_exp.append(_id_exp)

        seq = seq[:max_len]
        seq_exp = seq_exp[:max_len]
        return seq, seq_exp

    def _get_oov_dict(self, qa_line, fact_line):
        oov_token2id = dict()
        oov_id2token = dict()
        oov_index = self.src_vocab_size
        # just calculate question, ignoring the target tokens
        ques_line = qa_line.strip().split('\t')[1]
        for token in ques_line.strip().split(' '):
            token = token.strip()
            _id = self.src_token_ids.get(token, self.unk_id)
            if _id == self.unk_id:
                if token not in oov_token2id:
                    oov_token2id[token] = oov_index
                    oov_id2token[oov_index] = token
                    oov_index += 1

        elems = fact_line.strip().split('\t')
        for fact in elems[1:self.args.fact_number + 1]:
            for token in fact.strip().split(' '):
                token = token.strip()
                _id = self.src_token_ids.get(token, self.unk_id)
                if _id == self.unk_id:
                    if token not in oov_token2id:
                        oov_token2id[token] = oov_index
                        oov_id2token[oov_index] = token
                        oov_index += 1

        return oov_token2id, oov_id2token

    def _deal_qa_line(self, index, line, oov_token2id):
        elems = line.strip().split('\t')

        text = elems[1].strip()
        seq, seq_exp = self._line2ids(text, self.src_len, oov_token2id)
        src = helper_fn.pad_with_start_end(seq, self.max_src_len, self.start_id, self.end_id, self.pad_id)
        src_exp = helper_fn.pad_with_start_end(seq_exp, self.max_src_len, self.start_id, self.end_id, self.pad_id)

        # used for multi_task. If there is no fact, use src as the answer
        seq, seq_exp = self._line2ids(text, self.tar_len, oov_token2id)
        src_tar = helper_fn.pad_with_start(seq, self.max_tar_len, self.start_id, self.pad_id)
        src_tar_loss = helper_fn.pad_with_end(seq, self.max_tar_len, self.end_id, self.pad_id)

        text = elems[2].strip()
        seq, seq_exp = self._line2ids(text, self.tar_len, oov_token2id)
        tar = helper_fn.pad_with_start(seq, self.max_tar_len, self.start_id, self.pad_id)
        tar_exp = helper_fn.pad_with_start(seq_exp, self.max_tar_len, self.start_id, self.pad_id)

        tar_loss = helper_fn.pad_with_end(seq, self.max_tar_len, self.end_id, self.pad_id)
        tar_loss_exp = helper_fn.pad_with_end(seq_exp, self.max_tar_len, self.end_id, self.pad_id)

        return src, tar, tar_loss, src_exp, tar_exp, tar_loss_exp, src_tar, src_tar_loss

    def _deal_fact_line(self, index, line, oov_token2id):
        line = line.strip()
        cur_fact_ids = []
        cur_fact_ids_exp = []
        fact_tar = None
        fact_tar_loss = None
        elems = line.split('\t')
        no_fact_tag = False
        # if there is no fact, add pad sequence
        if elems[1] == config.NO_FACT or elems[1] == config.NO_CONTEXT:
            cur_fact_ids.append(self.pad_fact_seqs)
            cur_fact_ids_exp.append(self.pad_fact_seqs)
            no_fact_tag = True
        else:
            for index, text in enumerate(elems[1:]):
                seq, seq_exp = self._line2ids(text, self.max_fact_len, oov_token2id)
                new_seq = helper_fn.pad_with_pad(seq, self.max_fact_len, self.pad_id)
                cur_fact_ids.append(new_seq)
                new_seq_exp = helper_fn.pad_with_pad(seq_exp, self.max_fact_len, self.pad_id)
                cur_fact_ids_exp.append(new_seq_exp)
                if index == 0:
                    seq, seq_exp = self._line2ids(text, self.tar_len, oov_token2id)
                    fact_tar = helper_fn.pad_with_start(seq, self.max_tar_len, self.start_id, self.pad_id)
                    fact_tar_loss = helper_fn.pad_with_end(seq, self.max_tar_len, self.start_id, self.pad_id)

        # pad fact number
        cur_fact_ids = cur_fact_ids[:self.args.fact_number]
        cur_fact_ids_exp = cur_fact_ids_exp[:self.args.fact_number]

        cur_fact_ids = cur_fact_ids + [self.pad_fact_seqs] * (self.args.fact_number - len(cur_fact_ids))
        cur_fact_ids_exp = cur_fact_ids_exp + [self.pad_fact_seqs] * (self.args.fact_number - len(cur_fact_ids_exp))

        return no_fact_tag, cur_fact_ids, cur_fact_ids_exp, fact_tar, fact_tar_loss

    def _deal_conv_line(self, index, line, oov_token2id):
        line = line.strip()
        cur_conv_ids = []
        cur_conv_ids_exp = []
        conv_tar = None
        conv_tar_loss = None
        elems = line.split('\t')
        no_conv_tag = False
        # if there is no conv, add pad sequence
        if elems[1] == config.NO_CONTEXT:
            cur_conv_ids.append(self.pad_conv_seqs)
            cur_conv_ids_exp.append(self.pad_conv_seqs)
            no_conv_tag = True
        else:
            for index, text in enumerate(elems[1:]):
                seq, seq_exp = self._line2ids(text, self.max_conv_len, oov_token2id)
                new_seq = helper_fn.pad_with_pad(seq, self.max_conv_len, self.pad_id)
                cur_conv_ids.append(new_seq)
                new_seq_exp = helper_fn.pad_with_pad(seq_exp, self.max_conv_len, self.pad_id)
                cur_conv_ids_exp.append(new_seq_exp)
                if index == 0:
                    seq, seq_exp = self._line2ids(text, self.tar_len, oov_token2id)
                    conv_tar = helper_fn.pad_with_start(seq, self.max_tar_len, self.start_id, self.pad_id)
                    conv_tar_loss = helper_fn.pad_with_end(seq, self.max_tar_len, self.start_id, self.pad_id)

        # pad conv number
        cur_conv_ids = cur_conv_ids[:self.args.conv_number]
        cur_conv_ids_exp = cur_conv_ids_exp[:self.args.conv_number]

        cur_conv_ids = cur_conv_ids + [self.pad_conv_seqs] * (self.args.conv_number - len(cur_conv_ids))
        cur_conv_ids_exp = cur_conv_ids_exp + [self.pad_conv_seqs] * (self.args.conv_number - len(cur_conv_ids_exp))

        return no_conv_tag, cur_conv_ids, cur_conv_ids_exp, conv_tar, conv_tar_loss

    def _check_and_shuffle(self, _batch, have_conv=None):
        #print('in check and shuffle...')
        #print('src_ids len: ', len(_batch.src_ids))
        #print('tar_ids len: ', len(_batch.tar_ids))
        #print('tar_loss_ids len: ', len(_batch.tar_loss_ids))
        #print('fact_ids len: ', len(_batch.fact_ids))

        assert (len(_batch.src_ids) == len(_batch.tar_ids))
        assert (len(_batch.src_ids) == len(_batch.tar_loss_ids))
        assert (len(_batch.src_ids) == len(_batch.fact_ids))
        assert (len(_batch.src_ids) == len(_batch.fact_tar_ids))
        assert (len(_batch.src_ids_exp) == len(_batch.tar_ids_exp))
        assert (len(_batch.src_ids_exp) == len(_batch.tar_loss_ids_exp))
        assert (len(_batch.src_ids_exp) == len(_batch.fact_ids_exp))

        assert (len(_batch.src_ids) > 0)
        assert (len(_batch.tar_ids) > 0)
        assert (len(_batch.tar_loss_ids) > 0)
        assert (len(_batch.fact_ids) > 0)
        assert (len(_batch.fact_tar_ids) > 0)

        assert (len(_batch.src_ids_exp) > 0)
        assert (len(_batch.tar_ids_exp) > 0)
        assert (len(_batch.tar_loss_ids_exp) > 0)
        assert (len(_batch.fact_ids_exp) > 0)

        if have_conv != None:
            assert (len(_batch.src_ids) == len(_batch.conv_ids))

        return _batch.batch_shuffle(have_conv)

    def _fit_model(self, _batch, file_type, model_type):
        '''
        Please carefully choose the output type to fit with your model's inputs
        '''

        fit_output_loss = np.zeros((_batch.src_ids.shape[0], 1))

        no_fact_list = ['pg_only_ques_transformer', 'universal_transformer', 'transformer', 'seq2seq', 'copy_mechanism']
        fact_list = ['sr', 'ted', 'spg_transformer', 'pg_transformer', 'memnn']
        vspgt_list = ['mvspg_transformer', 'vspg_transformer']

        if model_type in no_fact_list:
            return ([_batch.src_ids, _batch.tar_ids], _batch.tar_loss_ids)
        elif model_type in fact_list:
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_ids], _batch.tar_loss_ids)
        elif model_type == 'multi_task':
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_tar_ids, _batch.fact_ids], \
                [_batch.tar_loss_ids, _batch.fact_tar_loss_ids, _batch.tar_loss_ids])
        elif model_type == 'multi_task_decex_transformer':
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_tar_ids, _batch.fact_ids], \
                [_batch.tar_loss_ids, _batch.fact_tar_loss_ids])
        elif model_type == 'merge_multi_task_decex_transformer':
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_tar_ids, _batch.fact_ids], \
                [_batch.tar_loss_ids])
        elif model_type in vspgt_list:
            if file_type == 'test':
                return ([_batch.src_ids, _batch.tar_ids, _batch.fact_ids, \
                    _batch.src_ids_exp, _batch.fact_ids_exp], _batch.tar_loss_ids_exp, _batch.oov_ids_dict)
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_ids, \
                _batch.src_ids_exp, _batch.fact_ids_exp], _batch.tar_loss_ids_exp)
        elif model_type == 'vspg_only_ques_transformer':
            if file_type == 'test':
                return ([_batch.src_ids, _batch.tar_ids, _batch.src_ids_exp], _batch.tar_loss_ids_exp, _batch.oov_ids_dict)
            return ([_batch.src_ids, _batch.tar_ids, _batch.src_ids_exp], _batch.tar_loss_ids_exp)

        elif model_type == 'cvspg_transformer':
            if file_type == 'test':
                return ([_batch.src_ids, _batch.tar_ids, _batch.conv_ids, _batch.fact_ids, \
                    _batch.src_ids_exp, _batch.fact_ids_exp], _batch.tar_loss_ids_exp, _batch.oov_ids_dict)
            return ([_batch.src_ids, _batch.tar_ids, _batch.conv_ids, _batch.fact_ids, \
                _batch.src_ids_exp, _batch.fact_ids_exp], _batch.tar_loss_ids_exp)
        elif model_type == 'fcvspg_transformer':
            if file_type == 'test':
                return ([_batch.src_ids, _batch.tar_ids, _batch.fact_ids, _batch.conv_ids, \
                    _batch.src_ids_exp, _batch.fact_ids_exp, _batch.conv_ids_exp], _batch.tar_loss_ids_exp, _batch.oov_ids_dict)
            return ([_batch.src_ids, _batch.tar_ids, _batch.fact_ids, _batch.conv_ids, \
                _batch.src_ids_exp, _batch.fact_ids_exp, _batch.conv_ids_exp], _batch.tar_loss_ids_exp)

        else:
            raise ValueError('The input model type: %s is not available. ' \
                'Please chech the file: data_reader.py line: _fit_model' % model_type)

    # This is a data generator, which is suitable for large-scale data set
    def data_generator(self, file_type, model_type):
        '''
        :param file_type: This is supposed to be: train, valid, or test
        :param model_type: This is supposed to be different models' name
        '''
        print('This is in data generator...')
        assert(self.max_src_len > 0)
        assert(self.max_tar_len > 0)
        assert(self.max_fact_len > 0)
        assert(self.max_conv_len > 0)
        assert file_type == 'train' or file_type == 'valid' or file_type == 'test'

        if file_type == 'train':
            qa_path = self.train_set_path
            conv_path = self.train_conv_path
            fact_path = self.train_sent_fact_path
        elif file_type == 'valid':
            qa_path = self.valid_set_path
            conv_path = self.valid_conv_path
            fact_path = self.valid_sent_fact_path
        elif file_type == 'test':
            qa_path = self.test_set_path
            conv_path = self.test_conv_path
            fact_path = self.test_sent_fact_path

        batch = Batch()

        def _read_qa_fact(batch):
            while True:
                f_qa = open(qa_path)
                f_fact = open(fact_path)
                print(qa_path)
                print(fact_path)
                for index, (qa_line, fact_line) in enumerate(zip(f_qa, f_fact)):
                    qa_id = qa_line.strip().split('\t')[0]
                    fact_id = fact_line.strip().split('\t')[0]
                    assert (qa_id == fact_id)

                    # get oov dict first before deal with each line
                    oov_token2id, oov_id2token = self._get_oov_dict(qa_line, fact_line)

                    src, tar, tar_loss, src_exp, tar_exp, tar_loss_exp, src_tar, src_tar_loss = \
                        self._deal_qa_line(index, qa_line, oov_token2id)
                    no_fact_tag, facts, facts_exp, fact_tar, fact_tar_loss = self._deal_fact_line(index, fact_line, oov_token2id)

                    batch.src_ids.append(src)
                    batch.src_ids_exp.append(src_exp)
                    batch.tar_ids.append(tar)
                    batch.tar_ids_exp.append(tar_exp)
                    batch.tar_loss_ids.append(tar_loss)
                    batch.tar_loss_ids_exp.append(tar_loss_exp)
                    batch.fact_ids.append(facts)
                    batch.fact_ids_exp.append(facts_exp)

                    if no_fact_tag is True:
                        batch.fact_tar_ids.append(src_tar)
                        batch.fact_tar_loss_ids.append(src_tar_loss)
                    else:
                        batch.fact_tar_ids.append(fact_tar)
                        batch.fact_tar_loss_ids.append(fact_tar_loss)

                    batch.oov_ids_dict.append(oov_id2token)

                    if ((index + 1) % self.args.batch_size == 0):
                        #print('index + 1: ', index+1)
                        ret = deepcopy(batch)
                        #self._check_and_shuffle(ret)
                        ret.np_format()
                        batch = Batch()
                        yield self._fit_model(ret, file_type, model_type)

                if (len(batch.src_ids) != 0):
                    ret = deepcopy(batch)
                    #self._check_and_shuffle(ret)
                    ret.np_format()
                    batch = Batch()
                    yield self._fit_model(ret, file_type, model_type)

        def _read_qa_fact_conv(batch):
            while True:
                f_qa = open(qa_path)
                f_fact = open(fact_path)
                f_conv = open(conv_path)
                print(qa_path)
                print(fact_path)
                print(conv_path)
                for index, (qa_line, fact_line, conv_line) in enumerate(zip(f_qa, f_fact, f_conv)):
                    qa_id = qa_line.strip().split('\t')[0]
                    fact_id = fact_line.strip().split('\t')[0]
                    conv_id = conv_line.strip().split('\t')[0]
                    assert (qa_id == fact_id)
                    assert (qa_id == conv_id)

                    # get oov dict first before deal with each line
                    oov_token2id, oov_id2token = self._get_oov_dict(qa_line, fact_line)

                    src, tar, tar_loss, src_exp, tar_exp, tar_loss_exp, src_tar, src_tar_loss = \
                        self._deal_qa_line(index, qa_line, oov_token2id)
                    no_fact_tag, facts, facts_exp, fact_tar, fact_tar_loss = self._deal_fact_line(index, fact_line, oov_token2id)
                    no_conv_tag, convs, convs_exp, conv_tar, conv_tar_loss = self._deal_conv_line(index, conv_line, oov_token2id)

                    batch.src_ids.append(src)
                    batch.src_ids_exp.append(src_exp)
                    batch.tar_ids.append(tar)
                    batch.tar_ids_exp.append(tar_exp)
                    batch.tar_loss_ids.append(tar_loss)
                    batch.tar_loss_ids_exp.append(tar_loss_exp)
                    batch.fact_ids.append(facts)
                    batch.fact_ids_exp.append(facts_exp)
                    batch.conv_ids.append(convs)
                    batch.conv_ids_exp.append(convs_exp)

                    if no_fact_tag is True:
                        batch.fact_tar_ids.append(src_tar)
                        batch.fact_tar_loss_ids.append(src_tar_loss)
                    else:
                        batch.fact_tar_ids.append(fact_tar)
                        batch.fact_tar_loss_ids.append(fact_tar_loss)

                    batch.oov_ids_dict.append(oov_id2token)

                    if ((index + 1) % self.args.batch_size == 0):
                        #print('index + 1: ', index+1)
                        ret = deepcopy(batch)
                        #self._check_and_shuffle(ret)
                        ret.np_format()
                        batch = Batch()
                        yield self._fit_model(ret, file_type, model_type)

                if (len(batch.src_ids) != 0):
                    ret = deepcopy(batch)
                    #self._check_and_shuffle(ret)
                    ret.np_format()
                    batch = Batch()
                    yield self._fit_model(ret, file_type, model_type)

        if self.args.use_conv == True:
            return _read_qa_fact_conv(batch)
        else:
            return _read_qa_fact(batch)

    def __read_global_ids(self, token_path):
        f = open(token_path)
        token_ids = dict()
        id_tokens = dict()
        vocab_size = 0
        for line in f:
            elems = line.strip().split('\t')
            word = elems[0]
            index = int(elems[1])
            token_ids[word] = index
            id_tokens[index] = word
            vocab_size += 1

        self.start_id = token_ids.get(config.START_TOKEN, -1)
        self.end_id = token_ids.get(config.END_TOKEN, -1)
        self.pad_id = token_ids.get(config.PAD_TOKEN, -1)
        self.unk_id = token_ids.get(config.UNK_TOKEN, -1)
        assert(self.start_id != -1)
        assert(self.end_id != -1)
        assert(self.pad_id != -1)
        assert(self.unk_id != -1)

        return token_ids, id_tokens, vocab_size


text:
import numpy as np
import os
import oandapyV20
import oandapyV20.endpoints.instruments as instruments
import oandapyV20.endpoints.orders as orders
from time import sleep
from dotenv import load_dotenv

class MovingAverageTrader:

    def __init__(self, short_timeframe=50, long_timeframe=200, instrument='EUR_USD', granularity='S5', volume = 1000):
        self.params = {'granularity': granularity, 'count': long_timeframe}
        self.accountID = os.environ['ACCOUNT_ID']
        self.client = oandapyV20.API(access_token=os.environ['TOKEN'])
        self.short_timeframe = short_timeframe
        self.long_timeframe = long_timeframe
        self.instrument = instrument
        self.granularity = granularity
        self.volume = volume
        self.stock = 0

    def get_candles(self):
        output = instruments.InstrumentsCandles(instrument=self.instrument, params=self.params)
        candles = self.client.request(output)

        return candles

    def create_trade(self, position):
        data = {"order": {"units": position, "instrument": self.instrument, "timeInForce": "FOK", "type": "MARKET",
                          "positionFill": "DEFAULT"}}
        r = orders.OrderCreate(self.accountID, data=data)
        self.client.request(r)

        return r.response

    def create_average(self, instrument_list):
        short_list = []
        long_list = []
        # TODO: Make more Pythonic
        for i, r in enumerate(instrument_list['candles'][-self.long_timeframe:]):
            long_list.append(instrument_list['candles'][i]['mid']['c'])
        for y, r in enumerate(instrument_list['candles'][-self.short_timeframe:]):
            short_list.append(instrument_list['candles'][y]['mid']['c'])

        return np.array(short_list).astype(np.float).mean(), np.array(long_list).astype(np.float).mean()

n = MovingAverageTrader()
load_dotenv()

while 1 == 1:
    candle_list = n.get_candles()
    short_list, long_list = n.create_average(candle_list)

    if short_list > long_list:
        if n.stock < n.volume:
            n.create_trade((n.volume / 10))
            n.stock += (n.volume / 10)
            print('Just bought {} {}'.format(n.volume / 10, n.instrument))

    if long_list > short_list:
        if n.stock > - n.volume:
            n.create_trade((- n.volume / 10))
            n.stock -= (n.volume / 10)
            print('Just sold {} {}'.format(n.volume / 10, n.instrument))

    print('As of now we own {} {}'.format(n.stock, n.instrument))
    sleep(5)


text:
"""
This module contains functions and classes for arbitrary-precision
numerical computations.

>>> from sympy.numerics import Float
>>> from sympy.numerics.functions import exp, log, sqrt
>>> from sympy.numerics.constants import pi_float
>>> Float.setdps(50)
>>> print pi_float()
3.1415926535897932384626433832795028841971693993751
>>> print exp(1)
2.7182818284590452353602874713526624977572470937000
>>> print log(2)
0.69314718055994530941723212145817656807550013436025
>>> print sqrt(2)
1.4142135623730950488016887242096980785696718753769

"""

from float_ import *
#from constants import *
#from functions import *
#from functions2 import *
from optimize import polyroots, bisect, secant
from quad import nintegrate
from evalf_ import evalf, polyfunc

max_stars_repo_name: certik/sympy-oldcore

max_issues_repo_name: certik/sympy-oldcore

max_forks_repo_name: certik/sympy-oldcore

reason: from sympy


text:
# Author: Simon Blanke
# Email: simon.blanke@yahoo.com
# License: MIT License

import random
import numpy as np
from .search_tracker import SearchTracker
from ..converter import Converter
from ..results_manager import ResultsManager

class BaseOptimizer(SearchTracker):
    def __init__(
        self,
        search_space,
        initialize={"grid": 4, "random": 2, "vertices": 4},
    ):
        super().__init__()
        self.conv = Converter(search_space)
        self.results_mang = ResultsManager(self.conv)
        self.initialize = initialize

        self.optimizers = [self]

    def move_random(self):
        position = []
        for search_space_pos in self.conv.search_space_positions:
            pos_ = random.choice(search_space_pos)
            position.append(pos_)

        return np.array(position)

    def track_nth_iter(func):
        def wrapper(self, *args, **kwargs):
            self.nth_iter = len(self.pos_new_list)
            pos = func(self, *args, **kwargs)
            self.pos_new = pos
            return pos

        return wrapper

    def random_restart(func):
        def wrapper(self, *args, **kwargs):
            if self.rand_rest_p > random.uniform(0, 1):
                return self.move_random()
            else:
                return func(self, *args, **kwargs)

        return wrapper

    def init_pos(self, pos):
        self.pos_new = pos
        self.nth_iter = len(self.pos_new_list)

    def evaluate(self, score_new):
        self.score_new = score_new

        if self.pos_best is None:
            self.pos_best = self.pos_new
            self.pos_current = self.pos_new

            self.score_best = score_new
            self.score_current = score_new

        # self._evaluate_new2current(score_new)
        # self._evaluate_current2best()


text:
"""
@Date: 2021/08/12
@description:
"""

import torch
import torch.nn as nn
import numpy as np

from visualization.grad import get_all

class GradLoss(nn.Module):
    def __init__(self):
        super().__init__()
        self.loss = nn.L1Loss()
        self.cos = nn.CosineSimilarity(dim=-1, eps=0)

        self.grad_conv = nn.Conv1d(1, 1, kernel_size=3, stride=1, padding=0, bias=False, padding_mode='circular')
        self.grad_conv.weight = nn.Parameter(torch.tensor([[[1, 0, -1]]]).float())
        self.grad_conv.weight.requires_grad = False

    def forward(self, gt, dt):
        gt_direction, _, gt_angle_grad = get_all(gt['depth'], self.grad_conv)
        dt_direction, _, dt_angle_grad = get_all(dt['depth'], self.grad_conv)

        normal_loss = (1 - self.cos(gt_direction, dt_direction)).mean()
        grad_loss = self.loss(gt_angle_grad, dt_angle_grad)
        return [normal_loss, grad_loss]

if __name__ == '__main__':
    from dataset.mp3d_dataset import MP3DDataset
    from utils.boundary import depth2boundaries
    from utils.conversion import uv2xyz
    from visualization.boundary import draw_boundaries
    from visualization.floorplan import draw_floorplan

    def show_boundary(image, depth, ratio):
        boundary_list = depth2boundaries(ratio, depth, step=None)
        draw_boundaries(image.transpose(1, 2, 0), boundary_list=boundary_list, show=True)
        draw_floorplan(uv2xyz(boundary_list[0])[..., ::2], show=True, center_color=0.8)

    mp3d_dataset = MP3DDataset(root_dir='../src/dataset/mp3d', mode='train', patch_num=256)
    gt = mp3d_dataset.__getitem__(1)
    gt['depth'] = torch.from_numpy(gt['depth'][np.newaxis])  # batch size is 1
    dummy_dt = {
        'depth': gt['depth'].clone(),
    }
    # dummy_dt['depth'][..., 20] *= 3  # some different

    # show_boundary(gt['image'], gt['depth'][0].numpy(), gt['ratio'])
    # show_boundary(gt['image'], dummy_dt['depth'][0].numpy(), gt['ratio'])

    grad_loss = GradLoss()
    loss = grad_loss(gt, dummy_dt)
    print(loss)

max_stars_repo_name: zhigangjiang/LGT-Net

max_issues_repo_name: zhigangjiang/LGT-Net

max_forks_repo_name: zhigangjiang/LGT-Net


text:
"""
second order neural interrupting interactions
interaction is class, gamma is function, delta function

if need to run in here pull in lib folder to dir
"""
import sys
import os
# sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

from pathlib import Path
sys.path.append(Path(__file__).resolve().parents[1])
if __name__ == '__main__' and __package__ is None:
    __package__ = 'kurosc'

from lib.plotformat import setup

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

""" Eqn 13 + Fig 4
"""

class interaction(object):
    def __init__(self,
                 dim:tuple = (3,3)
                 ):
        self.dimension = dim

    def delta(self,
              phase_array: np.array = np.array([0,1,1,0])
              ) -> np.ndarray:
        """pase difference of element from global 1D array"""

        d = np.zeros([self.dimension[0]*self.dimension[1],
                      self.dimension[1]*self.dimension[0]])

        # phase_array = phase_array.reshape((self.dimension[0],self.dimension[1]))

        # TODO validat this index assignment with ravel()
        for (k,p) in enumerate(phase_array):
            d[k,:] = np.array((phase_array - p), dtype = float)  # TODO remove % np.pi  ?

        """
        # k=0
        # for j in np.arange(phase_array.shape[1]):
        #     for i in np.arange(phase_array.shape[0]):
        #         # print(i*j,j,i)
        #         d[k,...] = (phase_array - phase_array[i,j]) % np.pi
        #         k+=1
        """
        return d

    def gamma(self,
              x: np.ndarray,
              beta: float = 1/4,
              r: float = 8/10
              ) -> np.ndarray:
        sign = -1 if r else 1  #togles between eqn 13 & 11 w/ presence of 2nd order term
        return -np.sin(x+beta) + r*np.sin(2*x)

    def plot_phase(self,
                   X: np.ndarray,
                   plot_title:str = 'placeholder',
                   y_axis:str = 'y',
                   x_axis:str = 'x',
                   ):
        """
        """
        fmt = setup(plot_title)  # plotting format obj
        fig = plt.figure(figsize=(9,9))
        ax = fig.add_subplot(111)
        ax.plot(X[...,0]/np.pi,X[...,1],'-b')
        ax.plot(np.asarray([X[0,0],X[-1,0]])/np.pi,[0,0],'-k',linewidth=1)  #

        plt.title(plot_title)
        plt.xlabel(x_axis)
        plt.ylabel(y_axis)

        ax.xaxis.set_major_formatter(mpl.ticker.FormatStrFormatter('%g $\pi$'))
        ax.xaxis.set_major_locator(mpl.ticker.MultipleLocator(base=1))
        plt.grid(b=True, which='major', axis='both')
        plt.show()
        fig.savefig(fmt.plot_name(plot_title,'png'))
        plt.close('all')

def main():
    """plot needs lib in this wd
    """
    a = interaction()
    params = ({'beta': 0.25, 'r':0.95},
              {'beta': 0.25, 'r':0.8},
              {'beta': 0.25, 'r':0.7},
              )
    x = np.linspace(-np.pi,np.pi,1000)

    for p in params:
        g = a.gamma(x,**p)
        a.plot_phase(np.asarray([x,g]).T,
                    'R = {0}'.format(p['r']),
                    '',
                    # r'$\frac{d\theta}{dt}$',
                    r'$\phi$')

if __name__ == '__main__':
    main()

max_stars_repo_name: chriswilly/kuramoto-osc

max_issues_repo_name: chriswilly/kuramoto-osc

max_forks_repo_name: chriswilly/kuramoto-osc


text:
#*******************************************************************#
#-------------------------------------------------------------------#
#---------------------------- AstroPix -----------------------------#
#-------------------------------------------------------------------#
#------------- Michela Negro - GSFC/UMBC - 2019/2020 ---------------#
#-------------------------------------------------------------------#
#*******************************************************************#

import os
import sys
import argparse
import numpy as np

__description__ = 'Creates the AstroPix geometry configurations'

formatter = argparse.ArgumentDefaultsHelpFormatter
PARSER = argparse.ArgumentParser(description=__description__, formatter_class=formatter)
PARSER.add_argument('-c', '--config', type=str, required=True,
                    help='the input configuration file')

if (sys.version_info > (3, 0)):
    from importlib.machinery import SourceFileLoader
    def get_var_from_file(filename):
        f = open(filename)
        global data
        data = SourceFileLoader('data', filename).load_module()
        f.close()
else:
    import imp
    def get_var_from_file(filename):
        f = open(filename)
        global data
        data = imp.load_source('data', '', f)
        f.close()

def run_mkGeometries(**kwargs):
	assert(kwargs['config'].endswith('.py'))
	get_var_from_file(kwargs['config'])

	passive = data.PASSIVE
	thickness = data.THICKNESS
	voxelsize = data.VOXELSIZE
	voxelnum = list((100./(np.array(voxelsize)/10)).astype('int'))
	geo_base = data.GEO_BASE
	det_base = data.DET_BASE

	line_thickness = 4
	line_passive = 5
	line_include_det = 9
	line_voxelnum = 1

	for i, p in enumerate(passive):
		print('PASSIVE %i: %i%%' %(i, p*100))
		for j, t in enumerate(thickness):
			print('THICKNESS %i: %.2f cm' %(j, t/10000))
			for k, v in enumerate(voxelnum):
				print('VOXEL SIZE %i: %.2f mm' %(k, voxelsize[k]))
				print('--> NUM. VOXELS: %ix%i' %(v, v))
				#___Create new folders and modify files___#
				folder_name = 'geo_%.2f_%i_%.2f'%(p, t, voxelsize[k])
				if not os.path.exists('geometry/%s' %folder_name):
					os.system('mkdir geometry/%s' %folder_name)
				geo_new = os.path.basename(geo_base).replace('base', '%.2f_%i_%.2f'%(p, t, voxelsize[k]))
				det_new = os.path.basename(det_base).replace('base', '%.2f_%i_%.2f'%(p, t, voxelsize[k]))
				os.system('cp %s geometry/%s/.' %(geo_base, folder_name))
				os.system('mv geometry/%s/%s geometry/%s/%s' \
					%(folder_name, os.path.basename(geo_base), folder_name, geo_new))
				geo_new_f = open('geometry/%s/%s'%(folder_name, geo_new), 'r')
				lines = geo_new_f.readlines()
				if 'Constant thickness' not in lines[line_thickness]:
					print('ATT: line number for the thickness parameter is not correct!')
					sys.exit()
				if 'Constant passive_thickness' not in lines[line_passive]:
					print('ATT: line number for the passive parameter is not correct!')
					sys.exit()
				if 'Include' not in lines[line_include_det] and '.det' not in lines[line_include_det]:
					print('ATT: line number  for the include is not correct!')
					sys.exit()
				line_list_t = lines[line_thickness].split()
				line_list_t[-1] = str(t/10000)+'\n'
				lines[line_thickness] = ' '.join(line_list_t)
				line_list_p = lines[line_passive].split()
				line_list_p[-1] = str(p)+'\n'
				lines[line_passive] = ' '.join(line_list_p)
				line_list_i = lines[line_include_det].split()
				line_list_i[-1] = det_new+'\n'
				lines[line_include_det] = ' '.join(line_list_i)
				with open('geometry/%s/%s'%(folder_name, geo_new), 'w') as f:
					for line in lines:
						f.write(line)
				geo_new_f.close()

				os.system('cp %s geometry/%s/.' %(det_base, folder_name))
				os.system('mv geometry/%s/%s geometry/%s/%s' \
					%(folder_name, os.path.basename(det_base), folder_name, det_new))
				det_new_f = open('geometry/%s/%s'%(folder_name, det_new), 'r')
				lines = det_new_f.readlines()
				if 'Constant voxel_num' not in lines[line_voxelnum]:
					print('ATT: line number for the voxel parameter is not correct!')
					sys.exit()
				line_list_v = lines[line_voxelnum].split()
				line_list_v[-1] = str(v)+'\n'
				lines[line_voxelnum] = ' '.join(line_list_v)
				with open('geometry/%s/%s'%(folder_name, det_new), 'w') as f:
					for line in lines:
						f.write(line)
				det_new_f.close()
		print('\n')

if __name__ == '__main__':
	args = PARSER.parse_args()

	print("--------------------")
	print("---- Start  Run ----")
	print("--------------------")
	print('\n')

	run_mkGeometries(**args.__dict__)

	print("--------------------")
	print("----- End  Run -----")
	print("--------------------")

max_stars_repo_name: AstroPix/Simulations

max_issues_repo_name: AstroPix/Simulations

max_forks_repo_name: AstroPix/Simulations


text:
from abc import ABCMeta, abstractmethod
from typing import Dict, Generic, List, TypeVar

import numpy as np

from expats.common.instantiate import BaseConfig, ConfigFactoried
from expats.common.type import ClassificationOutput, RegressionOutput

T = TypeVar("T")
U = TypeVar("U")

class Converter(Generic[T, U], metaclass=ABCMeta):
    """Converting some data type into another one.
    This is used for converting classification task's output (e.g discrete label) into regression's one (e.g float value)
    """
    @abstractmethod
    def convert(self, inputs: List[T]) -> List[U]:
        raise NotImplementedError()

class ClassificationToRegression(Converter[ClassificationOutput, RegressionOutput], ConfigFactoried):
    pass

class RegressionToClassification(Converter[RegressionOutput, ClassificationOutput], ConfigFactoried):
    pass

@ClassificationToRegression.register
class PredifinedNumerizer(ClassificationToRegression):
    def __init__(self, mapper: Dict[str, RegressionOutput]):
        self._mapper = mapper

    def convert(self, inputs: List[ClassificationOutput]) -> List[RegressionOutput]:
        return [self._convert(input_) for input_ in inputs]

    def _convert(self, input_):
        return self._mapper[input_]

@ClassificationToRegression.register
class ToFloat(ClassificationToRegression):
    def convert(self, inputs: List[ClassificationOutput]) -> List[RegressionOutput]:
        return [float(input_) for input_ in inputs]

@RegressionToClassification.register
class RoundNearestInteger(RegressionToClassification):
    def convert(self, inputs: List[RegressionOutput]) -> List[ClassificationOutput]:
        return [str(int(np.rint(input_))) for input_ in inputs]

@RegressionToClassification.register
class MinMaxDenormalizedRoundNearestInteger(RegressionToClassification):
    class _Config(BaseConfig):
        x_min: float
        x_max: float

    config_class = _Config

    def __init__(self, x_min: float, x_max: float):
        self._x_min = x_min
        self._x_max = x_max

    def convert(self, inputs: List[RegressionOutput]) -> List[ClassificationOutput]:
        return [
            str(int(np.rint(_min_max_denormalization(input_, self._x_min, self._x_max))))
            for input_ in inputs
        ]

def _min_max_denormalization(x_norm: float, x_min: float, x_max: float) -> float:
    if (x_norm > 1) or (x_norm < 0) or (x_min > x_max):
        raise ValueError(
            f"Invalid setting to normalize: x_norm={x_norm}, x_min={x_min}, x_max={x_max}")
    return x_norm * (x_max - x_min) + x_min

max_stars_repo_name: octanove/expats

max_issues_repo_name: octanove/expats

max_forks_repo_name: octanove/expats