text:
import os
import shutil
from toolz import pipe as p

import numpy as np
import pandas as pd

def makeControlDir(dest_dir, keep_actions = None, drop_actions = None, n_total_images = 200, replace=True):

    dest_dir_exists = os.path.exists(dest_dir)
    if dest_dir_exists and replace:
        shutil.rmtree(dest_dir)
        os.mkdir(dest_dir)
    elif not dest_dir_exists:
        os.mkdir(dest_dir)

    action_counts = _loadActionCounts(keep_actions, drop_actions, n_total_images)

    src_dir = 'stanford_40/JPEGImages'

    for c in action_counts.index:
        num_c = action_counts.loc[c, 'number_of_images']
        class_fs = np.random.choice(
            [f for f in os.listdir(src_dir) if c in f], num_c, replace = False)
        for f in class_fs:
            shutil.copy(os.path.join(src_dir, f), os.path.join(dest_dir, f))

def _loadActionCounts(keep_actions = None, drop_actions = None, n_total_images=200):
    if keep_actions is not None and drop_actions is not None:
        raise ValueError('can only chose keep actions or drop actions')

    f = "stanford_40/ImageSplits/actions.txt"
    action_counts = pd.read_csv(f, delim_whitespace=True, index_col = 0)

    actions = p(action_counts.index, set)

    if keep_actions is None and drop_actions is not None:
        drop_actions = drop_actions
    elif keep_actions is None and drop_actions is None:
        drop_actions = []
    else:
        keep_actions = [keep_actions] if type(keep_actions) is str else keep_actions
        drop_actions = actions - set(keep_actions)

    action_counts = action_counts.drop(drop_actions)
    action_counts['ratio'] = action_counts.number_of_images/sum(action_counts.number_of_images)
    action_counts['number_of_images_orig'] = action_counts.number_of_images
    action_counts['number_of_images'] = round(action_counts.ratio * n_total_images).astype(int)

    return action_counts


text:
from scipy.io import wavfile
import pyaudio, os, wave, time
import tqdm
import numpy as np
from scipy.fft import *
import datetime
from random import randint

note_freq = 0
timestamp = 0
rec = "false"

quotes = ['Music is the language of the spirit. It opens the secret of life bringing peace, abolishing strife. -Kahlil Gibran', 'Music is the only language in which you cannot say a mean or sarcastic thing. -John Erskine', 'Music is the moonlight in the gloomy night of life. -John Paul Friedrich Richter', 'If Music is a Place — then Jazz is the City, Folk is the Wilderness, Rock is the Road, Classical is a Temple -Vera Nazarin', 'Love is friendship set to music. -Jackson Pollock', 'Music is the universal language of mankind. -Henry Wadsworth Longfellow', 'Music is the literature of the heart; it commences where speech ends. -Alphonse de Lamartine', 'Music is the art which is most nigh to tears and memory. -Oscar Wilde', 'Music is nothing else but wild sounds civilized into time and tune. -Thomas Fuller', 'Music is your own experience, your thoughts, your wisdom. If you don’t live it, it won’t come out of your horn. -Charlie Parker', 'One good thing about music, when it hits you, you feel no pain. -Bob Marley', 'Where words fail, music speaks. -Hans Christian Anderson', 'How is it that music can, without words, evoke our laughter, our fears, our highest aspirations? -Jane Swan', 'Music expresses feeling and thought, without language; it was below and before speech, and it is above and beyond all words. -Robert G. Ingersoll', 'A strange art – music – the most poetic and precise of all the arts, vague as a dream and precise as algebra. -Guy de Maupassant', 'Music is good to the melancholy, bad to those who mourn, and neither good nor bad to the deaf. -Baruch Spinoza']

def cls():
    rand = randint(0, len(quotes))
    os.system('cls' if os.name=='nt' else 'clear')
    print('\x1B[3m 🎵 '+quotes[rand-1]+' 🎶 \x1B[0m \n')

def main_menu():
    global rec
    rand = randint(0, len(quotes))
    print('\x1B[3m 🎵 '+quotes[rand-1]+' 🎶 \x1B[0m')
    input("\r\nWelcome to Pythune, a Command-Line based piano tuner built in Python. Press any key to continue...")
    cls()
    print("\rPythune is listening for sound. Once it has detected a piano note, it will output tuning info for it.")
    cls()
    rec = "true"
    while rec == "true":
        record()
        time.sleep(2)

def file_length(name):
    global file_dur
    Fs, data = wavfile.read(name)
    n = data.size
    t = (n / Fs)*1000
    return t

# slightly modified from https://stackoverflow.com/questions/54612204/trying-to-get-the-frequencies-of-a-wav-file-in-python
def freq(file, start_time, end_time):
    global note_freq, timestamp
    # Open the file and convert to mono
    sr, data = wavfile.read(file)
    if data.ndim > 1:
        data = data[:, 0]
    else:
        pass
    # Return a slice of the data from start_time to end_time
    dataToRead = data[int(start_time * sr / 1000) : int(end_time * sr / 1000) + 1]
    # Fourier Transform
    N = len(dataToRead)
    yf = rfft(dataToRead)
    xf = rfftfreq(N, 1 / sr)
    # Get the most dominant frequency and return it
    idx = np.argmax(np.abs(yf))
    freq = xf[idx]
    return freq

# http://people.csail.mit.edu/hubert/pyaudio/
def record():
    CHUNK = 1024
    FORMAT = pyaudio.paInt16
    CHANNELS = 2
    RATE = 44100
    RECORD_SECONDS = 2
    WAVE_OUTPUT_FILENAME = "output.wav"

    p = pyaudio.PyAudio()

    stream = p.open(format=FORMAT,
    channels=CHANNELS,
    rate=RATE,
    input=True,
    frames_per_buffer=CHUNK)

    frames = []

    cls()
    for i in tqdm.tqdm(range(0, int(RATE / CHUNK * RECORD_SECONDS))):
        data = stream.read(CHUNK)
        frames.append(data)

    stream.stop_stream()
    stream.close()
    p.terminate()

    wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
    wf.setnchannels(CHANNELS)
    wf.setsampwidth(p.get_sample_size(FORMAT))
    wf.setframerate(RATE)
    wf.writeframes(b''.join(frames))
    wf.close()
    timestamp = (datetime.datetime.now().time())
    timestamp = str(timestamp).replace(':','').replace('.','')

    note_freq = freq('output.wav', 0, file_length('output.wav'))
    match_frequency(note_freq)
    os.rename('output.wav', 'cache/'+str(timestamp)+'.wav')

def closest(lst, K):
    return lst[min(range(len(lst)), key = lambda i: abs(lst[i]-K))]

def match_frequency(frequency):
    frequencies = [27.5000,29.1353,30.8677,32.7032,34.6479,36.7081,38.8909,41.2035,43.6536,46.2493,48.9995,51.9130,55.0000,58.2705,61.7354,65.4064,69.2957,73.4162,77.7817,82.4069,87.3071,92.4986,97.9989,103.826,110.000,116.541,123.471,130.813,138.591,146.832,155.563,164.814,174.614,184.997,195.998,207.652,220.000,233.082,246.942,261.626,277.183,293.665,311.127,329.628,349.228,369.994,391.995,415.305,440.000,466.164,493.883,523.251,554.365,587.330,622.254,659.255,698.456,739.989,783.991,830.609,880.000,932.328,987.767,1046.50,1108.73,1174.66,1244.51,1318.51,1396.91,1479.98,1567.98,1661.22,1760.00,1864.66,1975.53,2093.00,2217.46,2349.32,2489.02,2637.02,2793.83,2959.96,3135.96,3322.44,3520.00,3729.31,3951.07, 4186.01]
    names = ['A0', 'A#0/Bb0', 'B0', 'C1', 'C#1/Db1', 'D1', 'D#1/Eb1', 'E1', 'F1', 'F#1/Gb1', 'G1', 'G#1/Ab1', 'A1', 'A#1/Bb1', 'B1', 'C2', 'C#2/Db2', 'D2', 'D#2/Eb2', 'E2', 'F2', 'F#2/Gb2', 'G2', 'G#2/Ab2', 'A2', 'A#2/Bb2', 'B2', 'C3', 'C#3/Db3', 'D3', 'D#3/Eb3', 'E3', 'F3', 'F#3/Gb3', 'G3', 'G#3/Ab3', 'A3', 'A#3/Bb3', 'B3', 'C4', 'C#4/Db4', 'D4', 'D#4/Eb4', 'E4', 'F4', 'F#4/Gb4', 'G4', 'G#4/Ab4', 'A4', 'A#4/Bb4', 'B4', 'C5', 'C#5/Db5', 'D5', 'D#5/Eb5', 'E5', 'F5', 'F#5/Gb5', 'G5', 'G#5/Ab5', 'A5', 'A#5/Bb5', 'B5', 'C6', 'C#6/Db6', 'D6', 'D#6/Eb6', 'E6', 'F6', 'F#6/Gb6', 'G6', 'G#6/Ab6', 'A6', 'A#6/Bb6', 'B6', 'C7', 'C#7/Db7', 'D7', 'D#7/Eb7', 'E7', 'F7', 'F#7/Gb7', 'G7', 'G#7/Ab7', 'A7', 'A#7/Bb7', 'B7', 'C8']
    closestNum = closest(frequencies, frequency)
    index = frequencies.index(closestNum)
    print("🎹 The frequency was: "+str(frequency)+", which is closest to the note: "+str(names[index])+", with: "+str(closestNum-frequency)+" to go until perfect pitch 🎹")

main_menu()

ext: py

reason: import numpy,from scipy


text:
# http://github.com/timestocome

# Test run to see if I can do some software simulations
# then reproduce them in hardware

# Braitenberg Vehicle Simulation
# https://mitpress.mit.edu/books/vehicles
# Elements of Robotics, Springer 2018

# input is distance measure from sound echo, front, rear, left, right
# right and left wheels can turn individually

import pygame
import sys
import numpy as np
import time
import math

# environment
white = (255, 255, 255)
green = (0, 255, 0)
red = (255, 0, 0)
blue = (0, 0, 255)
purple = (200, 0, 200)

size = width, height = 1000, 1000
screen = pygame.display.set_mode(size)
screen.fill(white)

n_love_cars = 6
n_explorer_cars = 6
n_coward_cars = 6
n_aggressive_cars = 6

car_count = 0
cars = []

n_trees = 7    # 7 possible image files
total_trees = 21
tree_count = 0
trees = []

# HC SR204 object finder
sensor_max = 400    # 400 cm
sensor_angle = 15   # 15'

# wheel speed
speed = 5

##########################################################
# utility functions
##########################################################

def degrees_to_radians(d):
    return d / 57.3

def radians_to_degrees(r):
    return r * 57.3

def distance(x1, y1, x2, y2):
    return np.sqrt( (x1-x2)**2 + (y1-y2)**2 )

#########################################################
# sprites
#########################################################

# add obstacles
class Tree():

    def __init__(self):

        count = tree_count % n_trees + 1

        # load image
        self.image_file = 'tree_' + str(count) + '.png'
        self.image = pygame.image.load(self.image_file)
        self.rect = self.image.get_rect()
        self.copy = self.image

        # plant in random location
        self.x = np.random.randint(width)
        self.y = np.random.randint(height)

        # move to location
        self.rect = self.rect.move((self.x, self.y))
        self.rect = self.image.get_rect(center=self.rect.center)

class Car_Love():

    def __init__(self):

        # images
        image_file = 'car_1.png'
        self.image = pygame.image.load(image_file)
        self.rect = self.image.get_rect()
        self.copy = self.image

        # random starting point
        degrees = np.random.randint(360)
        self.angle_rads = degrees_to_radians(degrees)
        self.x = np.random.randint(width)
        self.y = np.random.randint(height)

        # move to starting location
        self.rect = self.rect.move((self.x, self.y))
        self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))
        self.rect = self.image.get_rect(center=self.rect.center)

    def line_of_sight(self, x_obstacle, y_obstacle, direction):

        # check front only
        dr = np.sqrt((x_obstacle - self.x)**2 + (y_obstacle - self.y)**2)
        dx = x_obstacle - self.x
        dy = y_obstacle - self.y

        # if object in sensor range
        if dr > 0 and dr < sensor_max:

            # compute angle
            r_view_angle = (direction - 183) % 360  # game board has rotated compass
            l_view_angle = (direction - 177) % 360
            view_angle = (direction - 180) % 360

            # look left
            xl = sensor_max * np.sin(degrees_to_radians(l_view_angle)) + self.x
            yl = sensor_max * np.cos(degrees_to_radians(l_view_angle)) + self.y
            rl = np.sqrt( (xl - self.x)**2 + (yl - self.y)**2 )
            pygame.draw.line(screen, green, (self.x, self.y), (xl, yl))

            if (dr * rl) > 0:
                dot_l = dx * (xl - self.x) + dy * (yl - self.y)
                angle_l = radians_to_degrees(math.acos(dot_l / (dr * rl)))

            # look right
            xr = sensor_max * np.sin(degrees_to_radians(r_view_angle)) + self.x
            yr = sensor_max * np.cos(degrees_to_radians(r_view_angle)) + self.y
            rr = np.sqrt( (xr - self.x)**2 + (yr - self.y)**2)
            pygame.draw.line(screen, blue, (self.x, self.y), (xr, yr))

            if (dr * rr) > 0:
                dot_r = dx * (xr - self.x) + dy * (yr - self.y)
                angle_r = radians_to_degrees(math.acos(dot_r / (dr * rr)))

            # point sensor_max and sensor angle away from car
            sensor_x = sensor_max * np.sin(degrees_to_radians(view_angle)) + self.x
            sensor_y = sensor_max * np.cos(degrees_to_radians(view_angle)) + self.y
            sensor_r = np.sqrt( (sensor_x - self.x)**2 + (sensor_y - self.y)**2 )

            pygame.draw.line(screen, red, (self.x, self.y), (sensor_x, sensor_y))

            # angle between line of sight and object
            if dr > 0:
                dot = dx * (sensor_x - self.x) + dy * (sensor_y - self.y)
                angle = radians_to_degrees( math.acos( dot / (dr * sensor_r)) )

            # check if angle in sensor range
            if angle < sensor_angle:
                if angle_l < angle_r:
                    return (1, 1)
                else:
                    return (1, -1)

        return (0,0)

    def move(self):

        # update location
        self.x = self.rect.center[0]
        self.y = self.rect.center[1]

        # check sensor
        tripped_sensor = 0
        object_angle = 0

        for t in trees:
            tx = t.rect.center[0]
            ty = t.rect.center[1]

            object_detected, angle = self.line_of_sight(tx, ty, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        for c in cars:
            cx = c.rect.center[0]
            cy = c.rect.center[1]

            object_detected, angle = self.line_of_sight(cx, cy, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        # something is in our view range
        if object_angle < 0:
            self.angle_rads -= 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if object_angle > 0:
            self.angle_rads += 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if tripped_sensor > 0:

            vx = -speed * np.sin(self.angle_rads)
            vy = -speed * np.cos(self.angle_rads)

            self.rect = self.rect.move(vx, vy)
            self.rect = self.image.get_rect(center=self.rect.center)

        # edge detection, wrapping
        x = self.rect.center[0]
        y = self.rect.center[1]

        if self.rect.center[0] < 0: x = width + self.rect.center[0]
        if self.rect.center[0] > width: x = self.rect.center[0] -  width
        if self.rect.center[1] < 0: y = height + self.rect.center[1]
        if self.rect.center[1] > height: y = self.rect.center[1] - height
        self.rect.center = (x, y)

class Car_Explorer():

    def __init__(self):

        # images
        image_file = 'car_2.png'
        self.image = pygame.image.load(image_file)
        self.rect = self.image.get_rect()
        self.copy = self.image

        # random starting point
        degrees = np.random.randint(360)
        self.angle_rads = degrees_to_radians(degrees)
        self.x = np.random.randint(width)
        self.y = np.random.randint(height)

        # move to starting location
        self.rect = self.rect.move((self.x, self.y))
        self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))
        self.rect = self.image.get_rect(center=self.rect.center)

    def line_of_sight(self, x_obstacle, y_obstacle, direction):

        # check front only
        dr = np.sqrt((x_obstacle - self.x)**2 + (y_obstacle - self.y)**2)
        dx = x_obstacle - self.x
        dy = y_obstacle - self.y

        # if object in sensor range
        if dr > 0 and dr < sensor_max:

            # compute angle
            r_view_angle = (direction - 183) % 360  # game board has rotated compass
            l_view_angle = (direction - 177) % 360
            view_angle = (direction - 180) % 360

            # look left
            xl = sensor_max * np.sin(degrees_to_radians(l_view_angle)) + self.x
            yl = sensor_max * np.cos(degrees_to_radians(l_view_angle)) + self.y
            rl = np.sqrt( (xl - self.x)**2 + (yl - self.y)**2 )
            pygame.draw.line(screen, green, (self.x, self.y), (xl, yl))

            if (dr * rl) > 0:
                dot_l = dx * (xl - self.x) + dy * (yl - self.y)
                angle_l = radians_to_degrees(math.acos(dot_l / (dr * rl)))

            # look right
            xr = sensor_max * np.sin(degrees_to_radians(r_view_angle)) + self.x
            yr = sensor_max * np.cos(degrees_to_radians(r_view_angle)) + self.y
            rr = np.sqrt( (xr - self.x)**2 + (yr - self.y)**2)
            pygame.draw.line(screen, blue, (self.x, self.y), (xr, yr))

            if (dr * rr) > 0:
                dot_r = dx * (xr - self.x) + dy * (yr - self.y)
                angle_r = radians_to_degrees(math.acos(dot_r / (dr * rr)))

            # point sensor_max and sensor angle away from car
            sensor_x = sensor_max * np.sin(degrees_to_radians(view_angle)) + self.x
            sensor_y = sensor_max * np.cos(degrees_to_radians(view_angle)) + self.y
            sensor_r = np.sqrt( (sensor_x - self.x)**2 + (sensor_y - self.y)**2 )

            pygame.draw.line(screen, red, (self.x, self.y), (sensor_x, sensor_y))

            # angle between line of sight and object
            if dr > 0:
                dot = dx * (sensor_x - self.x) + dy * (sensor_y - self.y)
                angle = radians_to_degrees( math.acos( dot / (dr * sensor_r)) )

            # check if angle in sensor range
            if angle < sensor_angle:
                if angle_l < angle_r:
                    return (1, 1)
                else:
                    return (1, -1)

        return (0,0)

    def move(self):

        # update location
        self.x = self.rect.center[0]
        self.y = self.rect.center[1]

        # check sensor
        tripped_sensor = 0
        object_angle = 0

        for t in trees:
            tx = t.rect.center[0]
            ty = t.rect.center[1]

            object_detected, angle = self.line_of_sight(tx, ty, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        for c in cars:
            cx = c.rect.center[0]
            cy = c.rect.center[1]

            object_detected, angle = self.line_of_sight(cx, cy, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        # something is in our view range
        if object_angle < 0:
            self.angle_rads += 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if object_angle > 0:
            self.angle_rads -= 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if tripped_sensor > 0:

            vx = -speed * np.sin(self.angle_rads)
            vy = -speed * np.cos(self.angle_rads)

            self.rect = self.rect.move(vx, vy)
            self.rect = self.image.get_rect(center=self.rect.center)

        # edge detection, wrapping
        x = self.rect.center[0]
        y = self.rect.center[1]

        if self.rect.center[0] < 0: x = width + self.rect.center[0]
        if self.rect.center[0] > width: x = self.rect.center[0] -  width
        if self.rect.center[1] < 0: y = height + self.rect.center[1]
        if self.rect.center[1] > height: y = self.rect.center[1] - height
        self.rect.center = (x, y)

class Car_Coward():

    def __init__(self):

        # images
        image_file = 'car_3.png'
        self.image = pygame.image.load(image_file)
        self.rect = self.image.get_rect()
        self.copy = self.image

        # random starting point
        degrees = np.random.randint(360)
        self.angle_rads = degrees_to_radians(degrees)
        self.x = np.random.randint(width)
        self.y = np.random.randint(height)

        # move to starting location
        self.rect = self.rect.move((self.x, self.y))
        self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))
        self.rect = self.image.get_rect(center=self.rect.center)

    def line_of_sight(self, x_obstacle, y_obstacle, direction):

        # check front only
        dr = np.sqrt((x_obstacle - self.x)**2 + (y_obstacle - self.y)**2)
        dx = x_obstacle - self.x
        dy = y_obstacle - self.y

        # if object in sensor range
        if dr > 0 and dr < sensor_max:

            # compute angle
            r_view_angle = (direction - 183) % 360  # game board has rotated compass
            l_view_angle = (direction - 177) % 360
            view_angle = (direction - 180) % 360

            # look left
            xl = sensor_max * np.sin(degrees_to_radians(l_view_angle)) + self.x
            yl = sensor_max * np.cos(degrees_to_radians(l_view_angle)) + self.y
            rl = np.sqrt( (xl - self.x)**2 + (yl - self.y)**2 )
            pygame.draw.line(screen, green, (self.x, self.y), (xl, yl))

            if (dr * rl) > 0:
                dot_l = dx * (xl - self.x) + dy * (yl - self.y)
                angle_l = radians_to_degrees(math.acos(dot_l / (dr * rl)))

            # look right
            xr = sensor_max * np.sin(degrees_to_radians(r_view_angle)) + self.x
            yr = sensor_max * np.cos(degrees_to_radians(r_view_angle)) + self.y
            rr = np.sqrt( (xr - self.x)**2 + (yr - self.y)**2)
            pygame.draw.line(screen, blue, (self.x, self.y), (xr, yr))

            if (dr * rr) > 0:
                dot_r = dx * (xr - self.x) + dy * (yr - self.y)
                angle_r = radians_to_degrees(math.acos(dot_r / (dr * rr)))

            # point sensor_max and sensor angle away from car
            sensor_x = sensor_max * np.sin(degrees_to_radians(view_angle)) + self.x
            sensor_y = sensor_max * np.cos(degrees_to_radians(view_angle)) + self.y
            sensor_r = np.sqrt( (sensor_x - self.x)**2 + (sensor_y - self.y)**2 )

            pygame.draw.line(screen, red, (self.x, self.y), (sensor_x, sensor_y))

            # angle between line of sight and object
            if dr > 0:
                dot = dx * (sensor_x - self.x) + dy * (sensor_y - self.y)
                angle = radians_to_degrees( math.acos( dot / (dr * sensor_r)) )

            # check if angle in sensor range
            if angle < sensor_angle:
                if angle_r < angle_r:   # swap angles
                    return (1, 1)
                else:
                    return (1, -1)

        return (0,0)

    def move(self):

        # update location
        self.x = self.rect.center[0]
        self.y = self.rect.center[1]

        # check sensor
        tripped_sensor = 0
        object_angle = 0

        for t in trees:
            tx = t.rect.center[0]
            ty = t.rect.center[1]

            object_detected, angle = self.line_of_sight(tx, ty, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        for c in cars:
            cx = c.rect.center[0]
            cy = c.rect.center[1]

            object_detected, angle = self.line_of_sight(cx, cy, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        # something is in our view range
        if object_angle < 0:
            self.angle_rads += 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if object_angle > 0:
            self.angle_rads -= 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if tripped_sensor > 0:

            vx = -speed * np.sin(self.angle_rads)
            vy = -speed * np.cos(self.angle_rads)

            self.rect = self.rect.move(vx, vy)
            self.rect = self.image.get_rect(center=self.rect.center)

        # edge detection, wrapping
        x = self.rect.center[0]
        y = self.rect.center[1]

        if self.rect.center[0] < 0: x = width + self.rect.center[0]
        if self.rect.center[0] > width: x = self.rect.center[0] -  width
        if self.rect.center[1] < 0: y = height + self.rect.center[1]
        if self.rect.center[1] > height: y = self.rect.center[1] - height
        self.rect.center = (x, y)

class Car_Aggressive():

    def __init__(self):

        # images
        image_file = 'car_4.png'
        self.image = pygame.image.load(image_file)
        self.rect = self.image.get_rect()
        self.copy = self.image

        # random starting point
        degrees = np.random.randint(360)
        self.angle_rads = degrees_to_radians(degrees)
        self.x = np.random.randint(width)
        self.y = np.random.randint(height)

        # move to starting location
        self.rect = self.rect.move((self.x, self.y))
        self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))
        self.rect = self.image.get_rect(center=self.rect.center)

    def line_of_sight(self, x_obstacle, y_obstacle, direction):

        # check front only
        dr = np.sqrt((x_obstacle - self.x)**2 + (y_obstacle - self.y)**2)
        dx = x_obstacle - self.x
        dy = y_obstacle - self.y

        # if object in sensor range
        if dr > 0 and dr < sensor_max:

            # compute angle
            r_view_angle = (direction - 183) % 360  # game board has rotated compass
            l_view_angle = (direction - 177) % 360
            view_angle = (direction - 180) % 360

            # look left
            xl = sensor_max * np.sin(degrees_to_radians(l_view_angle)) + self.x
            yl = sensor_max * np.cos(degrees_to_radians(l_view_angle)) + self.y
            rl = np.sqrt( (xl - self.x)**2 + (yl - self.y)**2 )
            pygame.draw.line(screen, green, (self.x, self.y), (xl, yl))

            if (dr * rl) > 0:
                dot_l = dx * (xl - self.x) + dy * (yl - self.y)
                angle_l = radians_to_degrees(math.acos(dot_l / (dr * rl)))

            # look right
            xr = sensor_max * np.sin(degrees_to_radians(r_view_angle)) + self.x
            yr = sensor_max * np.cos(degrees_to_radians(r_view_angle)) + self.y
            rr = np.sqrt( (xr - self.x)**2 + (yr - self.y)**2)
            pygame.draw.line(screen, blue, (self.x, self.y), (xr, yr))

            if (dr * rr) > 0:
                dot_r = dx * (xr - self.x) + dy * (yr - self.y)
                angle_r = radians_to_degrees(math.acos(dot_r / (dr * rr)))

            # point sensor_max and sensor angle away from car
            sensor_x = sensor_max * np.sin(degrees_to_radians(view_angle)) + self.x
            sensor_y = sensor_max * np.cos(degrees_to_radians(view_angle)) + self.y
            sensor_r = np.sqrt( (sensor_x - self.x)**2 + (sensor_y - self.y)**2 )

            pygame.draw.line(screen, red, (self.x, self.y), (sensor_x, sensor_y))

            # angle between line of sight and object
            if dr > 0:
                dot = dx * (sensor_x - self.x) + dy * (sensor_y - self.y)
                angle = radians_to_degrees( math.acos( dot / (dr * sensor_r)) )

            # check if angle in sensor range
            if angle < sensor_angle:
                if angle_l > angle_r:    # swap angles
                    return (1, 1)
                else:
                    return (1, -1)

        return (0,0)

    def move(self):

        # update location
        self.x = self.rect.center[0]
        self.y = self.rect.center[1]

        # check sensor
        tripped_sensor = 0
        object_angle = 0

        for t in trees:
            tx = t.rect.center[0]
            ty = t.rect.center[1]

            object_detected, angle = self.line_of_sight(tx, ty, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        for c in cars:
            cx = c.rect.center[0]
            cy = c.rect.center[1]

            object_detected, angle = self.line_of_sight(cx, cy, radians_to_degrees(self.angle_rads))
            tripped_sensor += object_detected
            object_angle += angle

        # something is in our view range
        if object_angle < 0:
            self.angle_rads -= 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if object_angle > 0:
            self.angle_rads += 1/10
            self.image = pygame.transform.rotate(self.copy, radians_to_degrees(self.angle_rads))

        if tripped_sensor > 0:

            vx = -speed * np.sin(self.angle_rads)
            vy = -speed * np.cos(self.angle_rads)

            self.rect = self.rect.move(vx, vy)
            self.rect = self.image.get_rect(center=self.rect.center)

        # edge detection, wrapping
        x = self.rect.center[0]
        y = self.rect.center[1]

        if self.rect.center[0] < 0: x = width + self.rect.center[0]
        if self.rect.center[0] > width: x = self.rect.center[0] -  width
        if self.rect.center[1] < 0: y = height + self.rect.center[1]
        if self.rect.center[1] > height: y = self.rect.center[1] - height
        self.rect.center = (x, y)

# set up
for i in range(n_love_cars):
    new_car = Car_Love()
    car_count += 1
    cars.append(new_car)

for i in range(n_explorer_cars):
    new_car = Car_Explorer()
    car_count += 1
    cars.append(new_car)

for i in range(n_coward_cars):
    new_car = Car_Coward()
    car_count += 1
    cars.append(new_car)

for i in range(n_aggressive_cars):
    new_car = Car_Aggressive()
    car_count += 1
    cars.append(new_car)

for i in range(total_trees):
    new_tree = Tree()
    tree_count += 1
    trees.append(new_tree)

# main loop
while(True):

    for event in pygame.event.get():
        if event.type == pygame.QUIT: sys.exit()

    screen.fill(white)

    for v in cars:
        car = v
        car.move()
        screen.blit(car.image, car.rect)

    for t in trees:
        tree = t
        screen.blit(tree.image, tree.rect)

    pygame.display.flip()

max_stars_repo_name: timestocome/EvolvingCode

max_issues_repo_name: timestocome/EvolvingCode

max_forks_repo_name: timestocome/EvolvingCode

reason: import numpy


text:
import torch
import numpy as np
import scipy.linalg

from model.structure.box3d_list import Box3dList
from model.dataset.model_util_rscan import RScanDatasetConfig
from model.modeling.detector.utils.box_util import box3d_iou_depth, box3d_vol
from model.dataset.rscan_utils import flip_axis_to_camera, flip_axis_to_depth

from my_utils.my_nms import _box_nms
DC = RScanDatasetConfig()

def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="pred_score"):
    if nms_thresh <= 0:
        return boxlist
    boxes = boxlist.bbox
    score = boxlist.get_field(score_field)
    keep = _box_nms(boxes, score, nms_thresh)
    if max_proposals > 0:
        keep = keep[: max_proposals]
    boxes = boxes[keep]
    return boxes, keep

def remove_small_boxes(boxlist, min_size):
    keep = np.argwhere((boxlist.area(DC) >= min_size) > 0)
    boxlist.clip(keep)
    return boxlist

def boxlist_iou(boxlist1, boxlist2):
    """
    Arguments:
      box1: (box3d_list) bounding boxes
      box2: (box3d_list) bounding boxes

    Returns:
      (tensor) iou, sized [N,M].
      (tensor) bird's eye view 2D bounding box IoU, sized [N,M]

    """
    N = len(boxlist1)
    M = len(boxlist2)

    boxes1 = boxlist1.bbox.detach().cpu().numpy() # (N,8,3)
    boxes2 = boxlist2.bbox.detach().cpu().numpy()  # (M,8,3)

    # boxes1 = flip_axis_to_camera(boxes1.reshape(-1, 3)).reshape(-1, 8, 3)
    # boxes2 = flip_axis_to_camera(boxes2.reshape(-1, 3)).reshape(-1, 8, 3)

    iou = np.zeros(shape=[N, M], dtype=np.float)
    iou2d = np.zeros(shape=[N, M], dtype=np.float)

    for n, box1 in enumerate(boxes1):
        for m, box2 in enumerate(boxes2):
            iou[n, m], iou2d[n, m] = box3d_iou_depth(box1, box2)

    iou = torch.as_tensor(iou).cuda()
    iou2d = torch.as_tensor(iou2d).cuda()

    return iou, iou2d

def boxlist_iou_tensor(boxlist1, boxlist2):

    N = boxlist1.shape[0]
    M = boxlist2.shape[0]

    boxes1 = boxlist1.detach().cpu().numpy()
    boxes2 = boxlist2.detach().cpu().numpy()

    # boxes1 = flip_axis_to_camera(boxes1.reshape(-1, 3)).reshape(-1, 8, 3)
    # boxes2 = flip_axis_to_camera(boxes2.reshape(-1, 3)).reshape(-1, 8, 3)

    iou = np.zeros(shape=[N, M], dtype=np.float)
    iou2d = np.zeros(shape=[N, M], dtype=np.float)

    for n, box1 in enumerate(boxes1):
        for m, box2 in enumerate(boxes2):
            iou[n, m], iou2d[n, m] = box3d_iou_depth(box1, box2)

    iou = torch.as_tensor(iou).to(boxlist1.device)
    iou2d = torch.as_tensor(iou2d).to(boxlist1.device)

    return iou, iou2d

def boxlist_iou_tensor_faster(boxlist1, boxlist2):
    N = boxlist1.shape[0]
    M = boxlist2.shape[0]

    # boxes1 = flip_axis_to_camera(boxes1.reshape(-1, 3)).reshape(-1, 8, 3)
    # boxes2 = flip_axis_to_camera(boxes2.reshape(-1, 3)).reshape(-1, 8, 3)

    iou = torch.zeros((N, M), dtype=torch.float64).to(boxlist1.device)
    iou2d = torch.zeros((N, M), dtype=torch.float64).to(boxlist1.device)

    for n, box1 in enumerate(boxlist1):
        for m, box2 in enumerate(boxlist2):
            iou[n, m], iou2d[n, m] = box3d_iou_simple(box1, box2)

    return iou, iou2d

def box3d_iou_simple(box1, box2):
    x1_1 = torch.min(box1[:, 0])
    x1_2 = torch.min(box2[:, 0])
    y1_1 = torch.min(box1[:, 1])
    y1_2 = torch.min(box2[:, 1])
    z1_1 = torch.min(box1[:, 2])
    z1_2 = torch.min(box2[:, 2])
    x2_1 = torch.max(box1[:, 0])
    x2_2 = torch.max(box2[:, 0])
    y2_1 = torch.max(box1[:, 1])
    y2_2 = torch.max(box2[:, 1])
    z2_1 = torch.max(box1[:, 2])
    z2_2 = torch.max(box2[:, 2])

    xx1 = torch.max(torch.as_tensor([x1_1, x1_2]))
    yy1 = torch.max(torch.as_tensor([y1_1, y1_2]))
    zz1 = torch.max(torch.as_tensor([z1_1, z1_2]))
    xx2 = torch.min(torch.as_tensor([x2_1, x2_2]))
    yy2 = torch.min(torch.as_tensor([y2_1, y2_2]))
    zz2 = torch.min(torch.as_tensor([z2_1, z2_2]))

    l = np.maximum(0, xx2-xx1)
    w = np.maximum(0, yy2-yy1)
    h = np.maximum(0, zz2-zz1)

    overlap = l*w*h
    overlap2d = l*w
    area1 = (x2_1 - x1_1) * (y2_1 - y1_1) * (z2_1 - z1_1)
    area2 = (x2_2 - x1_2) * (y2_2 - y1_2) * (z2_2 - z1_2)
    area2d_1 = (x2_1 - x1_1) * (y2_1 - y1_1)
    area2d_2 = (x2_2 - x1_2) * (y2_2 - y1_2)
    return overlap/(area1+area2-overlap), overlap2d/(area2d_1 + area2d_2 - overlap2d)

reason: import numpy,import scipy


text:
import numpy as np
import math
import collections
import logging
from src.handlefiles import load_objects
from src.nanoporereads import *

class HMM:
    """
    A hidden markov model.
    Find the haplotype for SNPS based on Nanopore sequencing READS.
    """

    def __init__(self, snp_data, states, ob=None):
        self.SNPs = snp_data
        self.STATEs = states
        if ob is None:
            ob = ['A', 'T', 'G', 'C']
        self.observations = ob
        self.emission_probs = np.zeros((len(self.SNPs), len(self.STATEs), len(self.observations)))

        # model results
        self.read_assignments = {"m1": [], "m2": []}
        self.alleles = {"m1": {}, "m2": {}}
        self.haplotypes = {"m1": {}, "m2": {}}

    def init_emission_matrix(self):
        """
        Initialize emission probabilities of observing <A, T, G, C> bases on given SNP position.
        """
        for index, snp in enumerate(self.SNPs):
            emission_prob = [10, 10, 10, 10]
            emission_prob[self.observations.index(snp.ref)] = 40
            emission_prob[self.observations.index(snp.alt)] = 40
            self.emission_probs[index, 0, ] = np.random.dirichlet(emission_prob)
            self.emission_probs[index, 1, ] = np.random.dirichlet(emission_prob)
        return self.emission_probs

    def read_log_likelihood(self, state, read):
        """
        Calculate the log value of probability of a snp observed in a model.
        :param state: one of the model, int 0 or 1.
        :param read: (NanoporeRead) snp
        :return sum (log value of likelihood of a read base observed on a SNP position)

        base_llhd = t[read.snps_id[index], state, self.observations.index(read.get_base(snp.pos))]
        """
        read_llhd = 0
        for index, snp in enumerate(read.snps):
            t = self.get_emission()
            base_llhd = t[snp.id, state, self.observations.index(read.bases[snp.id])]
            if base_llhd < 0:
                logging.error("base emission prob is negative.")
            read_llhd += math.log(base_llhd)
        return read_llhd

    def assign_reads(self, Reads):
        """Calculate the parental-maternal likelihood of the READS.
        Expectation step in EM algorithm.
        SR_DICT
        Args:
            Reads: list of READS.
            :rtype: ndarray pm_llhd
        """
        pm_llhd = np.zeros((len(Reads), 2), dtype=float)
        self.read_assignments = {"m1": [], "m2": []}
        self.alleles = {"m1": {}, "m2": {}}
        for idx, read in enumerate(Reads):
            pm_llhd[idx, 0] = self.read_log_likelihood(0, read)
            pm_llhd[idx, 1] = self.read_log_likelihood(1, read)
            if pm_llhd[idx, 0] > pm_llhd[idx, 1]:
                self.read_assignments["m1"].append(idx)
                for snp_id in read.snps_id:
                    if snp_id in self.alleles["m1"]:
                        self.alleles["m1"][snp_id].append(read.bases[snp_id])
                    else:
                        self.alleles["m1"][snp_id] = [read.bases[snp_id]]
            elif pm_llhd[idx, 0] < pm_llhd[idx, 1]:
                self.read_assignments["m2"].append(idx)
                for snp_id in read.snps_id:
                    if snp_id in self.alleles["m2"]:
                        self.alleles["m2"][snp_id].append(read.bases[snp_id])
                    else:
                        self.alleles["m2"][snp_id] = [read.bases[snp_id]]
        return pm_llhd

    def update(self, Reads, sr_dict, pm_llhd, hard=False, pseudo_base=1e-1):
        """Update the probability matrix

        Args:
            Reads (List): List of READS.
            sr_dict(Dictionary): Dictionary of snp snp map.
            pm_llhd (Float): Matrix of shape [Read_num,2].
            hard (Boolean): If update in hard manner.
            pseudo_base(Float): Probability normalized by adding this pseudo_base,
                e.diffs. p[_] = (count[_]+pseudo_base)/(count[_]+4*pseudo_base).

        hard: count[self.observations.index(read.get_base(snp.pos)), pm_type] += 1
        fuzzy: count[self.observations.index(read.get_base(snp.pos)), 0] += (pm_llhd[read_id, 0] * 1)
        """
        for snp_id in sr_dict.keys():
            count = np.zeros((len(self.observations), 2))
            count[:] = pseudo_base
            for read_id in sr_dict[snp_id]:
                read = Reads[read_id]
                if hard:
                    pm_type = np.argmax(pm_llhd[read_id, :])
                    count[self.observations.index(read.bases[snp_id]), pm_type] += 1
                else:
                    count[self.observations.index(read.bases[snp_id]), 0] += np.exp(pm_llhd[read_id, 0])
                    count[self.observations.index(read.bases[snp_id]), 1] += np.exp(pm_llhd[read_id, 1])
            for state in range(len(self.STATEs)):
                self.emission_probs[snp_id, state, :] = (count[:, state] / np.sum(count[:, state]))

    def partly_update(self, ratio, Reads, sr_dict, pm_llhd, pseudo_base=1e-1):
        """Only consider a part of SNPS site in a read.
        soft update manner"""
        snps = random_choose(list(sr_dict.keys()), ratio)
        for snp_id in snps:
            snp = self.SNPs[snp_id]
            count = np.zeros((len(self.observations), 2))
            count[:] = pseudo_base
            for read_id in sr_dict[snp_id]:
                read = Reads[read_id]
                count[self.observations.index(read.bases[snp.id]), 0] += np.exp(pm_llhd[read_id, 0])
                count[self.observations.index(read.bases[snp.id]), 1] += np.exp(pm_llhd[read_id, 1])
            for state in range(len(self.STATEs)):
                self.emission_probs[snp_id, state, :] = (count[:, state] / np.sum(count[:, state]))

    def cluster_reads(self, Reads):
        """Use current emission prob matrix to cluster any input Reads."""
        pm_llhd = np.zeros((len(Reads), 2), dtype=float)
        read_assignments = {"m1": [], "m2": []}
        alleles = {"m1": {}, "m2": {}}
        for idx, read in enumerate(Reads):
            pm_llhd[idx, 0] = self.read_log_likelihood(0, read)
            pm_llhd[idx, 1] = self.read_log_likelihood(1, read)
            if pm_llhd[idx, 0] > pm_llhd[idx, 1]:
                read_assignments["m1"].append(idx)
                for snp_id in read.snps_id:
                    if snp_id in alleles["m1"]:
                        alleles["m1"][snp_id].append(read.bases[snp_id])
                    else:
                        alleles["m1"][snp_id] = [read.bases[snp_id]]
            elif pm_llhd[idx, 0] < pm_llhd[idx, 1]:
                read_assignments["m2"].append(idx)
                for snp_id in read.snps_id:
                    if snp_id in alleles["m2"]:
                        alleles["m2"][snp_id].append(read.bases[snp_id])
                    else:
                        alleles["m2"][snp_id] = [read.bases[snp_id]]
        h = self.get_hs(alleles)
        return h

    def get_alleles(self):
        """Return a dict."""
        return self.alleles

    def get_ordered_alleles(self):
        """Return a sorted (by allele loci) collection.OrderedDict dictionary."""
        m1a = collections.OrderedDict(sorted(self.alleles["m1"].items()))
        m2a = collections.OrderedDict(sorted(self.alleles["m2"].items()))
        return {"m1": m1a, "m2": m2a}

    def get_haplotypes(self):
        """When there is a tie, choose base has higher emission prob.
        Or use both in turn."""
        for key in self.alleles:
            for snp_id in self.alleles[key]:
                base = most_common(self.alleles[key][snp_id])
                self.haplotypes[key][snp_id] = (base, len(self.alleles[key][snp_id]))
        return self.haplotypes

    @staticmethod
    def get_hs(alleles):
        """When there is a tie, choose base has higher emission prob.
        Or use both in turn."""
        haplotypes = {"m1": {}, "m2": {}}
        for key in alleles:
            for snp_id in alleles[key]:
                base = most_common(alleles[key][snp_id])
                haplotypes[key][snp_id] = (base, len(alleles[key][snp_id]))
        return haplotypes

    def align_alleles(self):
        """
        Compare haplotype strings between 2 states.
        Find difference, identify different loci and bases.
        scenario 1: the locus was covered only in one state, the other state did not emit base on this locus, reflect by -.
        scenario 2: in both states, the locus was covered with different bases.
        """
        snps_loci = list(self.haplotypes["m1"].keys())
        for locus in self.haplotypes["m2"].keys():
            if locus not in snps_loci:
                snps_loci.append(locus)
        snps_loci.sort()
        h1 = ""; h2 = ""; blank = "-"
        for pos in snps_loci:
            if pos in self.haplotypes["m1"]:
                h1 += self.haplotypes["m1"][pos][0]
            else:
                h1 += blank
        for pos in snps_loci:
            if pos in self.haplotypes["m2"]:
                h2 += self.haplotypes["m2"][pos][0]
            else:
                h2 += blank
        return h1, h2

    def get_aligned_haplotypes(self, align=True):
        """Return two strings of haplotypes generated by two states."""
        if align:
            self.get_haplotypes()
            return self.align_alleles()
        else:
            h1 = ""; h2 = ""
            m1 = collections.OrderedDict(sorted(self.alleles["m1"].items()))
            m2 = collections.OrderedDict(sorted(self.alleles["m2"].items()))
            for k, v in m1.items():
                h1 += v
            for k, v in m2.items():
                h2 += v
            return h1, h2

    def get_snps_pos(self):
        """Return two lists of SNPS loci."""
        return sorted(list(self.alleles["m1"].keys())), sorted(list(self.alleles["m2"].keys()))

    def get_reads(self):
        """Return reads clusters."""
        return self.read_assignments

    def get_states(self):
        """Return hidden states of the model."""
        return self.STATEs

    def get_observations(self):
        """Return the observations of the model."""
        return self.observations

    def get_emission(self):
        """Return emission probability of
        observed elements from different states."""
        return self.emission_probs

    def set_states(self, states):
        """Set states."""
        self.STATEs = states

    def set_observations(self, ob):
        """Set observations from states."""
        self.observations = ob

def random_choose(length, ratio):
    """length must be one-d, a list or an integer. not dictionaries."""
    s = int(len(length) * ratio)
    return np.random.choice(length, size=s, replace=False)

def most_common(lst):
    """Find the most common elements in a list, excluding ties."""
    return max(set(lst), key=lst.count)

def snp_read_dict(Reads, limited=False, minimum=5):
    """
    Find the READS on a given snp position
    :param Reads: list of NanoporeRead objects
    :param limited: (bool) if filter snps based on their reads num
    :param minimum: (int) the bare minimum number of reads mapped to a snp position
    :return: (dict) {snp: read_id1, read_id2}
    """
    sr_dict = {}
    for read_id, read in enumerate(Reads):
        for index, snp in enumerate(read.snps):
            snp_id = read.snps_id[index]
            if snp_id not in sr_dict.keys():
                sr_dict[snp_id] = [read_id]
            else:
                sr_dict[snp_id].append(read_id)
    if not limited:
        return sr_dict
    else:
        sr_d = {}
        for snp in sr_dict:
            if len(sr_dict[snp]) > minimum:
                sr_d[snp] = sr_dict[snp]
        return sr_d

##############
# Run Models #
##############
def run_model(snps, Reads, iter_num, hard=False, updateAll=True, ratio=0.5):
    """Set iteration times, run a model.
    Alternative: iterate until theta converge.
    :param updateAll:
    :param hard:
    :param iter_num:
    :param Reads:
    :param snps:
    :type ratio: float number between 0 and 1."""
    model = HMM(snps, ["Parental", "Maternal"])
    model.init_emission_matrix()
    # = np.zeros((iter_num, 2))
    for _ in range(iter_num):
        sr_dict = snp_read_dict(Reads, True, minimum=10)
        pm_llhd = model.assign_reads(Reads)
        #print(model.emission_probs[9519,:])
        #print((np.sum(pm_llhd[:,1]),np.sum(pm_llhd[:,0])))
        #s[_, 0] = np.sum(pm_llhd[:, 0])
        #s[_, 1] = np.sum(pm_llhd[:, 1])
        if updateAll:
            model.update(Reads, sr_dict, pm_llhd, hard=hard)
        else:
            model.partly_update(Reads, sr_dict, pm_llhd, ratio)
    return model

def models_iterations(iter_times, snps, Reads, hard=False, updateAll=True):
    """Generate a markov model multiple times."""
    model = run_model(snps, Reads, 10, hard=hard)
    results = np.zeros((iter_times, iter_times))
    for i in range(iter_times+1):  # i = base model
        last_model = model
        model = run_model(snps, Reads, 10, hard=hard, updateAll=updateAll)
        r = compare_models(last_model, model)
        results[i, i] = 1
        results[i, i+1] = r[0]
    return results

###################
# Analyze Results #
###################
## Compare with true value
def gold_standard(g_dict, m_dict):
    """compare model dict with this gold standard dict
    only use SNPS loci that been covered by Nanopore read in the model
    no need to use all the SNPS from the VCF file"""
    diff = {}
    for key in g_dict:
        diff[key] = {}
        for snp_pos in g_dict[key]:
            if snp_pos in m_dict["m1"]:
                diff[key]["m1"] = {}
                if m_dict["m1"][snp_pos] != g_dict[key][snp_pos]:
                    diff[key]["m1"][snp_pos] = (m_dict["m1"][snp_pos], g_dict[key][snp_pos])
            if snp_pos in m_dict["m2"]:
                diff[key]["m2"] = {}
                if m_dict["m2"][snp_pos] != g_dict[key][snp_pos]:
                    diff[key]["m2"][snp_pos] = (m_dict["m2"][snp_pos], g_dict[key][snp_pos])
    return diff

def read_results(model, all_snps):
    """

    :param model: HMM model
    :param all_snps: SNPs
    :return:
    """
    #snp_pos_bases = {"1|0": {snp.ref: B+=1, snp.alt: A+=1}, "0|1": {snp.alt: B+=1, snp.ref: A+=1}}
    h = model.get_haplotypes()
    for group in h:
        A = 0; B = 0; O = 0
        for snp_pos in h[group]:
            snp = all_snps[snp_pos]
            snp_pos_bases = {"1|0": {snp.ref: B, snp.alt: A}, "0|1": {snp.alt: B, snp.ref: A}}
            ref = snp.ref
            alt = snp.alt
            allele = h[group][snp_pos][0]

            #try:
                #snp_pos_bases[snp.gt][allele] += 1
            #except KeyError:
                #O += 1

            if snp.gt == "1|0":
                if allele == alt:
                    A += 1
                elif allele == ref:
                    B += 1
                else:
                    O += 1
            elif snp.gt == "0|1":
                if allele == alt:
                    B += 1
                elif allele == ref:
                    A += 1
                else:
                    O += 1
        print("Cluster: ", group)
        print("A/Total: ", A/(A+B+O))
        print("B/Total: ", B/(A+B+O))
        print("O/Total: ", O/(A+B+O))

## Two haplotype groups
def same_snp_sites(h1, h2):
    """
    Find SNP sites with same alleles.
    """
    same = {}
    for snp_id in h1:
        if snp_id in h2 and h2[snp_id][0] == h1[snp_id][0]:
            same[snp_id] = (h1[snp_id][1], h2[snp_id][1], h1[snp_id])
    if len(h1) == 0:
        same_p = 0
    else:
        same_p = len(same) / len(h1)
    if len(h2) == 0:
        same_p2 = 0
    else:
        same_p2 = len(same) / len(h2)
    return same

def diff_snp_sites(h1, h2):
    diff = {}
    for snp_id in h1:
        if snp_id not in h2:
            diff[snp_id] = (h1[snp_id][1], 0)
        elif h2[snp_id][0] != h1[snp_id][0]:
            diff[snp_id] = (h1[snp_id][1], h2[snp_id][1])
    for snp in h2:
        if snp not in h1:
            diff[snp] = (0, h2[snp][1])

    if len(h1) == 0:
        diff_p = 0
    else:
        diff_p = len(diff) / len(h1)
    if len(h2) == 0:
        diff_p2 = 0
    else:
        diff_p2 = len(diff) / len(h2)
    print("{}% of the first haplotype alleles are different from the second haplotype alleles.".format(diff_p))
    print("{}% of the second haplotype alleles are different from the first haplotype alleles.".format(diff_p2))
    return diff

## Two models
def find_common_poses(m1, m4):
    m1h = m1.get_haplotypes()
    m4h = m4.get_haplotypes()
    snp_posesm1 = []
    for snp_pos in m1h["m1"]:
        if m1h["m1"][snp_pos][1] == 1:
            pass
        else:
            if snp_pos not in snp_posesm1:
                snp_posesm1.append(snp_pos)
    snp_posesm2 = []
    for snp_pos in m1h["m2"]:
        if m1h["m2"][snp_pos][1] == 1:
            pass
        else:
            if snp_pos not in snp_posesm2:
                snp_posesm2.append(snp_pos)
    snp_poses2m1 = []
    for snp_pos in m4h["m1"]:
        if m4h["m1"][snp_pos][1] == 1:
            pass
        else:
            if snp_pos not in snp_poses2m1:
                snp_poses2m1.append(snp_pos)
    snp_poses2m2 = []
    for snp_pos in m4h["m2"]:
        if m4h["m2"][snp_pos][1] == 1:
            pass
        else:
            if snp_pos not in snp_poses2m2:
                snp_poses2m2.append(snp_pos)
    positions = [snp_posesm1, snp_posesm2, snp_poses2m1, snp_poses2m2]
    same_pos = set(positions[0])
    for s in positions[1:]:
        same_pos.intersection_update(s)
    print(len(same_pos)/len(snp_posesm1))
    print(len(same_pos)/len(snp_posesm2))
    print(len(same_pos)/len(snp_poses2m1))
    print(len(same_pos)/len(snp_poses2m2))
    print(len(same_pos))
    return same_pos

def compare_models(m1, m2):
    """
    same. pos. read num. percentage.
    diff.
    """
    a1 = m1.get_haplotypes()
    a2 = m2.get_haplotypes()
    same_snp_sites(a1["m1"], a2["m1"])
    same_snp_sites(a1["m1"], a2["m2"])
    same_snp_sites(a1["m2"], a2["m1"])
    same_snp_sites(a1["m2"], a2["m1"])
    diff_snp_sites(a1["m1"], a2["m1"])
    diff_snp_sites(a1["m1"], a2["m2"])
    diff_snp_sites(a1["m2"], a2["m1"])
    diff_snp_sites(a1["m2"], a2["m2"])

def common_pos_haplotypes(m1, m4, same_pos):
    h1 = m1.get_haplotypes()
    h4 = m4.get_haplotypes()
    new_h1 = {"m1": {}, "m2": {}}
    new_h4 = {"m1": {}, "m2": {}}
    for pos in same_pos:
        new_h1["m1"][pos] = h1["m1"][pos]
        new_h1["m2"][pos] = h1["m2"][pos]
        new_h4["m1"][pos] = h4["m1"][pos]
        new_h4["m2"][pos] = h4["m2"][pos]
    return new_h1, new_h4

## New input/testing data
def compare_clustering(m1, m2, data):
    mm1 = m1.cluster_reads(data)
    mm4 = m2.cluster_reads(data)
    same_snp_sites(mm1["m1"], mm4["m1"])
    same_snp_sites(mm1["m1"], mm4["m2"])
    same_snp_sites(mm1["m2"], mm4["m1"])
    same_snp_sites(mm1["m2"], mm4["m1"])
    diff_snp_sites(mm1["m1"], mm4["m1"])
    diff_snp_sites(mm1["m1"], mm4["m2"])
    diff_snp_sites(mm1["m2"], mm4["m1"])
    diff_snp_sites(mm1["m2"], mm4["m2"])

if __name__ == "__main__":
    SNPS = load_objects("../data/chr19_snps.obj")
    READS = load_objects("../data/reads_no_indels.obj")
    start = time.clock()
    m1 = run_model(SNPS, READS, 50)
    elapsed = (time.clock() - start)
    print("run one model: Time used:", elapsed)
    read_results(m1, SNPS)

    #mm1 = m1.cluster_reads(IR_READS)
    #m2 = run_model(SNPS, READS, 50)
    #mm4 = m2.cluster_reads(IR_READS)
    #compare_clustering(m1, m2, IR_READS)
    #h1 = m1.get_haplotypes()
    #h4 = m2.get_haplotypes()
    #stt = time.clock()
    #compare_models(m1, m2)
    #elapsed = (time.clock() - stt)
    #print("compare two models: Time used:", elapsed)


text:
import numpy as np
import matplotlib.pyplot as plt
import os

N_classes_default = 16
N_attributes_default = 40
class_names_default = ['basketball_layup',
               'bowling',
               'clean_and_jerk',
               'discus_throw',
               'diving_platform_10m',
               'diving_springboard_3m',
               'hammer_throw',
               'high_jump',
               'javelin_throw',
               'long_jump',
               'pole_vault',
               'shot_put',
               'snatch',
               'tennis_serve',
               'triple_jump',
               'vault'
               ]

def parse_attributes(attr_file=None,
                     save_npz=True,
                     attr_npz_file='attributes',
                     attr_npz_path=None,
                     N_classes=None,
                     N_attributes=None,
                     class_names=None):
    '''
    Parse a Olympic_Attributes like txt file, that contains attributes list and a 0's and 1's code assigning to each class. Defaults reading the resources attribute annotations, however also receives a path to a custom txt. It can save into a npz file for faster access or just parse. If saved you shoud call method read_attributes(path_to_npz_attributes) for fast access.

    :param attr_file: File to the 0's and 1's assignment txt file
    :param save_npz: boolean to save npz file
    :param attr_npz_file:  name of the attribute file, defaults attributes.npz
    :param attr_npz_path: path to directory where npz file is saved, defaults None, uses the current directory
    :param N_classes: number of classes considered in the file of assignment, defaults 16
    :param N_attributes: number of attributes considere in the file of assignment, defaults 40
    :param class_names: names of the classes defaults to `olympic_sports.class_names`
    :return: tuple of tree arrays: attributes N_classes X N_attributes, string array of class names, string array of attributes
    '''
    if class_names is None:
        class_names = class_names_default

    if N_classes is None:
        N_classes = N_classes_default

    if N_attributes is None:
        N_attributes = N_attributes_default

    if attr_file is None:
        attr_file = os.path.join(os.path.split(__file__)[0], "resources", "Olympic_Attributes.txt")

    if attr_npz_path is None:
        attr_npz_path = os.path.join(os.getcwd(), attr_npz_file)
    cnt_line = 0
    cnt_classes = 0
    cnt_attributes = 0
    attribute_names = []
    attributes = np.zeros((N_classes, N_attributes))

    file = open(attr_file, 'r')
    for line in file:
        line = line.rstrip('\n')
        if cnt_line == 0:
            print('----------------------------------------------')
            print('Parsing attribute name')
            print('')
            cnt_classes = 0
            cnt_attributes += 1
            # parsing attribute name and index
            attribute_names.append(line)
        else:
            # parsing attribute value
            attributes[cnt_classes, cnt_attributes - 1] = int(line)
            print('Class {:02d} \t| Attribute {:02d}'.format(cnt_classes + 1, cnt_attributes + 1))
            cnt_classes += 1
        if cnt_classes == N_classes:
            cnt_line = 0
        else:
            cnt_line += 1
    if save_npz:
        print('Saving arrays into a npz file: ', attr_npz_path)
        np.savez(attr_npz_path, attributes, class_names, attribute_names)
    return attributes, class_names, attribute_names

def plot_attibutes_table(attr_file=None):
    '''
    Plots assignemnt file .txt (e.g. resources/Olympic_Attributes) as color matrix image.

    :param attr_file: path the 0's and 1's attributs file.
    :return: None
    '''
    if attr_file is None:
        attr_file = os.path.join(os.path.split(__file__)[0], "resources", "Olympic_Attributes.txt")
    attributes, class_names, attribute_names = parse_attributes(attr_file=attr_file, save_npz=False)
    plt.figure(figsize=(5, 20))
    fig, ax = plt.subplots()
    im = ax.imshow(attributes)
    # We want to show all ticks...
    ax.set_xticks(np.arange(len(attribute_names)))
    ax.set_yticks(np.arange(len(class_names)))
    # ... and label them with the respective list entries
    ax.set_xticklabels(attribute_names)
    ax.set_yticklabels(class_names)
    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")
    # Loop over data dimensions and create text annotations.
    for i in range(len(class_names)):
        for j in range(len(attribute_names)):
            text = ax.text(j, i, attributes[i, j],
                           ha="center", va="center", color="w")
    ax.set_title("Classes & attributes")
    fig.tight_layout()
    plt.show()

def read_attributes(attr_file=None):
    '''
    Read attributes file npz already parsed or a txt file

    :param attr_file: If None, defaults to resources module file).
    :type attr_file: str
    :return: tuple of tree arrays: attributes N_classes X N_attributes, string array of class names, string array of attributes
    '''
    if attr_file is None:
        attr_file = os.path.join(os.path.split(__file__)[0], "resources", "attributes.npz")
        nzfile = np.load(attr_file)
        attributes, class_names, attribute_names = nzfile.files
        attributes = nzfile[attributes]
        class_names = nzfile[class_names]
        attribute_names = nzfile[attribute_names]
    elif attr_file.endswith('.npz'):
        nzfile = np.load(attr_file)
        attributes, class_names, attribute_names = nzfile.files
        attributes = nzfile[attributes]
        class_names = nzfile[class_names]
        attribute_names = nzfile[attribute_names]
    elif attr_file.endswith('.txt'):
        attributes, class_names, attribute_names = parse_attributes(attr_file=attr_file, save_npz=False)

    return attributes, class_names, attribute_names


text:
'''
Created on April 27, 2018

@author: Edwin Simpson
'''
from evaluation.experiment import Experiment
import data.load_data as load_data
import numpy as np
import os

gt, annos, doc_start, text, gt_nocrowd, doc_start_nocrowd, text_nocrowd, gt_val, _ = \
    load_data.load_ner_data(False)

# debug with subset -------
s = 100
idxs = np.argwhere(gt!=-1)[:s, 0]
gt = gt[idxs]
annos = annos[idxs]
doc_start = doc_start[idxs]
text = text[idxs]
gt_val = gt_val[idxs]
# -------------------------

num_reps = 10
batch_frac = 0.03
AL_iters = 10

output_dir = os.path.join(load_data.output_root_dir, 'ner_al')
if not os.path.isdir(output_dir):
    os.mkdir(output_dir)

# ACTIVE LEARNING WITH UNCERTAINTY SAMPLING
for rep in range(1, num_reps):

    beta0_factor = 0.1
    alpha0_diags = 1  # best_diags
    alpha0_factor = 1  # 9 # best_factor
    exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text,
                     alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor,
                     max_iter=20, crf_probs=True, rep=rep)
    exp.methods =  [
        'bac_seq_integrateIF',
        'HMM_crowd',
    ]
    results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods(
        active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters
    )

    beta0_factor = 0.1
    alpha0_diags = 100 # best_diags
    alpha0_factor = 0.1 #9 # best_factor
    exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text,
                     alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor,
                     max_iter=20, crf_probs=True, rep=rep)
    # run all the methods that don't require tuning here
    exp.methods =  [
        'bac_ibcc_integrateIF',
    ]

    results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods(
                        active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters
    )

    beta0_factor = 10
    alpha0_diags = 1 # best_diags
    alpha0_factor = 1#9 # best_factor
    exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text,
                     alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor,
                     max_iter=20, crf_probs=True, rep=rep)
    exp.methods =  [
        'bac_vec_integrateIF',
    ]

    results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods(
                        active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters
    )

    beta0_factor = 0.1
    alpha0_diags = 1 # best_diags
    alpha0_factor = 0.1#9 # best_factor
    exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text,
                     alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor,
                     max_iter=20, crf_probs=True, rep=rep)
    exp.methods =  [
        'ibcc',
        'ds',
        'majority'
    ]

    results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods(
                        active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters
    )


text:
import numpy as np
import random

import torch
from torch.utils.data import DataLoader
from torch.utils.data.sampler import Sampler
import torchvision.transforms as transforms

from dataset import Omniglot, MNIST

'''
Helpers for loading class-balanced few-shot tasks
from datasets
'''

class ClassBalancedSampler(Sampler):
    '''
    Samples class-balanced batches from 'num_cl' pools each
    of size 'num_inst'
    If 'batch_cutoff' is None, indices for iterating over batches
    of the entire dataset will be returned
    Otherwise, indices for the number of batches up to the batch_cutoff
    will be returned
    (This is to allow sampling with replacement across training iterations)
    '''

    def __init__(self, num_cl, num_inst, batch_cutoff=None):
       self.num_cl = num_cl
       self.num_inst = num_inst
       self.batch_cutoff = batch_cutoff

    def __iter__(self):
       '''return a single list of indices, assuming that items will be grouped by class '''
       # First construct batches of 1 instance per class
       batches = [[i+j*self.num_inst for i in torch.randperm(self.num_inst)] for j in range(self.num_cl)]
       batches = [[batches[j][i] for j in range(self.num_cl)] for i in range(self.num_inst)]
       # Shuffle within each batch so that classes don't always appear in same order
       for sublist in batches:
           random.shuffle(sublist)

       if self.batch_cutoff is not None:
           random.shuffle(batches)
           batches = batches[:self.batch_cutoff]

       batches = [item for sublist in batches for item in sublist]

       return iter(batches)

    def __len__(self):
       return 1

def get_data_loader(task, batch_size=1, split='train'):
    # NOTE: batch size here is # instances PER CLASS
    if task.dataset == 'mnist':
        normalize = transforms.Normalize(mean=[0.13066, 0.13066, 0.13066], std=[0.30131, 0.30131, 0.30131])
        dset = MNIST(task, transform=transforms.Compose([transforms.ToTensor(), normalize]), split=split)
    else:
        normalize = transforms.Normalize(mean=[0.92206, 0.92206, 0.92206], std=[0.08426, 0.08426, 0.08426])
        dset = Omniglot(task, transform=transforms.Compose([transforms.ToTensor(), normalize]), split=split)
    sampler = ClassBalancedSampler(task.num_cl, task.num_inst, batch_cutoff = (None if split != 'train' else batch_size))
    loader = DataLoader(dset, batch_size=batch_size*task.num_cl, sampler=sampler, num_workers=1, pin_memory=True)
    return loader

max_stars_repo_name: katerakelly/pytorch-maml

max_issues_repo_name: lolinkun/pytorch-maml

max_forks_repo_name: lolinkun/pytorch-maml


text:
import numpy as np
import tensorflow as tf
import tensorflow_datasets as tfds
import os
import sys

import input_pipeline

from tensorflow.keras.layers import Dense, Flatten, Conv2D,Embedding,LSTM
from tensorflow.keras import Model

sys.path.append(os.path.normpath(os.path.dirname(os.getcwd())))
from mandarin_common_tf2 import *

##############################################################
###Location
##############################################################

print("---------------------------------------------------------")
print("USE WITH TF 2.0")
print("---------------------------------------------------------")

os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
# The GPU id to use, usually either "0" or "1"
os.environ["CUDA_VISIBLE_DEVICES"]="0"

bs = 50
cutoff=4000
cut_or_pad=cutoff

ds_train, _, ds_test, mencoder = input_pipeline.get_tc_datasets(
  n_devices=1,
  task_name='imdb_reviews',
  data_dir=None,
  batch_size=bs,
  fixed_vocab=None,
  max_length=cutoff)

inputs = tf.keras.layers.Input(shape=(cutoff))

nb_patches=1300
nb_filters_conv1d = 256
l2_reg = 0.005
dr=0.25
nb_stacks = 2
incoming_proj = 24
pooling_size = 4

x = inputs
x = Embedding(input_dim=257,output_dim=256,trainable=True)(x)

x = IGLOO1D_BLOCK(x,nb_patches,nb_filters_conv1d,patch_size=4,nb_stacks=nb_stacks,DR=dr,l2_reg=l2_reg,transformer_style=True,pooling_size=pooling_size,blockstyle="v1",incoming_proj=incoming_proj)

x = Dense(80)(x)
x = Activation("relu")(x)
x = Dropout(dr)(x)

x = Dense(2, activation="softmax")(x)

model = Model(inputs=inputs, outputs=x, name="igloo_model")

model.compile(
    loss='sparse_categorical_crossentropy',
    optimizer=tf.keras.optimizers.Adam(0.00015),
    metrics=['accuracy'],
)

print(model.summary())

model.fit(
    ds_train,
    epochs=120,
    validation_data=ds_test,
)


text:
import json
import os
from os import listdir
import os.path as osp
import pickle
import time
from collections import OrderedDict, Counter
import argparse
import numpy as np
import pathlib
import joblib

from gym.wrappers.monitoring.video_recorder import VideoRecorder

from TeachMyAgent.run_utils.environment_args_handler import EnvironmentArgsHandler
from TeachMyAgent.run_utils.student_args_handler import StudentArgsHandler
from TeachMyAgent.teachers.teacher_controller import param_vec_to_param_dict

from TeachMyAgent.students.spinup.utils.test_policy import load_policy_and_env as spinup_load_policy
from TeachMyAgent.students.openai_baselines.ppo2.ppo2 import get_model as get_baselines_model
from TeachMyAgent.students.ppo_utils import create_custom_vec_normalized_envs

def get_student_type(save_path):
    '''
        Returns 'spinup' or 'baselines' depending on how the logs look like.

        Args:
            save_path (str): Path containing logs
    '''
    for root, _, files in os.walk(save_path):
        if 'progress.txt' in files: # Spinup
            return 'spinup'
        elif 'progress.csv' in files: # OpenAI Baselines
            return 'baselines'

def load_training_infos(save_path):
    '''
        Load hyperparameters stored in config.json.

        Args:
            save_path (str): Path containing logs
    '''
    with open(osp.join(save_path, 'config.json')) as json_file:
        training_config = json.load(json_file)
    return training_config

def get_baselines_last_checkpoint(path):
    '''
        OpenAI Baselines students save multiple checkpoints of the model. This function only loads the last one.

        Args:
            save_path (str): Path containing checkpoints
    '''
    last_checkpoint = -1
    for f in listdir(path):
        if osp.isfile(osp.join(path, f)):
            try:
                checkpoint = int(f)
                last_checkpoint = f if checkpoint > int(last_checkpoint) else last_checkpoint
            except Exception:
                continue
    return last_checkpoint

def load_env_params(save_path):
    '''
        Load book-keeped information (e.g. training and test tasks along with the obtained reward).

        Args:
            save_path (str): Path containing logs
    '''
    with open(osp.join(save_path, 'env_params_save.pkl'), "rb") as file:
        teacher_dict = pickle.load(file)
    return teacher_dict

def get_training_test_size(teacher_dict):
    '''
        Calculate size of test set used during training.

        Args:
            teacher_dict (dict): Dictionary of loaded logs.
    '''
    param_to_count = teacher_dict["env_params_test"][0]
    nb_of_epochs = 0
    for param in teacher_dict["env_params_test"]:
        if (param_to_count == param).all():
            nb_of_epochs += 1

    return int(len(teacher_dict["env_params_test"]) / nb_of_epochs)

def load_training_test_set(save_path, order_by_best_rewards=None):
    '''
        Load test set used during training.

        Args:
            save_path (str): Path containing logs
            order_by_best_rewards (str): If None => Do not order test set.
                                   If True => Order test set using rewards obtained from greatest to lowest
                                   If False => Order test set using rewards obtained from lowest to greatest

        Returns:
            list of tasks and list associated rewards
    '''
    ### Get last training test episodes and sort them by total reward
    teacher_dict = load_env_params(save_path)
    test_set_size = get_training_test_size(teacher_dict)

    test_params_to_use = teacher_dict["env_params_test"][-test_set_size:]  # nth last
    test_rewards_to_use = teacher_dict["env_test_rewards"][-test_set_size:]
    if order_by_best_rewards is not None:
        print("Getting test set tasks ordered by last return from {} to {} ..."
              .format("greatest" if order_by_best_rewards else "lowest",
                      "lowest" if order_by_best_rewards else "greatest"))
        sorted_indexes_of_test_episodes = sorted(range(test_set_size),
                                                 key=lambda k: test_rewards_to_use[k],
                                                 reverse=order_by_best_rewards)  # Sort with best results first
    else:
        print("Getting test set tasks as defined...")
        sorted_indexes_of_test_episodes = range(test_set_size)

    teacher_param_env_bounds = OrderedDict(teacher_dict["env_param_bounds"])
    env_params_list = [param_vec_to_param_dict(teacher_param_env_bounds, test_params_to_use[i])
                       for i in sorted_indexes_of_test_episodes]
    associated_rewards_list = [test_rewards_to_use[i] for i in sorted_indexes_of_test_episodes]
    return env_params_list, associated_rewards_list

def load_fixed_test_set(save_path, test_set_name):
    '''
        Load a test set from a file.

        Args:
            save_path (str): Path containing test sets
            test_set_name (str): Name of the file containing the test set (do not add the extension)

        Returns:
            list of tasks
    '''
    teacher_dict = load_env_params(save_path)
    teacher_param_env_bounds = OrderedDict(teacher_dict["env_param_bounds"])
    test_param_vec = np.array(pickle.load(open("TeachMyAgent/teachers/test_sets/" + test_set_name + ".pkl", "rb")))

    return [param_vec_to_param_dict(teacher_param_env_bounds, vec) for vec in test_param_vec]

def load_env(save_path, load_test_env=False):
    '''
        Load saved environment.

        Args:
            save_path (str): Path containing logs
            load_test_env (str): Name of the file containing the test set (do not add the extension)

        Returns:
            loaded environment
    '''
    try:
        filename = osp.join(save_path, 'vars.pkl')
        state = joblib.load(filename)
        if load_test_env:
            env = state['test_env']
        else:
            env = state['env']
    except Exception as err:
        print("Unable to load envs : {}".format(err))
        env = None

    return env

def load_vectorized_env(save_path, env):
    try:
        filename = osp.join(save_path, 'vars.pkl')
        state = joblib.load(filename)
        env.__load_rms__(state["ob_rms"], state["ret_rms"])
    except Exception as err:
        print("Unable to load Running Mean Stds : {}".format(err))

def run_policy(env, get_action, env_params_list, max_ep_len=None, episode_id=0, record=False, recording_path=None,
               no_render=False, use_baselines=False):
    '''
        Run an episode of a trained policy.

        Args:
            env: Environment
            get_action: Policy function
            env_params_list: List of tasks among one must be loaded
            max_ep_len: Maximum number of steps allowed in the episode
            episode_id: Id of the episode to load in `env_params_list`
            record: Whether a video of the episode should be recorded
            recording_path: Path on which the video must be saved
            no_render: Whether the episode must be ran without a frame rendering it
            use_baselines: Whether the policy was trained using OpenAI Baselines
    '''
    if record:
        if os.name == "nt":
            full_path = os.path.join(pathlib.Path().absolute(), recording_path)
            full_path_len = len(full_path)
            nb_char_to_remove = full_path_len - 245
            if nb_char_to_remove > 0:
                recording_path = recording_path[:-nb_char_to_remove]
        video_recorder = VideoRecorder(env, recording_path + "_ep" + str(episode_id) + ".mp4", enabled=True)

    if use_baselines:
        env.get_raw_env().set_environment(**env_params_list[episode_id])
    else:
        env.set_environment(**env_params_list[episode_id])

    if use_baselines:
        _, o = env.reset()
    else:
        o = env.reset()

    r, d, ep_ret, ep_len, n = 0, False, 0, 0, 0
    while True:
        if record and video_recorder.enabled:
            video_recorder.capture_frame()
        if not record and not no_render:
            env.render()
            time.sleep(1e-3)

        a = get_action(o)
        o, r, d, i = env.step(a)
        if use_baselines:
            ep_ret += i[0]["original_reward"][0]
        else:
            ep_ret += r
        ep_len += 1

        if d or (ep_len == max_ep_len):
            print('Episode %d \t EpRet %.3f \t EpLen %d'%(episode_id, ep_ret, ep_len))
            if record and video_recorder.enabled:
                video_recorder.close()
                video_recorder.enabled = False
            break
    return ep_ret

def main(args):
    '''
        Test a learned policy on tasks.

        Args:
            args: arguments defining what has to be run
    '''
    if args.fixed_test_set is None:
        # training_config = load_training_infos(args.fpath)
        # nb_test_episodes_during_training = training_config["num_test_episodes"] \
        #     if "num_test_episodes" in training_config \
        #     else training_config["nb_test_episodes"]
        test_set_params, _ = load_training_test_set(args.fpath, args.bests)
    else:
        test_set_params = load_fixed_test_set(args.fpath, args.fixed_test_set)

    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
    student_type = get_student_type(args.fpath)

    env = None
    if args.load_env:
        env = load_env(args.fpath, args.use_test_env is not None)

    if env is None:
        env_fn, _, _, _ = EnvironmentArgsHandler.get_object_from_arguments(args)
        if student_type == "spinup":
            env = env_fn()
        elif student_type == "baselines":
            env, _ = create_custom_vec_normalized_envs(env_fn)
            load_vectorized_env(args.fpath, env)

    if student_type == 'spinup':
        get_action = spinup_load_policy(args.fpath,
                                        args.itr if args.itr >= 0 else 'last',
                                        args.deterministic)
        env._SET_RENDERING_VIEWPORT_SIZE(600, 400)
    elif student_type == 'baselines':
        ac_kwargs = dict()
        ac_kwargs['hidden_sizes'] = [int(layer) for layer in args.hidden_sizes.split("/")]
        nbatch_train = args.nb_env_steps * 1e6 // int(args.sample_size//args.batch_size)

        model = get_baselines_model(network=args.network, nbatch_train=nbatch_train, ob_space=env.observation_space,
                                    ac_space=env.action_space, env=env, nsteps=args.sample_size, ent_coef=args.ent_coef,
                                    vf_coef=args.vf_coef, hidden_sizes=ac_kwargs['hidden_sizes'])
        last_checkpoint = get_baselines_last_checkpoint(args.fpath + "/checkpoints/")
        model.load(args.fpath + "/checkpoints/" + last_checkpoint)
        # Careful : The recurrent version is not implemented here yet
        get_action = lambda o: model.step(o)[0]
        env.get_raw_env()._SET_RENDERING_VIEWPORT_SIZE(600, 400)
    else:
        raise Exception('Unknown student type.')

    if args.episode_ids == "-1":
        print("Testing the policy on the whole test set...")
        episodes = [i for i in range(len(test_set_params))]
    else:
        episodes = [int(id) for id in args.episode_ids.split("/")]

    rewards = []
    for episode_id in episodes:
        r = run_policy(env, get_action, test_set_params, args.len, episode_id, args.record, args.recording_path,
                       args.norender, use_baselines=student_type == 'baselines')
        rewards.append(r)
    env.close()
    return rewards

def str2bool(v):
    if isinstance(v, bool):
       return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def get_parser():
    '''
        Define arguments that can be used when testing a policy.
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('--fpath', type=str)
    parser.add_argument('--len', '-l', type=int, default=0)
    parser.add_argument('--norender', '-nr', action='store_true')
    parser.add_argument('--itr', '-i', type=int, default=-1)
    parser.add_argument('--deterministic', '-d', action='store_true')
    parser.add_argument('--episode_ids', '-id', type=str, default="0")
    parser.add_argument('--bests', type=str2bool, default=None)
    parser.add_argument('--fixed_test_set', '-ts', type=str, default=None)
    parser.add_argument('--load_env', action='store_true')
    parser.add_argument('--use_test_env', action='store_true')

    parser.add_argument('--record', type=str2bool, default=False)
    parser.add_argument('--recording_path', type=str, default=None)
    EnvironmentArgsHandler.set_parser_arguments(parser)
    StudentArgsHandler.set_parser_arguments(parser)

    return parser

if __name__ == '__main__':
    parser = get_parser()
    args = parser.parse_args()
    main(args)

max_stars_repo_name: flowersteam/TeachMyAgent

max_issues_repo_name: flowersteam/TeachMyAgent

max_forks_repo_name: flowersteam/TeachMyAgent


text:
import argparse
import dataset
from model import NESA, get_metrics
import numpy as np
import os
import pickle
import random
import torch

def get_dataset(cfg, trained_dict_path):
    print('Creating the test dataset pickle..', )
    nets_dictionary = pickle.load(open(trained_dict_path, 'rb'))
    test_set = dataset.NETSDataset(cfg, nets_dictionary)
    if len(test_set.test_data) == 0:
        print('no events')
        return None
    pickle.dump(test_set, open(cfg.preprocess_save_path, 'wb'))
    return test_set

def get_model(widx2vec, model_path, dvc, idx2dur, arg):
    model_dir, model_filename = os.path.split(model_path)
    checkpoint = torch.load(model_path,
                            map_location=None if 'cuda' == dvc.type else 'cpu')
    ckpt_config = checkpoint['config']
    ckpt_dict = vars(ckpt_config)
    ckpt_dict['yes_cuda'] = arg.yes_cuda  # overriding

    model = \
        NESA(ckpt_config, widx2vec,
             idx2dur=idx2dur if ckpt_config.use_duration_scala > 0
             else None).to(dvc)
    model.config.checkpoint_dir = model_dir + '/'
    model.load_checkpoint(filename=model_filename[:-4])  # .pth
    # import pprint
    # pprint.PrettyPrinter().pprint(_model.config.__dict__)
    return model, ckpt_config

def measure_performance(test_set, model, conf, dvc, batch_size=1):
    performance_dict = dict()
    performance_dict['recall1'] = 0.
    performance_dict['recall5'] = 0.
    performance_dict['mrr'] = 0.
    performance_dict['ieuc'] = 0.
    performance_dict['count'] = 0
    performance_dict['steps'] = 0.

    model = model.eval()

    _, _, test_loader = test_set.get_dataloader(batch_size=batch_size)
    with torch.no_grad():
        for d_idx, ex in enumerate(test_loader):
            labels = ex[-1].to(dvc)
            outputs = model(*ex[:-1])
            metrics = get_metrics(outputs, labels, model.n_day_slots,
                                  model.n_classes,
                                  ex_targets=ex[-2].to(device)
                                  if conf.ex_pre_events > 0 else None)

            performance_dict['recall1'] += metrics[0]
            performance_dict['recall5'] += metrics[1]
            performance_dict['mrr'] += metrics[2]
            performance_dict['ieuc'] += metrics[3]
            performance_dict['count'] += outputs.data.size()[0]
            performance_dict['steps'] += 1.

            if d_idx % 1000 == 0 and d_idx > 0:
                print(d_idx)

    steps = performance_dict['steps']
    recall1 = performance_dict['recall1'] / steps
    recall5 = performance_dict['recall5'] / steps
    mrr = performance_dict['mrr'] / steps
    ieuc = performance_dict['ieuc'] / steps

    print('recall@1 %.4f' % recall1)
    print('recall@5 %.4f' % recall5)
    print('mrr      %.4f' % mrr)
    print('ieuc     %.4f' % ieuc)
    print('#events', performance_dict['count'])

def set_seed_all(seed):
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument("--input_path", type=str,
                            default='./data/sample_data.csv')
    arg_parser.add_argument("--serialized_data_path", type=str,
                            default='./data/preprocess_test.pkl')
    arg_parser.add_argument("--model_path", type=str,
                            default='./data/nesa_180522_0.pth')
    arg_parser.add_argument("--trained_dict_path", type=str,
                            default='./data/dataset_180522_dict.pkl')
    arg_parser.add_argument("--seed", type=int, default=3)
    arg_parser.add_argument('--yes_cuda', type=int, default=1)
    args = arg_parser.parse_args()

    use_cuda = args.yes_cuda > 0 and torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")
    print('CUDA device_count {0}'.format(torch.cuda.device_count())
          if use_cuda else 'CPU')

    set_seed_all(args.seed)

    config = dataset.Config()
    config.test_path = args.input_path
    config.preprocess_save_path = args.serialized_data_path
    config.preprocess_load_path = args.serialized_data_path

    print('Loading test dataset..')
    test_dataset = get_dataset(config, args.trained_dict_path)
    assert test_dataset is not None

    print('Loading NESA model..')
    nesa_model, nesa_conf = get_model(test_dataset.widx2vec, args.model_path,
                                      device, test_dataset.idx2dur, args)

    print('\nMeasuring NESA performance on test data..')
    measure_performance(test_dataset, nesa_model, nesa_conf, device)

max_stars_repo_name: dmis-lab/nesa

max_issues_repo_name: donghyeonk/nets

max_forks_repo_name: donghyeonk/nets


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-
## Autor: David Ochoa

import vispy as vpy
import numpy as np
import sys

std_ln_dic = dict(color=(1.00, 1.00, 1.00, 0.25))

class Bworld():
    """Ball and bubble world class.
            This clas initialize the view, handle the keypresses and update
            every object every time_step.
    """
    def __init__(self, bubble_list, boundaries = np.asarray([[0, 10], [0, 10], [0, 10]])):
        self.auto = False
        self.time_step = 0.05
        self.bubbles = bubble_list
        self.bound = boundaries
        for bble in self.bubbles:
            bble.set_bound(self.bound)
        canvas = vpy.scene.SceneCanvas(show=True, title=sys.argv[0])
        # canvas.measure_fps()
        view = canvas.central_widget.add_view()
        if self.bound is not None:
            bound_pt = np.array([[self.bound[0, 0], self.bound[1, 0], self.bound[2, 0]],
                                 [self.bound[0, 1], self.bound[1, 0], self.bound[2, 0]],
                                 [self.bound[0, 1], self.bound[1, 1], self.bound[2, 0]],
                                 [self.bound[0, 0], self.bound[1, 1], self.bound[2, 0]],
                                 [self.bound[0, 0], self.bound[1, 0], self.bound[2, 0]],
                                 [self.bound[0, 0], self.bound[1, 0], self.bound[2, 1]],
                                 [self.bound[0, 1], self.bound[1, 0], self.bound[2, 1]],
                                 [self.bound[0, 1], self.bound[1, 1], self.bound[2, 1]],
                                 [self.bound[0, 0], self.bound[1, 1], self.bound[2, 1]],
                                 [self.bound[0, 0], self.bound[1, 0], self.bound[2, 1]]],
                                 dtype=np.float32)
            bound_vi = vpy.scene.visuals.Line(pos=bound_pt, **std_ln_dic)
            view.add(bound_vi)
        view.camera = 'turntable'
        for bble in self.bubbles:
            bble.init_visual(view)

        def update(ev):
            for bble in self.bubbles:
                bble.step(self.time_step)

        timer = vpy.app.Timer()
        timer.connect(update)

        @canvas.events.key_press.connect
        def on_key_press(event):
            if event.key == 'Right':
                for bble in self.bubbles:
                    bble.step(self.time_step)
            if event.key == 'Space':
                if self.auto:
                    timer.stop()
                    self.auto = False
                else:
                    timer.start(self.time_step)
                    self.auto = True
            if event.key == 's':
                for bble in self.bubbles:
                    bble.shake()

        vpy.app.run()


text:
import numpy as np
import scipy.io as sio
import matplotlib.pyplot as plt

loss_val = np.load("valLoss2_Gauss.npy")[20:]
loss_train = np.load("trainLoss2_Gauss.npy")[20:]

"""
L = len(loss_val)

plt.figure()
plt.plot(range(L), loss_val, "r", label = "loss_val")
plt.plot(range(L), loss_train, "b", label = "loss_train")
plt.legend()
plt.savefig("newloss2.png")
"""

v1 = np.load("v1_Gauss.npy")
v2 = np.load("v2_Gauss.npy")

sio.savemat("OLS2015011208.mat", {"v1": v1, "v2": v2})

max_stars_repo_name: UltronAI/Stochastic-Process

max_issues_repo_name: UltronAI/Stochastic-Process

max_forks_repo_name: UltronAI/Stochastic-Process


text:
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_digits

digits = load_digits()
X_scale = StandardScaler()
X = X_scale.fit_transform(digits.data)
y = digits.target  # target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)

# this function convert digit value to vector in output layer of network
def convert_y_to_vect(y):
    y_vect = np.zeros((len(y), 10))
    for i in range(len(y)):
        y_vect[i, y[i]] = 1
    return y_vect

y_v_train = convert_y_to_vect(y_train)
y_v_test = convert_y_to_vect(y_test)
y_train[0], y_v_train[0]

print(y_train[0],y_v_train[0])

max_stars_repo_path: seting up the out put layer for nnetwork.py

max_stars_repo_name: m-okhravi/sklearn

max_issues_repo_name: m-okhravi/Neural-network

max_forks_repo_name: m-okhravi/Neural-network


text:
# https://www.geeksforgeeks.org/single-neuron-neural-network-python/
# Python program to implement a
# single neuron neural network

# import all necessery libraries
from numpy import exp, array, random, dot, tanh

# Class to create a neural
# network with single neuron
class NeuralNetwork():

	def __init__(self):

		# Using seed to make sure it'll
		# generate same weights in every run
		random.seed(1)

		# 3x1 Weight matrix
		self.weight_matrix = 2 * random.random((3, 1)) - 1

	# tanh as activation fucntion
	def tanh(self, x):
		return tanh(x)

	# derivative of tanh function.
	# Needed to calculate the gradients.
	def tanh_derivative(self, x):
		return 1.0 - tanh(x) ** 2

	# forward propagation
	def forward_propagation(self, inputs):
		return self.tanh(dot(inputs, self.weight_matrix))

	# training the neural network.
	def train(self, train_inputs, train_outputs,
							num_train_iterations):

		# Number of iterations we want to
		# perform for this set of input.
		for iteration in range(num_train_iterations):
			output = self.forward_propagation(train_inputs)

			# Calculate the error in the output.
			error = train_outputs - output

			# multiply the error by input and then
			# by gradient of tanh funtion to calculate
			# the adjustment needs to be made in weights
			adjustment = dot(train_inputs.T, error *
							self.tanh_derivative(output))

			# Adjust the weight matrix
			self.weight_matrix += adjustment

# Driver Code
if __name__ == "__main__":

	neural_network = NeuralNetwork()

	print ('Random weights at the start of training')
	print (neural_network.weight_matrix)

	train_inputs = array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 1]])
	train_outputs = array([[0, 1, 1, 0]]).T

	neural_network.train(train_inputs, train_outputs, 10000)

	print ('New weights after training')
	print (neural_network.weight_matrix)

	# Test the neural network with a new situation.
	print ("Testing network on new examples ->")
	print (neural_network.forward_propagation(array([1, 0, 0])))

reason: from numpy