text:
'''
This module is designed for testing the module "autoencoder"
'''
# %% Gernerate dataset
'''
The data generating process is
                r(i, t) = beta[z(i, t-1)] * f(t) + u(i, t).
The expectation of factor risk premium f(t) is supposed to be larger than 0,
and its volatility is supposed to be larger than that of z(~).
The characteristic of firm z(i, t-1) is supposed to be stationary and less volatile.
The expectation of the residue u(i, t) is zero,
and its volatility is prportional to the length of the horizon.
The beta is the linear function of the z(i, t-1):
                beta[z(i, t-1)] = z(i, t-1)*c(i) + v(i, t-1)
Following the idea from the RFS, the distributions of parameters are
                f(t) ~ Gaussian(0.1, 0.40)
                z(i, t-1) ~ Uniform(-0.1, 0.1)
                u(i,t) ~ Gaussian(0.0, 0.10)
                c(i) ~ Gaussian(1, 0.50)
                v(i, t-1) ~ Gaussian(0.0, 0.10)
The factor number is 10
The characteristic is 30
The data length is 12 months * 30 = 360
The data size ranges from 500 ~ 6000, including 500, 1000, 3000, 4500, 6000.
The mapping table
sample number  1     2     3     4     5
size           500   1000  3000  4500  6000
shape          *360  *360  *360  *360  *360
'''
import numpy as np

def data_generate(dsize, dshape=360):
    ret = np.zeros(shape=(dsize, dshape))
    c = np.random.normal(loc=1.0, scale=0.5, size=(30, 10))
    factors = np.zeros(shape=(dshape, 10, 1))
    characters = np.zeros(shape=(dshape, dsize, 30))
    for i in range(dshape):
        f = np.random.normal(loc=0.1, scale=0.4, size=(10, 1))
        z = np.random.uniform(low=-0.1, high=0.1, size=(dsize, 30))
        v = np.random.normal(loc=0.0, scale=0.1, size=(dsize, 10))
        u = np.random.normal(loc=0.0, scale=0.1, size=(1,1))

        beta = z.dot(c) + v
        r = beta.dot(f) + u

        factors[i] = f
        characters[i] = z
        ret[:, i] = np.reshape(r, len(r))

    return ret, factors, characters

sample1 = data_generate(500)
sample2 = data_generate(1000)
sample3 = data_generate(3000)
sample4 = data_generate(4500)
sample5 = data_generate(6000)

# %% Train complex model
import sys,os
sys.path.append(os.path.abspath(".."))
from autoencoder import combined_model
import tensorflow as tf

model = combined_model()

loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.SGD()

train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.MeanSquaredError(name='train_accuracy')

@tf.function
def train_step(factors, characters, ret):
    with tf.GradientTape() as tape:
        predictions = model(N_character=characters, factor=factors, K = 10)
        loss = loss_object(ret, predictions)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))

    train_loss(loss)
    train_accuracy(ret, predictions)

EPOCHS = 5

for epoch in range(EPOCHS):
    train_loss.reset_states()
    train_accuracy.reset_states()

    for i in range(360):
        train_step(sample1[2][i], sample1[1][i], sample1[0][:, i])

    print(
        f'Epoch {epoch + 1}, '
        f'Loss: {train_loss.result()}, '
        f'Accuracy: {train_accuracy.result() * 100}, '
        f'Test Loss: {train_loss.result()}, '
        f'Test Accuracy: {train_accuracy.result() * 100}'
    )

# %% Train simple model
import sys,os
sys.path.append(os.path.abspath(".."))
from autoencoder import simple_model
import tensorflow as tf

model = simple_model()

loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.SGD()

train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.MeanSquaredError(name='train_accuracy')

@tf.function
def train_step(factors, characters, ret):
    with tf.GradientTape() as tape:
        predictions = model(N_character=characters, factor=factors, K = 10)
        loss = loss_object(ret, predictions)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))

    train_loss(loss)
    train_accuracy(ret, predictions)

EPOCHS = 5

for epoch in range(EPOCHS):
    train_loss.reset_states()
    train_accuracy.reset_states()

    for i in range(360):
        train_step(sample1[2][i], sample1[1][i], sample1[0][:, i])

    print(
        f'Epoch {epoch + 1}, '
        f'Loss: {train_loss.result()}, '
        f'Accuracy: {train_accuracy.result() * 100}, '
        f'Test Loss: {train_loss.result()}, '
        f'Test Accuracy: {train_accuracy.result() * 100}'
    )

# %%

max_stars_repo_name: whyecofiliter/EAP

max_issues_repo_name: whyecofiliter/EAP

max_forks_repo_name: whyecofiliter/EAP


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributions import MultivariateNormal
import numpy as np
from numpy_ringbuffer import RingBuffer

from rlworldclient import RlWorldClient
import time
import os
import datetime as dt
import random
import math
import pickle

from tqdm.auto import tqdm

from torch.utils.tensorboard import SummaryWriter

id = random.randint(0,1000)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# device = torch.device("cpu")
print("device:", device)

class Memory:
	def __init__(self):
		self.actions = []
		self.states = []
		self.logprobs = []
		self.rewards = []
		self.is_terminals = []

	def clear_memory(self):
		del self.actions[:]
		del self.states[:]
		del self.logprobs[:]
		del self.rewards[:]
		del self.is_terminals[:]

	def save_memory(self):
		time_id = math.floor(time.time())
		# store what we're deleting
		with open(f"./data/{time_id}__{id}.pkl", 'wb') as f:
			pickle.dump({
				"actions": self.actions[:],
				"states": self.states[:],
				"logprobs": self.logprobs[:],
				"rewards": self.rewards[:],
				"is_terminals": self.is_terminals[:],
				}, f)

	def find_experiences(self, seconds_ago=15):
		now = dt.datetime.now()
		ago = now-dt.timedelta(seconds=seconds_ago)

		found_experiences = 0

		try:
			for root, dirs, files in os.walk('./data/'):
				for fname in files:
					path = os.path.join(root, fname)
					st = os.stat(path)
					mtime = dt.datetime.fromtimestamp(st.st_mtime)

					filename = path.split('/')[-1]
					filename_ext = filename[-3:]
					filename_id = filename.split('__')[-1][:3]

					if mtime > ago:
						# if it's not our experience
						if filename_id != str(id):
							self.load_memory(path)
							found_experiences += 1

					else:
						# cleaning up our own mess
						if mtime > now-dt.timedelta(seconds=seconds_ago*8) and filename_id == str(id):
							print("deleting:", filename, filename_ext, filename_id)
							os.remove(path)
		except Exception as e:
			print("caught exception:", e)

		return found_experiences

	def load_memory(self, path):
		print("loading experience:", path)
		new_experiences = pickle.load(open(path, "rb"))

		self.actions.extend(new_experiences['actions'])
		self.states.extend(new_experiences['states'])
		self.logprobs.extend(new_experiences['logprobs'])
		self.rewards.extend(new_experiences['rewards'])
		self.is_terminals.extend(new_experiences['is_terminals'])

class ActorCritic(nn.Module):
	def __init__(self, state_dim, action_dim, action_std):
		super(ActorCritic, self).__init__()
		# action mean range -1 to 1
		self.hidden_size = 64
		self.actor =  nn.Sequential(
				nn.Linear(state_dim, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, action_dim),
				nn.Tanh(),
				)
		# critic
		self.critic = nn.Sequential(
				nn.Linear(state_dim, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, self.hidden_size),
				nn.ReLU(),
				nn.Linear(self.hidden_size, 1)
				)
		self.action_var = torch.full((action_dim,), action_std*action_std).to(device)

	def forward(self):
		raise NotImplementedError

	def act(self, state, memory):
		action_mean = self.actor(state)
		cov_mat = torch.diag(self.action_var).to(device)

		dist = MultivariateNormal(action_mean, cov_mat)
		action = dist.sample()
		action_logprob = dist.log_prob(action)

		memory.states.append(state)
		memory.actions.append(action)
		memory.logprobs.append(action_logprob)

		return action.detach()

	def evaluate(self, state, action):
		action_mean = self.actor(state)

		action_var = self.action_var.expand_as(action_mean)
		cov_mat = torch.diag_embed(action_var).to(device)

		dist = MultivariateNormal(action_mean, cov_mat)

		action_logprobs = dist.log_prob(action)
		dist_entropy = dist.entropy()
		state_value = self.critic(state)

		return action_logprobs, torch.squeeze(state_value), dist_entropy

class PPO:
	def __init__(self, state_dim, action_dim, action_std, lr, betas, gamma, K_epochs, eps_clip):
		self.lr = lr
		self.betas = betas
		self.gamma = gamma
		self.eps_clip = eps_clip
		self.K_epochs = K_epochs

		self.policy = ActorCritic(state_dim, action_dim, action_std).to(device)
		self.optimizer = torch.optim.Adam(self.policy.parameters(), lr=lr, betas=betas)

		self.policy_old = ActorCritic(state_dim, action_dim, action_std).to(device)
		self.policy_old.load_state_dict(self.policy.state_dict())

		self.MseLoss = nn.MSELoss()

	def select_action(self, state, memory):
		state = torch.FloatTensor(state.reshape(1, -1)).to(device)
		return self.policy_old.act(state, memory).cpu().data.numpy().flatten()

	def update(self, memory):
		# Monte Carlo estimate of rewards:
		print("Updating networks...")
		rewards = []
		discounted_reward = 0
		for reward, is_terminal in zip(reversed(memory.rewards), reversed(memory.is_terminals)):
			if is_terminal:
				discounted_reward = 0
			discounted_reward = reward + (self.gamma * discounted_reward)
			rewards.insert(0, discounted_reward)

		# Normalizing the rewards:
		rewards = torch.tensor(rewards).to(device)
		rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-5)

		# convert list to tensor
		old_states = torch.squeeze(torch.stack(memory.states).to(device), 1).detach()
		old_actions = torch.squeeze(torch.stack(memory.actions).to(device), 1).detach()
		old_logprobs = torch.squeeze(torch.stack(memory.logprobs), 1).to(device).detach()

		min_size = min(len(rewards), len(old_states), len(old_actions), len(old_logprobs))

		rewards = rewards[:min_size]
		old_states = old_states[:min_size]
		old_actions = old_actions[:min_size]
		old_logprobs = old_logprobs[:min_size]

		tmp_loss = 0
		# Optimize policy for K epochs:
		for _ in range(self.K_epochs):
			# Evaluating old actions and values :
			logprobs, state_values, dist_entropy = self.policy.evaluate(old_states, old_actions)

			# Finding the ratio (pi_theta / pi_theta__old):
			ratios = torch.exp(logprobs - old_logprobs.detach())

			# Finding Surrogate Loss:
			advantages = rewards - state_values
			surr1 = ratios * advantages
			surr2 = torch.clamp(ratios, 1-self.eps_clip, 1+self.eps_clip) * advantages
			loss = -torch.min(surr1, surr2) + 0.5*self.MseLoss(state_values, rewards) - 0.01*dist_entropy

			# take gradient step
			self.optimizer.zero_grad()
			loss.mean().backward()
			tmp_loss += loss.mean()
			self.optimizer.step()

		# Copy new weights into old policy:
		# print("updating policy, loss:", loss/self.K_epochs)
		self.policy_old.load_state_dict(self.policy.state_dict())

		return tmp_loss / self.K_epochs

def main():
	############## Hyperparameters ##############
	env_name = f"{math.floor(time.time())}__id_{id}"
	writer = SummaryWriter("./logs/"+env_name)
	tick_time = 0.20
	log_interval = 2           # print avg reward in the interval
	max_episodes = 10000        # max training episodes
	max_timesteps = int(15 * (1/tick_time))        # max actions in one episode

	update_timestep = max_timesteps * 2 # update policy every n timesteps
	action_std = 0.5            # constant std for action distribution (Multivariate Normal)
	K_epochs = 10               # update policy for K epochs
	eps_clip = 0.2              # clip parameter for PPO
	gamma = 0.99                # discount factor

	lr = 0.0003                 # parameters for Adam optimizerd
	betas = (0.9, 0.999)

	random_seed = None
	#############################################

	# creating environment
	num_captures = 3
	num_tanks = 15
	action_dim = 3
	state_dim = (num_tanks*2)*(num_captures)+(action_dim*num_captures) # 15 tanks with (x,y) * 3 captures

	print(
		"state_dim:", state_dim, "\n",
		"action_dim:", action_dim, "\n",
		"max_timesteps:", max_timesteps, "\n",
		"update_timestep:", update_timestep, "\n",
	)

	if random_seed:
		print("Random Seed: {}".format(random_seed))
		torch.manual_seed(random_seed)
		env.seed(random_seed)
		np.random.seed(random_seed)

	memory = Memory()
	ppo = PPO(state_dim, action_dim, action_std, lr, betas, gamma, K_epochs, eps_clip)
	print(lr,betas)

	# logging variables
	running_reward = 0
	avg_length = 0
	time_step = 0

	def get_dist(x: float, y: float) -> float:
		return math.sqrt(math.pow(x, 2) + math.pow(y, 2))

	def parse_distances(arr: tuple):
		return get_dist(arr[0], arr[1])

	def parse_observations(observations: dict, old_obs: dict) -> (list, int, bool):
		"""
		parse observation dict into desirable data structures
		"""

		# print(observations)

		if not observations:
			# print("no observations.. there is no one around me?")
			observations = {
				'radarScan': 	[],
			}
			observations.update(old_obs)

		# grab the 15 closest tanks
		tank_locations = [ (float(tank_dic['x']), float(tank_dic['y']) ) for tank_dic in observations['radarScan']]
		tank_locations = sorted(tank_locations, key=parse_distances)
		# print([(parse_distances(dis), 1/parse_distances(dis)) for dis in tank_locations])

		ret_obs = {
			'killCount':	observations['killCount'],
			'hitCount':		observations['hitCount'],
			'deathCount':	observations['deathCount']
		}

		# List of the closest tank coordinates
		ret_state = np.zeros((num_tanks,2))

		for i, loc in enumerate(tank_locations):
			ret_state[i] = loc

		return ret_state, ret_obs, True if observations['deathCount'] >= 3 else False

	# training loop
	for i_episode in tqdm(range(1, max_episodes+1)):
		batch_time_begin = time.time()
		try:
			# env = RlWorldClient("129.127.147.237", 1337)
			env = RlWorldClient("10.90.159.11", 1337)
			state = RingBuffer(capacity=state_dim, dtype=np.float32)
			# instantiate buffer with all zeros
			state.extend(np.zeros((state_dim)))

			global_obs = {
				'killCount':	0,
				'hitCount': 	0,
				'deathCount':	0,
			}
			obs = global_obs
			for t in range(max_timesteps):
				time_step += 1
				start_time = time.time()

				state_current, obs, done = parse_observations(env.read_observation_dict(), obs)
				state.extend(state_current.reshape(-1))

				action = ppo.select_action(np.array(state), memory)
				state.extend(action.reshape(-1))

				# calculate rewards
				mm_clip = lambda x, l, u: max(l, min(u, x))

				# Rewards
				delta_kills = obs['killCount']	- global_obs['killCount']
				delta_hits 	= obs['hitCount']	- global_obs['hitCount']
				delta_death = obs['deathCount']	- global_obs['deathCount']

				# dist_start = (((num_tanks*2)+(action_dim)) * max(num_captures-1, 1))
				dist_start = 0
				dist_step = (num_tanks*2)+action_dim
				dist_end = dist_step*num_captures

				# tmp_distances = [get_dist(x_cord, y_cord) for x_cord, y_cord in zip(state[dist_start:dist_end:dist_step], state[dist_start+1:dist_end:dist_step])]
				# reward_velocity_to_closest = mm_clip(sum(np.diff(tmp_distances))/(tick_time*num_captures), 0, 5)
				# reward_smallest_distance = sum(mm_clip((-1/10)*dist+1, 0, 1) if dist>0 else 0 for dist in tmp_distances)
				# print('looking at:',state[dist_start:dist_end:dist_step])
				# print('dist_diff:', np.diff(tmp_distances))
				# print('tmp_distances:', tmp_distances)
				# print('reward_velocity_to_closest:', reward_velocity_to_closest)
				# print('reward_smallest_distance:', [mm_clip((-1/10)*dist+1, 0, 1) if dist>0 else 0 for dist in tmp_distances])

				time_discount = mm_clip((-1/(max_timesteps*1.1)) * t + 1, 0, 1)

				reward = [
					5	 	* delta_kills * time_discount,
					1    	* delta_hits * time_discount,
				]

				global_obs['killCount'] += delta_kills
				global_obs['hitCount'] += delta_hits
				global_obs['deathCount'] += delta_death

				# action = [mm_clip(float(act), -1, 1) for act in action]
				action = [float(act) for act in action]
				# print(
				# 	"action:", action, "\n",
				# 	"state:", state, "\n",
				# 	"global_obs:", global_obs, "\n",
				# 	"done:", done, "\n",
				# 	"reward:", reward, '\t', sum(reward), "\n",
				# )

				# Apply action to dict
				action_dict = {
					"name": f"swarm_{(i_episode/(max_episodes+1))*100:.1f}%_id:{id}",
					"colour": "#7017a1",
					"moveForwardBack": 	action[0],
					"moveRightLeft": 	action[1],
					"turnRightLeft": 	action[2],
					"fire": True,
				}

				env.send_action_dict(action_dict)

				# Saving reward and is_terminals:
				memory.rewards.append(sum(reward))
				memory.is_terminals.append(done)

				running_reward += sum(reward)

				if done:
					death_count += 1
					break

				avg_length += t

				# sleep
				end_time = time.time()
				time_diff = end_time-start_time
				time.sleep(0 if time_diff > tick_time else tick_time-time_diff)

			writer.add_scalar('rewards/ep_killCount', global_obs['killCount'], i_episode)
			writer.add_scalar('rewards/ep_hitCount', global_obs['hitCount'], i_episode)
			writer.add_scalar('rewards/ep_deathCount', global_obs['deathCount'], i_episode)

			if i_episode % log_interval == 0 and len(memory.states) > 0:
				# load memory from other saved batches
				batch_time_end = time.time()
				batch_time_diff = batch_time_end - batch_time_begin
				# print("batch_time_diff", batch_time_diff)
				memory.save_memory()
				found_count = memory.find_experiences(seconds_ago=batch_time_diff*1.95)
				loss = ppo.update(memory)
				memory.clear_memory()
				time_step = 0

				# logging
				# if i_episode % log_interval == 0:
				avg_length = avg_length//log_interval
				running_reward = running_reward/log_interval

				print(f'Episode {i_episode} \t Avg length: {avg_length} \t Avg reward: {running_reward:.3f}')
				writer.add_scalar('rewards/reward', running_reward, i_episode)
				writer.add_scalar('rewards/avg_length', avg_length, i_episode)
				writer.add_scalar('debug/found_exp', found_count, i_episode)
				writer.add_scalar('debug/loss', -float(loss.sum().detach()), i_episode)
				running_reward = 0
				avg_length = 0
				torch.save(ppo.policy.state_dict(), "./weights/" + env_name + ".pt")

		except Exception as e:
			print("caught exception:", e)
			time.sleep(1)

if __name__ == '__main__':
	main()

# while True:
#     try:
#         client = RlWorldClient("129.127.147.237",1337)
#         while True:
#             # print("read")
#             print(client.read_observation_dict())
#             # print("send")

#             action_dict = {
#                 "name": "Adrian " + str(id),
#                 "colour": "#7017a1",
#                 "moveForwardBack":1.0,
#                 "moveRightLeft": 0.0,
#                 "turnRightLeft": random.uniform(-1.0,1.0),
#                 "fire": True,
#             }

#             client.send_action_dict(action_dict)
#             # print("sleep")
#             time.sleep(0.5)
#     except Exception as e:
#         print(e)
#         time.sleep(1)

ext: py

max_stars_repo_name: AdrianOrenstein/AIML-rl-agent

max_issues_repo_name: AdrianOrenstein/AIML-rl-agent

max_forks_repo_name: AdrianOrenstein/AIML-rl-agent

reason: import numpy,from numpy


text:
import tensorflow as tf
import matplotlib.patches as patches
import matplotlib.pyplot as plt
import numpy as np
from collections import namedtuple

CLASSES = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog',
           'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']

COCO_CLASSES = ['person',
                'bicycle',
                'car',
                'motorcycle',
                'airplane',
                'bus',
                'train',
                'truck',
                'boat',
                'traffic light',
                'fire hydrant',
                'stop sign',
                'parking meter',
                'bench',
                'bird',
                'cat',
                'dog',
                'horse',
                'sheep',
                'cow',
                'elephant',
                'bear',
                'zebra',
                'giraffe',
                'backpack',
                'umbrella',
                'handbag',
                'tie',
                'suitcase',
                'frisbee',
                'skis',
                'snowboard',
                'sports ball',
                'kite',
                'baseball bat',
                'baseball glove',
                'skateboard',
                'surfboard',
                'tennis racket',
                'bottle',
                'wine glass',
                'cup',
                'fork',
                'knife',
                'spoon',
                'bowl',
                'banana',
                'apple',
                'sandwich',
                'orange',
                'broccoli',
                'carrot',
                'hot dog',
                'pizza',
                'donut',
                'cake',
                'chair',
                'couch',
                'potted plant',
                'bed',
                'dining table',
                'toilet',
                'tv',
                'laptop',
                'mouse',
                'remote',
                'keyboard',
                'cell phone',
                'microwave',
                'oven',
                'toaster',
                'sink',
                'refrigerator',
                'book',
                'clock',
                'vase',
                'scissors',
                'teddy bear',
                'hair drier',
                'toothbrush']

def rgb2bgr(tpl):
    return (tpl[2], tpl[1], tpl[0])

Label = namedtuple('Label', ['name', 'color'])

def voc_color_map(index):
    label_defs = [
        Label('aeroplane', rgb2bgr((0, 0, 0))),
        Label('bicycle', rgb2bgr((111, 74, 0))),
        Label('bird', rgb2bgr((81, 0, 81))),
        Label('boat', rgb2bgr((128, 64, 128))),
        Label('bottle', rgb2bgr((244, 35, 232))),
        Label('bus', rgb2bgr((230, 150, 140))),
        Label('car', rgb2bgr((70, 70, 70))),
        Label('cat', rgb2bgr((102, 102, 156))),
        Label('chair', rgb2bgr((190, 153, 153))),
        Label('cow', rgb2bgr((150, 120, 90))),
        Label('diningtable', rgb2bgr((153, 153, 153))),
        Label('dog', rgb2bgr((250, 170, 30))),
        Label('horse', rgb2bgr((220, 220, 0))),
        Label('motorbike', rgb2bgr((107, 142, 35))),
        Label('person', rgb2bgr((52, 151, 52))),
        Label('pottedplant', rgb2bgr((70, 130, 180))),
        Label('sheep', rgb2bgr((220, 20, 60))),
        Label('sofa', rgb2bgr((0, 0, 142))),
        Label('train', rgb2bgr((0, 0, 230))),
        Label('tvmonitor', rgb2bgr((119, 11, 32)))]
    return label_defs[index]

def coco_color_map(index):
    label_defs = [
        Label('aeroplane', rgb2bgr((0, 0, 0))),
        Label('bicycle', rgb2bgr((111, 74, 0))),
        Label('bird', rgb2bgr((81, 0, 81))),
        Label('boat', rgb2bgr((128, 64, 128))),
        Label('bottle', rgb2bgr((244, 35, 232))),
        Label('bus', rgb2bgr((230, 150, 140))),
        Label('car', rgb2bgr((70, 70, 70))),
        Label('cat', rgb2bgr((102, 102, 156))),
        Label('chair', rgb2bgr((190, 153, 153))),
        Label('cow', rgb2bgr((150, 120, 90))),
        Label('diningtable', rgb2bgr((153, 153, 153))),
        Label('dog', rgb2bgr((250, 170, 30))),
        Label('horse', rgb2bgr((220, 220, 0))),
        Label('motorbike', rgb2bgr((107, 142, 35))),
        Label('person', rgb2bgr((52, 151, 52))),
        Label('pottedplant', rgb2bgr((70, 130, 180))),
        Label('sheep', rgb2bgr((220, 20, 60))),
        Label('sofa', rgb2bgr((0, 0, 142))),
        Label('train', rgb2bgr((0, 0, 230))),
        Label('tvmonitor', rgb2bgr((119, 11, 32))),
        Label('aeroplane', rgb2bgr((0, 0, 0))),
        Label('bicycle', rgb2bgr((111, 74, 0))),
        Label('bird', rgb2bgr((81, 0, 81))),
        Label('boat', rgb2bgr((128, 64, 128))),
        Label('bottle', rgb2bgr((244, 35, 232))),
        Label('bus', rgb2bgr((230, 150, 140))),
        Label('car', rgb2bgr((70, 70, 70))),
        Label('cat', rgb2bgr((102, 102, 156))),
        Label('chair', rgb2bgr((190, 153, 153))),
        Label('cow', rgb2bgr((150, 120, 90))),
        Label('diningtable', rgb2bgr((153, 153, 153))),
        Label('dog', rgb2bgr((250, 170, 30))),
        Label('horse', rgb2bgr((220, 220, 0))),
        Label('motorbike', rgb2bgr((107, 142, 35))),
        Label('person', rgb2bgr((52, 151, 52))),
        Label('pottedplant', rgb2bgr((70, 130, 180))),
        Label('sheep', rgb2bgr((220, 20, 60))),
        Label('sofa', rgb2bgr((0, 0, 142))),
        Label('train', rgb2bgr((0, 0, 230))),
        Label('tvmonitor', rgb2bgr((119, 11, 32))),
        Label('aeroplane', rgb2bgr((0, 0, 0))),
        Label('bicycle', rgb2bgr((111, 74, 0))),
        Label('bird', rgb2bgr((81, 0, 81))),
        Label('boat', rgb2bgr((128, 64, 128))),
        Label('bottle', rgb2bgr((244, 35, 232))),
        Label('bus', rgb2bgr((230, 150, 140))),
        Label('car', rgb2bgr((70, 70, 70))),
        Label('cat', rgb2bgr((102, 102, 156))),
        Label('chair', rgb2bgr((190, 153, 153))),
        Label('cow', rgb2bgr((150, 120, 90))),
        Label('diningtable', rgb2bgr((153, 153, 153))),
        Label('dog', rgb2bgr((250, 170, 30))),
        Label('horse', rgb2bgr((220, 220, 0))),
        Label('motorbike', rgb2bgr((107, 142, 35))),
        Label('person', rgb2bgr((52, 151, 52))),
        Label('pottedplant', rgb2bgr((70, 130, 180))),
        Label('sheep', rgb2bgr((220, 20, 60))),
        Label('sofa', rgb2bgr((0, 0, 142))),
        Label('train', rgb2bgr((0, 0, 230))),
        Label('tvmonitor', rgb2bgr((119, 11, 32))),
        Label('aeroplane', rgb2bgr((0, 0, 0))),
        Label('bicycle', rgb2bgr((111, 74, 0))),
        Label('bird', rgb2bgr((81, 0, 81))),
        Label('boat', rgb2bgr((128, 64, 128))),
        Label('bottle', rgb2bgr((244, 35, 232))),
        Label('bus', rgb2bgr((230, 150, 140))),
        Label('car', rgb2bgr((70, 70, 70))),
        Label('cat', rgb2bgr((102, 102, 156))),
        Label('chair', rgb2bgr((190, 153, 153))),
        Label('cow', rgb2bgr((150, 120, 90))),
        Label('diningtable', rgb2bgr((153, 153, 153))),
        Label('dog', rgb2bgr((250, 170, 30))),
        Label('horse', rgb2bgr((220, 220, 0))),
        Label('motorbike', rgb2bgr((107, 142, 35))),
        Label('person', rgb2bgr((52, 151, 52))),
        Label('pottedplant', rgb2bgr((70, 130, 180))),
        Label('sheep', rgb2bgr((220, 20, 60))),
        Label('sofa', rgb2bgr((0, 0, 142))),
        Label('train', rgb2bgr((0, 0, 230))),
        Label('tvmonitor', rgb2bgr((119, 11, 32)))
    ]

    return label_defs[index]

@tf.function
def convert_locations_to_boxes(locations, priors, center_variance,
                               size_variance):
    """네트워크의 회귀 위치 결과를 (center_x, center_y, h, w) 형식의 box로 변환하는 과정
     변환 :
         $$ predicted :_center * center_variance = frac {real_center - prior_center} {prior_hw}$$
         $$ exp (예측_hw * size_variance) = frac {real_hw} {prior_hw} $$

     Args :
         locations (batch_size, num_priors, 4) : 네트워크의 회귀 출력. 출력도 포함
         priors (num_priors, 4) 또는 (batch_size / 1, num_priors, 4) : priors box
         center_variance : 중심 스케일을 변경 상수
         size_variance : 크기 스케일 변경 상수
     Returns:
         bbox : priors : [[center_x, center_y, h, w]]
             이미지 크기에 상대적입니다.
     """
    if tf.rank(priors) + 1 == tf.rank(locations):
        priors = tf.expand_dims(priors, 0)
    return tf.concat([
        locations[..., :2] * center_variance * priors[..., 2:] + priors[..., :2],
        tf.math.exp(locations[..., 2:] * size_variance) * priors[..., 2:]
    ], axis=tf.rank(locations) - 1)

@tf.function
def convert_boxes_to_locations(center_form_boxes, center_form_priors, center_variance, size_variance):
    if tf.rank(center_form_priors) + 1 == tf.rank(center_form_boxes):
        center_form_priors = tf.expand_dims(center_form_priors, 0)

    return tf.concat([
        (center_form_boxes[..., :2] - center_form_priors[..., :2]) / center_form_priors[..., 2:] / center_variance,
        tf.math.log(center_form_boxes[..., 2:] / center_form_priors[..., 2:]) / size_variance
    ], axis=tf.rank(center_form_boxes) - 1)

# experimental_relax_shapes=True인 경우 인스턴스 객체를 생성할 때 작은 graph를 생성할 수 있게 함 (XLA 타입으로 컴파일)
# 참고 자료 : https://www.tensorflow.org/xla?hl=ko
@tf.function(experimental_relax_shapes=True)
def area_of(left_top, right_bottom):
    """bbox 좌표값 (좌상단, 우하단)으로 사각형 넓이 계산.
    Args:
        left_top (N, 2): left 좌상단 좌표값.
        right_bottom (N, 2): 우하단 좌표값.
    Returns:
        area (N): 사각형 넓이.
    """

    hw = tf.clip_by_value(right_bottom - left_top, 0.0, 10000)
    return hw[..., 0] * hw[..., 1]

@tf.function(experimental_relax_shapes=True)
def iou_of(boxes0, boxes1, eps=1e-5):
    """두 bbox간 iou 계산.
    Args:
        boxes0 (N, 4): ground truth boxes 좌표값.
        boxes1 (N or 1, 4): predicted boxes 좌표값.
        eps: 0으로 치환되는 것을 막기위한 엡실론 상수값 .
    Returns:
        iou (N): IoU 값.
    """
    overlap_left_top = tf.maximum(boxes0[..., :2], boxes1[..., :2])
    overlap_right_bottom = tf.minimum(boxes0[..., 2:], boxes1[..., 2:])

    overlap_area = area_of(overlap_left_top, overlap_right_bottom)
    area0 = area_of(boxes0[..., :2], boxes0[..., 2:])
    area1 = area_of(boxes1[..., :2], boxes1[..., 2:])
    return overlap_area / (area0 + area1 - overlap_area + eps)

@tf.function
def center_form_to_corner_form(locations):
    output = tf.concat([locations[..., :2] - locations[..., 2:] / 2,
                        locations[..., :2] + locations[..., 2:] / 2], tf.rank(locations) - 1)

    return output

@tf.function
def corner_form_to_center_form(boxes):
    return tf.concat([
        (boxes[..., :2] + boxes[..., 2:]) / 2,
        boxes[..., 2:] - boxes[..., :2]
    ], tf.rank(boxes) - 1)

reason: import numpy


text:
"""
generate_test_spectrum.py

Script to generate a spectrum for testing PySpecTools.

This uses the `pyspectools.fast.gaussian` to generate Gaussian
peaks at uniform random locations. The amplitudes are sampled
from a normal distribution.

Every peak is grouped together using random integers; these
will be used later to test the `assignment` module mechanisms.
"""

from pyspectools.lineshapes import gaussian
import numpy as np
import pandas as pd
import yaml

SEED = 42
NPEAKS = 200
NGROUPS = 10
RESOLUTION = 0.02
FREQ_RANGE = (8000.0, 19000.0)

# Set a specific random seed to get consistent results
_ = np.random.seed(SEED)

# Gaussian parameters
AMPS = np.random.normal(loc=5.0, scale=2.0, size=(NPEAKS))
CENTERS = np.random.uniform(
    low=min(FREQ_RANGE) + 50.0, high=max(FREQ_RANGE) - 50.0, size=(NPEAKS)
)
# 50 kHz sigma
WIDTH = 0.05
# This generates encodings for each of the peaks as fake assignments
ASSIGNMENTS = np.random.randint(low=0, high=NGROUPS, size=(NPEAKS))

X = np.arange(min(FREQ_RANGE), max(FREQ_RANGE), RESOLUTION)
Y = np.random.normal(loc=0.0, scale=0.2, size=(X.size))
Y = np.abs(Y)

# Loop over each of parameters, and generate a Gaussian
for A, x0 in zip(AMPS, CENTERS):
    Y += gaussian(X, A, x0, WIDTH)

# Save the spectrum for tests
spectral_df = pd.DataFrame({"Frequency": X, "Intensity": Y})
spectral_df.to_csv("test-spectrum.csv", index=False)

# Package up the parameters used to generate the spectrum
# for testing
param_dict = {
    "NPEAKS": NPEAKS,
    "RESOLUTION": RESOLUTION,
    "FREQ_RANGE": FREQ_RANGE,
    "CENTERS": np.round(CENTERS, 4).tolist(),
    "AMPS": np.round(AMPS, 4).tolist(),
    "ASSIGNMENTS": ASSIGNMENTS.tolist(),
    "WIDTH": WIDTH,
}
with open("spectrum-parameters.yml", "w+") as write_file:
    yaml.dump(param_dict, write_file)

max_stars_repo_name: aowen-uwmad/PySpecTools

max_issues_repo_name: aowen-uwmad/PySpecTools

max_forks_repo_name: aowen-uwmad/PySpecTools


text:
from .pipeline2d import Pipeline2D
from . import fillareautils

import numpy
import vcs
import vtk

from .. import vcs2vtk

class BoxfillPipeline(Pipeline2D):

    """Implementation of the Pipeline interface for VCS boxfill plots.

    Internal variables:
        - self._contourLabels: Contour labels.
        - self._mappers: Mappers produced by this pipeline.
            TODO _mappers should be removed and replaced by a more specific
            set of ivars (at minimum, identify what the mappers are rendering).
    """

    def __init__(self, gm, context_, plot_keyargs):
        super(BoxfillPipeline, self).__init__(gm, context_, plot_keyargs)

        self._contourLabels = None
        self._mappers = None
        self._customBoxfillArgs = {}
        self._needsCellData = True

    def _updateScalarData(self):
        """Overrides baseclass implementation."""
        # Update data1 if this is a log10 boxfill:
        data = self._originalData1.clone()
        X = self.convertAxis(data.getAxis(-1), "x")
        Y = self.convertAxis(data.getAxis(-2), "y")
        data.setAxis(-1, X)
        data.setAxis(-2, Y)
        if self._gm.boxfill_type == "log10":
            data = numpy.ma.log10(data)

        self._data1 = self._context().trimData2D(data)
        self._data2 = self._context().trimData2D(self._originalData2)

    def _updateContourLevelsAndColors(self):
        """Overrides baseclass implementation."""
        if self._gm.boxfill_type != "custom":
            self._updateContourLevelsAndColorsForBoxfill()
        else:
            self._updateContourLevelsAndColorsGeneric()

        if isinstance(self._contourLevels, numpy.ndarray):
            self._contourLevels = self._contourLevels.tolist()

    def _updateContourLevelsAndColorsForBoxfill(self):
        """Set contour information for a standard boxfill."""
        self._contourLevels = self._gm.getlevels(self._scalarRange[0], self._scalarRange[1])
        self._contourLabels = self._gm.getlegendlabels(self._contourLevels)
        # Use consecutive colors:
        self._contourColors = list(range(self._gm.color_1, self._gm.color_2 + 1))

    def _plotInternal(self):
        """Overrides baseclass implementation."""
        # Special case for custom boxfills:
        if self._gm.boxfill_type != "custom":
            self._plotInternalBoxfill()
        else:
            self._plotInternalCustomBoxfill()

        if self._maskedDataMapper is not None:
            self._mappers.insert(0, self._maskedDataMapper)

        plotting_dataset_bounds = self.getPlottingBounds()
        x1, x2, y1, y2 = plotting_dataset_bounds

        if self._vtkGeoTransform:
            x1 = self._vtkDataSetBoundsNoMask[0]
            x2 = self._vtkDataSetBoundsNoMask[1]
            y1 = self._vtkDataSetBoundsNoMask[2]
            y2 = self._vtkDataSetBoundsNoMask[3]
            drawAreaBounds = vcs2vtk.computeDrawAreaBounds([x1, x2, y1, y2], self._context_flipX, self._context_flipY)
        else:
            drawAreaBounds = vcs2vtk.computeDrawAreaBounds([x1, x2, y1, y2])

        # And now we need actors to actually render this thing
        actors = []
        cti = 0
        ctj = 0
        _colorMap = self.getColorMap()
        _style = self._gm.fillareastyle
        vp = self._resultDict.get(
            'ratio_autot_viewport',
            [self._template.data.x1, self._template.data.x2,
             self._template.data.y1, self._template.data.y2])

        fareapixelspacing, fareapixelscale = self._patternSpacingAndScale()

        # view and interactive area
        view = self._context().contextView
        area = vtk.vtkInteractiveArea()
        view.GetScene().AddItem(area)

        [renWinWidth, renWinHeight] = self._context().renWin.GetSize()
        # vp = vcs2vtk.adjustBounds(vp, 0.9, 0.9)
        geom = vtk.vtkRecti(int(round(vp[0] * renWinWidth)),
                            int(round(vp[2] * renWinHeight)),
                            int(round((vp[1] - vp[0]) * renWinWidth)),
                            int(round((vp[3] - vp[2]) * renWinHeight)))

        vcs2vtk.configureContextArea(area, drawAreaBounds, geom)

        midx = 0

        for mapper in self._mappers:
            act = vtk.vtkActor()
            act.SetMapper(mapper)
            mapper.Update()
            poly = mapper.GetInput()

            if _style == "solid":
                if self._needsCellData:
                    attrs = poly.GetCellData()
                else:
                    attrs = poly.GetPointData()

                data = attrs.GetScalars()
                deleteColors = False

                if data:
                    lut = mapper.GetLookupTable()
                    range = mapper.GetScalarRange()
                    lut.SetRange(range)
                    mappedColors = lut.MapScalars(data, vtk.VTK_COLOR_MODE_DEFAULT, 0)
                    deleteColors = True
                else:
                    loc = 'point'
                    numTuples = poly.GetNumberOfPoints()
                    if self._needsCellData:
                        loc = 'cell'
                        numTuples = poly.GetNumberOfCells()
                    msg = 'WARNING: boxfill pipeline: poly does not have Scalars'
                    msg = msg + 'array on {0} data, using solid color'.format(loc)
                    print(msg)
                    color = [0, 0, 0, 255]
                    mappedColors = vcs2vtk.generateSolidColorArray(numTuples, color)

                mappedColors.SetName('Colors')

                item = vtk.vtkPolyDataItem()
                item.SetPolyData(poly)

                if self._needsCellData:
                    item.SetScalarMode(vtk.VTK_SCALAR_MODE_USE_CELL_DATA)
                else:
                    item.SetScalarMode(vtk.VTK_SCALAR_MODE_USE_POINT_DATA)

                item.SetMappedColors(mappedColors)
                if deleteColors:
                    mappedColors.FastDelete()
                area.GetDrawAreaItem().AddItem(item)

                if mapper is self._maskedDataMapper:
                    actors.append([item, self._maskedDataMapper, plotting_dataset_bounds])
                else:
                    actors.append([item, plotting_dataset_bounds])

            if mapper is not self._maskedDataMapper:
                if self._gm.boxfill_type == "custom":
                    # Patterns/hatches creation for custom boxfill plots
                    patact = None

                    tmpColors = self._customBoxfillArgs["tmpColors"]
                    if ctj >= len(tmpColors[cti]):
                        ctj = 0
                        cti += 1
                    # Since pattern creation requires a single color, assuming the first
                    c = self.getColorIndexOrRGBA(_colorMap, tmpColors[cti][ctj])

                    patact = fillareautils.make_patterned_polydata(
                        poly,
                        fillareastyle=_style,
                        fillareaindex=self._customBoxfillArgs["tmpIndices"][cti],
                        fillareacolors=c,
                        fillareaopacity=self._customBoxfillArgs["tmpOpacities"][cti],
                        fillareapixelspacing=fareapixelspacing,
                        fillareapixelscale=fareapixelscale,
                        size=self._context().renWin.GetSize(),
                        screenGeom=self._context().renWin.GetSize())

                    ctj += 1

                    if patact is not None:
                        patMapper = patact.GetMapper()
                        patMapper.Update()
                        patPoly = patMapper.GetInput()

                        patItem = vtk.vtkPolyDataItem()
                        patItem.SetPolyData(patPoly)

                        patItem.SetScalarMode(vtk.VTK_SCALAR_MODE_USE_CELL_DATA)
                        colorArray = patPoly.GetCellData().GetArray('Colors')

                        patItem.SetMappedColors(colorArray)
                        area.GetDrawAreaItem().AddItem(patItem)

                        actors.append([patItem, plotting_dataset_bounds])

            midx += 1

        self._resultDict["vtk_backend_actors"] = actors

        z, t = self.getZandT()

        kwargs = {
            "vtk_backend_grid": self._vtkDataSet,
            "dataset_bounds": self._vtkDataSetBounds,
            "plotting_dataset_bounds": plotting_dataset_bounds,
            "vtk_dataset_bounds_no_mask": self._vtkDataSetBoundsNoMask,
            "vtk_backend_geo": self._vtkGeoTransform,
            "vtk_backend_draw_area_bounds": drawAreaBounds,
            "vtk_backend_viewport_scale": [
                self._context_xScale,
                self._context_yScale
            ]
        }
        if ("ratio_autot_viewport" in self._resultDict):
            kwargs["ratio_autot_viewport"] = vp
        self._resultDict.update(self._context().renderTemplate(
            self._template,
            self._data1,
            self._gm, t, z, **kwargs))

        if getattr(self._gm, "legend", None) is not None:
            self._contourLabels = self._gm.legend

        if self._gm.ext_1:
            if isinstance(self._contourLevels[0], list):
                if numpy.less(abs(self._contourLevels[0][0]), 1.e20):
                    # Ok we need to add the ext levels
                    self._contourLevels.insert(
                        0, [-1.e20, self._contourLevels[0][0]])
            else:
                if numpy.less(abs(self._contourLevels[0]), 1.e20):
                    # need to add an ext
                    self._contourLevels.insert(0, -1.e20)
        if self._gm.ext_2:
            if isinstance(self._contourLevels[-1], list):
                if numpy.less(abs(self._contourLevels[-1][1]), 1.e20):
                    # need ext
                    self._contourLevels.append([self._contourLevels[-1][1],
                                                1.e20])
            else:
                if numpy.less(abs(self._contourLevels[-1]), 1.e20):
                    # need exts
                    self._contourLevels.append(1.e20)

        # Do not pass patterning parameters for color bar rendering if the
        # boxfill type is non-custom
        patternArgs = {}
        if self._gm.boxfill_type == "custom":
            patternArgs['style'] = self._gm.fillareastyle
            patternArgs['index'] = self._gm.fillareaindices
            patternArgs['opacity'] = self._gm.fillareaopacity
            # Compensate for the different viewport size of the colorbar
            patternArgs['pixelspacing'] = [int(fareapixelspacing[0] / (vp[1] - vp[0])),
                                           int(fareapixelspacing[1] / (vp[3] - vp[2]))]
            patternArgs['pixelscale'] = fareapixelscale / (vp[1] - vp[0])

        self._resultDict.update(
            self._context().renderColorBar(self._template, self._contourLevels,
                                           self._contourColors,
                                           self._contourLabels,
                                           self.getColorMap(),
                                           **patternArgs))

        projection = vcs.elements["projection"][self._gm.projection]
        kwargs['xaxisconvert'] = self._gm.xaxisconvert
        kwargs['yaxisconvert'] = self._gm.yaxisconvert
        if self._data1.getAxis(-1).isLongitude() and self._data1.getAxis(-2).isLatitude():
            self._context().plotContinents(self._plot_kargs.get("continents", self._useContinents),
                                           plotting_dataset_bounds, projection,
                                           self._dataWrapModulo,
                                           vp, self._template.data.priority, **kwargs)

    def _plotInternalBoxfill(self):
        """Implements the logic to render a non-custom boxfill."""
        # Prep mapper
        mapper = vtk.vtkPolyDataMapper()
        self._mappers = [mapper]

        if self._gm.ext_1 and self._gm.ext_2:
            # mapper.SetInputConnection(self._vtkPolyDataFilter.GetOutputPort())
            mapper.SetInputData(self._vtkDataSetFittedToViewport)
            self._resultDict["vtk_backend_geofilters"] = \
                [self._vtkPolyDataFilter]
        else:
            thr = vtk.vtkThreshold()
            # thr.SetInputConnection(self._vtkPolyDataFilter.GetOutputPort())
            thr.SetInputData(self._vtkDataSetFittedToViewport)
            if not self._gm.ext_1 and not self._gm.ext_2:
                thr.ThresholdBetween(self._contourLevels[0],
                                     self._contourLevels[-1])
            elif self._gm.ext_1 and not self._gm.ext_2:
                thr.ThresholdByLower(self._contourLevels[-1])
            elif not self._gm.ext_1 and self._gm.ext_2:
                thr.ThresholdByUpper(self._contourLevels[0])

            geoFilter2 = vtk.vtkDataSetSurfaceFilter()
            geoFilter2.SetInputConnection(thr.GetOutputPort())
            mapper.SetInputConnection(geoFilter2.GetOutputPort())
            self._resultDict["vtk_backend_geofilters"] = [geoFilter2]

        # Colortable bit
        # make sure length match
        numLevels = len(self._contourLevels) - 1
        while len(self._contourColors) < numLevels:
            self._contourColors.append(self._contourColors[-1])

        lut = vtk.vtkLookupTable()
        lut.SetNumberOfTableValues(numLevels)
        _colorMap = self.getColorMap()
        for i in range(numLevels):
            r, g, b, a = self.getColorIndexOrRGBA(_colorMap, self._contourColors[i])
            lut.SetTableValue(i, r / 100., g / 100., b / 100., a / 100.)

        mapper.SetLookupTable(lut)
        if numpy.allclose(self._contourLevels[0], -1.e20):
            lmn = self._min - 1.
        else:
            lmn = self._contourLevels[0]
        if numpy.allclose(self._contourLevels[-1], 1.e20):
            lmx = self._mx + 1.
        else:
            lmx = self._contourLevels[-1]
        mapper.SetScalarRange(lmn, lmx)
        self._resultDict["vtk_backend_luts"] = [[lut, [lmn, lmx, True]]]

    def _plotInternalCustomBoxfill(self):
        """Implements the logic to render a custom boxfill."""
        self._mappers = []

        self._customBoxfillArgs = self._prepContours()
        tmpLevels = self._customBoxfillArgs["tmpLevels"]
        tmpColors = self._customBoxfillArgs["tmpColors"]
        tmpOpacities = self._customBoxfillArgs["tmpOpacities"]

        style = self._gm.fillareastyle

        luts = []
        geos = []
        wholeDataMin, wholeDataMax = vcs.minmax(self._originalData1)
        _colorMap = self.getColorMap()
        for i, l in enumerate(tmpLevels):
            # Ok here we are trying to group together levels can be, a join
            # will happen if: next set of levels continues where one left off
            # AND pattern is identical

            # TODO this should really just be a single polydata/mapper/actor:
            for j, color in enumerate(tmpColors[i]):
                mapper = vtk.vtkPolyDataMapper()
                lut = vtk.vtkLookupTable()
                th = vtk.vtkThreshold()
                th.ThresholdBetween(l[j], l[j + 1])
                # th.SetInputConnection(self._vtkPolyDataFilter.GetOutputPort())
                th.SetInputData(self._vtkDataSetFittedToViewport)
                geoFilter2 = vtk.vtkDataSetSurfaceFilter()
                geoFilter2.SetInputConnection(th.GetOutputPort())
                # Make the polydata output available here for patterning later
                geoFilter2.Update()
                geos.append(geoFilter2)
                mapper.SetInputConnection(geoFilter2.GetOutputPort())
                lut.SetNumberOfTableValues(1)
                r, g, b, a = self.getColorIndexOrRGBA(_colorMap, color)
                if style == 'solid':
                    tmpOpacity = tmpOpacities[j]
                    if tmpOpacity is None:
                        tmpOpacity = a / 100.
                    else:
                        tmpOpacity = tmpOpacities[j] / 100.
                    lut.SetTableValue(0, r / 100., g / 100., b / 100., tmpOpacity)
                else:
                    lut.SetTableValue(0, 1., 1., 1., 0.)
                mapper.SetLookupTable(lut)
                mapper.SetScalarRange(l[j], l[j + 1])
                luts.append([lut, [l[j], l[j + 1], False]])
                # Store the mapper only if it's worth it?
                # Need to do it with the whole slab min/max for animation
                # purposes
                if not (l[j + 1] < wholeDataMin or l[j] > wholeDataMax):
                    self._mappers.append(mapper)

        self._resultDict["vtk_backend_luts"] = luts
        if len(geos) > 0:
            self._resultDict["vtk_backend_geofilters"] = geos

max_stars_repo_name: scottwittenburg/vcs

max_issues_repo_name: scottwittenburg/vcs

max_forks_repo_name: scottwittenburg/vcs


text:
import sys
import itk
import itk_helpers as Functions
import numpy as np
import dicom_functions as dfun

def drr(inputImage,output_filename,rot,t,focalPoint,originOutput,sizeOutput,cor,spaceOutput,directionOutput,threshold,InputImageType,OutputImageType,verbose):
    #%% Transfomration
    inOrigin     = inputImage.GetOrigin()                   # Get the origin of the image.
    inSpacing    = inputImage.GetSpacing()                  # Get the resolution of the input image.
    inSize       = inputImage.GetBufferedRegion().GetSize() # Get the size of the input image.
    inDirection  = inputImage.GetDirection()

    #%% ------------------ Transformation
    # This part is inevitable since the interpolator (Ray-cast) and resample Image
    # image filter uses a Transformation -- Here we set it to identity.
    TransformType = itk.CenteredEuler3DTransform[itk.D]
    transform     = TransformType.New()

    direction_mat = Functions.get_vnl_matrix(inDirection.GetVnlMatrix())

    rot_mod = np.dot(-1,rot)           # Due to Direction of transform mapping ( 8.3.1 in the ITK manual)
    t_mod   = np.dot(-1,t  )           # Due to Direction of transform mapping ( 8.3.1 in the ITK manual)
    # Since this transform is for the movement of x-ray source and not the rigid body, therefore, no need to invert the rotation.
    #
    #
    rot_mod = direction_mat.dot(np.transpose(rot_mod))
    t_mod   = direction_mat.dot(np.transpose(t_mod  ))

    transform.SetRotation(np.deg2rad(rot_mod[0]),np.deg2rad(rot_mod[1]),np.deg2rad(rot_mod[2])) # Setting the rotation of the transform
    transform.SetTranslation(itk.Vector.D3(t_mod))    # Setting the translation of the transform
    transform.SetComputeZYX(True)  # The order of rotation will be ZYX.

    center = direction_mat.dot(inOrigin)+ np.multiply(inSpacing,inSize)/2. # Setting the center of rotation as center of 3D object + offset determined by cor.
    center = direction_mat.dot(center)-t # Convert the image to the local coordinate system.
    transform.SetCenter(center)                     # Setting the center of rotation.

    if verbose:
        print(transform)

    #%% Raycast interpolator
    ScalarType = itk.D
    InterpolatorType = itk.RayCastInterpolateImageFunction[InputImageType,ScalarType]       # Defining the interpolator type from the template.
    interpolator     = InterpolatorType.New()               # Pointer to the interpolator

    interpolator.SetInputImage(inputImage)                  # Setting the input image data
    interpolator.SetThreshold(threshold)                    # Setting the output threshold
    interpolator.SetFocalPoint(itk.Point.D3(focalPoint))    # Setting the focal point (x-ray source location)
    interpolator.SetTransform(transform)                    # Setting the transform --

    if verbose:
        print(interpolator)
    #%% filtering
    #%----------------- Resample Image Filter ------------------------
        # In this part the resample image filter to map a 3D image to 2D image plane with desired specs is designed

    FilterType = itk.ResampleImageFilter[InputImageType,OutputImageType]                    # Defining the resample image filter type.
    resamplefilter = FilterType.New()               # Pointer to the filter

    resamplefilter.SetInput(inputImage)             # Setting the input image data
    resamplefilter.SetDefaultPixelValue( 0 )      # Setting the default Pixel value
    resamplefilter.SetInterpolator(interpolator)    # Setting the interpolator
    resamplefilter.SetTransform(transform)          # Setting the transform
    resamplefilter.SetSize(sizeOutput)              # Setting the size of the output image.
    resamplefilter.SetOutputSpacing(spaceOutput)    # Setting the spacing(resolution) of the output image.
    resamplefilter.SetOutputOrigin(originOutput)    # Setting the output origin of the image
    Functions.change_image_direction(oldDirection=resamplefilter.GetOutputDirection(),newDirection=directionOutput,DimensionOut=3)     # Setting the output direction of the image  --- resamplefilter.SetImageDirection(args) was not working properly
    resamplefilter.Update()                         # Updating the resample image filter.

    filteringOutput = resamplefilter.GetOutput()

    if verbose:
        print(resamplefilter)
    #%%---------------- Rescaler Image Filter --------------------------
    RescalerFilterType = itk.RescaleIntensityImageFilter[InputImageType,OutputImageType]    # Defining the rescale image filter.
    rescaler = RescalerFilterType.New()             # Pointer to the rescale filter
    rescaler.SetOutputMinimum(0)                    # Minimum output
    rescaler.SetOutputMaximum(255)                  # Maximum output
    rescaler.SetInput(resamplefilter.GetOutput())   # Setting the input to the image filter.

    filteringOutput = rescaler.GetOutput()

    if verbose:
        print(rescaler)

    #%%---------------- Flip Axis filter ------------------------------

    FlipFilterType = itk.FlipImageFilter[OutputImageType]
    flipfilter = FlipFilterType.New()

    flipfilter.SetFlipAxes([0,0,0]) # Flip the axes along x and y but leave z intact.
    flipfilter.SetInput(filteringOutput) # Setting the input to the flip filter.

    filteringOutput = flipfilter.GetOutput()

    if verbose:
        print(flipfilter)

    last_filter_output = filteringOutput

    #%% ------------------ Writer ------------------------------------
    # The output of the resample filter can then be passed to a writer to
    # save the DRR image to a file.

    WriterType = itk.ImageFileWriter[OutputImageType]
    writer = WriterType.New()

    writer.SetFileName(output_filename)
    writer.SetInput(last_filter_output) # set the input as the output of filtering process.

    try:
        print("Writing image: " + output_filename)
        writer.Update()
        print("Image Printed Successfully")
    except ValueError:
        print("ERROR: ExceptionObject cauth! \n")
        print(ValueError)
        sys.exit()

    if verbose:
        print('Details of image: ')
        print(last_filter_output)

max_stars_repo_name: BrysonGray/DRR-Studies

max_issues_repo_name: BTAROCKET/DRR-Studies

max_forks_repo_name: BTAROCKET/DRR-Studies


text:
from typing import Optional
from typing import Tuple
from ..classes.train_data import TrainData

from pytorch_lightning import LightningDataModule
from torch.utils.data import DataLoader, WeightedRandomSampler
from torch.utils.data import Dataset
from torch.utils.data import random_split
import torch

from sklearn.model_selection import StratifiedShuffleSplit
import numpy as np

from ..datasets.binary_image_classification_ds import BinaryImageClassificationDS

class MultilabelImageClassificationDM(LightningDataModule):

    """
    Example of LightningDataModule for MNIST dataset.

    A DataModule implements 5 key methods:
        - prepare_data (things to do on 1 GPU/TPU, not on every GPU/TPU in distributed mode)
        - setup (things to do on every accelerator in distributed mode)
        - train_dataloader (the training dataloader)
        - val_dataloader (the validation dataloader(s))
        - test_dataloader (the test dataloader(s))

    This allows you to share a full dataset without explaining how to download,
    split, transform and process the data

    Read the docs:
        https://pytorch-lightning.readthedocs.io/en/latest/extensions/datamodules.html
    """

    def __init__(
        self,
        train_data: TrainData,
        scaling: int,
        test_size: Tuple[float, float, float] = 0.2,
        batch_size: int = 64,
        num_workers: int = 0,
        pin_memory: bool = False,
        **kwargs,
    ):
        super().__init__()

        self.train_data_object = train_data

        # self.paths = np.array([_.as_posix() for _ in train_data.paths])
        # self.labels = np.array(train_data.labels)
        self.scaling = scaling
        self.test_size = test_size
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.pin_memory = pin_memory
        self.num_classes = train_data.n_classes()

        self.data_train: Optional[Dataset] = None
        self.data_val: Optional[Dataset] = None
        self.data_test: Optional[Dataset] = None

    def prepare_data(self):
        """Download data if needed. This method is called only from a single GPU.
        Do not use it to assign state (self.x = y)."""
        pass

    def setup(self, stage: Optional[str] = None):
        """Load data. Set variables: self.data_train, self.data_val, self.data_test."""

        # sss = StratifiedShuffleSplit(n_splits = 1, test_size = 0.1, random_state = 42)
        # train_indexes, test_indexes = [_ for _ in sss.split(
        #     self.train_data_object.paths,
        #     self.train_data_object.labels
        # )][0]

        # x_train, x_test, y_train, y_test = self.train_data_object.group_strat_split(
        #     test_size=self.test_size)
        labels = np.array(self.train_data_object.labels)
        paths = np.array([_.as_posix() for _ in self.train_data_object.paths])
        is_val = np.array(self.train_data_object.is_val)

        x_train = paths[~is_val]
        x_test = paths[is_val]
        y_train = labels[~is_val]
        y_test = labels[is_val]

        class_sample_count = np.array(
            [len(np.where(y_train == t)[0]) for t in np.unique(y_train)])

        weight = 1. / class_sample_count
        samples_weight = np.array([weight[t] for t in y_train])
        samples_weight = torch.from_numpy(samples_weight)
        samples_weight = samples_weight.double()
        self.train_sampler = WeightedRandomSampler(samples_weight, len(samples_weight))

        y_train = torch.from_numpy(y_train).long()

        self.train_dataset = BinaryImageClassificationDS(
            x_train, y_train, self.scaling
        )

        self.test_dataset = BinaryImageClassificationDS(
            x_test, y_test, self.scaling
        )

    def train_dataloader(self):
        return DataLoader(
            dataset=self.train_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            pin_memory=self.pin_memory,
            sampler = self.train_sampler
        )

    def val_dataloader(self):
        return DataLoader(
            dataset=self.test_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            pin_memory=self.pin_memory,
            shuffle=False,
        )

    # def test_dataloader(self):
    #     return DataLoader(
    #         dataset=self.data_test,
    #         batch_size=self.batch_size,
    #         num_workers=self.num_workers,
    #         pin_memory=self.pin_memory,
    #         shuffle=False,
    #     )

max_stars_repo_name: Maddonix/ukw-ml-tools

max_issues_repo_name: Maddonix/ukw-ml-tools

max_forks_repo_name: Maddonix/ukw-ml-tools


text:
# -*- coding: utf-8 -*-

import unittest

from numpy import int64

from pipesnake.transformers.converter import Category2Number

class TestConverter(unittest.TestCase):
    def test_category_to_number(self):
        from _data import x_str
        from _data import y_str

        cat2num = Category2Number(x_cols=['x_5'], y_cols=['y_3'])
        x_new, y_new = cat2num.fit_transform(x_str, y_str)

        for item in x_new['x_5'].values:
            self.assertEqual(type(item) == int64, True, 'error: {} should be an int got: {}'.format(item, type(item)))

        for item in y_new['y_3'].values:
            self.assertEqual(type(item) == int64, True, 'error: {} should be an int got: {}'.format(item, type(item)))

        x_org, y_org = cat2num.inverse_transform(x_new, y_new)

        for a, b in zip(x_str['x_5'].values, x_org['x_5'].values):
            self.assertEqual(a, b, 'mismatch value {} != {}'.format(a, b))

        for a, b in zip(y_str['y_3'].values, y_org['y_3'].values):
            self.assertEqual(a, b, 'mismatch value {} != {}'.format(a, b))

        del x_str, y_str, x_new, y_new

max_stars_repo_name: pierluigi-failla/pipesnake

max_issues_repo_name: pierluigi-failla/pipesnake

max_forks_repo_name: pierluigi-failla/pipesnake

reason: from numpy


text:
from . import *

__all__ = [
    'yearly_counts_grid','matrix','key2name','yearly_counts_table',
    'author_pub_table'
]

def key2name(tname, truncate=None):
    tstr = tname.split("|")
    if len(tstr) == 3:
        tstr = "%s (%s)" % (tstr[0], tstr[1])
    else:
        tstr = "%s %s" % (tstr[0], tstr[1])
    tstr = tstr.title()

    if truncate is not None:
        if tstr[:truncate] != tstr:
            tstr = tstr[:truncate] + "..."

    return tstr

def _normalize(mat, axis=0):
    if axis==0: # rows
        return mat/mat.sum(axis=1)[:,None]
    elif axis==1: #columns
         return (mat.T/mat.T.sum(axis=1)[:,None]).T
    else:
        raise Exception('axis != 0/1')

# colormaps...
# https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html

def matrix(db, typ='docs', plot=False, norm=None, trans=False, includeComplement=False, **kwargs):

    assert(isinstance(db, Dataset))

    cnt = db.by( *list(kwargs) )

    # by1=range(35,50), by2=range(10,20)
    if len(list(kwargs)) != 2:
        raise Exception("matrix with dim != 2 not supported")

    if typ == 'docs':
        c = cnt.docs
    elif typ == 'cits':
        c = cnt.cits
    else:
        raise Exception("wtf... docs or cits bro")

    key1, key2 = sorted(kwargs)
    by1 = kwargs[key1]
    by2 = kwargs[key2]

    import numpy as np
    from collections import defaultdict

    def mk_array():
        for a1 in by1:
            my_series = np.array([c[(a1, a2)] for a2 in by2])

            if includeComplement:
                out = 0
                for (pp1, pp2), count in c.items():
                    # to account for a temporary bug
                    if pp1 < 0 or pp2 < 0:
                        continue

                    # skip what's already included
                    if pp1 != a1:
                        continue
                    if pp2 in by2:
                        continue

                    out += count

                my_series = np.array(list(my_series)+[out])

            yield my_series

        if includeComplement:
            cLast = defaultdict(int)

            outside_both = 0
            for (pp1, pp2),count in c.items():
                # to account for a temporary bug
                if pp1 < 0 or pp2 < 0:
                    continue

                # skip what's already included
                if pp1 in by1:
                    continue

                if pp2 not in by2:
                    outside_both += count

                #print(pp1, pp2, count)
                cLast[pp2] += count

            cLast = [cLast[x] for x in by2] + [outside_both]
            #print(cLast)
            yield cLast

    mat = np.array(list(mk_array()))

    if norm is not None:
        from . import matrix_normalize
        if norm not in kwargs:
            raise Exception('norm should be the axis along which you want to normalize')

        norm_i = sorted(kwargs).index(norm)
        mat = matrix_normalize(mat, axis=norm_i)

    if trans:
        mat = mat.transpose()

    if plot:
        from . import plt
        fig = plt.figure(figsize=(20, 4))  # plt.subplots()#
        plt.imshow(mat, interpolation='nearest', cmap=plt.cm.hsv)
        # IMS.cmap.set_under('yellow')
        # IMS.cmap.set_over('orange')
        plt.colorbar()

        if trans:
            xtick = by1
            ytick = by2
        else:
            xtick = by2
            ytick = by1

        tick_target = 5

        xstep = max(len(xtick) // tick_target,1)
        ystep = max(len(ytick) // tick_target,1)

        plt.xticks(
            range(0, len(xtick),xstep),
            xtick[::xstep],
        )
        plt.yticks(
            range(0, len(ytick),ystep),
            ytick[::ystep],
        )

        if trans:
            plt.xlabel(key1)
            plt.ylabel(key2)
        else:
            plt.xlabel(key2)
            plt.ylabel(key1)
    else:
        return mat

def yearly_counts_grid(names, dataset, myname=None,
    overwrite=True,
    markers={}, print_names=None,
    ctype='c', yearly_prop=False,
    xlim=None, cols = 5,
    rows_per_group = 2,
        count_unit='doc', label_num='max'):

    assert(type(dataset) == Dataset)

    if xlim is None:
        xlim = (dataset.start, dataset.end)

    eps = (xlim[1]-xlim[0])/10
    xlim_display = (xlim[0]-eps, xlim[1]+eps)

    doing_both = False
    if type(count_unit) != str:
        if set(count_unit) == {'doc','cit'}:
            doing_both = True
        else:
            raise Exception('whoopsie')

    rows = len(names) // cols + bool(len(names) % cols) # = 15 for 5

    groupsize = rows_per_group * cols
    gs = rows // rows_per_group + bool(rows % rows_per_group)

    existing_files = Path(env.figure_dir).glob("%s.*" % myname)
    for fn in existing_files:
        fn.unlink()

    for groupi in range(gs):
        myFnName = "%s.%s" % (myname,groupi)
        if Path(myFnName).exists() and not overwrite:
            continue

        plt.figure(figsize=(cols*4,rows*1.2))
        plt.subplots_adjust(hspace=0.6, wspace=0)

        for i,name in enumerate(names[ groupi*groupsize : (groupi+1)*groupsize]):
            print(name)

            plt.subplot(rows,cols,i+1)

            yrstart,yrend = xlim
            years = range(yrstart, yrend+1)

            if count_unit == 'cit' or 'cit' in count_unit:
                vals = dataset.trend(ctype, name).cits(yrstart, yrend)

                if yearly_prop:
                    all_by_year = [ dataset(fy=YY).cits for YY in years ]
                    vals = np.array(vals) / np.array(all_by_year)

            else:
                vals = dataset.trend(ctype, name).docs(yrstart,yrend)

                if yearly_prop:
                    all_by_year = [ dataset(fy=YY).docs for YY in years ]
                    vals = np.array(vals) / np.array(all_by_year)

            # fill that shit up
            plt.fill_between(years,vals,color='black',alpha=0.4)

            if doing_both:
                vals2 = np.array(dataset.trend(ctype, name).docs())

                if yearly_prop:
                    all_by_year = [ dataset(fy=YY).docs for YY in years ]
                    vals2 = np.array(vals2) / np.array(all_by_year)

                # this one is scaled...
                vals2 = np.max(vals) * vals2 * 0.5 / np.max(vals2)

                plt.plot(years, vals2, color='black')

            if print_names is not None:
                title = print_names(name)
            else:
                title = name

            if len(title) >= 40:
                title = title[:37]+"..."

            #t = plt.text(min(years) + (max(years)-min(years))/2, 1.2*(max(vals))/1,title, fontsize=13)
            plt.title(title)

            plt.axis('off')

            from math import ceil, floor

            XAXIS_YSHIFT = -max(vals)/25
            lines = []

            if False:
                # lines on the x-axis...
                for decade in range(floor(yrstart/10)*10,ceil(yrend/10)*10,10):
                    lines += [
                        (decade+1, decade+10-1),
                        #(-max(vals)/5, -max(vals)/5),
                        #(0,0),
                        (XAXIS_YSHIFT, XAXIS_YSHIFT),
                        'black'
                    ]

            # labeling the last
            if label_num == 'last':
                last_num = vals[-1]

                if yearly_prop:
                    last_nump = "%0.1f%%"%(last_num*100)
                else:
                    last_nump = last_num

                lines += [
                    (xlim_display[1]-10, xlim_display[1]-5),
                    (last_num, last_num),
                    "black"
                ]
                plt.text(xlim_display[1]-3, last_num, last_nump, fontsize=12, verticalalignment='center', horizontalalignment='left')#*3+min(vals)

            elif label_num == 'max':
                #print(vals, title)
                max_num = np.max(vals)

                if yearly_prop:
                    max_nump = "%0.1f%%"%(max_num*100)
                else:
                    max_nump = max_num

                lines += [
                    (xlim_display[1]-10, xlim_display[1]-5),
                    (max_num, max_num),
                    "black"
                ]
                plt.text(xlim_display[1]-3, max_num, max_nump, fontsize=12, verticalalignment='center', horizontalalignment='left')#*3+min(vals)

            elif label_num is not None:
                raise Exception('don\'t recognize that label_num...')

            plt.plot(*lines)

            # dots on the x-axis
            half_centuries = range( ceil(xlim[0]/50)*50, floor(xlim[1]/50)*50 + 50, 50 )
            decades = list(set(range( ceil(xlim[0]/10)*10, floor(xlim[1]/10)*10 + 10, 10 )).difference(set(half_centuries)))
            plt.scatter(
                decades,
                [0]*len(decades),
                color='black',
                s=5
            )

            #print('max',np.max(vals))
            #print(list(vals))
            if True:
                plt.vlines(
                    half_centuries,
                    -np.max(vals)/5,
                    0
                )

            plt.xlim(xlim_display)
            plt.ylim((-np.max(vals)/5,np.max(vals)))

            # put the markers on the time seriesss
            if name in markers:
                ms = markers[ name ]
                if type(ms) == list:
                    #print('here!')
                    plt.scatter(
                        ms,
                        [-max(vals)/10]*len(ms),
                        color='red',
                        s=30,
                        marker="^"
                    )
                elif type(ms) == dict:
                    for k,v in ms.items():
                        plt.text(k, -max(vals)/2, v)

        dataset.save_figure(myFnName)
        plt.show();

def yearly_counts_table_simp(
        dta, who,
        NCOLS=2, NPAGES=None,
        markranges={}, yearlim=(1950,2015),
        tickstep=20,
        print_names={}, number_start=1,
        print_numbers = False):

    wper = 16 / 3
    hper = 14 / 20
    NROWS = (len(who) // NCOLS) + int(len(who) % NCOLS != 0)

    # top_end = (2080-2015) /  (2015-1950)
    # bottom_end = (1950 - 1880) / (2015 - 1950)
    # year_wid = yearlim[1] - yearlim[0]
    yearmin, yearmax = yearlim

    # xmin, xmax = (yearlim[0] - bottom_end * year_wid, yearlim[1] + top_end * year_wid)
    # def dist_trans(nyears):
    #    return year_wid * nyears / (2015-1950)

    plt.figure(figsize=(wper * NCOLS, hper * NROWS))

    for col_i in range(NCOLS):

        plt.subplot(1, NCOLS, col_i + 1)

        trends = []
        for i, tname in enumerate(who[col_i * NROWS: (col_i + 1) * NROWS]):
            tt = np.array(dta.trend('c', tname).cits(yearmin, yearmax))

            # print(yearmin,yearmax)
            trends.append((tname, tt))

        for i, (tname, tt) in enumerate(trends):
            ttm = tt.max()
            ttl = tt[-1]
            ttl15 = tt[-15:].mean()

            BASE = (NROWS - 1 - i) * 2

            tt = tt / ttm
            tt = tt + BASE

            years = range(yearmin, yearmax + 1)

            plt.axis('off')
            plt.fill_between(years, tt, y2=BASE, color='gray')

            if tname in print_names:
                tstr = print_names[tname]
            else:
                tstr = key2name(tname, truncate=25)

            tstr = "(%s) " % ((col_i) * NROWS + i + number_start) * print_numbers + tstr
            plt.text(2020, BASE, tstr, fontsize=13)
            # plt.text(yearmax + dist_trans(2030-2015), BASE, int(ttm), horizontalalignment='center')
            # plt.text(yearmax + dist_trans(2050-2015), BASE, int(ttl), horizontalalignment='center')
            # plt.text(yearmax + dist_trans(2070-2015), BASE, "%0.1f" % ttl15, horizontalalignment='center')
            # plt.text(2025, i*2, int(ttm), horizontalalignment='center')

            if tname in markranges:
                # xmin,xmax = plt.xlim()
                ymin, ymax = [BASE, BASE + 1.4]
                mk = markranges[tname]
                plt.fill_between(mk, [ymin, ymin], [ymax, ymax], color='blue', alpha=0.2)

        plt.vlines(range(yearmin, yearmax, tickstep), 0, NROWS * 2, color='black', alpha=0.3)

        # table lines...
        # plt.vlines([ yearmax + dist_trans(DD-2015) for DD in range(2020, 2080 + 20, 20)], 0, NROWS * 2, color='black', alpha=1)  # table lines

        for y in range(yearmin, yearmax, tickstep):
            plt.text(y, -1, y, horizontalalignment='center')

        plt.xlim(yearmin, yearmax + 50)

def yearly_counts_table(
        dta, who,
        NCOLS=2, NPAGES=None,
        markranges={}, yearlim=(1950,2015),
        tickstep=20,
        print_names={}, number_start=1
):
    if NPAGES is not None:
        nper = len(who) // NPAGES + int( len(who) % NPAGES != 0 )
        for i in range(NPAGES):
            yearly_counts_table(
                dta,
                who[i*nper:(i+1)*nper],
                NCOLS,None,
                markranges,
                yearlim,
                tickstep,
                print_names=print_names, number_start=number_start
            )
        return

    #print(yearlim)

    wper = 17 / 3
    hper = 10 / 20
    NROWS = (len(who) // NCOLS) + int(len(who) % NCOLS != 0)

    top_end = (2080-2015) /  (2015-1950)
    bottom_end = (1950 - 1850) / (2015 - 1950)
    year_wid = yearlim[1] - yearlim[0]
    yearmin, yearmax = yearlim

    xmin, xmax = (yearlim[0] - bottom_end * year_wid, yearlim[1] + top_end * year_wid)
    def dist_trans(nyears):
        return year_wid * nyears / (2015-1950)

    plt.figure(figsize=(wper * NCOLS, hper * NROWS))

    for col_i in range(NCOLS):

        plt.subplot(1, NCOLS, col_i + 1)

        trends = []
        for i, tname in enumerate(who[col_i * NROWS: (col_i + 1) * NROWS]):
            tt = np.array(dta.trend('c', tname).cits(yearmin, yearmax))

            #print(yearmin,yearmax)
            trends.append((tname, tt))

        for i, (tname, tt) in enumerate(trends):
            ttm = tt.max()
            ttl = tt[-1]
            ttl15 = tt[-15:].mean()

            BASE = (NROWS - 1 - i) * 2

            tt = tt / ttm
            tt = tt + BASE

            years = range(yearmin, yearmax+1)

            plt.axis('off')
            plt.fill_between(years, tt, y2=BASE, color='gray')

            if tname in print_names:
                tstr = print_names[tname]
            else:
                tstr = key2name(tname, truncate=25)

            tstr = "(%s) %s" % ((col_i) * NROWS + i + number_start, tstr)
            plt.text(yearmin - dist_trans(1950-1820), BASE, tstr)
            plt.text(yearmax + dist_trans(2030-2015), BASE, int(ttm), horizontalalignment='center')
            plt.text(yearmax + dist_trans(2050-2015), BASE, int(ttl), horizontalalignment='center')
            plt.text(yearmax + dist_trans(2070-2015), BASE, "%0.1f" % ttl15, horizontalalignment='center')
            # plt.text(2025, i*2, int(ttm), horizontalalignment='center')

            if tname in markranges:
                #xmin,xmax = plt.xlim()
                ymin,ymax = [BASE,BASE+1.4]
                mk = markranges[tname]
                plt.fill_between( mk, [ymin,ymin], [ymax,ymax], color='blue', alpha=0.2 )

        plt.vlines(range(yearmin, yearmax, tickstep), 0, NROWS * 2, color='black', alpha=0.3)

        # table lines...
        plt.vlines([ yearmax + dist_trans(DD-2015) for DD in range(2020, 2080 + 20, 20)], 0, NROWS * 2, color='black', alpha=1)  # table lines

        for y in range(yearmin, yearmax, tickstep):
            plt.text(y, -1, y, horizontalalignment='center')

        plt.xlim(xmin, xmax)

    # plt.savefig("top_cited.png", dpi=300)

def author_pub_table(dta, who, **kwargs):
    yearly_counts_table(dta, dta.search('c', who + "|"), **kwargs)

max_stars_repo_name: amcgail/knowknow

max_issues_repo_name: amcgail/knowknow

max_forks_repo_name: amcgail/knowknow


text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst
import pytest
from astropy.tests.helper import remote_data
from numpy.testing import assert_allclose
from ..fetch import fetch_tiles
from ..survey import HipsSurveyProperties
from ..tile import HipsTileMeta

TILE_FETCH_TEST_CASES = [
    dict(
        tile_indices=[69623, 69627, 69628, 69629, 69630, 69631],
        tile_format='fits',
        order=7,
        url='http://alasky.unistra.fr/DSS/DSS2Merged/properties',
        progress_bar=True,
        data=[2101, 1945, 1828, 1871, 2079, 2336],
        fetch_package='urllib',
    ),
    dict(
        tile_indices=[69623, 69627, 69628, 69629, 69630, 69631],
        tile_format='fits',
        order=7,
        url='http://alasky.unistra.fr/DSS/DSS2Merged/properties',
        progress_bar=True,
        data=[2101, 1945, 1828, 1871, 2079, 2336],
        fetch_package='aiohttp',
    ),
]

def make_tile_metas(hips_survey, pars):
    for healpix_pixel_index in pars['tile_indices']:
        yield HipsTileMeta(
            order=pars['order'],
            ipix=healpix_pixel_index,
            frame=hips_survey.astropy_frame,
            file_format=pars['tile_format'],
        )

@pytest.mark.parametrize('pars', TILE_FETCH_TEST_CASES)
@remote_data
def test_fetch_tiles(pars):
    hips_survey = HipsSurveyProperties.fetch(pars['url'])

    tile_metas = list(make_tile_metas(hips_survey, pars))

    tiles = fetch_tiles(
        tile_metas, hips_survey,
        progress_bar=pars['progress_bar'],
        fetch_package=pars['fetch_package'],
    )

    for idx, val in enumerate(pars['data']):
        assert_allclose(tiles[idx].data[0][5], val)

max_stars_repo_name: alex-w/hips

max_issues_repo_name: alex-w/hips

max_forks_repo_name: alex-w/hips

reason: from numpy,from astropy


text:
import os
import cv2
from flask import Flask, render_template, request, Response, send_from_directory
from flask_cors import CORS, cross_origin
import base64
import jsonpickle
from lib.device import Camera
from lib.processors_noopenmdao import findFaceGetPulse
from lib.interface import plotXY, imshow, waitKey, destroyWindow
from cv2 import moveWindow
import argparse
import numpy as np
import datetime
#TODO: work on serial port comms, if anyone asks for it
#from serial import Serial
import socket
import sys

global raw_img
global App
global BPM
BPM = 0

app = Flask(__name__, static_folder='static', template_folder='template')
CORS(app)

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                          'favicon.ico',mimetype='image/vnd.microsoft.icon')

@app.route('/')
def index():
   return render_template('index.html')

@app.route('/take_pic', methods = ['POST'])
def CalcBPM():
	encoded_img = request.get_data()
	# print(encoded_img[:20])
	nparr = np.fromstring(base64.b64decode(encoded_img), np.uint8)
	raw_img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
	App = getPulseApp()
	bpm = App.main_loop(raw_img)
	bpm_value = {BPM: bpm}
	response = jsonpickle.encode(bpm_value)
	return Response(response = response, status = 200, mimetype = 'application/json')

class getPulseApp():

    """
    Python application that finds a face in a webcam stream, then isolates the
    forehead.

    Then the average green-light intensity in the forehead region is gathered
    over time, and the detected person's pulse is estimated.
    """

    # def __init__(self, args):
    #     # Imaging device - must be a connected camera (not an ip camera or mjpeg
    #     # stream)
    #     serial = args.serial
    #     baud = args.baud
    #     self.send_serial = False
    #     self.send_udp = False
    #     if serial:
    #         self.send_serial = True
    #         if not baud:
    #             baud = 9600
    #         else:
    #             baud = int(baud)
    #         self.serial = Serial(port=serial, baudrate=baud)

    #     udp = args.udp
    #     if udp:
    #         self.send_udp = True
    #         if ":" not in udp:
    #             ip = udp
    #             port = 5005
    #         else:
    #             ip, port = udp.split(":")
    #             port = int(port)
    #         self.udp = (ip, port)
    #         self.sock = socket.socket(socket.AF_INET, # Internet
    #              socket.SOCK_DGRAM) # UDP

    #     self.cameras = []
    #     self.selected_cam = 0
    #     for i in range(3):
    #         camera = Camera(camera=i)  # first camera by default
    #         if camera.valid or not len(self.cameras):
    #             self.cameras.append(camera)
    #         else:
    #             break
    #     self.w, self.h = 0, 0
    #     self.pressed = 0
    #     # Containerized analysis of recieved image frames (an openMDAO assembly)
    #     # is defined next.

    #     # This assembly is designed to handle all image & signal analysis,
    #     # such as face detection, forehead isolation, time series collection,
    #     # heart-beat detection, etc.

    #     # Basically, everything that isn't communication
    #     # to the camera device or part of the GUI
    #     self.processor = findFaceGetPulse(bpm_limits=[50, 160],
    #                                       data_spike_limit=2500.,
    #                                       face_detector_smoothness=10.)

    #     # Init parameters for the cardiac data plot
    #     self.bpm_plot = False
    #     self.plot_title = "Data display - raw signal (top) and PSD (bottom)"

    #     # Maps keystrokes to specified methods
    #     #(A GUI window must have focus for these to work)
    #     self.key_controls = {"s": self.toggle_search,
    #                          "d": self.toggle_display_plot,
    #                          "c": self.toggle_cam,
    #                          "f": self.write_csv}

    # def toggle_cam(self):
    #     if len(self.cameras) > 1:
    #         self.processor.find_faces = True
    #         self.bpm_plot = False
    #         destroyWindow(self.plot_title)
    #         self.selected_cam += 1
    #         self.selected_cam = self.selected_cam % len(self.cameras)

    # def write_csv(self):
    #     """
    #     Writes current data to a csv file
    #     """
    #     fn = "Webcam-pulse" + str(datetime.datetime.now())
    #     fn = fn.replace(":", "_").replace(".", "_")
    #     data = np.vstack((self.processor.times, self.processor.samples)).T
    #     np.savetxt(fn + ".csv", data, delimiter=',')
    #     print("Writing csv")

    # def toggle_search(self):
    #     """
    #     Toggles a motion lock on the processor's face detection component.

    #     Locking the forehead location in place significantly improves
    #     data quality, once a forehead has been sucessfully isolated.
    #     """
    #     #state = self.processor.find_faces.toggle()
    #     state = self.processor.find_faces_toggle()
    #     print("face detection lock =", not state)

    # def toggle_display_plot(self):
    #     """
    #     Toggles the data display.
    #     """
    #     if self.bpm_plot:
    #         print("bpm plot disabled")
    #         self.bpm_plot = False
    #         destroyWindow(self.plot_title)
    #     else:
    #         print("bpm plot enabled")
    #         if self.processor.find_faces:
    #             self.toggle_search()
    #         self.bpm_plot = True
    #         self.make_bpm_plot()
    #         moveWindow(self.plot_title, self.w, 0)

    # def make_bpm_plot(self):
    #     """
    #     Creates and/or updates the data display
    #     """
    #     plotXY([[self.processor.times,
    #              self.processor.samples],
    #             [self.processor.freqs,
    #              self.processor.fft]],
    #            labels=[False, True],
    #            showmax=[False, "bpm"],
    #            label_ndigits=[0, 0],
    #            showmax_digits=[0, 1],
    #            skip=[3, 3],
    #            name=self.plot_title,
    #            bg=self.processor.slices[0])

    # def key_handler(self):
    #     """
    #     Handle keystrokes, as set at the bottom of __init__()

    #     A plotting or camera frame window must have focus for keypresses to be
    #     detected.
    #     """

    #     self.pressed = waitKey(10) & 255  # wait for keypress for 10 ms
    #     if self.pressed == 27:  # exit program on 'esc'
    #         print("Exiting")
    #         for cam in self.cameras:
    #             cam.cam.release()
    #         if self.send_serial:
    #             self.serial.close()
    #         sys.exit()

    #     for key in self.key_controls.keys():
    #         if chr(self.pressed) == key:
    #             self.key_controls[key]()

    def main_loop(self, raw_img):
        """
        Single iteration of the application's main loop.
        """
        # Get current image frame from the camera
        frame = raw_img
        self.h, self.w, _c = frame.shape

        # display unaltered frame
        # imshow("Original",frame)

        # set current image frame to the processor's input
        self.processor = findFaceGetPulse(bpm_limits=[50, 160],
                                          data_spike_limit=2500.,
                                          face_detector_smoothness=10.)
        self.processor.frame_in = frame
        # process the image frame to perform all needed analysis
        bpm = self.processor.run()
        return bpm
        # collect the output frame for display
        # output_frame = self.processor.frame_out

        # show the processed/annotated output frame
        # imshow("Processed", output_frame)

        # create and/or update the raw data display if needed
        # if self.bpm_plot:
        #     self.make_bpm_plot()

        # if self.send_serial:
        #     self.serial.write(str(self.processor.bpm) + "\r\n")

        # if self.send_udp:
        #     self.sock.sendto(str(self.processor.bpm), self.udp)

        # # handle any key presses
        # self.key_handler()

if __name__ == '__main__':
      app.run(host="127.0.0.1", port=2221,debug=True,ssl_context=('cert.pem', 'key.pem'))
      # app.run(host="127.0.0.1", port=2221,debug=True)

max_stars_repo_name: loginabhay/web-based-pulse-evaluation

max_issues_repo_name: loginabhay/web-based-pulse-evaluation

max_forks_repo_name: loginabhay/web-based-pulse-evaluation


text:
import json
import torch
import numpy as np

class Config():
    '''
    Abstracts over a model configuration.
    While it currently does not offer any advantages over working with a simple dict,
    it makes it possible to simply add functionality concerning configurations:
    - Checking the validity of a configuration file
    - Automatically loading and saving configuration files

    Parameters
    ----------
    path : str
        The path to the json with the config

    Attributes
    ----------
    architecture : str
        Stores the model architecture type. Currently ignored (only have CGNet), but can be used in the future
    lr : dict
        The learning rate used to train the model
    fields : [str]
        A dictionary mapping from variable names to normalisation statistics
    description : str
        Stores an uninterpreted description string for the model. Put anything you want here.
    '''

    def __init__(self, path: str):
        self.config_dict = json.load(open('config.json'))

        # TODO: Check structure

        self.architecture = self.config_dict['architecture']
        self.lr = self.config_dict['lr']
        self.seed = self.config_dict['seed']
        self.train_batch_size = self.config_dict['train_batch_size']
        self.pred_batch_size = self.config_dict['pred_batch_size']
        self.epochs = self.config_dict['epochs']
        self.fields = self.config_dict['fields']
        self.labels = self.config_dict['labels']
        self.description = self.config_dict['description']

        # Make reproducible
        torch.manual_seed(self.seed)
        np.random.seed(self.seed)

    def save(self, save_path: str):
        with open(save_path, 'w', encoding='utf-8') as f:
            json.dump(self.config_dict, f, ensure_ascii=False, indent=4)

max_stars_repo_name: andregraubner/ClimateNet

max_issues_repo_name: andregraubner/ClimateNet

max_forks_repo_name: andregraubner/ClimateNet


text:
import theano
import numpy
import scipy.sparse

from theano import gof, tensor, scalar, sparse
from theano.tensor import blas

from theano.sparse.basic import (
    as_sparse_variable, SparseType, add_s_s, neg,
    mul_s_s, mul_s_d, dot,
    CSMProperties, CSM, register_specialize,
    _is_sparse_variable, _is_dense_variable, CSC, CSR,
    csm_properties, csm_data, csm_indices, csm_indptr, csm_shape,
    _is_sparse,
    # To maintain compatibility
    Remove0, remove0,
    Cast, bcast, wcast, icast, lcast, fcast, dcast, ccast, zcast,
    HStack, hstack, VStack, vstack,
    AddSSData, add_s_s_data,
    MulSV, mul_s_v,
    Multinomial, multinomial, Poisson, poisson,
    Binomial, csr_fbinomial, csc_fbinomial, csr_dbinomial, csc_dbinomial,
    structured_monoid,
    structured_sigmoid, structured_exp, structured_log, structured_pow,
    structured_minimum, structured_maximum, structured_add,
    StructuredAddSV, structured_add_s_v,
    SamplingDot, sampling_dot)

# Also for compatibility
from theano.sparse.opt import (
    MulSDCSC, mul_s_d_csc, MulSDCSR, mul_s_d_csr,
    MulSVCSR, mul_s_v_csr,
    StructuredAddSVCSR, structured_add_s_v_csr,
    SamplingDotCSR, sampling_dot_csr,
    local_mul_s_d, local_mul_s_v,
    local_structured_add_s_v, local_sampling_dot_csr)

# Alias to maintain compatibility
EliminateZeros = Remove0
eliminate_zeros = remove0

max_stars_repo_name: jaberg/Theano

max_issues_repo_name: jaberg/Theano

max_forks_repo_name: jaberg/Theano

reason: import numpy,import scipy,import theano,from theano