text:
import itertools
import logging
import traceback
from abc import ABC, abstractmethod
from typing import Tuple, Dict, Iterable, TYPE_CHECKING, Optional, TypeVar, Generic, Any

import gym
import numpy as np

from assembly_gym.environment.generic import Environment

if TYPE_CHECKING:
    from .controllers import Controller
    from .sensors import Sensor
    from .rewards import Reward

class ResetFailedException(Exception):
    pass

EnvironmentType = TypeVar("EnvironmentType", bound=Environment)

class BaseTask(gym.Env, ABC, Generic[EnvironmentType]):
    """
    An abstract base class for gym environments that use the robot (real or simulated).
    """

    def __init__(self, controllers: Iterable["Controller[BaseTask]"],
                 sensors: Iterable["Sensor[BaseTask]"], rewards: Iterable["Reward[BaseTask]"],
                 time_step: float, time_limit_steps: Optional[int] = None):
        """
        :param controllers:             A sequence of controller objects that define the actions on the environment
        :param sensors:                 A sequence of sensors objects that provide observations to the agent
        :param rewards:                 A sequence of rewards objects that provide rewards to the agent
        :param time_step:               The time between two controller updates (actions of the agent)
        :param time_limit_steps:        The number of steps until the episode terminates (if no other termination
                                        criterion is reached)
        """
        self.__cumulative_rewards_dict: Dict["Reward", float] = {}

        self.__sensors: Iterable[Sensor] = sensors
        self.__rewards: Iterable["Reward"] = rewards

        self.__time_limit_steps: int = time_limit_steps
        self.__current_step: Optional[int] = None    # needed to enforce time_limit_steps

        self.__env: Optional[EnvironmentType] = None

        self.__logger: logging.Logger = logging.getLogger("task")
        self.__time_step: float = time_step

        self.__controllers = tuple(controllers)

        self.__auto_restart: Optional[bool] = None
        self.__latest_action: Optional[Dict[str, np.ndarray]] = None

    def initialize(self, env: EnvironmentType, auto_restart: bool = False) -> None:
        """
        Initialize the gym environment. Needs to be called before reset() is called for the first time.

        :param env:             the environment object that should be used to interact with the (simulated or real)
                                scene
        :param auto_restart:    whether the environment should be restarted if an error occurred during reset() or
                                step()
        """
        self.__env = env
        self.__auto_restart = auto_restart
        self.__env.initialize(self.__time_step)

        self._initialize()

        obs_space_dict = {}
        for sensor in self.__sensors:
            new_obs_spaces = sensor.initialize(self)
            assert len(set(new_obs_spaces.keys()).intersection(set(obs_space_dict.keys()))) == 0, \
                "Duplicate observation name"
            obs_space_dict.update(new_obs_spaces)

        self.observation_space = gym.spaces.Dict(obs_space_dict)

        action_space_dict = {}
        for controller in self.__controllers:
            new_action_space = controller.initialize(self)
            assert controller.name not in action_space_dict, "Duplicate action name"
            action_space_dict[controller.name] = new_action_space
        self.action_space = gym.spaces.Dict(action_space_dict)

        for reward in self.__rewards:
            reward.initialize(self)

    def _initialize(self) -> None:
        pass

    def step(self, action: Dict[str, np.ndarray]) -> Tuple[Dict[str, np.ndarray], float, bool, Dict[str, Any]]:
        """
        Executes one action of the agent and calculates the resulting observation, reward, and done signal.

        :param action:          the action that should be executed
        :return:                the feedback from the environment as a (observation, reward, done, info) tuple
        """
        try:
            self.__current_step += 1
            self.__latest_action = action

            for controller in self.__controllers:
                controller.actuate(action[controller.name])

            self.__env.step()

            done, task_info = self._step_task()
            if self.__time_limit_steps is not None:
                done = done or self.__time_limit_steps <= self.__current_step

            current_rewards = {reward: reward.calculate_reward(done) for reward in self.__rewards}
            reward = sum(current_rewards.values())

            reward_info = {reward.name: value for reward, value in current_rewards.items()}
            info = {
                "reward": reward_info,
                **task_info
            }

            obs = {k: v for k, v in itertools.chain(*(sensor.observe().items() for sensor in self.__sensors))}

            for reward_obj, value in current_rewards.items():
                self.__cumulative_rewards_dict[reward_obj] += value

            if done:
                self.environment.terminate_episode()

                cumulative_reward = sum(self.__cumulative_rewards_dict.values())
                current_reward = sum(current_rewards.values())
                self.__logger.info("")
                self.__logger.info("Cumulative reward: {: .6f} ({})".format(
                    cumulative_reward,
                    "  ".join([
                        "{}: {: .6f} [{: 8.4f}%]".format(r.name_abbreviation, v, v / cumulative_reward * 100)
                        for r, v in self.__cumulative_rewards_dict.items()])))
                self.__logger.info("Final reward:      {: .6f} ({})".format(
                    current_reward,
                    "  ".join([
                        "{}: {: .6f} [{: 8.4f}%]".format(r.name_abbreviation, v, v / current_reward * 100)
                        for r, v in current_rewards.items()])))
                for h in self.__logger.handlers:
                    h.flush()
            return obs, reward, done, info
        except KeyboardInterrupt:
            raise
        except:
            traceback.print_exc()
            if self.__auto_restart:
                self._restart_env()
            else:
                raise
            # Now obtain some valid observation
            self.reset()
            obs, reward, _, task_info = self.step(action)
            return obs, reward, True, task_info

    def reset(self) -> Dict[str, np.ndarray]:
        """
        Resets the gym environment. Must be called at the beginning of each episode.

        :return:        the initial observation for the episode
        """
        try:
            for i in range(10):
                self.__current_step = 0
                self.__latest_action = None
                self.__env.reset_scene()

                for controller in self.__controllers:
                    controller.initialize(self)

                try:
                    self._reset_task()
                    for reward in self.__rewards:
                        reward.reset()

                    self.__cumulative_rewards_dict = {reward: 0 for reward in self.__rewards}

                    return {k: v for k, v in itertools.chain(*(sensor.reset().items() for sensor in self.__sensors))}
                except ResetFailedException as e:
                    self.__logger.warning("Failed to reset the task. {} Retrying...".format(str(e)))
            raise ResetFailedException()
        except KeyboardInterrupt:
            raise
        except:
            traceback.print_exc()
            if self.__auto_restart:
                self._restart_env()
            else:
                raise
            return self.reset()

    def close(self) -> None:
        """
        Shuts the environment down. Should be called if the environment is not used anymore.
        """
        self.__env.shutdown()

    def render(self, mode='human'):
        assert mode == "human"

    @abstractmethod
    def _restart_env(self) -> None:
        """
        Restarts the environment in case an error occurred.
        """
        pass

    @abstractmethod
    def _step_task(self) -> Tuple[bool, Dict]:
        """
        Execute the task-specific components of gym.step().

        :return:                        a (done, info) tuple, where info contains the task-specific
                                        components of the infos, respectively
        """
        pass

    @abstractmethod
    def _reset_task(self) -> None:
        """
        Execute the task-specific components of gym.reset().
        """
        pass

    @property
    def controllers(self) -> Tuple["Controller", ...]:
        return self.__controllers

    @property
    def sensors(self) -> Iterable["Sensor"]:
        return self.__sensors

    @property
    def rewards(self) -> Iterable["Reward"]:
        return self.__rewards

    @property
    def environment(self) -> EnvironmentType:
        return self.__env

    @property
    def time_step(self) -> float:
        return self.__time_step

    @property
    def time_limit_steps(self) -> int:
        return self.__time_limit_steps

    @property
    def current_step(self) -> int:
        return self.__current_step

    @property
    def logger(self) -> logging.Logger:
        return self.__logger

    @property
    def latest_action(self) -> Optional[Dict[str, np.ndarray]]:
        return self.__latest_action


text:
"""
File: scipyoptimizer.py
Project: msalib
File Created: Thursday, 13th December 2018 10:08:17 pm
Author: Qianxiao Li (liqix@ihpc.a-star.edu.sg)
-----
Copyright - 2018 Qianxiao Li, IHPC, A*STAR
License: MIT License
"""

import tensorflow as tf
import numpy as np

Base = tf.contrib.opt.ScipyOptimizerInterface

class ScipyOptimizer(Base):
    """Scipy optimizer interface with option to
       randomize initial condition
    """

    def __init__(self, *args, perturb_init=None, **kwargs):
        self.perturb_init = perturb_init
        super().__init__(*args, **kwargs)

    def minimize(self, session=None, feed_dict=None, fetches=None,
                 step_callback=None, loss_callback=None, **run_kwargs):
        """Minimize a scalar `Tensor`.
        Variables subject to optimization are updated in-place at the end of
        optimization.
        Note that this method does *not* just return a minimization `Op`, unlike
        `Optimizer.minimize()`; instead it actually performs minimization by
        executing commands to control a `Session`.
        Args:
        session: A `Session` instance.
        feed_dict: A feed dict to be passed to calls to `session.run`.
        fetches: A list of `Tensor`s to fetch and supply to `loss_callback`
            as positional arguments.
        step_callback: A function to be called at each optimization step;
            arguments are the current values of all optimization variables
            flattened into a single vector.
        loss_callback: A function to be called every time the loss and gradients
            are computed, with evaluated fetches supplied as positional arguments.
        **run_kwargs: kwargs to pass to `session.run`.
        """
        session = session or tf.get_default_session()
        feed_dict = feed_dict or {}
        fetches = fetches or []

        loss_callback = loss_callback or (lambda *fetches: None)
        step_callback = step_callback or (lambda xk: None)

        # Construct loss function and associated gradient.
        loss_grad_func = self._make_eval_func(
            [self._loss, self._packed_loss_grad],
            session, feed_dict, fetches, loss_callback)

        # Construct equality constraint functions and associated gradients.
        equality_funcs = self._make_eval_funcs(
            self._equalities, session, feed_dict, fetches)
        equality_grad_funcs = self._make_eval_funcs(
            self._packed_equality_grads, session, feed_dict, fetches)

        # Construct inequality constraint functions and associated gradients.
        inequality_funcs = self._make_eval_funcs(
            self._inequalities, session, feed_dict, fetches)
        inequality_grad_funcs = self._make_eval_funcs(
            self._packed_inequality_grads, session, feed_dict, fetches)

        # Get initial value from TF session.
        initial_packed_var_val = session.run(self._packed_var)

        # Apply a perturbation to initial value fed into L-BFGS-B
        if self.perturb_init:
            initial_packed_var_val += self.perturb_init * np.random.uniform(
                low=-1.0, high=1.0, size=initial_packed_var_val.shape)

        # Perform minimization.
        packed_var_val = self._minimize(
            initial_val=initial_packed_var_val,
            loss_grad_func=loss_grad_func,
            equality_funcs=equality_funcs,
            equality_grad_funcs=equality_grad_funcs,
            inequality_funcs=inequality_funcs,
            inequality_grad_funcs=inequality_grad_funcs,
            packed_bounds=self._packed_bounds,
            step_callback=step_callback,
            optimizer_kwargs=self.optimizer_kwargs)
        var_vals = [
            packed_var_val[packing_slice] for packing_slice
            in self._packing_slices
        ]

        # Set optimization variables to their new values.
        session.run(
            self._var_updates,
            feed_dict=dict(zip(self._update_placeholders, var_vals)),
            **run_kwargs)

ext: py

max_stars_repo_name: bmkor/MaxPrinciple-DeepLearning

max_issues_repo_name: bmkor/MaxPrinciple-DeepLearning

max_forks_repo_name: bmkor/MaxPrinciple-DeepLearning

reason: import numpy


text:
# Copyright (c) The InferLO authors. All rights reserved.
# Licensed under the Apache License, Version 2.0.
import os
from dataclasses import dataclass
import tempfile
from typing import Dict

import numpy as np
import wget

from inferlo import GenericGraphModel
from inferlo.datasets.uai_reader import UaiReader
from inferlo.datasets.uai_writer import UaiWriter

REPO_URL = 'https://raw.githubusercontent.com/akxlr/tbp/master/tests/uai/'
UAI_PROB_URL = REPO_URL + 'MAR_prob'
UAI_SOL_URL = REPO_URL + 'MAR_sol'

# Copied from http://sli.ics.uci.edu/~ihler/uai-data/
# Values here are log10.
# TODO: load data from remote data repository.
UAI_PF = {
    'DBN_11.uai': 58.5307,
    'CSP_11.uai': 13.563,
    'CSP_12.uai': 16.4536,
    'CSP_13.uai': 15.3037,
    'Promedus_11.uai': -8.39145,
    'Promedus_12.uai': -3.16462,
    'Promedus_13.uai': -4.50703,
    'Promedus_14.uai': -7.80675,
    'Promedus_15.uai': -3.63667,
    'Promedus_16.uai': -6.97522,
    'Promedus_17.uai': -9.4592,
    'Promedus_18.uai': -4.67228,
    'Promedus_19.uai': -4.34464,
    'Promedus_20.uai': -7.06065,
    'Promedus_21.uai': -5.58012,
    'Promedus_22.uai': -2.49474,
    'Promedus_23.uai': -11.3151,
    'Promedus_24.uai': -5.86181,
    'Promedus_25.uai': -9.41791,
    'Promedus_26.uai': -7.30489,
    'Promedus_27.uai': -8.13576,
    'Promedus_28.uai': -8.13152,
    'Promedus_29.uai': -10.4527,
    'Promedus_30.uai': -22.1005,
    'Promedus_31.uai': -1.79979,
    'Promedus_32.uai': -2.20193,
    'Promedus_33.uai': -2.80866,
    'Promedus_34.uai': -3.07996,
    'Promedus_35.uai': -1.74295,
    'Promedus_36.uai': -1.7427,
    'Promedus_37.uai': -4.1711,
    'Promedus_38.uai': -4.98235,
    'linkage_11.uai': -31.1776,
    'linkage_12.uai': -69.7017,
    'linkage_13.uai': -76.0389,
    'linkage_14.uai': -30.7614,
    'linkage_15.uai': -76.6058,
    'linkage_16.uai': -38.5556,
    'linkage_17.uai': -64.8245,
    'linkage_18.uai': -78.5222,
    'linkage_19.uai': -59.0249,
    'linkage_20.uai': -64.2292,
    'linkage_21.uai': -29.6304,
    'linkage_22.uai': -54.2777,
    'linkage_23.uai': -116.58,
    'linkage_24.uai': -83.7331,
    'linkage_25.uai': -87.8848,
    'linkage_26.uai': -115.772,
    'linkage_27.uai': -63.4735,
}

@dataclass
class Dataset:
    """Dataset containing GM and true answers.

    Fields:
        * ``model`` - Graphical model.
        * ``true_marginals`` - known true exact marginal probabilities for all variables.
        * ``true_log_pf`` - known true exact natural logarithm of the partition function.
        * ``name`` - name of the dataset.
    """
    model: GenericGraphModel
    true_marginals: np.array
    true_log_pf: float
    name: str

class DatasetLoader:
    """Loads graphical models from named datasets."""

    def __init__(self, data_dir=None):
        """
        :param data_dir: Where to store cached datasets. Specify if you want
          datasets being cached locally. If not set, default system temporary
          directory will be used.
        """
        if data_dir is None:
            data_dir = os.path.join(tempfile.gettempdir(), 'inferlo_data')
        self.data_dir = os.path.expanduser(data_dir)
        if not os.path.exists(self.data_dir):
            os.makedirs(self.data_dir)
        self.custom_dir = os.path.join(self.data_dir, 'custom')
        if not os.path.exists(self.custom_dir):
            os.makedirs(self.custom_dir)
        self.uai_reader = UaiReader()
        self.uai_writer = UaiWriter()

    def load_file_(self, url_prefix, file_name):
        """Loads file from the web to local file.

        If local file already exists, does nothing.
        Returns local path to loaded file.
        """
        path = os.path.join(self.data_dir, file_name)
        if not os.path.exists(path):
            url = url_prefix + '/' + file_name
            wget.download(url, out=path)
            assert os.path.exists(path)
        return path

    def load_uai_dataset(self, dataset_name) -> Dataset:
        """Loads named dataset from UAI competition.

        :param dataset_name: Name of dataset, e.g. "Promedus_11.uai". For full
            list of UAI datasets, see http://sli.ics.uci.edu/~ihler/uai-data/.
            Not all of them are currently supported.
        :return: `Dataset` object, containing graphical model
            (as `GenericGraphModel` object), true logarithm of the partition
            function and true marginals.
        """
        if dataset_name not in UAI_PF:
            raise ValueError("Unknown UAI dataset: " + dataset_name)
        data_file = self.load_file_(UAI_PROB_URL, dataset_name)
        model = self.uai_reader.read_model(data_file)

        # True marginals.
        mar_file = self.load_file_(UAI_SOL_URL, dataset_name + '.MAR')
        marginals = self.uai_reader.read_marginals(mar_file)
        return Dataset(model=model,
                       true_marginals=marginals,
                       true_log_pf=UAI_PF[dataset_name] * np.log(10),
                       name=dataset_name)

    def save_custom_dataset(self, dataset):
        """Saves dataset locally."""
        # Store model.
        self.uai_writer.write_model(dataset.model,
                                    os.path.join(self.custom_dir, dataset.name + '.uai'))

        # Store true answers.
        ans_path = os.path.join(self.custom_dir, dataset.name + '.ans.npy')
        ans = {'true_marginals': dataset.true_marginals, 'true_log_pf': dataset.true_log_pf}
        np.save(ans_path, ans)

    def load_custom_dataset(self, name: str) -> Dataset:
        """Loads previously saved datataset."""
        model_path = os.path.join(self.custom_dir, name + '.uai')
        assert os.path.exists(model_path)
        model = self.uai_reader.read_model(model_path)

        ans_path = os.path.join(self.custom_dir, name + '.ans.npy')
        assert os.path.exists(model_path)
        ans = np.load(ans_path, allow_pickle=True).item()  # type: Dict
        return Dataset(model=model,
                       true_log_pf=ans['true_log_pf'],
                       true_marginals=ans['true_marginals'],
                       name=name)

    def custom_dataset_exists(self, name: str):
        """Checks if custom dataset exists."""
        return os.path.exists(os.path.join(self.custom_dir, name + '.uai'))

max_stars_repo_name: InferLO/inferlo

max_issues_repo_name: InferLO/inferlo

max_forks_repo_name: InferLO/inferlo


text:
import numpy as np
import os, glob, pyBigWig, argparse
from scipy import stats
import statsmodels.stats.weightstats as smw
from . import config
import warnings
warnings.filterwarnings("ignore")

#OUTPUT abs PSI for query group
def read_PsiMatrix(fn, delim,srr_list=None):
	with open(fn, 'r') as f:
		ele = f.readline().strip().split(delim)
		header = {ele[i].split('.')[0]:i for i in range(len(ele)) }
		if srr_list is None:
			srr_list = [ x.split('.')[0] for x in ele[8::] ]
		for line in f:
			ele = line.strip().split(delim)
			eid = ':'.join([ ele[header['chr']], ele[header['strand']],
				ele[header['upstreamEE']], ele[header['exonStart_0base']], ele[header['exonEnd']], ele[header['downstreamES']]])
			psi_list = np.empty(len(srr_list))
			psi_list[:] = np.nan
			for i in range(len(srr_list)):
				srr = srr_list[i]
				psi_list[i] = ele[ header[srr] ]
			yield ( eid, srr_list, psi_list )

def index_PsiMatrix(fn,outdir,delim):
	out_fp = outdir+'/'+fn.split('/')[-1]+'.idx'
	line_formatter =  "{id}\t{offset}\n"
	offset = 0
	with open(fn, 'r') as fin:
		with open(out_fp, 'w') as fout:
			offset += len(fin.readline())
			for line in fin:
				ele = line.strip().split(delim)
				eid = ':'.join([ele[0].split('_')[0].split('.')[0]]+ele[1:8])
				fout.write( line_formatter.format(id=eid, offset=offset) )
				offset += len(line)
	return

def read_PsiMatrix_index(fn,outdir):
	index = {}
	for line in open(outdir+'/'+fn.split('/')[-1]+'.idx', 'r'):
		ele = line.strip().split()
		index[ele[0]] = int(ele[1])
	return index

def fetch_PsiMatrix(eid, fn, outdir, delim, index=None):
	if index is None:
		index = read_PsiMatrix_index(fn,outdir)
	with open(fn, 'r') as f:
		ele = f.readline().strip().split(delim)
		header = np.asarray([ x.split('.')[0] if x.startswith('SRR') else x for x in ele ])
		f.seek(index[eid], 0)
		data = np.asarray(f.readline().strip().split(delim))
	return (header, data)

def openTestingFout(outdir, out_prefix, summary_file, ref_list, test_mode):
	header=['as_event','meanPSI','Q1PSI','Q3PSI']
	if test_mode=='group':
		header_prefix=['_pVal','_deltaPSI','_tumorFC']
	if test_mode=='personalized':
		header_prefix=['_modifiedPctl','_deltaPSI','_tumorFC']
	fout=open(outdir+'/'+out_prefix+'.test.all.txt','w')
	if summary_file==False:
		header+=['\t'.join(map(lambda x:ref+x ,header_prefix)) for ref in ref_list if ref!=out_prefix]
	fout.write('\t'.join(header)+'\n')
	return fout

def openScreeningFout(outdir, out_prefix, fout_name):
	fout=open(outdir+'/'+out_prefix+'.'+fout_name+'.txt','w')
	fout.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format('as_event','meanPSI','Q1PSI','Q3PSI','deltaPSI','fc_of_tumor_isoform','tissue_matched_normal_panel','tumor_panel','normal_panel','tag','mappability','mappability_tag'))
	return fout

def writeSummaryFile(out_prefix, db_dir, index, fout): #no screening, preparing for MS search
	fin_name=db_dir+'/'+out_prefix+'/splicing_matrix/splicing_matrix.SE.cov10.'+group_name+'.txt'
	index[out_prefix]=read_PsiMatrix_index(fin_name,'/'.join(fin_name.split('/')[:-1]))
	for j,k in enumerate(index[out_prefix]):
		psi_event=map(float,fetch_PsiMatrix(k,fin_name,'.','\t',index[out_prefix])[1][8:])
		query_mean=[np.nanmean(psi_event),np.nanpercentile(psi_event,25),np.nanpercentile(psi_event,75)]
		result=[k]+query_mean
		fout.write('\t'.join(map(str,result))+'\n')
	fout.close()

def loadMappability(bigwig_fin):
	bw_map=pyBigWig.open(bigwig_fin)
	d=45
	return bw_map,d

def getMappability(splicing_event,bw_map,d):
	arr=splicing_event.split(':')
	chrom,strand,start,end,up,down=arr[2],arr[3],int(arr[4]),int(arr[5]),int(arr[6]),int(arr[7])
	up_mean=bw_map.stats("%s"%chrom,up-d,up,type="mean")[0]
	down_mean=bw_map.stats("%s"%chrom,down,down+d,type="mean")[0]
	if abs(start-end)<2*d:
		target_mean=bw_map.stats("%s"%chrom,start,end,type="mean")[0]
	else:
		target_left=bw_map.stats("%s"%chrom,start,start+d,type="mean")[0]
		target_right=bw_map.stats("%s"%chrom,end-d,end,type="mean")[0]
		target_mean=(target_right+target_left)/2
	if strand=='-':  #switch the order
		li=[up_mean,down_mean]
		up_mean,down_mean=li[1],li[0]

	mappability=[str(up_mean),str(target_mean),str(down_mean)]
	return mappability

def calcTumorFormFoc(delta_psi, mean_psi):
	if delta_psi>0:
		return mean_psi/(mean_psi - delta_psi+10**-8)
	elif delta_psi<0:
		return (1- mean_psi)/ (1- mean_psi+ delta_psi+10**-8)
	else:
		return 1

def one2N(p1, g1, test_type):
	p1=np.array(p1)
	g1=np.array(g1)
	Q1=np.nanpercentile(g1,25)
	Q3=np.nanpercentile(g1,75)
	delta_psi=float(p1-np.nanmean(g1))
	tumor_foc=calcTumorFormFoc(delta_psi, float(p1))
	IQR=Q3-Q1
	low_bound=max(Q1-1.5*IQR,0)
	high_bound=min(Q3+1.5*IQR,1)
	points_add=np.append(g1,p1)
	empirical_percentile = stats.percentileofscore(points_add,p1)
	if empirical_percentile>50:
		empirical_percentile=100- empirical_percentile
	outlier_val=float(max(low_bound-p1,p1-high_bound))
	if test_type=='sig' and outlier_val>=0:
		return [empirical_percentile,delta_psi,tumor_foc]
	elif test_type=='equ' and outlier_val<=0:
		return [50-empirical_percentile, delta_psi, tumor_foc]
	else:
		return [np.nan,delta_psi,tumor_foc]

#def groupTest(g1,g2, test_type, threshold_tost=0.05):
def groupTest(g1,g2, test_type, direction='two-sided'):
	g1=np.array(g1)
	g2=np.array(g2)
	g1=g1[~np.isnan(g1)]
	g2=g2[~np.isnan(g2)]
	delta_psi=np.nanmean(g1)-np.nanmean(g2)
	tumor_foc=calcTumorFormFoc(delta_psi,np.nanmean(g1))
	if test_type=='sig':
		t1=stats.ttest_ind(g1,g2)[1]
	elif test_type=='equ':
	 	t1=smw.ttest_ind(g1,g2,alternative='two-sided')[1]
	# 	t1=smw.ttost_ind(g1,g2,-threshold_tost,threshold_tost,usevar='unequal')[0] #equalvalence test
	return [t1, delta_psi, tumor_foc]

def getDirection(filter1, psi, test):
	psi_primary=[]
	deltaPSI_primary=[]
	for primary_group in filter1:
		if test[primary_group]!=['-']*3:
			psi_primary+=psi[primary_group]
			deltaPSI_primary+=test[primary_group]
	if psi_primary==[]: #in case tissue-matched normal doesn't have data
		return [],[]
	else:
		direction='larger'
		if np.median(deltaPSI_primary)<=0:
			direction='smaller'
		return psi_primary, direction

def summarizeTestResult(filter1_cutoff_pval, filter1_cutoff_dpsi, filter1_cutoff_foc,filter2_cutoff_pval, filter2_cutoff_dpsi, filter2_cutoff_foc, filter3_cutoff_pval, filter3_cutoff_dpsi, filter3_cutoff_foc, primary, tumor_rec, pval, deltaPSI, foc, testing_type_index):
	differential,equal,positive,negative=[0,0,0,0]
	testable=0
	primary_result, primary_result_foc=[[],[]] #take care of multiple tiisue matched norm
	deltapsi_list,foc_list=[[],[]] #if no tissue-matched norm, use median
	for i,group_type in enumerate(testing_type_index):
		if pval[i]=='-':
			continue
		if i<primary:
			primary_result.append(float(deltaPSI[i]))
			primary_result_foc.append(float(foc[i]))
			if float(pval[i])<=filter1_cutoff_pval and abs(float(deltaPSI[i]))>=filter1_cutoff_dpsi and float(foc[i])>=filter1_cutoff_foc:
					differential+=1
					continue
		if testing_type_index[i]=='equ':
			if float(pval[i])<=filter2_cutoff_pval and abs(float(deltaPSI[i]))>=filter2_cutoff_dpsi:
					equal+=1
					continue
		if testing_type_index[i]=='sig' and i>=(primary+tumor_rec):
			testable+=1
			if float(pval[i])<=filter3_cutoff_pval and float(foc[i])>=filter3_cutoff_foc:
				deltapsi_list.append(float(deltaPSI[i]))
				foc_list.append(float(foc[i]))
				if float(deltaPSI[i])>=filter3_cutoff_dpsi:
					positive+=1
					continue
				if float(deltaPSI[i])<=-filter3_cutoff_dpsi:
					negative+=1
					continue
	return differential,equal,positive,negative,testable, np.median(primary_result), np.median(primary_result_foc), deltapsi_list,foc_list

def defineTumorEvents(filter1_group_cutoff,filter2_group_cutoff,filter3_group_cutoff, primary, norm_tissue, differential, equal, positive, negative, testable, primary_result, primary_result_foc, deltapsi_list,foc_list, use_ratio):
	tag=[]
 	if differential>=filter1_group_cutoff:
 		tag.append('associated')
 	if equal>=filter2_group_cutoff:
 		tag.append('recurrent')
	if primary==0:
		tissue_specificity=max(positive,negative)
		ratio=False if use_ratio==False else tissue_specificity/(testable+10**-8)>=filter3_group_cutoff/(norm_tissue+0.0)
		if tissue_specificity>=filter3_group_cutoff or ratio:
			tag.append('specific')
	else:
		if primary_result>0:
			tissue_specificity=positive
			ratio=False if use_ratio==False else positive/(testable+10**-8)>=filter3_group_cutoff/(norm_tissue+0.0)
			if positive>=filter3_group_cutoff or ratio:
				tag.append('specific')
		else:
			tissue_specificity=negative
			ratio=False if use_ratio==False else negative/(testable+10**-8)>=filter3_group_cutoff/(norm_tissue+0.0)
			if negative>=filter3_group_cutoff or ratio:
				tag.append('specific')
	primary_deltapsi=primary_result if primary_result!=0 else np.median(deltapsi_list)
	primary_foc=primary_result_foc if primary_result!=0 else np.median(foc_list)
	return primary_deltapsi, primary_foc, tissue_specificity, tag

def mappability_write(k, bw_map, calc_length):
	mappability_list=getMappability(k, bw_map, calc_length)
	mappability_tag='PASS'
	min_map_score=min(map(float, mappability_list))
	if min_map_score<0.8:
		mappability_tag='MID'
		if min_map_score<0.6:#from 0.7
			mappability_tag='LOW'
	return mappability_tag, mappability_list

def loadBlacklistEvents(fin):
	BlacklistEvents={}
	for l in open(fin):
		ls=l.strip().split('\t')
		des=ls[0].split(':')
		ensg=des[0].split('_')[0].split('.')[0]
		des=':'.join([ensg]+des[1:]).strip(':')
		BlacklistEvents[des]=''
	return BlacklistEvents

def translationCMD(ref_genome, outdir, out_prefix, fout_name):
	#uSE name space
	#Namespace(outdir='test1', parameter_fin='/u/home/p/panyang/bigdata-nobackup/Glioma_test/GBM.prioritze.par', subcommand='screening', translating=False)
	cmd_translation='IRIS translation '+outdir+'/'+out_prefix+'.'+fout_name+'.txt '+' -o '+outdir+'/'+fout_name+' -g '+ref_genome
	print '[INFO] Working on translating: '+fout_name
	os.system(cmd_translation)

def loadParametersRow(filter_para, ref_list):
	if len(filter_para.split(' '))==6:
		filter_cutoff_pval, filter_cutoff_dpsi, filter_cutoff_foc, filter_group_cutoff, filter_list =filter_para.split(' ')[1:]
		filter_cutoff_pval=float(filter_cutoff_pval)
		filter_cutoff_dpsi=float(filter_cutoff_dpsi)
		filter_cutoff_foc=float(filter_cutoff_foc)
		filter_group_cutoff=int(filter_group_cutoff)
		filter_list=filter_list.split(',')
		ref_list+=filter_list
	else:
		filter_cutoff_pval, filter_cutoff_dpsi, filter_cutoff_foc, filter_group_cutoff, filter_list =['','','','',[]]
	return filter_cutoff_pval, filter_cutoff_dpsi, filter_cutoff_foc, filter_group_cutoff, filter_list, ref_list

def main(args):
	###Loading Parameters####
	index={}
	fin_list={}
	para_fin=args.parameter_fin
	out_prefix,db_dir,filter1_para,filter2_para,filter3_para,test_mode,use_ratio,blacklist_path,mappability_path,ref_genome=[l.strip() for l in open(para_fin)]
	ref_list=[out_prefix]
	use_ratio=True if use_ratio=='True' else False
	if blacklist_path=='':
		blacklist_path=config.BRAIN_BLACKLIST_PATH
	blacklist_events=loadBlacklistEvents(blacklist_path)
	bw_map,calc_length=loadMappability(mappability_path)

	filter1_cutoff_pval, filter1_cutoff_dpsi, filter1_cutoff_foc, filter1_group_cutoff, filter1, ref_list =loadParametersRow(filter1_para, ref_list)
	filter2_cutoff_pval, filter2_cutoff_dpsi, filter2_cutoff_foc, filter2_group_cutoff, filter2, ref_list =loadParametersRow(filter2_para, ref_list)
	filter3_cutoff_pval, filter3_cutoff_dpsi, filter3_cutoff_foc, filter3_group_cutoff, filter3, ref_list =loadParametersRow(filter3_para, ref_list)
	if filter1==[] and filter2==[] and filter3==[] and test_mode!='summary':
		exit("[Error] No filtering required in parameteres file. exit!")
	group_test=False if test_mode!='group' else True
	individual_test=False if test_mode!='personalized' else True
	summary_file=False if test_mode!='summary' else True
	if [group_test,individual_test,summary_file]==[False,False,False]:
		exit('[Error] Need to choose one mode.exit!')
	primary=len(filter1)
	tumor_rec=len(filter2)
	norm_tissue=len(filter3)
	filter_count=sum(1 for i in [primary, tumor_rec, norm_tissue] if i!=0)
	testing_type_index=['sig']*primary+['equ']*tumor_rec+['sig']*norm_tissue

	db_dir=db_dir.rstrip('/')
	outdir=args.outdir.rstrip('/')
	os.system('mkdir -p  '+outdir)

	###Create Folders/Output####
	fout= openTestingFout(outdir, out_prefix, summary_file, ref_list, test_mode)
	if summary_file:
		writeSummaryFile(out_prefix, db_dir, index, fout)
		exit()

	fout_filtered=open(outdir+'/'+out_prefix+'.notest.txt','w')
	fout_primary=openScreeningFout(outdir, out_prefix, 'primary')
	fout_prioritized=openScreeningFout(outdir,out_prefix, 'prioritized')

	for group_name in ref_list:##Load IRIS reference panels
		fin_list[group_name]=db_dir+'/'+group_name+'/splicing_matrix/splicing_matrix.SE.cov10.'+group_name+'.txt'

	for group in fin_list.keys():
		if not os.path.isfile(fin_list[group]+'.idx'):
			exit('[Error] Need to index '+fin_list[group])
		index[group]=read_PsiMatrix_index(fin_list[group],'/'.join(fin_list[group].split('/')[:-1]))

	has={}
	#for j,k in enumerate(['ENSG00000083520:DIS3:chr13:-:73345041:73345126:73343050:73345218','ENSG00000110075:PPP6R3:chr11:+:68350510:68350597:68343511:68355265']):
	tot=len(index[out_prefix])-1
	print '[INFO] IRIS screening started. Total input events:', tot+1
	for event_idx,k in enumerate(index[out_prefix]):
		config.update_progress(event_idx/(0.0+tot))

		for group in ref_list:#Initiate
			if group!=out_prefix:
				has[group]=True
		psi={}
		has_count=0
		for group in ref_list:
			if k in index[group]:
				psi[group]=map(float,fetch_PsiMatrix(k,fin_list[group],'.','\t',index[group])[1][8:])
				has_count+=1
			else:
				has[group]=False

		cat_psi=[]
		for i in psi:
			cat_psi+=psi[i]
		if abs(max(cat_psi)-min(cat_psi))<0.05:#if change less than 5% skipped and no comparison available
			fout_filtered.write('[LowVar]{}\t{}\t{}\n'.format(k,str(abs(max(cat_psi)-min(cat_psi))),str(has_count)))
			continue
		if k in blacklist_events:
			fout_filtered.write('[Blacklisted]{}\t{}\t{}\n'.format(k,'-',str(has_count)))
			continue
		if  has_count<=1:
			fout_filtered.write('[NoTest]{}\t{}\t{}\n'.format(k,'-',str(has_count)))
			continue

		if group_test:
			test={}
			query_mean=[np.nanmean(psi[out_prefix]),np.nanpercentile(psi[out_prefix],25),np.nanpercentile(psi[out_prefix],75)]
			for j,group in enumerate(ref_list[1:]):
				test[group]=['-']*3
				if has[group]:
					if testing_type_index[j]=='equ':
						psi_primary, direction= getDirection(filter1, psi, test)
						if psi_primary==[]: #in case tissue-matched normal doesn't have data
							continue
						test[group]=groupTest(psi[group],psi_primary, testing_type_index[j], direction)
					else:
						test[group]=groupTest(psi[out_prefix],psi[group],testing_type_index[j])

			result=[k]+query_mean+['\t'.join(map(str,test[t])) for t in ref_list if t!=out_prefix]
			fout.write('\t'.join(map(str,result))+'\n')

			## summarize test result and prioritze
			pval=[test[t][0] for t in ref_list if t!=out_prefix]
			deltaPSI=[test[t][1] for t in ref_list if t!=out_prefix]# select deltapsi col of screening result
			foc=[test[t][2] for t in ref_list if t!=out_prefix]

			differential,equal,positive,negative,testable,primary_result, primary_result_foc, deltapsi_list,foc_list=summarizeTestResult(filter1_cutoff_pval, filter1_cutoff_dpsi, filter1_cutoff_foc,filter2_cutoff_pval, filter2_cutoff_dpsi, filter2_cutoff_foc, filter3_cutoff_pval, filter3_cutoff_dpsi, filter3_cutoff_foc, primary, tumor_rec, pval, deltaPSI, foc, testing_type_index)

			primary_deltapsi, primary_foc, tissue_specificity, tag = defineTumorEvents(filter1_group_cutoff,filter2_group_cutoff,filter3_group_cutoff, primary, norm_tissue, differential,equal,positive,negative,testable,primary_result, primary_result_foc, deltapsi_list,foc_list,use_ratio)

			if tag!=[]:
				if tag[0]=='associated':
					mappability_tag, mappability_list=mappability_write(k, bw_map, calc_length)
					fout_primary.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(k,query_mean[0],query_mean[1],query_mean[2],primary_deltapsi, primary_foc, str(differential)+'/'+str(primary),str(equal)+'/'+str(tumor_rec),str(tissue_specificity)+'/'+str(norm_tissue),';'.join(tag), ';'.join(mappability_list),mappability_tag))

			if filter_count==len(tag):
				if mappability_tag=='':#only if not in associated, then load mappability here
					mappability_tag, mappability_list=mappability_write(k, bw_map, calc_length)
				fout_prioritized.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(k,query_mean[0],query_mean[1],query_mean[2],primary_deltapsi, primary_foc, str(differential)+'/'+str(primary),str(equal)+'/'+str(tumor_rec),str(tissue_specificity)+'/'+str(norm_tissue),';'.join(tag),';'.join(mappability_list),mappability_tag))

			mappability_tag, mappability_list=['',''] #clear

		elif individual_test:
			test={}
			query_mean=[psi[out_prefix][0],'-','-']
			for j,group in enumerate(ref_list[1:]):
				test[group]=['-']*3
				if has[group]:
					test[group]=one2N(psi[out_prefix],psi[group],testing_type_index[j])

			result=[k]+query_mean+['\t'.join(map(str,test[t])) for t in ref_list if t!=out_prefix]
			fout.write('\t'.join(map(str,result))+'\n')

			pval=[test[t][0] for t in ref_list if t!=out_prefix]
			deltaPSI=[test[t][1] for t in ref_list if t!=out_prefix]# select deltapsi col of screening result
			foc=[test[t][2] for t in ref_list if t!=out_prefix]

			differential,equal,positive,negative,testable,primary_result, primary_result_foc, deltapsi_list,foc_list=summarizeTestResult(filter1_cutoff_pval, filter1_cutoff_dpsi, filter1_cutoff_foc,filter2_cutoff_pval, filter2_cutoff_dpsi, filter2_cutoff_foc, filter3_cutoff_pval, filter3_cutoff_dpsi, filter3_cutoff_foc, primary, tumor_rec, pval, deltaPSI, foc, testing_type_index)

			primary_deltapsi, primary_foc, tissue_specificity, tag = defineTumorEvents(filter1_group_cutoff,filter2_group_cutoff,filter3_group_cutoff, primary, norm_tissue, differential,equal,positive,negative,testable,primary_result, primary_result_foc, deltapsi_list,foc_list,use_ratio)

			if tag!=[]:
				if tag[0]=='associated':
					mappability_tag, mappability_list=mappability_write(k, bw_map, calc_length)
					fout_primary.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(k,query_mean[0],query_mean[1],query_mean[2],primary_deltapsi, primary_foc, str(differential)+'/'+str(primary),str(equal)+'/'+str(tumor_rec),str(tissue_specificity)+'/'+str(norm_tissue),';'.join(tag), ';'.join(mappability_list),mappability_tag))

			if filter_count==len(tag):
				if mappability_tag=='':#only if not in associated, then load mappability here
					mappability_tag, mappability_list=mappability_write(k, bw_map, calc_length)
				fout_prioritized.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(k,query_mean[0],query_mean[1],query_mean[2],primary_deltapsi, primary_foc, str(differential)+'/'+str(primary),str(equal)+'/'+str(tumor_rec),str(tissue_specificity)+'/'+str(norm_tissue),';'.join(tag),';'.join(mappability_list),mappability_tag))

			mappability_tag, mappability_list=['',''] #clear
		else:
			exit('[Error] no test performed')
	fout.close()
	fout_filtered.close()
	fout_primary.close()
	fout_prioritized.close()

	if args.translating:
		translationCMD(ref_genome, outdir, out_prefix, 'primary')
		translationCMD(ref_genome, outdir, out_prefix, 'prioritized')

if __name__ == '__main__':
	main()

max_stars_repo_name: Xinglab/IRIS

max_issues_repo_name: Xinglab/IRIS

max_forks_repo_name: Xinglab/IRIS

reason: import numpy,from scipy,import statsmodels


text:
# Some tests for the Cartesian coordinates handler

import numpy as np

from yt.testing import \
    fake_amr_ds, \
    assert_equal

# Our canonical tests are that we can access all of our fields and we can
# compute our volume correctly.

def test_cartesian_coordinates():
    # We're going to load up a simple AMR grid and check its volume
    # calculations and path length calculations.
    ds = fake_amr_ds()
    axes = list(set(ds.coordinates.axis_name.values()))
    axes.sort()
    for i, axis in enumerate(axes):
        dd = ds.all_data()
        fi = ("index", axis)
        fd = ("index", "d%s" % axis)
        fp = ("index", "path_element_%s" % axis)
        ma = np.argmax(dd[fi])
        assert_equal(dd[fi][ma] + dd[fd][ma] / 2.0, ds.domain_right_edge[i])
        mi = np.argmin(dd[fi])
        assert_equal(dd[fi][mi] - dd[fd][mi] / 2.0, ds.domain_left_edge[i])
        assert_equal(dd[fd].min(), ds.index.get_smallest_dx())
        assert_equal(dd[fd].max(), (ds.domain_width/ds.domain_dimensions)[i])
        assert_equal(dd[fd], dd[fp])
    assert_equal(dd["cell_volume"].sum(dtype="float64"), ds.domain_width.prod())

max_stars_repo_name: kastalpes/yt

max_issues_repo_name: kastalpes/yt

max_forks_repo_name: kastalpes/yt


text:
class load_knmi_data:
    def __init__(self, filename):
        self.filename = filename
        self.load_file()
        self.process_reference_data()

    def load_file(self):
        '''
        function description
        '''
        import zipfile
        import csv
        import os
        import StringIO
        from numpy import vstack
        # load the zip file
        zipf = zipfile.ZipFile(self.filename)
        # name of csv name in zip file
        txtname = os.path.splitext(os.path.basename(self.filename))[0] + '.txt'
        # read the data in the txt file
        data = StringIO.StringIO(zipf.read(txtname))
        reader = csv.reader(data)
        # file content is ignored while start_data==False
        start_data = False
        # loop through all rows of the txt file
        for row in reader:
            if not start_data:
                if '# STN' in row:  # look for header definition in csv file
                    header = [item.strip() for item in row]
                    # found the header
                    # set start_data = True ==>> use content from here
                    start_data = True
            else:
                # we found the header definition, data starts from here
                if len(row) > 0:
                    # strip data in row and convert to int
                    # empty fiels are filled with -999
                    try:
                        data_row = [int(item.strip()) if item.strip()
                                    else -999 for item in row if item]
                    except ValueError:
                        import pdb; pdb.set_trace()
                    # create array with output data
                    try:
                        csvdata = vstack((csvdata, data_row))
                    except UnboundLocalError:
                        csvdata = data_row
        # create a dictionary from the header and output data
        self.csvdata = dict(zip(header, csvdata.T))

    def process_reference_data(self):
        '''
        process the reference csv data
        '''
        from datetime import datetime
        from datetime import date
        from datetime import timedelta
        from numpy import zeros
        ## Convert time to datetime.datetime object
        # hours should be 0-23 instead of 1-24
        self.csvdata['HH'] = [item if item!=24 else 0 for item in
                              self.csvdata['HH']]
        # combine date and hour into one string
        dstring = [str(self.csvdata['YYYYMMDD'][idx]) +
                   str(self.csvdata['HH'][idx]).zfill(2) for idx in
                   range(0,len(self.csvdata['YYYYMMDD']))]
        # create datetime object
        self.csvdata['datetime'] = [datetime.strptime(
            str(item), ('%Y%m%d%H')) for item in dstring]
        # Correct conversion of the datestring
        # the date of the night -> HH=24 is HH=0 on the next day!
        self.csvdata['datetime'] = [c+ timedelta(days=1) if c.hour==0 else
                                    c for c in self.csvdata['datetime']]
        # rain is (-1 for <0.05 mm), set to 0
        self.csvdata['RH'] = [0 if item == -1 else
                                  item for item in self.csvdata['RH']]
        # process all variables that need to be divided by 10
        for variable in ['T10', 'T', 'RH', 'FF', 'TD']:
            # T10: temperature at 10 cm height, divide by 10 to convert to degC
            # T: temperature at 1.50 m height, divide by 10 to convert to degC
            # RH: rain
            # FF: wind speed
            self.csvdata[variable] = [round(0.1 * item, 1) if item != -999 else
                                item for item in self.csvdata[variable]]
        # SWD
        self.csvdata['Q'] = [round(float(10000 * item)/3600, 5) if
                             item != -999 else item for item in
                             self.csvdata['Q']]

reason: from numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 19 15:13:16 2019

@author: rocco
"""

from sklearn.decomposition import KernelPCA
from sklearn.preprocessing import MinMaxScaler
import h5py
import pickle
import numpy as np

#filter microwindows
r,c = np.triu_indices(142,1)
wk = np.where(((c < 106) | (c > 137)) & ((r < 106) | (r > 137)))[0]

new_file = h5py.File("../data/mipas_blabeled_reinhold_features/rf_mipas_complete", "r")
btd_mipas = new_file["btd_complete"][:, wk]
new_file.close()

scaler = MinMaxScaler()
btd_mipas_scaled = scaler.fit_transform(btd_mipas)

kpca = KernelPCA(kernel = "polynomial", n_components = 7)
kpc = kpca.fit_transform(btd_mipas_scaled)

filename = "model_kpca_btd_scaled_red"
pickle.dump(kpca, open(filename, 'wb'))


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of the SCICO package. Details of the copyright
# and user license can be found in the 'LICENSE.txt' file distributed
# with the package.

r"""
Deconvolution Microscopy
========================

This example partially replicates a [GlobalBioIm example](https://biomedical-imaging-group.github.io/GlobalBioIm/examples.html) using the [microscopy data](http://bigwww.epfl.ch/deconvolution/bio/) provided by the EPFL Biomedical Imaging Group.

The deconvolution problem is solved using class [admm.ADMM](../_autosummary/scico.admm.rst#scico.admm.ADMM) to solve an image deconvolution problem with isotropic total variation (TV) regularization.

  $$\mathrm{argmin}_{\mathbf{x}} \; \| M (\mathbf{y} - A \mathbf{x}) \|_2^2 + \lambda \| C \mathbf{x} \|_{2,1} + \iota_{\mathrm{NN}}(\mathbf{x}) \;,$$

where $M$ is a mask operator, $A$ is circular convolution, $\mathbf{y}$ is the blurred image, $C$ is a convolutional gradient operator, $\iota_{mathrm{NN}}$ is the indicator function of the non-negativity constraint, and $\mathbf{x}$ is the desired image.
"""

import glob
import os
import tempfile
import zipfile

import numpy as np

import imageio

import scico.numpy as snp
from scico import functional, linop, loss, plot, util
from scico.admm import ADMM, CircularConvolveSolver

"""
Define helper functions.
"""

def volread(path, ext="tif"):
    """Read a 3D volume from a set of files in the specified directory"""

    slices = []
    for file in sorted(glob.glob(os.path.join(path, "*." + ext))):
        image = imageio.imread(file)
        slices.append(image)
    return np.dstack(slices)

def block_avg(im, N):
    """
    Average distinct NxNxN blocks of im, return the resulting smaller image
    """

    im = snp.mean(snp.reshape(im, (-1, N, im.shape[1], im.shape[2])), axis=1)
    im = snp.mean(snp.reshape(im, (im.shape[0], -1, N, im.shape[2])), axis=2)
    im = snp.mean(snp.reshape(im, (im.shape[0], im.shape[1], -1, N)), axis=3)

    return im

"""
Download data or access it from cache directory.
"""
data_base_url = "http://bigwww.epfl.ch/deconvolution/bio/"
data_zip_files = ["CElegans-CY3.zip", "CElegans-DAPI.zip", "CElegans-FITC.zip"]
psf_zip_files = ["PSF-CElegans-CY3.zip", "PSF-CElegans-DAPI.zip", "PSF-CElegans-FITC.zip"]
cache_path = os.path.join(os.path.expanduser("~"), ".cache", "scico", "epfl_big")
if not os.path.isdir(cache_path):
    os.makedirs(cache_path)
    temp_dir = tempfile.TemporaryDirectory()
    for zip_file in data_zip_files + psf_zip_files:
        data = util.url_get(data_base_url + zip_file)
        f = open(os.path.join(temp_dir.name, zip_file), "wb")
        f.write(data.read())
        f.close()
    for zip_file in data_zip_files + psf_zip_files:
        with zipfile.ZipFile(os.path.join(temp_dir.name, zip_file), "r") as zip_ref:
            zip_ref.extractall(cache_path)

image_list = []
for zip_file in data_zip_files:
    image_list.append(volread(os.path.join(cache_path, zip_file[:-4])).astype(np.float32))
y = snp.stack(image_list)
del image_list
psf_list = []
for zip_file in psf_zip_files:
    psf_list.append(volread(os.path.join(cache_path, zip_file[:-4])).astype(np.float32))
psf = snp.stack(psf_list)
del psf_list

"""
Preprocess data. We downsample by a factor of 4 for purposes of the example.
Reducing the downsampling rate will be slower and more memory-intensive.
If your GPU does not have enough memory, you can try setting the environment
variable `JAX_PLATFORM_NAME=cpu` to run on CPU.
"""

channel = 0
downsampling_rate = 4

y = block_avg(y[channel], downsampling_rate)
psf = block_avg(psf[channel], downsampling_rate)

y -= y.min()
y /= y.max()

psf /= psf.sum()

"""
Pad data and create mask.
"""

padding = [[0, p] for p in snp.array(psf.shape) - 1]
y_pad = snp.pad(y, padding)
mask = snp.pad(snp.ones_like(y), padding)

"""
Define problem and algorithm parameters.
"""

λ = 2e-6  # L1 norm regularization parameter
ρ0 = 1e-3  # ADMM penalty parameter for first auxiiary variable
ρ1 = 1e-3  # ADMM penalty parameter for second auxiiary variable
ρ2 = 1e-3  # ADMM penalty parameter for third auxiiary variable
maxiter = 100  # Number of ADMM iterations

"""
Create operators.
"""

M = linop.Diagonal(mask)
C0 = linop.CircularConvolve(h=psf, input_shape=mask.shape, h_center=snp.array(psf.shape) / 2 - 0.5)
C1 = linop.FiniteDifference(input_shape=mask.shape, circular=True)
C2 = linop.Identity(mask.shape)

"""
Create functionals.
"""

g0 = loss.SquaredL2Loss(y=y_pad, A=M)  # Loss function (forward model)
g1 = λ * functional.L21Norm()  # TV penalty (when applied to gradient)
g2 = functional.NonNegativeIndicator()  # Non-negativity constraint

"""
Set up ADMM solver object and solve problem.
"""
##
solver = ADMM(
    f=None,
    g_list=[g0, g1, g2],
    C_list=[C0, C1, C2],
    rho_list=[ρ0, ρ1, ρ2],
    maxiter=maxiter,
    verbose=True,
    x0=y_pad,
    subproblem_solver=CircularConvolveSolver(),
)

t = util.Timer()
with util.ContextTimer(t):
    solver.solve()
solve_stats = solver.itstat_object.history(transpose=True)
x_pad = solver.x
x = x_pad[: y.shape[0], : y.shape[1], : y.shape[2]]

##

"""
Show the recovered image.
"""

def make_slices(x, sep_width=10):
    """
    Make an image with xy, xz, and yz slices from an input volume.
    """
    fill_val = -1.0
    out = snp.concatenate(
        (
            x[:, :, x.shape[2] // 2],
            snp.full((x.shape[0], sep_width), fill_val),
            x[:, x.shape[1] // 2, :],
        ),
        axis=1,
    )

    out = snp.concatenate(
        (
            out,
            snp.full((sep_width, out.shape[1]), fill_val),
            snp.concatenate(
                (
                    x[x.shape[0] // 2, :, :].T,
                    snp.full((x.shape[2], x.shape[2] + sep_width), fill_val),
                ),
                axis=1,
            ),
        ),
        axis=0,
    )

    out = snp.where(out == fill_val, out.max(), out)

    return out

fig, ax = plot.subplots(nrows=1, ncols=2, figsize=(10, 5))
plot.imview(make_slices(y), title="Blurred measurements", fig=fig, ax=ax[0])
plot.imview(make_slices(x), title="Deconvolved image", fig=fig, ax=ax[1])
fig.show()

"""
Plot convergence statistics.
"""

fig, ax = plot.subplots(nrows=1, ncols=2, figsize=(12, 5))
plot.plot(
    solve_stats.Objective,
    title="Objective function",
    xlbl="Iteration",
    ylbl="Functional value",
    fig=fig,
    ax=ax[0],
)
plot.plot(
    snp.vstack((solve_stats.Primal_Rsdl, solve_stats.Dual_Rsdl)).T,
    ptyp="semilogy",
    title="Residuals",
    xlbl="Iteration",
    lgnd=("Primal", "Dual"),
    fig=fig,
    ax=ax[1],
)
fig.show()

input("\nWaiting for input to close figures and exit")

max_stars_repo_name: lukepfister/scico

max_issues_repo_name: lukepfister/scico

max_forks_repo_name: lukepfister/scico


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 15 14:17:40 2020

@author: rodri
"""

import pandas as pd
from scipy import stats
import statsmodels.api as sm
from statsmodels.formula.api import ols
import matplotlib.pyplot as plt # gerar os gráficos
from statsmodels.stats.multicomp import MultiComparison

tratamento = pd.read_csv('anova.csv',sep=';') # abrir csv com separados ";"

tratamento.boxplot(by = 'Remedio', grid=False) # criar box plot dos dados em função de Remedio e grid tira linhas

plt.scatter(tratamento.Remedio,tratamento.Horas)

modelo1=ols("Horas ~ Remedio", data=tratamento).fit() # cria modelo através de data / fit faz o treinamento (regressão)
resultados1 = sm.stats.anova_lm(modelo1) # teste do anova

modelo2=ols("Horas ~ Remedio*Sexo", data=tratamento).fit() # cria modelo através de data + de 1 coluna/ fit faz o treinamento (regressão)
resultados2 = sm.stats.anova_lm(modelo2) # teste do anova

#Existe variação significativa ? Método Turkey

ac = MultiComparison (tratamento['Horas'], tratamento['Remedio'])
resultado_teste=ac.tukeyhsd()
print(resultado_teste)
resultado_teste.plot_simultaneous()

max_stars_repo_path: Python/Metodo Anova.py

max_issues_repo_path: Python/Metodo Anova.py

max_forks_repo_path: Python/Metodo Anova.py

reason: from scipy,import statsmodels,from statsmodels


text:
#!/usr/bin/env python3
# Description:
# - Subscribes to real-time streaming video from your built-in webcam.
#
# Author:
# - Addison Sears-Collins
# - https://automaticaddison.com

# Import the necessary libraries
import rospy # Python library for ROS
from sensor_msgs.msg import Image # Image is the message type
import cv2 # OpenCV library
import numpy as np
import detector

def callback(data):

	# Output debugging information to the terminal
	rospy.loginfo("receiving video frame")

	# Convert ROS Image message to OpenCV image
	np_arr = np.fromstring(data.data, np.uint8)
	current_frame = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)

	# Display image
	#cv2.imshow("camera", current_frame)
	cv2.imshow("camera", detector.perform_obj_detection(current_frame))
	cv2.waitKey(1)

def receive_message():

	# Tells rospy the name of the node.
	# Anonymous = True makes sure the node has a unique name. Random
	# numbers are added to the end of the name.
	rospy.init_node('video_sub_py', anonymous=True)

	# Node is subscribing to the video_frames topic
	#rospy.Subscriber('video_frames', Image, callback)
	rospy.Subscriber('/camera/image_raw', Image, callback)

	# spin() simply keeps python from exiting until this node is stopped
	rospy.spin()

	# Close down the video stream when done
	cv2.destroyAllWindows()

if __name__ == '__main__':
	receive_message()


text:
# %% [markdown]
# # Imports
import os
import pickle
import warnings
from operator import itemgetter
from pathlib import Path
from timeit import default_timer as timer

import colorcet as cc
import matplotlib.colors as mplc
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import pandas as pd
import seaborn as sns
from bokeh.embed import file_html
from bokeh.io import output_file, output_notebook, show
from bokeh.layouts import gridplot
from bokeh.models import ColumnDataSource, FactorRange, Legend, Span, PreText, Circle
from bokeh.palettes import Spectral4, all_palettes
from bokeh.plotting import curdoc, figure, output_file, show
from bokeh.resources import CDN
from bokeh.sampledata.stocks import AAPL, GOOG, IBM, MSFT
from joblib import Parallel, delayed
from matplotlib.cm import ScalarMappable
from mpl_toolkits.axes_grid1 import make_axes_locatable
from sklearn.metrics import adjusted_rand_score
from sklearn.model_selection import ParameterGrid
from sklearn.neighbors import NearestNeighbors

from graspy.cluster import AutoGMMCluster, GaussianCluster
from graspy.embed import AdjacencySpectralEmbed, LaplacianSpectralEmbed
from graspy.plot import degreeplot, edgeplot, gridplot, heatmap, pairplot
from graspy.utils import symmetrize
from src.cluster import DivisiveCluster
from src.data import load_everything, load_metagraph, load_networkx
from src.embed import ase, lse, preprocess_graph
from src.graph import MetaGraph
from src.hierarchy import signal_flow
from src.io import savefig, saveobj, saveskels
from src.utils import (
    get_blockmodel_df,
    get_sbm_prob,
    invert_permutation,
    meta_to_array,
    savefig,
)
from src.visualization import (
    bartreeplot,
    get_color_dict,
    get_colors,
    remove_spines,
    sankey,
    screeplot,
)

from bokeh.layouts import column, row
from bokeh.models import Select
from bokeh.palettes import Spectral5
from bokeh.plotting import curdoc, figure
from scipy.linalg import orthogonal_procrustes

def pair_augment(mg):
    pair_df = pd.read_csv(
        "maggot_models/data/raw/Maggot-Brain-Connectome/pairs/bp-pairs-2020-01-13_continuedAdditions.csv"
    )

    skeleton_labels = mg.meta.index.values

    # extract valid node pairings
    left_nodes = pair_df["leftid"].values
    right_nodes = pair_df["rightid"].values

    left_right_pairs = list(zip(left_nodes, right_nodes))

    left_nodes_unique, left_nodes_counts = np.unique(left_nodes, return_counts=True)
    left_duplicate_inds = np.where(left_nodes_counts >= 2)[0]
    left_duplicate_nodes = left_nodes_unique[left_duplicate_inds]

    right_nodes_unique, right_nodes_counts = np.unique(right_nodes, return_counts=True)
    right_duplicate_inds = np.where(right_nodes_counts >= 2)[0]
    right_duplicate_nodes = right_nodes_unique[right_duplicate_inds]

    left_nodes = []
    right_nodes = []
    for left, right in left_right_pairs:
        if left not in left_duplicate_nodes and right not in right_duplicate_nodes:
            if left in skeleton_labels and right in skeleton_labels:
                left_nodes.append(left)
                right_nodes.append(right)

    pair_nodelist = np.concatenate((left_nodes, right_nodes))
    not_paired = np.setdiff1d(skeleton_labels, pair_nodelist)
    sorted_nodelist = np.concatenate((pair_nodelist, not_paired))

    # sort the graph and metadata according to this
    sort_map = dict(zip(sorted_nodelist, range(len(sorted_nodelist))))
    inv_perm_inds = np.array(itemgetter(*skeleton_labels)(sort_map))
    perm_inds = invert_permutation(inv_perm_inds)

    mg.reindex(perm_inds)

    side_labels = mg["Hemisphere"]
    side_labels = side_labels.astype("<U2")
    for i, l in enumerate(side_labels):
        if mg.meta.index.values[i] in not_paired:
            side_labels[i] = "U" + l
    mg["Hemisphere"] = side_labels
    n_pairs = len(left_nodes)
    return mg, n_pairs

def max_symmetrize(mg, n_pairs):
    """ assumes that mg is sorted

    Parameters
    ----------
    mg : [type]
        [description]
    n_pairs : [type]
        [description]
    """
    adj = mg.adj
    left_left_adj = adj[:n_pairs, :n_pairs]
    left_right_adj = adj[:n_pairs, n_pairs : 2 * n_pairs]
    right_right_adj = adj[n_pairs : 2 * n_pairs, n_pairs : 2 * n_pairs]
    right_left_adj = adj[n_pairs : 2 * n_pairs, :n_pairs]

    # max, average gives similar results
    sym_ipsi_adj = np.maximum(left_left_adj, right_right_adj)
    sym_contra_adj = np.maximum(left_right_adj, right_left_adj)

    sym_adj = adj.copy()
    sym_adj[:n_pairs, :n_pairs] = sym_ipsi_adj
    sym_adj[n_pairs : 2 * n_pairs, n_pairs : 2 * n_pairs] = sym_ipsi_adj
    sym_adj[:n_pairs, n_pairs : 2 * n_pairs] = sym_contra_adj
    sym_adj[n_pairs : 2 * n_pairs, :n_pairs] = sym_contra_adj

    sym_mg = MetaGraph(sym_adj, mg.meta)  # did not change indices order so this ok

    return sym_mg

FNAME = os.path.basename(__file__)[:-3]
print(FNAME)
BRAIN_VERSION = "2020-01-21"

# %% [markdown]
# #
thresh = 2
mg = load_metagraph("Gad", BRAIN_VERSION)
mg.adj[mg.adj < thresh] = 0
mg, n_pairs = pair_augment(mg)

left_pair_ids = mg["Pair ID"][:n_pairs]
right_pair_ids = mg["Pair ID"][n_pairs : 2 * n_pairs]
print((left_pair_ids == right_pair_ids).all())

sym_mg = max_symmetrize(mg, n_pairs)
left_pair_ids = sym_mg["Pair ID"][:n_pairs]
right_pair_ids = sym_mg["Pair ID"][n_pairs : 2 * n_pairs]
print((left_pair_ids == right_pair_ids).all())

sym_mg.make_lcc()
n_pairs = sym_mg.meta["Pair ID"].nunique() - 1
left_pair_ids = sym_mg["Pair ID"][:n_pairs]
right_pair_ids = sym_mg["Pair ID"][n_pairs : 2 * n_pairs]
print((left_pair_ids == right_pair_ids).all())

uni_pair, counts = np.unique(sym_mg["Pair ID"], return_counts=True)
print(np.min(counts))
# %% [markdown]
# #
left_pair_ids = sym_mg["Pair ID"][:n_pairs]
right_pair_ids = sym_mg["Pair ID"][n_pairs : 2 * n_pairs]

latent = lse(sym_mg.adj, n_components=None)
left_latent = latent[:n_pairs, :]
right_latent = latent[n_pairs : 2 * n_pairs, :]
R, scalar = orthogonal_procrustes(left_latent, right_latent)

n_components = latent.shape[1]
class_labels = sym_mg["Class 1"]
n_unique = len(np.unique(class_labels))
sym_mg.meta["Original index"] = range(len(sym_mg.meta))
left_df = sym_mg.meta[sym_mg.meta["Hemisphere"] == "L"]
left_inds = left_df["Original index"].values
left_latent = latent[left_inds, :]
left_latent = left_latent @ R
latent[left_inds, :] = left_latent

latent_cols = [f"dim {i}" for i in range(latent.shape[1])]
latent_df = pd.DataFrame(data=latent, index=sym_mg.meta.index, columns=latent_cols)

df = pd.concat((sym_mg.meta, latent_df), axis=1)
df.index.name = "Skeleton ID"

x = "dim 0"
y = "dim 1"

color_by = "Class 1"
uni_color_classes = np.unique(df[color_by])
colormap = dict(zip(uni_color_classes, cc.glasbey_light))
colors = np.array(itemgetter(*df[color_by])(colormap))
df["colors"] = colors
source = ColumnDataSource(df)

TOOLS = "pan,reset,zoom_in,zoom_out,box_zoom,lasso_select,xbox_select"

scatter_tooltips = [
    ("ID", "@{Skeleton ID}"),
    ("Class 1", "@{Class 1}"),
    ("Class 2", "@{Class 2}"),
    ("Side", "@Hemisphere"),
    ("Name", "@neuron_name"),
    ("Lineage", "@lineage"),
]

stats = PreText(text="HERE", width=500)

def create_figure():

    scatter_fig = figure(
        tools=TOOLS,
        plot_width=900,
        plot_height=800,
        min_border=10,
        min_border_left=50,
        toolbar_location="above",
        x_axis_location=None,
        y_axis_location=None,
        tooltips=scatter_tooltips,
    )
    scatter_fig.xgrid.visible = False
    scatter_fig.ygrid.visible = False

    legend_it = []
    for c in uni_color_classes:
        temp_df = df[df[color_by] == c]
        temp_source = ColumnDataSource(temp_df)
        out = scatter_fig.circle(
            x=x.value,
            y=y.value,
            source=temp_source,
            alpha=0.6,
            line_color=None,
            fill_color="colors",
            # legend_group=color_by,
        )
        legend_it.append((c, [out]))
        # selected = Circle(fill_alpha=1)
        # nonselected = Circle(fill_alpha=0.2)
        # out.selection_glyph = selected
        # out.nonselection_glyph = nonselected

    out = scatter_fig.circle(
        x=x.value,
        y=y.value,
        source=source,
        alpha=0.1,
        line_color=None,
        fill_color="colors",
        # legend_group=color_by,
    )

    legend = Legend(items=legend_it[: len(legend_it) // 2], location=(0, 0))
    legend.click_policy = "hide"
    legend.label_text_font_size = "5pt"
    legend.orientation = "horizontal"
    legend.padding = 0
    legend.margin = 0
    legend2 = Legend(items=legend_it[len(legend_it) // 2 :], location=(0, 0))
    legend2.click_policy = "hide"
    legend2.label_text_font_size = "6pt"
    legend2.orientation = "horizontal"
    legend2.padding = 0
    legend2.margin = 0
    scatter_fig.add_layout(legend, "below")
    scatter_fig.add_layout(legend2, "below")
    return scatter_fig

def update(attr, old, new):
    layout.children[0] = create_figure()

cols = ["Skeleton ID", "Merge Class", "Hemisphere", "Pair ID", "lineage"]

def update_stats(data):
    stats.text = data[cols].to_string(index=False)

def selection_change(attr, old, new):
    selected = source.selected.indices
    data = source.to_df()
    if selected:
        update_stats(data.iloc[selected])

x = Select(title="X-Axis", value="dim 0", options=latent_cols)
x.on_change("value", update)

y = Select(title="Y-Axis", value="dim 1", options=latent_cols)
y.on_change("value", update)

source.selected.on_change("indices", selection_change)

controls = column(x, y, width=200)
left_col = column(create_figure())
right_col = column(row(controls), row(stats))
# bottom_row = row(controls, stats)
layout = row(left_col, right_col)

curdoc().add_root(layout)
curdoc().title = BRAIN_VERSION
show(layout)
# html = file_html(scatter_fig, CDN, "mb_adj")
# with open("full_new_adj.html", "w") as f:
#     f.write(html)

# %%

reason: import numpy,from scipy,import networkx


text:
# First import the libraries that we need to use
import pandas as pd
import requests
import json
from urllib.request import urlopen
from bs4 import BeautifulSoup
from fileDirectoryCreation import *
from tqdm import tqdm
from ta import add_all_ta_features
from ta.momentum import *
from ta.volume import *
from ta.volatility import *
from ta.trend import *
from ta.others import *
from ta.utils import dropna
import glob
import datetime
import numpy as np

year = str(datetime.date.today().year)
print(year)
checkFolderIntegrity()

def fetchCryptoCurrencies():
    '''

    Fetches a list of cryptocurrencies from a JSON file through the Crypto Compare API, and
    then parses the JSON file into a more usable CSV file.

    Source used: https://min-api.cryptocompare.com/data/all/coinlist

    '''
    URL = 'https://min-api.cryptocompare.com/data/all/coinlist'
    response = requests.get(URL)
    storage = response.json()

    df_crypto = pd.DataFrame()

    cryptoID = ''
    cryptoURL = ''
    cryptoImageURL = ''
    cryptoCreationDate = ''
    cryptoName = ''
    cryptoSymbol = ''
    cryptoCoinName = ''
    cryptoFullName = ''
    cryptoDescription = ''
    cryptoAssetTokenStatus = ''
    cryptoAlgorithm = ''
    cryptoProofType = ''
    cryptoSortOrder = ''
    cryptoSponsored = ''
    cryptoTaxonomyAccess = ''
    cryptoTaxonomyFCA = ''
    cryptoTaxonomyFINMA = ''
    cryptoTaxonomyIndustry = ''
    cryptoTaxonomyCollateralizedAsset = ''
    cryptoTaxonomyCollateralizedAssetType = ''
    cryptoTaxonomyCollateralType = ''
    cryptoTaxonomyCollateralInfo = ''
    cryptoWeissRating = ''
    cryptoWeissTechnologyAdoptionRating = ''
    cryptoWeissMarketPerformanceRating = ''
    cryptoIsTrading = ''
    cryptoTotalCoinsMined = ''
    cryptoCirculatingSupply = ''
    cryptoBlockNumber = ''
    cryptoNetHashesPerSecond = ''
    cryptoBlockReward = ''
    cryptoBlockTime = ''
    cryptoAssetLaunchDate = ''
    cryptoAssetWhitepaperUrl = ''
    cryptoAssetWebsiteUrl = ''
    cryptoMaxSupply = ''
    cryptoMktCapPenalty = ''
    cryptoIsUsedInDefi = ''
    cryptoIsUsedInNft = ''

    for bill in tqdm(storage['Data'],ascii=True, bar_format='{l_bar}{bar:30}{r_bar}{bar:-30b}'):
        try:
            cryptoID = storage['Data'][bill]['Id']
        except:
            cryptoID = ''

        try:
            cryptoURL = storage['Data'][bill]['Url']
        except:
            cryptoURL =''

        try:
            cryptoImageURL = storage['Data'][bill]['ImageUrl']
        except:
            cryptoImageURL = ''

        try:
            cryptoCreationDate = storage['Data'][bill]['ContentCreatedOn']
        except:
            cryptoCreationDate = ''

        try:
            cryptoName = storage['Data'][bill]['Name']
        except:
            cryptoName = ''

        try:
            cryptoSymbol = storage['Data'][bill]['Symbol']
        except:
            cryptoSymbol = ''

        try:
            cryptoCoinName = storage['Data'][bill]['CoinName']
        except:
            cryptoCoinName = ''

        try:
            cryptoFullName = storage['Data'][bill]['FullName']
        except:
            cryptoFullName = ''

        try:
            cryptoDescription = storage['Data'][bill]['Description']
        except:
            cryptoDescription = ''

        try:
            cryptoAssetTokenStatus = storage['Data'][bill]['AssetTokenStatus']
        except:
            cryptoAssetTokenStatus = ''

        try:
            cryptoAlgorithm = storage['Data'][bill]['Algorithm']
        except:
            cryptoAlgorithm = ''

        try:
            cryptoProofType = storage['Data'][bill]['ProofType']
        except:
            cryptoProofType = ''

        try:
            cryptoSortOrder = storage['Data'][bill]['SortOrder']
        except:
            cryptoSortOrder = ''

        try:
            cryptoSponsored = storage['Data'][bill]['Sponsored']
        except:
            cryptoSponsored = ''

        try:
            cryptoTaxonomyAccess = storage['Data'][bill]['Taxonomy']['Access']
        except:
            cryptoTaxonomyAccess = ''

        try:
            cryptoTaxonomyFCA = storage['Data'][bill]['Taxonomy']['FCA']
        except:
            cryptoTaxonomyFCA = ''

        try:
            cryptoTaxonomyFINMA = storage['Data'][bill]['Taxonomy']['FINMA']
        except:
            cryptoTaxonomyFINMA = ''

        try:
            cryptoTaxonomyIndustry = storage['Data'][bill]['Taxonomy']['Industry']
        except:
            cryptoTaxonomyIndustry = ''

        try:
            cryptoTaxonomyCollateralizedAsset = storage['Data'][bill]['Taxonomy']['CollateralizedAsset']
        except:
            cryptoTaxonomyCollateralizedAsset = ''

        try:
            cryptoTaxonomyCollateralizedAssetType = storage['Data'][bill]['Taxonomy']['CollateralizedAssetType']
        except:
            cryptoTaxonomyCollateralizedAssetType = ''

        try:
            cryptoTaxonomyCollateralType = storage['Data'][bill]['Taxonomy']['CollateralType']
        except:
            cryptoTaxonomyCollateralType = ''

        try:
            cryptoTaxonomyCollateralInfo = storage['Data'][bill]['Taxonomy']['CollateralInfo']
        except:
            cryptoTaxonomyCollateralInfo = ''

        try:
            cryptoWeissRating = storage['Data'][bill]['Rating']['Weiss']['Rating']
        except:
            cryptoWeissRating = ''

        try:
            cryptoWeissTechnologyAdoptionRating = storage['Data'][bill]['Rating']['Weiss']['TechnologyAdoptionRating']
        except:
            cryptoWeissTechnologyAdoptionRating = ''

        try:
            cryptoWeissMarketPerformanceRating = storage['Data'][bill]['Rating']['Weiss']['MarketPerformanceRating']
        except:
            cryptoWeissMarketPerformanceRating = ''

        try:
            cryptoIsTrading = storage['Data'][bill]['IsTrading']
        except:
            cryptoIsTrading = ''

        try:
            cryptoTotalCoinsMined = storage['Data'][bill]['TotalCoinsMined']
        except:
            cryptoTotalCoinsMined = ''

        try:
            cryptoCirculatingSupply = storage['Data'][bill]['CirculatingSupply']
        except:
            cryptoCirculatingSupply = ''

        try:
            cryptoBlockNumber = storage['Data'][bill]['BlockNumber']
        except:
            cryptoBlockNumber = ''

        try:
            cryptoNetHashesPerSecond = storage['Data'][bill]['NetHashesPerSecond']
        except:
            cryptoNetHashesPerSecond = ''

        try:
            cryptoBlockReward = storage['Data'][bill]['BlockReward']
        except:
            cryptoBlockReward = ''

        try:
            cryptoBlockTime = storage['Data'][bill]['BlockTime']
        except:
            cryptoBlockTime = ''

        try:
            cryptoAssetLaunchDate = storage['Data'][bill]['AssetLaunchDate']
        except:
            cryptoAssetLaunchDate = ''

        try:
            cryptoAssetWhitepaperUrl = storage['Data'][bill]['AssetWhitepaperUrl']
        except:
            cryptoAssetWhitepaperUrl = ''

        try:
            cryptoAssetWebsiteUrl = storage['Data'][bill]['AssetWebsiteUrl']
        except:
            cryptoAssetWebsiteUrl = ''

        try:
            cryptoMaxSupply = storage['Data'][bill]['MaxSupply']
        except:
            cryptoMaxSupply = ''

        try:
            cryptoMktCapPenalty = storage['Data'][bill]['MktCapPenalty']
        except:
            cryptoMktCapPenalty = ''

        try:
            cryptoIsUsedInDefi = storage['Data'][bill]['IsUsedInDefi']
        except:
            cryptoIsUsedInDefi = ''

        try:
            cryptoIsUsedInNft = storage['Data'][bill]['IsUsedInNft']
        except:
            cryptoIsUsedInNft = ''

        new_row = {
        'cryptoID':cryptoID,
        'cryptoURL':cryptoURL,
        'cryptoImageURL':cryptoImageURL,
        'cryptoCreationDate':cryptoCreationDate,
        'cryptoName':cryptoName,
        'cryptoSymbol':cryptoSymbol,
        'cryptoCoinName':cryptoCoinName,
        'cryptoFullName':cryptoFullName,
        'cryptoDescription':cryptoDescription,
        'cryptoAssetTokenStatus':cryptoAssetTokenStatus,
        'cryptoAlgorithm':cryptoAlgorithm,
        'cryptoProofType':cryptoProofType,
        'cryptoSortOrder':cryptoSortOrder,
        'cryptoSponsored':cryptoSponsored,
        'cryptoTaxonomyAccess':cryptoTaxonomyAccess,
        'cryptoTaxonomyFCA':cryptoTaxonomyFCA,
        'cryptoTaxonomyFINMA':cryptoTaxonomyFINMA,
        'cryptoTaxonomyIndustry':cryptoTaxonomyIndustry,
        'cryptoTaxonomyCollateralizedAsset':cryptoTaxonomyCollateralizedAsset,
        'cryptoTaxonomyCollateralizedAssetType':cryptoTaxonomyCollateralizedAssetType,
        'cryptoTaxonomyCollateralType':cryptoTaxonomyCollateralType,
        'cryptoTaxonomyCollateralInfo':cryptoTaxonomyCollateralInfo,
        'cryptoWeissRating':cryptoWeissRating,
        'cryptoWeissTechnologyAdoptionRating':cryptoWeissTechnologyAdoptionRating,
        'cryptoWeissMarketPerformanceRating':cryptoWeissMarketPerformanceRating,
        'cryptoIsTrading':cryptoIsTrading,
        'cryptoTotalCoinsMined':cryptoTotalCoinsMined,
        'cryptoCirculatingSupply':cryptoCirculatingSupply,
        'cryptoBlockNumber':cryptoBlockNumber,
        'cryptoNetHashesPerSecond':cryptoNetHashesPerSecond,
        'cryptoBlockReward':cryptoBlockReward,
        'cryptoBlockTime':cryptoBlockTime,
        'cryptoAssetLaunchDate':cryptoAssetLaunchDate,
        'cryptoAssetWhitepaperUrl':cryptoAssetWhitepaperUrl,
        'cryptoAssetWebsiteUrl':cryptoAssetWebsiteUrl,
        'cryptoMaxSupply':cryptoMaxSupply,
        'cryptoMktCapPenalty':cryptoMktCapPenalty,
        'cryptoIsUsedInDefi':cryptoIsUsedInDefi,
        'cryptoIsUsedInNft':cryptoIsUsedInNft
        }
        df_crypto = df_crypto.append(new_row, ignore_index=True)
        #print(cryptoID+': '+cryptoName)
    df_crypto.to_csv(fileDirectory+'/cryptoABV.csv',index=False)
    print(df_crypto)

def fetchCoinbaseSymbols(file):
    '''

    Utilizes the Coinbase PRO API to grab a list of cryptocurencies supported
    by Coinbase, and their online platform, and saves them to a local CSV file.

    To start, download the file from 'https://api.pro.coinbase.com/products' as a JSON file,
    then feed it through this function to get desired results.

    Args:
    file: the directory your saved JSON file exists in.

    '''
    print(file)
    jsonFile = file
    df = pd.read_json(jsonFile)
    df.to_csv(fileDirectory+'/Crypto/coinbase_listings.csv')
    print(df)

def fetchDailyData(symbol):
    '''
    args: symbol
    You need to input the Cryptocurrency, and the currency you are trying to compare agianst.
    For example, if you want the price of Bitcoin (BTC) agianst the United States Dollar (USD) you input
    etch_daily_data(symbol="BTC/USD")

    Code pre-developed from CryptoDataDownload.com

    Grabs the historical price data for a provided cryptocurrency agianst another currency.

    '''
    # Code pre-developed from CryptoDataDownload
    pair_split = symbol.split('/')  # symbol must be in format XXX/XXX ie. BTC/EUR
    symbol = pair_split[0] + '-' + pair_split[1]
    url = f'https://api.pro.coinbase.com/products/{symbol}/candles?granularity=86400'
    response = requests.get(url)
    if response.status_code == 200:  # check to make sure the response from server is good
        data = pd.DataFrame(json.loads(response.text), columns=['unix', 'low', 'high', 'open', 'close', 'volume'])
        data.sort_values(by=['unix'],inplace=True)
        data['date'] = pd.to_datetime(data['unix'], unit='s')  # convert to a readable date
        data['vol_fiat'] = data['volume'] * data['close']      # multiply the BTC volume by closing price to approximate fiat volume
        data['Symbol'] = symbol
        Momentum_AOI = AwesomeOscillatorIndicator(high=data.high, low=data.low)
        data['AOI'] = Momentum_AOI.awesome_oscillator()

        Momentum_KARMA = KAMAIndicator(close=data.close)
        data['KAMA'] = Momentum_KARMA.kama()

        Momentum_PPO = PercentagePriceOscillator(close=data.close)
        data['PPO'] = Momentum_PPO.ppo()
        data['PPO_Histogram'] = Momentum_PPO.ppo_hist()
        data['PPO_Signal'] = Momentum_PPO.ppo_hist()

        Momentum_PVO = PercentageVolumeOscillator(volume=data.volume)
        data['PVO'] = Momentum_PVO.pvo()
        data['PVO_Histogram'] = Momentum_PVO.pvo_hist()
        data['PVO_Signal'] = Momentum_PVO.pvo_signal()

        Momentum_RSI = RSIIndicator(close=data.close,window=14)
        data['RSI'] = Momentum_RSI.rsi()

        Momentum_StochasticOscillator = StochasticOscillator(data.high, data.low, data.close)
        data['Stochastic Oscillator'] = Momentum_StochasticOscillator.stoch()
        data['Stochastic Oscillator_Signial'] = Momentum_StochasticOscillator.stoch_signal()

        Momentum_UltimateOscillator = UltimateOscillator(high=data.high, low=data.low, close=data.close)
        data['Ultimate Oscillator'] = Momentum_UltimateOscillator.ultimate_oscillator()

        Momentum_Williams_R = WilliamsRIndicator(high=data.high,low=data.low,close=data.close)
        data['Williams %R'] = Momentum_Williams_R.williams_r()

        Volume_ADI = AccDistIndexIndicator(high=data.high,low=data.low,close=data.close,volume=data.volume)
        data['ADI'] = Volume_ADI.acc_dist_index()

        Volume_CMF = ChaikinMoneyFlowIndicator(high=data.high,low=data.low,close=data.close,volume=data.volume)
        data['CMF'] = Volume_CMF.chaikin_money_flow()

        Volume_FI = ForceIndexIndicator(close=data.close,volume=data.volume)
        data['FI'] = Volume_FI.force_index()

        Volume_MFI = MFIIndicator(high=data.high,low=data.low,close=data.close,volume=data.volume)
        data['MFI'] = Volume_MFI.money_flow_index()

        Volume_VWAP = VolumeWeightedAveragePrice(high=data.high,low=data.low,close=data.close,volume=data.volume)
        data['VWAP'] = Volume_VWAP.volume_weighted_average_price()

        Volitility_Bollinger = BollingerBands(close=data.close)
        data['Bollinger High Band'] = Volitility_Bollinger.bollinger_hband()
        data['Bollinger High Band Indicator'] = Volitility_Bollinger.bollinger_hband_indicator()
        data['Bollinger Low Band'] = Volitility_Bollinger.bollinger_lband()
        data['Bollinger Low Band Indicator'] = Volitility_Bollinger.bollinger_lband_indicator()
        data['Bollinger Middle Band'] = Volitility_Bollinger.bollinger_mavg()
        data['Bollinger Percetage Band'] = Volitility_Bollinger.bollinger_pband()
        data['Bollinger Band Width'] = Volitility_Bollinger.bollinger_wband()

        Volitility_Donchian = DonchianChannel(high=data.high,low=data.low,close=data.close)
        data['Donchian Channel High Band'] = Volitility_Donchian.donchian_channel_hband()
        data['Donchian Channel Low Band'] = Volitility_Donchian.donchian_channel_lband()
        data['Donchian Channel Middle Band'] = Volitility_Donchian.donchian_channel_mband()
        data['Donchian Channel Percentage Band'] = Volitility_Donchian.donchian_channel_pband()
        data['Donchian Channel Band Width'] = Volitility_Donchian.donchian_channel_wband()

        Volatility_Keltner = KeltnerChannel(high=data.high,low=data.low,close=data.close)
        data['Keltner Channel High Band'] = Volatility_Keltner.keltner_channel_hband()
        data['Keltner Channel Indicator Crossing High Band'] = Volatility_Keltner.keltner_channel_hband_indicator()
        data['Keltner Channel Low Band'] = Volatility_Keltner.keltner_channel_lband()
        data['Keltner Channel Indicator Crossing Low Band'] = Volatility_Keltner.keltner_channel_lband_indicator()
        data['Keltner Channel Middle Band'] = Volatility_Keltner.keltner_channel_mband()
        data['Keltner Channel Percentage Band'] = Volatility_Keltner.keltner_channel_pband()
        data['Keltner Channel Band Width'] = Volatility_Keltner.keltner_channel_wband()

        Volatility_Ulcer = UlcerIndex(close=data.close)
        data['Ulcer Index'] = Volatility_Ulcer.ulcer_index()

        try:
            Volatility_ATR = average_true_range(high=data.high,low=data.low,close=data.close)
            data['ATR'] = Volatility_ATR
        except:
            pass

        Trend_Aroon = AroonIndicator(close=data.close)
        data['Aroon Down Channel'] = Trend_Aroon.aroon_down()
        data['Aroon Indicator'] = Trend_Aroon.aroon_indicator()
        data['Aroon Up Channel'] = Trend_Aroon.aroon_up()

        Trend_CCI = CCIIndicator(high=data.high,low=data.low,close=data.close)
        data['CCI'] = Trend_CCI.cci()

        Trend_DPO = DPOIndicator(close=data.close)
        data['DPO'] = Trend_DPO.dpo()

        Trend_EMA = EMAIndicator(close=data.close)
        data['EMA'] = Trend_EMA.ema_indicator()

        Trend_Ichimoku = IchimokuIndicator(high=data.high,low=data.low)
        data['Ichimoku Senkou Span A'] = Trend_Ichimoku.ichimoku_a()
        data['Ichimoku Senkou Span B'] = Trend_Ichimoku.ichimoku_b()
        data['Ichimoku Kujun-sen'] = Trend_Ichimoku.ichimoku_base_line()
        data['Ichimoku Tenkan-Sen'] = Trend_Ichimoku.ichimoku_conversion_line()

        Trend_KST = KSTIndicator(close=data.close)
        data['KST'] = Trend_KST.kst()
        data['Diff KST'] = Trend_KST.kst_diff()
        data['Signal Line KST'] = Trend_KST.kst_sig()

        Trend_MACD = MACD(close=data.close)
        data['MACD Line'] = Trend_MACD.macd()
        data['MACD Histogram'] = Trend_MACD.macd_diff()
        data['MACD Signal Line'] = Trend_MACD.macd_signal()

        Trend_MI = MassIndex(high=data.high,low=data.low)
        data['Mass Index'] = Trend_MI.mass_index()

        Trend_PSAR = PSARIndicator(high=data.high,low=data.low,close=data.close)
        data['PSAR'] = Trend_PSAR.psar()
        data['PSAR Down'] = Trend_PSAR.psar_down()
        data['PSAR Down Indicator'] = Trend_PSAR.psar_down_indicator()
        data['PSAR Up'] = Trend_PSAR.psar_up()
        data['PSAR Up Indicator'] = Trend_PSAR.psar_up_indicator()

        Trend_STC = STCIndicator(close=data.close)
        data['STC'] = Trend_STC.stc()

        Trend_TRIX = TRIXIndicator(close=data.close)
        data['TRIX'] = Trend_TRIX.trix()

        Trend_Vortex = VortexIndicator(high=data.high,low=data.low,close=data.close)
        data['VI'] = Trend_Vortex.vortex_indicator_diff()
        data['VI-'] = Trend_Vortex.vortex_indicator_neg()
        data['VI+'] = Trend_Vortex.vortex_indicator_pos()

        Trend_WMA = WMAIndicator(close=data.close)
        data['WMA'] = Trend_WMA.wma()

        Other_CR = CumulativeReturnIndicator(close=data.close)
        data['CR'] = Other_CR.cumulative_return()

        Other_DLR = DailyLogReturnIndicator(close=data.close)
        data['DLR'] = Other_DLR.daily_log_return()

        Other_DR = DailyReturnIndicator(close=data.close)
        data['DR'] = Other_DR.daily_return()
        # if we failed to get any data, print an error...otherwise write the file
        if data is None:
            print("Did not return any data from Coinbase for this symbol")
        else:
            if(pair_split[1] == "EUR"):
                data.to_csv(fileDirectory+'/Crypto/EUR/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+'dailydata.csv', index=False)
            elif(pair_split[1] == "GBP"):
                data.to_csv(fileDirectory+'/Crypto/GBP/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "USD"):
                data.to_csv(fileDirectory+'/Crypto/USD/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "USDC"):
                data.to_csv(fileDirectory+'/Crypto/USDC/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "USDT"):
                data.to_csv(fileDirectory+'/Crypto/USDT/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "UST"):
                data.to_csv(fileDirectory+'/Crypto/UST/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "BTC"):
                data.to_csv(fileDirectory+'/Crypto/USDT/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "ETH"):
                data.to_csv(fileDirectory+'/Crypto/UST/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            elif(pair_split[1] == "DAI"):
                data.to_csv(fileDirectory+'/Crypto/UST/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)
            else:
                data.to_csv(fileDirectory+'/Crypto/Coinbase_'+ pair_split[0] +'-'+ pair_split[1] + '_'+year+ '_dailydata.csv', index=False)

    else:
        print("Did not receieve OK response from Coinbase API")

def fetchAllCoinbase():
    coinbaseSymbols = pd.read_csv(fileDirectory+'/Crypto/coinbase_listings.csv')
    df = coinbaseSymbols['id'].to_numpy()
    for c in tqdm(df.T, ascii=True, bar_format='{l_bar}{bar:30}{r_bar}{bar:-30b}'):
        fetchDailyData(str(c).replace('-','/'))

def mergeCoinbaseFiles():
    main_df = pd.DataFrame()
    l = fileDirectory + "/Crypto/USD"
    f = 0
    for file in glob.iglob(l+"/*csv"):
        f = f + 1

    for file in tqdm(glob.iglob(l+"/*csv"),total=f,ascii=True, bar_format='{l_bar}{bar:30}{r_bar}{bar:-30b}'):
        df = pd.read_csv(file)
        main_df = pd.concat([main_df, df], ignore_index=True)
    main_df.to_csv(fileDirectory + "/Crypto/Merged"+year+".csv", index=False)

#if __name__ == "__main__":
#    # we set which pair we want to retrieve data for
#    pair = "BTC/USD"
#    fetch_daily_data(symbol="BTC/USD")

max_stars_repo_name: armstjc/Investing-wrapper

max_issues_repo_name: armstjc/Investing-wrapper

max_forks_repo_name: armstjc/Investing-wrapper


text:
from __future__ import absolute_import
import numpy as np
from keras_contrib import backend as K
from keras.initializers import *
from keras.initializers import _compute_fans

class ConvolutionAware(Initializer):
    """
    Initializer that generates orthogonal convolution filters in the fourier
    space. If this initializer is passed a shape that is not 3D or 4D,
    orthogonal intialization will be used.
    # Arguments
        eps_std: Standard deviation for the random normal noise used to break
        symmetry in the inverse fourier transform.
        seed: A Python integer. Used to seed the random generator.
    # References
        Armen Aghajanyan, https://arxiv.org/abs/1702.06295
    """

    def __init__(self, eps_std=0.05, seed=None):
        self.eps_std = eps_std
        self.seed = seed
        self.orthogonal = Orthogonal()

    def __call__(self, shape):
        rank = len(shape)

        if self.seed is not None:
            np.random.seed(self.seed)

        fan_in, fan_out = _compute_fans(shape, K.image_data_format())
        variance = 2 / fan_in

        if rank == 3:
            row, stack_size, filters_size = shape

            transpose_dimensions = (2, 1, 0)
            kernel_shape = (row,)
            correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0])
            correct_fft = np.fft.rfft

        elif rank == 4:
            row, column, stack_size, filters_size = shape

            transpose_dimensions = (2, 3, 0, 1)
            kernel_shape = (row, column)
            correct_ifft = np.fft.irfft2
            correct_fft = np.fft.rfft2

        elif rank == 5:
            x, y, z, stack_size, filters_size = shape

            transpose_dimensions = (3, 4, 0, 1, 2)
            kernel_shape = (x, y, z)
            correct_fft = np.fft.rfftn
            correct_ifft = np.fft.irfftn
        else:
            return K.variable(self.orthogonal(shape), dtype=K.floatx())

        kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape

        init = []
        for i in range(filters_size):
            basis = self._create_basis(
                stack_size, np.prod(kernel_fourier_shape))
            basis = basis.reshape((stack_size,) + kernel_fourier_shape)

            filters = [correct_ifft(x, kernel_shape) +
                       np.random.normal(0, self.eps_std, kernel_shape) for
                       x in basis]

            init.append(filters)

        # Format of array is now: filters, stack, row, column
        init = np.array(init)
        init = self._scale_filters(init, variance)
        return init.transpose(transpose_dimensions)

    def _create_basis(self, filters, size):
        if size == 1:
            return np.random.normal(0.0, self.eps_std, (filters, size))

        nbb = filters // size + 1
        li = []
        for i in range(nbb):
            a = np.random.normal(0.0, 1.0, (size, size))
            a = self._symmetrize(a)
            u, _, v = np.linalg.svd(a)
            li.extend(u.T.tolist())
        p = np.array(li[:filters], dtype=K.floatx())
        return p

    def _symmetrize(self, a):
        return a + a.T - np.diag(a.diagonal())

    def _scale_filters(self, filters, variance):
        c_var = np.var(filters)
        p = np.sqrt(variance / c_var)
        return filters * p

    def get_config(self):
        return {
            'eps_std': self.eps_std,
            'seed': self.seed
        }

max_stars_repo_name: arodiss/keras-contrib

max_issues_repo_name: arodiss/keras-contrib

max_forks_repo_name: arodiss/keras-contrib


text:
import os
import sys
from typing import *

import numpy as np
import matplotlib.pyplot as plt

class WeightPlotters(object):
    @staticmethod
    def wgts_hist(
            dirname : str,
            idx_str : str,
            bins_gen : Tuple[float,float,int] = (-3.0, 2.0, 50),
            sample_range : Tuple[int, int, int] = (0, 3000, 100),
            show : bool = True,
        ):
        """plot the histogram of weights and how they change over time

        for n in the sample range, grabs the files '{dirname}{idx_str}/weights_{idx_str}_e-{n}.csv'
        and plots a histogram of weights changing over time

        # Parameters:
         - `dirname : str`
           directory to look in
         - `idx_str : str`
           string corresponding to the run
         - `bins_gen : Tuple[float,float,int]`
           passed to `np.linspace` to generate bins. fmt is: start, stop, n_bins
           (defaults to `(-3.0, 2.0, 50)`)
         - `sample_range : Tuple[int, int, int]`
            for which sample counts to plot. fmt is: start, stop, step. make sure the step actually matches the sample counts at which the weights were saved.
           (defaults to `(0, 3000, 100)`)
         - `show : bool`
           whether to show the plot before returning. Set to false if youre plotting this in something else
           (defaults to `True`)
        """

        fig, ax = plt.subplots(1,1)
        bins = np.linspace(*bins_gen)

        print('reading files:')
        for n in range(*sample_range):
            # filename = dirname + idx_str + '/weights_' + idx_str + '_e-%d.csv' % n
            filename = f'{dirname}{idx_str}/weights_{idx_str}_e-{n}.csv'
            print('\t' + filename)

            if not os.path.isfile(filename):
                break
            else:
                all_weights = np.genfromtxt(filename, delimiter=',').flatten()
                hist, _ = np.histogram(all_weights,bins)
                ax.set_title(dirname)

                plt.plot(
                    bins[:-1],
                    hist,
                    label = f'e={n}',
                    c = [(n/100) / 30, 0.2, 0.2],
                )
        if show:
            plt.show()

    @staticmethod
    def wgts_hist_neg_percent(
            dirname : str,
            idx_str : str,
            bins_gen : Tuple[float,float,int] = (-3.0, 2.0, 50),
            sample_range : Tuple[int, int, int] = (0, 3000, 100),
            show : bool = True,
        ):
        """shows how the proportion of negative weights changes as more samples are included

        # Parameters:
         - `dirname : str`
           directory to look in
         - `idx_str : str`
           string corresponding to the run
         - `bins_gen : Tuple[float,float,int]`
           passed to `np.linspace` to generate bins. fmt is: start, stop, n_bins
           (defaults to `(-3.0, 2.0, 50)`)
         - `sample_range : Tuple[int, int, int]`
            for which sample counts to plot. fmt is: start, stop, step. make sure the step actually matches the sample counts at which the weights were saved.
           (defaults to `(0, 3000, 100)`)
         - `show : bool`
           whether to show the plot before returning. Set to false if youre plotting this in something else
           (defaults to `True`)
        """

        bins = np.linspace(*bins_gen)
        x = []
        y = []
        print('reading files:')
        for n in range(*sample_range):
            x.append(n)
            filename = f'{dirname}{idx_str}/weights_{idx_str}_e-{n}.csv'
            print('\t' + filename)

            if not os.path.isfile(filename):
                break
            else:
                all_weights = np.genfromtxt(filename, delimiter=',').flatten()
                n_neg = 0
                for w in all_weights:
                    if w < 0:
                        n_neg += 1
                y.append(sample_range[2] * n_neg / len(all_weights))

        plt.plot(x, y)

        if show:
            plt.show()

    @staticmethod
    def percept(
            dirname : str,
            idx_str : str,
            sample : int,
            figax_objs : Optional[Tuple[plt.figure, plt.axes]] = None,
            show : bool = True,
        ):
        """plot the perceptive field for the given run

        gets the file using the pattern:
        '{dirname}{idx_str}/weights_{idx_str}_e-{sample}.csv'

        # Parameters:
         - `dirname : str`
         - `idx_str : str`
         - `sample : int`
           sample count at which to get the weights. make sure the weights were actually saved for this sample count
         - `figax_objs : Optional[Tuple[plt.figure, plt.axes]]`
           figure and axis on which to plot this. if none given, creates a few figure
           (defaults to `None`)
         - `show : bool`
           whether to show the figure. set to false if wrapping
           (defaults to `True`)
        """

        if figax_objs is None:
            fig, ax = plt.subplots(1,1)
        else:
            fig, ax = figax_objs

        filename = f'{dirname}{idx_str}/weights_{idx_str}_e-{sample}.csv'
        W = np.genfromtxt(filename, delimiter=',')

        X = range(0, W.shape[0])
        Y = range(0, W.shape[1])

        im = ax.imshow(W, cmap='seismic', interpolation = 'nearest')

        ax.set_title("sample = {sample}, idx_str = {idx_str}")
        cbar = ax.figure.colorbar(im, ax=ax)

        if show:
            plt.show()

    @staticmethod
    def percept_toarr(
            dirname : str,
            idx_str : str,
            fig : plt.figure, ax : plt.axes,
            sample : int,
            row_idx : int,
        ):
        """used for plotting the big grid of perceptive fields"""
        filename = f'{dirname}{idx_str}/weights_{idx_str}_e-{sample}.csv'
        W = np.genfromtxt(filename, delimiter=',')
        W = W[row_idx][:-1]
        W = W.reshape((28, 28))

        X = range(0, W.shape[0])
        Y = range(0, W.shape[1])

        im = ax.imshow(W, cmap='seismic', interpolation='nearest')

    #    ax.set_title("e %d, r %d" % (sample, row_idx))
    #    cbar = ax.figure.colorbar(im, ax=ax)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

    @staticmethod
    def perceptive_fields(
            dirname : str,
            sample : int = 1900,
            save : Optional[str] = 'perceptive_fields.png',
            show : bool = True,
        ):
        """plots a grid of perceptive fields

        # Parameters:
         - `dirname : str`
           where to find the data
         - `sample : int`
           which sample count to make the plot for
           (defaults to `1900`)
         - `save : Optional[str]`
           where to save the figure. skips if `Non`
           (defaults to `'perceptive_fields.png'`)
         - `show : bool`
           whether to show the figure
           (defaults to `True`)
        """

        fig, axs = plt.subplots(10, 10)
        for r in range(100):
            print(r)
            WeightPlotters.percept_toarr(dirname, 'W1', fig, axs[r % 10][r // 10], sample, r)
        plt.subplots_adjust(
                left=0.0,
                right=0.45,
                bottom=0.1,
                top=0.9,
                wspace=0.0,
                hspace=0.1)

        #    fig.tight_layout()

        if save is not None:
            plt.savefig(save)

        if show:
            plt.show()

if __name__ == "__main__":
    import fire
    fire.Fire(WeightPlotters)

#    fig, axs = plt.subplots(4,1)
#    heatmap(sys.argv[1], 'W2', fig, axs[0], 0)
#    heatmap(sys.argv[1], 'W2', fig, axs[1], 3000)
#    heatmap(sys.argv[1], 'W2', fig, axs[2], 7000)
#    heatmap(sys.argv[1], 'W2', fig, axs[3], 11900)

#    plot_weights_hist(sys.argv[1], "W1")
#    plot_weights_neg_percent(sys.argv[1], "W1")

#    epochs = [0, 500, 1000, 1500, 1900 ]
#    rows = [70 + i for i in range(30)]
#    fig, axs = plt.subplots(len(epochs), len(rows))
#
#    for i, r in enumerate(rows):
#        for j, e in enumerate(epochs):
#            print(i, j)
#            heatmap2(sys.argv[1], 'W1', fig, axs[j][i], e, r)

max_stars_repo_name: knc-neural-calculus/HH-neurons-learning

max_issues_repo_name: knc-neural-calculus/HH-neurons-learning

max_forks_repo_name: knc-neural-calculus/HH-neurons-learning