text:
import pandas as pd
import numpy as np
import logging
import os
import geojson

import math
import itertools

import geopandas as gpd
from geopy.geocoders import Nominatim
from shapely.geometry import Point, Polygon, MultiPolygon, shape
import shapely.ops
from pyproj import Proj

from bs4 import BeautifulSoup
import requests
from abc import ABC, abstractmethod

from .geolocations import *
from .visualizations import *

logging.basicConfig(level=logging.WARNING)

class OpinionNetworkModel(ABC):
    """ Abstract base class for network model """

    def __init__(self,
                probabilities = [.45,.1,.45],
                power_law_exponent = 1.5,
                openness_to_neighbors = 1.5,
                openness_to_influencers = 1.5,
                distance_scaling_factor = 1/10,
                importance_of_weight = 1.6,
                importance_of_distance = 8.5,
                include_opinion = True,
                include_weight = True,
                left_reach = 0.8,
                right_reach = 0.8,
                threshold = -1
                ):
        """Returns initialized OpinionNetworkModel.

        Inputs:
            probabilities: (list) probabilities of each mode; these are the
                values "p_0,p_1,p_2" from [1].
            power_law_exponent: (float) exponent of power law, must be > 0;
                this is "gamma" from [1].
            openness_to_neighbors: (float) maximum inter-mode distance that agents
                can influence; this is "b" from [1].
            openness_to_influencers: (float) distance in opinion space that
                mega-influencers can reach; this is "epsilon" from [1].
            distance_scaling_factor: (float) Scale distancy by this amount, must
                be >0; this is "lambda" from [1].
            importance_of_weight: (float) Raise weights to this power, must be > 0;
                this is "alpha" from [1].
            importance_of_distance: (float) Raise adjusted distance to this power,
                must be > 0; this is "delta" from [1].
            include_opinion: (boolean) If True, include distance in opinion space
                in the probability measure.
            include_weight: (boolean) If True, include influencer weight in the
                probability measure.
            left_reach: (float) this is the proportion of the susceptible population
                that the left mega-influencers will actually reach, must be between
                0 and 1; this is p_L from [1]
            right_reach: (float) this is the proportion of the susceptible population
                that the right mega-influencers will actually reach, must be between
                0 and 1; this is p_R from [1]

            threshold: (int) value below which opinions no longer change.

        Outputs:
            Fully initialized OpinionNetwork instance.
        """
        self.probabilities = probabilities
        self.power_law_exponent = power_law_exponent
        self.openness_to_neighbors = openness_to_neighbors
        self.openness_to_influencers = openness_to_influencers
        self.distance_scaling_factor = distance_scaling_factor
        self.importance_of_weight = importance_of_weight
        self.importance_of_distance = importance_of_distance
        self.include_opinion = include_opinion
        self.include_weight = include_weight
        self.left_reach = left_reach
        self.right_reach = right_reach
        self.threshold = threshold

        self.agent_df = None
        self.belief_df = None
        self.prob_df = None
        self.adjacency_df = None
        self.mega_influencer_df = None

        self.clustering_coefficient = 0
        self.mean_degree = 0

    def populate_model(self, num_agents = None, geo_df = None, bounding_box = None, show_plot = False):
        """ Fully initialized but untrained OpinionNetworkModel instance.

        Input:
            num_agents: (int) number of agents to plot.
            geo_df: (dataframe) geographic datatframe including county geometry.
            bounding_box: (list) list of 4 vertices determining a bounding box
                where agents are to be added.  If no box is given, agents are added
                to a random triangle.
            show_plot: (bool) if true then plot is shown.

        Output:
            OpinionNetworkModel instance.
        """
        if bounding_box is None:
            agent_df = self.add_random_agents_to_triangle(num_agents = num_agents,
                                                        geo_df = geo_df,
                                                        show_plot = False)
        else:
            if geo_df is None:
                raise ValueError("If a bounding box is specified, then a "
                    "geo_df must also be given.")
            agent_df = self.add_random_agents_to_triangles(geo_df = geo_df,
                                                        bounding_box = bounding_box,
                                                        show_plot = False)

        logging.info("\n {} agents added.".format(agent_df.shape[0]))

        belief_df = self.assign_weights_and_beliefs(agent_df)
        logging.info("\n Weights and beliefs assigned.")

        prob_df = self.compute_probability_array(belief_df)
        adjacency_df = self.compute_adjacency(prob_df)
        logging.info("\n Adjacencies computed.")

        # Connect mega-influencers
        mega_influencer_df = self.connect_mega_influencers(belief_df)

        # Compute network statistics.
        logging.info("\n Computing network statistics...")
        cc, md = self.compute_network_stats(adjacency_df)
        logging.info("\n Clustering Coefficient: {}".format(cc))
        logging.info("\n Mean Degree: {}".format(md))

        self.agent_df = agent_df
        self.belief_df = belief_df
        self.prob_df = prob_df
        self.adjacency_df = adjacency_df
        self.mega_influencer_df = mega_influencer_df
        self.clustering_coefficient = cc
        self.mean_degree = md

        if show_plot == True:
            self.plot_initial_network()

        return None

    def plot_initial_network(self):
        plot_network(self)
        return None

    def add_random_agents_to_triangle(self, num_agents, geo_df = None, triangle_object = None,
        show_plot = False):
        """ Assign N points on a triangle using Poisson point process.

        Input:
            num_agents: (int) number of agents to add to the triangle.  If None,
                then agents are added according to density.
            geo_df: (dataframe) geographic datatframe including county geometry.
            triangle_object: (Polygon) bounded triangular region to be populated.
            show_plot: (bool) if true then plot is shown.

        Returns:
            An num_agents x 2 dataframe of point coordinates.
        """
        if triangle_object is None:
            # If no triangle is given, initialize triangle with area 1 km^2.
            triangle_object = Polygon([[0,0],[1419,0], [1419/2,1419],[0,0]])

            # If density is specified, adjust triangle size.
            if geo_df is not None:
                density = geo_df.loc[0,"density"]
                b = 1419 * (num_agents/density) ** (1/2)
                triangle_object = Polygon([[0,0],[b,0], [b/2,b],[0,0]])

        bnd = list(triangle_object.boundary.coords)
        gdf = gpd.GeoDataFrame(geometry = [triangle_object])

        # Establish initial CRS
        gdf.crs = "EPSG:3857"

        # Set CRS to lat/lon
        gdf = gdf.to_crs(epsg=4326)

        # Extract coordinates
        co = list(gdf.loc[0,"geometry"].exterior.coords)
        lon, lat = zip(*co)
        pa = Proj(
            "+proj=aea +lat_1=37.0 +lat_2=41.0 +lat_0=39.0 +lon_0=-106.55")
        x, y = pa(lon, lat)
        coord_proj = {"type": "Polygon", "coordinates": [zip(x, y)]}
        area = shape(coord_proj).area / (10 ** 6) # area in km^2

        # Get Vertices
        V1 = np.array(bnd[0])
        V2 = np.array(bnd[1])
        V3 = np.array(bnd[2])

        # Sample from uniform distribution on [0,1]
        U = np.random.uniform(0,1,num_agents)
        V = np.random.uniform(0,1,num_agents)

        UU = np.where(U + V > 1, 1-U, U)
        VV = np.where(U + V > 1, 1-V, V)

        # Shift triangle into origin and and place points.
        agents = (UU.reshape(len(UU),-1) * (V2 - V1).reshape(-1,2)) + (
                                VV.reshape(len(VV),-1) * (V3 - V1).reshape(-1,2))

        # Shift points back to original position.
        agents = agents + V1.reshape(-1,2)
        agent_df = pd.DataFrame(agents, columns = ["x","y"])

        if show_plot == True:
            plot_agents_on_triangle(triangle_object, agent_df)

        return agent_df

    def add_random_agents_to_triangles(self, geo_df, bounding_box = None, show_plot = False):
        """ Plots county with triangular regions.

        Inputs:
            geo_df: (dataframe) geographic datatframe including county geometry.
            bounding_box: (list) list of 4 vertices determining a bounding box
                where agents are to be added.  If no box is given, then the
                bounding box is taken as the envelope of the county.
            show_plot: (bool) if true then plot is shown.

        Returns:
            Populated triangles in specified county enclosed in the given
            bounding box where regions are filled with proper density using a Poisson
            point process.
        """
        tri_dict = make_triangulation(geo_df)
        tri_df = gpd.GeoDataFrame({"geometry":[Polygon(t) for t in tri_dict["geometry"]["coordinates"]]})

        # Establish initial CRS
        tri_df.crs = "EPSG:3857"

        # Set CRS to lat/lon.
        tri_df = tri_df.to_crs(epsg=4326)

        # Get triangles within bounding box.
        if bounding_box is None:
            geo_df.crs = "EPSG:3857"
            geo_df = geo_df.to_crs(epsg=4326)
            sq_df = gpd.GeoDataFrame(geo_df["geometry"])
        else:
            sq_df = gpd.GeoDataFrame({"geometry":[Polygon(bounding_box)]})
        inset = [i for i in tri_df.index if tri_df.loc[i,"geometry"].within(sq_df.loc[0,"geometry"])]

        # Load triangle area.
        agent_df = pd.DataFrame()
        for i in inset:
            co = list(tri_df.loc[i,"geometry"].exterior.coords)
            lon, lat = zip(*co)
            pa = Proj(
                "+proj=aea +lat_1=37.0 +lat_2=41.0 +lat_0=39.0 +lon_0=-106.55")
            x, y = pa(lon, lat)
            coord_proj = {"type": "Polygon", "coordinates": [zip(x, y)]}
            area = shape(coord_proj).area / (10 ** 6) # area in km^2
            num_agents = int(area * geo_df.loc[0,"density"])
            df = pd.DataFrame(columns = ["x","y"])
            if num_agents > 0:
                df = self.add_random_agents_to_triangle(num_agents,
                                                    geo_df = geo_df,
                                                    triangle_object = tri_df.loc[i,"geometry"],
                                                    show_plot = False)
            agent_df = pd.concat([agent_df,df])

        agent_df.reset_index(drop = True, inplace = True)

        # Plot triangles.
        if show_plot == True:
            fig, ax = plt.subplots(figsize = (10,10))
            tri_df.loc[inset,:].boundary.plot(ax = ax, alpha=1,
                                 linewidth = 3,
                                 edgecolor = COLORS["light_blue"])

            ax.scatter(agent_df["x"], agent_df["y"], s = 3)
            ax.set_axis_off()
            ax.set_aspect(.9)
            plt.show()

        return agent_df

    def assign_weights_and_beliefs(self, agent_df, show_plot = False):
        """ Assign weights and beliefs (i.e. modes) accoring to probabilities.

        Inputs:
            agent_df: (dataframe) xy-coordinates for agents.
            show_plot: (bool) if true then plot is shown.

        Returns:
            Dataframe with xy-coordinates, beliefs, and weights for each point.
        """
        belief_df = agent_df.copy()
        power_law_exponent = self.power_law_exponent
        k = -1/(power_law_exponent)
        modes = [i for i in range(len(self.probabilities))]

        assert np.sum(np.array(self.probabilities)) == 1, "Probabilities must sum to 1."

        belief_df["weight"] = np.random.uniform(0,1,belief_df.shape[0]) ** (k)
        belief_df["belief"] = np.random.choice(modes, belief_df.shape[0],
                                p = self.probabilities)
        belief_df["decile"] = pd.qcut(belief_df["weight"], q = 100, labels = [
                                i for i in range(1,101)])

        if show_plot == True:
            plot_agents_with_belief_and_weight(belief_df)

        return belief_df

    def compute_probability_array(self, belief_df):
        """ Return dataframe of probability that row n influences column m.

        Inputs:
            belief_df: (dataframe) xy-coordinats, beliefs and weights of
                agents.

        Returns:
            Dataframe with the probability that agent n influces agent m
            in row n column m.
        """
        n = belief_df.index.shape[0]
        prob_array = np.ones((n,n))

        dist_array = np.zeros((n,n))
        for i in range(n):
            point_i = Point(belief_df.loc[i,"x"],belief_df.loc[i,"y"])

            # Get distances from i to each other point.
            dist_array[i,:] = [point_i.distance(Point(belief_df.loc[j,"x"],
                                                       belief_df.loc[j,"y"])
                                                    ) for j in belief_df.index]

        # Compute the dimensionless distance metric.
        diam = dist_array.max().max()
        lam = (self.distance_scaling_factor * diam)
        delta = -1 * self.importance_of_distance

        dist_array = np.where(dist_array == 0,np.nan, dist_array)
        dist_array = (1 + (dist_array/lam)) ** delta

        prob_array = prob_array * dist_array

        # Only allow connections to people close in opinion space.
        if self.include_opinion == True:
            op_array = np.zeros((n,n))
            # If row i connects to column j, that means person i is
            # influencing person j.  This can only happen if j is
            # already sufficiently close to person i in opinion space.
            for i in range(n):
                i_current_belief = belief_df.loc[i,"belief"]
                opinion_diff = np.where(
                    np.abs(belief_df["belief"] - i_current_belief
                        ) > self.openness_to_neighbors,0, 1)
                op_array[i,:] = opinion_diff

            prob_array = prob_array * op_array

        # Incentivize connections with heavily weighted people.
        if self.include_weight == True:
            wt_array = belief_df["weight"] ** self.importance_of_weight
            wt_array = wt_array.values.reshape(-1,1)

            prob_array = prob_array * wt_array

        prob_df = pd.DataFrame(prob_array)
        prob_df = prob_df.clip(upper = 1)
        prob_df = prob_df.replace(np.nan,0)

        return prob_df

    def compute_adjacency(self, prob_df):
        """ Compute NxN adjacency dataframe.

        Inputs:
            prob_df: (dataframe) num_agents x num_agents dataframe giving
                probabiltiy of influce row n on column m.

        Outputs:
            Dataframe where row n and column n is a 1 if n influces m,
            and a 0 otherwise.
        """
        adjacency_df = pd.DataFrame(0,index = [int(i) for i in prob_df.index],
            columns = [int(i) for i in prob_df.columns])
        U = np.random.uniform(0,1,(prob_df.shape[0], prob_df.shape[1]))
        adjacency_df = pd.DataFrame(np.where(U< prob_df.values,1,0))

        return adjacency_df

    def compute_network_stats(self, adjacency_df):
        """ Return clustering coefficient and mean degree.

        Inputs:
            adjacency_df: (dataframe) num_agents x num_agents dataframe, where
                a 1 in row n column m indicates that agent n influences agent m.

        Returns:
            Tuple of clustering coefficient and mean degree of
            the network.
        """
        cc = 0
        degrees = []
        for i in adjacency_df.index:
            nbhd = np.where(adjacency_df.loc[:,i] != 0)[0]
            deg = len(nbhd)
            degrees.append(deg)
            cc_i = 0
            if deg > 2:
                C = list(itertools.permutations(nbhd,2))
                cc_i = np.sum([int(adjacency_df.loc[c[0],c[1]] > 0) for c in C]
                                                    ) / math.perm(deg,2)

            cc += cc_i
        clustering_coefficient = cc / adjacency_df.shape[0]

        return clustering_coefficient, np.mean(degrees)

    def connect_mega_influencers(self, belief_df):
        """ Returns mega_influencer reach dataframe

        Inputs:
            belief_df: (dataframe) xy-coordinats, beliefs and weights of
                agents.

        Outputs:
            Dataframe where rows are the keys of reach dict and columns
            are the index of belief_df, where a 1 in row n column m indicates
            that influencer n reaches agent m.
        """
        reach_dict = {0:self.left_reach, 2:self.right_reach}
        mega_influencer_df = pd.DataFrame(0, index = [0,2], columns = list(
            belief_df.index))

        for i in mega_influencer_df.index:
            ind = np.where(np.abs(belief_df["belief"] - i) != 2)[0]
            U = np.random.uniform(0,1,len(ind))
            mega_influencer_df.loc[i,ind[np.where(U < reach_dict[i])[0]]] = 1

        return mega_influencer_df

class NetworkSimulation(ABC):
    """ Abstract base class for learning algorithm.
    """
    def __init__(self):
        self.model = None
        self.stopping_thresh = None
        self.results = []
        self.dynamic_belief_df = None

    def run_simulation(self, model, stopping_thresh = .01, show_plot = False, store_results = False):
        """ Carry out simulation.

        Inputs:
            model: OpinionNetworkModel instance.
            stopping_thresh: (float) threshold for stopping criterion.
            show_plot: (bool) if true, shows ridge plot.
            store_results: (bool) if True, stores results for all phases, if False
                then only updating beliefs are stored.

        Outputs:
            Complete simulation.
        """

        results = []
        self.model = model
        new_belief_df = model.belief_df.copy()
        new_adjacency_df = model.adjacency_df.copy()
        new_mega_influencer_df = model.mega_influencer_df.copy()

        df = pd.DataFrame()
        df[0] = new_belief_df["belief"].values

        self.stopping_thresh = stopping_thresh
        stopping_df = pd.DataFrame()

        i = 0
        while True:
            phase_dict = {}
            new_belief_df, new_mega_influencer_df = self.one_dynamics_iteration(
                            belief_df = new_belief_df,
                            adjacency_df = new_adjacency_df,
                            openness_to_influencers = model.openness_to_influencers,
                            mega_influencer_df = new_mega_influencer_df,
                            threshold = model.threshold)

            phase_dict["belief_df"] = new_belief_df
            prob_df = model.compute_probability_array(new_belief_df)
            new_adjacency_df = model.compute_adjacency(prob_df)
            new_adjacency_df.columns = [int(i) for i in new_adjacency_df.columns]
            phase_dict["adjacency_df"] = new_adjacency_df
            clust_coeff, mean_degree = model.compute_network_stats(new_adjacency_df)
            phase_dict["clust_coeff"] = clust_coeff
            phase_dict["mean_degree"] = mean_degree

            results.append(phase_dict)
            df[i+1] = new_belief_df["belief"].values

            stopping_df[i] = df.iloc[:,i+1] - df.iloc[:,i]
            i = i+1
            # Check that at least 5 iterations have been carried out.
            if stopping_df.shape[1] > 5:
                # If rolling average change is less than stopping_thresh, break.
                if stopping_df.rolling(window = 5, axis = 1).mean().iloc[:,-1
                                                    ].abs().mean() < stopping_thresh:
                    break

        self.dynamic_belief_df = df

        if show_plot == True:
            self.plot_simulation_results()
        if store_results == True:
            self.results = results

        return None

    def plot_simulation_results(self):
        """ Return ridgeplot of simulation.
        """
        df = self.dynamic_belief_df
        phases = df.shape[1] - 1
        if df is None:
            raise ValueError("It looks like the simulation hasn't been run yet.")
        if phases < 5:
            plot_phases = [t for t in range(phases +1)]
        else:
            t = phases // 5
            plot_phases = [0] + [t * (i+1) for i in range(1,5)]
        get_ridge_plot(df, plot_phases, reach_dict = {0:self.model.left_reach,
                                                    2:self.model.right_reach})
        return None

    def one_dynamics_iteration(self,
                        belief_df,
                        adjacency_df,
                        openness_to_influencers,
                        mega_influencer_df,
                        threshold):
        """ Returns updated belief_df.
        Inputs:
            belief_df: (dataframe)
            adjacency_df: (dataframe)
            openness_to_influencers: (float) distance in opinion space that
                mega-influencers can reach; this is "epsilon" from [1].
            threshold: (int) value below which opinions no longer change.

        Returns:
            Updated belief_df after one round of Hegselmann-Krause.
        """
        df_new = belief_df.copy()
        new_mega_influencer_df = mega_influencer_df.copy()

        for i in belief_df.index:
            current_belief = belief_df.loc[belief_df.index[i], "belief"]

            if current_belief > threshold:

                # Sum over column
                edges = np.where(adjacency_df[i] == 1)[0]
                n_edges = len(edges)

                new_belief = np.sum(belief_df.loc[edges,"belief"]) + current_belief

                if openness_to_influencers > 0:
                    # Am  I connected to the right mega-influencer?
                    if mega_influencer_df.loc[2,i] == 1:
                        # Do I listen to them?
                        if np.abs(current_belief - 2) <= openness_to_influencers:
                            n_edges = n_edges + 1
                            new_belief = new_belief + 2
                        # If not, flip a biased coin to decide if my influencer connection
                        # changes affiliation.
                        else:
                            new_mega_influencer_df.loc[2,i] == 0
                            u = np.random.uniform(0,1)
                            if u < self.model.left_reach:
                                new_mega_influencer_df.loc[0,i] == 1

                    # Am  I connected to the left mega-influencer?
                    if mega_influencer_df.loc[0,i] == 1:
                        # Do I listen to them?
                        if np.abs(current_belief  - 0) <= openness_to_influencers:
                            n_edges = n_edges + 1
                            new_belief = new_belief + 0
                        # If not, flip a biased coin to decide if my influencer connection
                        # changes affiliation.
                        else:
                            new_mega_influencer_df.loc[0,i] == 0
                            u = np.random.uniform(0,1)
                            if u < self.model.right_reach:
                                new_mega_influencer_df.loc[2,i] == 1

                new_belief = new_belief / (n_edges + 1)

                df_new.loc[i,"belief"] = new_belief

        return df_new, new_mega_influencer_df

max_stars_repo_name: annahaensch/ODyN

max_issues_repo_name: annahaensch/ODyN

max_forks_repo_name: annahaensch/ODyN


text:
import numpy as np
from scipy.sparse import issparse, csr_matrix
from typing import Union, List

def update_rep(rep: str) -> str:
    """ If rep is None, return rep as mat, which refers to the whole expression matrix
    """
    return rep if rep is not None else "mat"

def X_from_rep(data: "AnnData", rep: str) -> np.array:
    """
    If rep is not mat, first check if X_rep is in data.obsm. If not, raise an error.
    If rep is None, return data.X as a numpy array
    """
    if rep != "mat":
        rep_key = "X_" + rep
        if rep_key not in data.obsm.keys():
            raise ValueError("Cannot find {0} matrix. Please run {0} first".format(rep))
        return data.obsm[rep_key]
    else:
        return data.X if not issparse(data.X) else data.X.toarray()

def W_from_rep(data: "AnnData", rep: str) -> "csr_matrix":
    """
    Return affinity matrix W based on representation rep.
    """
    rep_key = "W_" + rep
    if rep_key not in data.uns:
        raise ValueError("Affinity matrix does not exist. Please run neighbors first!")
    return data.uns[rep_key]

def knn_is_cached(
    data: "AnnData", indices_key: str, distances_key: str, K: int
) -> bool:
    return (
        (indices_key in data.uns)
        and (distances_key in data.uns)
        and data.uns[indices_key].shape[0] == data.shape[0]
        and (K <= data.uns[indices_key].shape[1] + 1)
    )

# slicing is not designed to work at extracting one element, convert to dense matrix
def slicing(X: Union[csr_matrix, np.ndarray], row: Union[List[bool], List[int], int] = slice(None), col: Union[List[bool], List[int], int] = slice(None), copy: bool = False, squeeze: bool = True) -> np.ndarray:
    result = X[row, col]
    if issparse(X):
        result = result.toarray()
    elif copy:
        result = result.copy()
    if squeeze:
        result = np.squeeze(result)
        if result.ndim == 0:
            result = result.item()
    return result

def calc_expm1(X: Union[csr_matrix, np.ndarray]) -> np.ndarray:
    if not issparse(X):
        return np.expm1(X)
    res = X.copy()
    np.expm1(res.data, out = res.data)
    return res

ext: py

max_stars_repo_name: slowkow/pegasus

max_issues_repo_name: slowkow/pegasus

max_forks_repo_name: slowkow/pegasus

reason: import numpy,from scipy


text:
from argparse import ArgumentParser
from os import path, remove
from typing import Dict

import numpy
from tqdm import tqdm

from data_preprocessing.preprocess_code2seq_data import DATA_FOLDER
from utils.common import count_lines_in_file
from random import shuffle, seed

SEED = 7

def _get_id2value_from_csv(path_: str) -> Dict[str, str]:
    return dict(numpy.genfromtxt(path_, delimiter=",", dtype=(str, str))[1:])

def preprocess_csv(data_folder: str, dataset_name: str, holdout_name: str, is_shuffled: bool):
    """
    Preprocessing for files tokens.csv, paths.csv, node_types.csv
    """
    dataset_path = path.join(data_folder, dataset_name)
    id_to_token_data_path = path.join(dataset_path, f"tokens.{holdout_name}.csv")
    id_to_type_data_path = path.join(dataset_path, f"node_types.{holdout_name}.csv")
    id_to_paths_data_path = path.join(dataset_path, f"paths.{holdout_name}.csv")
    path_contexts_path = path.join(dataset_path, f"path_contexts.{holdout_name}.csv")
    output_c2s_path = path.join(dataset_path, f"{dataset_name}.{holdout_name}.c2s")

    id_to_paths = _get_id2value_from_csv(id_to_paths_data_path)
    id_to_paths = {index: [n for n in nodes.split()] for index, nodes in id_to_paths.items()}

    id_to_node_types = _get_id2value_from_csv(id_to_type_data_path)
    id_to_node_types = {index: node_type.rsplit(" ", maxsplit=1)[0] for index, node_type in id_to_node_types.items()}

    id_to_tokens = _get_id2value_from_csv(id_to_token_data_path)

    if path.exists(output_c2s_path):
        remove(output_c2s_path)
    with open(path_contexts_path, "r") as path_contexts_file, open(output_c2s_path, "a+") as c2s_output:
        output_lines = []
        for line in tqdm(path_contexts_file, total=count_lines_in_file(path_contexts_path)):
            label, *path_contexts = line.split()
            parsed_line = [label]
            for path_context in path_contexts:
                from_token_id, path_types_id, to_token_id = path_context.split(",")
                from_token, to_token = id_to_tokens[from_token_id], id_to_tokens[to_token_id]
                nodes = [id_to_node_types[p_] for p_ in id_to_paths[path_types_id]]
                parsed_line.append(",".join([from_token, "|".join(nodes), to_token]))
            output_lines.append(" ".join(parsed_line + ["\n"]))
        if is_shuffled:
            shuffle(output_lines)
        c2s_output.write("".join(output_lines))

if __name__ == "__main__":
    arg_parser = ArgumentParser()
    arg_parser.add_argument("data", type=str)
    arg_parser.add_argument("--shuffle", action="store_true")
    args = arg_parser.parse_args()
    data_path = path.join(DATA_FOLDER, args.data)

    seed(SEED)
    for holdout_name in ["train", "test", "val"]:
        preprocess_csv(DATA_FOLDER, args.data, holdout_name, args.shuffle)

reason: import numpy


text:
import glob
import os
import sys
utils_path = os.path.join(os.path.abspath(os.getenv('PROCESSING_DIR')),'utils')
if utils_path not in sys.path:
    sys.path.append(utils_path)
import util_files
import util_cloud
import util_carto
import logging
from ftplib import FTP
import urllib
import numpy as np
import pandas as pd
from zipfile import ZipFile

# Set up logging
# Get the top-level logger object
logger = logging.getLogger()
for handler in logger.handlers: logger.removeHandler(handler)
logger.setLevel(logging.INFO)
# make it print to the console.
console = logging.StreamHandler()
logger.addHandler(console)
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# name of table on Carto where you want to upload data
# this should be a table name that is not currently in use
dataset_name = 'dis_017_storm_events_us' #check

logger.info('Executing script for dataset: ' + dataset_name)
# create a new sub-directory within your specified dir called 'data'
# within this directory, create files to store raw and processed data
data_dir = util_files.prep_dirs(dataset_name)

'''
Download data and save to your data directory.
Two types of files are being used: details and locations.
Using bulk download via FTP, link to NOAA's storm events database:
https://www.ncdc.noaa.gov/stormevents/ftp.jsp
'''
# connect to the FTP server and login anonymously
ftp = FTP('ftp.ncdc.noaa.gov', timeout = 30)
ftp.login()
# navigate to the correct directory and get a list of all filenames
ftp.cwd('/pub/data/swdi/stormevents/csvfiles/')
filenames = ftp.nlst()

# retrieve a sorted list of the details files
details_files = []
for filename in filenames:
    if not filename.startswith('StormEvents_details-ftp_v1.0_d'):
        continue
    details_files.append(filename)
details_files.sort()

# retrieve a sorted list of the locations files
locations_files = []
for filename in filenames:
    if not filename.startswith('StormEvents_locations-ftp_v1.0_d'):
        continue
    locations_files.append(filename)
locations_files.sort()

def ftp_download(file_dir):
    '''
    download data
    INPUT   file_dir: ftp location of file to download (string)
    '''
    for filename in file_dir:
     with open(os.path.join(data_dir, filename), 'wb') as fo:
        ftp.retrbinary("RETR " + filename, fo.write)
# download data from the source FTP
ftp_download(details_files)
ftp_download(locations_files)

'''
Process data
'''
#Concatenating details and locations files

raw_data_file = glob.glob(os.path.join(data_dir, "*.gz"))     # advisable to use os.path.join as this makes concatenation OS independent

details_list = []
locations_list = []

# go through each file, turn it into a dataframe, and append that df to one of two lists, based on if it
# is a details file or a locations file
for file in raw_data_file:
    if file.startswith('data/StormEvents_details-ftp_v1.0_d'):
        df = pd.read_csv(file)
        details_list.append(df)
    elif file.startswith('data/StormEvents_locations-ftp_v1.0_d'):
        df_1 = pd.read_csv(file)
        locations_list.append(df_1)
    else: print('error')

# concatenate tables for every year into one table for the details files and one table for the locations files
details_concatenated = pd.concat(details_list, ignore_index=True)
locations_concatenated = pd.concat(locations_list, ignore_index=True)

# Select columns of interest from dataset and clean locations dataset
event_locations = locations_concatenated[['EVENT_ID', "LOCATION", "LATITUDE", "LONGITUDE"]]
event_locations = event_locations.replace(to_replace="\s\s*",value = '',regex=True)
# Merge details and details files 'EVENT_ID' and changing column names to lowercase
events = pd.merge(details_concatenated, event_locations, on='EVENT_ID')
# make column names lowercase because Carto only uses lowercase column names
events.columns= events.columns.str.strip().str.lower()

#save processed dataset to csv
processed_data_file = os.path.join(data_dir, dataset_name+'_edit.csv')
events.to_csv(processed_data_file, index=False)

'''
Upload processed data to Carto
'''
logger.info('Uploading processed data to Carto.')
util_carto.upload_to_carto(processed_data_file, 'LINK')

'''
Upload original data and processed data to Amazon S3 storage
'''
# initialize AWS variables
aws_bucket = 'wri-public-data'
s3_prefix = 'resourcewatch/'

logger.info('Uploading original data to S3.')
# Upload raw data file to S3

# Copy the raw data into a zipped file to upload to S3
raw_data_dir = os.path.join(data_dir, dataset_name+'.zip')
with ZipFile(raw_data_dir,'w') as zipped:
    for file in raw_data_file:
        zipped.write(file, os.path.basename(file))

# Upload raw data file to S3
uploaded = util_cloud.aws_upload(raw_data_dir, aws_bucket, s3_prefix + os.path.basename(raw_data_dir))
logger.info('Uploading processed data to S3.')

# Copy the processed data into a zipped file to upload to S3
processed_data_dir = os.path.join(data_dir, dataset_name+'_edit.zip')
with ZipFile(processed_data_dir,'w') as zip:
    zip.write(processed_data_file, os.path.basename(processed_data_file))

# Upload processed data file to S3
uploaded = util_cloud.aws_upload(processed_data_dir, aws_bucket, s3_prefix + os.path.basename(processed_data_dir))

max_stars_repo_name: resource-watch/data-pre-processing

max_issues_repo_name: resource-watch/data-pre-processing

max_forks_repo_name: resource-watch/data-pre-processing


text:
# Copyright (C) 2017 Zhixian MA <zxma_sjtu@qq.com>

"""
Utilities for dicom images processing

Requirements
============
astropy
pydicom
pandas
xlrd

References
==========
[1] pydicom documentation
https://pydicom.readthedoces.io/en/stable/working_with_pixel_data.html
[2] Read and write excel with pandas
http://blog.csdn.net/qq_24683561/article/details/54576520

Methods
=======
load_info(infopath="patient_list.xlsx")
    Load the patient names and identities
sort_filename(foldname)
    readjust filenames and sort
gen_fits(dcmpath,header=None)
    transform the dcm image to fits
gen_fits_cube(foldname, header=None)
    transform the group of dcm files into a cube
"""

import os
import dicom
import pandas as pd
import numpy as np
from astropy.io import fits
from matplotlib import pyplot as plt

def load_info(infopath):
    """
    Load the patient names and identities

    input
    =====
    infopath: str
        The path to save the info file.

    output
    ======
    namelist: dict
        The list hoding name and id of the patients
    """
    # load the excel
    try:
        p_list = pd.read_excel(infopath)
    except:
        print("Something wrong when loading the info file.")
        return None

    # sort
    idx = np.argsort(p_list['name'])
    p_name = p_list['name'][idx]
    p_name.index = np.arange(0, p_name.shape[0])
    p_id = p_list['id'][idx]
    p_id.index = np.arange(0, p_id.shape[0])

    # dict
    namelist = {'name': p_name, 'id': p_id}

    return namelist

def sort_filename(foldname):
    """
    readjust filenames and sort
    """
    try:
        filelist = os.listdir(foldname)
    except:
        print("Folder %s does not exist." % foldname)
        return None

    # first sort
    filelist.sort()
    numfiles = len(filelist)
    filesort = ['' for n in range(numfiles)]

    for f in filelist:
        f_split = f.split('.')
        idx = int(f_split[-2]) - 1
        filesort[idx] = f

    return filesort

def gen_fits(dcmname, header=None):
    """
    Transform dcm image to fits image

    inputs
    ======
    dcmname: str
        File path of the dcm image
    header: dict
        <TODO> The header of the fits file

    output
    ======
    hdu: astropy.io.fits.PrimarHDU
        The single image
    """
    # read the dcm image
    try:
        img_dcm = dicom.read_file(dcmname)
    except:
        print("Something wrong when read the dcm image.")
        return None

    # Initialize hdu
    hdu = fits.ImageHDU()
    if header is not None:
        hdu.header.update(header)
    hdu.data = np.flipud(img_dcm.pixel_array)

    return hdu

def save_fits(hdu, savepath):
    """Save the fits image"""
    hdu.writeto(savepath, clobber=True)

def gen_fits_cube(samplepath, header=None):
    """Generate the fits cube from dcm group

    inputs
    ======
    samplepath: str
        Path of the sample folder
    header: dict
        The header of the fits file

    output
    ======
    cube_hdu: astropy.io.fits.PrimaryHDU
        The fits cube
    """
    # Init a hdu cube
    cube_hdu = fits.ImageHDU()

    # load single extension
    fitslist = sort_filename(samplepath)
    fitslist = fitslist[1:]

    # Initialize the data cube
    data_cube = []

    for f in fitslist:
        # print("Processing on dcm %s " % f[0:-4])
        dcmpath = os.path.join(samplepath,f)
        try:
            img_dcm = dicom.read_file(dcmpath)
        except:
            print("Cannot load dcm %s." % f)
            continue

        data_cube.append(np.flipud(img_dcm.pixel_array))

    cube_hdu.data = np.array(data_cube)
    if header is not None:
        cube_hdu.header.update(header)

    return cube_hdu

def gen_infodict(namelist,keys_idx,p_idx):
    """Genearte the info dict"""
    infodict = []
    for kid in keys_idx:
        key = namelist.keys()[kid]
        value = str(namelist[key][p_idx])
        temp_dict = key + ": " + value
        infodict.append(temp_dict)

    return infodict

def gen_mark_image(infodict,savepath,imgshape=(128,128)):
    """Generate blank image with info marked."""
    # Init
    img = np.zeros(imgshape,dtype='uint8')

    fig = plt.figure(figsize=(imgshape[0]/10,imgshape[1]/10))
    ax = plt.subplot(111)
    ax.imshow(img,cmap='Greys')
    # mark text
    x0 = 0
    y0 = 0
    step = 15
    for info_text in infodict:
        y0 = y0 + step
        ax.text(x0, y0, info_text,fontsize=60)

    plt.box('off')
    ax.set_xticks([])
    ax.set_yticks([])
    # save
    fig.savefig(savepath, dpi=10)

max_stars_repo_name: myinxd/ct-fits

max_issues_repo_name: myinxd/ct-fits

max_forks_repo_name: myinxd/ct-fits

reason: import numpy,from astropy


text:
import unittest

import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn

from pytorch_metric_learning.losses import InstanceLoss

from .. import TEST_DEVICE, TEST_DTYPES

# original loss copied from here
# https://github.com/layumi/Person_reID_baseline_pytorch/blob/master/instance_loss.py
# as a reference to test against
def l2_norm(v):
    fnorm = torch.norm(v, p=2, dim=1, keepdim=True) + 1e-6
    v = v.div(fnorm.expand_as(v))
    return v

class OriginalInstanceLoss(nn.Module):
    def __init__(self, gamma=1) -> None:
        super().__init__()
        self.gamma = gamma

    def forward(self, feature, label=None) -> Tensor:
        # Dual-Path Convolutional Image-Text Embeddings with Instance Loss, ACM TOMM 2020
        # https://zdzheng.xyz/files/TOMM20.pdf
        # using cross-entropy loss for every sample if label is not available. else use given label.
        normed_feature = l2_norm(feature)
        sim1 = torch.mm(normed_feature * self.gamma, torch.t(normed_feature))
        # sim2 = sim1.t()
        if label is None:
            sim_label = torch.arange(sim1.size(0)).cuda().detach()
        else:
            _, sim_label = torch.unique(label, return_inverse=True)
        loss = F.cross_entropy(sim1, sim_label)  # + F.cross_entropy(sim2, sim_label)
        return loss

class TestInstanceLoss(unittest.TestCase):
    def test_instance_loss(self):
        torch.manual_seed(3029)
        for dtype in TEST_DTYPES:
            for gamma in range(1, 256, 32):
                loss_fn = InstanceLoss(gamma=gamma)
                original_loss_fn = OriginalInstanceLoss(gamma=gamma)
                embeddings = torch.randn(
                    32, 128, device=TEST_DEVICE, dtype=dtype, requires_grad=True
                )
                labels = torch.randint(0, 10, size=(32,), device=TEST_DEVICE)
                loss = loss_fn(embeddings, labels)
                loss.backward()

                original_loss = original_loss_fn(embeddings, labels)
                rtol = 1e-3 if dtype == torch.float16 else 1e-6
                self.assertTrue(
                    np.isclose(loss.item(), original_loss.item(), rtol=rtol)
                )

if __name__ == "__main__":
    unittest.main()


text:
"""
Copyright 2021 Intel Corporation

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

# import multiprocessing as mp
# import threading
import numpy as np

loaders = {}

class loader:
    def __init__(self, ftype, is_dist, aliases=[]):
        self.ftype = ftype
        self.is_dist = is_dist
        self.getinfo = globals()[ftype + "_getinfo"]
        self.read = globals()[ftype + "_read"]
        self.readall = globals()[ftype + "_readall"]
        loaders[ftype] = self
        for i in aliases:
            loaders[i] = self

def get_load_handler(fname, ftype=None):
    if ftype is None:
        ftype = fname.split("/")[-1].split(".")[-1].lower()
    if ftype not in loaders:
        print("Error:  unknown file type", ftype)
        ftype = "hdf5"
    return loaders[ftype]

try:
    import h5py

    def hdf5_get_part(fname, nm):
        f = h5py.File(fname, "r")
        if nm is None or nm == "":
            return f
        l = nm.split("/")
        p = f
        for i in l:
            p = p[i]
        return p

    def hdf5_getinfo(fname, arr_path):
        p = hdf5_get_part(fname, arr_path)
        return p.shape, p.dtype

    def hdf5_read(fname, arr, src_index, arr_path, dst_index=None):
        p = hdf5_get_part(fname, arr_path)
        print("hdf5 read direct", p.shape, src_index)
        if dst_index is None:
            p.read_direct(arr, src_index)
        else:
            p.read_direct(arr, src_index, dst_index)

    def hdf5_readall(fname, arr_path):
        p = hdf5_get_part(fname, arr_path)
        return p[:]

    loader("hdf5", True, aliases=["h5ad"])
except:
    print("No HDF5 support")

try:
    from PIL import Image

    # TODO: should handle frames, depths other than 8-bit
    def pil_getinfo(fname):
        img = Image.open(fname)
        h = img.height
        w = img.width
        c = len(img.getbands())
        shape = (c, h, w)
        dtype = np.uint8
        return shape, dtype

    def pil_read():  # distributed, partial loads not supported
        pass

    def pil_readall(fname):
        img = Image.open(fname)
        arr = np.array(img)
        if arr.ndim > 2:
            arr = np.transpose(arr, (2, 0, 1))  # convert from HxWxC to CxHxW
        return arr

    loader("pil", False, aliases=["jpg", "jpeg", "png", "tif", "tiff"])

except:
    print("No PIL support")

max_stars_repo_name: Python-for-HPC/ramba

max_issues_repo_name: Python-for-HPC/ramba

max_forks_repo_name: Python-for-HPC/ramba


text:
__author__ = 'Dmitry Patashov'

import numpy as np

def myHistEqualization(Img):

    HistL = np.zeros(256)
    for i in range(256):
        HistL[i] = len(Img[Img == np.float64(i)])

    PDFfun = HistL / np.sum(HistL)

    CDFfun = np.zeros(PDFfun.shape)
    CDFfun[0] = PDFfun[0]
    for k in range(1, len(CDFfun)):
        CDFfun[k] = CDFfun[k - 1] + PDFfun[k]

    RepImg = np.float64(Img)
    for j in range(256):
        RepImg[RepImg == j] = CDFfun[j] * 255

    return np.uint8(np.round(RepImg))

max_issues_repo_name: haimadrian/MachineLearningUsingPython

max_forks_repo_name: haimadrian/MachineLearningUsingPython


text:
import collections
import numpy as np

def mag(vec):
    """Determines the Euclidean norm of a vector in iterable form.

    Args:
        vec (iterable): some iterable that contains components.

    Returns:
        float: the Euclidean norm of the input.
    """
    leSum = 0
    for i in range(len(vec)):
        leSum += vec[i] ** 2
    return np.sqrt(leSum)

def difference(v1, v2):
    """Returns the difference between two vectors in iterable form, i.e. v2 - v1.

    Args:
        v1 (iterable): some vector.
        v2 (iterable): some vector.

    Returns:
        iterable: the resultant vector.
    """
    if type(v1) == tuple and type(v2) == tuple:
        return tuple(np.subtract(v2, v1))
    else:
        return [ai - bi for ai, bi in zip(v2, v1)]

def subtraction(v1, v2):
    """
    Just difference(), but in reverse, i.e. v1 - v2. Much more intuitive and this is a
    non-breaking change.

    Args:
        v1 (iterable): some vector.
        v2 (iterable): some vector.

    Returns:
        iterable: the resultant vector.
    """
    return difference(v2, v1)

def addition(v1, v2):
    """Returns the addition of two vectors in iterable form, i.e. v1 + v2.

    Args:
        v1 (iterable): some vector.
        v2 (iterable): some vector.

    Returns:
        iterable: the resultant vector.
    """
    if type(v1) == tuple and type(v2) == tuple:
        return tuple(np.add(v1, v2))
    else:
        return [ai + bi for ai, bi in zip(v2, v1)]

def multiply(k, v1):
    """Returns the k*v1 where k is multiplied by each element in v1.

    Args:
        k (float): scale factor.
        v1 (iterable): a vector in iterable form.

    Returns:
        iterable: the resultant vector.
    """
    newIterable = [k * i for i in v1]
    return tuple(newIterable) if type(v1) == tuple else newIterable

def negate(vec):
    """Negates a vector in iterable form.

    Args:
        vec (iterable): a vector iterable.

    Returns:
        iterable: the resultant vector.
    """
    if type(vec) == tuple:
        return tuple([-i for i in vec])
    else:
        return [-i for i in vec]

def flatten(someList):
    """
    Flattens a nested list into a list with singular elements. Example:
    [[[0, 1, 3], [4, 7]], [0, 1], 2]
    turns into
    [0, 1, 3, 4, 7, 0, 1, 2]

    Args:
        someList (iterable): a nested list or iterable.

    Returns:
        list: the flattened version of the list.
    """
    if isinstance(someList, collections.Iterable):
        return [a for i in someList for a in flatten(i)]
    else:
        return [someList]

def flattenOnce(someList):
    """Only flattens a list by one level. Example:
    [[[0, 1, 3], [4, 7]], [0, 1], 2]
    turns into
    [[0, 1, 3], [4, 7], 0, 1, 2].

    Args:
        someList (iterable): a nested iterable.

    Returns:
        list: resultant list.
    """
    newList = []
    for i in someList:
        if isinstance(i, collections.Iterable):
            for j in i:
                newList.append(j)
        else:
            newList.append(i)
    return newList

def deepenOnce(someList):
    """Deepens list by one level. Example:
    [0, 1, 2, 3, 4, 5]
    turns into
    [[0], [1], [2], [3], [4], [5]]

    Args:
        someList (iterable): some iterable.

    Returns:
        list: deepened resultant list
    """
    return [[a] for a in someList]

def alternator(iterable, frontFirst=True):
    """
    Alternates an iterable from front to back and returns the alternation as a list.
    Example:
    [0, 1, 2, 3, 4, 5]
    turns into
    [0, 5, 1, 4, 2, 3] if frontFirst is True and
    [5, 0, 4, 1, 3, 2] if frontFirst is False.

    Args:
        iterable (iterable): some iterable to alternate.
        frontFirst (bool, optional): whether to use the front element first or the
            back element first. Defaults to True, so front element first.

    Returns:
        list: the alternated iterable.
    """
    res = iterable.copy()
    idx = 0
    flipper = frontFirst
    while len(iterable) > 0:
        if flipper:
            res[idx] = iterable.pop(0)
        else:
            res[idx] = iterable.pop(-1)
        flipper = not flipper
        idx += 1
    return res

def splitByMax(iterable):
    """
    Takes a list of nodes and spits out that same list reordered by choosing the first
    node and maximizing the distance to subsequent nodes.

    Args:
        iterable (iterable): a list of nodes.

    Returns:
        list: the list reordered by maximizing distances to subsequent nodes.
    """
    ogLen = len(iterable)
    # create a 2D-list of all the distances to each node
    matrix = [[0 for x in range(ogLen)] for y in range(ogLen)]
    for i, a in enumerate(iterable):
        for j, b in enumerate(iterable):
            matrix[i][j] = mag(subtraction(a, b))
    # gotta start somewhere, so start at the top of the list
    currentSet = [0]
    sums = ogLen * [0]
    while len(currentSet) < ogLen:
        # determine the next spot by examining the sums of distances
        for i in range(ogLen):
            if i in currentSet:
                sums[i] = 0
            else:
                sums[i] += matrix[i][currentSet[-1]]
        newIdx = sums.index(max(sums))
        currentSet.append(newIdx)
    return [iterable[i] for i in currentSet]

max_stars_repo_name: hmomin/peeps

max_issues_repo_name: hmomin/peeps

max_forks_repo_name: hmomin/peeps


text:
#https://stackoverflow.com/questions/63589249/plotly-dash-display-real-time-data-in-smooth-animation/63681810

import dash
import dash_html_components as html
import dash_core_components as dcc
import numpy as np

from dash.dependencies import Input, Output, State

# Example data (a circle).
resolution = 1000
t = np.linspace(0, np.pi * 2, resolution)
x, y = np.cos(t), np.sin(t)
# Example app.
figure = dict(data=[{'x': [], 'y': []}], layout=dict(xaxis=dict(range=[-1, 1]), yaxis=dict(range=[-1, 1])))
app = dash.Dash(__name__, update_title=None)  # remove "Updating..." from title
app.layout = html.Div([
    dcc.Graph(id='graph', figure=dict(figure)), dcc.Interval(id="interval", interval=200),
    dcc.Store(id='offset', data=0), dcc.Store(id='store', data=dict(x=x, y=y, resolution=resolution)),
])
app.clientside_callback(
    """
    function (n_intervals, data, offset) {
        offset = offset % data.x.length;
        const end = Math.min((offset + 10), data.x.length);
        return [[{x: [data.x.slice(offset, end)], y: [data.y.slice(offset, end)]}, [0], 500], end]
    }
    """,
    [Output('graph', 'extendData'), Output('offset', 'data')],
    [Input('interval', 'n_intervals')], [State('store', 'data'), State('offset', 'data')]
)

if __name__ == '__main__':
    app.run_server()


text:
#!/usr/local/bin/anaconda3/bin/python

#Este archivo recibe las secuencias de ADN de cada cromosoma de cada especie ya "limpia", es decir sin genes (transcritos) ortólogos.
#Con esto crea la base de datos con mediciones de las 9 variables en ventanas de tamaño 300 pares de bases (por ahora).
#SI SE AÑADEN NUEVAS ESPECIES AL ESTUDIO ENTONCES ESTE ARCHIVO SE TIENE QUE MODIFICAR PARA INCLUIRLAS.
import random
import csv
from Bio import SeqIO
import numpy as np

#DESDE AQUÍ SE DEBE(N) AÑADIR LA(S) NUEVA(S) ESPECIE(S)
cromosomas_ciona=14
cromosomas_mosca=7
cromosomas_pez_zebra=25
cromosomas_gusano=6
cromosomas_homo=24
cromosomas_levadura=16
cromosomas_pollo=34 #33 cromosomas excepto el 29, más el W y el Z
cromosomas_raton=21

#defino arreglos con los nombres de los cromosomas por cada especie en el estudio.
a_ciona=[i for i in range(1,cromosomas_ciona+1)]
a_gusano=['I','II','III','IV','V','X']
a_homo=[i for i in range(1,cromosomas_homo-1)]
a_homo.append('X')
a_homo.append('Y')
a_levadura=['I','II','III','IV','V','VI','VII','VIII','IX','X','XI','XII','XIII','XIV','XV','XVI']
a_mosca=['2L','2R','3L','3R',4,'X','Y']
a_pollo=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,30,31,32,33,'W','Z']
a_raton=[i for i in range(1,cromosomas_raton-1)]
a_raton.append('X')
a_raton.append('Y')
a_pez=[i for i in range(1,cromosomas_pez_zebra+1)]

arr_as=[a_levadura, a_ciona, a_mosca, a_pez, a_pollo, a_raton, a_homo, a_gusano]

Especies=["sec_Saccharomyces_cerevisiae_crom_","sec_Ciona_intestinalis_crom_","sec_Drosophila_melanogaster_crom_","sec_Danio_rerio_crom_","sec_Gallus_gallus_crom_","sec_Mus_musculus_crom_","sec_Homo_sapiens_crom_","sec_Caenorhabditis_elegans_crom_"]

#FUNCIONES
def selec_crom_esp(esp):
    cromosomas=0
    res="/u/scratch/mauricio/code/secuencias/"
    if esp=="sec_Ciona_intestinalis_crom_":
        cromosomas=cromosomas_ciona
        res2="ciona_intestinalis"
    elif esp=="sec_Drosophila_melanogaster_crom_":
        cromosomas=cromosomas_mosca
        res2="mosca"
    elif esp=="sec_Danio_rerio_crom_":
        cromosomas=cromosomas_pez_zebra
        res2="zebra_fish"
    elif esp=="sec_Caenorhabditis_elegans_crom_":
        cromosomas=cromosomas_gusano
        res2="gusano"
    elif esp=="sec_Homo_sapiens_crom_":
        cromosomas=cromosomas_homo
        res2="homo_sapiens"
    elif esp=="sec_Saccharomyces_cerevisiae_crom_":
        cromosomas=cromosomas_levadura
        res2="levadura"
    elif esp=="sec_Gallus_gallus_crom_":
        cromosomas=cromosomas_pollo
        res2="pollo"
    elif esp=="sec_Mus_musculus_crom_":
        cromosomas=cromosomas_raton
        res2="raton"
    else:
        print("ERROR, NO SE DIO LA ESPECIE CORRECTA")
    return cromosomas, res, res2

def obten_info_secuencia(esp, a):
    cromosomas, cad, espe=selec_crom_esp(esp)
    res=[i for i in range(cromosomas)]
    i=0
    for ind in a:
        arch=cad+esp+str(ind)+".fa"
        res[i]=lee_sec(arch)
        i+=1
    return res, cromosomas, espe

def lee_sec(arch):
    fasta_sequences=SeqIO.parse(arch, 'fasta')
    for seq_record in fasta_sequences:
        cad=str(seq_record.seq)
        lon=len(cad)
        N=cad.count("N")
        A=cad.count("A")
        G=cad.count("G")
        C=cad.count("C")
        T=cad.count("T")
        Ast=cad.count("*")
        N=N/lon
        A=A/lon
        G=G/lon
        C=C/lon
        T=T/lon
        Ast=Ast/lon
    return [N,A,G,C,T,Ast,lon]

#MAIN----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

#0->levadura
#1->ciona_intestinalis
#2->mosca
#3->pez_zebra
#4->pollo
#5->raton
#6->humano
#7->gusano
lon=len(arr_as)
for indi in range(lon):
    lista, crom, espe=obten_info_secuencia(Especies[indi], arr_as[indi])
    with open('info_calidad_'+espe+'.csv','w') as newFile:
        newFileWriter=csv.writer(newFile)
        newFileWriter.writerow(["N","A","G","C","T","*","Cromosoma"])
        for ind in range(crom):
            lista[ind].append(arr_as[indi][ind])
            newFileWriter.writerow(lista[ind])


text:
from distutils.core import setup
from Cython.Build import cythonize

import numpy as np

setup(
    name = 'opt app',
    ext_modules = cythonize("opt.pyx"),
    include_dirs = [np.get_include()],
)

max_stars_repo_name: cclauss/statarb

max_issues_repo_name: cclauss/statarb

max_forks_repo_name: cclauss/statarb


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 18 11:42:53 2020

@author: iagorosa
"""

#%%
import numpy as np
import pandas as pd
from PIL import Image as pil
#import matplotlib.pyplot as plt
from os import listdir
from os.path import isfile, join
from sklearn_extensions.extreme_learning_machines.elm import ELMRegressor, ELMClassifier

from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler

import ClassELM as elm
import aux_functions as af

from itertools import product, combinations

from sklearn.preprocessing import LabelBinarizer

from tensorflow.keras.datasets import mnist

import leitura_geral as lg
from sklearn.model_selection import train_test_split

import leitura_geral as lg

#%%

#X = np.loadtxt("./dataset/smallNORB/smallNORB-32x32.csv", delimiter=',', skiprows=1)
# col 0: label
#X_train = X[:, 1:]
#y_train = X[:, 0]

#XX = np.loadtxt("./dataset/smallNORB/smallNORB-32x32(test).csv", delimiter=',', skiprows=1)
# col 0: label
#X_test = XX[:, 1:]
#y_test = XX[:, 0]

#%%

(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(X_train.shape[0], X_train.shape[1]*X_train.shape[2]).astype('float')
y_train = y_train.astype('int')

X_test = X_test.reshape(X_test.shape[0], X_test.shape[1]*X_test.shape[2]).astype('float')
y_test = y_test.astype('int')
#%%

# Normalização

X_train = X_train / 255.0
X_test  = X_test / 255.0

#%%

#X = lg.leitura('umist')
X = lg.leitura('umist').astype('float')
X[:, 2:] = X[:, 2:] / 255.0

X_train,X_test,y_train,y_test=train_test_split(X[:, 2:],X[:,0],test_size=1.0/3,random_state=None)

pl.hist(y_train)
pl.hist(y_test)
#%%

db = lg.Databases('yale')
(X_train, y_train), (X_test, y_test) = db.load_train_test(pTrain=8, conf_op=10)
#%%
######### SETTING

func_hidden_layer = af.normal_random_layer
random_state = 10
activation_func='tanh'
n_hidden=1000

celm = elm.ELMClassifier(n_hidden=n_hidden, activation_func='sigmoid', func_hidden_layer = func_hidden_layer, bias=True, random_state= random_state, regressor = 'ls_dual', degree=3, lbd=2)
cmelm = elm.ELMMLPClassifier(n_hidden=n_hidden, activation_func='relu', func_hidden_layer = func_hidden_layer, random_state= random_state, regressor = 'pinv')

skt_elm = ELMClassifier(n_hidden=n_hidden, random_state=random_state, activation_func=activation_func, binarizer=LabelBinarizer(0, 1))
#r2 = 0
#%%
######### TRAIN

celm.fit(X_train, y_train)
cmelm.fit(X_train, y_train)
#skt_elm.fit(X_train, y_train)

#%%
######### TEST

r1 = celm.predict(X_test, y_test)[0]
r2 = cmelm.predict(X_test, y_test)[0]
#r3 = skt_elm.score(X_test, y_test)
r3=0

#%%

print('r1:', r1, '\nr2:', r2, '\nr3:', r3)

#%%

for img0 in X[X[:,1] == 20][:,2:]:
    print(img0.shape)
    m0 = img0.reshape(112,92).T
    img = pil.fromarray(m0, 'L')
    img.show()

max_stars_repo_path: em desuso/main_smallNORB.py

max_issues_repo_path: em desuso/main_smallNORB.py

max_forks_repo_path: em desuso/main_smallNORB.py


text:
###############################################################################
# Building the Model

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.externals import joblib
# import pickle

# opening the databases
train_df = pd.read_csv('data/train_data_modified.csv')
test_df = pd.read_csv('data/test_data_modified.csv')

# now let's find mean based prediction
mean_sales = train_df['Item_Outlet_Sales'].mean()

# making baseline models helps in setting a benchmark.
# If the predictive algorithm is below this,
# there is something going seriously wrong with the algorithm or data

baseline = pd.DataFrame({
    'Item_Identifier': test_df['Item_Identifier'],
    'Outlet_Identifier': test_df['Outlet_Identifier'],
    'Item_Outlet_Sales': mean_sales
}, columns=['Item_Identifier', 'Outlet_Identifier', 'Item_Outlet_Sales'])

# Export the baseline result
baseline.to_csv('data/mean_result.csv', index=False)

def trainModel(regressor,   X_train, y_train, filename):
    # fitting the training set on the model
    regressor.fit(X_train, y_train)
    # print the regressor accuracy
    print(filename,"score on training set: %.4f" %(lr.score(X_train,y_train)))
    # Applying K-Fold cross validation
    from sklearn.model_selection import cross_val_score

    accuracies = cross_val_score(estimator=regressor, X=X_train, y=y_train,
                                 cv=10, n_jobs=-1)
    print("mean accuracy of "+filename+" Model on training set: %.4f"
          %(accuracies.mean()))
    print("standard deviation of "+filename+" Model on training set: %.4f"
          %(accuracies.std()))
    # Save the trained model
    joblib.dump(regressor,str("models/"+filename+".sav"))
    # Save the Trained Model
    # pickle.dump(regressor, open("filename.sav", 'wb'))
    # regressor = picle.load(open("filename.sav",'wb'))

# Extracting the training set
X_train = train_df.drop(['Item_Outlet_Sales', 'Item_Identifier',
                         'Outlet_Identifier'], axis=1)
y_train = train_df['Item_Outlet_Sales']

# Linear Regression Model
from sklearn.linear_model import LinearRegression
lr = LinearRegression(normalize=True)

trainModel(lr, X_train, y_train, 'linear_regressor')

# Decision Tree Model
from sklearn.tree import DecisionTreeRegressor
tree = DecisionTreeRegressor(max_depth=9, min_samples_leaf=150)
trainModel(tree, X_train, y_train, 'decision_tree_regressor')

# RandomForest
from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor(n_estimators=300, max_depth=8, min_samples_leaf=100, n_jobs=-1)
trainModel(lr, X_train, y_train, 'random_forest_regressor')

# SVM
from sklearn.svm import SVR
svr_reg = SVR(kernel='rbf', gamma=0.5)
trainModel(lr, X_train, y_train, 'svm_regressor')

# Gradiant Boosting Regression Model
from sklearn.ensemble import GradientBoostingRegressor
gbr = GradientBoostingRegressor(n_estimators=300, min_samples_leaf=100, max_depth=8)

trainModel(gbr, X_train, y_train, "gradient_boost_regressor")

print("The models are successfully trained")


text:
#########################################################################################################################
##   Distribution code Version 1.0 -- 14/10/2021 by Tak Ming Wong Copyright 2021, University of Siegen
##
##   The Code is created based on the method described in the following paper
##   [1] "Deep Optimization Prior for THz Model Parameter Estimation", T.M. Wong, H. Bauermeister, M. Kahl, P. Haring Bolivar, M. Moeller, A. Kolb,
##   Winter Conference on Applications of Computer Vision (WACV) 2022.
##
##   If you use this code in your scientific publication, please cite the mentioned paper.
##   The code and the algorithm are for non-comercial use only.
##
##   For other details, please visit website https://github.com/tak-wong/Deep-Optimization-Prior
#########################################################################################################################
import math
import os
import torch
import numpy as np
import scipy.io
import datetime

from PIL import Image
from torchvision import transforms

from .model_thz import *

class model_thz_reg(model_thz):
    # -----------------------------------------------------------------------
    def __init__(self, device, dataset, hp, SHOULD_PROJECT_BEFORE = False, verbose = True):
        super(model_thz_reg, self).__init__(device, dataset, SHOULD_PROJECT_BEFORE, verbose)

        self.noise_real = None
        self.noise_imag = None
        self.reg = None

        self.hp = hp

        if self.verbose:
            print("model_thz_reg.init")

    # -----------------------------------------------------------------------
    def get_name(self):
        return "model_thz_reg"

    # -----------------------------------------------------------------------
    def prepare_tensors(self, batch = None):
        # generate z-axis tensor
        z_reshape = self.z_np.reshape((1, 1, self.NZ, 1, 1))
        z_tile = np.tile(z_reshape, (1, 1, 1, self.NX, self.NY))
        self.z_tensor = torch.tensor(z_tile, requires_grad=False).float().to(self.device)

    # -----------------------------------------------------------------------
    def initialize_noise(self):
        # store to a single tensor
        noise_tensor = torch.zeros((1, 1, self.NZ, self.NX, self.NY), dtype=torch.float, requires_grad = False)

        return noise_tensor

    # -----------------------------------------------------------------------
    def get_p(self):
        params = {
            "sigma" : self.sigma,
            "mu" : self.mu,
            "ehat" : self.ehat,
            "phi" : self.phi,
        }

        if (self.noise_real is not None):
            params["noise_real"] = self.noise_real

        if (self.noise_imag is not None):
            params["noise_imag"] = self.noise_imag

        return params

    # -----------------------------------------------------------------------
    def forward_reg(self, batch_shape, batch_predict, noise_real, noise_imag):
        # this is the forward model, as a decoder
        model_predict = torch.zeros(batch_shape, dtype=torch.float, requires_grad=True, device=self.device).clone()

        reg = torch.zeros((2), dtype=torch.float, requires_grad=True, device=self.device).clone()

        """
        Model based decoder
        """
        sigma_norm = batch_predict[:, :, 0, :, :]
        mu_norm = batch_predict[:, :, 1, :, :]
        ehat_norm = batch_predict[:, :, 2, :, :]
        phic_norm = batch_predict[:, :, 3, :, :]
        phis_norm = batch_predict[:, :, 4, :, :]

        # do projection
        if (self.SHOULD_PROJECT_BEFORE):
            sigma_proj, mu_proj, ehat_proj, phic_proj, phis_proj = self.projection(sigma_norm, mu_norm, ehat_norm, phic_norm, phis_norm)
        else:
            # extend all parameters in z-axis
            sigma_proj = sigma_norm
            mu_proj = mu_norm
            ehat_proj = ehat_norm
            phic_proj = phic_norm
            phis_proj = phis_norm

        # calculate all predicted parameters
        sigma_one, mu_one, ehat_one, phic_one, phis_one = self.scaling_unfold(sigma_proj, mu_proj, ehat_proj, phic_proj, phis_proj)

        # fold to phage angle
        phi_one = torch.atan2( phis_one, phic_one )

        # extend all parameters in z-axis
        sigma = sigma_one.unsqueeze(2).repeat(1, 1, self.NZ, 1, 1)
        mu = mu_one.unsqueeze(2).repeat(1, 1, self.NZ, 1, 1)
        ehat = ehat_one.unsqueeze(2).repeat(1, 1, self.NZ, 1, 1)
        phi = phi_one.unsqueeze(2).repeat(1, 1, self.NZ, 1, 1)

        # calculate sinc(x) = sin(pi * x) / (pi * x)
        pi_sigma_mu = math.pi * sigma * (self.z_tensor - mu)
        sin_pi_sigma_mu = torch.sin( math.pi * sigma * (self.z_tensor - mu) )
        sin_pi_sigma_mu[ torch.abs(pi_sigma_mu) < self.FLT_EPSILON ] = 1
        pi_sigma_mu[ torch.abs(pi_sigma_mu) < self.FLT_EPSILON ] = 1
        sinc_sigma_mu = (sin_pi_sigma_mu) / (pi_sigma_mu)

        # calculate phase item
        omega_z_phi = self.OMEGA * self.z_tensor - phi

        # compute the model
        if noise_real is None or noise_imag is None:
            model_predict[:, self.INDEX_REAL, :, :, :] =      ehat * sinc_sigma_mu * torch.cos(omega_z_phi)
            model_predict[:, self.INDEX_IMAG, :, :, :] = -1 * ehat * sinc_sigma_mu * torch.sin(omega_z_phi)
        else:
            model_predict[:, self.INDEX_REAL, :, :, :] =      ehat * sinc_sigma_mu * torch.cos(omega_z_phi) + self.hp.ALPHA_S * noise_real
            model_predict[:, self.INDEX_IMAG, :, :, :] = -1 * (ehat * sinc_sigma_mu * torch.sin(omega_z_phi) + self.hp.ALPHA_S * noise_imag)

        # regularization
        if self.hp.ALPHA_S > 0.0:
            noise_reg = self.hp.ALPHA_S * self.hp.LAMBDA_S * (self.Lq_norm(noise_real, self.hp.NORM_Q, dim=(0, 1, 2)) + self.Lq_norm(noise_imag, self.hp.NORM_Q, dim=(0, 1, 2)))
            reg[0] = torch.mean( noise_reg )
        else:
            reg[0] = 0.0

        reg[1] = 0.0
        if np.count_nonzero(self.hp.LAMBDA_I) > 0:
            if self.hp.LAMBDA_I[0] > 0.0:
                ehat_grad = torch.mean(torch.abs(ehat[:, :, :, 0:self.NX-1, :] - ehat[:, :, :, 1:self.NX, :])) + torch.mean(torch.abs(ehat[:, :, :, :, 0:self.NY-1] - ehat[:, :, :, :, 1:self.NY]))
                reg[1] += np.square(self.hp.LAMBDA_I[0] * self.MAX_VALUE) * ehat_grad

            if self.hp.LAMBDA_I[1] > 0.0:
                sigma_grad = torch.mean(torch.abs(sigma[:, :, :, 0:self.NX-1, :] - sigma[:, :, :, 1:self.NX, :])) + torch.mean(torch.abs(sigma[:, :, :, :, 0:self.NY-1] - sigma[:, :, :, :, 1:self.NY]))
                reg[1] += np.square(self.hp.LAMBDA_I[1] * self.MAX_VALUE) * sigma_grad

            if self.hp.LAMBDA_I[2] > 0.0:
                mu_grad = torch.mean(torch.abs(mu[:, :, :, 0:self.NX-1, :] - mu[:, :, :, 1:self.NX, :])) + torch.mean(torch.abs(mu[:, :, :, :, 0:self.NY-1] - mu[:, :, :, :, 1:self.NY]))
                reg[1] += np.square(self.hp.LAMBDA_I[2] * self.MAX_VALUE) * mu_grad

            if self.hp.LAMBDA_I[3] > 0.0:
                phic_grad = torch.mean(torch.abs(torch.cos(phi[:, :, :, 0:self.NX-1, :]) - torch.cos(phi[:, :, :, 1:self.NX, :]))) + torch.mean(torch.abs(torch.cos(phi[:, :, :, :, 0:self.NY-1]) - torch.cos(phi[:, :, :, :, 1:self.NY])))
                reg[1] += np.square(self.hp.LAMBDA_I[3] * self.MAX_VALUE) * phic_grad

            if self.hp.LAMBDA_I[4] > 0.0:
                phis_grad = torch.mean(torch.abs(torch.sin(phi[:, :, :, 0:self.NX-1, :]) - torch.sin(phi[:, :, :, 1:self.NX, :]))) + torch.mean(torch.abs(torch.sin(phi[:, :, :, :, 0:self.NY-1]) - torch.sin(phi[:, :, :, :, 1:self.NY])))
                reg[1] += np.square(self.hp.LAMBDA_I[4] * self.MAX_VALUE) * phis_grad

        with torch.no_grad():
            self.sigma = sigma_one.clone().requires_grad_(False).squeeze_()
            self.mu = mu_one.clone().requires_grad_(False).squeeze_()
            self.ehat = ehat_one.clone().requires_grad_(False).squeeze_()
            self.phi = phi_one.clone().requires_grad_(False).squeeze_()
            if noise_real is None or noise_imag is None:
                self.noise_real = None
                self.noise_imag = None
            else:
                self.noise_real = torch.sum(noise_real, dim=(0, 1, 2), keepdim=False).clone().requires_grad_(False)
                self.noise_imag = torch.sum(noise_imag, dim=(0, 1, 2), keepdim=False).clone().requires_grad_(False)

        return model_predict, reg

    # -----------------------------------------------------------------------
    def save_mat(self, path_file, sigma = None, mu = None, ehat = None, phi = None, noise_real = None, noise_imag = None, loss_map = None):
        if (self.verbose):
            print("Writing mat File")

        if sigma is None:
            sigma = self.sigma

        if mu is None:
            mu = self.mu

        if ehat is None:
            ehat = self.ehat

        if phi is None:
            phi = self.phi

        if noise_real is None:
            noise_real = self.noise_real

        if noise_imag is None:
            noise_imag = self.noise_imag

        matfiledata = {} # make a dictionary to store the MAT data in
        matfiledata[u'Ehat'] = ehat.cpu().detach().numpy()
        matfiledata[u'Sigma'] = sigma.cpu().detach().numpy()
        matfiledata[u'Mu'] = mu.cpu().detach().numpy()
        matfiledata[u'Phi'] = phi.cpu().detach().numpy()

        if (loss_map is not None):
            matfiledata[u'Loss'] = loss_map.cpu().detach().numpy()

        if (noise_real is not None):
            matfiledata[u'Noise_real'] = noise_real.cpu().detach().numpy()

        if (noise_imag is not None):
            matfiledata[u'Noise_imag'] = noise_imag.cpu().detach().numpy()

        scipy.io.savemat(path_file, matfiledata)

        if (self.verbose):
            print("{} is saved".format(path_file))

        return path_file

max_stars_repo_name: tak-wong/Deep-Optimization-Prior

max_issues_repo_name: tak-wong/Deep-Optimization-Prior

max_forks_repo_name: tak-wong/Deep-Optimization-Prior

reason: import numpy,import scipy


text:
import numpy as np
import matplotlib.pyplot as plt
from scripts.forest import Forest
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
from sklearn.svm import OneClassSVM
import scripts.timeseries as ts
import pandas as pd
import time
import scipy.io as sio
from sklearn import metrics

datasets = ['nyc_taxi','ambient_temperature_system_failure','cpu_utilization_asg_misconfiguration','machine_temperature_system_failure','ec2_request_latency_system_failure']
L = len(datasets)
trials = 5
run_lof_svm = 1

for i in range(0,L):
    data = pd.read_csv('../data/numenta/'+datasets[i]+'.csv')
    value = list(data["value"])
    arr = np.array(value)
    y = np.array(list(data["label"]))
    X = ts.shingle(arr, 10)
    X = np.transpose(X)
    t1, _ = np.shape(X)
    file_name = 'experiment_results/' + datasets[i] + '.txt'
    File_object = open(file_name,"w")
    time_all = np.zeros((trials,4))
    precision_all = np.zeros((trials,4))
    auc_all = np.zeros((trials,4))

    for j in range(0,trials):

        print('\n\n******'+datasets[i]+' trial '+str(j+1)+'*******\n\n')

        print('\n******Iso-Forest*******\n')
        start = time.time()
        clf = IsolationForest(contamination = 0.1, behaviour = 'new')
        clf.fit(X)
        end = time.time()
        time_all[j,0] = end - start
        iso_scores = clf.score_samples(X)

        if run_lof_svm == 0:
            lof_scores = iso_scores
            osvm_scores = iso_scores
        elif j == 0:
            print('\n******LOF*******\n')
            start = time.time()
            lof = LocalOutlierFactor()
            lof.fit(X)
            end = time.time()
            time_all[j,1] = end - start
            lof_scores = lof.negative_outlier_factor_

            print('\n******1-class SVM*******\n')
            start = time.time()
            osvm = OneClassSVM(kernel='rbf')
            osvm.fit(X)
            end = time.time()
            time_all[j,2] = end - start
            osvm_scores = osvm.score_samples(X)

        print('\n******Our Algo*******\n')
        start = time.time()
        #n_samples = int(t1/50)
        n_samples = 100
        kwargs = {'max_depth': 10, 'n_trees':50,  'max_samples': n_samples, 'max_buckets': 3, 'epsilon': 0.1, 'sample_axis': 1,
          'threshold': 0}
        forest = Forest(**kwargs)
        forest.fit(np.transpose(X))
        indices, outliers, scores , pst, our_scores = forest.predict(np.transpose(X), err = 0.1, pct = 0)
        end = time.time()
        time_all[j,3] = end - start

        precision_iso, recall_iso, thresholds_iso = metrics.precision_recall_curve(y[:t1], -iso_scores, pos_label=1)
        precision_lof, recall_lof, thresholds_lof = metrics.precision_recall_curve(y[:t1], -lof_scores, pos_label=1)
        precision_osvm, recall_osvm, thresholds_osvm = metrics.precision_recall_curve(y[:t1], -osvm_scores, pos_label=1)
        precision_our, recall_our, thresholds_our = metrics.precision_recall_curve(y[:t1], -our_scores, pos_label=1)
        precision_all[j,0] = max(2*precision_iso*recall_iso/(precision_iso+recall_iso))
        precision_all[j,1] = max(2*precision_lof*recall_lof/(precision_lof+recall_lof))
        precision_all[j,2] = max(2*precision_osvm*recall_osvm/(precision_osvm+recall_osvm))
        precision_all[j,3] = max(2*precision_our*recall_our/(precision_our+recall_our))

        auc_all[j,0] = metrics.roc_auc_score(y[:t1], -iso_scores)
        auc_all[j,1] = metrics.roc_auc_score(y[:t1], -lof_scores)
        auc_all[j,2] = metrics.roc_auc_score(y[:t1], -osvm_scores)
        auc_all[j,3] = metrics.roc_auc_score(y[:t1], -our_scores)

        for k in range(0,4):
            print('{:.4f}\t'.format( precision_all[j,k] ))
        print('\n')

        for k in range(0,4):
            print('{:.4f}\t'.format( auc_all[j,k] ))
        print('\n')

    File_object.write(str(kwargs))

    File_object.write('\n\nIF\tLOF\tSVM\tOur-Algo\n\n')

    for j in range(0,trials):
        for k in range(0,4):
            File_object.write('{:.4f}\t'.format( precision_all[j,k] ))
        File_object.write('\n')

    File_object.write('\n')

    for k in range(0,4):
            File_object.write('{:.4f}\t'.format( np.mean(precision_all[:,k]) ))
    File_object.write('\n')

    for k in range(0,4):
            File_object.write('{:.4f}\t'.format( np.std(precision_all[:,k]) ))
    File_object.write('\n')

    File_object.write('\nIF\tLOF\tSVM\tOur-Algo\n\n')

    for j in range(0,trials):
        for k in range(0,4):
            File_object.write('{:.4f}\t'.format( auc_all[j,k] ))
        File_object.write('\n')

    File_object.write('\n')

    for k in range(0,4):
            File_object.write('{:.4f}\t'.format( np.mean(auc_all[:,k]) ))
    File_object.write('\n')

    for k in range(0,4):
            File_object.write('{:.4f}\t'.format( np.std(auc_all[:,k]) ))
    File_object.write('\n')

    File_object.close()

    file_name = 'experiment_results/' + datasets[i] + '_results.mat'
    sio.savemat(file_name, {'time_all':time_all, 'precision_all':precision_all, 'auc_all':auc_all})

max_stars_repo_name: janfrancu/pidforest

max_issues_repo_name: janfrancu/pidforest

max_forks_repo_name: janfrancu/pidforest


text:
try:
    import cupy as xp
except ImportError:
    import numpy as xp

from common import crop, pack
from pinger import pingplot, trigplot

class Trigger:
    def __init__(self, L_interval, L_search, fir_rise_time,
        trigger_plot=False, ping_plot=False):
        assert fir_rise_time >= 1, 'Filter rise time must be at least 1'

        self._L_interval = L_interval
        self._L_search = L_search
        self._fir_rise_time = fir_rise_time

        self._trigger_plot = (
            trigplot.TriggerPlot() if trigger_plot else None)
        self._ping_plot = pingplot.PingPlot() if ping_plot else None

        self._sig_pkr = pack.Packer(L_interval)
        self._sub_hdgs_pkr = pack.Packer(L_interval)

    def push(self, sig, sub_hdgs):
        packed_sig = self._sig_pkr.push(sig)
        packed_sub_hdgs = self._sub_hdgs_pkr.push(sub_hdgs)
        if packed_sig is not None:
            ampl = xp.abs(packed_sig).sum(axis=0)
            ampl_delayed = xp.roll(ampl, self._fir_rise_time)
            mean_ampl = ampl.mean()

            trigger_f = (ampl + mean_ampl) / (ampl_delayed + mean_ampl)
            trigger_f[: self._fir_rise_time] = 0

            ampl_peak_pos = int(ampl.argmax())
            (search_start, search_end) = crop.find_bounds(
                self._L_interval, self._L_search, ampl_peak_pos)
            trigger_f[: search_start] = 0
            trigger_f[search_end :] = 0

            ping_pos = int(trigger_f.argmax())
            ping_phase = xp.angle(packed_sig[:, ping_pos])
            sub_hdg = packed_sub_hdgs[0, ping_pos]

            if self._trigger_plot is not None:
                self._trigger_plot.plot(ampl, trigger_f, ping_pos)

            if self._ping_plot is not None:
                self._ping_plot.plot(packed_sig, ping_pos)

            return (ping_phase, sub_hdg)

        return None

max_stars_repo_name: cuauv/software

max_issues_repo_name: cuauv/software

max_forks_repo_name: cuauv/software

reason: import numpy,import cupy


text:
"""Methods to handle input/output files."""

# Authors: Guillaume Lemaitre <g.lemaitre58@gmail.com>
#          Cedric Lemaitre
# License: BSD 3 clause

import os
from collections import defaultdict

import pandas as pd
import numpy as np
import six

from fitparse import FitFile

# 'timestamp' will be consider as the index of the DataFrame later on
FIELDS_DATA = ('timestamp', 'power', 'heart_rate', 'cadence', 'distance',
               'altitude', 'speed')

def check_filename_fit(filename):
    """Method to check if the filename corresponds to a fit file.

    Parameters
    ----------
    filename : str
        The fit file to check.

    Returns
    -------
    filename : str
        The checked filename.

    """

    # Check that filename is of string type
    if isinstance(filename, six.string_types):
        # Check that this is a fit file
        if filename.endswith('.fit'):
            # Check that the file is existing
            if os.path.isfile(filename):
                return filename
            else:
                raise ValueError('The file does not exist.')
        else:
            raise ValueError('The file is not a fit file.')
    else:
        raise ValueError('filename needs to be a string. Got {}'.format(
            type(filename)))

def load_power_from_fit(filename):
    """Method to open the power data from FIT file into a pandas dataframe.

    Parameters
    ----------
    filename : str,
        Path to the FIT file.

    Returns
    -------
    data : DataFrame
        Power records of the ride.

    """
    filename = check_filename_fit(filename)
    activity = FitFile(filename)
    activity.parse()
    records = activity.get_messages(name='record')

    data = defaultdict(list)
    for rec in records:
        values = rec.get_values()
        for key in FIELDS_DATA:
            data[key].append(values.get(key, np.NaN))

    data = pd.DataFrame(data)
    if data.empty:
        raise IOError('The file {} does not contain any data.'.format(
            filename))

    # rename the columns for consistency
    data.rename(columns={'heart_rate': 'heart-rate', 'altitude': 'elevation'},
                inplace=True)

    data.set_index(FIELDS_DATA[0], inplace=True)
    del data.index.name

    return data

max_stars_repo_name: nogrets/scikit-cycling

max_issues_repo_name: nogrets/scikit-cycling

max_forks_repo_name: nogrets/scikit-cycling


text:
#Copyright (c) 2018-2020 Gavin W. Wilson

#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

def snv2vcf_cmd(cargs):
    import numpy as NP
    import pandas as pd
    from .snvcmp import SampleData
    from .calc import write_MTX
    import sys, argparse, os
    import itertools

    parser = argparse.ArgumentParser(description='Create a minimal VCF and mtx matrices from SNV data by removing potential edits in REDIportal or overapping Alu elements')
    parser.add_argument('--max-homozygous', help='Percent of cells that only express the SNV must be less than this', type=float, default=99.9)
    parser.add_argument('--min-cells', help='Number of cells with coverage at a given SNV', type=int, default=10)
    parser.add_argument('--min-alts', help='Number of cells with a given SNV', type=int, default=5)
    parser.add_argument('--min-af', help='Minimum overall allele fraction (total alt / [total ref + alt])', type=float, default=0)
    parser.add_argument('-o', '--output', help='Output folder', type=str)

    parser.add_argument('-r', '--ref', help='Ref lengths text generated by scsnv index (to generate vcf header)', type=str)
    parser.add_argument('-f', '--fasta', help='Fasta reference file (for vcf header)', type=str)
    egroup = parser.add_mutually_exclusive_group(required=True)
    egroup.add_argument('-e', '--remove-edits', help='Filter strand corrected A-to-G changes that overlap REDIportal and/or Alu elements', action='store_true')
    egroup.add_argument('-k', '--keep-annotated', help='Only keep annotated edits (1000 genomes)', action='store_true')
    #egroup.add_argument('-n', '--only-edits', help='Filter strand corrected A-to-G changes that overlap REDIportal and/or Alu elements', action='store_true')
    #egroup.add_argument('-b', '--keep-unannotated', help='Only keep unannotated edits (1000 genomes)', action='store_true')

    group = parser.add_mutually_exclusive_group(required=True)
    #group.add_argument('-c', '--csv', help='Write CSV files compatible with scSplit', action='store_true')
    group.add_argument('-m', '--mtx', help='Write vartrix like MTX files compatible with souporcell / vireo', action='store_true')
    group.add_argument('-x', '--flamm', help='Write flammkuchen H5 file', action='store_true')

    parser.add_argument('annotated', help='Annotated mutation flammkuchen/deepdish file')
    args = parser.parse_args(cargs[1:])
    print(args)

    sdata = args.annotated
    gt_dir = args.output
    rdata = args.ref
    data = SampleData(sdata, 'scSNV')
    if not os.path.isdir(gt_dir):
        os.mkdir(gt_dir)

    tref = data.strand_ref
    talt = data.strand_alt

    snvs = data.snvs

    cell_counts = []
    alt_counts = []
    BB_counts = []
    AFs = []

    for s, e in zip(tref.indptr, tref.indptr[1:]):
        d1 = tref.data[s:e]
        d2 = talt.data[s:e]
        rsum = d1.sum()
        asum = d2.sum()
        AFs.append(100.0 * asum / (rsum + asum))
        alt_counts.append(NP.count_nonzero(d2))
        cell_counts.append(len(d2))
        BB_counts.append(100.0 * ((d2 > 0) & (d1 == 0)).sum() / len(d1))

    BB_counts = NP.array(BB_counts)
    cell_counts = NP.array(cell_counts)
    alt_counts = NP.array(alt_counts)
    AFs = NP.array(AFs)

    keep = NP.full(len(snvs), True, dtype='bool')

    if args.min_cells > 0:
        keep &= (cell_counts >= args.min_cells)
    if args.min_alts > 0:
        keep &= (alt_counts >= args.min_alts)
    if args.max_homozygous > 0:
        keep &= (BB_counts < args.max_homozygous)
    if args.min_af < 101:
        keep &= (AFs >= args.min_af)

    if args.remove_edits:
        snvs['repeat_type'] = ''
        snvs.loc[snvs.rep_family != '', 'repeat_type'] = 'Other'
        snvs.loc[snvs.rep_family.str.contains('Alu'), 'repeat_type'] = 'Alu'
        snvs['potential_edit'] = (snvs.strand_ref == 'A') & (snvs.strand_alt == 'G') & ((snvs.repeat_type == 'Alu') | snvs.REDIportal)
        if args.remove_edits:
            print("Removing edits")
            keep &= ~snvs['potential_edit']
    elif args.keep_annotated:
        print("Only keeping annotated SNVs")
        keep &= snvs.g1000

    fsnvs = snvs[keep].copy()

    print(f'Using {len(fsnvs)} / {len(snvs)} SNVs for genotyping')

    ridx = fsnvs.snv_idx.values.copy()
    tref = tref[ridx]
    talt = talt[ridx]
    if args.mtx:
        print(tref.data.dtype, tref.indptr.dtype, tref.indices.dtype, talt.data.dtype)
        print(__file__)
        write_MTX(tref, talt, os.path.join(gt_dir, f'refs.mtx'), os.path.join(gt_dir, f'alts.mtx'))
        print("Testing")
        #mmdebug(os.path.join(gt_dir, f'refs.mtx'))
        #mmdebug(os.path.join(gt_dir, f'alts.mtx'))
        if not os.path.isfile(f'{gt_dir}/barcodes.txt'):
            fout = open(f'{gt_dir}/barcodes.txt', 'w')
            for b in data.barcodes:
                fout.write(b + "\n")
            fout.close()
    elif args.csv:
        tref = tref.toarray()
        rows = [f'{i[0]}:{i[1] + 1}' for i in fsnvs.index]
        print("Writing reference allele csv")
        df = pd.DataFrame(data=tref, columns=data.barcodes, index=rows)
        df.to_csv(f'{gt_dir}/refs.csv')

        talt = talt.toarray()
        print("Writing alternative allele csv")
        df = pd.DataFrame(data=talt, columns=data.barcodes, index=rows)
        df.to_csv(f'{gt_dir}/alts.csv')

    elif args.flamm:
        print("not supporting yet")

    rdata = pd.read_csv(rdata, sep='\t', names=['chrom', 'rlen', 'comment'])
    print("Writing VCF")
    with open(f'{gt_dir}/snvs.vcf', 'w') as fp:
        fp.write('##fileformat=VCFv4.2\n')
        fp.write(f'##reference={args.fasta}\n')
        for r in rdata.itertuples():
            fp.write(f'##contig=<ID={r.chrom},length={r.rlen}>\n')
        fp.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n")
        for i, r in fsnvs.iterrows():
            fp.write(f'{i[0]}\t{i[1] + 1}\tscsnv_idx_{r.snv_idx}\t{r.ref}\t{r.alt}\t.\tPASS\t.\n')

    print("Writing SNV csv file")
    fsnvs.to_csv(f'{gt_dir}/snvs.csv')

max_stars_repo_name: GWW/sctx

max_issues_repo_name: GWW/sctx

max_forks_repo_name: GWW/sctx


text:
import argparse
import os
os.environ['CUDA_VISIBLE_DEVICES']='1'
import numpy as np
import soundfile as sf
import librosa
import wave
from Denoise.enhance_testsets import denoisy

# parser = argparse.ArgumentParser()
# parser.add_argument('--inputwav', default='input/noisy.wav', help="ckpt dir")
# parser.add_argument('--outputwav', default='output/denoisy.wav', help="Directory containing params.json")
# args = parser.parse_args()

# def main():

#     noisywav, sr = librosa.load(args.inputwav,sr = 16000)
#     denoisy2 = denoisy(noisywav)
#     sf.write(data=denoisy2,file =args.outputwav, samplerate=16000)
# def noisy_processing(inputwav: str,outputwav: str,sampleRate: str):
def noisy_processing(inputwav: str,outputwav: str):

    # f = wave.open(inputwav)
    # SampleRate = f.getframerate() #获取输入音频的采样率
    SampleRate = 24000 # 设置的采样率
    print('获取的采样率为：',SampleRate)
    noisywav, sr = librosa.load(inputwav,sr = SampleRate)
    denoisy2 = denoisy(noisywav)
    sf.write(data=denoisy2,file =outputwav, samplerate=SampleRate)
    print('返回信号处理完的音频为：',outputwav)
    return outputwav

if __name__ == '__main__':
    main()

max_stars_repo_name: X-CCS/TensorFlowTTS

max_issues_repo_name: X-CCS/TensorFlowTTS

max_forks_repo_name: X-CCS/TensorFlowTTS


text:
import numpy as np
import pandas as pd
import plotly.graph_objects as go
from geopy.geocoders import Nominatim
import plotly.express as px

coordinates = {'Boden': (65.833333, 21.666667),
               'Borås': (57.7210839, 12.9407407),
               'Eskilstuna': (59.3717379, 16.5051474),
               'Falun': (60.6070068, 15.6323059),
               'Gävle': (60.6750132, 17.1467019),
               'Göteborg': (57.7072326, 11.9670171),
               'Halmstad': (56.6739826, 12.8574827),
               'Haparanda': (65.833333, 24.1),
               'Helsingborg': (56.0441984, 12.7040684),
               'Hudiksvall': (61.7281607, 17.105575),
               'Jönköping': (57.7825634, 14.165719),
               'Kalmar': (57.02784235, 16.575243899947836),
               'Karlskrona': (56.1621073, 15.5866422),
               'Karlstad': (59.3809146, 13.5027631),
               'Kiruna': (67.8550724, 20.2255482),
               'Kristianstad': (56.0293778, 14.1566859),
               'Lidköping': (58.5037196, 13.1576427),
               'Linköping': (58.4098135, 15.6245252),
               'Luleå': (65.5831187, 22.1459535),
               'Malmö': (55.6052931, 13.0001566),
               'Motala': (58.5420395, 15.041261),
               'Norrköping': (58.5909124, 16.1903511),
               'Nyköping': (58.7545409, 17.0120656),
               'Sandviken': (60.619422, 16.7724214),
               'Skellefteå': (64.7520185, 20.959339),
               'Skövde': (58.3898453, 13.8443792),
               'Stockholm': (59.3251172, 18.0710935),
               'Sundsvall': (62.3907552, 17.3071024),
               'Trelleborg': (55.37592, 13.1461522),
               'Uddevalla': (58.3490555, 11.9382855),
               'Umeå': (63.8256568, 20.2630745),
               'Uppsala': (59.8586126, 17.6387436),
               'Varberg': (57.1057412, 12.2502949),
               'Vetlanda': (57.36554305, 15.167830505066155),
               'Vänersborg': (58.3811988, 12.3226877),
               'Västervik': (57.7594186, 16.6385035),
               'Västerås': (59.6110992, 16.5463679),
               'Växjö': (56.8787183, 14.8094385),
               'Örebro': (59.2747287, 15.2151181),
               'Örnsköldsvik': (63.2888613, 18.7160209),
               'Östersund': (63.1793655, 14.6357061)}

df_cities = pd.DataFrame.from_dict(
    coordinates, orient='index', columns=["Latitude", "Longitude"])
df_cities = df_cities.reset_index().rename(columns={'index': "City"})

def add_edge_to_df(df_cities, df_edges, from_city, to_city):
    start_lon = df_cities[df_cities['City'] == from_city]['Longitude'].iloc[0]
    start_lat = df_cities[df_cities['City'] == from_city]['Latitude'].iloc[0]
    end_lon = df_cities[df_cities['City'] == to_city]['Longitude'].iloc[0]
    end_lat = df_cities[df_cities['City'] == to_city]['Latitude'].iloc[0]

    df_edges = df_edges.append({'start_lat': start_lat, 'start_lon': start_lon,
                               'end_lat': end_lat, 'end_lon': end_lon}, ignore_index=True)
    return df_edges

def show(city_sol, edge_sol):

    city_sol = pd.concat([df_cities, city_sol], axis=1, join='inner')

    # Build df with edges from - to coordinates
    df_edges = pd.DataFrame(
        columns=['start_lat', 'start_lon', 'end_lat', 'end_lon'])
    for index, row in edge_sol.iterrows():

        df_edges = add_edge_to_df(df_cities, df_edges, row['From'], row['To'])

    fig = go.Figure()

    # draw edges
    for i in range(len(df_edges)):
        if edge_sol.iloc[i]['Type'] == 'CORE':
            color = 'red'
        else:
            color = 'blue'

        fig.add_trace(
            go.Scattermapbox(
                lon=[df_edges['start_lon'][i], df_edges['end_lon'][i]],
                lat=[df_edges['start_lat'][i], df_edges['end_lat'][i]],
                mode='markers+lines',
                line=dict(width=3, color=color),
                opacity=0.7
            )
        )

    # draw cities
    for i in range(len(city_sol)):
        if city_sol['IsControlCenter'][i]:
            text = city_sol['Name'][i] + ' is control center'
            color = 'green'
            size = 8
        else:
            text = city_sol['Name'][i]
            color = 'red'
            size = 4

        ingoing = round(city_sol['IngoingFlow'][i], 1)
        outgoing = round(city_sol['OutgoingFlow'][i], 1)

        text += ', Ingoing flow: ' + str(ingoing)
        text += ', Outgoing flow: ' + str(outgoing)
        fig.add_trace(go.Scattermapbox(

            lon=[city_sol['Longitude'][i]],
            lat=[city_sol['Latitude'][i]],
            hoverinfo='text',
            text=text,
            mode='markers',
            marker=dict(
                size=size,
                color=color,
            )))

    fig.update_layout(
        title_text='Automax',
        margin={'l': 0, 't': 0, 'b': 0, 'r': 0},
        mapbox={
            'center': {'lon': 10, 'lat': 10},
            'style': "carto-positron",
            'center': {'lon': -20, 'lat': -20},
            'zoom': 1},
        autosize=False,
        width=1000,
        height=1000,
        showlegend=False,)

    fig.show()