text:
import argparse
import pdb
import sys

import matplotlib.pyplot as plt
import numpy as np
import torch
from model import MyAwesomeModel
from torch import nn, optim

from data import mnist

class TrainOREvaluate(object):
    """Helper class that will help launch class methods as commands
    from a single script
    """

    def __init__(self):
        parser = argparse.ArgumentParser(
            description="Script for either training or evaluating",
            usage="python main.py <command>",
        )
        parser.add_argument("command", help="Subcommand to run")
        args = parser.parse_args(sys.argv[1:2])
        if not hasattr(self, args.command):
            print("Unrecognized command")

            parser.print_help()
            exit(1)
        # use dispatch pattern to invoke method with same name
        getattr(self, args.command)()

    def train(self):
        print("Training day and night")
        parser = argparse.ArgumentParser(description="Training arguments")
        parser.add_argument("--lr", default=0.1)
        # add any additional argument that you want
        args = parser.parse_args(sys.argv[2:])
        # print(args)

        # TODO: Implement training loop here
        model = MyAwesomeModel()
        train_set, _ = mnist()
        train_set = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)

        criterion = nn.NLLLoss()
        optimizer = optim.Adam(model.parameters(), lr=1e-2)

        epochs = 30
        steps = 0

        train_losses, test_losses = [], []
        for e in range(epochs):
            running_loss = 0

            for images, labels in train_set:
                # View images as vectors
                images = images.view(images.shape[0], -1)

                # zero gradients
                optimizer.zero_grad()

                # Compute loss, step and save running loss
                log_ps = model(images)
                loss = criterion(log_ps, labels)
                loss.backward()
                optimizer.step()
                steps += 1
                # pdb.set_trace()

                running_loss += loss.item()

            print(
                "Epoch: {}/{}.. ".format(e + 1, epochs),
                "Training Loss: {:.3f}.. ".format(running_loss / len(train_set)),
            )
            train_losses.append(running_loss / len(train_set))

        plt.plot(train_losses)
        plt.xlabel("Epoch")
        plt.ylabel("Loss")
        plt.title("Training loss plot")
        torch.save(model.state_dict(), "model.pth")

    def evaluate(self):
        print("Evaluating until hitting the ceiling")
        parser = argparse.ArgumentParser(description="Training arguments")
        parser.add_argument("load_model_from", default="")
        # add any additional argument that you want
        args = parser.parse_args(sys.argv[2:])
        print(args)

        # TODO: Implement evaluation logic here
        model = MyAwesomeModel()
        state_dict = torch.load(args.load_model_from)
        model.load_state_dict(state_dict)

        _, test_set = mnist()
        test_set = torch.utils.data.DataLoader(test_set, batch_size=64, shuffle=True)

        criterion = nn.NLLLoss()

        accuracy = []
        with torch.no_grad():
            for images, labels in test_set:
                model.eval()
                # View images as vectors
                images = images.view(images.shape[0], -1)

                # Evaluation mode - compute loss
                loss_valid = criterion(model(images), labels)

                log_ps_valid = torch.exp(model(images))
                top_p, top_class = log_ps_valid.topk(1, dim=1)
                equals = top_class == labels.view(*top_class.shape)

                accuracy.append(sum(equals) / len(equals))
                print(f"Validation loss: {loss_valid.item()}")

        print(f"Mean accuracy: {torch.mean(sum(accuracy)/len(accuracy))*100}%")

if __name__ == "__main__":
    TrainOREvaluate()

max_stars_repo_name: fredemandensGit/CookieCutterProject

max_issues_repo_name: fredemandensGit/CookieCutterProject

max_forks_repo_name: fredemandensGit/CookieCutterProject


text:
import argparse, umap, os, hdbscan
import numpy
import numpy as np
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.cluster import AgglomerativeClustering
from utils import read_json, write_json, check_make
import csv

this_script = os.path.dirname(os.path.realpath(__file__))

parser = argparse.ArgumentParser(description='Reduce the data from an analysis file')
parser.add_argument('-i', '--infile', type=str, help='The input folder to analyse')
parser.add_argument('-o', '--outfile', type=str, default='reduce.json', help='The output json')
parser.add_argument('-n', '--neighbours', type=int, default='7', help='Number of neighbours for UMAP.')
parser.add_argument('-m', '--mindist', type=float, default='0.1', help='Minimum distance for neighbourhood for UMAP')
parser.add_argument('-c', '--components', type=int, default='3', help='Number of components to reduce to')
parser.add_argument('-a', '--algorithm', type=str, default='hdbscan', help='algorithm for clustering. options are agglom(erative) or hdbscan.')
parser.add_argument('--sample',type=int, default=100000, help='Maximum elements to fit on before transformation')
parser.add_argument('-d', '--depth', type=int, default=250, help='number of clusters to look for (if agglomerative)')

args = parser.parse_args()

# feature = read_json(args.infile)
def read_db(csvfile):
    db = {"filenames":None,"mat":None}
    rows = []
    filenames = []
    with open(csvfile) as fd:
        reader = csv.reader(fd)
        for row in reader:
            filename = row[0]
            rest = [int(x) for x in row[1:]]
            index = rest[0]
            vals = rest[1:]
            filenames.append((filename,index))
            rows.append(vals)
    db["mat"] = numpy.array(rows)
    # print(csvfile,db["mat"].shape)
    db["filenames"] = filenames
    return db

feature = read_db( args.infile )

print("Loading DB")
# raw_data = [v for v in feature.values()]
raw_data = feature["mat"]
# keys = [k for k in feature.keys()]
keys = [str(x) for x in  feature["filenames"]]

scaler = StandardScaler()
unstandardised_data = np.array(raw_data)
data = scaler.fit_transform(unstandardised_data)

print("Create Reduction")
# Create class for computing UMAP
reduction = umap.UMAP(
    n_components=args.components,
    n_neighbors=args.neighbours,
    min_dist=args.mindist
)

# Execute fit and transform
print("Execute fit and transform")

if unstandardised_data.shape[0] > args.sample:
    print(f"Sampling {args.sample}")
    row_indices = np.random.choice(unstandardised_data.shape[0], size=args.sample, replace=False)
    print(f"Fitting {args.sample}")
    reduction = reduction.fit( unstandardised_data[row_indices,:] )
    print(f'Transforming {unstandardised_data.shape[0]}')
    data = reduction.transform( unstandardised_data )
else:
    data = reduction.fit_transform(unstandardised_data)

# data = reduction.fit_transform(data)

# Normalisation for sane coordinates
print("Normalisation for sane coordinates")
post_normalisation = MinMaxScaler()
normalised_data = post_normalisation.fit_transform(data)

########## CLUSTERING ##########
print("CLUSTERING")
if args.algorithm == 'hdbscan':
    db = hdbscan.HDBSCAN().fit(normalised_data)
if args.algorithm == 'agglom':
    db = AgglomerativeClustering(n_clusters=args.depth).fit(data)

### D3.JS ###

print("D3.JS Clusters")
num_clusters = 0
for _, cluster in zip(keys, db.labels_):
    if cluster > num_clusters:
        num_clusters = cluster

# Pack the keys and data into a dictionary
d3_coordinates = {}
d3_inner_list = []

print("Key to cluster")
key_to_cluster = {}
for audio, cluster in zip(keys, db.labels_):
    key_to_cluster[audio] = int(cluster)

print("KV")
for key, value in zip(keys, normalised_data):
    # print(key)
    t_dict = {}
    coords = value.tolist()
    t_dict["name"] = key
    t_dict["x"] = coords[0]
    t_dict["y"] = coords[1]
    t_dict["z"] = coords[2]
    #for audio, cluster in zip(keys, db.labels_):
    #    if audio == key:
    #        t_dict["cluster"] = int(cluster)
    if key in key_to_cluster:
            t_dict["cluster"] = key_to_cluster[key]
    d3_inner_list.append(t_dict)

d3_coordinates["data"] = d3_inner_list
d3_coordinates["meta"] = [int(num_clusters)]

# Write it out
print('Write it out')
json_dir = os.path.join(this_script, '_reduction')
json_path = os.path.join(json_dir, args.outfile)
write_json(json_path, d3_coordinates)

ext: py

max_stars_repo_name: abramhindle/ptomely-umap-scratcher

max_issues_repo_name: abramhindle/ptomely-umap-scratcher

max_forks_repo_name: abramhindle/ptomely-umap-scratcher

reason: import numpy


text:
import os, tqdm
import numpy as np
import matplotlib.pyplot as plt
from astropy.io import fits
from muchbettermoments import quadratic_2d
from astropy.wcs import WCS
from astropy.nddata import Cutout2D
from astropy.utils.data import download_file
import requests
from bs4 import BeautifulSoup
import warnings

from .mast import tic_by_contamination

def use_pointing_model(coords, pointing_model):
    """Applies pointing model to correct the position of star(s) on postcard.

    Parameters
    ----------
    coords : tuple
        (`x`, `y`) position of star(s).

    pointing_model : astropy.table.Table
        pointing_model for ONE cadence.

    Returns
    -------
    coords : tuple
        Corrected position of star(s).
    """
    pointing_model = np.reshape(list(pointing_model), (3,3))
    A = np.column_stack([coords[0], coords[1], np.ones_like(coords[0])])
    fhat = np.dot(A, pointing_model)
    return fhat[:,0:2]

class ffi:
    """This class allows the user to download all full-frame images for a given sector,
         camera, and chip. It also allows the user to create their own pointing model
         based on each cadence for a given combination of sector, camera, and chip.

    No individual user should have to download all of the full-frame images because
         stacked postcards will be available for the user to download from MAST.

    Parameters
    ----------
    sector : int, optional
    camera : int, optional
    chip : int, optional
    """
    def __init__(self, sector=None, camera=None, chip=None):
        self.sector = sector
        self.camera = camera
        self.chip   = chip

    def download_ffis(self, download_dir=None):
        """
        Downloads entire sector of data into FFI download directory.

        Parameters
        ----------
        download_dir : str
            Location where the data files will be stored.
            Defaults to "~/.eleanor/sector_{}/ffis" if `None` is passed.
        """

        def findAllFFIs(ca, ch):
            nonlocal year, days, url
            calFiles, urlPaths = [], []
            for d in days:
                path = '/'.join(str(e) for e in [url, year, d, ca])+'-'+str(ch)+'/'
                for fn in BeautifulSoup(requests.get(path).text, "lxml").find_all('a'):
                    if fn.get('href')[-7::] == 'ic.fits':
                        calFiles.append(fn.get('href'))
                        urlPaths.append(path)
            return calFiles, urlPaths

        if self.sector in np.arange(1,14,1):
            year=2019
        else:
            year=2020
        # Current days available for ETE-6
        days = np.arange(129,158,1)

        # This URL applies to ETE-6 simulated data ONLY
        url = 'https://archive.stsci.edu/missions/tess/ete-6/ffi/'
        files, urlPaths = findAllFFIs(self.camera, self.chip)

        if download_dir is None:
            # Creates hidden .eleanor FFI directory
            ffi_dir = self._fetch_ffi_dir()
        else:
            ffi_dir = download_dir

        files_in_dir = os.listdir(ffi_dir)

        local_paths = []
        for i in range(len(files)):
            if files[i] not in files_in_dir:
                os.system('cd {} && curl -O -L {}'.format(ffi_dir, urlPaths[i]+files[i]))
            local_paths.append(ffi_dir+files[i])
        self.local_paths = np.array(local_paths)
        return

    def _fetch_ffi_dir(self):
        """Returns the default path to the directory where FFIs will be downloaded.

        By default, this method will return "~/.eleanor/sector_{}/ffis" and create
        this directory if it does not exist.  If the directory cannot be
        access or created, then it returns the local directory (".").

        Returns
        -------
        download_dir : str
            Path to location of `ffi_dir` where FFIs will be downloaded
        """
        download_dir = os.path.join(os.path.expanduser('~'), '.eleanor',
                                    'sector_{}'.format(self.sector), 'ffis')
        if os.path.isdir(download_dir):
            return download_dir
        else:
            # if it doesn't exist, make a new cache directory
            try:
                os.mkdir(download_dir)
            # downloads locally if OS error occurs
            except OSError:
                warnings.warn('Warning: unable to create {}. '
                              'Downloading FFIs to the current '
                              'working directory instead.'.format(download_dir))
                download_dir = '.'

        return download_dir

    def sort_by_date(self):
        """Sorts FITS files by start date of observation."""
        dates, time = [], []
        for f in self.local_paths:
            hdu = fits.open(f)
            hdr = hdu[1].header
            dates.append(hdr['DATE-OBS'])
        dates, fns = np.sort(np.array([dates, self.local_paths]))
        self.local_paths = fns
        self.dates = dates
        return

    def build_pointing_model(self, pos_predicted, pos_inferred, outlier_removal=False):
        """Builds an affine transformation to correct the positions of stars
           from a possibly incorrect WCS.

        Parameters
        ----------
        pos_predicted : tuple
            Positions taken straight from the WCS; [[x,y],[x,y],...] format.
        pos_inferred : tuple
            Positions taken using any centroiding method; [[x,y],[x,y],...] format.
        outlier_removal : bool, optional
            Whether to clip 1-sigma outlier frames. Default `False`.

        Returns
        -------
        xhat : np.ndarray
            (3, 3) affine transformation matrix between WCS positions
            and inferred positions.
        """
        A = np.column_stack([pos_predicted[:,0], pos_predicted[:,1], np.ones_like(pos_predicted[:,0])])
        f = np.column_stack([pos_inferred[:,0], pos_inferred[:,1], np.ones_like(pos_inferred[:,0])])
        if outlier_removal == True:
            dist = np.sqrt(np.sum((A - f)**2, axis=1))
            mean, std = np.nanmean(dist), np.nanstd(dist)
            lim = 1.0
            A = A[dist < mean + lim*std]
            f = f[dist < mean + lim*std]
        ATA = np.dot(A.T, A)
        ATAinv = np.linalg.inv(ATA)
        ATf = np.dot(A.T, f)
        xhat = np.dot(ATAinv, ATf)
        fhat = np.dot(A, xhat)

        return xhat

    def pointing_model_per_cadence(self):
        """Step through build_pointing_model for each cadence."""

        def find_isolated(x, y):
            """Finds the most isolated, least contaminated sources for pointing model."""
            isolated = []
            for i in range(len(x)):
                x_list  = np.delete(x, np.where(x==x[i]))
                y_list  = np.delete(y, np.where(y==y[i]))
                closest = np.sqrt( (x[i]-x_list)**2 + (y[i]-y_list)**2 ).argmin()
                dist    = np.sqrt( (x[i]-x_list[closest])**2+ (y[i]-y_list[closest])**2 )
                if dist > 8.0:
                    isolated.append(i)
            return np.array(isolated)

        def isolated_center(x, y, image):
            """Finds the centroid of each isolated source with quadratic_2d.

            Parameters
            ----------
            x : array-like
                Initial guesses of x positions for sources.
            y : array-like
                Initial guesses of y positions for sources.
            image : np.ndarray
                FFI flux data.

            Returns
            -------
            cenx : array-like
                Controid x coordinates for all good input sources.
            ceny : array-like
                Centroid y coordinates for all good input sources.
            good : list
                Indexes into input arrays corresponding to good sources.
            """
            cenx, ceny, good = [], [], []
            print("Finding isolated centers of sources")
            for i in range(len(x)):
                 if x[i] > 0. and y[i] > 0.:
                     tpf = Cutout2D(image, position=(x[i], y[i]), size=(7,7), mode='partial')
                     origin = tpf.origin_original
                     cen = quadratic_2d(tpf.data)
                     cenx.append(cen[0]+origin[0]); ceny.append(cen[1]+origin[1])
                     good.append(i)
            cenx, ceny = np.array(cenx), np.array(ceny)
            return cenx, ceny, good

        def apply_pointing_model(xy, matrix):
            pointing_model = matrix
            centroid_xs, centroid_ys = [], []
            new_coords = use_pointing_model(xy, pointing_model)
            return np.array(new_coords)

        pm_fn = 'pointingModel_{}_{}-{}.txt'.format(self.sector, self.camera, self.chip)
        with open(pm_fn, 'w') as tf:
            tf.write('0 1 2 3 4 5 6 7 8\n')

        hdu = fits.open(self.local_paths[0])
        hdr = hdu[1].header
        pos = [hdr['CRVAL1'], hdr['CRVAL2']]

        r = 6.0#*np.sqrt(1.2)
        contam = [0.0, 5e-3]
        tmag_lim = 12.5

        t  = tic_by_contamination(pos, r, contam, tmag_lim)

        for fn in self.local_paths:
            print(fn)
            hdu = fits.open(fn)
            hdr = hdu[1].header
            xy = WCS(hdr).all_world2pix(t['ra'], t['dec'], 1)
            # Triple checks the sources are on the FFI
            onFrame = np.where( (xy[0]>10) & (xy[0]<2092-10) & (xy[1]>10) & (xy[1]<2048-10) )[0]
            xy  = np.array([xy[0][onFrame], xy[1][onFrame]])
            iso = find_isolated(xy[0], xy[1])
            xy  = np.array([xy[0][iso], xy[1][iso]])
            cenx, ceny, good = isolated_center(xy[0], xy[1], hdu[1].data)

            # Triple checks there are no nans; Nans make people sad
            no_nans = np.where( (np.isnan(cenx)==False) & (np.isnan(ceny)==False))
            pos_inferred = np.array( [cenx[no_nans], ceny[no_nans]] )
            xy = np.array( [xy[0][no_nans], xy[1][no_nans]] )

            solution = self.build_pointing_model(xy.T, pos_inferred.T)

            xy = apply_pointing_model(xy.T, solution)
            matrix = self.build_pointing_model(xy, pos_inferred.T, outlier_removal=True)

            sol    = np.dot(matrix, solution)
            sol    = sol.flatten()

            with open(pm_fn, 'a') as tf:
                tf.write('{}\n'.format(' '.join(str(e) for e in sol) ) )

max_stars_repo_name: megbedell/ELLIE

max_issues_repo_name: megbedell/ELLIE

max_forks_repo_name: megbedell/ELLIE

reason: import numpy,from astropy


text:
# AsteScorer
#   by: Camilla Padgitt-Coles
#   Katya Rozanova
#   Nicolas Escarpentier

import numpy  as np
import random as rng
import json
import dateutil.parser as dateparser

# MIDI to frequency function
def mtof(m):
	f = 440. * np.exp(.057762265 * (m - 69.))
	return f

# == JSON data ==
aster_file = open('asteroids_comets.json')
aster_data = json.load(aster_file)
aster_file.close()

# == GLOBAL VARIABLES ==
score_file = []
month_interval = 5. # in sec
year_mult = 12 # multiplier (how many months in a year)
total_time = month_interval * 12 * 6

# scale - 3 octaves
base = 58   # b flat
minor = [0, 2, 3, 5, 7, 8, 10]
scale = []
for i in range(21):
    octave = int(i/7)
    scale.append( base +(12*octave) +minor[i%len(minor)] )

# orbit types for defining instruments
types = []
[types.append(a['orbit_class']) for a in aster_data if a['orbit_class'] not in types]

# == F TABLE & HARMONICS ==
score_file.append("f1 0 16384 10 1." + "\n")
score_file.append("f2 0 16384 10 1 0.8 0.9 0.2 0.5" + "\n")
score_file.append("f3 0 16384 10 1 0.1 0.1 0.8 0.9" + "\n")
score_file.append("f4 0 16384 10 1 0.9 0.9 0.5 0.2" + "\n")
score_file.append("\n")

# == LOOP over asteroids ==
for a in aster_data:
    # get raw data
    datetime = dateparser.parse(a['discovery_date'])
    yea = datetime.year       # starting time
    mon = datetime.month      # starting time
    day = datetime.day        # starting time
    deg = float(a['i_deg'] )  # sets the freq
    orbit = a['orbit_class']  # selects the instrument
    qau1 = float(a['q_au_1']) # sets the duration

    # sound parameters
    start = ((yea-2010)*year_mult + mon + day/30.) * month_interval # first occurrence (starting in Jan 1, 2010)
    dur  = qau1 * 2
    freq = mtof( scale[ int(deg/9) ] )
    amp  = rng.random()*2000 + 4000
    pan  = rng.random()*0.6 + 0.2
    instr = types.index(orbit) +1
    # write line
    note_list = ["i", instr, start, dur, freq, amp, pan, "\n"]
    note_str = " ".join(str(item) for item in note_list)
    score_file.append(note_str)

    # recursion
    if orbit != 'Parabolic Comet':
        recur = float(a['period_yr'])  # convert to number
        start += recur *year_mult
        while start < total_time:
            # write line - as we already have all the parameters
            note_list = ["i", instr, start, dur, freq, amp, pan, "\n"]
            note_str = " ".join(str(item) for item in note_list)
            score_file.append(note_str)
            # jump to the next apparition
            start += recur *year_mult

# == END ==
score_file.append("e")

# == WRITE FILE ==
filename = "asterscore.sco"
# clear the file
open(filename,"w").close()
# write the lines
sco = open(filename,"w")
sco.writelines(score_file)
sco.close()


text:
import numpy as np
import matplotlib.pyplot as plt
epsilon = 1e-7 # global variable for fixing nan issues caused.

class Activation_Functions:

  def sigmoid(self,Z):
    A = 1/(1+np.exp(-Z))
    return A

  def tanh(self,Z):
    A = np.tanh(Z)
    return A

  def relu(self,Z):
    A = np.maximum(0,Z)
    return A

  def softmax(self,Z):
    A = np.exp(Z)/(np.sum(np.exp(Z))+epsilon)
    return A

  def relu_backward(self,dA, cache):
    Z = cache
    dZ = np.array(dA, copy=True)
    dZ[Z <= 0] = 0
    return dZ

  def sigmoid_backward(self,dA, cache):
    Z = cache
    s = 1/(1+np.exp(-Z))
    dZ = dA * s * (1-s)
    return dZ

  def tanh_backward(self,dA, cache):
    A = cache
    dZ = np.multiply(dA, 1 - np.power(A, 2))
    return dZ

  def softmax_backward(self,dA,cache):
    Z = cache
    global epsilon
    Z = np.exp(Z)
    numerator = np.multiply( Z , np.sum(Z,axis=0,keepdims=True) - Z )
    denominator = np.square(np.sum(Z,axis=0,keepdims=True))+epsilon
    dZ = np.divide(numerator,denominator)
    return dZ

class Weight_Initialization:

  def weight_initialize_randomize(self,curr_layer,prev_layer,random_initializer):

    W = np.random.randn(curr_layer,prev_layer) * random_initializer
    b = np.zeros((curr_layer,1))
    return W,b

  def weight_initialize_xavier(self,curr_layer,prev_layer):

    W = np.random.randn(curr_layer,prev_layer) * np.sqrt(2/(curr_layer+prev_layer))
    b = np.zeros((curr_layer,1))
    return W,b

  def weight_initialize_he(self,curr_layer,prev_layer):

    W = np.random.randn(curr_layer, prev_layer) / np.sqrt(prev_layer) #np.random.randn(curr_layer,prev_layer) * np.sqrt(2/(prev_layer))
    b = np.zeros((curr_layer,1))
    return W,b

class Regularization_Functions:

  def cost_L1(self,W,reg_lambda):
    regularizer_cost = reg_lambda * np.sum(np.absolute(W))
    return regularizer_cost

  def cost_L2(self,W,reg_lambda):
    regularizer_cost = reg_lambda * np.sum(np.square(W))
    return regularizer_cost

  def compute_regularizer_cost(self,m,parameters,regularizer_layers):
    regularizer_compute_cost = 0
    for i in regularizer_layers:
      W = parameters["W"+i[1]]
      reg_lambda = regularizer_layers[i][1]
      if regularizer_layers[i][0] == "L1":
        regularizer_compute_cost += self.cost_L1(W,reg_lambda)
      elif regularizer_layers[i][0] == "L2":
        regularizer_compute_cost += self.cost_L2(W,reg_lambda)

    regularizer_compute_cost = regularizer_compute_cost/(2*m)
    return regularizer_compute_cost

  def backward_propagation_L1(self,W,m,reg_lambda):
    return reg_lambda/(2*m)

  def backward_propagation_L2(self,W,m,reg_lambda):
    return reg_lambda*W/m

  def backward_propagation_regularizer(self,m,parameters,regularizer_layers):
    grads_regularizers = {}
    for i in regularizer_layers:
      W = parameters["W"+i[1]]
      reg_lambda = regularizer_layers[i][1]
      if regularizer_layers[i][0] == "L1":
        grads_regularizers["dW"+i[1]] = self.backward_propagation_L1(W,m,reg_lambda)
      elif regularizer_layers[i][0] == "L2":
        grads_regularizers["dW"+i[1]] = self.backward_propagation_L2(W,m,reg_lambda)

    return grads_regularizers

class Drop_Out:

  def inverted_dropout_cache(self,A,dropout):

      D = np.random.randn(A.shape[0],A.shape[1]) < dropout
      A = A * D
      A = A / dropout

      return A,D

  def inverted_dropout_grads(self,dA,D,dropout):

    dA = dA * D
    dA = dA / dropout

    return dA

class Optimizers:

  def initialize_momentum(self,parameters,total_layers):

    v = {}
    for i in range(1,total_layers):
      v["dW" + str(i)] = np.zeros(parameters["W" + str(i)].shape)
      v["db" + str(i)] = np.zeros(parameters["b" + str(i)].shape)

    return v

  def initialize_rmsProp(self,parameters,total_layers):

    s = {}
    for i in range(1,total_layers):
      s["dW" + str(i)] = np.zeros(parameters["W" + str(i)].shape)
      s["db" + str(i)] = np.zeros(parameters["b" + str(i)].shape)

    return s

  def initialize_adam(self,parameters,total_layers):

    v = {}
    s = {}

    v = self.initialize_momentum(parameters,total_layers)
    s = self.initialize_rmsProp(parameters,total_layers)

    return v,s

  def gradient(self,parameters,total_layers,learning_rate,grads):

    for l in range(1,total_layers):
      parameters["W" + str(l)] = parameters["W" + str(l)] - learning_rate*grads["dW"+str(l)]
      parameters["b" + str(l)] = parameters["b" + str(l)] - learning_rate*grads["db"+str(l)]

    return parameters

  def momentum(self,parameters,v,total_layers,learning_rate,grads,beta):

    for l in range(1,total_layers):
      v["dW" + str(l)] = beta * v["dW" + str(l)] + (1 - beta) * grads['dW' + str(l)]
      v["db" + str(l)] = beta * v["db" + str(l)] + (1 - beta) * grads['db' + str(l)]
      parameters["W" + str(l)] = parameters["W" + str(l)] - learning_rate * v["dW" + str(l)]
      parameters["b" + str(l)] = parameters["b" + str(l)] - learning_rate * v["db" + str(l)]

    return parameters

  def rmsProp(self,parameters,s,total_layers,learning_rate,grads,beta):
    global epsilon
    for l in range(1,total_layers):
      s["dW" + str(l)] = beta * s["dW" + str(l)] + (1 - beta) * np.square(grads['dW' + str(l)])
      s["db" + str(l)] = beta * s["db" + str(l)] + (1 - beta) * np.square(grads['db' + str(l)])
      parameters["W" + str(l)] = parameters["W" + str(l)] - learning_rate * np.divide(grads["dW" + str(l)],np.sqrt(s["dW" + str(l)])+epsilon)
      parameters["b" + str(l)] = parameters["b" + str(l)] - learning_rate * np.divide(grads["db" + str(l)],np.sqrt(s["db" + str(l)])+epsilon)

    return parameters

  def adam(self,parameters,v,s,t,total_layers,learning_rate,grads,beta1,beta2):
    v_corrected = {}
    s_corrected = {}
    epsilon = 1e-8

    for l in range(1,total_layers):
      v["dW" + str(l)] = beta1 * v["dW" + str(l)] + (1 - beta1) * grads['dW' + str(l)]
      v["db" + str(l)] = beta1 * v["db" + str(l)] + (1 - beta1) * grads['db' + str(l)]

      v_corrected["dW" + str(l)] = v["dW" + str(l)] / (1 - np.power(beta1, t))
      v_corrected["db" + str(l)] = v["db" + str(l)] / (1 - np.power(beta1, t))

      s["dW" + str(l)] = beta2 * s["dW" + str(l)] + (1 - beta2) * np.power(grads['dW' + str(l)], 2)
      s["db" + str(l)] = beta2 * s["db" + str(l)] + (1 - beta2) * np.power(grads['db' + str(l)], 2)

      s_corrected["dW" + str(l)] = s["dW" + str(l)] / (1 - np.power(beta2, t))
      s_corrected["db" + str(l)] = s["db" + str(l)] / (1 - np.power(beta2, t))

      parameters["W" + str(l)] = parameters["W" + str(l)] - learning_rate * v_corrected["dW" + str(l)] / np.sqrt(s_corrected["dW" + str(l)] + epsilon)
      parameters["b" + str(l)] = parameters["b" + str(l)] - learning_rate * v_corrected["db" + str(l)] / np.sqrt(s_corrected["db" + str(l)] + epsilon)

    return parameters

class Cost_Functions:

  def crossEntropy_cost(self,Y_pred,Y,m):

    cost = (1./m) * (-np.dot(Y,np.log(Y_pred).T) - np.dot(1-Y, np.log(1-Y_pred).T))
    return cost

  def meanSquare_cost(self,Y_pred,Y,m):

    cost = (1/m) * np.sum(np.square(Y - Y_pred))
    return cost

  def crossEntropy_derivative(self,Y_pred,Y):

    dA = -(np.divide(Y, Y_pred+epsilon) - np.divide(1 - Y, 1 - Y_pred+epsilon))
    return dA

  def meanSquare_derivative(self,Y_pred,Y):

    dA = Y_pred - Y
    return dA

  def categorical_crossEntropy_cost(self,Y_pred,Y,m):

    cost = (-1 / m) * np.sum(np.multiply(Y, np.log(Y_pred+epsilon)))
    return cost

  def categorical_crossEntropy_derivative(self,Y_pred,Y):

    dA = -np.divide(Y,Y_pred+epsilon)
    return dA

def generate_mini_batches(X, Y, batch_size = 64):

    m = X.shape[1]
    mini_batches = []

    permutation = list(np.random.permutation(m))
    shuffled_X = X[:, permutation]
    shuffled_Y = Y[:, permutation].reshape((Y.shape[0],m))

    num_complete_minibatches = int(np.floor(m/batch_size))

    for k in range(num_complete_minibatches):
        mini_batch_X = shuffled_X[:,k * batch_size:(k + 1) * batch_size]
        mini_batch_Y = shuffled_Y[:,k * batch_size:(k + 1) * batch_size]
        mini_batch = (mini_batch_X, mini_batch_Y)
        mini_batches.append(mini_batch)

    if m % batch_size != 0:
        mini_batch_X = shuffled_X[:,num_complete_minibatches *batch_size:]
        mini_batch_Y = shuffled_Y[:,num_complete_minibatches * batch_size:]
        mini_batch = (mini_batch_X, mini_batch_Y)
        mini_batches.append(mini_batch)
        num_complete_minibatches += 1

    return mini_batches,num_complete_minibatches

def forward_propagation(X,total_layers,parameters,activations,dropout_layers):

  cache = {}
  cache["A0"] = X

  activation_function = Activation_Functions()
  drop_out = Drop_Out()

  for i in range(1,total_layers):

    cache["Z" + str(i)] = np.dot(parameters["W" + str(i)],cache["A"+str(i-1)]) + parameters["b" + str(i)]

    if activations["AF"+str(i)] == "sigmoid":
      cache["A" + str(i)] = activation_function.sigmoid(cache["Z" + str(i)])

    elif activations["AF"+str(i)] == "tanh":
      cache["A" + str(i)] = activation_function.tanh(cache["Z" + str(i)])

    elif activations["AF"+str(i)] == "relu":
      cache["A" + str(i)] = activation_function.relu(cache["Z" + str(i)])

    elif activations["AF"+str(i)] == "softmax":
      cache["A" + str(i)] = activation_function.softmax(cache["Z" + str(i)])

    if "h" + str(i) in dropout_layers.keys():
      cache["A" + str(i)],cache["D" + str(i)] = drop_out.inverted_dropout_cache(cache["A" + str(i)],dropout_layers["h" + str(i)])

  return cache

def compute_cost(Y_pred,Y,cost_function,parameters,regularizer_layers):

  regularization_function = Regularization_Functions()
  cost_functions = Cost_Functions()

  m = Y.shape[1]
  reg_cost = regularization_function.compute_regularizer_cost(m,parameters,regularizer_layers)

  if cost_function == "crossEntropy":
    cost = cost_functions.crossEntropy_cost(Y_pred,Y,m)

  elif cost_function == "meanSquare":
    cost = cost_functions.meanSquare_cost(Y_pred,Y,m)

  elif cost_function == "categoricalCrossEntropy":
    cost = cost_functions.categorical_crossEntropy_cost(Y_pred,Y,m)

  cost = cost + reg_cost
  cost = np.squeeze(cost)

  return cost

def backward_propagation(Y,cost_function,parameters,cache,activations,total_layers,regularizer_layers,dropout_layers):

  Y_pred = cache["A" + str(total_layers)]
  m = Y_pred.shape[1]
  Y = Y.reshape(Y_pred.shape)
  grads = {}
  propogate = {}

  activation_function = Activation_Functions()
  cost_functions = Cost_Functions()
  drop_out = Drop_Out()
  regularization_function = Regularization_Functions()

  if cost_function == "crossEntropy":
    propogate["dA"+str(total_layers)] = cost_functions.crossEntropy_derivative(Y_pred,Y)

  elif cost_function == "meanSquare":
    propogate["dA"+str(total_layers)] = cost_functions.meanSquare_derivative(Y_pred,Y)

  elif cost_function == "categoricalCrossEntropy":
    propogate["dA"+str(total_layers)] = cost_functions.categorical_crossEntropy_derivative(Y_pred,Y)

  for i in range(total_layers,0,-1):

    if (activations["AF" + str(i)] == "sigmoid"):
      propogate["dZ"+str(i)] = activation_function.sigmoid_backward(propogate["dA"+str(i)], cache["Z" + str(i)])

    elif (activations["AF" + str(i)] == "relu"):
      propogate["dZ"+str(i)] = activation_function.relu_backward(propogate["dA"+str(i)], cache["Z" + str(i)])

    elif (activations["AF" + str(i)] == "tanh"):
      propogate["dZ"+str(i)] = activation_function.tanh_backward(propogate["dA"+str(i)], cache["A" + str(i)])

    elif (activations["AF" + str(i)] == "softmax"):
      propogate["dZ"+str(i)] = activation_function.softmax_backward(propogate["dA"+str(i)], cache["Z" + str(i)])

    grads["dW"+str(i)] = (1 / m) * np.dot(propogate["dZ"+str(i)], cache["A"+str(i-1)].T)

    grads["db"+str(i)] = (1 / m) * np.sum(propogate["dZ"+str(i)], axis=1, keepdims=True)

    propogate["dA"+str(i-1)] = np.dot(parameters["W"+str(i)].T,propogate["dZ"+str(i)])

    if "D"+str(i-1) in  cache.keys():
      propogate["dA"+str(i-1)] = drop_out.inverted_dropout_grads(propogate["dA"+str(i-1)],cache["D"+str(i-1)],dropout_layers["h"+str(i-1)])

  grads_regularizers = regularization_function.backward_propagation_regularizer(m,parameters,regularizer_layers)
  for i in grads_regularizers:
    grads[i] += grads_regularizers[i]

  return grads

def update_parameters(parameters,v,s,t,grads,learning_rate,total_layers,optimizer,momentum_beta,rms_beta,adam_beta1,adam_beta2):

  optimizers = Optimizers()

  if optimizer == "gradient":
    parameters = optimizers.gradient(parameters,total_layers,learning_rate,grads)

  elif optimizer == "momentum":
    parameters = optimizers.momentum(parameters,v,total_layers,learning_rate,grads,momentum_beta)

  elif optimizer == "rmsProp":
    parameters = optimizers.rmsProp(parameters,s,total_layers,learning_rate,grads,rms_beta)

  elif optimizer == "adam":
    parameters = optimizers.adam(parameters,v,s,t,total_layers,learning_rate,grads,adam_beta1,adam_beta2)

  return parameters

class Architecture:

  def __init__(self):

    self.parameters = {}
    self.activations = {}
    self.cache = {}
    self.grads = {}
    self.layer_dims = []
    self.total_layers = 0
    self.cost_function = ''
    self.optimizer = ''
    self.learning_rate = 0
    self.momentum_beta = 0
    self.rms_beta = 0
    self.adam_beta1 = 0
    self.adam_beta2 = 0
    self.adam_t = 0
    self.cost = 0
    self.cost_graph = []
    self.weight_initialization = ''
    self.random_initializer = 0
    self.regularizer_layers = {}
    self.dropout_layers = {}
    self.momentum_parameters = {}
    self.rms_parameters = {}
    self.batch_size = 0
    self.mini_batches = []
    self.mini_iterations = 0
    self.input_mean = 0
    self.input_variance = 0

  def add_inputLayer(self,input_units,normalization = False,weight_initialization="randomize",random_initializer = 0.01):

    self.normalization = normalization
    self.layer_dims.append(input_units)
    self.total_layers += 1
    self.weight_initialization = weight_initialization
    self.random_initializer = random_initializer
    #np.random.seed(1)

  def add_hiddenLayer(self,hidden_units, activation_function = "sigmoid", dropout = 1,regularizer = None, regularizer_lambda=0.1):

    weight_initializer = Weight_Initialization()

    if (regularizer != None):
      self.regularizer_layers["h"+str(self.total_layers)] = (regularizer,regularizer_lambda)

    if dropout < 1:
      self.dropout_layers["h" + str(self.total_layers)] = dropout

    if (self.weight_initialization == "randomize"):
      self.parameters["W"+str(self.total_layers)],self.parameters["b"+str(self.total_layers)] = weight_initializer.weight_initialize_randomize(hidden_units,self.layer_dims[-1],self.random_initializer)

    elif (self.weight_initialization == "xavier"):
      self.parameters["W"+str(self.total_layers)],self.parameters["b"+str(self.total_layers)] = weight_initializer.weight_initialize_xavier(hidden_units,self.layer_dims[-1])

    elif (self.weight_initialization == "he"):
      self.parameters["W"+str(self.total_layers)],self.parameters["b"+str(self.total_layers)] = weight_initializer.weight_initialize_he(hidden_units,self.layer_dims[-1])

    self.activations["AF"+str(self.total_layers)] = activation_function
    self.layer_dims.append(hidden_units)
    self.total_layers += 1

  def compile(self,cost_function = "crossEntropy", optimizer = 'gradient', learning_rate = 0.01, momentum_beta = 0.9, rms_beta = 0.999, adam_beta1=0.9, adam_beta2=0.999,batch_normalization = False):

    optimizers = Optimizers()

    self.cost_function = cost_function
    self.optimizer = optimizer
    self.learning_rate = learning_rate
    self.momentum_beta = momentum_beta
    self.rms_beta = rms_beta
    self.adam_beta1 = adam_beta1
    self.adam_beta2 = adam_beta2

    if optimizer == "momentum":
      self.momentum_parameters = optimizers.initialize_momentum(self.parameters,self.total_layers)

    elif optimizer == "rmsProp":
      self.rms_parameters = optimizers.initialize_rmsProp(self.parameters,self.total_layers)

    elif optimizer == "adam":
      self.momentum_parameters,self.rms_parameters = optimizers.initialize_adam(self.parameters,self.total_layers)

  def train(self,X,Y,number_of_epoch = 1000,batch_size=0,print_every = 100):
    #np.random.seed(1)
    if self.normalization == True:
      self.input_mean = np.mean(X,axis=1,keepdims = True)
      self.input_variance = np.var(X,axis=1, keepdims = True)
      X = np.divide(X-self.input_mean,np.sqrt(self.input_variance)+epsilon)

    self.batch_size = batch_size

    if (self.batch_size == 0):
      self.mini_batches.append((X,Y))
      self.mini_iterations = 1

    else:
      self.mini_batches,self.mini_iterations = generate_mini_batches(X, Y, self.batch_size)

    for i in range(number_of_epoch):

      if (self.mini_iterations != 1):
        self.mini_batches,self.mini_iterations = generate_mini_batches(X, Y, self.batch_size)

      for minibatch in self.mini_batches:

        (minibatch_X,minibatch_Y) = minibatch

        self.cache = forward_propagation(minibatch_X,self.total_layers,self.parameters,self.activations,self.dropout_layers)

        Y_pred = self.cache["A"+str(self.total_layers-1)]
        self.cost = compute_cost(Y_pred,minibatch_Y,self.cost_function,self.parameters,self.regularizer_layers)

        self.grads = backward_propagation(minibatch_Y,self.cost_function,self.parameters,self.cache,self.activations,self.total_layers-1,self.regularizer_layers,self.dropout_layers)

        self.adam_t += 1
        self.parameters = update_parameters(self.parameters,self.momentum_parameters,self.rms_parameters,self.adam_t,self.grads,self.learning_rate,self.total_layers,self.optimizer,self.momentum_beta,self.rms_beta,self.adam_beta1,self.adam_beta2)

      if i % print_every == 0:
        print ("Cost after iteration %i: %f" % (i,self.cost))
        self.cost_graph.append(self.cost)

    plt.figure(figsize=(10,6))
    plt.plot(np.squeeze(self.cost_graph))
    plt.ylabel('cost')
    plt.xlabel('iterations every'+str(print_every))
    plt.title("Learning rate =" + str(self.learning_rate))
    plt.show()

max_stars_repo_name: Aiden-Python/Megamind

max_issues_repo_name: Aiden-Python/Megamind

max_forks_repo_name: Aiden-Python/Megamind


text:
import numpy as np
import scipy as sp
import pylab as pl
import sys
sys.path.append('../../lablib')
import ourgui

filename = ourgui.openFile()
meas = np.loadtxt(filename, comments='#')
n = len(meas)
rate = 20000.0
totalt = n / rate
t = np.linspace(0, totalt, n)

pl.subplot(211)
pl.plot(t, meas)

dt = t[1]-t[0]
freq = np.fft.fftfreq(t.shape[-1], d=dt)
print max(freq)
sp = np.fft.fft(meas)
spmag = np.sqrt(sp.real**2 + sp.imag**2)
spmag = spmag[freq >= 0]
freq = freq[freq >= 0]
print spmag
pl.subplot(212)
pl.semilogy(freq, spmag)
pl.xlim([0, 6250])

out = zip(freq, spmag)
np.save(filename+'.fft', out)

pl.show()

max_stars_repo_name: imrehg/labhardware

max_issues_repo_name: imrehg/labhardware

max_forks_repo_name: imrehg/labhardware

reason: import numpy,import scipy


text:
DESC="""Defines multiscale network class and provides read/write
functionality. A multiscale network consists of (1) a sequence of networks
(specified as ((0.nodes,0.edges),(1.nodes,1.edges),...) and (2) a hierarchy
tree (hierarchy.tree) that specifies the parent-child relationship between
nodes of different levels.

Pending:
- Mandatory column testing.
- To verify the hierarchy tree, it is read in as a string dataframe. This is a
  potential performance issue. However, it permits using different data types
  at different levels.
By: MS and AA
"""
import argparse
import logging
import numpy as np
import pandas as pd
import pdb
from glob import glob
import networkx as nx
from pdb import set_trace

EPI="""The file format for node file is:
node,attr1,attr2
1,0,0
2,0,0
3,0,0
4,0,0
The file format for edge file is:
source,target,Time,Weight,Condition
1,2,0,0,0
2,3,0,0,0
2,4,0,0,0
4,3,0,0,0
4,1,0,0,0
The node and the edge file have levelnumber.node, levelnumber.edge naming convention.
"""

class MultiScaleNet:

    def __init__(self):

        self.name=None
        self.nodes=[]
        self.edges=[]
        self.hierarchy=None
        self.number_of_levels=0

    def read_from_file(self,networkFolder):

        self.name=networkFolder

        # Number of levels
        nodeFiles = glob(networkFolder+'/*.nodes')
        edgeFiles = glob(networkFolder+'/*.edges')
        self.number_of_levels=len(nodeFiles)
        if self.number_of_levels != len(edgeFiles):
            raise Exception('Number of node files and number of edge files do not match.')

        # Read nodes
        nodeLevelMapper=pd.DataFrame(columns = {'node': [], 'level': []})
        for l in range(self.number_of_levels):
            nodeFileName='%s/%d.nodes' %(networkFolder,l)
            try:
                currentLevelNodes=pd.read_csv(nodeFileName)
                if 'node' not in currentLevelNodes.columns:
                    raise Exception(f"{nodeFileName}: 'node' column should be present in node file.")
                elif currentLevelNodes.node.dtype!=int:
                    raise Exception(f"{nodeFileName}: 'node' should be of 'int' type.")
                self.nodes.append(currentLevelNodes)
            except FileNotFoundError:
                raise  FileNotFoundError('Expected file %s is absent.' %nodeFileName)
            df=self.nodes[l]['node'].to_frame()
            df['level']=l
            nodeLevelMapper=pd.concat([nodeLevelMapper,df], ignore_index=True) #make into pd series with node =index

        nodeLevelMapper=nodeLevelMapper.append(
                {'node': -1,'level': self.number_of_levels},ignore_index=True)

        # Check if there are any duplicate node ids. That is, ids which occur in
        # more than one level.
        x=nodeLevelMapper.node.drop_duplicates()
        if x.shape[0]<nodeLevelMapper.shape[0]:
            raise Exception("A node id has been used in multiple levels.")

        nodeLevelMapper = nodeLevelMapper.set_index('node',drop=True).squeeze()

        # Read hierarchy and check if hierarchy conditions are met
        self.hierarchy=pd.read_csv(f'{networkFolder}/hierarchy.tree')
        if self.hierarchy.parent.dtype!=int or self.hierarchy.child.dtype!=int:
            raise Exception("Hierarchy file: All nodes should be integers.")
        levelMappedHierarchy=self.hierarchy.replace(nodeLevelMapper)

        if np.less(levelMappedHierarchy.parent,levelMappedHierarchy.child).any():  # Each child and each parents level compare
            raise Exception("A node has a parent at a lower level. Check input.")
        if not nx.is_tree(nx.from_pandas_edgelist(self.hierarchy,'parent','child')):
            raise Exception("The hierachy relationship is not a tree.")

        # Read edges
        for l in range(self.number_of_levels):
            edgeFileName='%s/%d.edges' %(networkFolder,l)
            try:
                x=pd.read_csv(edgeFileName)
                if ('source' and 'target' not in x.columns):
                    raise ValueError('Source and target columns should be present')
                self.edges.append(x)
            except FileNotFoundError:
                raise  FileNotFoundError('Expected file %s is absent.' %edgeFileName)
            # MS: throw an error if mandatory columns are not present
        self.compute_summary()

    def compute_summary(self):
        self.number_of_nodes=[]
        self.number_of_edges=[]
        for l in range(self.number_of_levels):
            self.number_of_nodes.append(self.nodes[l].shape[0])
            self.number_of_edges.append(self.edges[l].shape[0])
        return

    def display_summary(self):
        logging.info(f"Name: {self.name}")
        logging.info(f"Levels: {self.number_of_levels}")
        logging.info(f"Nodes: {self.number_of_nodes}")
        logging.info(f"Edges: {self.number_of_edges}")
        return

if __name__=='__main__':
    # parser
    parser=argparse.ArgumentParser(description=DESC, epilog=EPI,formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("network", help="input network folder which contains (i.nodes,i.edges) for i=0,1,2,... and hierarchy.tree files.")

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG)

    net=MultiScaleNet()
    net.read_from_file(args.network)
    net.display_summary()

reason: import numpy,import networkx


text:
# coding=utf-8
# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from abc import ABC, abstractmethod
from pickle import UnpicklingError
from typing import Dict

import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.serialization import to_bytes
from flax.traverse_util import unflatten_dict
from jax.random import PRNGKey

from .configuration_utils import PretrainedConfig
from .file_utils import WEIGHTS_NAME, cached_path, hf_bucket_url, is_remote_url
from .utils import logging

logger = logging.get_logger(__name__)

@jax.jit
def gelu(x):
    r"""
    Gaussian error linear unit activation function.

    Computes the element-wise function:

    .. math::
      \mathrm{gelu}(x) = \frac{x}{2} \left(1 + \mathrm{tanh} \left(
        \sqrt{\frac{2}{\pi}} \left(x + 0.044715 x^3 \right) \right) \right)

    We explicitly use the approximation rather than the exact formulation for speed. For more information, see
    `Gaussian Error Linear Units (GELUs) <https://arxiv.org/abs/1606.08415>`_, section 2.
    """
    return x * 0.5 * (1.0 + jax.lax.erf(x / jnp.sqrt(2.0)))

ACT2FN = {
    "gelu": nn.gelu,
    "relu": nn.relu,
    "silu": nn.swish,
    "swish": nn.swish,
    "gelu_new": gelu,
}

class FlaxPreTrainedModel(ABC):
    config_class = None
    pretrained_model_archive_map = {}
    base_model_prefix = ""
    model_class = None

    def __init__(self, config: PretrainedConfig, module: nn.Module, params: Dict, seed: int = 0):
        if config is None:
            raise ValueError("config cannot be None")

        if module is None:
            raise ValueError("module cannot be None")

        if params is None:
            raise ValueError("state cannot be None")

        # Those are private to be exposed as typed property on derived classes.
        self._config = config
        self._module = module

        # Those are public as their type is generic to every derived classes.
        self.key = PRNGKey(seed)
        self.params = params
        self.model = module

    @property
    def config(self) -> PretrainedConfig:
        return self._config

    @staticmethod
    @abstractmethod
    def convert_from_pytorch(pt_state: Dict, config: PretrainedConfig) -> Dict:
        raise NotImplementedError()

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
        r"""
        Instantiate a pretrained Flax model from a pre-trained model configuration.
        """
        config = kwargs.pop("config", None)
        # state_dict = kwargs.pop("state_dict", None)
        cache_dir = kwargs.pop("cache_dir", None)
        # from_tf = kwargs.pop("from_tf", False)
        force_download = kwargs.pop("force_download", False)
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        # output_loading_info = kwargs.pop("output_loading_info", False)
        local_files_only = kwargs.pop("local_files_only", False)
        revision = kwargs.pop("revision", None)

        # Load config if we don't provide a configuration
        if not isinstance(config, PretrainedConfig):
            config_path = config if config is not None else pretrained_model_name_or_path
            config, model_kwargs = cls.config_class.from_pretrained(
                config_path,
                *model_args,
                cache_dir=cache_dir,
                return_unused_kwargs=True,
                force_download=force_download,
                resume_download=resume_download,
                proxies=proxies,
                local_files_only=local_files_only,
                revision=revision,
                **kwargs,
            )
        else:
            model_kwargs = kwargs

        # Load model
        if pretrained_model_name_or_path is not None:
            if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
                archive_file = pretrained_model_name_or_path
            else:
                archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME, revision=revision)

            # redirect to the cache, if necessary
            try:
                resolved_archive_file = cached_path(
                    archive_file,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    local_files_only=local_files_only,
                )
            except EnvironmentError as err:
                logger.error(err)
                msg = (
                    f"Can't load weights for '{pretrained_model_name_or_path}'. Make sure that:\n\n"
                    f"- '{pretrained_model_name_or_path}' is a correct model identifier listed on 'https://huggingface.co/models'\n\n"
                    f"- or '{pretrained_model_name_or_path}' is the correct path to a directory containing a file named {WEIGHTS_NAME}.\n\n"
                )
                raise EnvironmentError(msg)

            if resolved_archive_file == archive_file:
                logger.info(f"loading weights file {archive_file}")
            else:
                logger.info(f"loading weights file {archive_file} from cache at {resolved_archive_file}")
        else:
            resolved_archive_file = None

        # Instantiate model.
        with open(resolved_archive_file, "rb") as state_f:
            try:
                from flax.serialization import from_bytes

                state = from_bytes(cls.model_class, state_f)
            except TypeError:
                try:
                    import torch

                    state = torch.load(state_f)
                    state = {k: v.numpy() for k, v in state.items()}
                    state = cls.convert_from_pytorch(state, config)
                    state = unflatten_dict({tuple(k.split(".")[1:]): v for k, v in state.items()})
                except UnpicklingError:
                    raise EnvironmentError(
                        f"Unable to convert model {archive_file} to Flax deserializable object. "
                        "Supported format are PyTorch archive or Flax msgpack"
                    )

        return cls(config, state, *model_args, **model_kwargs)

    def save_pretrained(self, folder):
        folder_abs = os.path.abspath(folder)

        if not os.path.exists(folder_abs):
            os.mkdir(folder_abs)

        with open(os.path.join(folder_abs, f"{self._config.model_type}.flax", "wb")) as f:
            model_bytes = to_bytes(self.params)
            f.write(model_bytes)

max_issues_repo_name: tutussss/transformers

max_forks_repo_name: tutussss/transformers

reason: import jax,from jax


text:
#!/usr/bin/env python
# Copyright 2014-2019 The PySCF Developers. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
1-electron Spin-free X2C approximation
'''

from functools import reduce
import numpy
import scipy.linalg
from pyscf import lib
from pyscf import gto
from pyscf.lib import logger
from pyscf.scf import hf
from pyscf.scf import ghf
from pyscf.x2c import x2c
from pyscf.data import nist

def sfx2c1e(mf):
    '''Spin-free X2C.
    For the given SCF object, it updates the hcore constructor.  All integrals
    are computed in the real spherical GTO basis.

    Args:
        mf : an SCF object

    Returns:
        An SCF object

    Examples:

    >>> mol = gto.M(atom='H 0 0 0; F 0 0 1', basis='ccpvdz', verbose=0)
    >>> mf = scf.RHF(mol).sfx2c1e()
    >>> mf.scf()

    >>> import pyscf.x2c.sfx2c1e
    >>> mol.symmetry = 1
    >>> mol.build(0, 0)
    >>> mf = pyscf.x2c.sfx2c1e.sfx2c1e(scf.UHF(mol))
    >>> mf.scf()
    '''
    assert isinstance(mf, hf.SCF)

    if isinstance(mf, x2c._X2C_SCF):
        if mf.with_x2c is None:
            mf.with_x2c = SpinFreeX2CHelper(mf.mol)
            return mf
        elif not isinstance(mf.with_x2c, SpinFreeX2CHelper):
            # An object associated to x2c.SpinOrbitalX2CHelper
            raise NotImplementedError
        else:
            return mf

    mf_class = mf.__class__
    if mf_class.__doc__ is None:
        doc = ''
    else:
        doc = mf_class.__doc__

    class SFX2C1E_SCF(x2c._X2C_SCF, mf_class):
        __doc__ = doc + '''
        Attributes for spin-free X2C:
            with_x2c : X2C object
        '''
        def __init__(self, mol, *args, **kwargs):
            mf_class.__init__(self, mol, *args, **kwargs)
            self.with_x2c = SpinFreeX2CHelper(mf.mol)
            self._keys = self._keys.union(['with_x2c'])

        def get_hcore(self, mol=None):
            if self.with_x2c:
                hcore = self.with_x2c.get_hcore(mol)
                if isinstance(self, ghf.GHF):
                    hcore = scipy.linalg.block_diag(hcore, hcore)
                return hcore
            else:
                return mf_class.get_hcore(self, mol)

        def dump_flags(self, verbose=None):
            mf_class.dump_flags(self, verbose)
            if self.with_x2c:
                self.with_x2c.dump_flags(verbose)
            return self

        def reset(self, mol):
            self.with_x2c.reset(mol)
            return mf_class.reset(self, mol)

        def dip_moment(self, mol=None, dm=None, unit='Debye', verbose=logger.NOTE,
                       picture_change=True, **kwargs):
            r''' Dipole moment calculation with picture change correction

            Args:
                 mol: an instance of :class:`Mole`
                 dm : a 2D ndarrays density matrices

            Kwarg:
                picture_chang (bool) : Whether to compute the dipole moment with
                picture change correction.

            Return:
                A list: the dipole moment on x, y and z component
            '''
            if mol is None: mol = self.mol
            if dm is None: dm = self.make_rdm1()
            log = logger.new_logger(mol, verbose)

            if not (isinstance(dm, numpy.ndarray) and dm.ndim == 2):
                # UHF denisty matrices
                dm = dm[0] + dm[1]

            if isinstance(self, ghf.GHF):
                nao = mol.nao_nr()
                dm = dm[:nao,:nao] + dm[nao:,nao:]

            with mol.with_common_orig((0,0,0)):
                if picture_change:
                    xmol = self.with_x2c.get_xmol()[0]
                    nao = xmol.nao
                    prp = xmol.intor_symmetric('int1e_sprsp').reshape(3,4,nao,nao)[:,0]
                    ao_dip = self.with_x2c.picture_change(('int1e_r', prp))
                else:
                    ao_dip = mol.intor_symmetric('int1e_r')

            el_dip = numpy.einsum('xij,ji->x', ao_dip, dm).real

            charges = mol.atom_charges()
            coords  = mol.atom_coords()
            nucl_dip = numpy.einsum('i,ix->x', charges, coords)
            mol_dip = nucl_dip - el_dip

            if unit.upper() == 'DEBYE':
                mol_dip *= nist.AU2DEBYE
                log.note('Dipole moment(X, Y, Z, Debye): %8.5f, %8.5f, %8.5f', *mol_dip)
            else:
                log.note('Dipole moment(X, Y, Z, A.U.): %8.5f, %8.5f, %8.5f', *mol_dip)
            return mol_dip

    with_x2c = SpinFreeX2CHelper(mf.mol)
    return mf.view(SFX2C1E_SCF).add_keys(with_x2c=with_x2c)

sfx2c = sfx2c1e

class SpinFreeX2CHelper(x2c.X2CHelperMixin):
    '''1-component X2c (spin-free part only)
    '''
    def get_hcore(self, mol=None):
        '''1-component X2c Foldy-Wouthuysen (FW Hamiltonian  (spin-free part only)
        '''
        if mol is None: mol = self.mol
        if mol.has_ecp():
            raise NotImplementedError

        xmol, contr_coeff = self.get_xmol(mol)
        c = lib.param.LIGHT_SPEED
        assert('1E' in self.approx.upper())
        t = xmol.intor_symmetric('int1e_kin')
        v = xmol.intor_symmetric('int1e_nuc')
        s = xmol.intor_symmetric('int1e_ovlp')
        w = xmol.intor_symmetric('int1e_pnucp')
        if 'get_xmat' in self.__dict__:
            # If the get_xmat method is overwritten by user, build the X
            # matrix with the external get_xmat method
            x = self.get_xmat(xmol)
            h1 = x2c._get_hcore_fw(t, v, w, s, x, c)

        elif 'ATOM' in self.approx.upper():
            atom_slices = xmol.offset_nr_by_atom()
            nao = xmol.nao_nr()
            x = numpy.zeros((nao,nao))
            for ia in range(xmol.natm):
                ish0, ish1, p0, p1 = atom_slices[ia]
                shls_slice = (ish0, ish1, ish0, ish1)
                t1 = xmol.intor('int1e_kin', shls_slice=shls_slice)
                s1 = xmol.intor('int1e_ovlp', shls_slice=shls_slice)
                with xmol.with_rinv_at_nucleus(ia):
                    z = -xmol.atom_charge(ia)
                    v1 = z * xmol.intor('int1e_rinv', shls_slice=shls_slice)
                    w1 = z * xmol.intor('int1e_prinvp', shls_slice=shls_slice)
                x[p0:p1,p0:p1] = x2c._x2c1e_xmatrix(t1, v1, w1, s1, c)
            h1 = x2c._get_hcore_fw(t, v, w, s, x, c)

        else:
            h1 = x2c._x2c1e_get_hcore(t, v, w, s, c)

        if self.basis is not None:
            s22 = xmol.intor_symmetric('int1e_ovlp')
            s21 = gto.intor_cross('int1e_ovlp', xmol, mol)
            c = lib.cho_solve(s22, s21)
            h1 = reduce(numpy.dot, (c.T, h1, c))
        if self.xuncontract and contr_coeff is not None:
            h1 = reduce(numpy.dot, (contr_coeff.T, h1, contr_coeff))
        return h1

    def picture_change(self, even_operator=(None, None), odd_operator=None):
        '''Picture change for even_operator + odd_operator

        even_operator has two terms at diagonal blocks
        [ v  0 ]
        [ 0  w ]

        odd_operator has the term at off-diagonal blocks
        [ 0    p ]
        [ p^T  0 ]

        v, w, and p can be strings (integral name) or matrices.
        '''
        mol = self.mol
        xmol, c = self.get_xmol(mol)
        pc_mat = self._picture_change(xmol, even_operator, odd_operator)

        if self.basis is not None:
            s22 = xmol.intor_symmetric('int1e_ovlp')
            s21 = gto.mole.intor_cross('int1e_ovlp', xmol, mol)
            c = lib.cho_solve(s22, s21)

        elif self.xuncontract:
            pass

        else:
            return pc_mat

        if pc_mat.ndim == 2:
            return lib.einsum('pi,pq,qj->ij', c, pc_mat, c)
        else:
            return lib.einsum('pi,xpq,qj->xij', c, pc_mat, c)

    def get_xmat(self, mol=None):
        if mol is None:
            xmol = self.get_xmol(mol)[0]
        else:
            xmol = mol
        c = lib.param.LIGHT_SPEED
        assert('1E' in self.approx.upper())

        if 'ATOM' in self.approx.upper():
            atom_slices = xmol.offset_nr_by_atom()
            nao = xmol.nao_nr()
            x = numpy.zeros((nao,nao))
            for ia in range(xmol.natm):
                ish0, ish1, p0, p1 = atom_slices[ia]
                shls_slice = (ish0, ish1, ish0, ish1)
                t1 = xmol.intor('int1e_kin', shls_slice=shls_slice)
                s1 = xmol.intor('int1e_ovlp', shls_slice=shls_slice)
                with xmol.with_rinv_at_nucleus(ia):
                    z = -xmol.atom_charge(ia)
                    v1 = z * xmol.intor('int1e_rinv', shls_slice=shls_slice)
                    w1 = z * xmol.intor('int1e_prinvp', shls_slice=shls_slice)
                x[p0:p1,p0:p1] = x2c._x2c1e_xmatrix(t1, v1, w1, s1, c)
        else:
            t = xmol.intor_symmetric('int1e_kin')
            v = xmol.intor_symmetric('int1e_nuc')
            s = xmol.intor_symmetric('int1e_ovlp')
            w = xmol.intor_symmetric('int1e_pnucp')
            x = x2c._x2c1e_xmatrix(t, v, w, s, c)
        return x

    def _get_rmat(self, x=None):
        '''The matrix (in AO basis) that changes metric from NESC metric to NR metric'''
        xmol = self.get_xmol()[0]
        if x is None:
            x = self.get_xmat(xmol)
        c = lib.param.LIGHT_SPEED
        s = xmol.intor_symmetric('int1e_ovlp')
        t = xmol.intor_symmetric('int1e_kin')
        s1 = s + reduce(numpy.dot, (x.conj().T, t, x)) * (.5/c**2)
        return x2c._get_r(s, s1)

    def hcore_deriv_generator(self, mol=None, deriv=1):
        from pyscf.x2c import sfx2c1e_grad
        from pyscf.x2c import sfx2c1e_hess
        if deriv == 1:
            return sfx2c1e_grad.hcore_grad_generator(self, mol)
        elif deriv == 2:
            return sfx2c1e_hess.hcore_hess_generator(self, mol)
        else:
            raise NotImplementedError

SpinFreeX2C = SpinFreeX2CHelper

if __name__ == '__main__':
    mol = gto.Mole()
    mol.build(
        verbose = 0,
        atom = [["O" , (0. , 0.     , 0.)],
                [1   , (0. , -0.757 , 0.587)],
                [1   , (0. , 0.757  , 0.587)] ],
        basis = 'ccpvdz-dk',
    )

    method = hf.RHF(mol)
    enr = method.kernel()
    print('E(NR) = %.12g' % enr)

    method = sfx2c1e(hf.RHF(mol))
    esfx2c = method.kernel()
    print('E(SFX2C1E) = %.12g' % esfx2c)
    method.with_x2c.basis = 'unc-ccpvqz-dk'
    print('E(SFX2C1E) = %.12g' % method.kernel())
    method.with_x2c.approx = 'atom1e'
    print('E(SFX2C1E) = %.12g' % method.kernel())

max_stars_repo_name: umamibeef/pyscf

max_issues_repo_name: umamibeef/pyscf

max_forks_repo_name: umamibeef/pyscf


text:
"""

"""

# Built-in
try:
    from itertools import ifilterfalse
except ImportError:
    from itertools import filterfalse as ifilterfalse

# Libs
import numpy as np

# Pytorch
import torch
from torch import nn
from torch.autograd import Variable
from torch.nn import functional as F

# Own modules

class LossClass(nn.Module):
    """
    The base class of loss metrics, all loss metrics should inherit from this class
    This class contains a function that defines how loss is computed (def forward) and a loss tracker that keeps
    updating the loss within an epoch
    """
    def __init__(self):
        super(LossClass, self).__init__()
        self.loss = 0
        self.cnt = 0

    def forward(self, pred, lbl):
        raise NotImplementedError

    def update(self, loss, size):
        """
        Update the current loss tracker
        :param loss: the computed loss
        :param size: #elements in the batch
        :return:
        """
        self.loss += loss.item() * size
        self.cnt += 1

    def reset(self):
        """
        Reset the loss tracker
        :return:
        """
        self.loss = 0
        self.cnt = 0

    def get_loss(self):
        """
        Get mean loss within this epoch
        :return:
        """
        return self.loss / self.cnt

class LossMeter(LossClass):
    """
    A meter for calculated loss
    """
    def __init__(self, name_ext='', func=None):
        super(LossMeter, self).__init__()
        self.name = 'meter_' + name_ext
        self.func = func

    def forward(self, pred, lbl):
        if self.func:
            return self.func(pred, lbl)

class CrossEntropyLoss(LossClass):
    """
    Cross entropy loss function used in training
    """
    def __init__(self, class_weights=(1., 1.)):
        super(CrossEntropyLoss, self).__init__()
        self.name = 'xent'
        class_weights = torch.tensor([float(a) for a in class_weights])
        self.criterion = nn.CrossEntropyLoss(class_weights)

    def forward(self, pred, lbl):
        if len(lbl.shape) == 4 and lbl.shape[1] == 1:
            lbl = lbl[:, 0, :, :]
        return self.criterion(pred, lbl)

class PixelWeightedCrossEntropyLoss(LossClass):
    """
    Cross entropy loss function used in training
    """
    def __init__(self, class_weights=(1., 1.)):
        super(PixelWeightedCrossEntropyLoss, self).__init__()
        self.name = 'pw_xent'
        class_weights = torch.tensor([float(a) for a in class_weights])
        self.criterion = nn.CrossEntropyLoss(class_weights, reduction='none')

    def forward(self, pred, lbl, weight_map):
        if len(lbl.shape) == 4 and lbl.shape[1] == 1:
            lbl = lbl[:, 0, :, :]
        return torch.mean(self.criterion(pred, lbl) * weight_map)

class BCEWithLogitLoss(LossClass):
    """
    Cross entropy loss function used in training
    """
    def __init__(self, device, class_weights=(1., 1.)):
        super(BCEWithLogitLoss, self).__init__()
        self.name = 'bcelogits'
        class_weights = torch.tensor([float(a) for a in class_weights]).to(device)
        self.criterion = nn.BCEWithLogitsLoss(pos_weight=class_weights)

    def forward(self, pred, lbl):
        return self.criterion(pred, lbl)

class SoftIoULoss(LossClass):
    """
    Soft IoU loss that is differentiable
    This code comes from https://discuss.pytorch.org/t/one-hot-encoding-with-autograd-dice-loss/9781/5
    Paper: http://campar.in.tum.de/pub/milletari2016Vnet/milletari2016Vnet.pdf
    """
    def __init__(self, device, delta=1e-12):
        super(SoftIoULoss, self).__init__()
        self.name = 'softIoU'
        self.device = device
        self.delta = delta

    def forward(self, pred, lbl):
        num_classes = pred.shape[1]
        if num_classes == 1:
            true_1_hot = torch.eye(num_classes + 1)[lbl.squeeze(1)].to(self.device)
            true_1_hot = true_1_hot.permute(0, 3, 1, 2).float()
            true_1_hot_f = true_1_hot[:, 0:1, :, :]
            true_1_hot_s = true_1_hot[:, 1:2, :, :]
            true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1)
            pos_prob = torch.sigmoid(pred)
            neg_prob = 1 - pos_prob
            probas = torch.cat([pos_prob, neg_prob], dim=1)
        else:
            true_1_hot = torch.eye(num_classes)[lbl.squeeze(1)].to(self.device)
            true_1_hot = true_1_hot.permute(0, 3, 1, 2).float()
            probas = F.softmax(pred, dim=1)
        true_1_hot = true_1_hot.type(pred.type())
        dims = (0,) + tuple(range(2, lbl.ndimension()))
        intersection = torch.sum(probas * true_1_hot, dims)
        cardinality = torch.sum(probas + true_1_hot, dims)
        dice_loss = (2. * intersection / (cardinality + self.delta)).mean()
        return (1 - dice_loss)

class IoU(LossClass):
    """
    IoU metric that is not differentiable in training
    """
    def __init__(self, delta=1e-7):
        super(IoU, self).__init__()
        self.name = 'IoU'
        self.numerator = 0
        self.denominator = 0
        self.delta = delta

    def forward(self, pred, lbl):
        class_n = list(pred.shape)[1]
        if class_n == 2:
            truth = lbl.flatten().float()
            _, pred = torch.max(pred[:, :, :, :], 1)
            pred = pred.flatten().float()
            intersect = truth * pred
            return torch.sum(intersect == 1), torch.sum(truth + pred >= 1)
        else:
            # multi-label class, code comes from https://github.com/keras-team/keras/issues/11350
            truth = lbl.flatten().float()
            _, pred = torch.max(pred[:, :, :, :], 1)
            pred = pred.flatten().float()
            intersect, union = 0, 0
            for i in range(class_n):
                true_labels = torch.eq(truth, i)
                pred_labels = torch.eq(pred, i)
                intersect += torch.sum(true_labels * pred_labels == 1)
                union += torch.sum(true_labels + pred_labels >= 1)
            return intersect, union

    def update(self, loss, size):
        self.numerator += loss[0].item() * size
        self.denominator += loss[1].item() * size

    def reset(self):
        self.numerator = 0
        self.denominator = 0

    def get_loss(self):
        return self.numerator / (self.denominator + self.delta)

class FocalLoss(LossClass):
    """
    Focal loss: this code comes from
    https://github.com/mbsariyildiz/focal-loss.pytorch/blob/6551bd3e433ce41020b6bc8d99221eb6cd10ae17/focalloss.py#L33
    """
    def __init__(self, device, gamma=0, alpha=None, size_average=True):
        super(FocalLoss, self).__init__()
        self.name = 'focal'
        self.gamma = gamma
        self.alpha = alpha
        if isinstance(alpha, (float, int)): self.alpha = torch.Tensor([alpha, 1 - alpha]).to(device)
        if isinstance(alpha, list): self.alpha = torch.Tensor(alpha).to(device)
        self.size_average = size_average

    def forward(self, input, target):
        if input.dim() > 2:
            input = input.view(input.size(0), input.size(1), -1)  # N,C,H,W => N,C,H*W
            input = input.transpose(1, 2)                         # N,C,H*W => N,H*W,C
            input = input.contiguous().view(-1, input.size(2))    # N,H*W,C => N*H*W,C
        target = target.view(-1, 1)

        logpt = F.log_softmax(input, dim=1)
        logpt = logpt.gather(1,target)
        logpt = logpt.view(-1)
        pt = logpt.exp()

        if self.alpha is not None:
            if self.alpha.type() != input.data.type():
                self.alpha = self.alpha.type_as(input.data)
            at = self.alpha.gather(0, target.data.view(-1))
            logpt = logpt * at

        loss = -1 * (1 - pt)**self.gamma * logpt
        if self.size_average: return loss.mean()
        else: return loss.sum()

class LovaszSoftmax(LossClass):
    """
    LovaszSoftmax loss, this code comes from:
    https://github.com/bermanmaxim/LovaszSoftmax/blob/7d48792d35a04d3167de488dd00daabbccd8334b/pytorch/lovasz_losses.py
    """
    def __init__(self, classes='present', per_image=False, ignore=None):
        super(LovaszSoftmax, self).__init__()
        self.name = 'lovasz'
        self.classes = classes
        self.per_image = per_image
        self.ignore = ignore

    def forward(self, pred, lbl):
        if self.per_image:
            loss = self.mean(
                self.lovasz_softmax_flat(*self.flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), self.ignore))
                for prob, lab in zip(pred, lbl))
        else:
            loss = self.lovasz_softmax_flat(*self.flatten_probas(pred, lbl, self.ignore))
        return loss

    def lovasz_softmax_flat(self, probas, labels):
        """
        Multi-class Lovasz-Softmax loss
          probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1)
          labels: [P] Tensor, ground truth labels (between 0 and C - 1)
          classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.
        """
        if probas.numel() == 0:
            # only void pixels, the gradients should be 0
            return probas * 0.
        C = probas.size(1)
        losses = []
        class_to_sum = list(range(C)) if self.classes in ['all', 'present'] else self.classes
        for c in class_to_sum:
            fg = (labels == c).float()  # foreground for class c
            if (self.classes is 'present' and fg.sum() == 0):
                continue
            if C == 1:
                if len(self.classes) > 1:
                    raise ValueError('Sigmoid output possible only with 1 class')
                class_pred = probas[:, 0]
            else:
                class_pred = probas[:, c]
            errors = (Variable(fg) - class_pred).abs()
            errors_sorted, perm = torch.sort(errors, 0, descending=True)
            perm = perm.data
            fg_sorted = fg[perm]
            losses.append(torch.dot(errors_sorted, Variable(self.lovasz_grad(fg_sorted))))
        return self.mean(losses)

    @staticmethod
    def mean(l, ignore_nan=False, empty=0):
        """
        nan mean compatible with generators.
        """
        def isnan(x):
            return x != x
        l = iter(l)
        if ignore_nan:
            l = ifilterfalse(isnan, l)
        try:
            n = 1
            acc = next(l)
        except StopIteration:
            if empty == 'raise':
                raise ValueError('Empty mean')
            return empty
        for n, v in enumerate(l, 2):
            acc += v
        if n == 1:
            return acc
        return acc / n

    @staticmethod
    def flatten_probas(probas, labels, ignore=None):
        """
        Flattens predictions in the batch
        """
        if probas.dim() == 3:
            # assumes output of a sigmoid layer
            B, H, W = probas.size()
            probas = probas.view(B, 1, H, W)
        B, C, H, W = probas.size()
        probas = probas.permute(0, 2, 3, 1).contiguous().view(-1, C)  # B * H * W, C = P, C
        labels = labels.view(-1)
        if ignore is None:
            return probas, labels
        valid = (labels != ignore)
        vprobas = probas[valid.nonzero().squeeze()]
        vlabels = labels[valid]
        return vprobas, vlabels

    @staticmethod
    def lovasz_grad(gt_sorted):
        """
        Computes gradient of the Lovasz extension w.r.t sorted errors
        See Alg. 1 in paper
        """
        p = len(gt_sorted)
        gts = gt_sorted.sum()
        intersection = gts - gt_sorted.float().cumsum(0)
        union = gts + (1 - gt_sorted).float().cumsum(0)
        jaccard = 1. - intersection / union
        if p > 1:  # cover 1-pixel case
            jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]
        return jaccard

def iou_metric(truth, pred, divide=False, eval_class=(1,)):
    """
    Compute IoU, i.e., jaccard index
    :param truth: truth data matrix, should be H*W
    :param pred: prediction data matrix, should be the same dimension as the truth data matrix
    :param divide: if True, will return the IoU, otherwise return the numerator and denominator
    :param eval_class: the label class to be evaluated
    :return:
    """
    truth = truth.flatten()
    pred = pred.flatten()
    iou_score = np.zeros((2, len(eval_class)), dtype=float)
    for c_cnt, curr_class in enumerate(eval_class):
        iou_score[0, c_cnt] += np.sum(((truth == curr_class) * (pred == curr_class)) == 1)
        iou_score[1, c_cnt] += np.sum(((truth == curr_class) + (pred == curr_class)) >= 1)
    if not divide:
        return iou_score
    else:
        return np.mean(iou_score[0, :] / iou_score[1, :])

if __name__ == '__main__':
    pass

max_stars_repo_name: energydatalab/mrs


text:
import unittest
from dpipe import *
import numpy as np
from random import choice
from string import ascii_letters, digits
import tensorflow as tf
import time
from dpipe.factories import from_function

class Counter(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = [1,2,3,4]
    def __len__(self):
        return 4
    def read(self,index):
        print('index at counter', index)
        return self.values[index]

class ListNumerics(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = [[1,2,3,4],[5,6,7,8],[9,10,11,12]]
    def __len__(self):
        return len(self.values)
    def read(self,index):
        print('index at counter', index, 'type', type(index))
        return tuple(self.values[index])

def make_random_string(lenght=None):
    if lenght is None:
        lenght = np.random.randint(15)
    return ''.join([choice(ascii_letters+digits) for _ in range(lenght)])
class Strings(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = [make_random_string() for _ in range(10)]
    def __len__(self):
        return 10
    def read(self,index):
        print('index at counter', index)
        return self.values[index]
class ListOfStrings(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = [[make_random_string() for _ in range(3)] for _ in range(10)]
    def __len__(self):
        return 10
    def read(self,index):
        print('index at counter', index)
        return tuple(self.values[index])
class ImageWithLabel(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = 10*[(np.random.rand(3,10,10),np.random.randint(9))]
    def __len__(self):
        return 10
    def read(self,index):
        print('index at counter', index)
        return self.values[index]
class DoubleImage(object):
    """ Dummy counter class
    """
    def __init__(self,length=10):
        self.values = length*[(np.random.rand(3,10,10),np.random.rand(3,10,10))]
        self.length = length
    def __len__(self):
        return self.length
    def read(self,index):
        print('index at counter', index)
        return self.values[index]
class DoubleImageList(object):
    """ Dummy counter class
    """
    def __init__(self):
        self.values = 10*[[np.random.rand(3,10,10),np.random.rand(3,10,10)]]
    def __len__(self):
        return 10
    def read(self,index):
        print('index at counter', index)
        return self.values[index]

class GeneratorSingleListOfNumbers(object):
    """
    Dummy counter class with generator
    """
    def __init__(self):
        self.values = list(range(10))
    def __len__(self):
        return 10
    def read(self,index):
        # returns a counter generator over the list from 0 to the given index.
        i = 0
        index = 9 if index>10 else index
        while i <= index:
            yield self.values[i]
            i += 1

def get_all_methods(object):
    return [method_name for method_name in dir(object) if callable(getattr(object, method_name))]

class TestFactoriesMethods(unittest.TestCase):

    def test_from_object_one_output(self):
        c = Counter()
        dataset_builder = from_object(c,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)

    def test_from_object_two_outputs(self):
        imgs = ImageWithLabel()
        dataset_builder = from_object(imgs,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        imgs = DoubleImage()
        dataset_builder = from_object(imgs,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
    def test_from_object_two_outputs_list(self):

        imgs = DoubleImageList()
        dataset_builder = from_object(imgs,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)
    def test_from_object_string_outputs(self):
        strs = Strings()
        dataset_builder = from_object(strs,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)
    def test_from_object_list_string_outputs(self):
        strs = ListOfStrings()
        dataset_builder = from_object(strs,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)
    def test_from_object_list_numerics_outputs(self):
        numerics = ListNumerics()
        dataset_builder = from_object(numerics,'read','values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec )
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)
    def test_from_object_with_generator(self):
        genSingleList = GeneratorSingleListOfNumbers()
        dataset_builder = from_object(genSingleList, getitem_fcn='read',itemlist_name='values')
        dataset = dataset_builder.build()
        # print(list(dataset))
        print("element spec: ", dataset.element_spec)
        for element in dataset:
            print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)

class ImageFlatWithLabel(object):
    """ Dummy counter class
    """
    def __init__(self,length=10):
        # image 10x10 and a label
        self.list = length*[(np.random.rand(100),np.random.randint(9))]
        self.length = length
    def __len__(self):
        return self.length
    def read(self,index):
        print('index at counter', index)
        return self.list[index]

def make_model():
    inputs = tf.keras.Input(shape=(100,))
    dense = tf.keras.layers.Dense(64, activation='relu')
    x = dense(inputs)
    x = tf.keras.layers.Dense(64, activation='relu')(x)
    outputs = tf.keras.layers.Dense(10)(x)
    model = tf.keras.Model(inputs=inputs, outputs=outputs)
    return model

class TestFactoryDatasetBuilder(unittest.TestCase):
    def test_build(self):
        c = DoubleImage(length=100)
        dataset_builder = from_object(c,'read')
        dataset = dataset_builder.batch(20).cache().build()
        print("Build arguments: ", dataset.built_args)
        print("element spec: ", dataset.element_spec )
        # for element in dataset:
        #     print('element of the dataset: ', element)
            # print('methods: ', get_all_methods(element))
        self.assertEqual(0, 0)
    def test_build(self):
        LENGTH = 50
        c = ImageFlatWithLabel(length=LENGTH)
        dataset_builder = from_object(c,'read')
        dataset = dataset_builder.build()
        self.assertEqual(len(list(dataset.as_numpy_iterator())) ,LENGTH)
    def test_batching(self):
        BATCH = 10
        LENGTH = 50
        c = ImageFlatWithLabel(length=LENGTH)
        dataset_builder = from_object(c, 'read')
        dataset = dataset_builder.batch(BATCH).build()
        self.assertEqual(len(list(dataset.as_numpy_iterator())), LENGTH//BATCH)
    def test_shuffle(self):
        c = Counter()
        dataset_builder = from_object(c,'read','values')
        dataset = dataset_builder.shuffle(len(c), reshuffle_each_iteration=True).build()
        list1 = list(dataset.as_numpy_iterator())
        list2 = list(dataset.as_numpy_iterator())
        self.assertFalse(list1==list2)
    def test_training(self):
        EPOCHS = 10
        LENGTH = 50
        c = ImageFlatWithLabel(length=LENGTH)
        dataset_builder = from_object(c,'read')
        dataset = dataset_builder.shuffle(LENGTH, reshuffle_each_iteration=True).batch(2).repeat().build()
        model = make_model()
        model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.RMSprop())
        print("Build arguments: ", dataset.built_args)
        model.fit(x=dataset, epochs=EPOCHS,**dataset.built_args)

TEXT_FILE = 'text_dataset/text_emotion.csv'

def csv_to_lists(csv_file):
    with open(csv_file) as f:
        line = f.readline()
        # it skips header
        line = f.readline()
        emotions, messages = [], []
        while line:
            tokens = line.strip().split(',')
            emotions.append(tokens[1])
            messages.append(' '.join(tokens[3:]).replace('"',''))
            line = f.readline()
    return messages, emotions

def compute_dictionary(tokens):
    codes_dict = {c: i for i, c in enumerate(set(tokens))}
    return codes_dict

def text_encode(message, codes_dict):
    return np.array([codes_dict[word] for word in message.split(' ')], dtype=int)

class TestFactoryFunctionList(unittest.TestCase):
    def test_read_idem(self):
        # extract messages and emotions from csv file
        messages, emotions = csv_to_lists(TEXT_FILE)
        dataset_builder = from_function(lambda x: x, list(zip(messages,emotions)))
        dataset = dataset_builder.build()
        self.assertEqual(len(list(dataset.as_numpy_iterator())), 40000)
        self.assertEqual(str(dataset.element_spec),
                         '(TensorSpec(shape=(), dtype=tf.string, name=None), TensorSpec(shape=(), dtype=tf.string, '
                         'name=None))')
    def test_read_function_float_gen(self):
        def counter_float(v):
            return 1.0

        def counter_float_ndarray(v):
            return np.array(1.0)

        dataset = from_function(counter_float, [1,2,3,4]).build()
        print(list(dataset.as_numpy_iterator()))
        dataset = from_function(counter_float_ndarray, [1, 2, 3, 4]).build()
        print(list(dataset.as_numpy_iterator()))

    def test_read_text_file(self):
        def read_fcn(filename):
            messages, emotions = csv_to_lists(filename)
            # find message unique dictionary encoding
            message_dict = compute_dictionary(' '.join(messages).split(' '))
            emotions_dict = compute_dictionary(' '.join(emotions).split(' '))
            encoding_x_fnc = lambda x: text_encode(x, message_dict)
            encoding_y_fnc = lambda x: text_encode(x, emotions_dict)
            messages = list(map(encoding_x_fnc,messages))
            emotions = list(map(encoding_y_fnc, emotions))
            for m,e in zip(messages, emotions):
                yield m,e
        N = 2
        filenames = N*[TEXT_FILE]
        # comparing the tf normal  and the implementation of file concurrent reading with dpipe function way
        dataset_builder = from_function(read_fcn, filenames,
                                        undetermined_shape=([0], []) # messages are shape [1] with undetermined shape at the dim 0
                                                                     # the second correspond to the emotions which is an scalar text
                                        )
        t0 = time.time()
        dataset = dataset_builder.build()
        self.assertEqual(len(list(dataset.as_numpy_iterator())), N*40000)
        t1 = time.time()-t0
        print('dadspipe implementation took: ', t1, 'seconds')

    def test_training(self):
        EPOCHS = 10
        LENGTH = 50
        c = ImageFlatWithLabel(length=LENGTH)
        dataset_builder = from_object(c,'read')
        dataset = dataset_builder.shuffle(LENGTH, reshuffle_each_iteration=True).batch(2).repeat().build()
        model = make_model()
        model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.RMSprop())
        print("Build arguments: ", dataset.built_args)
        model.fit(x=dataset, epochs=EPOCHS,**dataset.built_args)

    def test_parallel_training(self):
        EPOCHS = 10
        LENGTH = 50
        c = ImageFlatWithLabel(length=LENGTH)
        dataset_builder = from_object(c,'read')
        dataset = dataset_builder.shuffle(LENGTH, reshuffle_each_iteration=True).batch(2).repeat().\
            parallelize_extraction(num_parallel_calls=2).build()
        model = make_model()
        model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.RMSprop())
        print("Build arguments: ", dataset.built_args)
        model.fit(x=dataset, epochs=EPOCHS,**dataset.built_args)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: aiporre/multidataloader

max_issues_repo_name: aiporre/multidataloader

max_forks_repo_name: aiporre/multidataloader


text:
from setuptools import setup
from Cython.Build import cythonize

import numpy as np

setup(
    include_dirs=[np.get_include()],
    ext_modules = cythonize('*.pyx'),
)