text:
#!/usr/bin/env python3

from __future__ import division, print_function
import string, time, random, msvcrt
import sounddevice as sd
import numpy as np
from scipy import io
import scipy.io.wavfile

import morse

SPS = 8000
LETTERS = string.ascii_uppercase
FREQ = 750
WPM = 25
FS = 10
AUDIO_PADDING = 0.5  # Seconds
CLICK_SMOOTH = 2  # Tone periods

class Person:
  def __init__(self, sign, name, location, rig, pwr, ant):
    self.sign = sign
    self.name = name
    self.location = location
    self.rig = rig
    self.pwr = pwr
    self.ant = ant
    self.rst = str(random.randrange(1,5)) + str(random.randrange(1,9)) + str(random.randrange(1,9))

def main(freq, wpm, fs, force, outFile, inFile):

  call_signs = []

  # Read from file in file provided
  #if len(inFile) > 0:
  #  file_handle = open(inFile, 'r')
  #  lines = file_handle.readlines()
  #  for line in lines:
  #    call_signs.append(line[:-1])
  #else:
  #  call_signs = ['KD2WAI', 'KO4HMB', 'KD2HCU', 'N1PRR', 'W8OV']
  operator = []

  operator.append(Person('KD2WAI', 'MATT', 'SCHNECTADY, NY',      'IC7300',      '100W', 'EFHW'))
  operator.append(Person('KO4HMB', 'DAVE', 'DANVILLE, VA',        'FT991A',      '5W',   'HAMSTICK'))
  operator.append(Person('KD2HCU', 'ERIC', 'YAPSHANK, NY',        'YEASU 817',   '50W',  'WIRE'))
  operator.append(Person('N1PRR',  'GARY', 'LITCHFIELD PARK, AZ', 'IC705',       '50W',  'SOTA DIPOLE'))
  operator.append(Person('W8OV',   'DAVE', 'COLLIN, TX',          'ELECRAFT K3', '100W', 'W RADIALS'))

  sps = SPS

  print('Audio samples per second =', sps)
  print('Tone period     =', round(1000/freq, 1), 'ms')

  dps = morse.wpmToDps(wpm)  # Dots per second
  mspd = 1000/dps  # Dot duration in milliseconds
  farnsworthScale = morse.farnsworthScaleFactor(wpm, fs)

  print()
  print('Dot width       =', round(mspd, 1), 'ms')
  print('Dash width      =', int(round(mspd * morse.DASH_WIDTH)), 'ms')
  print('Character space =', int(round(mspd * morse.CHAR_SPACE * farnsworthScale)), 'ms')
  print('Word space      =', int(round(mspd * morse.WORD_SPACE * farnsworthScale)), 'ms')

  print()
  print("Hit <ENTER> to start.")
  input()

  caller, callee = random.choices(operator, k=2)

  # CALLER

  # Is this frequency in use...
  playAndCheckMessage('QRL? QRL?', sps, wpm, fs, freq)
  playAndCheckMessage(random.choice(['QRL', 'A', 'L', 'Y', 'YES', '']), sps, wpm, fs, freq)

  # Request contact
  playAndCheckMessage('CQ CQ CQ DE ' + caller.sign + ' ' +caller.sign + ' K', sps, wpm, fs, freq)

  # CALLEE

  # Respond to CQ
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign + ' ' + callee.sign + ' K', sps, wpm, fs, freq)

  # CALLER

  # Response confirmation
  playAndCheckMessage(callee.sign + ' DE ' + caller.sign + ' BT', sps, wpm, fs, freq)

  # Request slower speed
  playAndCheckMessage(random.choice(['QRS', 'QRS PLS', 'QRS PSE', '']), sps, wpm, fs, freq)

  # Send thanks for the call
  playAndCheckMessage(random.choice(['GM ', 'GA ', 'GE ', '']) + 'THX ' + random.choice(['FOR ', 'FER ', '']) + 'CALL BT', sps, wpm, fs, freq)

  # Signal report
  playAndCheckMessage('UR RST IS '+ callee.rst + ' ' + callee.rst, sps, wpm, fs, freq)

  # Location
  playAndCheckMessage(random.choice(['IN ', 'QTH ']) + caller.location + ' ' + caller.location, sps, wpm, fs, freq)

  # Name
  playAndCheckMessage('NAME IS ' + caller.name + ' ' + caller.name, sps, wpm, fs, freq)

  # Back to callee
  playAndCheckMessage(random.choice(['BTU', 'HW CPY?']), sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(callee.sign + ' DE ' + caller.sign + ' K', sps, wpm, fs, freq)

  # CALLEE

  # Confirmation info
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign +  ' K', sps, wpm, fs, freq)

  # Signal report
  playAndCheckMessage('THX UR RST IS '+ caller.rst + ' ' + caller.rst, sps, wpm, fs, freq)

  # Location
  playAndCheckMessage(random.choice(['IN ', 'QTH ']) + callee.location + ' ' + callee.location, sps, wpm, fs, freq)

  # Name
  playAndCheckMessage('NAME IS ' + callee.name + ' ' + callee.name, sps, wpm, fs, freq)

  # Back to caller
  playAndCheckMessage(random.choice(['BTU', 'HW CPY?']), sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign + ' K', sps, wpm, fs, freq)

  # CALLER

  # Confirmation info
  playAndCheckMessage(callee.sign + ' DE ' + caller.sign, sps, wpm, fs, freq)

  # Send thanks for info
  playAndCheckMessage(random.choice(['THX ', 'THANKS ']) + random.choice(['FOR ', 'FER ', '']) + 'INFO BT',  sps, wpm, fs, freq)

  # Rig info
  playAndCheckMessage('RIG ' + random.choice(['HERE ', 'HR ', ''])  + 'IS ' + caller.rig + ' AT ' + caller.pwr + ' BT', sps, wpm, fs, freq)

  # Antenna
  playAndCheckMessage('ANT IS ' + caller.ant + ' BT', sps, wpm, fs, freq)

  # Back to callee
  playAndCheckMessage(random.choice(['BTU', 'HW CPY?']), sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(callee.sign + ' DE ' + caller.sign + ' K', sps, wpm, fs, freq)

  # CALLEE

  # Confirmation info
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign, sps, wpm, fs, freq)

  # Send thanks for info
  playAndCheckMessage(random.choice(['THX ', 'THANKS ']) + random.choice(['FOR ', 'FER ', '']) + 'INFO BT',  sps, wpm, fs, freq)

  # Rig info
  playAndCheckMessage('RIG ' + random.choice(['HERE ', 'HR ', ''])  + 'IS ' + callee.rig + ' AT ' + callee.pwr + ' BT', sps, wpm, fs, freq)

  # Antenna
  playAndCheckMessage('ANT IS ' + callee.ant + ' BT', sps, wpm, fs, freq)

  # Back to caller
  playAndCheckMessage(random.choice(['BTU', 'HW CPY?']), sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign + ' K', sps, wpm, fs, freq)

  # CALLER

  # Send thanks for info
  playAndCheckMessage(random.choice(['THX ', 'THANKS ']) + random.choice(['FOR ', 'FER ', '']) + 'QSO ' + callee.name + ' ES 73',  sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(callee.sign + ' DE ' + caller.sign + ' SK', sps, wpm, fs, freq)

  # CALLEE

  # Send thanks for info
  playAndCheckMessage(random.choice(['THX ', 'THANKS ']) + random.choice(['FOR ', 'FER ', '']) + 'QSO ' + caller.name + ' ES 73',  sps, wpm, fs, freq)

  # Confirmation info
  playAndCheckMessage(caller.sign + ' DE ' + callee.sign + ' SK', sps, wpm, fs, freq)

def playAndCheckMessage(message, sps, wpm, fs, freq):

  if message == '':
    return

  playMessage(message, sps, wpm, fs, freq)

  print('Enter message:')
  start = time.time()
  check = input()

  if check.upper() == message.upper():
    end = time.time()
    print('Correct! [', '{:.2f}'.format(end - start), 's]')
  else:
    print('Wrong. The correct answer is ', message)

  print("Hit <ENTER> to continue.")
  input()

def playMessage(message, sps, wpm, fs, freq):

  audio = stringToMorseAudio(message, sps, wpm, fs, freq, 0.5, None, promptVolume=0.3)
  audio /= 2

  playBlock(audio, sps)
  time.sleep(0.1)

def addAudio(base, new, offset):
  if base is None:
    base = np.array([], dtype=np.float32)
  assert offset >= 0
  lenBase, lenNew = len(base), len(new)
  if offset+lenNew > lenBase:
    # Make base longer by padding with zeros
    base = np.append(base, np.zeros(offset+lenNew-lenBase))
  base[offset:offset+lenNew] += new
  return base

def boolArrToSwitchedTone(boolArr, freq, sps, volume=1.0):
  ''' Create the tone audio from a bool array representation of morse code. '''
  weightLen = int(CLICK_SMOOTH*sps/freq)
  if weightLen % 2 == 0:
    weightLen += 1  # Make sure the weight array is odd length
  smoothingWeights = np.concatenate((np.arange(1, weightLen//2+1), np.arange(weightLen//2+1, 0, -1)))
  smoothingWeights = smoothingWeights / np.sum(smoothingWeights)
  numSamplesPadding = int(sps*AUDIO_PADDING) + int((weightLen-1)/2)
  padding = np.zeros(numSamplesPadding, dtype=bool)
  boolArr = np.concatenate((padding, boolArr, padding)).astype(np.float32)
  if CLICK_SMOOTH <= 0:
    smoothBoolArr = boolArr
  else:
    smoothBoolArr = np.correlate(boolArr, smoothingWeights, 'valid')
  numSamples = len(smoothBoolArr)
  x = np.arange(numSamples)
  toneArr = np.sin(x * (freq*2*np.pi/sps)) * volume
  toneArr *= smoothBoolArr
  return toneArr

def stringToMorseAudio(message, sps=SPS, wpm=WPM, fs=FS, freq=FREQ, volume=1.0, letterPrompts=None, promptVolume=1.0):
  message = message.upper()
  code = morse.stringToMorse(message)
  boolArr = morse.morseToBoolArr(code, sps, wpm, fs)
  audio = boolArrToSwitchedTone(boolArr, freq, sps, volume)
  numSamplesPadding = int(sps*AUDIO_PADDING)
  if letterPrompts is not None:
    for i in range(len(message)):
      l = message[i]
      if l in letterPrompts:
        offsetPlus = morse.morseSampleDuration(morse.stringToMorse(message[:i+1]), sps, wpm, fs)
        letterDuration = morse.morseSampleDuration(morse.letterToMorse(message[i]), sps, wpm, fs)
        offset = numSamplesPadding + offsetPlus - letterDuration
        audio = addAudio(audio, letterPrompts[l][1]*promptVolume, offset)
  return audio

def loadLetterNames(pathTemplate='audio/letter-names/%s_.wav', letters=LETTERS):
  out = {}
  for letter in letters:
    fName = pathTemplate % letter
    out[letter] = loadWav(fName)
  return out
def loadWav(fName):
  rate, data = io.wavfile.getHistory(fName, mmap=True)
  dataScale = data.astype(np.float32) / maxDtypeVolume(data.dtype)
  return rate, dataScale
def maxDtypeVolume(dtype):
  try:
    return np.iinfo(dtype).max  # Integer data type
  except ValueError:
    return 1.0  # Float data type
def playLetterNamesBlock(letterNames):
  sps = letterNames[LETTERS[0]][0]
  letterNameList = [letterNames[l][1] for l in LETTERS]
  alphabetAudio = np.concatenate(letterNameList)
  playBlock(alphabetAudio, sps)

def genTone(frequency, duration, sps=SPS, volume=1.0):
  return np.sin(np.arange(sps*duration)*(frequency*2*np.pi/sps))*volume
def playTone(*args, **kwargs):
  play(genTone(*args, **kwargs))
def play(array, sps=SPS):
  sd.play(array.astype(np.float32), sps)
def waitFor(array, sps=SPS):
  duration = len(array) / sps
  time.sleep(duration)
def playBlock(array, sps=SPS):
  play(array, sps)
  waitFor(array, sps)

if __name__ == '__main__':
  import sys, argparse

  parser = argparse.ArgumentParser(description='Convert text to morse code audio.')
  parser.add_argument('-f', type=float, default=FREQ, help='Tone frequency')
  parser.add_argument('--wpm', type=float, default=WPM, help='Words per minute')
  parser.add_argument('--fs', type=float, default=FS, help='Farnsworth speed')
  parser.add_argument('--force', action='store_true', default=False, help='Force user to get the answer correct before completing')
  parser.add_argument('-o', type=str, default='', help='Output to given WAV file instead of playing sound')
  parser.add_argument('-i', type=str, default='', help='Input from text file')
  args = parser.parse_args()

  main(args.f, args.wpm, args.fs, args.force, args.o, args.i)

max_stars_repo_name: matthew-a-adams/morse-practice-scripts

max_issues_repo_name: matthew-a-adams/morse-practice-scripts

max_forks_repo_name: matthew-a-adams/morse-practice-scripts

reason: import numpy,import scipy,from scipy


text:
# ===========================================
#
# mian Analysis Data Mining/ML Library
# @author: tbj128
#
# ===========================================
import pandas as pd
from scipy import stats, math
import numpy as np
from sklearn.model_selection import train_test_split
import random

from mian.analysis.alpha_diversity import AlphaDiversity
from mian.analysis.analysis_base import AnalysisBase
from mian.core.statistics import Statistics
from mian.model.otu_table import OTUTable
from mian.model.genes import Genes
from mian.model.metadata import Metadata
from mian.model.user_request import UserRequest

class CorrelationsSelection(AnalysisBase):

    def run(self, user_request):
        table = OTUTable(user_request.user_id, user_request.pid)
        table.load_phylogenetic_tree_if_exists()
        metadata = table.get_sample_metadata()
        otu_metadata = table.get_otu_metadata()
        otu_table, headers, sample_labels = table.get_table_after_filtering_and_aggregation_and_low_count_exclusion(user_request)
        phylogenetic_tree = table.get_phylogenetic_tree()

        self.table = table

        return self.analyse(user_request, otu_table, headers, sample_labels, metadata, otu_metadata, phylogenetic_tree)

    def analyse(self, user_request, otu_table, headers, sample_labels, metadata, otu_metadata, phylogenetic_tree):
        select = user_request.get_custom_attr("select")
        against = user_request.get_custom_attr("against")
        expvar = user_request.get_custom_attr("expvar")
        pvalthreshold = user_request.get_custom_attr("pvalthreshold")

        seed = int(user_request.get_custom_attr("seed")) if user_request.get_custom_attr("seed") is not "" else random.randint(0, 100000)
        training_proportion = float(user_request.get_custom_attr("trainingProportion"))

        if training_proportion < 1:
            otu_table, _, sample_labels, _ = train_test_split(otu_table, sample_labels, train_size=training_proportion, random_state=seed)

        if expvar == "":
            return {"correlations": []}

        against_vals = []
        if against == "gene":
            genes = Genes(user_request.user_id, user_request.pid)
            gene_list = expvar.split(",")
            against_vals = genes.get_multi_gene_values(gene_list, sample_labels=sample_labels)
        elif against == "metadata":
            metadata_values = metadata.get_metadata_column_table_order(sample_labels, expvar)
            against_vals = list(map(float, metadata_values))
        elif against == "alpha":
            expvar_vals = expvar.split(",")
            alpha = AlphaDiversity()

            # We should only calculated the alpha diversity on an unfiltered OTU table
            unfiltered_user_request = UserRequest(user_request.user_id, user_request.pid, taxonomy_filter_count=0,
                                                  taxonomy_filter_prevalence=0, taxonomy_filter = -2,
                                                  taxonomy_filter_role="Include", taxonomy_filter_vals=[], sample_filter=user_request.sample_filter,
                                                  sample_filter_role=user_request.sample_filter_role,
                                                  sample_filter_vals=user_request.sample_filter_vals, level=user_request.level, catvar="")

            unfiltered_otu_table, unfiltered_headers, unfiltered_sample_labels = self.table.get_table_after_filtering_and_aggregation_and_low_count_exclusion(unfiltered_user_request)

            unfiltered_otu_table, _, unfiltered_sample_labels, _ = train_test_split(unfiltered_otu_table, unfiltered_sample_labels, train_size=training_proportion, random_state=seed)

            if int(user_request.level) == -1:
                # OTU tables are returned as a CSR matrix
                unfiltered_otu_table = pd.DataFrame.sparse.from_spmatrix(unfiltered_otu_table, columns=unfiltered_headers,
                                                              index=range(unfiltered_otu_table.shape[0]))

            against_vals = alpha.calculate_alpha_diversity(unfiltered_otu_table, unfiltered_sample_labels, unfiltered_headers, phylogenetic_tree, expvar_vals[1], expvar_vals[0])
        else:
            expvar_arr = expvar.split(",")
            relevant_cols = []
            c = 0
            for header in headers:
                if header in expvar_arr:
                    relevant_cols.append(c)
                c += 1
            r = 0
            while r < otu_table.shape[0]:
                total_row_sum = 0
                for c in relevant_cols:
                    total_row_sum += otu_table[r, c]
                against_vals.append(total_row_sum)
                r += 1

        taxonomy_map = otu_metadata.get_taxonomy_map()

        if select == "gene":
            abunObj = self.analyse_select_gene(user_request, sample_labels, against_vals, pvalthreshold)
        elif select == "metadata":
            abunObj = self.analyse_select_metadata(user_request, sample_labels, against_vals, pvalthreshold)
        else:
            abunObj = self.analyse_select_otu(user_request, otu_table, headers, against_vals, taxonomy_map, pvalthreshold)

        abunObj["seed"] = seed
        return abunObj

    def analyse_select_gene(self, user_request, sample_labels, against_vals, pvalthreshold):
        corrMethod = user_request.get_custom_attr("corrMethod")

        genes = Genes(user_request.user_id, user_request.pid)
        gene_table, gene_list, gene_sample_labels = genes.get_as_table()

        sample_labels_map = {}
        for i, sample_label in enumerate(gene_sample_labels):
            sample_labels_map[sample_label] = i

        label_index_ordered = []
        for label in sample_labels:
            if label in gene_sample_labels:
                label_index_ordered.append(gene_sample_labels[label])
            else:
                label_index_ordered.append(-1)

        correlations = []
        pvals = []

        c = 0
        for gene in gene_list:
            gene_vals = []
            for r in label_index_ordered:
                if r == -1:
                    gene_vals.append(0)
                else:
                    gene_vals.append(float(gene_table[r, c]))
                r += 1

            if corrMethod == "pearson":
                coef, pval = stats.pearsonr(against_vals, gene_vals)
            elif corrMethod == "spearman":
                coef, pval = stats.spearmanr(against_vals, gene_vals)
            else:
                raise NotImplementedError("Corr method is not implemented")

            if math.isnan(coef):
                coef = 0
            if math.isnan(pval):
                pval = 1
            correlations.append({"otu": gene, "coef": coef, "pval": pval})
            if len(correlations) > 100:
                break
            pvals.append(pval)
            c += 1

        qvals = Statistics.getFDRCorrection(pvals)

        correlations_retval = []
        i = 0
        while i < len(qvals):
            if correlations[i]["pval"] <= float(pvalthreshold):
                correlations[i]["qval"] = qvals[i]
                correlations_retval.append(correlations[i])
            i += 1

        return {"correlations": correlations_retval}

    def analyse_select_metadata(self, user_request, sample_labels, against_vals, pvalthreshold):
        corrMethod = user_request.get_custom_attr("corrMethod")
        metadata = Metadata(user_request.user_id, user_request.pid)
        metadata_table, metadata_headers, _ = metadata.get_as_table_in_table_order(sample_labels)

        correlations = []
        pvals = []

        c = 0
        while c < len(metadata_headers):
            metadata = metadata_headers[c]
            metadata_vals = []
            is_numeric = True
            r = 0
            while r < len(metadata_table):
                if not metadata_table[r][c].isnumeric():
                    is_numeric = False
                    break
                metadata_vals.append(float(metadata_table[r][c]))
                r += 1

            if is_numeric:
                if corrMethod == "pearson":
                    coef, pval = stats.pearsonr(against_vals, metadata_vals)
                elif corrMethod == "spearman":
                    coef, pval = stats.spearmanr(against_vals, metadata_vals)
                else:
                    raise NotImplementedError("Corr method is not implemented")
                if math.isnan(coef):
                    coef = 0
                if math.isnan(pval):
                    pval = 1
                correlations.append({"otu": metadata, "coef": coef, "pval": pval})
                pvals.append(pval)
            c += 1

        qvals = Statistics.getFDRCorrection(pvals)

        correlations_retval = []
        i = 0
        while i < len(qvals):
            if correlations[i]["pval"] <= float(pvalthreshold):
                correlations[i]["qval"] = qvals[i]
                correlations_retval.append(correlations[i])
            i += 1

        return {"correlations": correlations_retval}

    def analyse_select_otu(self, user_request, base, headers, metadata_values, taxonomy_map, pvalthreshold):
        corrMethod = user_request.get_custom_attr("corrMethod")
        otu_to_genus = {}
        if int(user_request.level) == -1:
            # We want to display a short hint for the OTU using the genus (column 5)
            for header in headers:
                if header in taxonomy_map and len(taxonomy_map[header]) > 5:
                    otu_to_genus[header] = taxonomy_map[header][5]
                else:
                    otu_to_genus[header] = ""

        correlations = []
        pvals = []

        c = 0
        while c < base.shape[1]:
            otu_name = headers[c]
            otu_vals = []
            r = 0
            while r < base.shape[0]:
                otu_vals.append(base[r, c])
                r += 1

            if corrMethod == "pearson":
                coef, pval = stats.pearsonr(metadata_values, otu_vals)
            elif corrMethod == "spearman":
                coef, pval = stats.spearmanr(metadata_values, otu_vals)
            else:
                raise NotImplementedError("Corr method is not implemented")

            if math.isnan(coef):
                coef = 0
            if math.isnan(pval):
                pval = 1

            if int(user_request.level) == -1 and otu_name in otu_to_genus:
                correlations.append({"otu": otu_name, "coef": coef, "pval": pval, "hint": otu_to_genus[otu_name]})
            else:
                correlations.append({"otu": otu_name, "coef": coef, "pval": pval})
            pvals.append(pval)

            c += 1

        qvals = Statistics.getFDRCorrection(pvals)

        correlations_retval = []
        i = 0
        while i < len(qvals):
            if correlations[i]["pval"] <= float(pvalthreshold):
                correlations[i]["qval"] = qvals[i]
                correlations_retval.append(correlations[i])
            i += 1

        return {"correlations": correlations_retval}

ext: py

reason: import numpy,from scipy


text:
import torch
import numpy

def generate_test_input(
    model_format:str,
    model_name:str,
    time_dim: int,
    hidden_size:int=512,
    half_precision:bool=False):
    """
    outputs a test input based on the model format ("pytorch" or "onnx") and the model name
    Arguments
        time_dim: time_dimension into the model
        hidden_size (int): size of RNN/LSTM cell
    """
    batch_size = 1
    layer_count = 5

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    dtype = torch.float16 if half_precision else torch.float32

    if model_format == "pytorch":
        if model_name == "super_resolution":
            return torch.randn(batch_size, 1, 224, 224, requires_grad=True, device=device)
        elif model_name == "resnet18" or model_name == "alexnet":
            return torch.randn(batch_size, 3, 224,224, requires_grad=True, device=device)
        elif model_name == "lstm":
            return (torch.randn(5, 3, 10, device=device),
                    torch.randn(layer_count * 2, 3, 20, device=device),
                    torch.randn(layer_count * 2, 3, 20, device=device)
                    )
        else:
            return (torch.randn(1,time_dim, 257, device=device).type(dtype),
                    (torch.randn(layer_count * 1, 1, hidden_size, device=device).type(dtype),
                    torch.randn(layer_count * 1, 1, hidden_size, device=device).type(dtype))
                    )
    else:
        raise ValueError("model_format parameters must be 'pytorch' or 'onnx'")

max_stars_repo_name: dzubke/speech-lite

max_issues_repo_name: dzubke/speech-lite

max_forks_repo_name: dzubke/speech-lite

reason: import numpy


text:
import sympy as sp
from sympy.logic.boolalg import Or, And

_op_identity = {And: True, Or: False}

def check_value(order):
    """ Checks that the value of order is between 0 and 1 """
    if not (0 <= order <= 1):
        raise ValueError

def unique(sequence):
    """ Returns the sequence without duplicate elements """
    return list(set(sequence))

def conjuncts(sentence):
    """ Returns a list of all the conjuncts in the given sentence """
    return dissociate(And, [sentence])

def disjuncts(sentence):
    """ Returns a list of all the disjuncts in the given sentence """
    return dissociate(Or, [sentence])

def dissociate(op, args):
    """ Given an associative operator, return a flattened list result """
    result = []

    def collect(subargs):
        for arg in subargs:
            if arg.func == op:
                collect(arg.args)
            else:
                result.append(arg)

    collect(args)
    return result

def associate(op, args):
    """ Given an associative operator, return an flattened expression
    such that nested instances of the same operator is at the top level.
    """
    args = dissociate(op, args)
    if len(args) == 0:
        return _op_identity[op]
    elif len(args) == 1:
        return args[0]
    else:
        return op(*args)

def first(iterable, default=None):
    """ Returns the first element of an iterable """
    return next(iter(iterable), default)

max_stars_repo_name: simonamtoft/belief-revision-agent

max_issues_repo_name: simonamtoft/belief-revision-agent

max_forks_repo_name: simonamtoft/belief-revision-agent

reason: import sympy,from sympy


text:
"""Metrics for evaluating IRL algorithm performance"""

import warnings

import numpy as np
from scipy.stats import norm

from mdp_extras import vi, v2q, OptimalPolicy, pi_eval

def ile_evd(
    xtr,
    phi,
    reward_gt,
    reward_test,
    *,
    p=1,
    vi_kwargs={},
    policy_kwargs={},
    pe_kwargs={},
    ret_gt_value=False,
    gt_policy_value=None,
):
    """Find Inverse Learning Error and Expected Value Difference metrics

    Inverse Learning Error is defined in "Inverse reinforcement learning in partially
    observable environments." by Choi and Kim, 2011.

    Expected Value Difference is defined in "Nonlinear inverse reinforcement learning
    with gaussian processes." by Levine, et al. 2011. EVD is essentially a weighted
    version of ILE, that only considers states with non-zero starting probability.

    Args:
        xtr (mdp_extras.DiscreteExplicitExtras) MDP extras object
        phi (mdp_extras.FeatureFunction) Feature function for MDP
        reward_gt (mdp_extras.RewardFunction): Ground Truth reward function for MDP
        reward_test (mdp_extras.RewardFunction): Learned reward function for MDP

        p (int): p-Norm to use for ILE, Choi and Kim and other papers recommend p=1
        vi_kwargs (dict): Extra keyword args for mdp_extras.vi Value Iteration method
        policy_kwargs (dict): Extra keyword args for mdp_extras.OptimalPolicy
        pe_kwargs (dict): Extra keyword args for mdp_extras.pi_eval Policy Evaluation method
        ret_gt_value (bool): If true, also return the GT policy state value function,
            can be used for speeding up future calls
        gt_policy_value (numpy array): Optional ground truth policy state value function
            - used for speeding up this function with multiple calls

    Returns:
        (float): Inverse Learning Error metric
        (float): Expected Value Difference metric
    """

    if gt_policy_value is None:
        # Get GT policy state value function
        gt_policy_value, _ = vi(xtr, phi, reward_gt, **vi_kwargs)

    # Get test policy state value function under GT reward
    v_star_test, q_star_test = vi(xtr, phi, reward_test, **vi_kwargs)
    pi_star_test = OptimalPolicy(q_star_test, stochastic=False, **policy_kwargs)
    test_policy_value = pi_eval(xtr, phi, reward_gt, pi_star_test, **pe_kwargs)

    value_delta = gt_policy_value - test_policy_value
    ile = np.linalg.norm(value_delta, ord=p)
    evd = xtr.p0s @ value_delta

    if evd < 0:
        warnings.warn(
            f"EVD is < 0 (by {0 - evd}) - possible loss of accuracy due to numerical rounding"
        )
        evd = 0.0
    if ile < 0:
        warnings.warn(
            f"ILE is < 0 (by {0 - ile}) - possible loss of accuracy due to numerical rounding"
        )
        ile = 0.0

    if not ret_gt_value:
        return ile, evd
    else:
        return ile, evd, gt_policy_value

def inner_angle(v1, v2):
    """Find inner angle between two n-vectors in radians

    Args:
        v1 (numpy array): First vector
        v2 (numpy array): Second vector

    Returns:
        (float): Inner angle in radians
    """

    assert len(v1.shape) == 1
    assert v1.shape == v2.shape

    return np.arccos(v1 @ v2 / (np.linalg.norm(v1) * np.linalg.norm(v2)))

def mean_ci(values, confidence_level=0.95):
    """Compute mean and symmetric confidence interval for a list of values

    Args:
        values (list): List of float
        confidence_level (float): Confidence level

    Returns:
        (float): Lower confidence interval
        (float): Mean value
        (float): Upper confidence interval
    """
    mean = np.mean(values)
    std = np.std(values)
    num_repeats = len(values)

    # Compute Z-factor
    critical_value = 1.0 - confidence_level
    z_factor = norm().ppf(1 - critical_value / 2)

    # Compute CI
    ci = z_factor * std / np.sqrt(num_repeats)

    return mean - ci, mean, mean + ci

def mean_ci_agg(values, *args):
    """Aggregator for use with Pandas groupby() function

    Args:
        values (pandas DataFrame): DataFrame with floating point values to apply aggregator to

        *args: Optional args to pass to mean_ci function

    Returns:
        (pandas DataFrame): DataFrame containing strings of the form "Mean ± CI"
    """
    if np.any(values.isna().to_numpy()):
        return "N/A"
    low, med, high = mean_ci(values, *args)
    interval = high - low
    return "{:03.2f} $\pm$ {:05.2f}".format(med, interval)

def median_ci(values, confidence_level=0.95):
    """Compute median and approximate confidence interval for a list of values

    The method of computing the CI is taken from
    https://www.ucl.ac.uk/child-health/short-courses-events/about-statistical-courses/research-methods-and-statistics/chapter-8-content-8

    Args:
        values (list): List of float
        confidence_level (float): Confidence level

    Returns:
        (float): Lower approximate confidence interval
        (float): Median value
        (float): Upper approximate confidence interval
    """
    median = np.median(values)
    num_repeats = len(values)

    # Compute Z-factor
    critical_value = 1.0 - confidence_level
    z_factor = norm().ppf(1 - critical_value / 2)

    # Compute CI rankings
    low_ci_rank = int(round(num_repeats / 2 - z_factor * np.sqrt(num_repeats) / 2))
    high_ci_rank = int(round(1 + num_repeats / 2 + z_factor * np.sqrt(num_repeats) / 2))
    values_sorted = sorted(values)

    return values_sorted[low_ci_rank], median, values_sorted[high_ci_rank]

def median_ci_agg(values, *args):
    """Aggregator for use with Pandas groupby() function

    Args:
        values (pandas DataFrame): DataFrame with floating point values to apply aggregator to

        *args: Optional args to pass to median_ci function

    Returns:
        (pandas DataFrame): DataFrame containing strings of the form "Median [Low - High]"
    """
    if np.any(values.isna().to_numpy()):
        return "N/A"
    low, med, high = median_ci(values, *args)
    return "{:.2f} [{:.2f} - {:.2f}]".format(low, med, high)

max_stars_repo_name: aaronsnoswell/UniModal-IRL

max_issues_repo_name: aaronsnoswell/UniModal-IRL

max_forks_repo_name: aaronsnoswell/UniModal-IRL


text:
import numpy as np
from .utils import setDimensions,accum_np
from .OBSstruct import OBSstruct
from netCDF4 import Dataset
#from .sgrid import SGrid

def superob(S,superprov=0):
    '''
    This function checks the provided observation data and creates
    super observations when there are more than one meassurement of
    the same state variable per grid cell.

    This is basically a python version of super_obs.m, distributed
    through myroms.org
    '''
    if not isinstance(S,OBSstruct):
       fid = Dataset(S)
       OBS = OBSstruct(fid)
    else:
       OBS=OBSstruct(S)

    field_list=OBS.getfieldlist()

    binfields = OBS.getfieldlist(); binfields.remove('time'); binfields.remove('type')

    # Find observations associated with the same state variable
    state_vars = np.unique(OBS.type)
    Nstate = len(np.unique(OBS.type))

    Sout = OBSstruct()
    Sout.globalatts = OBS.globalatts
    Sout.Nsurvey = OBS.Nsurvey
    Sout.Nstate = len(OBS.variance)
    Sout.spherical = OBS.spherical
    Sout.survey_time = OBS.survey_time
    Sout.variance = OBS.variance
    Sout.Nobs = np.zeros_like(OBS.Nobs).tolist()
    Sout_std = []
    # Compute super observations when needed
    for m in range(0, OBS.Nsurvey):

        T=OBSstruct()
        # Extract observations with the same survey time
        OBS.toarray()
        ind_t = np.where(OBS.time == OBS.survey_time[m])[0]

        for names in field_list:
           T.__dict__[names]=OBS.__dict__[names][ind_t]
        t_std=np.zeros_like(ind_t)
        std=OBS.error[ind_t]
        for n in range (0,Nstate):
            # Now extract observations associated with the same state variable
            ind_v = np.where(T.type == state_vars[n])[0]
            if (len(ind_v)==0):
                continue
            V=OBSstruct()
            for names in field_list:
                V.__dict__[names]=T.__dict__[names][ind_v]
            v_std=np.zeros_like(ind_v)
            vstd=T.error[ind_v]

            # Set binning parameters. The processing is done in
            #fractional (x,y,z) grid locations

            Xmin=np.min(V.Xgrid)
            Xmax=np.max(V.Xgrid)

            Ymin=np.min(V.Ygrid)
            Ymax=np.max(V.Ygrid)

            Zmin=np.min(V.Zgrid)
            Zmax=np.max(V.Zgrid)

            dx=1.; dy=1.; dz=1.
            minObs=1

            # Compute the index in each dimension of the grid cell in which the
            # observation is located

            Xbin = 1.0 + np.floor((V.Xgrid -Xmin)/dx)
            Ybin = 1.0 + np.floor((V.Ygrid -Ymin)/dy)
            Zbin = 1.0 + np.floor((V.Zgrid -Zmin)/dz)

            # Similarly, compute the maximum averaging grid size

            Xsize = int(1.0 + np.floor((Xmax - Xmin)/dx))
            Ysize = int(1.0 + np.floor((Ymax - Ymin)/dy))
            Zsize = int(1.0 + np.floor((Zmax - Zmin)/dz))

            A=np.ones([Ysize,Xsize,Zsize])
            # Combine the indices in each dimension into one index.
            # It is like stacking all the matrix in one column vector
            varInd= Ybin + (Xbin-1)*A.shape[0] + (Zbin-1)*A.shape[0]*A.shape[1]
            onesCol=np.ones_like(varInd)
            maxVarInd=np.max(varInd)
            # Accumulate values in bins usig accum_np function.
            # Count how many observations fall in each bin
            #count=accum_np(varInd,onesCol)
            count=np.bincount(varInd.astype(int),onesCol)

            # Bins with no observations are not kept. Index vector "isdata"
            # will be used in the conversion from the sparse output
            # from accumarray to the corresponding full vector assigned to
            # Sout
            isdata = np.where(count != 0)[0]
            Nsuper = len(isdata)
            # bin all fields
            # remove time and type from field_list used in this loop:

            for names in binfields:
                binned=np.bincount(varInd.astype(int),V.__dict__[names])
                binned=binned[isdata]/count[isdata]

                if names =='provenance':
                    for s in range(len(binned)):
                        if not binned[s] in set(V.provenance):
                            binned[s] = superprov + state_vars[n]

                Sout.__dict__[names]  = np.append( Sout.__dict__[names], binned[:])

                if names =='value':
                    Vmean=binned
            binned = np.bincount(varInd.astype(int),V.value**2)
            binned=binned[isdata]/count[isdata] - Vmean**2
            #Sout.std.extend(np.sqrt(binned))

            Sout.type  = np.append( Sout.type, np.ones([Nsuper])*state_vars[n] )
            Sout.time  = np.append( Sout.time, np.ones([Nsuper])*OBS.survey_time[m] )

            #Sout.type.extend(np.ones([Nsuper])*state_vars[n])
            #Sout.time.extend(np.ones([Nsuper])*OBS.survey_time[m])
            Sout.Nobs[m] =  Sout.Nobs[m]+Nsuper

            if hasattr(OBS,'provenance'):
               if len(OBS.provenance):
                    Sout.provenance = np.floor(Sout.provenance).tolist()

    Sout=setDimensions(Sout)
    Sout.toarray()
    return Sout

max_stars_repo_name: metno/pyromsobs

max_issues_repo_name: metno/pyromsobs

max_forks_repo_name: metno/pyromsobs


text:
import sys
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
import os
import numpy

a = torch.Tensor([[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1],[1, 1, 1, 1]])
b = torch.Tensor([[-1, -1, -1, -2], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]])
print(a.numpy())

def custom_loss(prediction, label):
    loss = torch.abs(label-prediction)
    avg_throttle_loss = numpy.mean(loss.numpy()[:, 0])
    sum_throttle_loss = sum(loss.numpy()[:, 0])
    total_loss = torch.sum(loss)
    return loss, total_loss, avg_throttle_loss, sum_throttle_loss

J = custom_loss(a, b)

print(J)
print(J[1])
print(J[2])
print(J[3])

c="([[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1],[1, 1, 1, 1]])"
print(c.strip("()"))

max_stars_repo_name: CerberusLatrans/AlphaSlow

max_issues_repo_name: CerberusLatrans/AlphaSlow

max_forks_repo_name: CerberusLatrans/AlphaSlow


text:
from __future__ import absolute_import, division, print_function

import os
import pickle
from itertools import product
from multiprocessing import cpu_count

import dask
import dask.array as da
import dask.dataframe as dd
import numpy as np
import pandas as pd
import pytest
from dask.base import tokenize
from dask.callbacks import Callback
from dask.delayed import delayed
from dask.utils import tmpdir
from distributed import Client, Nanny, Variable
from distributed.utils_test import cluster, loop  # noqa
from sklearn.datasets import load_iris, make_classification
from sklearn.decomposition import PCA
from sklearn.ensemble import RandomForestClassifier
from sklearn.exceptions import FitFailedWarning, NotFittedError
from sklearn.feature_selection import SelectKBest
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import (
    GridSearchCV,
    GroupKFold,
    GroupShuffleSplit,
    KFold,
    LeaveOneGroupOut,
    LeaveOneOut,
    LeavePGroupsOut,
    LeavePOut,
    PredefinedSplit,
    ShuffleSplit,
    StratifiedKFold,
    StratifiedShuffleSplit,
    TimeSeriesSplit,
)
from sklearn.model_selection._split import _CVIterableWrapper
from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.svm import SVC

import dask_ml.model_selection as dcv
from dask_ml._compat import DISTRIBUTED_2_11_0
from dask_ml.model_selection import check_cv, compute_n_splits
from dask_ml.model_selection._search import _normalize_n_jobs
from dask_ml.model_selection.methods import CVCache
from dask_ml.model_selection.utils_test import (
    AsCompletedEstimator,
    CheckXClassifier,
    FailingClassifier,
    MockClassifier,
    MockClassifierWithFitParam,
    ScalingTransformer,
)

def _passthrough_scorer(estimator, *args, **kwargs):
    """Function that wraps estimator.score"""
    return estimator.score(*args, **kwargs)

class assert_dask_compute(Callback):
    def __init__(self, compute=False):
        self.compute = compute

    def __enter__(self):
        self.ran = False
        super(assert_dask_compute, self).__enter__()

    def __exit__(self, *args):
        if not self.compute and self.ran:
            raise ValueError("Unexpected call to compute")
        elif self.compute and not self.ran:
            raise ValueError("Expected call to compute, but none happened")
        super(assert_dask_compute, self).__exit__(*args)

    def _start(self, dsk):
        self.ran = True

def test_visualize():
    pytest.importorskip("graphviz")

    X, y = make_classification(n_samples=100, n_classes=2, flip_y=0.2, random_state=0)
    clf = SVC(random_state=0, gamma="auto")
    grid = {"C": [0.1, 0.5, 0.9]}
    gs = dcv.GridSearchCV(clf, param_grid=grid).fit(X, y)

    assert hasattr(gs, "dask_graph_")

    with tmpdir() as d:
        gs.visualize(filename=os.path.join(d, "mydask"))
        assert os.path.exists(os.path.join(d, "mydask.png"))

    # Doesn't work if not fitted
    gs = dcv.GridSearchCV(clf, param_grid=grid)
    with pytest.raises(NotFittedError):
        gs.visualize()

np_X = np.random.normal(size=(20, 3))
np_y = np.random.randint(2, size=20)
np_groups = np.random.permutation(list(range(5)) * 4)
da_X = da.from_array(np_X, chunks=(3, 3))
da_y = da.from_array(np_y, chunks=3)
da_groups = da.from_array(np_groups, chunks=3)
del_X = delayed(np_X)
del_y = delayed(np_y)
del_groups = delayed(np_groups)

@pytest.mark.parametrize(
    ["cls", "has_shuffle"],
    [
        (KFold, True),
        (GroupKFold, False),
        (StratifiedKFold, True),
        (TimeSeriesSplit, False),
    ],
)
def test_kfolds(cls, has_shuffle):
    assert tokenize(cls(n_splits=3)) == tokenize(cls(n_splits=3))
    assert tokenize(cls(n_splits=3)) != tokenize(cls(n_splits=4))
    if has_shuffle:
        assert tokenize(cls(shuffle=True, random_state=0, n_splits=3)) == tokenize(
            cls(shuffle=True, random_state=0, n_splits=3)
        )

        rs = np.random.RandomState(42)
        assert tokenize(cls(shuffle=True, random_state=rs, n_splits=3)) == tokenize(
            cls(shuffle=True, random_state=rs, n_splits=3)
        )

        assert tokenize(cls(shuffle=True, random_state=0, n_splits=3)) != tokenize(
            cls(shuffle=True, random_state=2, n_splits=3)
        )

        assert tokenize(cls(shuffle=False, random_state=None, n_splits=3)) == tokenize(
            cls(shuffle=False, random_state=None, n_splits=3)
        )

    cv = cls(n_splits=3)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == 3

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == 3

@pytest.mark.parametrize(
    "cls", [ShuffleSplit, GroupShuffleSplit, StratifiedShuffleSplit]
)
def test_shuffle_split(cls):
    assert tokenize(cls(n_splits=3, random_state=0)) == tokenize(
        cls(n_splits=3, random_state=0)
    )

    assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
        cls(n_splits=3, random_state=2)
    )

    assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
        cls(n_splits=4, random_state=0)
    )

    cv = cls(n_splits=3)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == 3

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == 3

@pytest.mark.parametrize("cvs", [(LeaveOneOut(),), (LeavePOut(2), LeavePOut(3))])
def test_leave_out(cvs):
    tokens = []
    for cv in cvs:
        assert tokenize(cv) == tokenize(cv)
        tokens.append(cv)
    assert len(set(tokens)) == len(tokens)

    cv = cvs[0]
    sol = cv.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == sol

    with assert_dask_compute(True):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == sol

    with assert_dask_compute(False):
        assert compute_n_splits(cv, np_X, da_y, da_groups) == sol

@pytest.mark.parametrize(
    "cvs", [(LeaveOneGroupOut(),), (LeavePGroupsOut(2), LeavePGroupsOut(3))]
)
def test_leave_group_out(cvs):
    tokens = []
    for cv in cvs:
        assert tokenize(cv) == tokenize(cv)
        tokens.append(cv)
    assert len(set(tokens)) == len(tokens)

    cv = cvs[0]
    sol = cv.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == sol

    with assert_dask_compute(True):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == sol

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, np_groups) == sol

def test_predefined_split():
    cv = PredefinedSplit(np.array(list(range(4)) * 5))
    cv2 = PredefinedSplit(np.array(list(range(5)) * 4))
    assert tokenize(cv) == tokenize(cv)
    assert tokenize(cv) != tokenize(cv2)

    sol = cv.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == sol

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == sol

def test_old_style_cv():
    cv1 = _CVIterableWrapper(
        [
            np.array([True, False, True, False] * 5),
            np.array([False, True, False, True] * 5),
        ]
    )
    cv2 = _CVIterableWrapper(
        [
            np.array([True, False, True, False] * 5),
            np.array([False, True, True, True] * 5),
        ]
    )
    assert tokenize(cv1) == tokenize(cv1)
    assert tokenize(cv1) != tokenize(cv2)

    sol = cv1.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv1, np_X, np_y, np_groups) == sol
    with assert_dask_compute(False):
        assert compute_n_splits(cv1, da_X, da_y, da_groups) == sol

def test_check_cv():
    # No y, classifier=False
    cv = check_cv(3, classifier=False)
    assert isinstance(cv, KFold) and cv.n_splits == 3
    cv = check_cv(5, classifier=False)
    assert isinstance(cv, KFold) and cv.n_splits == 5

    # y, classifier = False
    dy = da.from_array(np.array([1, 0, 1, 0, 1]), chunks=2)
    with assert_dask_compute(False):
        assert isinstance(check_cv(y=dy, classifier=False), KFold)

    # Binary and multi-class y
    for y in [
        np.array([0, 1, 0, 1, 0, 0, 1, 1, 1]),
        np.array([0, 1, 0, 1, 2, 1, 2, 0, 2]),
    ]:
        cv = check_cv(5, y, classifier=True)
        assert isinstance(cv, StratifiedKFold) and cv.n_splits == 5

        dy = da.from_array(y, chunks=2)
        with assert_dask_compute(True):
            cv = check_cv(5, dy, classifier=True)
        assert isinstance(cv, StratifiedKFold) and cv.n_splits == 5

    # Non-binary/multi-class y
    y = np.array([[1, 2], [0, 3], [0, 0], [3, 1], [2, 0]])
    assert isinstance(check_cv(y=y, classifier=True), KFold)

    dy = da.from_array(y, chunks=2)
    with assert_dask_compute(True):
        assert isinstance(check_cv(y=dy, classifier=True), KFold)

    # Old style
    cv = [np.array([True, False, True]), np.array([False, True, False])]
    with assert_dask_compute(False):
        assert isinstance(check_cv(cv, y=dy, classifier=True), _CVIterableWrapper)

    # CV instance passes through
    y = da.ones(5, chunks=2)
    cv = ShuffleSplit()
    with assert_dask_compute(False):
        assert check_cv(cv, y, classifier=True) is cv
        assert check_cv(cv, y, classifier=False) is cv

def test_grid_search_dask_inputs():
    # Numpy versions
    np_X, np_y = make_classification(n_samples=15, n_classes=2, random_state=0)
    np_groups = np.random.RandomState(0).randint(0, 3, 15)
    # Dask array versions
    da_X = da.from_array(np_X, chunks=5)
    da_y = da.from_array(np_y, chunks=5)
    da_groups = da.from_array(np_groups, chunks=5)
    # Delayed versions
    del_X = delayed(np_X)
    del_y = delayed(np_y)
    del_groups = delayed(np_groups)

    cv = GroupKFold(n_splits=3)
    clf = SVC(random_state=0, gamma="auto")
    grid = {"C": [1]}

    sol = SVC(C=1, random_state=0, gamma="auto").fit(np_X, np_y).support_vectors_

    for X, y, groups in product(
        [np_X, da_X, del_X], [np_y, da_y, del_y], [np_groups, da_groups, del_groups]
    ):
        gs = dcv.GridSearchCV(clf, param_grid=grid, cv=cv)

        with pytest.raises(ValueError) as exc:
            gs.fit(X, y)
        assert "parameter should not be None" in str(exc.value)

        gs.fit(X, y, groups=groups)
        np.testing.assert_allclose(sol, gs.best_estimator_.support_vectors_)

def test_grid_search_dask_dataframe():
    iris = load_iris()
    X = iris.data
    y = iris.target

    df = pd.DataFrame(X)
    ddf = dd.from_pandas(df, 2)

    dy = pd.Series(y)
    ddy = dd.from_pandas(dy, 2)

    clf = LogisticRegression(multi_class="auto", solver="lbfgs", max_iter=200)

    param_grid = {"C": [0.1, 1, 10]}
    gs = GridSearchCV(clf, param_grid, cv=5)
    dgs = dcv.GridSearchCV(clf, param_grid, cv=5)
    gs.fit(df, dy)
    dgs.fit(ddf, ddy)

    assert gs.best_params_ == dgs.best_params_

def test_pipeline_feature_union():
    iris = load_iris()
    X, y = iris.data, iris.target

    pca = PCA(random_state=0)
    kbest = SelectKBest()

    empty_union = FeatureUnion([("first", "drop"), ("second", "drop")])
    empty_pipeline = Pipeline([("first", None), ("second", None)])
    scaling = Pipeline([("transform", ScalingTransformer())])
    svc = SVC(kernel="linear", random_state=0)

    pipe = Pipeline(
        [
            ("empty_pipeline", empty_pipeline),
            ("scaling", scaling),
            ("missing", None),
            (
                "union",
                FeatureUnion(
                    [
                        ("pca", pca),
                        ("missing", "drop"),
                        ("kbest", kbest),
                        ("empty_union", empty_union),
                    ],
                    transformer_weights={"pca": 0.5},
                ),
            ),
            ("svc", svc),
        ]
    )

    param_grid = dict(
        scaling__transform__factor=[1, 2],
        union__pca__n_components=[1, 2, 3],
        union__kbest__k=[1, 2],
        svc__C=[0.1, 1, 10],
    )

    gs = GridSearchCV(pipe, param_grid=param_grid, cv=3)
    gs.fit(X, y)
    dgs = dcv.GridSearchCV(pipe, param_grid=param_grid, scheduler="sync", cv=3)
    dgs.fit(X, y)

    # Check best params match
    assert gs.best_params_ == dgs.best_params_

    # Check PCA components match
    sk_pca = gs.best_estimator_.named_steps["union"].transformer_list[0][1]
    dk_pca = dgs.best_estimator_.named_steps["union"].transformer_list[0][1]
    np.testing.assert_allclose(sk_pca.components_, dk_pca.components_)

    # Check SelectKBest scores match
    sk_kbest = gs.best_estimator_.named_steps["union"].transformer_list[2][1]
    dk_kbest = dgs.best_estimator_.named_steps["union"].transformer_list[2][1]
    np.testing.assert_allclose(sk_kbest.scores_, dk_kbest.scores_)

    # Check SVC coefs match
    np.testing.assert_allclose(
        gs.best_estimator_.named_steps["svc"].coef_,
        dgs.best_estimator_.named_steps["svc"].coef_,
    )

def test_pipeline_sub_estimators():
    iris = load_iris()
    X, y = iris.data, iris.target

    scaling = Pipeline([("transform", ScalingTransformer())])

    pipe = Pipeline(
        [
            ("setup", None),
            ("missing", None),
            ("scaling", scaling),
            ("svc", SVC(kernel="linear", random_state=0)),
        ]
    )

    param_grid = [
        {"svc__C": [0.1, 0.1]},  # Duplicates to test culling
        {
            "setup": [None],
            "svc__C": [0.1, 1, 10],
            "scaling": [ScalingTransformer(), None],
        },
        {
            "setup": [SelectKBest()],
            "setup__k": [1, 2],
            "svc": [
                SVC(kernel="linear", random_state=0, C=0.1),
                SVC(kernel="linear", random_state=0, C=1),
                SVC(kernel="linear", random_state=0, C=10),
            ],
        },
    ]

    gs = GridSearchCV(pipe, param_grid=param_grid, return_train_score=True, cv=3,)
    gs.fit(X, y)
    dgs = dcv.GridSearchCV(
        pipe, param_grid=param_grid, scheduler="sync", return_train_score=True, cv=3
    )
    dgs.fit(X, y)

    # Check best params match
    assert gs.best_params_ == dgs.best_params_

    # Check cv results match
    res = pd.DataFrame(dgs.cv_results_)
    sol = pd.DataFrame(gs.cv_results_)

    # TODO: Failures on Py36 / sklearn dev with order here.
    res = res.reindex(columns=sol.columns)

    pd.util.testing.assert_index_equal(res.columns, sol.columns)
    skip = ["mean_fit_time", "std_fit_time", "mean_score_time", "std_score_time"]
    res = res.drop(skip, axis=1)
    sol = sol.drop(skip, axis=1)
    pd.util.testing.assert_frame_equal(
        res, sol, check_exact=False, check_less_precise=1
    )

    # Check SVC coefs match
    np.testing.assert_allclose(
        gs.best_estimator_.named_steps["svc"].coef_,
        dgs.best_estimator_.named_steps["svc"].coef_,
    )

def check_scores_all_nan(gs, bad_param, score_key="score"):
    bad_param = "param_" + bad_param
    n_candidates = len(gs.cv_results_["params"])
    keys = ["split{}_test_{}".format(s, score_key) for s in range(gs.n_splits_)]
    assert all(
        np.isnan([gs.cv_results_[key][cand_i] for key in keys]).all()
        for cand_i in range(n_candidates)
        if gs.cv_results_[bad_param][cand_i] == FailingClassifier.FAILING_PARAMETER
    )

@pytest.mark.xfail(reason="https://github.com/dask/dask-ml/issues/672")
@pytest.mark.parametrize(
    "weights", [None, (None, {"tr0": 2, "tr2": 3}, {"tr0": 2, "tr2": 4})]
)
def test_feature_union(weights):
    X = np.ones((10, 5))
    y = np.zeros(10)

    union = FeatureUnion(
        [
            ("tr0", ScalingTransformer()),
            ("tr1", ScalingTransformer()),
            ("tr2", ScalingTransformer()),
        ]
    )

    factors = [(2, 3, 5), (2, 4, 5), (2, 4, 6), (2, 4, None), (None, None, None)]
    params, sols, grid = [], [], []
    for constants, w in product(factors, weights or [None]):
        p = {}
        for n, c in enumerate(constants):
            if c is None:
                p["tr%d" % n] = None
            elif n == 3:  # 3rd is always an estimator
                p["tr%d" % n] = ScalingTransformer(c)
            else:
                p["tr%d__factor" % n] = c
        sol = union.set_params(transformer_weights=w, **p).transform(X)
        sols.append(sol)
        if w is not None:
            p["transformer_weights"] = w
        params.append(p)
        p2 = {"union__" + k: [v] for k, v in p.items()}
        p2["est"] = [CheckXClassifier(sol[0])]
        grid.append(p2)

    # Need to recreate the union after setting estimators to `None` above
    union = FeatureUnion(
        [
            ("tr0", ScalingTransformer()),
            ("tr1", ScalingTransformer()),
            ("tr2", ScalingTransformer()),
        ]
    )

    pipe = Pipeline([("union", union), ("est", CheckXClassifier())])
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False, cv=2, n_jobs=1)

    gs.fit(X, y)

def test_feature_union_fit_failure():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)

    pipe = Pipeline(
        [
            (
                "union",
                FeatureUnion(
                    [("good", MockClassifier()), ("bad", FailingClassifier())],
                    transformer_weights={"bad": 0.5},
                ),
            ),
            ("clf", MockClassifier()),
        ]
    )

    grid = {"union__bad__parameter": [0, 1, 2]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False, scoring=None)

    # Check that failure raises if error_score is `'raise'`
    with pytest.raises(ValueError):
        gs.fit(X, y)

    # Check that grid scores were set to error_score on failure
    gs.error_score = float("nan")
    with pytest.warns(FitFailedWarning):
        gs.fit(X, y)
    check_scores_all_nan(gs, "union__bad__parameter")

def test_feature_union_fit_failure_multiple_metrics():
    scoring = {"score_1": _passthrough_scorer, "score_2": _passthrough_scorer}
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)

    pipe = Pipeline(
        [
            (
                "union",
                FeatureUnion(
                    [("good", MockClassifier()), ("bad", FailingClassifier())],
                    transformer_weights={"bad": 0.5},
                ),
            ),
            ("clf", MockClassifier()),
        ]
    )

    grid = {"union__bad__parameter": [0, 1, 2]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False, scoring=scoring)

    # Check that failure raises if error_score is `'raise'`
    with pytest.raises(ValueError):
        gs.fit(X, y)

    # Check that grid scores were set to error_score on failure
    gs.error_score = float("nan")
    with pytest.warns(FitFailedWarning):
        gs.fit(X, y)

    for key in scoring:
        check_scores_all_nan(gs, "union__bad__parameter", score_key=key)

def test_failing_classifier_fails():
    clf = dcv.GridSearchCV(
        FailingClassifier(),
        {
            "parameter": [
                FailingClassifier.FAILING_PARAMETER,
                FailingClassifier.FAILING_SCORE_PARAMETER,
            ]
        },
        refit=False,
        return_train_score=False,
    )

    X, y = make_classification()

    with pytest.raises(ValueError, match="Failing"):
        clf.fit(X, y)

    clf = clf.set_params(error_score=-1)

    with pytest.warns(FitFailedWarning):
        clf.fit(X, y)

    for result in ["mean_fit_time", "mean_score_time", "mean_test_score"]:
        assert not any(np.isnan(clf.cv_results_[result]))

def test_pipeline_fit_failure():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)

    pipe = Pipeline(
        [
            ("bad", FailingClassifier()),
            ("good1", MockClassifier()),
            ("good2", MockClassifier()),
        ]
    )

    grid = {
        "bad__parameter": [
            0,
            FailingClassifier.FAILING_PARAMETER,
            FailingClassifier.FAILING_PREDICT_PARAMETER,
            FailingClassifier.FAILING_SCORE_PARAMETER,
        ]
    }
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False)

    # Check that failure raises if error_score is `'raise'`
    with pytest.raises(ValueError):
        gs.fit(X, y)

    # Check that grid scores were set to error_score on failure
    gs.error_score = float("nan")
    with pytest.warns(FitFailedWarning):
        gs.fit(X, y)

    check_scores_all_nan(gs, "bad__parameter")

@pytest.mark.filterwarnings("ignore")
@pytest.mark.parametrize("in_pipeline", [False, True])
def test_estimator_predict_failure(in_pipeline):
    X, y = make_classification()
    if in_pipeline:
        clf = Pipeline([("bad", FailingClassifier())])
        key = "bad__parameter"
    else:
        clf = FailingClassifier()
        key = "parameter"

    grid = {
        key: [
            0,
            FailingClassifier.FAILING_PARAMETER,
            FailingClassifier.FAILING_PREDICT_PARAMETER,
            FailingClassifier.FAILING_SCORE_PARAMETER,
        ]
    }
    gs = dcv.GridSearchCV(
        clf, param_grid=grid, refit=False, error_score=float("nan"), cv=2
    )
    gs.fit(X, y)

def test_pipeline_raises():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)

    pipe = Pipeline([("step1", MockClassifier()), ("step2", MockClassifier())])

    grid = {"step3__parameter": [0, 1, 2]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False)
    with pytest.raises(ValueError):
        gs.fit(X, y)

    grid = {"steps": [[("one", MockClassifier()), ("two", MockClassifier())]]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False)
    with pytest.raises(NotImplementedError):
        gs.fit(X, y)

def test_feature_union_raises():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)

    union = FeatureUnion([("tr0", MockClassifier()), ("tr1", MockClassifier())])
    pipe = Pipeline([("union", union), ("est", MockClassifier())])

    grid = {"union__tr2__parameter": [0, 1, 2]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False)
    with pytest.raises(ValueError):
        gs.fit(X, y)

    grid = {"union__transformer_list": [[("one", MockClassifier())]]}
    gs = dcv.GridSearchCV(pipe, param_grid=grid, refit=False)
    with pytest.raises(NotImplementedError):
        gs.fit(X, y)

def test_bad_error_score():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)
    gs = dcv.GridSearchCV(
        MockClassifier(), {"foo_param": [0, 1, 2]}, error_score="badparam"
    )

    with pytest.raises(ValueError):
        gs.fit(X, y)

class CountTakes(np.ndarray):
    count = 0

    def take(self, *args, **kwargs):
        self.count += 1
        return super().take(*args, **kwargs)

    def __getitem__(self, *args, **kwargs):
        self.count += 1
        return super().__getitem__(*args, **kwargs)

def test_cache_cv():
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)
    X2 = X.view(CountTakes)
    gs = dcv.GridSearchCV(
        MockClassifier(),
        {"foo_param": [0, 1, 2]},
        cv=3,
        cache_cv=False,
        scheduler="sync",
    )
    gs.fit(X2, y)
    assert X2.count == 2 * 3 * 3  # (1 train + 1 test) * n_params * n_splits

    X2 = X.view(CountTakes)
    assert X2.count == 0
    gs.cache_cv = True
    gs.fit(X2, y)
    assert X2.count == 2 * 3  # (1 test + 1 train) * n_splits

def test_CVCache_serializable():
    inds = np.arange(10)
    splits = [(inds[:3], inds[3:]), (inds[3:], inds[:3])]
    X = np.arange(100).reshape((10, 10))
    y = np.zeros(10)
    cache = CVCache(splits, pairwise=True, cache=True)

    # Add something to the cache
    r1 = cache.extract(X, y, 0)
    assert cache.extract(X, y, 0) is r1
    assert len(cache.cache) == 1

    cache2 = pickle.loads(pickle.dumps(cache))
    assert len(cache2.cache) == 0
    assert cache2.pairwise == cache.pairwise
    assert all(
        (cache2.splits[i][j] == cache.splits[i][j]).all()
        for i in range(2)
        for j in range(2)
    )

def test_normalize_n_jobs():
    assert _normalize_n_jobs(-1) is None
    assert _normalize_n_jobs(-2) == cpu_count() - 1
    with pytest.raises(TypeError):
        _normalize_n_jobs("not an integer")

@pytest.mark.parametrize(
    "scheduler,n_jobs",
    [
        (None, 4),
        ("threading", 4),
        ("threading", 1),
        ("synchronous", 4),
        ("sync", 4),
        ("multiprocessing", 4),
        pytest.param(dask.get, 4, marks=[pytest.mark.filterwarnings("ignore")]),
    ],
)
def test_scheduler_param(scheduler, n_jobs):
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)
    gs = dcv.GridSearchCV(
        MockClassifier(),
        {"foo_param": [0, 1, 2]},
        cv=3,
        scheduler=scheduler,
        n_jobs=n_jobs,
    )
    gs.fit(X, y)

def test_scheduler_param_distributed(loop):  # noqa
    X, y = make_classification(n_samples=100, n_features=10, random_state=0)
    with cluster() as (s, [a, b]):
        with Client(s["address"], loop=loop) as client:
            gs = dcv.GridSearchCV(MockClassifier(), {"foo_param": [0, 1, 2]}, cv=3)
            gs.fit(X, y)

            def f(dask_scheduler):
                return len(dask_scheduler.transition_log)

            assert client.run_on_scheduler(f)  # some work happened on cluster

@pytest.mark.skip(reason="https://github.com/dask/dask-ml/issues/611 TimeoutError")
def test_as_completed_distributed(loop):  # noqa
    cluster_kwargs = dict(active_rpc_timeout=10, nanny=Nanny)
    if DISTRIBUTED_2_11_0:
        cluster_kwargs["disconnect_timeout"] = 10
    with cluster(**cluster_kwargs) as (s, [a, b]):
        with Client(s["address"], loop=loop) as c:
            counter_name = "counter_name"
            counter = Variable(counter_name, client=c)
            counter.set(0)
            lock_name = "lock"

            killed_workers_name = "killed_workers"
            killed_workers = Variable(killed_workers_name, client=c)
            killed_workers.set({})

            X, y = make_classification(n_samples=100, n_features=10, random_state=0)
            gs = dcv.GridSearchCV(
                AsCompletedEstimator(killed_workers_name, lock_name, counter_name, 7),
                param_grid={"foo_param": [0, 1, 2]},
                cv=3,
                refit=False,
                cache_cv=False,
                scheduler=c,
            )
            gs.fit(X, y)

            def f(dask_scheduler):
                return dask_scheduler.transition_log

            def check_reprocess(transition_log):
                finished = set()
                for transition in transition_log:
                    key, start_state, end_state = (
                        transition[0],
                        transition[1],
                        transition[2],
                    )
                    assert key not in finished
                    if (
                        "score" in key
                        and start_state == "memory"
                        and end_state == "forgotten"
                    ):
                        finished.add(key)

            check_reprocess(c.run_on_scheduler(f))

def test_cv_multiplemetrics():
    X, y = make_classification(random_state=0)

    param_grid = {"max_depth": [1, 5]}
    a = dcv.GridSearchCV(
        RandomForestClassifier(n_estimators=10),
        param_grid,
        refit="score1",
        scoring={"score1": "accuracy", "score2": "accuracy"},
        cv=3,
    )
    b = GridSearchCV(
        RandomForestClassifier(n_estimators=10),
        param_grid,
        refit="score1",
        scoring={"score1": "accuracy", "score2": "accuracy"},
        cv=3,
    )
    a.fit(X, y)
    b.fit(X, y)

    assert a.best_score_ > 0
    assert isinstance(a.best_index_, type(b.best_index_))
    assert isinstance(a.best_params_, type(b.best_params_))

def test_cv_multiplemetrics_requires_refit_metric():
    X, y = make_classification(random_state=0)

    param_grid = {"max_depth": [1, 5]}
    a = dcv.GridSearchCV(
        RandomForestClassifier(n_estimators=10),
        param_grid,
        refit=True,
        scoring={"score1": "accuracy", "score2": "accuracy"},
    )

    with pytest.raises(ValueError):
        a.fit(X, y)

def test_cv_multiplemetrics_no_refit():
    X, y = make_classification(random_state=0)

    param_grid = {"max_depth": [1, 5]}
    a = dcv.GridSearchCV(
        RandomForestClassifier(),
        param_grid,
        refit=False,
        scoring={"score1": "accuracy", "score2": "accuracy"},
    )
    b = GridSearchCV(
        RandomForestClassifier(),
        param_grid,
        refit=False,
        scoring={"score1": "accuracy", "score2": "accuracy"},
    )
    assert hasattr(a, "best_index_") is hasattr(b, "best_index_")
    assert hasattr(a, "best_estimator_") is hasattr(b, "best_estimator_")
    assert hasattr(a, "best_score_") is hasattr(b, "best_score_")

@pytest.mark.parametrize("cache_cv", [True, False])
def test_gridsearch_with_arraylike_fit_param(cache_cv):
    # https://github.com/dask/dask-ml/issues/319
    X, y = make_classification(random_state=0)
    param_grid = {"foo_param": [0.0001, 0.1]}

    a = dcv.GridSearchCV(
        MockClassifierWithFitParam(), param_grid, cv=3, refit=False, cache_cv=cache_cv,
    )
    b = GridSearchCV(MockClassifierWithFitParam(), param_grid, cv=3, refit=False)

    b.fit(X, y, mock_fit_param=[0, 1])
    a.fit(X, y, mock_fit_param=[0, 1])

def test_mock_with_fit_param_raises():
    X, y = make_classification(random_state=0)

    clf = MockClassifierWithFitParam()

    with pytest.raises(ValueError):
        clf.fit(X, y)

max_stars_repo_name: GueroudjiAmal/dask-ml

max_issues_repo_name: GueroudjiAmal/dask-ml

max_forks_repo_name: GueroudjiAmal/dask-ml


text:
# This code is supporting material for the book
# Building Machine Learning Systems with Python
# by Willi Richert and Luis Pedro Coelho
# published by PACKT Publishing
#
# It is made available under the MIT License

import numpy as np
import mahotas as mh

# Load our example image:
image = mh.imread('../SimpleImageDataset/building05.jpg')

# Convert to greyscale
image = mh.colors.rgb2gray(image, dtype=np.uint8)

# Compute a threshold value:
thresh = mh.thresholding.otsu(image)
print('Otsu threshold is {0}'.format(thresh))

# Compute the thresholded image
otsubin = (image > thresh)
print('Saving thresholded image (with Otsu threshold) to otsu-threshold.jpeg')
mh.imsave('otsu-threshold.jpeg', otsubin.astype(np.uint8) * 255)

# Execute morphological opening to smooth out the edges
otsubin = mh.open(otsubin, np.ones((15, 15)))
mh.imsave('otsu-closed.jpeg', otsubin.astype(np.uint8) * 255)

# An alternative thresholding method:
thresh = mh.thresholding.rc(image)
print('Ridley-Calvard threshold is {0}'.format(thresh))
print('Saving thresholded image (with Ridley-Calvard threshold) to rc-threshold.jpeg')
mh.imsave('rc-threshold.jpeg', (image > thresh).astype(np.uint8) * 255)

max_stars_repo_path: Chapter 10/threshold.py

max_issues_repo_path: Chapter 10/threshold.py

max_forks_repo_path: Chapter 10/threshold.py


text:
import h5py
import matplotlib.pyplot as plt
from ipywidgets import interact, fixed
from pandas import read_hdf
import numpy as np

def view_eems(database_name):
    file_names = np.array(read_hdf(database_name, 'meta')['File_Name'])
    def plot_eem(database_name, i):

        fig = plt.figure(figsize=(4,3))
        with h5py.File(database_name, 'r') as f:
            eem = f['eems'][i]
        ex = eem[0,1:]
        em = eem[1:,0]
        fl = eem[1:,1:]
        plt.contourf(ex, em, fl)
        plt.colorbar()
        plt.title(file_names[i])
        return

    print(file_names.shape[0])
    interact(plot_eem, database_name=fixed(database_name), i=(0,file_names.shape[0]-1))
    return

#################################
####### Old visual functions from class work with Ben, need to adapt to new data structure
################################

import numpy as np
import pandas as pd
#import plotly.plotly as py
#import plotly.offline as offline
#import plotly.graph_objs as go
#from plotly.graph_objs import *
#offline.init_notebook_mode()

from scipy.ndimage import gaussian_filter

from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
from matplotlib import cm

#Disabled all plotly features because not currently using

def contourslice(ex, em, z, n, lower_level_frac, higher_level_frac, title=''):
    from mpl_toolkits.mplot3d import axes3d
    import matplotlib.pyplot as plt
    from matplotlib import cm

    '''Create a contour plot of your spectra data with a specific height and number of contour lines.
    ==========  ================================================
    Input    Description
    ==========  ================================================
    *ex*        Excitation wavelength as mesh grid of numpy.arrays
    *em*        Emission wavelength as mesh grid of numpy.arrays
    *em*        Intensity as mesh grid of numpy.arrays
    *n*         number of contour lines
    *lower*     percentile of lower contour
    *higher*    percentile of higher contour
    ==========  ================================================
    ex = excitation wavelengths as a np.meshgrid of an np.array
    em = emission wavelengths as a np.meshgrid of an np.array

    n is the number of contour lines that you want to view in the range selected.

    lower_level_frac and higher_level_frac are the percentile ranges of the
    intensity(z) levels to view. The range is from 0 to 1 over the range
    and lower_level_frac must be less than higher_level_frac.
    ============================================================'''

    assert higher_level_frac > lower_level_frac, 'higher must be greater than lower and both between 1 and 0'
    assert type(ex) == type(np.array([1])), 'ex should be np array'
    assert type(em) == type(np.array([1])), 'em should be np array'
    assert type(z) == type(np.array([1])), 'ez should be np array'
#     assert len(np.shape(ex))==2, 'shape is not the right dimensions'
#     assert np.shape(ex)==np.shape(em), 'shape of ex and em are not the same'
#     assert np.shape(ex)==np.shape(z), 'shape of z and ex are not the same'

    x = ex
    y = em

    # This is how we select the contour levels based on the input to the function
    bottom = z.min()
    top = z.max()
    diff = top - bottom
    higher_level = higher_level_frac*diff+bottom
    lower_level = lower_level_frac*diff+bottom
    level = np.linspace(lower_level, higher_level, n)

    fig = plt.figure(figsize=(5, 4))
    plt.contourf(x, y, z, n, levels=level)  # plots contour plot
    plt.title(title + '\n Slice from %1.2f' % lower_level_frac +
              ' to %1.2f' % higher_level_frac + ' of max')  # name function with info
    plt.xlabel('ex '+r'$\lambda$')  # label axis
    plt.ylabel('em '+r'$\lambda$')
    plt.colorbar()
    return fig

def contourslice_manual(ex, em, z, n, low_level, high_level, title=''):
    from mpl_toolkits.mplot3d import axes3d
    import matplotlib.pyplot as plt
    from matplotlib import cm

    '''Create a contour plot of your spectra data with a specific height and number of contour lines.
    ==========  ================================================
    Input    Description
    ==========  ================================================
    *ex*        Excitation wavelength as mesh grid of numpy.arrays
    *em*        Emission wavelength as mesh grid of numpy.arrays
    *em*        Intensity as mesh grid of numpy.arrays
    *n*         number of contour lines
    *low_level*     lower limit for contour levels
    *high_level*    high limit for contour levels
    ==========  ================================================
    ex = excitation wavelengths as a np.meshgrid of an np.array
    em = emission wavelengths as a np.meshgrid of an np.array

    n is the number of contour lines that you want to view in the range selected.

    lower_level_frac and higher_level_frac are the percentile ranges of the
    intensity(z) levels to view. The range is from 0 to 1 over the range
    and lower_level_frac must be less than higher_level_frac.
    ============================================================'''

    x = ex
    y = em

    # This is how we select the contour levels based on the input to the function
    level = np.linspace(low_level, high_level, n)

    fig = plt.figure(figsize=(5, 4))
    plt.contourf(x, y, z, n, levels=level)  # plots contour plot
    plt.title(title)  # name function with info
    plt.xlabel('ex '+r'$\lambda$')  # label axis
    plt.ylabel('em '+r'$\lambda$')
    plt.colorbar()
    return fig

def contour_line_slice(ex, em, z, n, lower_level_frac, higher_level_frac):
    from mpl_toolkits.mplot3d import axes3d
    import matplotlib.pyplot as plt
    from matplotlib import cm

    '''Create a contour plot of your spectra data with a specific height and number of contour lines.
    ==========  ================================================
    Input    Description
    ==========  ================================================
    *ex*        Excitation wavelength as mesh grid of numpy.arrays
    *em*        Emission wavelength as mesh grid of numpy.arrays
    *em*        Intensity as mesh grid of numpy.arrays
    *n*         number of contour lines
    *lower*     percentile of lower contour
    *higher*    percentile of higher contour
    ==========  ================================================
    ex = excitation wavelengths as a np.meshgrid of an np.array
    em = emission wavelengths as a np.meshgrid of an np.array

    n is the number of contour lines that you want to view in the range selected.

    lower_level_frac and higher_level_frac are the percentile ranges of the
    intensity(z) levels to view. The range is from 0 to 1 over the range
    and lower_level_frac must be less than higher_level_frac.
    ============================================================'''

    assert higher_level_frac > lower_level_frac, 'higher must be greater than lower and both between 1 and 0'
    assert type(ex) == type(np.array([1])), 'ex should be np array'
    assert type(em) == type(np.array([1])), 'em should be np array'
    assert type(z) == type(np.array([1])), 'ez should be np array'
#     assert len(np.shape(ex))==2, 'shape is not the right dimensions'
#     assert np.shape(ex)==np.shape(em), 'shape of ex and em are not the same'
#     assert np.shape(ex)==np.shape(z), 'shape of z and ex are not the same'

    x = ex
    y = em

    # This is how we select the contour levels based on the input to the function
    bottom = z.min()
    top = z.max()
    diff = top - bottom
    higher_level = higher_level_frac*diff+bottom
    lower_level = lower_level_frac*diff+bottom
    level = np.linspace(lower_level, higher_level, n)

    plt.contour(x, y, z, n, levels=level)  # plots contour plot
    plt.title('Contour Plot from %1.2f' % lower_level_frac +
              ' to %1.2f' % higher_level_frac + ' of normalized Intensity')  # name function with info
    plt.xlabel('ex '+r'$\lambda$')  # label axis
    plt.ylabel('em '+r'$\lambda$')
    plt.colorbar()
    return

def contour_line_compare(ex, em, eem_1, eem_2, n):
    import matplotlib.pyplot as plt
    from matplotlib import gridspec

    '''Create a contour plot of your spectra data with a specific height and number of contour lines.
    ==========  ================================================
    Input    Description
    ==========  ================================================
    *ex*        Excitation wavelength vector
    *em*        Emission wavelength vector
    *eem_1*     First EEM to plot (scale is based on maximum of this eem)
    *eem_2*     Second EEM to plot
    *n*         number of contour lines
    ==========  ================================================
    '''

    # Choose countour levels based on eem_1
    level = np.linspace(eem_1.min(), eem_1.max(), n)
    print(level)

    fig = plt.figure(figsize=(11, 5))
    gs = gridspec.GridSpec(1, 2, width_ratios=[1, 1.2])
    ax0 = plt.subplot(gs[0])

    plt0 = ax0.contour(ex, em, eem_1, n, levels=level)  # plots contour plot
    plt.title('')
    plt.xlabel('ex '+r'$\lambda$')  # label axis
    plt.ylabel('em '+r'$\lambda$')
    fig.colorbar(plt0, ax=ax0)

    ax1 = plt.subplot(gs[1])
    plt1 = ax1.contour(ex, em, eem_2, n, levels=level)  # plots contour plot
    plt.title('')
    plt.xlabel('ex '+r'$\lambda$')  # label axis
    plt.ylabel('em '+r'$\lambda$')
    fig.colorbar(plt1, ax=ax1)

    return

# def fingerprint_3D_plot(ex, em, z):
#     '''Create a 3D contour plot with contour and finger prints.

#     ==========  ================================================
#     Input    Description
#     ==========  ================================================
#     *ex*        Excitation wavelength as meshgrid of numpy.arrays
#     *em*        Emission wavelength as meshgrid of numpy.arrays
#     *z*         Intensity as meshgrid of numpy.arrays
#     ==========  ================================================'''
#     import plotly.plotly as py
#     import plotly.offline as offline
#     import plotly.graph_objs as go
#     from scipy.ndimage import gaussian_filter
#     offline.init_notebook_mode()

#     from mpl_toolkits.mplot3d import axes3d
#     import matplotlib.pyplot as plt
#     from matplotlib import cm

#     # set the figure size and the 3D axis
#     fig = plt.figure(figsize=(10, 10))
#     ax = fig.gca(projection='3d')

#     assert type(ex) == type(np.array([1])), 'ex should be np array'
#     assert type(em) == type(np.array([1])), 'em should be np array'
#     assert type(z) == type(np.array([1])), 'ez should be np array'
# #     assert len(np.shape(ex))==2, 'shape is not the right dimensions'
# #     assert np.shape(ex)==np.shape(em), 'shape of ex and em are not the same'
# #     assert np.shape(ex)==np.shape(z), 'shape of z and ex are not the same'

#     # name the variables for the function
#     X = em
#     Y = ex
#     Z = z

#     fig.suptitle('3D Surface Plot with 2D spectra and Contour', fontsize=20)

#     # Surface plot and the csets for the 2D spectra and contour on the wall
#     surf = ax.plot_surface(X, Y, Z, alpha=1, cmap=cm.coolwarm)
#     cset = ax.contour(X, Y, Z, zdir='z', offset=Z.max(), # these add wall plots
#                       cmap=cm.coolwarm)  # extend3d=True)
#     cset = ax.contour(X, Y, Z, zdir='x', offset=250, cmap=cm.cool)
#     #cset = ax.contour(X, Y, Z, zdir='y', offset=500, cmap=cm.spring)

#     fig.colorbar(surf, shrink=0.7, aspect=15,)

#     ax.set_ylabel('Excitation '+r'$\lambda$')
#     ax.set_ylim(Y.max(), Y.min())
#     ax.set_xlabel('Emission '+r'$\lambda$')
#     ax.set_xlim(X.min(), X.max())
#     ax.set_zlabel('Intensity')
#     ax.set_zlim(Z.min(), Z.max())
#     return

# def Interactive_3D_Plot(ex, em, z):

#     '''Create an interactive 3D contour plot.

#     ==========  ================================================
#     Input    Description
#     ==========  ================================================
#     *ex*        Excitation wavelength as meshgrid of numpy.arrays
#     *em*        Emission wavelength as meshgrid of numpy.arrays
#     *z*         Intensity as meshgrid of numpy.arrays
#     ==========  ================================================'''
#     import plotly.plotly as py
#     import plotly.offline as offline
#     import plotly.graph_objs as go

#     offline.init_notebook_mode()
#     from scipy.ndimage import gaussian_filter

#     assert type(ex) == type(np.array([1])), 'ex should be np array'
#     assert type(em) == type(np.array([1])), 'em should be np array'
#     assert type(z) == type(np.array([1])), 'ez should be np array'
# #     assert len(np.shape(ex))==2, 'shape is not the right dimensions'
# #     assert np.shape(ex)==np.shape(em), 'shape of ex and em are not the same'
# #     assert np.shape(ex)==np.shape(z), 'shape of z and ex are not the same'

#     surface = Surface(x=em, y=ex, z=z)
#     data = Data([surface])

#     layout = Layout(
#         title='3D Intensity Plot',
#         scene=Scene(
#             aspectmode="manual",
#             aspectratio=dict(x=1, y=1, z=1),

#             xaxis=dict(
#                 title='emission',
#                 gridcolor='rgb(255, 255, 255)',
#                 zerolinecolor='rgb(255, 255, 255)',
#                 showbackground=True,
#                 backgroundcolor='rgb(230, 230,230)'
#             ),
#             yaxis=YAxis(
#                 title='excitation',
#                 gridcolor='rgb(255, 255, 255)',
#                 zerolinecolor='rgb(255, 255, 255)',
#                 showbackground=True,
#                 backgroundcolor='rgb(230, 230,230)'
#             ),
#             zaxis=ZAxis(
#                 title='Intensity',
#                 gridcolor='rgb(255, 255, 255)',
#                 zerolinecolor='rgb(255, 255, 255)',
#                 showbackground=True,
#                 backgroundcolor='rgb(230, 230,230)'
#             )
#         )
#     )

#     fig = Figure(data=data, layout=layout)
#     offline.iplot(fig)  # , image='png')
#     return

max_stars_repo_name: jayruth/pyeem

max_issues_repo_name: jayruth/pyeem

max_forks_repo_name: jayruth/pyeem


text:
import numpy as np
import math

def unit(vector):
    return vector / np.linalg.norm(vector)

def close(p1,p2):
    return np.allclose(p1,p2,atol=0.1)

class Filmstrip(object):
    def __init__(self,face_obj):
        self.faces = [face_obj]
        self.pts = list(face_obj.coords)

    def touches(self,face_obj):
        face = face_obj.coords
        return close(self.pts[0],face[0]) or close(self.pts[0],face[-1]) or close(self.pts[-1],face[0]) or close(self.pts[-1],face[-1])

    def add_touching_face(self,face_obj):
        face = face_obj.coords

        if close(self.pts[0],face[0]):
            self.pts = list(face[::-1][:-1]) + self.pts
            self.faces.insert(0,face_obj)
        elif close(self.pts[0],face[-1]):
            self.pts = list(face[:-1]) + self.pts
            self.faces.insert(0,face_obj)
        elif close(self.pts[-1],face[0]):
            self.pts = self.pts + list(face[1:])
            self.faces.append(face_obj)
        elif close(self.pts[-1],face[-1]):
            self.pts = self.pts + list(face[:-1][::-1])
            self.faces.append(face_obj)
        else:
            raise Exception

    def point_is_left(self,centroid):
        if self.pts[0][0] < self.pts[-1][0]:
            left_point = self.pts[0]
            right_point = self.pts[-1]
            reverse = False
        else:
            left_point = self.pts[-1]
            right_point = self.pts[0]
            reverse = True
        c = centroid.coords[0]
        # find the left-most point of the segment
        d1 = (c[0] - left_point[0])*(right_point[1]-left_point[1]) - (c[1]-left_point[1])*(right_point[0]-left_point[0])
        # find the sign of a point guaranteed on the left
        test_point = (left_point[0], left_point[1] + 1)
        d2 = (test_point[0] - left_point[0])*(right_point[1]-left_point[1]) - (test_point[1]-left_point[1])*(right_point[0]-left_point[0])
        return d1 * d2 > 0

    # precondition: face touches current first or last point
    def join_angle(self,face_obj):
        face = face_obj.coords
        if close(self.pts[0],face[0]):
            vec0 = self.pts[1] - self.pts[0]
            vec1 = face[1] - face[0]
        elif close(self.pts[0],face[-1]):
            vec0 = self.pts[1] - self.pts[0]
            vec1 = face[-2] - face[-1]
        elif close(self.pts[-1],face[0]):
            vec0 = self.pts[-2] - self.pts[-1]
            vec1 = face[1] - face[0]
        elif close(self.pts[-1],face[-1]):
            vec0 = self.pts[-2] - self.pts[-1]
            vec1 = face[-2] - face[-1]
        else:
            raise Exception
        return np.arccos(np.clip(np.dot(unit(vec0),unit(vec1)),-1.0,1.0)) * 180 / math.pi

    def __repr__(self):
        return 'filmstrip ' + str(self.faces)

class StripMaker(object):
    def __init__(self,faces):
        face = faces.pop(0)
        faces_seen = 0
        strips = [Filmstrip(face)]
        while len(faces) != 0:
            # for every filmstrip, find any touching faces left
            face = faces.pop(0)
            matched = False
            for strip in strips:
                if strip.touches(face):
                    join_angle = strip.join_angle(face)
                    if join_angle > 90 + 15 and strip.join_angle(face) < 270 - 15:
                        matched = True
                        strip.add_touching_face(face)
                        faces_seen = 0
                        break

            if not matched:
                # if a face reaches the front of the list again without the length changing,
                # it is an orphan, make it a new filmstrip
                if len(faces)+1 == faces_seen:
                    strips.append(Filmstrip(face))
                    faces_seen = 0
                else:
                    faces_seen = faces_seen + 1
                    # put it at the back of the list
                    faces.append(face)
        self._strips = strips

    @property
    def strips(self):
        return self._strips

    def pruned_strips(self):
        def lots_then_length(x):
            return (len(x.faces),sum(f.length for f in x.faces))
        # remove any strips that are subsets of other strips
        # order the strips by the # of lots, and then by length
        sorted(self._strips, key=lots_then_length)

        pruned = []
        seen_lots = set()
        for strip in self._strips:
            these_lots = set([f.lot_id for f in strip.faces])
            if these_lots.issubset(seen_lots):
                pass
            else:
                pruned.append(strip)
                seen_lots.update(these_lots)

        return pruned


text:
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import unittest
import numpy as np
import paddle
from paddle.fluid.framework import _test_eager_guard

class TestSparseActivation(unittest.TestCase):
    def test_sparse_relu(self):
        with _test_eager_guard():
            x = [[0, -1, 0, 2], [0, 0, -3, 0], [4, 5, 0, 0]]
            dense_x = paddle.to_tensor(x, dtype='float32', stop_gradient=False)
            sparse_dim = 2
            sparse_x = dense_x.to_sparse_coo(sparse_dim)
            sparse_relu = paddle.sparse.ReLU()
            sparse_out = sparse_relu(sparse_x)
            dense_relu = paddle.nn.ReLU()
            #TODO: replace non_zero_elements() as values()
            dense_out = dense_relu(sparse_x.non_zero_elements())
            actual_result = sparse_out.non_zero_elements().numpy()
            assert np.array_equal(dense_out.numpy(), actual_result)
            dense_out.backward(dense_out)
            sparse_out.backward(sparse_out)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: LiYuRio/Paddle

max_issues_repo_name: LiYuRio/Paddle

max_forks_repo_name: LiYuRio/Paddle