text:
import pyvisa  # Keithley Module
import serial  # Arduino Module
import seabreeze

seabreeze.use("pyseabreeze")
import seabreeze.spectrometers as sb  # MayaLSL Modules for Ocean Spectrometer

import core_functions as cf

import thorlabs_apt as apt  # thorlabs apt for thorlabs motor

import sys
import time
import logging
import re
import numpy as np
import math
import copy

from PySide2 import QtCore, QtGui, QtWidgets

class ArduinoUno:
    """
    Class that manages all functionality of our arduino uno
    """

    def __init__(self, com_address):
        # Define a mutex
        self.mutex = QtCore.QMutex(QtCore.QMutex.Recursive)

        # Check for devices on the pc
        rm = pyvisa.ResourceManager()
        # The actual addresses for the Keithleys can be accessed via rm.list_resources()
        visa_resources = rm.list_resources()

        # Open COM port to Arduino
        if com_address not in visa_resources:
            cf.log_message(
                "The Arduino Uno seems to be missing. Try to reconnect to computer."
            )
            # self.queue.put(
            # "Arduino Uno seems to be missing."
            # + "\nPlease connect Arduino Uno to computer and try again."
            # )
            sys.exit()

        # Instead of letting the user explicitly define the COM port after he
        # already defined the com_address, search for the number in the
        # com_address to construct the right string
        uno_port = "COM" + re.findall(r"\d+", com_address)[0]

        # assign name to Arduino and assign short timeout to be able to do things fast
        self.uno = serial.Serial(uno_port, timeout=0.01)

        # Try to open the serial connection
        try:
            self.init_serial_connection()
        except serial.SerialException:
            # If the serial connection was already established, close it again and open it again
            try:
                self.uno.close()
                self.init_serial_connection()
            except IOError:
                cf.log_message(
                    "COM port to Arduino Uno already open. Reconnect arduino manually and try again."
                )
                # self.queue.put(
                # "COM2 port to Arduino Uno already open."
                # + "\nTry 'uno.close()' in your console or restarting IPython."
                # )
                sys.exit()

        # cf.log_message("Arduino successfully initiated")

    def init_serial_connection(self, wait=1):
        """
        Private function
        Initialise serial connection to com.

        com: func
            specify COM port. Needs to be opened prior to calling this function.
            e.g.:
                > uno = serial.Serial(2, timeout=0.2)
                > uno_init(com=uno)
        wait: flt
            time in seconds to wait before collecting initialisation message.
        """

        self.mutex.lock()

        # Open serial port
        try:
            self.uno.open()
        except serial.SerialException:
            # If port was already open, we do not have to open it obviously.
            cf.log_message("Arduino port was already open")

        # Wait for a defined period of time
        time.sleep(wait)

        # Read serial port result
        cf.log_message(
            "Arduino serial port successfully initialised with "
            + str(self.uno.readall())
        )
        # self.queue.put(com.readall())
        self.serial_connection_open = True
        self.mutex.unlock()

    def close_serial_connection(self):
        """
        Close connection to arduino
        """
        self.mutex.lock()
        self.uno.close()
        self.serial_connection_open = False
        self.mutex.unlock()

    def trigger_relay(self, relay):
        """
        Open or close a relay via COM on an Arduino with a 8-Relays shield.

        relay: int
            If relay == [1-8], the according relay opens or closes
            If relay == 0, all relays close
            If relay == 9, all relays open
        """
        self.mutex.lock()
        com = self.uno

        if self.serial_connection_open == False:
            self.init_serial_connection()

        # If the number is in the range [0, 9] the command is correct
        if relay not in np.arange(0, 10, 1):
            cf.log_message("Unknown arduino serial communication command")
        else:
            try:
                com.write(str.encode(str(relay)))
            except serial.SerialException:
                cf.log_message(
                    "Serial connection not established. Please make sure to do so before attempting to trigger a relay."
                )

        cf.log_message(com.readall())
        self.mutex.unlock()

    def close(self):
        """
        Function that is called before program is closed to make sure that
        all relays are closed
        """
        self.mutex.lock()
        self.trigger_relay(0)
        time.sleep(1)
        self.close_serial_connection()
        self.mutex.unlock()

class KeithleySource:
    """
    Class that manages all functionality of our Keithley voltage/current source
    """

    def __init__(self, keithley_source_address, current_compliance):
        """
        Initialise Hardware. This function must be improved later as well.
        For the time being it is probably alright.
        """
        # Define a mutex
        self.mutex = QtCore.QMutex(QtCore.QMutex.Recursive)

        # Keithley Finding Device
        rm = pyvisa.ResourceManager()
        # The actual addresses for the Keithleys can be accessed via rm.list_resources()
        visa_resources = rm.list_resources()

        # Check if keithley source is present at the given address
        if keithley_source_address not in visa_resources:
            cf.log_message("The SourceMeter seems to be absent or switched off.")
            raise IOError("The SourceMeter seems to be absent or switched off.")

        self.keith = rm.open_resource(keithley_source_address)

        # As a standard initialise the Keithley as a voltage source
        self.as_voltage_source(current_compliance)

        # Set voltage mode indicator
        self.mode = "voltage"

        # Reverse voltages
        self.reverse = 1

    def as_voltage_source(self, current_compliance):
        """
        Function that initalises the Keithley as a voltage source
        """
        self.mutex.lock()
        # Write operational parameters to Sourcemeter (Voltage to OLED)
        self.reset()
        # set voltage as source
        self.keith.write("Source:Function Volt")
        # choose current for measuring
        self.keith.write('Sense:Function "Current"')
        # set compliance
        self.keith.write("Source:Volt:ILimit " + str(current_compliance * 1e-3))
        # reads back the set voltage
        self.keith.write("Source:Volt:READ:BACK ON")
        # sets the read-out speed and accuracy (0.01 fastest, 10 slowest but highest accuracy)
        self.keith.write("Current:NPLCycles 1")
        self.keith.write("Current:AZero OFF")  # turn off autozero
        self.keith.write("Source:Volt:Delay:AUTO OFF")  # turn off autodelay

        # Set voltage mode indicator
        self.mode = "voltage"
        self.mutex.unlock()

    def as_current_source(self, voltage_compliance):
        """
        Initialise (or reinitialise) class as current source
        """
        self.mutex.lock()
        # Write operational parameters to Sourcemeter (Voltage to OLED)
        self.reset()
        # Write operational parameters to Sourcemeter (Current to OLED)
        self.keith.write("Source:Function Current")  # set current as source

        self.keith.write('Sense:Function "Volt"')  # choose voltage for measuring
        self.keith.write(
            "Source:Current:VLimit " + str(voltage_compliance)
        )  # set voltage compliance to compliance
        self.keith.write(
            "Source:Current:READ:BACK OFF"
        )  # record preset source value instead of measuring it anew. NO CURRENT IS MEASURED!!! (Costs approx. 1.5 ms)
        self.keith.write("Volt:AZero OFF")  # turn off autozero
        self.keith.write("Source:Current:Delay:AUTO OFF")  # turn off autodelay

        # Set current mode indicator
        self.mode = "current"
        self.mutex.unlock()

    def reset(self):
        """
        reset instrument
        """
        self.mutex.lock()
        self.keith.write("*rst")
        self.keith.write("Source:Volt:ILimit " + str(1.05))
        self.mutex.unlock()

    def init_buffer(self, buffer_name, buffer_length):
        """
        Initialise buffer of source meter
        """
        self.mutex.lock()
        # if the buffer already exists, delete it first to prevent the error
        # "parameter error TRACe:MAKE cannot use an existing reading buffer name keithley"
        # try:
        # self.keith.write('TRACe:DELete "' + buffer_name + '"')
        # except:
        # cf.log_message("Buffer " + buffer_name + " does not exist yet")
        self.keith.write(
            'Trace:Make "' + buffer_name + '", ' + str(max(buffer_length, 10))
        )

        # Keithley empties the buffer
        self.keith.write("Trace:Clear " + '"' + buffer_name + '"')
        self.buffer_name = buffer_name
        self.mutex.unlock()

    def empty_buffer(self, buffer_name):
        """
        Function that empties the Keithley's buffer for the next run
        """
        self.mutex.lock()
        self.keith.write("Trace:Clear " + '"' + buffer_name + '"')
        self.mutex.unlock()

    def activate_output(self):
        """
        Activate output
        """
        self.mutex.lock()
        self.keith.write("Output ON")
        self.mutex.unlock()

    def deactivate_output(self):
        """
        Turn power off
        """
        self.mutex.lock()
        self.keith.write("Output OFF")
        self.mutex.unlock()

    def read_current(self):
        """
        Read current on Keithley source meter
        """
        return self.reverse * float(self.keith.query("MEASure:CURRent:DC?"))

    def read_voltage(self):
        """
        Read voltage on Keithley source meter
        """
        return self.reverse * float(self.keith.query("MEASure:VOLTage:DC?"))

    def read_buffer(self, buffer_name):
        return float(self.keith.query('Read? "' + buffer_name + '"')[:-1])

    def set_voltage(self, voltage):
        """
        Set the voltage on the source meter (only in voltage mode)
        """

        self.mutex.lock()
        voltage = float(voltage)
        if self.mode == "voltage":
            self.keith.write("Source:Volt " + str(self.reverse * voltage))
        else:
            logging.warning(
                "You can not set the voltage of the Keithley source in current mode"
            )
        self.mutex.unlock()

    def set_current(self, current):
        """
        Set the current on the source meter (only in current mode)
        """
        self.mutex.lock()
        current = float(current)
        # set current to source_value
        if self.mode == "current":
            self.keith.write("Source:Current " + str(self.reverse * current * 1e-3))
        else:
            logging.warning(
                "You can not set the current of the Keithley source in voltage mode"
            )
        self.mutex.unlock()

class KeithleyMultimeter:
    """
    Class that manages all functionality of our Keithley multi meter
    """

    def __init__(self, keithley_multimeter_address):
        # Define a mutex
        self.mutex = QtCore.QMutex(QtCore.QMutex.NonRecursive)

        # Keithley Finding Device
        rm = pyvisa.ResourceManager()
        # The actual addresses for the Keithleys can be accessed via rm.list_resources()
        visa_resources = rm.list_resources()

        if keithley_multimeter_address not in visa_resources:
            cf.log_message("The Multimeter seems to be absent or switched off.")
            raise IOError("The Multimeter seems to be absent or switched off.")

        self.keithmulti = rm.open_resource(keithley_multimeter_address)

        # Write operational parameters to Multimeter (Voltage from Photodiode)
        # reset instrument
        self.reset()

    def reset(self):
        """
        Reset instrument
        """
        self.mutex.lock()
        self.keithmulti.write("*rst")

        # Write operational parameters to Multimeter (Voltage from Photodiode)
        # sets the voltage range
        self.keithmulti.write("SENSe:VOLTage:DC:RANGe 10")
        # sets the voltage resolution
        self.keithmulti.query("VOLTage:DC:RESolution?")
        # sets the read-out speed and accuracy (0.01 fastest, 10 slowest but highest accuracy)
        self.keithmulti.write("VOLTage:NPLCycles 1")
        # sets the trigger to activate immediately after 'idle' -> 'wait-for-trigger'
        self.keithmulti.write("TRIGer:SOURce BUS")
        # sets the trigger to activate immediately after 'idle' -> 'wait-for-trigger'
        self.keithmulti.write("TRIGer:DELay 0")
        # Activate wait for trigger mode
        self.keithmulti.write("INITiate")
        self.mutex.unlock()

    # def set_fixed_range(self, value):
    #     """
    #     Sets a fixed voltage range if the user selected so
    #     """
    #     # Turn off the auto range function of the multimeter
    #     self.keithmulti.write("SENSe:VOLTage:DC:RANGe:AUTO OFF")
    #     # Set the range of the multimeter to a fixed value
    #     self.keithmulti.write("CONF:VOLTage:DC:RANGe " + str(value))

    # def set_auto_range(self):
    #     """
    #     Sets automatic detection of the multimeter range
    #     """
    #     # Turn on the auto range function of the multimeter
    #     self.keithmulti.write("SENSe:VOLTage:DC:RANGe:AUTO ON")

    def measure_voltage(self, multimeter_range=0):
        """
        Returns an actual voltage reading on the keithley multimeter
        """
        if multimeter_range == 0:
            return float(self.keithmulti.query("MEASure:VOLTage:DC?"))
        else:
            return float(
                self.keithmulti.query("MEASure:VOLTage:DC? " + str(multimeter_range))
            )

class OceanSpectrometer:
    """
    Class to deal with the ocean spectrometer
    """

    def __init__(self, non_linearity_correction):
        # Define a mutex
        self.mutex = QtCore.QMutex(QtCore.QMutex.NonRecursive)
        # List all spectrometers
        maya_devices = sb.list_devices()

        # Select our spectrometer (probably only in the list)
        self.spectrometer = sb.Spectrometer(maya_devices[0])

        # Set integration time of spectrometer, doesn't really matter since it
        # is changed in the software on the fly anyways
        self.set_integration_time_ms(100)

        # Check if one shall correct for non-linearity
        self.non_linearity_correction = non_linearity_correction

    def measure(self):
        """
        Function to measure spectrum. The data structure might be already
        altered in this function (this I have to see on the hardware device)
        """
        self.mutex.lock()
        # The following yields lists of wavelength and intensity
        # wavelength = self.spec.wavelengths()
        # intensity = self.spec.intensities()
        data = self.spectrometer.spectrum(
            correct_nonlinearity=self.non_linearity_correction
        )

        self.mutex.unlock()
        time.sleep(1)

        return data

    def set_integration_time_ms(self, integration_time):
        """
        Function that allows the user to set the integration time even after
        initialisation of the spectrometer
        """
        # Integration time is a string when it is read from the global settings
        # file. To make sure that it is converted correctly, convert it first
        # to a float, multiply by 1000 to convert to us and then convert to int
        # as required by the api
        self.mutex.lock()
        self.integration_time = int(float(integration_time) * 1000)
        self.spectrometer.integration_time_micros(self.integration_time)
        cf.log_message(
            "Spectrometer integration time set to " + str(integration_time) + " ms"
        )
        self.mutex.unlock()

    def close_connection(self):
        """
        Closes connection to spectrometer
        """

        self.mutex.lock()
        self.spectrometer.close()
        self.mutex.unlock()

class ThorlabMotor:
    """
    Class to control the thorlab motors
    """

    # update_animation = QtCore.Signal(float)
    update_progress_bar = QtCore.Signal(str, float)

    def __init__(self, motor_run, motor_number, offset_angle, motor_speed, main_widget):
        # Define a mutex
        self.mutex = QtCore.QMutex(QtCore.QMutex.Recursive)

        # First cleanup before we can work with things (otherwise there might be
        # an open connection somewhere)
        self.clean_up()
        time.sleep(5)

        # from stage.motor_ini.core import find_stages

        # s = list(find_stages())
        # Set the motor to the number
        self.motor = apt.Motor(int(motor_number))
        time.sleep(1)
        # velocity MUST be set to avoid the motor moving slowly
        self.motor.set_velocity_parameters(0, 9, motor_speed)
        # ensures that the motor homes properly - home in reverse with reverse lim switches
        self.motor.set_hardware_limit_switches(5, 5)
        self.motor.set_move_home_parameters(2, 1, 10, 3)
        # Move the motor home first, so that we can work with absolute positions
        # self.motor.move_home(True)

        # self.update_animation.connect(main_widget.gw_animation.move)
        self.main_widget = main_widget
        self.motor_run = motor_run
        self.offset_angle = offset_angle
        self.top_emitting_reverse_angles = 1
        self.motor_run.offset_angle = self.offset_angle

    def change_velocity(self, velocity):
        """
        Change velocity of motor
        """
        if velocity <= 20 and velocity >= 0:
            self.motor.set_velocity_parameters(0, 9, velocity)
        else:
            cf.log_message("Motor velocity is exceeding its limits")

    def move_to(self, angle, blocking=False):
        """
        Call the move_to function of the apt package
        """
        self.mutex.lock()

        angle_corrected = angle * self.top_emitting_reverse_angles

        if math.isclose(abs(self.read_position()), 180, abs_tol=50):
            # Only if the current position is close to 180, allow crossing the 180°
            # otherwise go the full turn back
            self.motor_run.helper_angle = False
            self.motor.move_to((angle_corrected - float(self.offset_angle)) % 360)
        elif np.sign(np.round(self.read_position(), 2)) != np.sign(angle):
            # In case of a different sign, use a helper angle to make sure that
            # the zero is always crossed and the program does not take the
            # shortest way (friendlier for flexcable)
            self.motor_run.helper_angle = True
            self.motor.move_to((0 - float(self.offset_angle)) % 360)
        else:
            self.motor_run.helper_angle = False
            self.motor.move_to((angle_corrected - float(self.offset_angle)) % 360)

        # This here must not be angle_corrected
        self.motor_run.angle = angle

        self.motor_run.start()
        self.mutex.unlock()

    def change_offset_angle(self, offset_angle, relative=True):
        """
        Changes offset angle
        """
        if relative:
            self.offset_angle += offset_angle
            self.motor_run.offset_angle = self.offset_angle
        else:
            self.offset_angle = offset_angle
            self.motor_run.offset_angle = self.offset_angle

    def read_position(self):
        """
        Function that reads out the current motor position
        """
        self.mutex.lock()
        # Make sure that the motor position is returned as values between -180 to 180 (definition)
        if (self.motor.position + float(self.offset_angle)) % 360 > 180:

            motor_position_translated = (
                self.motor.position + float(self.offset_angle)
            ) % 360 - 360
        else:
            motor_position_translated = (
                self.motor.position + float(self.offset_angle)
            ) % 360

        motor_position_translated = (
            motor_position_translated * self.top_emitting_reverse_angles
        )
        self.mutex.unlock()
        return motor_position_translated

    def clean_up(self):
        """
        This solution is a little bit hacky but works. This is copied code
        from the thorlabs_api repo and enables controlled cleaning up of the
        connection to the motor when the program is closed.
        """
        self.mutex.lock()
        import ctypes
        import os
        from thorlabs_apt import _APTAPI

        if os.name != "nt":
            raise Exception(
                "Your operating system is not supported. "
                "Thorlabs' APT API only works on Windows."
            )
        lib = None
        filename = ctypes.util.find_library("APT")
        if filename is not None:
            lib = ctypes.windll.LoadLibrary(filename)
        else:
            filename = "%s/APT.dll" % os.path.dirname(__file__)
            lib = ctypes.windll.LoadLibrary(filename)
            if lib is None:
                filename = "%s/APT.dll" % os.path.dirname(sys.argv[0])
                lib = ctypes.windll.LoadLibrary(lib)
                if lib is None:
                    raise Exception("Could not find shared library APT.dll.")
        _APTAPI.set_ctypes_argtypes(lib)
        err_code = lib.APTInit()
        if err_code != 0:
            raise Exception("Thorlabs APT Initialization failed.")
        if lib.EnableEventDlg(False) != 0:
            raise Exception("Couldn't disable event dialog.")

        if lib is not None:
            lib.APTCleanUp()
            lib.APTInit()
        self.mutex.unlock()

class MotorMoveThread(QtCore.QThread):
    """
    QThread that shall do all the work for the current tester in the setup tab.
    It is mainly needed for updating the current reading. However, I figure
    at the moment, that it is probably also good to have the other current
    tester functionalities in this class. Mainly because it does not make
    sense to return a current reading while the keithley has to be resetted,
    for instance. So it is not only okay but kind of demanded to have this in
    a single thread.
    """

    # Define costum signals
    # https://stackoverflow.com/questions/36434706/pyqt-proper-use-of-emit-and-pyqtsignal
    # With pyside2 https://wiki.qt.io/Qt_for_Python_Signals_and_Slots
    # read_motor_position_signal = QtCore.Signal()
    update_animation_signal = QtCore.Signal(float)
    update_progress_bar_signal = QtCore.Signal(str, float)

    def __init__(
        self,
        angle,
        offset_angle,
        helper_angle,
        main_widget,
    ):

        super(MotorMoveThread, self).__init__()

        self.is_killed = False

        self.angle = angle
        self.offset_angle = offset_angle
        self.helper_angle = helper_angle
        self.main_widget = main_widget

        # Reset Arduino and Keithley
        # Connect signal to the updater from the parent class
        # self.read_motor_position_signal.connect(motor.read_position)
        self.update_animation_signal.connect(main_widget.gw_animation.move)
        self.update_progress_bar_signal.connect(main_widget.progressBar.setProperty)

    def run(self):
        """
        Class that continuously reads the current on the Keithley source and
        communicates with the main class. It has to be kept in separate
        classes to allow for threading. It is started with the .start()
        method from the QThread class
        """
        import pydevd

        pydevd.settrace(suspend=False)
        motor = self.main_widget.motor

        # self.show_progress_bar_signal.emit()
        self.update_progress_bar_signal.emit("value", 0)

        # Instead of defining a homeing time, just read the motor position and
        # only start the measurement when the motor is at the right position
        motor_position = motor.read_position()
        initial_position = copy.copy(motor_position)

        while not math.isclose(motor_position, self.angle, abs_tol=0.01):
            if self.helper_angle == True:
                if math.isclose(motor_position, 0, abs_tol=20):
                    motor.motor.move_to(
                        motor.top_emitting_reverse_angles * self.angle
                        - float(self.offset_angle)
                    )
                    self.helper_angle = False

            motor_position = motor.read_position()
            self.update_animation_signal.emit(motor_position)
            self.update_progress_bar_signal.emit(
                "value",
                int(abs(motor_position - initial_position))
                / max(int(abs(initial_position - self.angle)), 1)
                * 100,
            )
            time.sleep(0.1)

            if self.is_killed:
                self.quit()
                break

        # Update animation once more since the position might be 0.9 at this
        # point (int comparison in the above while loop)
        self.update_animation_signal.emit(motor_position)

        self.update_progress_bar_signal.emit("value", 100)
        # self.hide_progress_bar_signal.emit()

        cf.log_message("Motor moved to " + str(self.angle) + "° angle.")

    def kill(self):
        """
        Kill this thread by stopping the loop
        """

        # Trigger interruption of run sequence
        self.is_killed = True

max_stars_repo_name: GatherLab/OLED-jvl-measurement

max_issues_repo_name: GatherLab/OLED-jvl-measurement

max_forks_repo_name: GatherLab/OLED-jvl-measurement


text:
import numpy
import matplotlib.pyplot as plot
import scipy.signal

path = "/Users/borja/Documents/ResearchStay/MobileBIT/Logs/"
# myPath=path+"2014-05-20_10-18-27.txt"
myPath = path+"myECGraw.txt"

data = numpy.loadtxt(myPath, 'float')

maxLen = 3000 # length of the data

#noisy sequence:
nSeq = data[:,1]

#### LOW PASS FILTER #####

a = [1, -2, 1]
b = [1, 0, 0,  -2,   0, 0, 1]
xVal_p = scipy.signal.lfilter(b,[1],data[:,1],-1,None)
yVal_p = scipy.signal.lfilter(b,a,data[:,1],-1,None)

# Real time

xSeq = data[0:maxLen,1]
xVal_r = [xSeq[x] for x in range(0, 5, 1)]
xVal_r.extend([xSeq[x] - 2*xSeq[x-5] for x in range(5, 10, 1)])

for x in range(10, maxLen):
    aux = (2*xSeq[x-5]) - xSeq[x-10]
    xVal_r.insert(x, xSeq[x] - aux)

yVal_r = [xVal_r[0]]
aux = (2*yVal_r[0]) + xVal_r[1]
yVal_r.insert(1, aux )

for x in range(2, maxLen):
    aux = (2*yVal_r[x-1]) - yVal_r[x-2]
    yVal_r.insert(x,(aux + xVal_r[x]))

##### SlopeSumFunction #####

yDelta = numpy.diff(yVal_p)
uDelta = []
for x in range(0, maxLen-1):
  if yDelta[x] > 0:
    uDelta.insert(x, yDelta[x])
  else:
    uDelta.insert(x, 0)
uDelta.insert(x, 0)
#print "uDelta len: " + str(len(uDelta))

w = 6
zVal=[0,0,0,0,0,0,0,0]
for i in range(w+1, maxLen):
  #print "i: "+ str(i)
  sum_k = 0
  for k in range(i-w-1,i):
    #print "k: "+ str(k)
    sum_k += uDelta[k]
    #print "sum: "+str(sum_k)
  zVal.insert(i,sum_k)

print (zVal[maxLen])

### Decission Rule ####
##### Adaptive thresholding #####
num_seconds = 2
num_frames = 10*100*num_seconds
print num_frames
t_base = numpy.mean(zVal[0:num_frames])
print t_base

##### Local search #####

##### PLOTTING ####
plot.figure()
#plot.subplot(3,1,1)
#plot.plot(nSeq[0:maxLen])
#plot.subplot(3,1,2)
#plot.plot(yVal_p[0:maxLen])
#plot.subplot(3,1,3)
#plot.plot(zVal[0:maxLen])
plot.plot(zVal)

plot.show()

ext: py

reason: import numpy,import scipy


text:
"""
Returns the array of random variates for a probability distribution
assigned to positions corresponding to defined values in an input array.
"""

from __future__ import print_function

import numpy
import scipy.stats
import pyferret
import pyferret.stats

def ferret_init(id):
    """
    Initialization for the stats_rvs python-backed ferret external function
    """
    axes_values = [ pyferret.AXIS_IMPLIED_BY_ARGS ] * pyferret.MAX_FERRET_NDIM
    true_influences = [ True ] * pyferret.MAX_FERRET_NDIM
    false_influences = [ False ] * pyferret.MAX_FERRET_NDIM
    retdict = { "numargs": 3,
                "descript": "Returns random variates for a probability distribution",
                "axes": axes_values,
                "argnames": ("TEMPLATE", "PDNAME", "PDPARAMS"),
                "argdescripts": ("Template array for the array of random variates to be returned",
                                 "Name of a probability distribution",
                                 "Parameters for this probability distribution"),
                "argtypes": (pyferret.FLOAT_ARRAY, pyferret.STRING_ONEVAL, pyferret.FLOAT_ARRAY),
                "influences": (true_influences, false_influences, false_influences),
              }
    return retdict

def ferret_compute(id, result, resbdf, inputs, inpbdfs):
    """
    Assigns result with random variates of the probability distribution
    indicated by inputs[1] (a string) using the parameters given in
    inputs[2].  Random variates will be assigned to positions corresponding
    to defined positions in inputs[0].  For positions where the inputs[0]
    value is undefined, the result value will be undefined.
    """
    distribname = inputs[1]
    distribparams = inputs[2].reshape(-1)
    distrib = pyferret.stats.getdistrib(distribname, distribparams)
    badmask = ( numpy.fabs(inputs[0] - inpbdfs[0]) < 1.0E-5 )
    badmask = numpy.logical_or(badmask, numpy.isnan(inputs[0]))
    goodmask = numpy.logical_not(badmask)
    result[badmask] = resbdf
    # result[goodmask] is a flattened array
    result[goodmask] = distrib.rvs(len(result[goodmask]))

#
# The rest of this is just for testing this module at the command line
#
if __name__ == "__main__":
    # make sure ferret_init does not have problems
    info = ferret_init(0)

    # Get a large sample from a normal distribution with small variance
    # so the mean and stdev will be very close
    mu = 5.0
    sigma = 0.5
    # probability of randomly choosing -65536.0 is essentially zero
    undefval = -65536.0
    xdim = 19
    ydim = 21
    zdim = 17
    tdim = 23

    pfname = "norm"
    pfparams = numpy.array([mu, sigma], dtype=numpy.float64)
    inpbdfs = numpy.array([-1.0, 0.0, 0.0], dtype=numpy.float64)
    resbdf = numpy.array([undefval], dtype=numpy.float64)
    # template initialized to all zero != impbdfs[0]
    template = numpy.zeros((xdim, ydim, zdim, tdim, 1, 1), dtype=numpy.float64, order='F')
    expectedgood = numpy.empty((xdim, ydim, zdim, tdim, 1, 1), dtype=bool, order='F')
    index = 0
    for i in range(xdim):
        for j in range(ydim):
            for k in range(zdim):
                for l in range(tdim):
                    if (index % 53) == 1:
                        template[i, j, k, l, 0, 0] = inpbdfs[0]
                        expectedgood[i, j, k, l, 0, 0] = False
                    else:
                        expectedgood[i, j, k, l, 0, 0] = True
                    index += 1
    result = -8888.0 * numpy.ones((xdim, ydim, zdim, tdim, 1, 1), dtype=numpy.float64, order='F')
    ferret_compute(0, result, resbdf, (template, pfname, pfparams), inpbdfs)
    resultgood = ( result != resbdf )
    if numpy.any( resultgood !=  expectedgood ):
        raise ValueError("Assigned random variates does not match template")
    mean = numpy.mean(result[resultgood])
    if abs(mean - 5.0) > 5.0E-3:
        raise ValueError("Mean of random sample: expected: 5.0; found: %f" % mean)
    stdev = numpy.std(result[resultgood])
    if abs(stdev - 0.5) > 5.0E-3:
        raise ValueError("Standard deviation of random sample: expected: 0.5; found: %f" % stdev)

    # All successful
    print("Success")

max_stars_repo_name: josborne-noaa/PyFerret

max_issues_repo_name: josborne-noaa/PyFerret

max_forks_repo_name: josborne-noaa/PyFerret


text:
from pylab import *
from clawpack.visclaw.data import ClawPlotData

if 0:
    gdata = loadtxt('gauges.data',skiprows=7)
    ngauges = gdata.shape[0]
    print "Found %s gauges" % ngauges
    xc = gdata[:,1]
    yc = gdata[:,2]

# Load ClawplotData
plotdata = ClawPlotData()
plotdata.format = 'binary'
plotdata.outdir = '_output'

def plot_gauges(goffset=0):

    if goffset == 0:
        ngauges = 100 # top surface
    elif goffset == 200:
        ngauges = 100 # bottom of domain
    elif goffset == 300:
        ngauges = 50 # above fault plane
    elif goffset == 400:
        ngauges = 50 # below fault plane
    plot_okada = (goffset==0)

    figure()

    #for t in linspace(0.,120,7):
    for t in [100]:   #linspace(0.,100,11):
        xs = zeros(ngauges)
        ys = zeros(ngauges)
        xg = zeros(ngauges)
        for j in range(ngauges):
            gaugeno = goffset + j
            g = plotdata.getgauge(gaugeno)
            xg[j] = g.location[0]  # x-location of this gauge
            for k in range(1,len(g.t)):
                if g.t[k] > t:
                    break
                dt = g.t[k] - g.t[k-1]
                u = g.q[3,k]
                v = g.q[4,k]
                xs[j] = xs[j] + dt*u
                ys[j] = ys[j] + dt*v

        subplot(211)
        plot(xg,xs,label='t = %6.3f' % t)
        title("horizontal displacement")
        ax = subplot(212)
        plot(xg,ys,label='t = %6.3f' % t)
        title("vertical displacement")

    if plot_okada:
        import plot_okada
        ax = subplot(211)
        plot_okada.plot_okada_horiz(ax, 'r-')
        legend()

        ax = subplot(212)
        plot_okada.plot_okada_surface(ax, 'r-')
        legend()

def fault_slip(t):
    dip = 9.9  # degrees
    ngauges = 50
    xs_above = zeros(ngauges)
    ys_above = zeros(ngauges)
    xs_below = zeros(ngauges)
    ys_below = zeros(ngauges)
    xg = zeros(ngauges)
    slip = zeros(ngauges)
    for j in range(ngauges):
        g_above = plotdata.getgauge(j + 300)
        g_below = plotdata.getgauge(j + 400)
        xg[j] = g_above.location[0]  # x-location of this gauge
        for k in range(1,len(g_above.t)):
            if g_above.t[k] > t:
                break
            dt = g_above.t[k] - g_above.t[k-1]
            u_above = g_above.q[3,k]
            v_above = g_above.q[4,k]
            u_below = g_below.q[3,k]
            v_below = g_below.q[4,k]
            xs_above[j] = xs_above[j] + dt*u_above
            ys_above[j] = ys_above[j] + dt*v_above
            xs_below[j] = xs_below[j] + dt*u_below
            ys_below[j] = ys_below[j] + dt*v_below

    dipdir_above = xs_above*cos(dip*pi/180.) - ys_above*sin(dip*pi/180.)
    orthog_above = xs_above*sin(dip*pi/180.) + ys_above*cos(dip*pi/180.)
    dipdir_below = xs_below*cos(dip*pi/180.) - ys_below*sin(dip*pi/180.)
    orthog_below = xs_below*sin(dip*pi/180.) + ys_below*cos(dip*pi/180.)
    slip = dipdir_below - dipdir_above
    return xg, slip

def make_dtopo(xtopo, times):
    """
    Parse the gauge output for gauges on surface (seafloor)
    and turn into dtopo object.  Interpolate to the requested xtopo, times.
    """

    from scipy.interpolate import interp1d

    # custom dtopotools includes DTopography1d
    import clawpack.seismic.dtopotools_horiz_okada_and_1d as dtopotools

    goffset = 0  # gauges at top surface
    ngauges = 100

    # dtopo object for vertical displacement:
    dtopo_z = dtopotools.DTopography1d()
    dtopo_z.x = xtopo
    dtopo_z.times = times

    # dtopo object for horizontal displacement:
    dtopo_x = dtopotools.DTopography1d()
    dtopo_x.x = xtopo
    dtopo_x.times = times

    dZ_list_z = []
    dZ_list_x = []

    for t in times:
        xs = zeros(ngauges)
        ys = zeros(ngauges)
        xg = zeros(ngauges)
        for j in range(ngauges):
            gaugeno = goffset + j
            g = plotdata.getgauge(gaugeno)
            xg[j] = g.location[0]  # x-location of this gauge
            for k in range(1,len(g.t)):
                if g.t[k] > t:
                    break
                dt = g.t[k] - g.t[k-1]
                u = g.q[3,k]
                v = g.q[4,k]
                xs[j] = xs[j] + dt*u
                ys[j] = ys[j] + dt*v

        # vertical displacement:
        dz = interp1d(xg, ys, 'linear',bounds_error=False,
                fill_value=0.,assume_sorted=True)
        dZ_list_z.append(dz(xtopo))

        # horizontal displacement:
        dz = interp1d(xg, xs, 'linear',bounds_error=False,
                fill_value=0.,assume_sorted=True)
        dZ_list_x.append(dz(xtopo))

    dtopo_z.dZ = array(dZ_list_z, ndmin=2)
    dtopo_x.dZ = array(dZ_list_x, ndmin=2)

    return dtopo_z, dtopo_x

def save_dtopo_test1():
    """
    Test make_dtopo.
    Save full time history and also make a dtopo file with the final
    deformation shifted to t = 1 second.
    """

    import copy
    xtopo = linspace(-150e3,150e3,151)
    times = linspace(0,80,41)
    dtopo_z, dtopo_x = make_dtopo(xtopo,times)

    fname = 'dtopo_z_seismic.tt3'
    dtopo_z.write(fname, 3)
    print "Created ",fname

    fname = 'dtopo_x_seismic.tt3'
    dtopo_x.write(fname, 3)
    print "Created ",fname

    # make the deformation file with only the first and last columns of
    # dtopo.dZ (initial and final time) and set final time to 1 sec:
    dtopo_final = copy.copy(dtopo_z)
    dtopo_final.times = array([0., 1.])
    dtopo_final.dZ = dtopo_z.dZ[[0,-1],:]
    fname = 'dtopo_z_seismic_final.tt3'
    dtopo_final.write(fname, 3)
    print "Created ",fname

    dtopo_final = copy.copy(dtopo_x)
    dtopo_final.times = array([0., 1.])
    dtopo_final.dZ = dtopo_x.dZ[[0,-1],:]
    fname = 'dtopo_x_seismic_final.tt3'
    dtopo_final.write(fname, 3)
    print "Created ",fname

max_stars_repo_name: rjleveque/seismic

max_issues_repo_name: clawpack/seismic

max_forks_repo_name: rjleveque/seismic

reason: from scipy


text:
import numpy as np

class ChoiceEvaluator:

    """
    Evaluator for classifying one elements with n labels.

    Fast and slow scoring both use accuracy.
    """

    def score_batch(self, predictions):
        scores = [self.score_example(p) for p in predictions]
        return float(sum(scores)) / max(len(scores), 1)

    def compare_performance(self, old_performance, new_performance):
        return old_performance < new_performance

    def score_example(self, prediction):
        return float(prediction.get_prediction() == prediction.get_gold_answer())

    def get_optimal_performance(self):
        return 1.0

    def evaluate_stats(self, all_stats, split):
        all_stats = np.array(all_stats)

        return all_stats.mean()

    def get_stats(self, prediction):
        return [self.score_example(prediction)]

    def get_evaluation_metric_name(self):
        return "accuracy"

    def set_mode(self, mode):
        pass

max_stars_repo_name: S-Eggers/GraphMask

max_issues_repo_name: S-Eggers/GraphMask

max_forks_repo_name: S-Eggers/GraphMask

reason: import numpy


text:
"""
==================
Find EOG artifacts
==================

Locate peaks of EOG to spot blinks and general EOG artifacts.

"""
# Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#
# License: BSD (3-clause)

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import mne
from mne import io
from mne.datasets import sample
data_path = sample.data_path()

###############################################################################
# Set parameters
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'

# Setup for reading the raw data
raw = io.Raw(raw_fname)

event_id = 998
eog_events = mne.preprocessing.find_eog_events(raw, event_id)

# Read epochs
picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=True,
                        exclude='bads')
tmin, tmax = -0.2, 0.2
epochs = mne.Epochs(raw, eog_events, event_id, tmin, tmax, picks=picks)
data = epochs.get_data()

print("Number of detected EOG artifacts : %d" % len(data))

###############################################################################
# Plot EOG artifacts
plt.plot(1e3 * epochs.times, np.squeeze(data).T)
plt.xlabel('Times (ms)')
plt.ylabel('EOG (muV)')
plt.show()

max_stars_repo_name: TanayGahlot/mne-python

max_issues_repo_name: TanayGahlot/mne-python

max_forks_repo_name: TanayGahlot/mne-python


text:
# By Jeff Yuanbo Han (u6617017), 2018-05-01.
import torch
import numpy as np
from load_data import attributes

w = torch.load('net_weights')['0.weight'][0,:].numpy()  # File 'net_weights' is derived by bpNN.py
w = w.astype(np.int)

valid_attr = {}

end = 0
for attr in attributes:
    if attr == 'class':
        continue
    start = end
    end = start + attributes[attr]
    if np.size(np.nonzero(w[start:end])):
        valid_attr[attr] = w[start:end]

for attr in valid_attr:
    print('{}: {}'.format(attr, valid_attr[attr]))

max_stars_repo_name: ybhan/Mushroom-Edibility-Rules

max_issues_repo_name: ybhan/Mushroom-Edibility-Rules

max_forks_repo_name: ybhan/Mushroom-Edibility-Rules


text:
import numpy as np
import igraph as ig
from numba import jit

def make_symmetric(edges_mat):

    edges_mat += edges_mat.T

    # return edges_mat
    return np.triu(edges_mat,1)

def edges2_vertices(edges_mat):

    edges_nz = edges_mat.nonzero()

    weights = edges_mat[edges_nz]
    edges = [(edges_nz[0][t], edges_nz[1][t]) for t in range(len(edges_nz[0]))]

    # iGraph expects vertex names to be 0,1,2,..., so we need a mapping
    vertices = set()
    for e in edges: vertices |= set(e)
    vertex_mapping = list(vertices)
#    vertex_mapping.index(7)
    # update edges according to new mapping
    edges = [(vertex_mapping.index(e[0]), vertex_mapping.index(e[1])) for e in edges]

    return edges, weights, vertex_mapping

def edges_to_iGraph(edges_mat):

    edges, weights, vertex_mapping = edges2_vertices(edges_mat)

    g = ig.Graph()

    g.add_vertices(vertex_mapping)  # add a list of unique vertices to the graph
    g.add_edges(edges)  # add the edges to the graph..
    g.es['weight'] = weights

    g = g.simplify(combine_edges=sum)

    return g, vertex_mapping

def memberships_to_cluster_list(memberships):
    memberships = np.asarray(memberships)

    clusters_list = []
    for c in np.unique(memberships):
        clusters_list.append(np.nonzero(memberships == c)[0].tolist())
    return clusters_list

def cluster_adj_mat(edges_mat, q_thr=0.8, method='fastgreedy'):
    # weights = edges_mat[edges_mat.nonzero()]  # edge weights

    g, vertex_mapping = edges_to_iGraph(edges_mat)
    print('*** vertex mapping done ***')

    if method=='fastgreedy':
        dend = g.community_fastgreedy()

        if q_thr == 0:
            optimal_count = dend.optimal_count
        else:
            # func. below is added from: ...envs/tez/lib/python3.6/site-packages/igraph/clustering.py
            optimal_count = dend.optimal_count_ratio(ratio=q_thr)

        clus = dend.as_clustering(optimal_count)

    elif method=='louvain':
        clus = g.community_multilevel()

    memberships = clus.membership

    modularity = clus.modularity

    clusters_list = memberships_to_cluster_list(memberships)

    #
    for i,clus in enumerate(clusters_list):
        clusters_list[i] = [ vertex_mapping[c]  for c in clus]

    return clusters_list, memberships, modularity

def find_single_nodes_in_matrix(edges_mat):
    # find isolated nodes in the graph
    assert edges_mat.shape[0] == edges_mat.shape[1] # i.e. square
    nodes_to_delete = []
    for i in range(edges_mat.shape[0]):
        if sum(edges_mat[i,:]) == 0 and sum(edges_mat[:,i]) == 0:
            nodes_to_delete.append(i)
    return nodes_to_delete

def remove_single_nodes(edges_mat, nodes_to_delete):
    # delete nodes that dont have any connection
    edges_mat = np.delete(edges_mat, np.asarray(nodes_to_delete), axis=0 )
    edges_mat = np.delete(edges_mat, np.asarray(nodes_to_delete), axis=1 )

    return edges_mat

reason: import numpy,from numba


text:
# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import numpy as np

from libs.configs._base_.models.retinanet_r50_fpn import *
from libs.configs._base_.datasets.dota_detection import *
from libs.configs._base_.schedules.schedule_1x import *
from dataloader.pretrained_weights.pretrain_zoo import PretrainModelZoo

# schedule
BATCH_SIZE = 1
GPU_GROUP = "0,1"
NUM_GPU = len(GPU_GROUP.strip().split(','))
SAVE_WEIGHTS_INTE = 40000 * 2
DECAY_STEP = np.array(DECAY_EPOCH, np.int32) * SAVE_WEIGHTS_INTE
MAX_ITERATION = SAVE_WEIGHTS_INTE * MAX_EPOCH
WARM_SETP = int(WARM_EPOCH * SAVE_WEIGHTS_INTE)

# dataset
DATASET_NAME = 'DOTA2.0'
CLASS_NUM = 18

# model
# backbone
pretrain_zoo = PretrainModelZoo()
PRETRAINED_CKPT = pretrain_zoo.pretrain_weight_path(NET_NAME, ROOT_PATH)
TRAINED_CKPT = os.path.join(ROOT_PATH, 'output/trained_weights')

# loss
CLS_WEIGHT = 1.0
REG_WEIGHT = 1.0 / 5.0
REG_LOSS_MODE = 1  # IoU-Smooth L1

VERSION = 'RetinaNet_DOTA2.0_2x_20210427'

"""
RetinaNet-H + IoU-Smooth L1
FLOPs: 676601855;    Trainable params: 33148131
This is your evaluation result for task 1:

    mAP: 0.46306689567660153
    ap of each class:
    plane:0.7754947865172579,
    baseball-diamond:0.46434700034028786,
    bridge:0.39599890161126844,
    ground-track-field:0.5841579611023893,
    small-vehicle:0.347077308109511,
    large-vehicle:0.36984795081268984,
    ship:0.47937500010188855,
    tennis-court:0.7713614052919219,
    basketball-court:0.5712051108571043,
    storage-tank:0.5361292205829321,
    soccer-ball-field:0.4011674247207251,
    roundabout:0.5077467772636136,
    harbor:0.4002952079323899,
    swimming-pool:0.5416185098631832,
    helicopter:0.5162148043881953,
    container-crane:0.13870523708490112,
    airport:0.4057909924349168,
    helipad:0.1286705231636533

The submitted information is :

Description: RetinaNet_DOTA2.0_2x_20210427_104w
Username: yangxue
Institute: UCAS
Emailadress: yangxue16@mails.ucas.ac.cn
TeamMembers: yangxue
"""


text:
from godirect import GoDirect
import matplotlib.pyplot as plt
import queue
import numpy as np
from matplotlib.animation import FuncAnimation
from gdx import gdx

sample_rate = 10
gdx = gdx.gdx()
gdx.open_usb()
gdx.select_sensors(sensors=gdx.devices[0].list_sensors())
gdx.start(period=1000/sample_rate) # the unit is millisecond
data = queue.Queue()

# for i in range(0,40):
#     measurements = gdx.read()
#     if measurements == None:
#         break
#     print(data)
#     data.put(measurements)

# animation script
fig, ax = plt.subplots()
plt.xlabel('Time (s)')
plt.ylabel('Force (N)')
xdata, ydata = [], []
ln, = plt.plot([], [], 'r-')
time = 0

def init():
    ax.set_xlim(0, 20)
    ax.set_ylim(10, 50)
    return ln,

def update(frame):
    measurements = gdx.read()[0]
    time = len(xdata) / sample_rate
    xdata.append(time)
    ydata.append(measurements)
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()

    if time >= xmax:
        ax.set_xlim(xmin, 2*xmax)
        ax.figure.canvas.draw()
    if measurements >= ymax:
        ax.set_ylim(ymin, measurements + 5)
        ax.figure.canvas.draw()
    if measurements <= ymin:
        ax.set_ylim(measurements - 5, ymax)
        ax.figure.canvas.draw()
    ln.set_data(xdata, ydata)

    return ln,

ani = FuncAnimation(fig, update, frames=np.linspace(0, 1000, 200),
                    init_func=init, blit=True)
plt.show()
gdx.stop()
gdx.close()

max_stars_repo_name: WanHaoRan/godirect-examples

max_issues_repo_name: WanHaoRan/godirect-examples

max_forks_repo_name: WanHaoRan/godirect-examples


text:
import os

import numpy as np
import pandas as pd

from fedot.core.data.data import InputData
from fedot.core.pipelines.node import PrimaryNode, SecondaryNode
from fedot.core.pipelines.pipeline import Pipeline
from fedot.core.repository.dataset_types import DataTypesEnum
from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams
from fedot.core.utils import fedot_project_root, DEFAULT_PARAMS_STUB
from test.unit.pipelines.test_pipeline_parameters import small_ts_dataset
from test.unit.tasks.test_regression import get_synthetic_regression_data

def get_ts_pipeline(window_size):
    """ Function return pipeline with lagged transformation in it """
    node_lagged = PrimaryNode('lagged')
    node_lagged.custom_params = {'window_size': window_size}

    node_final = SecondaryNode('ridge', nodes_from=[node_lagged])
    pipeline = Pipeline(node_final)
    return pipeline

def get_ransac_pipeline():
    """ Function return pipeline with lagged transformation in it """
    node_ransac = PrimaryNode('ransac_lin_reg')
    node_final = SecondaryNode('linear', nodes_from=[node_ransac])
    pipeline = Pipeline(node_final)
    return pipeline

def test_lagged_with_invalid_params_fit_correctly():
    """ The function define a pipeline with incorrect parameters in the lagged
    transformation. During the training of the pipeline, the parameter 'window_size'
    is corrected
    """
    window_size = 600
    len_forecast = 50

    # The length of the time series is 500 elements
    project_root_path = str(fedot_project_root())
    file_path = os.path.join(project_root_path, 'test/data/short_time_series.csv')
    df = pd.read_csv(file_path)
    time_series = np.array(df['sea_height'])

    task = Task(TaskTypesEnum.ts_forecasting,
                TsForecastingParams(forecast_length=len_forecast))

    ts_input = InputData(idx=np.arange(0, len(time_series)), features=time_series,
                         target=time_series, task=task, data_type=DataTypesEnum.ts)

    # Get pipeline with lagged transformation in it
    pipeline = get_ts_pipeline(window_size)

    # Fit it
    pipeline.fit(ts_input)

    # Get lagged node
    lagged_node = pipeline.nodes[1]
    fixed_params = lagged_node.custom_params

    assert pipeline.is_fitted
    assert fixed_params['window_size'] == 439

def test_ransac_with_invalid_params_fit_correctly():
    """ Check that on a small dataset the RANSAC anomaly search algorithm can
    adjust the values of hyperparameters

    As stated in the sklearn documentation, min_samples is determined by default
    based on how many features are in the dataset
    Therefore, problems can arise when there are more attributes in a dataset
    than the number of objects
    """

    input_regression = get_synthetic_regression_data(n_samples=20, n_features=23)

    ransac_pipeline = get_ransac_pipeline()
    ransac_pipeline.fit(input_regression)
    predicted = ransac_pipeline.predict(input_regression)

    assert ransac_pipeline.is_fitted
    assert predicted is not None

def test_params_filter_correct_with_default():
    """
    Check custom_params returns updated parameters for lagged operation after fitting.
    Default params for operation loaded from json file
    """
    input_ts = small_ts_dataset()

    node_lagged = PrimaryNode('lagged')

    # Correct parameters during fit
    node_lagged.fit(input_ts)
    updated_params = node_lagged.custom_params
    assert 'window_size' in list(updated_params.keys())
    assert len(list(updated_params.keys())) == 1

def test_params_filter_with_non_default():
    """
    Check custom_params returns updated parameters only for changed keys.
    Default params for operation setting as string 'default_params'
    """
    input_data = InputData(idx=np.arange(0, 3),
                           features=np.array([[1, 0, 2],
                                              [2, 0, 3],
                                              [3, 1, 4]]),
                           target=np.array([[1], [0], [1]]),
                           task=Task(TaskTypesEnum.classification),
                           data_type=DataTypesEnum.table)
    # Params are default for now - 'default_params'
    node_knn = PrimaryNode('knn')
    default_params = node_knn.custom_params

    node_knn.fit(input_data)
    updated_params = node_knn.custom_params

    assert default_params == DEFAULT_PARAMS_STUB
    assert 'n_neighbors' in list(updated_params.keys())
    assert len(list(updated_params.keys())) == 1

max_stars_repo_name: rozlana-g/FEDOT

max_issues_repo_name: rozlana-g/FEDOT

max_forks_repo_name: rozlana-g/FEDOT


text:
"""Filtering tests."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import numpy as np
from nose.tools import assert_almost_equal as almeq

from spikedetekt2 import (read_raw, get_threshold, bandpass_filter,)

# -----------------------------------------------------------------------------
# Tests
# -----------------------------------------------------------------------------
def test_get_threshold_1():
    duration = 10.
    sample_rate = 20000.
    filter_low = 10.
    filter_high = 10000.
    nchannels = 5
    nsamples = int(duration * sample_rate)
    nexcerpts = 3
    excerpt_size = 10000

    X = np.random.randn(nsamples, nchannels)
    raw_data = read_raw(X)

    filter = bandpass_filter(filter_butter_order=3,
                             sample_rate=sample_rate,
                             filter_low=filter_low,
                             filter_high=filter_high,
                             )

    threshold1 = get_threshold(raw_data, filter=filter,
                              nexcerpts=nexcerpts,
                              excerpt_size=excerpt_size,
                              threshold_std_factor=(2., 4.))

    threshold2 = get_threshold(raw_data, filter=filter,
                              nexcerpts=nexcerpts,
                              excerpt_size=excerpt_size,
                              threshold_std_factor=(2., 4.),
                              use_single_threshold=False)

    threshold3 = get_threshold(raw_data, filter=filter,
                              nexcerpts=nexcerpts,
                              excerpt_size=excerpt_size,
                              threshold_std_factor=np.array([[2.5, 2., 2., 2.3, 1.2], [4.5, 3.2, 1., 2.3, 3.4]]),
                              use_single_threshold=False)

    # assert np.abs(np.array(threshold) - 4.5) < .5


text:
from .._tier0 import plugin_function
from .._tier0 import create_none
from .._tier0 import create
from .._tier0 import Image
from .._tier0 import execute

@plugin_function(output_creator=create_none)
def n_closest_points(distance_matrix : Image, indexlist_destination : Image = None, n : int = 1, ignore_background : bool = True, ignore_self : bool = True) -> Image:
    """Determine the n point indices with shortest distance for all points in
    a distance matrix.

    This corresponds to the n row indices with minimum values for each column of
    the distance matrix.

    Parameters
    ----------
    distance_matrix : Image
    indexlist_destination : Image
    n : Number
    ignore_background : bool
        The first column and row of the distance matrix will be ignored because they represent the background object.
    ignore_self : bool
        The x==y diagonale will be ignored because it represents the distance of the object to itself.

    Returns
    -------
    indexlist_destination

    References
    ----------
    .. [1] https://clij.github.io/clij2-docs/reference_nClosestPoints
    """
    from .._tier1 import copy
    from .._tier1 import crop
    from .._tier0 import create
    from .._tier1 import set_column
    from .._tier1 import set_row
    from .._tier1 import set_where_x_equals_y
    from .._tier1 import add_image_and_scalar
    import numpy as np

    max_value = np.finfo(np.float32).max

    if ignore_background:
        distance_matrix = crop(distance_matrix, start_x=1, start_y=1, width=distance_matrix.shape[0] - 1, height=distance_matrix.shape[0] - 1 )

    if ignore_self:
        if not ignore_background:
            distance_matrix = copy(distance_matrix)
        set_where_x_equals_y(distance_matrix, max_value)

    temp = create([int(n), distance_matrix.shape[1]])

    parameters = {
        "src_distancematrix":distance_matrix,
        "dst_indexlist":temp,
    }

    # todo: rename cl-file kernel to fulfill naming conventions
    execute(__file__, "../clij-opencl-kernels/kernels/n_shortest_points_x.cl", "find_n_closest_points", distance_matrix.shape, parameters)

    if (ignore_background):
        indexlist_destination = add_image_and_scalar(temp, indexlist_destination, scalar=1)
    else:
        indexlist_destination = copy(temp, indexlist_destination)

    return indexlist_destination


text:
# Necessary Libraries
import pandas as pd
import numpy as np
import pickle
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk

nltk.download('punkt')

# In[2]:

# Importing the dataset
df = pd.read_csv('/Users/sumitkumarkundu/PycharmProjects/Fake_News_Classifier/venv/Model/train.csv')

# In[3]:
print(df.head())

df.isnull().sum()

# In[4]:

df.sample()

# In[5]:

# df.info()

# In[6]:

# input column is text and output column is label and in text there are 39 missing data
# df['text'].shape

# In[7]:

# but you see total 20800 text rows are there so we can drop these 28 rows , it will do no effect
df['text'] = df['text'].fillna(method='ffill')

# In[8]:
# df['text'].shape
# In[9]:

X = df['text']
y = df['label']

# In[10]:

# let's split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)

# In[11]:

# now let's create a pipeline for removing stopwords and create bag of words
# applying multinomialNB as it gives us better results
pipe=Pipeline([('tfidf', TfidfVectorizer(stop_words='english')), ('nbmodel', MultinomialNB())])

# In[13]:

# fit the model
pipe.fit(X_train, y_train)

# In[15]:

prediction=pipe.predict(X_test)

# In[16]:

print(classification_report(y_test, prediction))

# In[17]:

print(confusion_matrix(y_test, prediction))

# In[18]:
# see true positive and true negative value are so much high
# In[19]:

# make the pickle file
with open('model.pickle', 'wb') as target:
    pickle.dump(pipe, target, protocol=pickle.HIGHEST_PROTOCOL)


text:
# OVERVIEW
#
# A simulator script for working with the SPOM model.
# Input:
#  - habitat matrix
#  - list of species
#
# Optional input:
#  - region stencil file (label each habitat cell with a region ID)
#  - initial occupancy matrices for each species
#
# Output:
# - occupancy matrices for each species
# - regional occupancy statistics (how many patches in each region were occupied)
#
# USAGE
#
# Run the script with parameter '-h' to get the list of options.
# To increase the level of logging run the script with parameters '-v' or '-vv' (for more detail).
#
# FILE FORMATS
#
# Habitat, stencil, and occupancy matrices are given as CSV files:
#  - comma ',' is used as a separator value
#  - row i, col j correspond to cell (i,j) in the matrix
#
# Species are given as a CSV file. Each row corresponds to a species and the parameters
# are given in the following order:
#  - phenotype (phi)
#  - colonization rate (c)
#  - extinction rate (e)
#  - inverse of dispersal distance (alpha)
#  - niche width (gamma)

import scipy
import scipy.stats
import csv
import sys
import argparse
import os
import time
from multiprocessing import Pool

from collections import defaultdict

import model.grid
from model.grid import Species, Grid

VERBOSITY = 0

class InputData(object):
    def __init__(self, habitat, species_parameters, stencil, locations):
        self.habitat = habitat
        self.h, self.w = habitat.shape
        self.stencil = stencil
        self.species_parameters = species_parameters
        self.locations = locations
        self.species = [Species(*s) for s in self.species_parameters]

    def count_occupied_in_regions(self, occupancy):
        """ Count the number of occupied patches for each region specified in the stencil.
            If stencil is None, the whole grid is treated as a single region. """
        res = []

        if self.stencil is None:
            res.append(occupancy.sum())
        else:
            for r in scipy.unique(self.stencil):
                X = [self.stencil == r]*occupancy
                occurences = X.sum()
                assert occurences <= self.number_of_cells(r)
                res.append(occurences)

        return res

    def number_of_regions(self):
        if self.stencil is None:
            return 1
        else:
            return len(scipy.unique(self.stencil))

    def number_of_cells(self, region):
        if self.stencil == None:
            return self.h*self.w
        return (self.stencil.flatten() == region).sum()

    def create_grid(self, i):
        s = self.species[i]
        g = Grid(self.h, self.w, s)
        g.initialize()
        g.calculate_fitness_from_habitat(self.habitat)
        if len(args.stochasticity) > 0:
            mean, stddev, wf = args.stochasticity
            log("Enabling regional stochasticity with parameters: mean=%f, stddev=%f, weight factor=%f" % tuple(args.stochasticity), 1)
            g.enable_regional_stochasticity(mean, stddev, wf)

        # If a location matrix was specified, initialize the occupancy matrix with that,
        # otherwise populate everything.
        if i < len(self.locations):
            g.occupancy[:] = self.locations[i]
        else:
            g.occupancy[:] = 1

        return g

def read_matrix(csv_filename):
    """ Read a matrix from a CSV file"""

    log("Reading from '%s'" % (csv_filename,), 2)
    f = open(csv_filename)
    csv_reader = csv.reader(f)
    rows = []
    for row in csv_reader:
        rows.append(map(float, row))

    return scipy.array(rows)

def write_matrix(matrix, csv_filename):
    """ Write a matrix to a CSV file """
    log("Writing to '%s'" % csv_filename, 2)
    f = open(csv_filename, 'w')
    for row in matrix:
        f.write(",".join(map(str, row)))
        f.write("\n")
    f.close()

def read_input():
    """ Parse command line parameters """
    def assert_size(A,B, name1, name2):
        assert A.shape == B.shape, "%s and %s matrices have different shape: %s != %s" % (name1, name2, A, B)

    log("Reading input files..", 1)

    habitat = read_matrix(args.habitat)
    log("Landscape size %ix%i" % habitat.shape, 1)

    species = read_matrix(args.species)
    log("Loaded %i species" % (len(species,)), 1)

    stencil = None
    if args.stencil is not None:
        stencil = read_matrix(args.stencil)
        assert_size(habitat, stencil, "habitat", "stencil")

    locations = [read_matrix(fname) for fname in args.locations]

    for i, l in enumerate(locations):
        assert_size(habitat, l, "habitat", "location %i" % (i,))

    return InputData(habitat, species, stencil, locations)

def initialize():
    """ Initialize the SPOM library """
    seed = int(time.time()*1000) if args.seed is None else args.seed
    log("Initializing the SPOM library (fft threads=%i, seed=%i)" % (args.fft_threads, seed), 1)
    model.grid.initialize_model(args.fft_threads, seed)

def sim_stencil_helper(p):
    i, data, time = p
    g = data.create_grid(i)
    hist = defaultdict(list)

    log("Simulating species %i" % i, 2)
    ps = g.occupancy.flatten().sum()
    for t in xrange(time+1):
        res = data.count_occupied_in_regions(g.occupancy)

        for region, patches in enumerate(res):
            hist[region].append(patches)

        if ps == 0:
            log("Species %i EXTINCT at %i" % (i,t), 2)
            break # Extinct, no need to simulate

        ps = g.step()

    return hist, g.occupancy

def simulate_stencil(data, time):
    """ Compute number of occupied patches for each species in each region """
    # TODO This is a very slow and wasteful implementation
    species = len(data.species)
    regions = data.number_of_regions()
    total_hist = defaultdict(list)

    log("Using %i threads for simulation" % (args.threads,), 1)
    pool = Pool(args.threads)
    res_list = pool.map(sim_stencil_helper, [(i,data,time) for i in xrange(species)])

    assert len(res_list) == species, "Did not get data from all species"
    hists, occs = zip(*res_list)

    for i in xrange(len(hists)):
        for k in xrange(len(hists[i])):
            total_hist[(i,k)] = hists[i][k]

    log("Simulations complete")

    return total_hist, occs

def write_occupancy(occs):
    prefix = os.path.join(args.output_dir, args.prefix)
    # Occupancy matrices for each species
    if args.output_locations:
        for i, om in enumerate(occs):
            write_matrix(om, prefix+'occupancy-%i.csv' % (i,))

def write_hist(hist, fname):
    log("Writing history to '%s'" % (fname,), 2)
    f = open(fname, 'w')
    for ((i,r), l) in sorted(hist.items()):
        line = [i,r] + l
        f.write(",".join(map(str,line)))
        f.write("\n")

def log(msg, lvl=1):
    timestamp = "%.3f" % (time.time() - START_TIME)
    if VERBOSITY >= lvl:
       print "%s -- %s" % (timestamp, msg)

def main():
    if args.output_dir is None:
        log("*** NO output files as output directory is not set ***", 0)

    initialize()
    data = read_input()

    r = data.number_of_regions()
    log("%i regions specified" % (r,), 1)
    for i in xrange(r):
        log("Region %i has %i cells" % (i, data.number_of_cells(i)), 2)

    for (i,s) in enumerate(data.species):
        log("Species %i: %s" % (i,s), 2)

    hist, occs = simulate_stencil(data, args.steps)

    if args.output_dir is not None:
        write_occupancy(occs)
        if hist is not None:
            fname = os.path.join(args.output_dir, args.prefix+"history.csv")
            write_hist(hist, fname)

    log("Finished", 0)

# --- argument parsing and calling of main ---

def make_parser():
    parser = argparse.ArgumentParser(description='Run a SPOM simulation from given input data')
    parser.add_argument('habitat', help='habitat matrix csv file')
    parser.add_argument('species', help='species parameters csv file')
    parser.add_argument('--stencil', '-s', help='the region stencil csv file')
    parser.add_argument('--seed', type=int, help='the seed for the random number generator')
    parser.add_argument('--locations', '-l', default=[], nargs='+', help='initial occupancy matrices for species. if not specified, all species are present at each patch')
    parser.add_argument('--fft_threads', '-ft', type=int, default=1, help='number of threads allocated to FFTW')
    parser.add_argument('--threads', '-t', type=int, default=2, help='number of threads for parallel map')
    parser.add_argument('--steps', type=int, default=1, help='the number of simulation steps')
    parser.add_argument('--output_dir', '-od', help='write result files into the output directory')
    parser.add_argument('--output_locations', '-ol', action='store_true', default=False, help='output resulting occupancy matrices')
    parser.add_argument('--prefix', '-p', default="", help='prefix for output files')
    parser.add_argument('--stochasticity', type=float, nargs=3, default=[], help='enable regional stochasticity. Give mean, stddev, weight factor as parameters.')
    parser.add_argument('--verbose', '-v', action='count', default=0)
    return parser.parse_args()

START_TIME = time.time()
if __name__ == '__main__':
    args = make_parser()
    VERBOSITY = args.verbose
    main()

max_stars_repo_name: rybicki/corridor-spom

max_issues_repo_name: rybicki/corridor-spom

max_forks_repo_name: rybicki/corridor-spom

reason: import scipy


text:
import numpy as np

# Range of power law indices permitted.
power_law_index_bounds = (0, 4)

# Assume a power law in power spectrum - Use a Bayesian marginal distribution
# to calculate the probability that the power spectrum has a power law index
# 'n'.
def bayeslogprob(f, I, n, m):
    """
    Return the log of the marginalized Bayesian posterior of an observed
    Fourier power spectra fit with a model spectrum Af^{-n}, where A is a
    normalization constant, f is a normalized frequency, and n is the power law
    index at which the probability is calculated. The marginal probability is
    calculated using a prior p(A) ~ A^{m}.
    The function returns log[p(n)] give.  The most likely value of 'n' is the
    maximum value of p(n).
    f : normalized frequencies
    I : Fourier power spectrum
    n : power law index of the power spectrum
    m : power law index of the prior p(A) ~ A^{m}
    """
    N = len(f)
    term1 = n * np.sum(np.log(f))
    term2 = (N - m - 1) * np.log(np.sum(I * f ** n))
    return term1 - term2

# Find the most likely power law index given a prior on the amplitude of the
# power spectrum.
def most_probable_power_law_index(f, I, m, n):
    blp = np.zeros_like(n)
    for inn, nn in enumerate(n):
        blp[inn] = bayeslogprob(f, I, nn, m)
    return n[np.argmax(blp)]

def _frequency_range_indices(f, frequency_limits):
    return [np.argmin(np.abs(f-frequency_limits[0])),
            np.argmin(np.abs(f-frequency_limits[1]))]

def range_index_estimate(f, frequency_limits=None):
    nf = len(f)
    if frequency_limits is None:
        ii = [int(0.025*nf), int(0.975*nf)]
    else:
        ii = _frequency_range_indices(f, frequency_limits)
    return ii

def range_amplitude_estimate(f, frequency_limits=None):
    nf = len(f)
    if frequency_limits is None:
        ii = [0, int(0.025*nf)]
    else:
        ii = _frequency_range_indices(f, frequency_limits)
    return ii

def range_background_estimate(f, frequency_limits=None):
    nf = len(f)
    if frequency_limits is None:
        ii = [int(0.975 * nf), nf]
    else:
        ii = _frequency_range_indices(f, frequency_limits)
    return ii

class InitialParameterEstimatePlC(object):
    def __init__(self, f, p, ir=None, ar=None, br=None,
                 bayes_search=(0, np.linspace(power_law_index_bounds[0], power_law_index_bounds[1], 50))):
        """
        Estimate of three parameters of the power law + constant observation model - the amplitude,
        the power law index, and the background value.

        Parameters
        ----------
        f :
            Positive frequencies of the power law spectrum

        p :
            Power at the frequencies

        ir :

        ar :

        br :

        bayes_search : ~tuple

        """
        self.f = f
        self.p = p
        self.ir = ir
        self.ar = ar
        self.br = br
        self.bayes_search = bayes_search

        self._ir = range_index_estimate(self.f, frequency_limits=self.ir)
        self._ar = range_index_estimate(self.f, frequency_limits=self.ar)
        self._br = range_background_estimate(self.f, frequency_limits=self.br)

        # The bit in-between the low and high end of the spectrum to estimate the power law index.
        self._index = most_probable_power_law_index(self.f[self._ir[0]:self._ir[1]],
                                                    self.p[self._ir[0]:self._ir[1]],
                                                    bayes_search[0], bayes_search[1])

        # Use the low-frequency end to estimate the amplitude, normalizing for the first frequency
        self._amplitude = np.mean(self.p[self._ar[0]:self._ar[1]]) * (self.f[0] ** self._index)

        # Use the high frequency part of the spectrum to estimate the constant value.
        self._background = np.exp(np.mean(np.log(self.p[self._br[0]:self._br[1]])))

    @property
    def index_range(self):
        return self._ir

    @property
    def amplitude_range(self):
        return self._ar

    @property
    def background_range(self):
        return self._br

    @property
    def index(self):
        return self._index

    @property
    def amplitude(self):
        return self._amplitude

    @property
    def background(self):
        return self._background


text:
#!/usr/bin/env python

from __future__ import absolute_import
import numpy
from scipy import ndimage

def label_region(data, all_neighbors=False, ulong=False):
    """
    Replicates the label_region function avaiable within IDL
    (Interactive Data Language, EXELISvis).
    Consecutively labels all regions/segments with a unique region
    index.

    :param data:
        A 2D NumPy array, ideally a bi-level array.

    :param all_neighbors:
        If set to True then all 8 neighbors of a pixel are used to
        determine connectivity. Default is False, only the 4 immediate
        neighbors of a pixel are used to determnine connectiviy.

    :param ulong:
        If set to True then the output array will be an 32 bit unsinged
        long integer. Default is False, and the output will be a 16 bit
        unsinged integer. Use this keyword if you expect > 65525 regions.

    :return:
        A 2D NumPy array either 16 or 32 bit unsigned integer, with
        each pixel containing its region/segment index. Zeros values
        are considered to be background.

    :notes:
        Only non-zero values are considered in the labelling process.
        Zero values are considered background data in the input data
        array, and ignored.

    :history:
        *  16/08/2014: Created

    :author:
        Josh Sixsmith; josh.sixsmith@gmail.com; joshua.sixsmith@ga.gov.au

    :copyright:
        Copyright (c) 2014, Josh Sixsmith
        All rights reserved.

        Redistribution and use in source and binary forms, with or without
        modification, are permitted provided that the following conditions are met:

        1. Redistributions of source code must retain the above copyright notice, this
           list of conditions and the following disclaimer.
        2. Redistributions in binary form must reproduce the above copyright notice,
           this list of conditions and the following disclaimer in the documentation
           and/or other materials provided with the distribution.

        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
        ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
        WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
        ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
        (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
        LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
        ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
        (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
        SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

        The views and conclusions contained in the software and documentation are those
        of the authors and should not be interpreted as representing official policies,
        either expressed or implied, of the FreeBSD Project.
    """
    if data.ndim != 2:
        raise ValueError('Error. Array must be 2 dimensional.')

    if all_neighbors:
        kernel = numpy.array([[1,1,1],[1,1,1],[1,1,1]]).reshape(3,3)
    else:
        kernel = numpy.array([[0,1,0],[1,1,1],[0,1,0]]).reshape(3,3)

    if ulong:
        result = numpy.zeros(data.shape, dtype='uint32')
    else:
        result = numpy.zeros(data.shape, dtype='uint16')

    # We don't return the number of features
    nfeatures = ndimage.label(data, structure=kernel, output=result)

    return result

reason: import numpy,from scipy


text:
from collections import defaultdict
import itertools
from .base_solver import *
from opt import PruningOp, SPruningOp, BertQuantizeOp, LowRankOp
# from opt import PruningOp, SPruningOp, BertQuantizeOp
from misc.train_bert import get_bert_FIM
from misc.translation import get_translation_FIM
import matplotlib.pyplot as plt
import cvxpy as cp
import pickle
import os
import random

class OneShotHessianSolver(BaseSolver):
    def __init__(self, net, ops, configs=None, tokenizer=None, logger=None, task_name=None):
        super().__init__(net, ops)
        self.configs = configs
        self.tokenizer = tokenizer
        self.logger = logger
        self.task_name = task_name
        if "MRPC" in task_name:
            self.get_FIM_func = get_bert_FIM
        elif "MarianMT-wmt16" in task_name:
            self.get_FIM_func = get_translation_FIM
        elif "t5-small-wmt16" in task_name:
            self.get_FIM_func = get_translation_FIM
        else:
            raise AttributeError("task name not existed")
        if "czy" in task_name:
            self._czy_init()
        else:
            self._init()

    def _init(self):
        info_fn = f"./info/{self.task_name}-info.pkl"
        if os.path.exists(info_fn):
            with open(info_fn, "rb") as fp:
                info = pickle.load(fp)
                print(f"Load saved {self.task_name}-info of OSHS solver.")
                all_l = np.array(info["l"])
                all_s = np.array(info["s"])

                self.all_l = all_l
                self.all_name = info["name"]
                self.all_s = all_s
                self.model_size = self.all_s[:, 0].sum()
                self.cand = self.operatable
            return
        layer_list = self.operatable
        all_name = []
        all_l = []
        all_s = []
        temp = 1
        self.cand = layer_list
        for i, layer_name in enumerate(self.cand):
            print(f"Get profile count: {i+1}/{len(self.cand)}")
            profile, storage, loss, _ = self.get_profile(layer_name)  # 0.35, 0.50, 0.30
            name_p = []
            name_q = []
            l_p = []
            l_q = []
            s_p = []
            s_q = []
            name = []
            l = []
            s = []
            for k, v in loss.items():
                _, op_name, attrs = k.split("@")
                if op_name == "upruning":
                    name_p.append(k)
                    l_p.append(v)
                    s_p.append(storage[k])
                elif op_name == "quantize":
                    name_q.append(k)
                    l_q.append(v)
                    s_q.append(storage[k])
            for i, n_p in enumerate(name_p):
                for j, n_q in enumerate(name_q):
                    name.append(n_p + "+" + n_q)
                    l.append(l_p[i]+l_q[j])
                    s.append(s_p[i]*s_q[j])
            print(name, l)
            all_name.append(name)
            all_l.append(l)
            all_s.append(s)
        info = {}
        info["name"] = all_name
        info["s"] = all_s
        info["l"] = all_l
        with open(info_fn, "wb") as fp:
            pickle.dump(info, fp)
        all_l = np.array(all_l)
        all_s = np.array(all_s)

        self.all_l = all_l
        self.all_name = all_name
        self.all_s = all_s
        self.model_size = self.all_s[:, 0].sum()

    def _czy_init(self):
        info_fn = f"./info/{self.task_name}-info.pkl"
        if os.path.exists(info_fn):
            with open(info_fn, "rb") as fp:
                info = pickle.load(fp)
                print(f"Load saved {self.task_name}-info of OSHS solver.")
                self.all_l = np.array(info["l"])
                self.all_name = info["name"]
                self.all_s = np.array(info["s"])
                self.model_size = self.all_s[:, 0].sum()
                self.cand = self.operatable
            return

        all_name,all_l,all_s = [], [], []
        self.cand = self.operatable
        for i, layer_name in enumerate(self.cand):
            print(f"Get profile count: {i+1}/{len(self.cand)}")
            _, storage, loss, original_size = self.get_profile(layer_name)
            temp_dic = defaultdict(list)
            for k, v in loss.items():
                _, op_name, attrs = k.split("@")
                temp_dic[op_name].append((k, v, storage[k]))  # name, loss, storage

            layer_name, layer_l, layer_s = [], [], []
            for comb in itertools.product(temp_dic['spruning'], temp_dic['lowrank'], temp_dic['upruning'], temp_dic['quantize']):
                layer_name.append("+".join([tup[0] for tup in comb]))
                layer_l.append(sum([tup[1] for tup in comb]))
                layer_s.append(original_size * np.prod([max(0.00, tup[2]) for tup in comb]))
            all_name.append(layer_name)
            all_l.append(layer_l)
            all_s.append(layer_s)

        info = {"name": all_name, "s": all_s, "l": all_l}
        if "try" not in info_fn:
            with open(info_fn, "wb") as fp:
                pickle.dump(info, fp)
        self.all_l = np.array(all_l)
        self.all_name = all_name
        self.all_s = np.array(all_s)
        self.model_size = self.all_s[:, 0].sum()

    def get_profile(self, layer_name: str):
        profile = {}
        storage = {}
        loss = {}
        original_size = None
        for Op in self.ops:
            op = Op(self.net)
            if layer_name in op.operatable:
                FIM = self.get_FIM_func(self.configs, self.net, self.tokenizer, layer_name, self.logger)
                if isinstance(op, PruningOp) or isinstance(op, SPruningOp):
                    for rate in np.arange(0.00, 1.05, 0.05):
                        _, diff, storage_save = op.apply([layer_name], amount=rate, with_profile=True)
                        # print(f"layer name: {layer_name}, pruning rate: {rate}, diff norm: {np.linalg.norm(diff[layer_name]*FIM)}.")
                        obj = (diff[layer_name] ** 2 * FIM).sum()
                        name = f"{layer_name}@{op.op_name}@{rate:.2f}"
                        loss[name] = obj
                        storage[name] = storage_save[layer_name]
                        profile[name] = storage[name] / (loss[name] + 1e-12)
                        if original_size is None:
                            original_size = diff[layer_name].size
                elif isinstance(op, BertQuantizeOp):
                    op.model.to("cpu")
                    for mode in [None, "fbgemm"]:
                        op.reset()
                        op.set_config(mode)
                        _, diff, storage_save = op.apply([layer_name], with_profile=True)
                        # print(f"layer name: {layer_name}, quantization mode: {mode}, diff norm: {np.linalg.norm(diff[layer_name]*FIM)}.")
                        obj = (diff[layer_name] ** 2 * FIM).sum()
                        name = f"{layer_name}@{op.op_name}@{op.mode}"
                        loss[name] = obj
                        storage[name] = storage_save[layer_name]
                        profile[name] = storage[name] / (loss[name] + 1e-12)
                        if original_size is None:
                            original_size = diff[layer_name].size
                elif isinstance(op, LowRankOp):
                    for r in np.arange(1.00, 0.00, -0.05):
                        _, diff, storage_save = op.apply([layer_name], rank_fraction=r, with_profile=True)
                        obj = (diff[layer_name] ** 2 * FIM).sum()
                        name = f"{layer_name}@{op.op_name}@{r:.2f}"
                        loss[name] = obj
                        storage[name] = storage_save[layer_name]
                        profile[name] = storage[name] / (loss[name] + 1e-12)
                        if original_size is None:
                            original_size = diff[layer_name].size
        return profile, storage, loss, original_size

    def get_assignment(self, storage_threshold: float, p_min:float=0.4, p_max:float=0.6):
        layer_loss = self.all_l[:, -1]
        mean_loss = layer_loss / self.all_s[:, 0]
        # print(f"mean loss: {mean_loss}")
        p = mean_loss / mean_loss.sum()
        p = p / (2*p.mean())
        p = np.clip(p, p_min, p_max)
        mem = (p*self.all_s[:, 0]).sum()

        p = (storage_threshold / mem) * p
        p[p >= 1.] = 1.
        assign_storage = p * self.all_s[:, 0]
        overflow_storage = storage_threshold - assign_storage.sum()

        while True:
            if np.isclose(overflow_storage, 0.):
                print(f"returned p: {p}")
                return assign_storage
            cand_flag = p < 1.
            mean_loss *= cand_flag

            p_inc = mean_loss / mean_loss.sum()
            p_inc = p_inc / (2 * p_inc.mean())
            p_inc = np.clip(p_inc, p_min, p_max)
            mem = (p_inc * self.all_s[:, 0]).sum()
            p_inc = (overflow_storage / mem) * p_inc
            p += p_inc
            p[p >= 1.] = 1.
            assign_storage = p * self.all_s[:, 0]
            overflow_storage = storage_threshold - assign_storage.sum()

    def should_skip(self, name:str, methods:set) -> bool:
        for name in name.split("+"):
            _, op_name, attrs = name.split("@")
            if op_name not in methods:
                if (op_name == "upruning" and float(attrs) != 0.00) or (
                    op_name == "quantize" and attrs != "none") or (
                    op_name == "lowrank" and float(attrs) != 1.00) or \
                   (op_name == "spruning" and float(attrs) != 0.00):
                    return True
        return False

    def get_solution(self, storage_threshold: float):

        select = cp.Variable(self.all_l.shape, boolean=True)

        selection_constraint = cp.sum(select, axis=1) == 1.
        storage_constraint = cp.sum(cp.multiply(self.all_s, select)) <= storage_threshold
        constraints = [storage_constraint, selection_constraint]
        cost = cp.sum(cp.multiply(self.all_l, select))

        problem = cp.Problem(cp.Minimize(cost), constraints=constraints)
        m = problem.solve(verbose=True)

        return

    def get_zzh_solution(self, storage_threshold: float):

        # p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size)/self.model_size)
        p = self.get_assignment(storage_threshold)
        # print(p)
        solution = []
        for i in range(len(self.all_name)):
            upb = p[i]
            # upb = storage_threshold * p[i]
            best = None
            best_name = None
            # print(upb, self.all_l[i, :], self.all_s[i, :])
            for j in range(self.all_l.shape[1]):
                if self.all_s[i, j] <= upb+1e-4:
                    if best is None:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
                    elif self.all_l[i, j] < best:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
            solution.append(best_name)
        return solution

    def get_filtered_solution(self, storage_threshold: float, methods: set):
        # p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size)/self.model_size)
        p = self.get_assignment(storage_threshold)
        solution = []
        for i in range(len(self.all_name)):
            # upb = self.all_s[i, 0] * p[i]
            upb = p[i]
            best = None
            best_name = None
            for j in range(self.all_l.shape[1]):
                if self.all_s[i, j] <= upb + 1e-4:
                    if self.should_skip(self.all_name[i][j], methods):
                        continue
                    if best is None or self.all_l[i, j] < best:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
            solution.append(best_name)
        return solution

    def get_quantize_solution(self, storage_threshold):
        layer_list = self.operatable

        quant_l = self.all_l[:, :2]
        quant_s = self.all_s[:, :2]

        print("Getting MIP solution for pure quantization")

        select = cp.Variable(quant_l.shape, boolean=True)

        selection_constraint = cp.sum(select, axis=1) == 1.
        storage_constraint = cp.sum(cp.multiply(quant_s, select)) <= storage_threshold
        constraints = [storage_constraint, selection_constraint]
        cost = cp.sum(cp.multiply(quant_l, select))

        problem = cp.Problem(cp.Minimize(cost), constraints=constraints)
        m = problem.solve(verbose=True)
        solution = []
        if select.value is not None:
            for i in range(len(self.all_name)):
                if int(select.value[i, 0]) == 1.:
                    solution.append(self.all_name[i][0])
                else:
                    solution.append(self.all_name[i][1])
            return solution
        else:
            return None

    def get_pruning_solution(self, storage_threshold):
        p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size)/self.model_size)
        # print(p)
        solution = []
        print("Getting solution for pure pruning")
        for i in range(len(self.all_name)):
            upb = self.all_s[i, 0] * p[i]
            best = None
            best_name = None
            # print(upb, self.all_l[i, :], self.all_s[i, :])
            for j in range(self.all_l.shape[1]):
                if j % 2 == 0 and self.all_s[i, j] <= upb:
                    if best is None:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
                    elif self.all_l[i, j] < best:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
            solution.append(best_name)
        return solution

    def get_lowrank_solution(self, storage_threshold):
        p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size)/self.model_size)
        # print(p)
        solution = []
        print("Getting solution for pure lowrank")
        for i in range(len(self.all_name)):
            upb = self.all_s[i, 0] * p[i]
            best = None
            best_name = None
            # print(upb, self.all_l[i, :], self.all_s[i, :])
            for j in range(self.all_l.shape[1]):
                if j % 2 == 0 and self.all_s[i, j] <= upb:
                    if best is None:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
                    elif self.all_l[i, j] < best:
                        best = self.all_l[i, j]
                        best_name = self.all_name[i][j]
            solution.append(best_name)
        return solution

    def get_random_solution(self, storage_threshold):
        p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size) / self.model_size)
        # print(p)
        solution = []
        print("Getting solution for random selection")
        for i in range(len(self.all_name)):
            upb = self.all_s[i, 0] * p[i]
            cand = []
            # print(upb, self.all_l[i, :], self.all_s[i, :])
            for j in range(self.all_l.shape[1]):
                if self.all_s[i, j] <= upb:
                    cand.append(self.all_name[i][j])
            solution.append(random.choice(cand))
        return solution

    def get_max_storage_solution(self, storage_threshold: float):

        p = np.ones((len(self.cand),)) * (min(storage_threshold, self.model_size)/self.model_size)
        # print(p)
        solution = []
        print("Getting solution for maximum storage selection")
        for i in range(len(self.all_name)):
            upb = self.all_s[i, 0] * p[i]
            best = None
            best_name = None
            # print(upb, self.all_l[i, :], self.all_s[i, :])
            for j in range(self.all_l.shape[1]):
                if self.all_s[i, j] <= upb:
                    if best is None:
                        best = self.all_s[i, j]
                        best_name = self.all_name[i][j]
                    elif self.all_s[i, j] > best:
                        best = self.all_s[i, j]
                        best_name = self.all_name[i][j]
            solution.append(best_name)
        return solution

        # layer_list = self.operatable
        #
        # prune_l = self.all_l[:, ::2]
        # prune_s = self.all_s[:, ::2]
        # print("Getting MIP solution for pure pruning")
        # print(prune_l[0, :], prune_s[0, :], self.all_name[0])
        #
        # select = cp.Variable(prune_l.shape, boolean=True)
        #
        # selection_constraint = cp.sum(select, axis=1) == 1.
        # storage_constraint = cp.sum(cp.multiply(prune_s, select)) <= storage_threshold
        # constraints = [storage_constraint, selection_constraint]
        # cost = cp.sum(cp.multiply(prune_l, select))
        #
        # problem = cp.Problem(cp.Minimize(cost), constraints=constraints)
        # m = problem.solve(verbose=True, cplex_params={"timelimit": 60})
        # solution = []
        # if select.value is not None:
        #     for i in range(len(self.all_name)):
        #         idx = np.where(select.value[i] == 1.)[0][0]
        #         solution.append(self.all_name[i][idx])
        #     return solution
        # else:
        #     return None

max_stars_repo_name: JackFram/Neural-Flow

max_issues_repo_name: JackFram/Neural-Flow

max_forks_repo_name: JackFram/Neural-Flow

reason: import cvxpy


text:
#-----------------------------------------------------------------------------
# Copyright (c) 2014, HFTools Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
import numpy as np
import os, pdb
import hftools.file_formats

from hftools.testing import TestCase

basepath = os.path.split(__file__)[0]

from hftools.file_formats import Comments

class TestComment1(TestCase):
    def setUp(self):
        self.comment = Comments(["V1:10", "V2[V]:13", "V3  [V]  :  14"])

    def test_1(self):
        self.assertEqual(self.comment.property["V1"], [10])

    def test_2(self):
        self.assertEqual(self.comment.property["V2"], [13])

    def test_3(self):
        self.assertEqual(self.comment.property["V3"], [14])

class TestComment2(TestComment1):
    def setUp(self):
        self.comment = Comments(["V1=10", "V2[V]=13", "V3  [V]  =  14"])

class TestComment3(TestCase):
    def setUp(self):
        self.comment = Comments(["Vgs[V] = -1 ",
                                 "Vds[V] = 5.3",
                                 "Ig[uA] = 0.3 ",
                                 "Id[mA] = 20 ",
                                 "P = 20 mW"])
    def test_1(self):
        self.assertEqual(self.comment.property["Vgs"], [-1])

    def test_2(self):
        self.assertEqual(self.comment.property["Vds"], [5.3])

    def test_3(self):
        self.assertEqual(self.comment.property["Ig"], [0.3e-6])

    def test_4(self):
        self.assertEqual(self.comment.property["Id"], [0.02])

    def test_5(self):
        self.assertEqual(self.comment.property["P"], [0.02])

max_issues_repo_name: arsenovic/hftools

max_forks_repo_name: arsenovic/hftools


text:
import numpy as np
import pandas as pd

from pycalib.models import IsotonicCalibration
from betacal import BetaCalibration
from pycalib.models import SigmoidCalibration

def construct_data_row(n_folds=None, binning_name=None, n_bins=None,
                       c_hat_distance_p=None, c_hat_distance_p_debiased=None,
                       c_hat_distance_p_square=None, c_hat_distance_p_square_debiased=None,
                       c_hat_distance_c=None, c_hat_distance_c_square=None, p_distance_c = None, p_distance_c_square = None):
    return {
        "n_folds": n_folds,
        "binning": binning_name,
        "n_bins": n_bins,

        "c_hat_distance_p": c_hat_distance_p,
        "c_hat_distance_p_debiased": c_hat_distance_p_debiased,

        "c_hat_distance_p_square": c_hat_distance_p_square,
        "c_hat_distance_p_square_debiased": c_hat_distance_p_square_debiased,

        "c_hat_distance_c": c_hat_distance_c,
        "c_hat_distance_c_square": c_hat_distance_c_square,
        "p_distance_c": p_distance_c,
        "p_distance_c_square": p_distance_c_square
    }

def construct_data_row_from_raw_data(n_folds=None, binning_name=None, n_bins=None, c_hat=None, p=None, c=None):
    return construct_data_row(n_folds=n_folds, binning_name=binning_name, n_bins=n_bins,
                              c_hat_distance_p=np.mean(np.abs(c_hat - p)),
                              c_hat_distance_p_debiased=np.mean(np.abs(c_hat - p)),
                              c_hat_distance_p_square=np.mean(np.square(c_hat - p)),
                              c_hat_distance_p_square_debiased=np.mean(np.square(c_hat - p)),
                              c_hat_distance_c=np.mean(np.abs(c_hat - c)),
                              c_hat_distance_c_square=np.mean(np.square(c_hat - c)),
                              p_distance_c=np.mean(np.abs(p - c)),
                              p_distance_c_square=np.mean(np.square(p - c)))

def run_isotonic(p, y, c):
    isotonic = IsotonicCalibration()
    isotonic.fit(p, y)
    c_hat = isotonic.predict(p)

    data_row = construct_data_row_from_raw_data(n_folds=None, binning_name="isotonic",
                                                n_bins=None,
                                                c_hat=c_hat, p=p, c=c)
    return data_row

def logit(x):
    return np.log(x/(1-x))

def logit_to_scale(x, eps = 1e-16):
    l_max = logit(1-eps)
    x = np.array(x, dtype = np.float64)
    x_clip = np.clip(x, eps, 1-eps)
    return (logit(x_clip) + l_max)/(2*l_max)

def run_platt(p, y, c):

    p = logit_to_scale(p)

    platt = SigmoidCalibration()
    platt.fit(p.reshape(-1, 1), y)
    c_hat = platt.predict_proba(p.reshape(-1, 1))

    data_row = construct_data_row_from_raw_data(n_folds=None, binning_name="platt", n_bins=None,
                                                c_hat=c_hat, p=p, c=c)
    return data_row

def run_beta(p, y, c):
    bc = BetaCalibration(parameters="abm")
    bc.fit(p.reshape(-1, 1), y)
    c_hat = bc.predict(p.reshape(-1, 1))

    data_row = construct_data_row_from_raw_data(n_folds=None, binning_name="beta", n_bins=None,
                                                c_hat=c_hat, p=p, c=c)
    return data_row

def run_brier_score(p, y):
    brier_score = np.mean(np.square(p - y))

    data_row = construct_data_row(n_folds=None, binning_name="brier_score", n_bins=None,
                                  c_hat_distance_p=brier_score,
                                  c_hat_distance_p_debiased=brier_score,
                                  c_hat_distance_p_square=brier_score,
                                  c_hat_distance_p_square_debiased=brier_score,
                                  c_hat_distance_c=brier_score,
                                  c_hat_distance_c_square=brier_score)
    return [data_row], []

def run_cross_entropy_loss(p, y):
    cross_entropy_loss = log_loss(y, p)

    data_row = construct_data_row(n_folds=None, binning_name="cross_entropy_loss", n_bins=None,
                                  c_hat_distance_p=cross_entropy_loss,
                                  c_hat_distance_p_debiased=cross_entropy_loss,
                                  c_hat_distance_p_square=cross_entropy_loss,
                                  c_hat_distance_p_square_debiased=cross_entropy_loss,
                                  c_hat_distance_c=cross_entropy_loss,
                                  c_hat_distance_c_square=cross_entropy_loss)

    return [data_row], []