text:
import json
import os
import re
from unicodedata import normalize

import PIL
from PIL import Image
import boto3
import numpy as np
from base_processor.imaging import utils
from botocore.client import Config
from base_processor.imaging.deepzoom import _get_or_create_path, _get_files_path
from openslide import ImageSlide, open_slide, OpenSlide, OpenSlideError, \
    deepzoom, PROPERTY_NAME_MPP_X, PROPERTY_NAME_MPP_Y
from base_image_microscopy_processor import BaseMicroscopyImageProcessor

def slugify(text):
    text = normalize('NFKD', text.lower()).encode('ascii', 'ignore').decode()
    return re.sub('[^a-z0-9]+', '-', text)

class SVSProcessorOutputs(object):
    def __init__(self, *args, **kwargs):
        self.slide = None
        self.associated_images = []
        self.associated_image_names = []
        self.ImageCount = kwargs.get('ImageCount', 1)

        self.img_dimensions = {}
        self.num_dimensions = -1
        self.file_path = None
        self.view_path = None
        self.ometiff_path = None

        self.img_data = kwargs.get('img_data', None)
        self.img_data_dtype = kwargs.get('img_data_dtype', None)
        self.metadata = kwargs.get('metadata', {})

        self.img_rdr = kwargs.get('img_rdr', None)
        self.DimensionOrder = kwargs.get('DimensionOrder', None)
        self.ImageCount = kwargs.get('ImageCount', -1)
        self.SizeX = kwargs.get('SizeX', -1)
        self.SizeY = kwargs.get('SizeY', -1)
        self.SizeZ = kwargs.get('SizeZ', -1)
        self.SizeC = kwargs.get('SizeC', -1)
        self.SizeT = kwargs.get('SizeT', -1)
        self.PixelType = kwargs.get('PixelType', None)
        self.RGBChannelCount = kwargs.get('RGBChannelCount', -1)

        self.isRGB = kwargs.get('isRGB', False)
        self.RGBDimension = kwargs.get('RGBDimension', -1)
        self.hasTimeDimension = kwargs.get('hasTimeDimension', False)
        self.TimeDimension = kwargs.get('TimeDimension', -1)

        self.view_format = 'dzi'
        self.optimize = kwargs.get('optimize', False)
        self.tile_size = kwargs.get('tile_size', 128)
        self.tile_overlap = kwargs.get('tile_overlap', 0)
        self.tile_format = kwargs.get('tile_format', "png")
        self.image_quality = kwargs.get('image_quality', 1.0)
        self.resize_filter = kwargs.get('resize_filter', "bicubic")

    def _load_and_save_assets(self, i_xyzct, n_xyzct):
        """Load image using OpenSlide"""

        # Parse parallelization arguments
        i_x, i_y, i_z, i_c, i_t = i_xyzct
        n_x, n_y, n_z, n_c, n_t = n_xyzct

        if i_x == -1 or n_x == -1:
            i_x = 0
            n_x = 1
        if i_y == -1 or n_y == -1:
            i_y = 0
            n_y = 1
        if i_z == -1 or n_z == -1:
            i_z = 0
            n_z = 1
        if i_c == -1 or n_c == -1:
            i_c = 0
            n_c = 1
        if i_t == -1 or n_t == -1:
            i_t = 0
            n_t = 1

        # Make view directory
        if not os.path.exists('%s-zoomed' % os.path.basename(self.file_path)):
            os.makedirs('%s-zoomed' % os.path.basename(self.file_path))
        asset_format = self.view_format

        try:
            self.slide = open_slide(self.file_path)
            z = 0
            t = 0
            for name, image in self.slide.associated_images.items():
                if 'thumbnail' in name:
                    continue
                # Increment z index
                z += 1
                self.associated_image_names.append(slugify(name))
                self.associated_images.append(ImageSlide(image))

                # Save thumbnails of associated images for slide viewer stack viewer
                if (i_x, i_y) == (0, 0) or (i_x, i_y) == (-1, -1):
                    timage = image.copy()
                    timage.thumbnail((200, 200), PIL.Image.ANTIALIAS)
                    timage.save(
                        os.path.join(
                            '%s-zoomed' % os.path.basename(self.file_path),
                            'dim_Z_slice_{slice_z}_dim_T_slice_{slice_t}_thumbnail.png'.format(
                                slice_z=z,
                                slice_t=t
                            )
                        )
                    )
                    # Create large thumbnail
                    timage = image.copy()
                    timage.thumbnail((1000, 1000), PIL.Image.ANTIALIAS)
                    timage.save(
                        os.path.join(
                            '%s-zoomed' % os.path.basename(self.file_path),
                            'dim_Z_slice_{slice_z}_dim_T_slice_{slice_t}_large_thumbnail.png'.format(
                                slice_z=z,
                                slice_t=t
                            )
                        )
                    )

                    filename = os.path.join(
                        '%s-zoomed' % os.path.basename(self.file_path),
                        'dim_Z_slice_{slice_z}_dim_T_slice_{slice_t}.{fmt}'.format(
                            slice_z=z, slice_t=t, fmt=asset_format)
                    )

                    utils.save_asset(
                        image.convert("RGB"),
                        asset_format,
                        filename,
                        optimize=self.optimize, tile_size=self.tile_size,
                        tile_overlap=self.tile_overlap, tile_format=self.tile_format,
                        image_quality=self.image_quality, resize_filter=self.resize_filter
                    )
        except OpenSlideError:
            raise OpenSlideError("file is recognized by OpenSlide but an error occurred")
        except IOError:
            raise IOError("File is not recognized at all")

        # Generate tile assets
        try:
            if isinstance(self.slide, ImageSlide) or isinstance(self.slide, OpenSlide):
                z = 0
                t = 0
                filename = os.path.join(
                    '%s-zoomed' % os.path.basename(self.file_path),
                    'dim_Z_slice_{slice_z}_dim_T_slice_{slice_t}.{fmt}'.format(
                        slice_z=z, slice_t=t, fmt=asset_format)
                )

                # Check to see if processor should process entire image view instead of just a sub-region
                if n_y == -1 or n_x == -1 \
                        or i_x == -1 or i_y == -1:
                    # Read ENTIRE image into memory
                    self.LOGGER.warn('Reading entire image {} into memory for encoding into DeepZoom'.format(self.file_path))
                    image = self.slide.read_region((0, 0), 0, self.slide.dimensions).convert("RGB")

                    # Encode into DZI
                    utils.encode_dzi(image, filename, tile_size=self.tile_size, tile_overlap=self.tile_overlap,
                                     tile_format=self.tile_format, image_quality=self.image_quality)
                else:
                    # Initialize DeepZoomGenerator from OpenSlide Python API
                    deepzoomgenerator = deepzoom.DeepZoomGenerator(osr=self.slide, tile_size=self.tile_size,
                                                                   overlap=self.tile_overlap)
                    # Create tiles using OpenSlide
                    image_files = _get_or_create_path(_get_files_path(filename))
                    for level in xrange(deepzoomgenerator.level_count):
                        level_dir = _get_or_create_path(os.path.join(image_files, str(level)))

                        # Determine number of rows and columns of tiles at this level
                        num_columns = deepzoomgenerator.level_tiles[level][0]
                        num_rows = deepzoomgenerator.level_tiles[level][1]

                        # Iterate over all tiles to determine which ones fall in svs-processor sub-region
                        for column in xrange(num_columns):
                            for row in xrange(num_rows):
                                # Calculate the appropriate sub-region column and row index for this tile
                                try:
                                    tile_i_x = int(column / (float(num_columns) / n_x))
                                except ZeroDivisionError:
                                    tile_i_x = 0
                                try:
                                    tile_i_y = int(row / (float(num_rows) / n_y))
                                except ZeroDivisionError:
                                    tile_i_y = 0

                                # Check if this tile falls in the given sub-region
                                if tile_i_x == i_x and \
                                        tile_i_y == i_y:
                                    # Get tile image and save it
                                    tile = deepzoomgenerator.get_tile(level, (column, row))
                                    format = self.tile_format
                                    tile_path = os.path.join(level_dir,
                                                             '%s_%s.%s' % (column, row, format))
                                    tile_file = open(tile_path, 'wb')
                                    if self.tile_format == 'jpg':
                                        jpeg_quality = int(self.image_quality * 100)
                                        tile.save(tile_file, 'JPEG', quality=jpeg_quality)
                                    else:
                                        tile.save(tile_file)
                    # Create descriptor only once; defaulted to when we're at the very first sub-region
                    if i_y == 0 and i_x == 0:
                        open(filename, 'w').write(deepzoomgenerator.get_dzi(self.tile_format))

                        # Generate and save thumbnail
                        timage = self.slide.get_thumbnail((200, 200))
                        timage.save(
                            os.path.join(
                                '%s-zoomed' % os.path.basename(self.file_path),
                                'dim_Z_slice_0_dim_T_slice_0_thumbnail.png'
                            )
                        )

                        # Generate properties metadata.json metadata
                        metadata = []
                        for property_key, property_value in self.slide.properties.items():
                            property = {}
                            property["key"] = property_key
                            property["value"] = property_value
                            property["dataType"] = "String"
                            property["category"] = "Blackfynn"
                            property["fixed"] = False
                            property["hidden"] = False
                            metadata.append(property)
                        with open('metadata.json','w') as fp:
                            json.dump(metadata, fp)
                self.view_path = os.path.join(os.getcwd(), '%s-zoomed' % os.path.basename(self.file_path))
        except OpenSlideError:
            pass
        return

    @property
    def file_size(self):
        """Return file size"""
        return os.path.getsize(self.file_path)

    @property
    def view_size(self):
        return os.path.getsize(self.view_path)

    def get_view_asset_dict(self, storage_bucket, upload_key):
        upload_key = upload_key.rstrip('/')
        json_dict = {
            "bucket": storage_bucket,
            "key": upload_key,
            "type": "View",
            "size": self.view_size,
            "fileType": "Image"
        }
        return json_dict

    def get_dim_assignment(self):
        """Retrieve inferred dimension assignment based on number of dimensions and length of each dimension. """
        return self.DimensionOrder

    def set_img_properties(self, asset_format='dzi'):
        """Create and assign properties for the image"""

        self.view_format = asset_format

        # Load image using openslide
        try:
            self.slide = open_slide(self.file_path)
            assert isinstance(self.slide, ImageSlide) or isinstance(self.slide, OpenSlide)
            z_index = 0
            for name, image in self.slide.associated_images.items():
                if 'thumbnail' in name:
                    continue
                # Increment z index
                z_index += 1
                self.associated_image_names.append(slugify(name))
                self.associated_images.append(ImageSlide(image))
        except OpenSlideError:
            raise OpenSlideError("file is recognized by OpenSlide but an error occurred")
        except IOError:
            raise IOError("File is not recognized at all")

        # Get image data details
        self.DimensionOrder = list('XYZCT')
        self.ImageCount = len(self.slide.associated_images.values()) + 1
        self.SizeX = self.slide.dimensions[0]
        self.SizeY = self.slide.dimensions[1]
        self.SizeC = 3
        self.SizeT = 1
        self.SizeZ = len(self.associated_images) + 1
        self.PixelType = 1
        self.RGBChannelCount = 3
        self.isRGB = True

        self.RGBDimension = 3
        self.hasTimeDimension = True
        self.TimeDimension = 4

        # Set data type
        if self.PixelType == 0:  # int8
            # self.img_data = self.img_data.astype(np.int8)
            self.img_data_dtype = np.int8
        elif self.PixelType == 1:  # uint8
            # self.img_data = self.img_data.astype(np.uint8)
            self.img_data_dtype = np.uint8
        elif self.PixelType == 2:  # int16
            # self.img_data = self.img_data.astype(np.int16)
            self.img_data_dtype = np.int16
        elif self.PixelType == 3:  # uint16
            # self.img_data = self.img_data.astype(np.uint16)
            self.img_data_dtype = np.uint16
        elif self.PixelType == 4:  # int32
            # self.img_data = self.img_data.astype(np.int32)
            self.img_data_dtype = np.int32
        elif self.PixelType == 5:  # uint32
            # self.img_data = self.img_data.astype(np.uint32)
            self.img_data_dtype = np.uint32
        elif self.PixelType == 6:  # float
            # self.img_data = self.img_data.astype(np.float)
            self.img_data_dtype = np.float
        elif self.PixelType == 7:  # double
            # self.img_data = self.img_data.astype(np.double)
            self.img_data_dtype = np.double

        # Set number of dimensions of image matrix
        self.num_dimensions = 5
        self.img_data_shape = [self.SizeX, self.SizeY, self.SizeZ, self.SizeC, self.SizeT]

        dim_assignment = list('XYZCT')  # Force assignment

        self.img_dimensions['filename'] = os.path.basename(self.file_path)
        self.img_dimensions['num_dimensions'] = self.num_dimensions
        self.img_dimensions['isColorImage'] = False
        self.img_dimensions['dimensions'] = {}

        for dim in range(self.num_dimensions):
            self.img_dimensions['dimensions'][dim] = {}
            self.img_dimensions['dimensions'][dim]["assignment"] = dim_assignment[dim]
            self.img_dimensions['dimensions'][dim]["length"] = self.img_data_shape[dim]
            self.img_dimensions['dimensions'][dim]["resolution"] = -1
            self.img_dimensions['dimensions'][dim]["units"] = "um"
            if dim_assignment[dim] == 'C' and self.isRGB:
                self.RGBDimension = dim
            if dim_assignment[dim] == 'T':
                self.hasTimeDimension = True
                self.TimeDimension = dim
        self.img_dimensions['isColorImage'] = self.isRGB

        # Get metadata
        metadata = self.slide.properties

        # Get resolution
        try:
            self.img_dimensions['dimensions'][0]["resolution"] = metadata[PROPERTY_NAME_MPP_X]
        except ValueError:
            self.img_dimensions['dimensions'][0]["resolution"] = -1
        try:
            self.img_dimensions['dimensions'][1]["resolution"] = metadata[PROPERTY_NAME_MPP_Y]
        except ValueError:
            self.img_dimensions['dimensions'][1]["resolution"] = -1

        return

    def load_and_save_assets(self, svs_file_path, i_xyzct=(-1, -1, -1, -1, -1), n_xyzct=(-1, -1, -1, -1, -1),
                             asset_format='dzi'):
        """Load image and generate view assets"""

        # Set file path
        self.file_path = svs_file_path

        # Set image properties
        self.set_img_properties(asset_format=asset_format)

        # Load image
        self._load_and_save_assets(i_xyzct=i_xyzct, n_xyzct=n_xyzct)

    def save_ometiff(self):
        """Save OME-TIFF images derived from SVS file"""

        # Not implemented because of possible memory issues when loading entire 2D plane to write to OME TIFF
        raise NotImplementedError

class SVSProcessor(BaseMicroscopyImageProcessor):
    required_inputs = ['file']

    def __init__(self, *args, **kwargs):
        super(SVSProcessor, self).__init__(*args, **kwargs)
        self.session = boto3.session.Session()
        self.s3_client = self.session.client('s3', config=Config(signature_version='s3v4'))

        self.file_path = self.inputs.get('file')

        self.upload_key = None
        try:
            self.optimize = utils.str2bool(self.inputs.get('optimize_view'))
        except AttributeError:
            self.optimize = False

        try:
            self.tile_size = int(self.inputs.get('tile_size'))
        except (ValueError, KeyError, TypeError) as e:
            self.tile_size = 128

        try:
            self.tile_overlap = int(self.inputs.get('tile_overlap'))
        except (ValueError, KeyError, TypeError) as e:
            self.tile_overlap = 0

        try:
            self.tile_format = self.inputs.get('tile_format')
            if self.tile_format is None:
                self.tile_format = "png"
        except KeyError:
            self.tile_format = "png"

        try:
            self.image_quality = float(self.inputs.get('image_quality'))
        except (ValueError, KeyError, TypeError) as e:
            self.image_quality = 1.0

        try:
            self.resize_filter = self.inputs.get('resize_filter')
        except KeyError:
            self.resize_filter = "bicubic"

    def load_and_save(self, i_xyzct=(-1, -1, -1, -1, -1), n_xyzct=(-1, -1, -1, -1, -1)):
        """Run load_image and save OME-TIFF for all output files"""
        if os.path.isfile(self.file_path):
            output_file = SVSProcessorOutputs(optimize=self.optimize, tile_size=self.tile_size,
                                        tile_overlap=self.tile_overlap, tile_format=self.tile_format,
                                        image_quality=self.image_quality, resize_filter=self.resize_filter)
            output_file.load_and_save_assets(self.file_path, i_xyzct=i_xyzct, n_xyzct=n_xyzct)
            self.outputs.append(output_file)

    def task(self):
        """Run main task which will load and save view assets for all CZI output files deocded"""

        # self._load_image()
        self.LOGGER.info('Got inputs {}'.format(self.inputs))

        # Get sub_region index
        try:
            sub_region = self.inputs['sub_region_file']
            sub_region_regex = r'sub_' \
                               r'x_([0-9]+)_([0-9]+)_' \
                               r'y_([0-9]+)_([0-9]+)_' \
                               r'z_([0-9]+)_([0-9]+)_' \
                               r'c_([0-9]+)_([0-9]+)_' \
                               r't_([0-9]+)_([0-9]+).txt'
            i_x = int(re.match(re.compile(sub_region_regex), sub_region).groups()[0])
            n_x = int(re.match(re.compile(sub_region_regex), sub_region).groups()[1])
            i_y = int(re.match(re.compile(sub_region_regex), sub_region).groups()[2])
            n_y = int(re.match(re.compile(sub_region_regex), sub_region).groups()[3])
            i_z = int(re.match(re.compile(sub_region_regex), sub_region).groups()[4])
            n_z = int(re.match(re.compile(sub_region_regex), sub_region).groups()[5])
            i_c = int(re.match(re.compile(sub_region_regex), sub_region).groups()[6])
            n_c = int(re.match(re.compile(sub_region_regex), sub_region).groups()[7])
            i_t = int(re.match(re.compile(sub_region_regex), sub_region).groups()[8])
            n_t = int(re.match(re.compile(sub_region_regex), sub_region).groups()[9])
        except (KeyError, IndexError, AttributeError):
            i_x = -1
            n_x = -1
            i_y = -1
            n_y = -1
            i_z = -1
            n_z = -1
            i_c = -1
            n_c = -1
            i_t = -1
            n_t = -1

        # Load and save view images
        self.load_and_save(i_xyzct=(i_x, i_y, i_z, i_c, i_t), n_xyzct=(n_x, n_y, n_z, n_c, n_t))

        # Save dimensions object as JSON in view/ directory (for now)
        if os.path.isfile(self.file_path):
            # Output file is just the one and only file in outputs
            output_file = self.outputs[0]

            # Save dimensions object as JSON in view/ directory (for now)
            with open(os.path.join('%s-zoomed' % os.path.basename(self.file_path), 'dimensions.json'), 'w') as fp:
                json.dump(output_file.img_dimensions, fp)

            # Create create-asset JSON object file called view_asset_info.json
            self.upload_key = os.path.join(
                self.settings.storage_directory,
                os.path.basename(output_file.file_path) + '-zoomed'
            )

            with open('view_asset_info.json', 'w') as fp:
                json.dump(output_file.get_view_asset_dict(
                    self.settings.storage_bucket,
                    self.upload_key
                ), fp)

            # Generate properties metadata.json metadata
            metadata = []
            img_dimensions = self.outputs[0].img_dimensions
            for dim in range(self.outputs[0].num_dimensions):
                for property_key_suffix in ["assignment", "length", "resolution", "units"]:
                    # Initialize property
                    property = {}

                    # Set property key and value
                    property_key = 'dimensions.%i.%s' % (dim, property_key_suffix)
                    property_value = str(img_dimensions['dimensions'][dim][property_key_suffix])

                    # Create property instance
                    property["key"] = property_key
                    property["value"] = property_value
                    property["dataType"] = "String"
                    property["category"] = "Blackfynn"
                    property["fixed"] = False
                    property["hidden"] = False
                    metadata.append(property)
            for property_key, property_value in self.outputs[0].slide.properties.items():
                property = {}
                property["key"] = property_key
                property["value"] = property_value
                property["dataType"] = "String"
                property["category"] = "Blackfynn"
                property["fixed"] = False
                property["hidden"] = False
                metadata.append(property)
            with open('metadata.json', 'w') as fp:
                json.dump(metadata, fp)

            ## Hack for backwards compatibility for DeepZoom viewer
            os.system('cp -r %s-zoomed/dim_Z_slice_0_dim_T_slice_0.dzi %s-zoomed/slide.dzi' %
                      (os.path.basename(self.file_path), os.path.basename(self.file_path)))
            os.system('cp -r %s-zoomed/dim_Z_slice_0_dim_T_slice_0_files %s-zoomed/slide_files' %
                      (os.path.basename(self.file_path), os.path.basename(self.file_path)))

max_stars_repo_name: Pennsieve/svs-processor

max_issues_repo_name: Pennsieve/svs-processor

max_forks_repo_name: Pennsieve/svs-processor


text:
import numpy as np
from integrator import *
from laser import *
import matplotlib.pyplot as plt
from const import *
class expectation_values(integrator):
    '''
    Evaluation of thermally averaged, time-dependent expectation values.
    '''

    def cos2(self, Cstor, Jweight):
        tt = np.linspace(0,5,1000)
        cos2 = np.zeros(tt.size,dtype = 'complex')
        for J in range(self.Jmax):
            for M in range(-J,J+1):
                for jj in range(self.Jmax-2):
                    w = 4*jj+6
                    phi = np.angle(Cstor[J,M,jj])-np.angle(Cstor[J,M,jj+2])
                    cos2 += Jweight[J]/(2*J+1)*(abs(Cstor[J,M,jj])**2*self.c2(jj,M) +
                            abs(Cstor[J,M,jj])*abs(Cstor[J,M,jj+2])*self.cp2(jj,M)*np.cos(w*tt+phi))
        return tt, cos2

    def cos2plot(self, tt, cos2, molecule):
        plt.figure()
        plt.plot(tt*hbar/self.B*10**12,np.real(cos2),'k-')
        plt.xlabel('Time [ps]')
        plt.ylabel('<cos$^2\Theta$>')
        plt.grid()
        plt.ylim(0,1)
        plt.show()
        return 0

ext: py

reason: import numpy


text:
from .sortingextractor import SortingExtractor
import numpy as np

# Encapsulates a subset of a spike sorted data file

class SubSortingExtractor(SortingExtractor):

    def __init__(self, parent_sorting, *, unit_ids=None, renamed_unit_ids=None, start_frame=None, end_frame=None):
        SortingExtractor.__init__(self)
        self._parent_sorting = parent_sorting
        self._unit_ids = unit_ids
        self._renamed_unit_ids = renamed_unit_ids
        self._start_frame = start_frame
        self._end_frame = end_frame
        if self._unit_ids is None:
            self._unit_ids = self._parent_sorting.get_unit_ids()
        if self._renamed_unit_ids is None:
            self._renamed_unit_ids = self._unit_ids
        if self._start_frame is None:
            self._start_frame = 0
        if self._end_frame is None:
            self._end_frame = float("inf")
        self._original_unit_id_lookup = {}
        for i in range(len(self._unit_ids)):
            self._original_unit_id_lookup[self._renamed_unit_ids[i]] = self._unit_ids[i]
        self.copy_unit_properties(parent_sorting, unit_ids=self._renamed_unit_ids)
        self.copy_unit_spike_features(parent_sorting, unit_ids=self._renamed_unit_ids)

    def get_unit_ids(self):
        return list(self._renamed_unit_ids)

    def get_unit_spike_train(self, unit_id, start_frame=None, end_frame=None):
        if start_frame is None:
            start_frame = 0
        if end_frame is None:
            end_frame = np.Inf
        if (isinstance(unit_id, (int, np.integer))):
            if (unit_id in self.get_unit_ids()):
                original_unit_id = self._original_unit_id_lookup[unit_id]
            else:
                raise ValueError("Non-valid unit_id")
        else:
            raise ValueError("unit_id must be an int")
        sf = self._start_frame + start_frame
        ef = self._start_frame + end_frame
        if sf < self._start_frame:
            sf = self._start_frame
        if ef > self._end_frame:
            ef = self._end_frame
        return self._parent_sorting.get_unit_spike_train(unit_id=original_unit_id, start_frame=sf,
                                                         end_frame=ef) - self._start_frame

    def get_sampling_frequency(self):
        return self._parent_sorting.get_sampling_frequency()

    def copy_unit_properties(self, sorting, unit_ids=None):
        if unit_ids is None:
            unit_ids = self.get_unit_ids()
        if isinstance(unit_ids, int):
            sorting_unit_id = unit_ids
            if sorting is self._parent_sorting:
                sorting_unit_id = self.get_original_unit_ids(unit_ids)
            curr_property_names = sorting.get_unit_property_names(unit_id=sorting_unit_id)
            for curr_property_name in curr_property_names:
                value = sorting.get_unit_property(unit_id=sorting_unit_id, property_name=curr_property_name)
                self.set_unit_property(unit_id=unit_ids, property_name=curr_property_name, value=value)
        else:
            for unit_id in unit_ids:
                sorting_unit_id = unit_id
                if sorting is self._parent_sorting:
                    sorting_unit_id = self.get_original_unit_ids(unit_id)
                curr_property_names = sorting.get_unit_property_names(unit_id=sorting_unit_id)
                for curr_property_name in curr_property_names:
                    value = sorting.get_unit_property(unit_id=sorting_unit_id, property_name=curr_property_name)
                    self.set_unit_property(unit_id=unit_id, property_name=curr_property_name, value=value)

    def copy_unit_spike_features(self, sorting, unit_ids=None):
        if unit_ids is None:
            unit_ids = self.get_unit_ids()
        if isinstance(unit_ids, int):
            sorting_unit_id = unit_ids
            if sorting is self._parent_sorting:
                sorting_unit_id = self.get_original_unit_ids(unit_ids)
            curr_feature_names = sorting.get_unit_spike_feature_names(unit_id=sorting_unit_id)
            for curr_feature_name in curr_feature_names:
                value = sorting.get_unit_spike_features(unit_id=sorting_unit_id, feature_name=curr_feature_name)
                self.set_unit_spike_features(unit_id=unit_ids, feature_name=curr_feature_name, value=value)
        else:
            for unit_id in unit_ids:
                sorting_unit_id = unit_id
                if sorting is self._parent_sorting:
                    sorting_unit_id = self.get_original_unit_ids(unit_id)
                curr_feature_names = sorting.get_unit_spike_feature_names(unit_id=sorting_unit_id)
                for curr_feature_name in curr_feature_names:
                    value = sorting.get_unit_spike_features(unit_id=sorting_unit_id, feature_name=curr_feature_name)
                    self.set_unit_spike_features(unit_id=unit_id, feature_name=curr_feature_name, value=value)

    def get_original_unit_ids(self, unit_ids):
        if isinstance(unit_ids, (int, np.integer)):
            if unit_ids in self.get_unit_ids():
                original_unit_ids = self._original_unit_id_lookup[unit_ids]
            else:
                raise ValueError("Non-valid unit_id")
        else:
            original_unit_ids = []
            for unit_id in unit_ids:
                if isinstance(unit_id, (int, np.integer)):
                    if unit_id in self.get_unit_ids():
                        original_unit_id = self._original_unit_id_lookup[unit_id]
                        original_unit_ids.append(original_unit_id)
                    else:
                        raise ValueError("Non-valid unit_id")
                else:
                    raise ValueError("unit_id must be an int")
        return original_unit_ids

max_stars_repo_name: dqx-github/spikeextractors

max_issues_repo_name: dqx-github/spikeextractors

max_forks_repo_name: dqx-github/spikeextractors


text:
import tensorflow as tf
import numpy as np
import os
import time
import datetime
import wget
from input_helpers import InputHelper
from siamese_network import SiameseLSTM
from random import random
from glove import read_glove_vectors

corpus_size = 10000  # val 33281

# Parameters
# ==================================================
tf.flags.DEFINE_integer("embedding_dim", 300, "Dimensionality of character embedding (default: 300)")
tf.flags.DEFINE_float("dropout_keep_prob", 0.5, "Dropout keep probability (default: 0.5)")
tf.flags.DEFINE_float("l2_reg_lambda", 0.0, "L2 regularizaion lambda (default: 0.0)")
tf.flags.DEFINE_string("training_files", "../data/similarity/train1.tsv", "training file (default: None)")
tf.flags.DEFINE_integer("hidden_units", 250, "Number of hidden units in softmax regression layer (default:50)")
tf.flags.DEFINE_integer("number_of_layers", 2, "Number of hidden layers for LSTM")
tf.flags.DEFINE_bool("bi_directional", False, "Whether or not to use BiDirectional LSTMS")
tf.flags.DEFINE_integer("max_document_length", 30, "The maximum size input elements (default: 30)")

# Training parameters
tf.flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)")
tf.flags.DEFINE_float("learning_rate", 0.001, "Learning rate (default: 0.001)")
tf.flags.DEFINE_float("lr_decay", 0.85, "Learning rate (default: 0.9)")
tf.flags.DEFINE_bool("use_gradient_clipping", False, "Use gradient clipping (default: False)")
tf.flags.DEFINE_integer("gradient_clipping", 5, "Gradient clipping threshold (default: 1.25)")
tf.flags.DEFINE_string("loss_function", "cross_entropy", "The loss function to use (default: cross_entropy)")
tf.flags.DEFINE_integer("num_epochs", 25, "Number of training epochs (default: 200)")
tf.flags.DEFINE_integer("evaluate_every", 2000, "Evaluate model on dev set after this many steps (default: 100)")
tf.flags.DEFINE_integer("checkpoint_every", 2000, "Save model after this many steps (default: 100)")
tf.flags.DEFINE_integer("lr_decay_every", 10000, "Decay learning rate after this many steps (default: 10000)")

# Misc Parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")

FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
print("batches per epoch", int(corpus_size / FLAGS.batch_size))
for attr, value in sorted(FLAGS.__flags.items()):
    print("{}={}".format(attr.upper(), value))
print("")

if FLAGS.training_files is None:
    print("Input Files List is empty. use --training_files argument.")
    exit()

inpH = InputHelper()
train_set, dev_set, vocab_processor, sum_no_of_batches, char_set = inpH.getDataSets(FLAGS.training_files, 5, FLAGS.batch_size, max_length=FLAGS.max_document_length)

embeddings_file = '../data/glove/glove.6B/glove.6B.300d.txt'
if not os.path.isfile(embeddings_file):
    os.makedirs('../data/glove/glove.6B')
    embeddings_url = 'http://learning-resources.jorgetravieso.com/embeddings/glove/glove.6B.300d.txt'
    wget.download(embeddings_url, out=embeddings_file)

embeddings = read_glove_vectors(vocab_processor.vocabulary_._mapping, embeddings_file, 300)

# Training
# ==================================================
print("starting graph def")
init = tf.global_variables_initializer()
with tf.Session() as sess:
    print("Started session")
    sess.run(init)
    siameseModel = SiameseLSTM(
        vocab_size=len(vocab_processor.vocabulary_),
        embedding_size=FLAGS.embedding_dim,
        hidden_units=FLAGS.hidden_units,
        number_of_layers=FLAGS.number_of_layers,
        l2_reg_lambda=FLAGS.l2_reg_lambda,
        batch_size=FLAGS.batch_size,
        word_embeddings=embeddings,
        n_chars=len(char_set),
        use_bi_lstm=FLAGS.bi_directional,
        loss_function=FLAGS.loss_function,
        max_length=FLAGS.max_document_length)

    # Define Training procedure
    global_step = tf.Variable(0, name="global_step", trainable=False)
    learning_rate = tf.train.exponential_decay(FLAGS.learning_rate, global_step, FLAGS.lr_decay_every, FLAGS.lr_decay, staircase=True)

    optimizer = tf.train.AdamOptimizer(learning_rate)
    print("Initialized siameseModel object")

    tr_op_set = None
    if FLAGS.use_gradient_clipping:
        gradients, variables = zip(*optimizer.compute_gradients(siameseModel.loss))
        gradients, _ = tf.clip_by_global_norm(gradients, FLAGS.gradient_clipping)
        tr_op_set = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step)
    else:
        grads_and_vars = optimizer.compute_gradients(siameseModel.loss)
        tr_op_set = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
    print("Defined training_ops")

    # Output directory for models and summaries
    timestamp = str(int(time.time()))
    out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
    print("Writing to {}\n".format(out_dir))

    # Checkpoint directory. TensorFlow assumes this directory already exists so we need to create it
    checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
    checkpoint_prefix = os.path.join(checkpoint_dir, "model")
    if not os.path.exists(checkpoint_dir):
        os.makedirs(checkpoint_dir)
    saver = tf.train.Saver(tf.all_variables(), max_to_keep=2)

    # Print Hyper-parameters to file:
    with open(os.path.join(out_dir, "README.txt"), "w") as readme:
        for attr, value in sorted(FLAGS.__flags.items()):
            readme.write("{}={}\n".format(attr.upper(), value))
        readme.write("\n")

    # Write vocabulary
    vocab_processor.save(os.path.join(checkpoint_dir, "vocab"))

    # Define Summary Writers
    train_writer = tf.summary.FileWriter(os.path.join(out_dir, "summaries", "train"), tf.get_default_graph())
    dev_writer = tf.summary.FileWriter(os.path.join(out_dir, "summaries", "dev"))

    # Initialize all variables
    sess.run(tf.initialize_all_variables())

    # Merged summaries
    merged = tf.summary.merge_all()

    def train_step(x1_batch, x2_batch, y_batch):
        """
		A single training step
		"""
        feed_dict = get_fd(x1_batch, x2_batch, y_batch)
        _, step, loss, accuracy, pred, summary = sess.run(
            [tr_op_set, global_step, siameseModel.loss, siameseModel.accuracy, siameseModel.pred, merged],
            feed_dict)
        time_str = datetime.datetime.now().isoformat()
        print("TRAIN {}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))

        def reverse_tokens(word_ids):
            return '\n'.join(vocab_processor.reverse(word_ids))

        for i in range(0):
            print("X1", reverse_tokens([x1_batch[i]]))
            print("X2", reverse_tokens([x2_batch[i]]))
            print(i, ", Target:", y_batch[i], ", Prediction:", '[{:g}, {:g}]'.format(pred[i][0], pred[i][1]), "\n")
        train_writer.add_summary(summary, step)
        # time.sleep()

    def get_lengths(x_batch):
        def compute_size(x):
            last_index = None
            for i, e in enumerate(x):
                if e is not 0:
                    last_index = i
            return len(x) if last_index is None else last_index + 1
        return [compute_size(x) for x in x_batch]

    def get_max_length(x_batch):
        return max(len(word_ids) for word_ids in x_batch)

    def swap(a, b):
        return b, a

    def get_fd(x1_batch, x2_batch, y_batch, dropout=FLAGS.dropout_keep_prob):
        if random() < 0.5:
            x1_batch, x2_batch = swap(x1_batch, x2_batch)
        return {
            siameseModel.input_x1: x1_batch,
            siameseModel.input_x1_lens: get_lengths(x1_batch),
            siameseModel.input_x2: x2_batch,
            siameseModel.input_x2_lens: get_lengths(x2_batch),
            siameseModel.input_y: y_batch,
            siameseModel.dropout_keep_prob: dropout,
        }

    def dev_step(x1_batch, x2_batch, y_batch):
        """
			A single training step
		"""
        step, loss, accuracy, pred, summary = sess.run(
            [global_step, siameseModel.loss, siameseModel.accuracy, siameseModel.pred, merged],
            get_fd(x1_batch, x2_batch, y_batch, dropout=1.0))
        time_str = datetime.datetime.now().isoformat()
        print("DEV {}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
        dev_writer.add_summary(summary, step)
        return accuracy

    # Generate batches
    batches = inpH.batch_iter(
        list(zip(train_set[0], train_set[1], train_set[2])), FLAGS.batch_size, FLAGS.num_epochs, shuffle=True)

    ptr = 0
    max_validation_acc = 0.0
    for nn in range(sum_no_of_batches * FLAGS.num_epochs):
        batch = next(batches)
        if len(batch) < 1:
            continue
        x1_batch, x2_batch, y_batch = zip(*batch)
        if len(y_batch) < 1:
            continue
        train_step(x1_batch, x2_batch, y_batch)
        current_step = tf.train.global_step(sess, global_step)
        sum_acc = 0.0
        counter = 0
        if current_step % FLAGS.evaluate_every == 0:
            print("\nEvaluation:")
            dev_batches = inpH.batch_iter(list(zip(dev_set[0], dev_set[1], dev_set[2])), FLAGS.batch_size, 1)
            for db in dev_batches:
                if len(db) < 1:
                    continue
                x1_dev_b, x2_dev_b, y_dev_b = zip(*db)
                if len(y_dev_b) < 1:
                    continue
                counter += 1
                acc = dev_step(x1_dev_b, x2_dev_b, y_dev_b)
                sum_acc = sum_acc + acc
            print("Averaged batch accuracy was: ", sum_acc / counter)
        time.sleep(2)
        if current_step % FLAGS.checkpoint_every == 0:
            if sum_acc >= max_validation_acc:
                max_validation_acc = sum_acc
                saver.save(sess, checkpoint_prefix, global_step=current_step)
                tf.train.write_graph(sess.graph.as_graph_def(), checkpoint_prefix, "graph" + str(nn) + ".pb",
                                     as_text=False)
                print("Saved model {} with sum_accuracy={} checkpoint to {}\n".format(nn, max_validation_acc,
                                                                                checkpoint_prefix))
            else:
                print("Ignoring the current model as the accuracy didn't improve")

max_stars_repo_name: jtraviesor/alfred-tf-trainer

max_issues_repo_name: jtraviesor/alfred-tf-trainer

max_forks_repo_name: jtraviesor/alfred-tf-trainer


text:
# Copyright 2021 QHAna plugin runner contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from http import HTTPStatus
from io import StringIO
from json import dumps, loads
from tempfile import SpooledTemporaryFile
from typing import Dict, Mapping, Optional

import marshmallow as ma
from celery.canvas import chain
from celery.result import AsyncResult
from celery.utils.log import get_task_logger
from flask import Response
from flask.app import Flask
from flask.globals import request
from flask.helpers import url_for
from flask.templating import render_template
from flask.views import MethodView
from marshmallow import EXCLUDE
from sqlalchemy.sql.expression import select

from qhana_plugin_runner.api.util import (
    FileUrl,
    FrontendFormBaseSchema,
    MaBaseSchema,
    SecurityBlueprint,
)
from qhana_plugin_runner.celery import CELERY
from qhana_plugin_runner.db.db import DB
from qhana_plugin_runner.db.models.tasks import ProcessingTask
from qhana_plugin_runner.requests import open_url
from qhana_plugin_runner.storage import STORE
from qhana_plugin_runner.tasks import save_task_error, save_task_result
from qhana_plugin_runner.util.plugins import QHAnaPluginBase, plugin_identifier

_plugin_name = "hybrid-autoencoder"
__version__ = "v0.1.0"
_identifier = plugin_identifier(_plugin_name, __version__)

HA_BLP = SecurityBlueprint(
    _identifier,
    __name__,
    description="Hybrid Autoencoder plugin API.",
    template_folder="hybrid_ae_templates",
)

class HybridAutoencoderResponseSchema(MaBaseSchema):
    name = ma.fields.String(required=True, allow_none=False, dump_only=True)
    version = ma.fields.String(required=True, allow_none=False, dump_only=True)
    identifier = ma.fields.String(required=True, allow_none=False, dump_only=True)

class HybridAutoencoderTaskResponseSchema(MaBaseSchema):
    name = ma.fields.String(required=True, allow_none=False, dump_only=True)
    task_id = ma.fields.String(required=True, allow_none=False, dump_only=True)
    task_result_url = ma.fields.Url(required=True, allow_none=False, dump_only=True)

class HybridAutoencoderPennylaneRequestSchema(FrontendFormBaseSchema):
    input_data = FileUrl(
        required=True, allow_none=False, load_only=True, metadata={"label": "Input Data"}
    )
    number_of_qubits = ma.fields.Integer(
        required=True,
        allow_none=False,
        load_only=True,
        metadata={"label": "Number of Qubits"},
    )
    embedding_size = ma.fields.Integer(
        required=True,
        allow_none=False,
        load_only=True,
        metadata={"label": "Embedding Size"},
    )
    qnn_name = ma.fields.String(
        required=True, allow_none=False, load_only=True, metadata={"label": "QNN Name"}
    )
    training_steps = ma.fields.Integer(
        required=True,
        allow_none=False,
        load_only=True,
        metadata={"label": "Training Steps"},
    )

@HA_BLP.route("/")
class PluginsView(MethodView):
    """Plugins collection resource."""

    @HA_BLP.response(HTTPStatus.OK, HybridAutoencoderResponseSchema())
    @HA_BLP.require_jwt("jwt", optional=True)
    def get(self):
        """Demo endpoint returning the plugin metadata."""
        return {
            "name": HybridAutoencoderPlugin.instance.name,
            "version": HybridAutoencoderPlugin.instance.version,
            "identifier": HybridAutoencoderPlugin.instance.identifier,
        }

@HA_BLP.route("/ui/")
class MicroFrontend(MethodView):
    """Micro frontend for the hybrid autoencoder plugin."""

    example_inputs = {
        "inputData": "data:text/plain,0,0,0,0,0,0,0,0,0,0",
        "numberOfQubits": 3,
        "embeddingSize": 2,
        "qnnName": "QNN3",
        "trainingSteps": 100,
    }

    @HA_BLP.html_response(
        HTTPStatus.OK, description="Micro frontend of the hybrid autoencoder plugin."
    )
    @HA_BLP.arguments(
        HybridAutoencoderPennylaneRequestSchema(
            partial=True, unknown=EXCLUDE, validate_errors_as_result=True
        ),
        location="query",
        required=False,
    )
    @HA_BLP.require_jwt("jwt", optional=True)
    def get(self, errors):
        """Return the micro frontend."""
        return self.render(request.args, errors)

    @HA_BLP.html_response(
        HTTPStatus.OK, description="Micro frontend of the hybrid autoencoder plugin."
    )
    @HA_BLP.arguments(
        HybridAutoencoderPennylaneRequestSchema(
            partial=True, unknown=EXCLUDE, validate_errors_as_result=True
        ),
        location="form",
        required=False,
    )
    @HA_BLP.require_jwt("jwt", optional=True)
    def post(self, errors):
        """Return the micro frontend with prerendered inputs."""
        return self.render(request.form, errors)

    def render(self, data: Mapping, errors: dict):
        print(">>>", errors)
        schema = HybridAutoencoderPennylaneRequestSchema()
        return Response(
            render_template(
                "hybrid_ae_template.html",
                name=HybridAutoencoderPlugin.instance.name,
                version=HybridAutoencoderPlugin.instance.version,
                schema=schema,
                values=data,
                errors=errors,
                process=url_for(f"{HA_BLP.name}.HybridAutoencoderPennylaneAPI"),
                example_values=url_for(
                    f"{HA_BLP.name}.MicroFrontend", **self.example_inputs
                ),
            )
        )

@HA_BLP.route("/process/pennylane/")
class HybridAutoencoderPennylaneAPI(MethodView):
    """Start a long running processing task."""

    @HA_BLP.response(HTTPStatus.OK, HybridAutoencoderTaskResponseSchema)
    @HA_BLP.arguments(
        HybridAutoencoderPennylaneRequestSchema(unknown=EXCLUDE), location="form"
    )
    @HA_BLP.require_jwt("jwt", optional=True)
    def post(self, req_dict):
        """Start the demo task."""
        db_task = ProcessingTask(
            task_name=hybrid_autoencoder_pennylane_task.name,
            parameters=dumps(req_dict),
        )
        db_task.save(commit=True)

        # all tasks need to know about db id to load the db entry
        task: chain = hybrid_autoencoder_pennylane_task.s(
            db_id=db_task.id
        ) | save_task_result.s(db_id=db_task.id)
        # save errors to db
        task.link_error(save_task_error.s(db_id=db_task.id))
        result: AsyncResult = task.apply_async()

        db_task.task_id = result.id
        db_task.save(commit=True)

        return {
            "name": hybrid_autoencoder_pennylane_task.name,
            "task_id": str(result.id),
            "task_result_url": url_for("tasks-api.TaskView", task_id=str(result.id)),
        }

class HybridAutoencoderPlugin(QHAnaPluginBase):

    name = _plugin_name
    version = __version__

    def __init__(self, app: Optional[Flask]) -> None:
        super().__init__(app)

    def get_api_blueprint(self):
        return HA_BLP

    def get_requirements(self) -> str:
        # return "git+ssh://git@github.com/UST-QuAntiL/MuseEmbeddings.git@6cc2f18fdd6b9483d5aaa68d12f8e01cb6329dde#egg=hybrid_autoencoders"
        return ""

TASK_LOGGER = get_task_logger(__name__)

@CELERY.task(
    name=f"{HybridAutoencoderPlugin.instance.identifier}.pennylane_hybrid_autoencoder_task",
    bind=True,
)
def hybrid_autoencoder_pennylane_task(self, db_id: int) -> str:
    import numpy as np
    from hybrid_autoencoders import simple_api

    TASK_LOGGER.info(
        f"Starting new hybrid autoencoder pennylane task with db id '{db_id}'"
    )
    task_data: ProcessingTask = DB.session.execute(
        select(ProcessingTask).filter_by(id=db_id)
    ).scalar_one()

    if task_data is None:
        msg = f"Could not load task data with id {db_id} to read parameters!"
        TASK_LOGGER.error(msg)
        raise KeyError(msg)

    params: Dict = loads(task_data.parameters or "{}")
    input_data_url: str = params.get("input_data", None)
    q_num: int = params.get("number_of_qubits", None)
    embedding_size: int = params.get("embedding_size", None)
    qnn_name: str = params.get("qnn_name", None)
    steps: int = params.get("training_steps", None)

    TASK_LOGGER.info(
        f"input_data: {input_data_url}, q_num: {q_num}, embedding_size: {embedding_size}, qnn_name: {qnn_name}, steps: {steps}"
    )

    if None in [input_data_url, q_num, embedding_size, qnn_name, steps]:
        raise ValueError("Request is missing one or more values.")

    with open_url(input_data_url, stream=True) as url_data:
        input_data_arr = np.genfromtxt(url_data.iter_lines(), delimiter=",")

    if input_data_arr.ndim == 1:
        input_data_arr = input_data_arr.reshape((1, -1))

    output_arr = simple_api.pennylane_hybrid_autoencoder(
        input_data_arr, q_num, embedding_size, qnn_name, steps
    )

    with SpooledTemporaryFile(mode="w") as output:
        np.savetxt(output, output_arr, delimiter=",")
        STORE.persist_task_result(
            db_id, output, "out.csv", "autoencoder-result", "text/csv"
        )
        output.seek(
            0
        )  # TODO remove separate output if task output is already persisted as file
        return "".join(output.readlines())

max_stars_repo_name: UST-QuAntiL/qhana-plugin-runner

max_issues_repo_name: UST-QuAntiL/qhana-plugin-runner

max_forks_repo_name: UST-QuAntiL/qhana-plugin-runner


text:
"""
Originally adapted from https://github.com/cajohare/AxionLimits/
"""

from numpy import *
import matplotlib.pyplot as plt

import sys
from Dirs import axionlimits_dir

from PlotFuncs import BlackHoleSpins, AxionPhoton, MySaveFig, UpperFrequencyAxis,col_alpha,FigSetup

fig,ax = FigSetup(Shape='Square',xlab = r'Axion mass $m_a$ [eV]',ylab='$|g_{a\gamma\gamma}|$ [GeV$^{-1}$]',mathpazo=True,\
                 m_min=1e-6,m_max=1e-3,g_min=2e-16,g_max=1e-9,lfs=40, tfs=35,xtick_rotation=0,FrequencyAxis=True,N_Hz=1e6,upper_xlabel=r"Frequency [MHz]",xlabel_pad=15)

y2 = 1e-9
#xscale = 1.03*(log10(3e-4) - log10(1e-6))/(log10(1e-3) - log10(1e-6))

f_ax = 0.9
scale = (f_ax)**-0.5

ADMX_col = 'crimson'

# CAST
AxionPhoton.Helioscopes(ax,projection=False,text_on=False)
plt.gcf().text(0.2*(1-0.005),0.805*(1+0.005),r'{\bf CAST}',color='k',fontsize=40,rotation=0,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.2,0.805,r'{\bf CAST}',color=col_alpha(ADMX_col,0.0),fontsize=40,rotation=0,rotation_mode='anchor',ha='center',va='center')

# QCD axion
AxionPhoton.QCDAxion(ax,text_on=False,C_center=abs(5/3-1.92)*(44/3-1.92)/2,C_width=0.7)
def g_x(C_ag,m_a):
        return 2e-10*C_ag*m_a
mvals = array([1e-6,1e-3])
plt.plot(mvals,g_x(44/3-1.95,mvals),'--',lw=3,color='brown',zorder=0.0,alpha=0.2)
plt.plot(mvals,g_x(abs(5/3-1.92),mvals),'--',lw=3,color='brown',zorder=0.1,alpha=0.2)
#plt.gcf().text(0.89,0.53,r'$E/N = 44/3$',fontsize=20,color='brown',rotation=13,rotation_mode='anchor',ha='right',alpha=0.9)
#plt.gcf().text(0.89,0.305,r'$E/N = 5/3$',fontsize=20,color='brown',rotation=13,rotation_mode='anchor',ha='right',alpha=0.9)
plt.gcf().text(0.89,0.49,r'KSVZ',fontsize=20,color='darkred',rotation=20,rotation_mode='anchor',ha='right',alpha=1)
plt.gcf().text(0.89,0.413,r'DFSZ',fontsize=20,color='darkred',rotation=20,rotation_mode='anchor',ha='right',alpha=1)

# RBF/UF
UF_col = [0.8,0,0]
RBF_col = 'Darkred'
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/RBF_UF_Haloscopes.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor='darkred',zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/UF.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=[0.8,0,0],zorder=0.1)
plt.gcf().text(0.315*(1-0.003),0.32*(1+0.003),r'{\bf UF}',color='k',fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.315,0.32,r'{\bf UF}',color=col_alpha(UF_col,1),fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.355*(1-0.003),0.55*(1+0.003),r'{\bf RBF}',color='k',fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.355,0.55,r'{\bf RBF}',color=col_alpha(RBF_col,0.1),fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')

# ADMX
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX2018.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX2019_1.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX2019_2.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX2021.txt")
plt.fill_between(dat[:,0],dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
plt.gcf().text(0.22*(1-0.005),0.4*(1+0.005),r'{\bf ADMX}',color='k',fontsize=35,rotation=90,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.22,0.4,r'{\bf ADMX}',color=col_alpha(ADMX_col,0.2),fontsize=35,rotation=90,rotation_mode='anchor',ha='center',va='center')

# HAYSTAC
HAYSTAC_col = 'orangered'
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/HAYSTAC_highres.txt")
dat2 = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/HAYSTAC_2020_highres.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=HAYSTAC_col,zorder=0.1)
plt.fill_between(dat2[:,0],scale*dat2[:,1],y2=y2,edgecolor='k',facecolor=HAYSTAC_col,zorder=0.1)
plt.gcf().text(0.436,0.357,r'{\bf HAYSTAC}',color=col_alpha(HAYSTAC_col,1),fontsize=17,rotation=90,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.473,0.42,r'{\bf HAYSTAC}',color=col_alpha(HAYSTAC_col,1),fontsize=17,rotation=90,rotation_mode='anchor',ha='center',va='center')

# CAPP
CAPP_col = [1, 0.1, 0.37]
dat1 = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/CAPP-1.txt")
dat2 = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/CAPP-2.txt")
dat3 = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/CAPP-3.txt")
plt.fill_between(dat1[:,0],scale*dat1[:,1],y2=y2,edgecolor='k',facecolor=CAPP_col,zorder=0.1)
plt.fill_between(dat2[:,0],scale*dat2[:,1],y2=y2,edgecolor='k',facecolor=CAPP_col,zorder=0.1)
plt.fill_between(dat3[:,0],scale*dat3[:,1],y2=y2,edgecolor='k',facecolor=CAPP_col,zorder=0.1)
plt.gcf().text(0.385,0.31,r'{\bf CAPP}',color=col_alpha(CAPP_col,1),fontsize=20,rotation=90,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.425,0.45,r'{\bf CAPP}',color=col_alpha(CAPP_col,1),fontsize=15,rotation=-90,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.347,0.35,r'{\bf CAPP}',color=col_alpha(CAPP_col,1),fontsize=15,rotation=-90,rotation_mode='anchor',ha='center',va='center')

# ORGAN
ORGAN_col = [0.7,0.2,0.05]
AxionPhoton.ORGAN(ax,text_on=False,col=ORGAN_col, f_ax = f_ax)
plt.gcf().text(0.645,0.615,r'{\bf ORGAN}',color=col_alpha(ORGAN_col,1),fontsize=20,rotation=90,rotation_mode='anchor',ha='center',va='center')

# GrAHal
GrAHal_col = '#b53e5a'
AxionPhoton.GrAHal(ax,text_on=False,col=GrAHal_col, f_ax = f_ax)
plt.gcf().text(0.497,0.51,r'{\bf GrAHal}',color=col_alpha(GrAHal_col,1),fontsize=15,rotation=-90,rotation_mode='anchor',ha='center',va='center')

# ADMX Sidecar
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX_Sidecar.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/ADMX_Sidecar_JTWPA.txt")
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor=ADMX_col,zorder=0.1)
plt.gcf().text(0.471,0.55,r'{\bf ADMX}',color=ADMX_col,fontsize=18,rotation=90,rotation_mode='anchor',ha='center',va='center')

# QUAX
QUAX_col = [0.6,0.1,0.3]
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/QUAX.txt")
dat2 = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/QUAX2.txt")
plt.plot([dat[0,0],dat[0,0]],[scale*dat[0,1],y2],color=QUAX_col,lw=2,zorder=0.05)
plt.plot([dat2[0,0],dat2[0,0]],[scale*dat2[0,1],y2],color=QUAX_col,lw=2,zorder=0.05)
plt.gcf().text(0.538,0.479,r'{\bf QUAX}-$a\gamma$',color=col_alpha(QUAX_col,1),fontsize=20,rotation=90,rotation_mode='anchor',ha='center',va='center')

# RADES
RADES_col = [0.95,0.1,0.1]
dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/RADES.txt")
plt.plot([dat[0,0],dat[0,0]],[scale*dat[0,1],y2],color=RADES_col,lw=2,zorder=0.05)
plt.gcf().text(0.517,0.53,r'{\bf RADES}',color=col_alpha(RADES_col,1),fontsize=15,rotation=90,rotation_mode='anchor',ha='center',va='center')

# Neutron stars
dat = loadtxt(axionlimits_dir +'limit_data/AxionPhoton/NeutronStars_GreenBank.txt')
plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor='ForestGreen',zorder=0.11)
#dat = loadtxt(axionlimits_dir +'limit_data/AxionPhoton/NeutronStars_VLA.txt')
#plt.fill_between(dat[:,0],scale*dat[:,1],y2=y2,edgecolor='k',facecolor='SeaGreen',zorder=0.11)
dat = loadtxt(axionlimits_dir +'limit_data/AxionPhoton/NeutronStars_Battye.txt')
plt.plot(dat[:,0],dat[:,1],color='k',lw=1,zorder=0.2)
plt.fill_between(dat[:,0],scale*dat[:,1],y2=1e0,zorder=0.2,color='#4ad46a')
plt.gcf().text(0.50*(1-0.003),0.81*(1+0.003),r'{\bf Neutron Stars}',color='k',fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')
plt.gcf().text(0.50,0.81,r'{\bf Neutron Stars}',color=col_alpha('Green',0.4),fontsize=30,rotation=0,rotation_mode='anchor',ha='center',va='center')

plt.gcf().text(0.34*(1-0.001),0.76*(1+0.001),r'{\bf Foster et al.}',color='k',fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')
plt.gcf().text(0.34,0.76,r'Foster et al.',color=col_alpha('ForestGreen',0.7),fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')

#plt.gcf().text(0.47*(1-0.001),0.76*(1+0.001),r'{\bf Darling}',color='k',fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')
#plt.gcf().text(0.47,0.76,r'Darling',color=col_alpha('SeaGreen',0.7),fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')

plt.gcf().text(0.58*(1-0.001),0.76*(1+0.001),r'{\bf Battye et al.}',color='k',fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')
plt.gcf().text(0.58,0.76,r'Battye et al.',color=col_alpha('#20c742',0.7),fontsize=25,rotation=0,rotation_mode='anchor',ha='center',va='center',multialignment='center')

#AxionPhoton.Haloscopes(ax,projection=True,fs=20,text_on=True,BASE_arrow_on=True)

text_on = True
AxionPhoton.ALPHA(ax,text_on=text_on, text_shift=[0.95, 1], f_ax = f_ax)
AxionPhoton.MADMAX(ax,text_on=text_on, text_shift=[1.4,1], f_ax = f_ax)
AxionPhoton.ORGAN(ax,projection=True,text_on=text_on, f_ax = f_ax)
#AxionPhoton.ADMX(ax,projection=True,text_on=text_on, f_ax = f_ax)

dat = loadtxt(axionlimits_dir + "limit_data/AxionPhoton/Projections/ADMX_Projected.txt")
plt.plot(dat[:,0],(f_ax**-0.5)*dat[:,1],'-',linewidth=1.5,color=ADMX_col,zorder=0)
plt.fill_between(dat[:,0],(f_ax**-0.5)*dat[:,1],y2=y2,edgecolor=None,facecolor=ADMX_col,zorder=0,alpha=0.1)

plt.text(3.5e-5,1e-15,r'{\bf ADMX}',fontsize=20,color=ADMX_col,rotation=0,ha='left',va='top',clip_on=True)
plt.plot([3e-5,2e-5],[1e-15,0.2e-14],'k-',lw=1.5)

INCLUDE_CADEX = False
if (INCLUDE_CADEX):
    #CADEx
    CADEx_col = [0.95,0.5,0.5]
    #CADEx_col = 'k'
    mvals = geomspace(330e-6, 460e-6)
    gvals = (f_ax**-0.5)*1.6e-13*mvals/370e-6
    plt.fill_between(mvals,gvals,y2=y2,edgecolor='k',facecolor=CADEx_col,zorder=0.1, linewidth=3.0, alpha=0.4)
    xvals = concatenate((mvals[0:1], mvals, mvals[-2:-1]))
    yvals = concatenate(([y2], gvals, [y2]))
    plt.plot(xvals, yvals, color='k', zorder=0.11, linewidth=3.0)
    #plt.fill_between(mvals,4e-13*mvals/400e-6,y2=y2,edgecolor='k',facecolor=None,zorder=0.1, linewidth=3.0)
    #plt.plot([330, 460],[dat[0,1],y2],color=CADEx_col,lw=2,zorder=0.05)
    plt.gcf().text(0.764,0.615,r'{\bf CADEx}',color='k',fontsize=30,rotation=90,rotation_mode='anchor',ha='center',va='center')

plt.gca().xaxis.labelpad = 0

plt.gcf().text(0.89,0.13,r'$f_\mathrm{ax} = 90\%; \,\,\rho_0 = 0.45$ GeV cm$^{-3}$',fontsize=25,ha='right', va='bottom')

#plt.title('0',color='w',pad=10)
MySaveFig(fig,'AxionPhoton_RadioFreqCloseup_withProjections', pngsave=False)

max_stars_repo_name: bradkav/DarkAxionPortal

max_issues_repo_name: bradkav/DarkAxionPortal

max_forks_repo_name: bradkav/DarkAxionPortal

reason: from numpy


text:
import numpy_eigen
import inspect

from .libsm_python import *
from .plotCoordinateFrame import plotCoordinateFrame
from .Progress import Progress
from .Progress import Progress2
from .saveFigTight import saveFigTight

def logInfo(message):
    if getLoggingLevel() > LoggingLevel.Info:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(LoggingLevel.Info, info.filename, info.lineno, info.function, message)

def logDebug(message):
    if getLoggingLevel() > LoggingLevel.Debug:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(LoggingLevel.Debug, info.filename, info.lineno, info.function, message)

def logWarn(message):
    if getLoggingLevel() > LoggingLevel.Warn:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(LoggingLevel.Warn, info.filename, info.lineno, info.function, message)

def logFatal(message):
    if getLoggingLevel() > LoggingLevel.Fatal:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(LoggingLevel.Fatal, info.filename, info.lineno, info.function, message)

def logError(message):
    if getLoggingLevel() > LoggingLevel.Error:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(LoggingLevel.Error, info.filename, info.lineno, info.function, message)

def log(level,message):
    if getLoggingLevel() > level:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLog(level, info.filename, info.lineno, info.function, message)

def logInfoNamed(name,message):
    if getLoggingLevel() > LoggingLevel.Info:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,LoggingLevel.Info, info.filename, info.lineno, info.function, message)

def logDebugNamed(name,message):
    if getLoggingLevel() > LoggingLevel.Debug:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,LoggingLevel.Debug, info.filename, info.lineno, info.function, message)

def logWarnNamed(name,message):
    if getLoggingLevel() > LoggingLevel.Warn:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,LoggingLevel.Warn, info.filename, info.lineno, info.function, message)

def logFatalNamed(name,message):
    if getLoggingLevel() > LoggingLevel.Fatal:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,LoggingLevel.Fatal, info.filename, info.lineno, info.function, message)

def logErrorNamed(name,message):
    if getLoggingLevel() > LoggingLevel.Error:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,LoggingLevel.Error, info.filename, info.lineno, info.function, message)

def logNamed(name,level,message):
    if getLoggingLevel() > level:
        return
    callerframerecord = inspect.stack()[1]
    frame = callerframerecord[0]
    info = inspect.getframeinfo(frame)
    rawLogNamed(name,level, info.filename, info.lineno, info.function, message)

max_stars_repo_name: bhirschel/kalibr

max_issues_repo_name: bhirschel/kalibr

max_forks_repo_name: bhirschel/kalibr


text:
import numpy as np
from bisect import bisect_right
from collections import defaultdict, OrderedDict

# ----------------------------------- #
# Level

def count_within(values, intervals):
    """Count number of values within each interval

    Args:
        values (n-list of nb)
        intervals (m-list of 2-nb-tuple)

    Return:
        (m-np.ndarray) count for each interval
    """
    rvals = np.reshape(values, [-1, 1])
    intervals_m = np.array(intervals)
    rlow = intervals_m[:, 0].reshape(1, -1)
    rhigh = intervals_m[:, 1].reshape(1, -1)

    flags = (rlow <= rvals) * (rvals < rhigh)
    return np.sum(flags, axis=0)

def count_within_bkd(values, intervals):
    """Return number of values within each interval in a dictionary

    Args:
        values (n-list of nb)
        intervals (m-list of 2-nb-tuple)

    Return:
        (OrderedDict) key=interv value=nb of values within interv
    """
    return OrderedDict([
        (tuple(i), c)
        for i, c in zip(intervals, count_within(values, intervals))
     ])

def index_lvl(a, thlds):
    """Index level(s) of value(s) given thlds b/w those levels

    Args:
        a (nb or n-np.ndarray)  : value or array of values
        thlds (list)            : list of sorted thresholds b/w levels

    Return:
        level index(es) (int or n-int-np.ndarray)
            res_i = index of level regarding thlds
            res_i = k means thld_(k-1) <= a_i < thld_k

    Example:
        >> index_lvl(np.array([0, 10, 20]), [10])
        [0, 1, 1]
    """
    if isinstance(a, (float, int)):
        return bisect_right(thlds, a)

    res = np.zeros(len(a))
    for thld in thlds:
        res += thld <= a
    return res.astype(int)

def label_lvl(a, thlds, labels):
    """Label level(s) of value(s) given thlds b/w those levels

    Args:
        a (nb or n-np.ndarray)  : value or array of values
        thlds (list)            : list of sorted thresholds b/w levels
        labels (m+1-list)       : level names
            label_k = level b/w thld_(k-1) & thld_k

    Return:
        level label(s) (int or n-int-np.ndarray)
            res_i = label_k if thld_(k-1) <= a_i < thld_k

    Example:
        >> label_lvl(np.array([0, 10, 20, 5, 7]), [10], ['low', 'high'])
        ['low', 'high', 'high', 'low', 'low']
    """
    if len(labels) != len(thlds) + 1:
        raise ValueError("Must be one more label than number of thresholds")
    lvl_indexes = index_lvl(a, thlds)
    return np.take(labels, lvl_indexes)

def bkd_lvl(a, thlds, labels):
    """Breakdown level spans per label given thlds b/w levels

    Args:
        a (n-np.ndarray)    : array of values
        thlds (list)        : list of sorted thresholds b/w levels
        labels (m+1-list)   : level names
            label_k = level b/w thld_(k-1) & thld_k

    Return:
        (dict)
        {
            <label>: (2-int-tuple-list)
        }

    Example:
        >> bkd_lvl(np.array([0, 10, 20, 5, 7]), [10], ['low', 'high'])
        {'low': [(0, 0), (3, 4)], 'high': [(1, 2)]}
    """
    res = defaultdict(list)
    flags = label_lvl(a, thlds, labels)
    lsti = 0
    case = flags[lsti]
    for i, flag in enumerate(flags[1:], 1):
        if flag == case:
            continue
        else:
            res[case].append((lsti, i-1))
            case = flag
            lsti = i
    if lsti != i:
        res[case].append((lsti, i))
    return dict(res)

# ----------------------------------- #
# In-betweens

def count_inbetweens(refticks, contenders, end=None):
    """Count number of intervals with a tick of contender in-b/w"""
    end = refticks[-1] if end is None else end

    refticks_a = np.reshape(refticks, (-1, 1))
    refticks_b = np.reshape(np.concatenate([refticks[1:], [end]]), (-1, 1))

    res = []
    for contender in contenders:
        inbetweens = np.sum(  # rtick b/w 2 cons. mtick (TCyc recovered in time)
            (contender - refticks_a >= 0) * (refticks_b - contender > 0),
            axis=1
        )
        res.append(inbetweens)
    return np.array(res)

def has_inbetweens(*args, **kwargs):
    return count_inbetweens(*args, **kwargs) > 0

def count_recoveries(*args, **kwargs):
    return np.sum(has_inbetweens(*args, **kwargs), axis=1)

# ----------------------------------- #
# Noise

def monotonic_dev(X, Y, gradsign=None, gradwindow=5, amplification=None):
    """Return total deviation from a monotonic behavior

    Gives and idea of total noise of signal.

    Args:
        X (n-np.ndarray)
        Y (n-np.ndarray)
        gradsign (float): sign of expected monotony
        gradwindow (int): size of window to smart compute gradsign
        amplification (callable): noise amplification
            (np.ndarray)-> (np.ndarray)

    Return:
        (float) monotonic deviation
    """
    def totdev(dY):
        """Return total deviation"""
        ampl = (lambda x: x) if amplification is None else amplification
        return np.sum(ampl(np.abs(dY)))

    if len(Y) == 0:
        return np.nan
    elif len(Y) == 1:
        return 0

    if gradsign is None:
        gradwindow = min(gradwindow, (len(Y) + 1) // 2)
        yfst = np.mean(Y[:gradwindow])
        ylst = np.mean(Y[-gradwindow:])
        gradsign = (ylst - yfst) if gradsign is None else gradsign

    dY = np.diff(Y)
    if gradsign == 0:
        return totdev(dY)
    elif gradsign < 0:
        return 2 * totdev(dY[dY > 0])
    else:  # gradsign > 0
        return 2 * totdev(dY[dY < 0])

def monotonic_devr(X, Y, **kwargs):
    """Return deviation ratio from a monotonic behavior

    Gives an idea of mean noise in signal.

    Args:
        X (n-np.ndarray)
        Y (n-np.ndarray)
        **kwargs: @see monotonic_dev

    Return:
        (float) monotonic deviation / n
    """
    return monotonic_dev(X, Y, **kwargs) / len(Y)

max_stars_repo_name: OctaveLauby/olfactory

max_issues_repo_name: OctaveLauby/olfactory

max_forks_repo_name: OctaveLauby/olfactory


text:
"""Visualization for sequential tracks."""
import numpy as np
import open3d as o3d

from ..data.loadmodels import track_palette

def make_geometries_from_dict(tracks, old_track_geometries=None,
                              palette=track_palette):
    track_line_sets = {}

    for track_id, points in tracks.items():
        points_pos = np.reshape([point[0] for point in points], (-1, 3))

        if points_pos.shape[0] > 1:
            line_set = o3d.geometry.LineSet()

            line_set.points = o3d.utility.Vector3dVector(points_pos)

            line_indices = np.column_stack((range(0, len(points) - 1),
                                            range(1, len(points))))
            line_set.lines = o3d.utility.Vector2iVector(np.asarray(line_indices))

            if old_track_geometries:
                if track_id in old_track_geometries:
                    old_color = np.asarray(old_track_geometries[track_id].colors)[0,:]
                    line_set.paint_uniform_color(old_color)
                else:
                    line_set.paint_uniform_color(np.asarray(next(palette)))
            else:
                line_set.paint_uniform_color(np.asarray(next(palette)))

            track_line_sets[track_id] = line_set

    return track_line_sets

def init_tracks(visualizer, tracks, palette=track_palette):
    """Add detections in proper form to open3d visualizer."""
    tracks_geometries = make_geometries_from_dict(tracks, palette=palette)

    for _, geometry in tracks_geometries.items():
        visualizer.add_geometry(geometry)

    return tracks_geometries

def update_tracks(visualizer, old_geometries, new_tracks,
                  palette=track_palette):
    """Update detections that has a previous geometry object."""
    for _, geometry in old_geometries.items():
        visualizer.remove_geometry(geometry, False)

    new_geometries = make_geometries_from_dict(new_tracks, old_geometries,
                                               palette=palette)

    for _, geometry in new_geometries.items():
        visualizer.add_geometry(geometry, False)

    return new_geometries


text:
import numpy as np
from matplotlib import pyplot as pyp,cm
from scipy.optimize import minimize, Bounds, fsolve, LinearConstraint
from joblib import Parallel, delayed
import sklearn.gaussian_process as gpr
from collections import OrderedDict
from mpl_toolkits.mplot3d import Axes3D

class LCB_AF():
    def __init__(self, model, exp_w, descale, **refmod):
        self.model = model
        self.exp_w = exp_w
        self.descale = descale
        if refmod:
            self.refmod = refmod['refmod']
        else:
            def zr(x):
                return 0
            self.refmod = zr
    def LCB(self, x):
        x = np.array([x]).reshape(-1,1); x=x.reshape(1,x.shape[0])
        mu, std = self.model.predict(x, return_std=True); std=std.reshape(-1,1)
        if str(type(self.refmod))=="<class '__main__.Network'>":
            yref = self.refmod(torch.from_numpy(self.descale(x)).float()).data.numpy()
        else:
            yref = self.refmod(self.descale(x))
        return (yref+mu-self.exp_w*std).flatten()

class BO():
    def __init__(self, ub, lb, dim, exp_w, kernel, system, bounds, **aux_mods):
        self.ub = ub
        self.lb = lb
        self.dim = dim
        self.exp_w = exp_w
        self.kernel = kernel
        self.system = system
        self.bounds = bounds
        self.dist_ref = {}
        if aux_mods:
            self.refmod = {'refmod': list(aux_mods.values())[0]}

    def descale(self, x, scaling_factor = 1):
        # b = (self.ub+self.lb)/2
        # m = (b-self.lb)/scaling_factor
        m = (self.ub-self.lb)/scaling_factor
        b = self.lb
        return m*x+b

    def scale(self, x, scaling_factor = 1):
        m = 2*scaling_factor/(self.ub-self.lb)
        b = scaling_factor-m*self.ub
        # m = scaling_factor/(self.ub-self.lb)
        # b = -self.lb/(self.ub-self.lb)
        return m*x+b

    def optimizergp(self, trials, xinit = None):
        print('Vanilla BO Run...')
        self.trialsgp = trials
        if xinit is None:
            # x = np.random.uniform(-1, 1, (1, self.dim))
            x = np.random.uniform(0, 1, (1, self.dim))
        else:
            x = xinit.reshape(1, self.dim)
        y = self.system(self.descale(x))
        modelgp = gpr.GaussianProcessRegressor(self.kernel, alpha = 1e-6,
                                                  normalize_y = True,
                                                  n_restarts_optimizer = 10)
        modelgp.fit(x, y)
        # x0 = np.random.uniform(-1, 1, (100, self.dim))
        x0 = np.random.uniform(0, 1, (100, self.dim))
        LCBgp = LCB_AF(modelgp, self.exp_w, self.descale).LCB
        opt = Parallel(n_jobs = -1)(delayed(minimize)(LCBgp, x0 = start_point,
                                                      method = 'L-BFGS-B',
                                                      bounds = self.bounds)
                                    for start_point in x0)
        xnxts = np.array([res.x for res in opt], dtype  = 'float')
        funs = np.array([np.atleast_1d(res.fun)[0] for res in opt])
        for i in range(self.trialsgp):
            xnxt = xnxts[np.argmin(funs)].reshape(1, self.dim)
            ynxt = self.system(self.descale(xnxt))
            x = np.vstack([x, xnxt])
            y = np.vstack([y, ynxt])
            modelgp.fit(x, y)
            # x0 = np.random.uniform(-1, 1, (100, self.dim))
            x0 = np.random.uniform(0, 1, (100, self.dim))
            opt = Parallel(n_jobs = -1)(delayed(minimize)(LCBgp, x0 = start_point,
                                                      method = 'L-BFGS-B',
                                                      bounds = self.bounds)
                                    for start_point in x0)
            xnxts = np.array([res.x for res in opt], dtype  = 'float')
            funs = np.array([np.atleast_1d(res.fun)[0] for res in opt])
        self.modelgp = modelgp
        self.xgp = self.descale(x)
        self.ygp = y
        self.ref_optim = False
        self.dist_optim = False
        self.distref_optim = False
        self.splt_optim = False
        self.spltvar_optim = False

    def optimizeref(self, trials, xinit = None):
        print('BO with Reference Model Run...')
        self.trialsref = trials
        refmod = self.refmod['refmod']
        if xinit is None:
            # x = np.random.uniform(-1, 1, (1, self.dim))
            x = np.random.uniform(0, 1, (1, self.dim))
        else:
            x = xinit.reshape(1, self.dim)
        y = self.system(self.descale(x))
        eps = y - refmod(self.descale(x))
        self.modeleps = gpr.GaussianProcessRegressor(self.kernel, alpha = 1e-6,
                                                  normalize_y = True,
                                                  n_restarts_optimizer = 10)
        self.modeleps.fit(x, eps)
        # x0 = np.random.uniform(-1, 1, (100, self.dim))
        x0 = np.random.uniform(0, 1, (100, self.dim))
        LCBref = LCB_AF(self.modeleps, self.exp_w, self.descale, **self.refmod).LCB
        opt = Parallel(n_jobs = -1)(delayed(minimize)(LCBref, x0 = start_point,
                                                      method = 'L-BFGS-B',
                                                      bounds = self.bounds)
                                    for start_point in x0)
        xnxts = np.array([res.x for res in opt], dtype  = 'float')
        funs = np.array([np.atleast_1d(res.fun)[0] for res in opt])
        for i in range(self.trialsref):
            xnxt = xnxts[np.argmin(funs)].reshape(1, self.dim)
            ynxt = self.system(self.descale(xnxt))
            epsnxt = ynxt - refmod(self.descale(xnxt))
            x = np.vstack([x, xnxt])
            y = np.vstack([y, ynxt])
            eps = np.vstack([eps, epsnxt])
            self.modeleps.fit(x, eps)
            # x0 = np.random.uniform(-1, 1, (100, self.dim))
            x0 = np.random.uniform(0, 1, (100, self.dim))
            opt = Parallel(n_jobs = -1)(delayed(minimize)(LCBref, x0 = start_point,
                                                      method = 'L-BFGS-B',
                                                      bounds = self.bounds)
                                    for start_point in x0)
            xnxts = np.array([res.x for res in opt], dtype  = 'float')
            funs = np.array([np.atleast_1d(res.fun)[0] for res in opt])
        self.ref_optim = True
        self.xref = self.descale(x)
        self.ytru = y
        self.eps = eps

    def plots(self, figure_name):
        itr = np.arange(1, self.trialsgp+2, 1)
        yliml = min(self.ygp)-0.1*abs(min(self.ygp))
        ylimu = max(self.ygp)+0.1*abs(max(self.ygp))
        fig, ax1 = pyp.subplots(1, 1, figsize=(11, 8.5))
        ax1.grid(color='gray', axis='both', alpha = 0.25)
        ax1.set_axisbelow(True)
        ax1.set_xlim((1, self.trialsgp+1))
        ax1.set_xlabel(r'$Sample\ Number$', fontsize = 24)
        pyp.xticks(fontsize = 24)
        ax1.set_ylabel(r'$Operating\ cost\ (10k\ USD/hr)$', fontsize = 24)
        pyp.yticks(fontsize = 24);
        ax1.scatter(itr, self.ygp, marker = 'o', color = 'white', edgecolor = 'black',
                    zorder = 3, s = 200);
        ax1.plot(itr, self.ygp, color = 'black', linewidth = 3, label = 'BO');
        if self.ref_optim:
            itr = np.arange(1, self.trialsref+2, 1)
            ax1.scatter(itr, self.ytru, marker = 'o', color = 'white', edgecolor = 'blue',
                    zorder = 3, s = 200);
            ax1.plot(itr, self.ytru, color = 'blue', linewidth = 3, label = 'BO+Ref');
            yliml = min(yliml, min(self.ytru)-0.1*abs(min(self.ytru)))
            ylimu = max(ylimu, max(self.ytru)+0.1*abs(max(self.ytru)))
        ax1.set_ylim(yliml, ylimu)
        pyp.legend(loc = 'upper right')
        pyp.show()
        pyp.savefig(figure_name+'.png', dpi = 300, edgecolor = 'white',
                    bbox_inches = 'tight', pad_inches = 0.1);

max_stars_repo_name: zavalab/bayesianopt

max_issues_repo_name: zavalab/bayesianopt

max_forks_repo_name: zavalab/bayesianopt

reason: import numpy,from scipy


text:
import os
import sys
import librosa
import tflearn
import tensorflow as tf
import librosa.display
import IPython.display
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.style as ms

os.chdir('DATA/')

learning_rate = 0.0001
width = 20  # mfcc features
height = 432
classes = 2  # digits

print("-> Loading")
print("Creating network... ")
net = tflearn.input_data([None, width, height])
#net = tflearn.embedding(net, input_dim=10000, output_dim=128)
net = tflearn.lstm(net, 256, dropout=0.8)
net = tflearn.fully_connected(net, classes, activation='softmax', regularizer='L2')
net = tflearn.fully_connected(net, classes, activation='softmax', regularizer='L2')
net = tflearn.fully_connected(net, classes, activation='softmax', regularizer='L2')
net = tflearn.regression(net, optimizer='adam', learning_rate=learning_rate, loss='categorical_crossentropy')
# Training
print("Creating model... ")
model = tflearn.DNN(net, tensorboard_verbose=0)

print("Loading weights... [tflearn.lstm.model]")
model.load("tflearn.lstm.model")

os.chdir('..')
print("-> Loading file")
if len(sys.argv) < 2:
    print("Please insert wav file as first argument!")
    sys.exit()
file = sys.argv[1]
if not file.endswith(".wav"):
    print("Please insert wav file!")
    sys.exit()
wave, sr = librosa.load(file, mono=True)
mfcc = librosa.feature.mfcc(wave, sr)
if (len(mfcc[0]) != height):
    print("Loading " + str(file) + " with " + str(np.array(mfcc).shape) +
            " reshaping " + str(np.shape(mfcc)) + " to ", end = '') # Debug
    mfcc.resize(width, height)
    print(str(np.shape(mfcc)))
else:
    print("Loading " + str(file) + " with " + str(np.array(mfcc).shape)) # Debug

print("-> Predicting data")
print(model.predict([mfcc])[0])

max_stars_repo_name: StoneLabs/dialektor-tensorflow

max_issues_repo_name: StoneLabs/dialektor-tensorflow

max_forks_repo_name: StoneLabs/dialektor-tensorflow


text:
# Data Set: http://archive.ics.uci.edu/ml/datasets/Abalone
# 1 - Sex / nominal / -- / M, F, and I (infant)
# 2 - Length / continuous / mm / Longest shell measurement
# 3 - Diameter / continuous / mm / perpendicular to length  # 直径
# 4 - Height / continuous / mm / with meat in shell
# 5 - Whole weight / continuous / grams / whole abalone
# 6 - Shucked weight / continuous / grams / weight of meat
# 7 - Viscera weight / continuous / grams / gut weight (after bleeding)
# 8 - Shell weight / continuous / grams / after being dried
# 9 - Rings / integer / -- / +1.5 gives the age in years

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

""" SK-Learn Library Import"""
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import Bunch
from sklearn import svm
from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score

import os

def load_data():
    """
    loading the dataset into a scikit learn Bunch object
    returns:
    ----
    data,target,feature_names,target_names
    """
    # 数据集存放目录
    data_dir = os.path.join(os.getcwd(), "knn_svm_proj\\abalone-dataset\\abalone.data")

    # 定义数据集标签
    xnlabs = ["Sex"]  # 非数值字段
    xqlabs = [  # 数值字段
        "Length",
        "Diameter",
        "Height",
        "Whole weight",
        "Shucked weight",
        "Viscera weight",
        "Shell weight",
    ]
    xlabs = xnlabs + xqlabs

    # 要预测的属性
    ylabs = ["Rings"]

    # 将数据加载到pandas模块的DataFrame对象中
    df = pd.read_csv(data_dir, header=None, sep=",", names=xlabs + ylabs)

    """将非数值字段的项值数值化为0,1,2"""
    df["Sex"] = LabelEncoder().fit_transform(df["Sex"].tolist())

    # 过滤除去无意义的字段
    df = df[df["Height"] > 0.0]
    df = df[df["Length"] > 0.0]
    df = df[df["Diameter"] > 0.0]

    # dummies = pd.get_dummies(df[xnlabs], prefix="Sex")
    # dfdummies = df[xqlabs + ylabs].join(dummies)
    # xqlabs = xqlabs + dummies.columns.tolist()

    # 返回Bunch对象
    return Bunch(
        data=df[xqlabs + xnlabs],
        target=df[ylabs],
        feature_names=xqlabs,
        target_names=ylabs,
    )

if __name__ == "__main__":

    # 加载数据集
    abl_dataset = load_data()

    """训练、预测特征"""
    X_train = abl_dataset["data"].drop(
        ["Sex"], axis=1
    )  # result_acc = 0.26256983240223464
    # X_train = abl_dataset["data"] # 0.2601755786113328
    y_train = abl_dataset["target"]

    # 划分训练集与测试集
    X_train, X_test, y_train, y_test = train_test_split(
        X_train, y_train, test_size=0.3, random_state=0
    )

    """创建决策树训练模型对象"""
    clsModel = DecisionTreeClassifier()
    """根据给定的训练数据拟合Decision Tree模型。"""
    clsModel.fit(X_train, y_train)
    """使用训练后的模型，基于测试集进行目标参数的预测"""
    y_pred = clsModel.predict(X_test)
    """计算训练结果精度"""
    result_acc = accuracy_score(y_test, y_pred)  # 0.19632881085395051

    # 生成分类结果汇报
    cls_report = "Classification Report:\n{}".format(
        classification_report(y_test, y_pred, zero_division=1)
    )

    # 使用最佳参数设置画决策树图
    plt.figure(figsize=(20, 10))
    tree.plot_tree(
        clsModel.fit(X_train, y_train),
        filled=True,
        fontsize=4,
        class_names=[
            "1",
            "2",
            "3",
            "4",
            "5",
            "6",
            "7",
            "8",
            "9",
            "10",
            "11",
            "12",
            "13",
            "14",
            "15",
            "16",
            "17",
            "18",
            "19",
            "20",
            "21",
            "22",
            "23",
            "24",
            "25",
            "26",
            "27",
            "28",
            "29",
        ],
        max_depth=5,
    )
    plt.show()
    plt.clf()

    # Plot result figure
    x = list(range(1, X_test.shape[0] + 1))
    y1 = y_test["Rings"].values
    y2 = y_pred
    plt.xlim(150, 550)
    plt.ylim(0, 25)
    plt.plot(x, y1, "g.", label="Actual Rings of test set")
    plt.plot(x, y2, "rx", label="Predicted Rings of test set")
    plt.title("Decision Tree Regression (Test set)")
    plt.xlabel("Index")
    plt.ylabel("Rings")
    plt.legend()
    plt.show()

    input()  # for debug console

max_stars_repo_name: Delreyaa/machine-learning

max_issues_repo_name: Delreyaa/machine-learning

max_forks_repo_name: Delreyaa/machine-learning


text:
import threading
import time
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import os
import signal
from itertools import islice
import sys
import random
import numpy as np
import random, string
import time, sys
import datetime
import traceback
import argparse
from io import SEEK_END
import multiprocessing as mp
import os
import itertools
from datetime import datetime
import errno

def current_time():
    return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]

def test(write, file_name, output_dir, data, chunk_number):
    print('batch {}: running  with  pid {}'.format(chunk_number, os.getpid()))
    even = []
    odd = []
    for line in data:
        if int(line) % 2 == 0:
            even.append("{}\n".format(line))
        else:
            odd.append("{}\n".format(line))
    all_groups = [even, odd]
    write(all_groups, file_name, output_dir, chunk_number)
    return len(data)

def split_dummy_test(file_name, output_dir, data, chunk_number, params):
    print(file_name, output_dir, chunk_number, params)

def split_data_by_year(file_name, output_dir, data, chunk_number, params):
    # print('process started', file_name, chunk_number, len(data))
    ls_1995_1996 = []
    for i in range(1995, 2006):
        ls_1995_1996.append([])
    for line in data:
        sys.stdout.write("\r" + random.choice(string.ascii_letters))
        sys.stdout.flush()
        if "1995" in line:
            ls_1995_1996[0].append(line)
        elif "1996" in line:
            ls_1995_1996[1].append(line)
        elif "1997" in line:
            ls_1995_1996[2].append(line)
        elif "1998" in line:
            ls_1995_1996[3].append(line)
        elif "1999" in line:
            ls_1995_1996[4].append(line)
        elif "2000" in line:
            ls_1995_1996[5].append(line)
        elif "2001" in line:
            ls_1995_1996[6].append(line)
        elif "2002" in line:
            ls_1995_1996[7].append(line)
        elif "2003" in line:
            ls_1995_1996[8].append(line)
        elif "2004" in line:
            ls_1995_1996[9].append(line)
        elif "2005" in line:
            ls_1995_1996[10].append(line)

    with ProcessPoolExecutor(max_workers=1) as pe:
        feature = pe.submit(write_to_file, file_name, output_dir, chunk_number)
        feature.add_done_callback(write_to_file_callback)

    # sync write_to_file(ls_1995_1996, file_name, output_dir, chunk_number)
    return file_name, chunk_number, len(data), params

def write_to_file_callback(future):
    print('files written to disk', future.result())

def time_milli_sec():
    x = lambda: int(round(time.time() * 1000))
    return x()

def write_to_file(lines, file_name, output_dir, chunk_number):
    # print("in writing file chunk {}".format(chunk_number))
    year = 1995
    lst_files = []
    for line_set in lines:
        dir_path = os.path.join(output_dir, str(year), os.path.basename(file_name))
        try:
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        # path = os.path.join(dir_path, "{}.part.{}".format(time_milli_sec(),str(chunk_number)))
        path = os.path.join(dir_path, "{}.{}.part.{}".format(current_time(), str(year), str(chunk_number)))
        lst_files.append((path,chunk_number,len(lines)))
        if (os.path.exists(path)):
            print("file path exit {}".format(path))
        path = path.replace("\\\\", "\\")
        # time.sleep(10000)
        with open(path, 'w') as fs:
            fs.writelines(line_set)
        year = year + 1

    return lst_files

def callback_function(future):
    # print('process finished', future.result())
    x = future.result()

def parse_lines(lines, file_name, output_dir):
    split_data_by_year(write_to_file, file_name, output_dir, lines)

def chunks(list_values, chunk_size):
    chunk_size = max(1, chunk_size)
    return (list_values[i:i + chunk_size] for i in range(0, len(list_values), chunk_size))

def chunks2(list_values, size):
    it = iter(list_values)
    item = list(itertools.islice(it, size))
    while item:
        yield item
        item = list(itertools.islice(it, size))
    return item

def test_batch_execution(file_name, output_dir):
    futures = []
    multi_lines = [str(i) for i in range(1, 40)]
    chunked_data = chunks(multi_lines, 10)
    chunk_number = 0
    with ProcessPoolExecutor(max_workers=10) as e:
        for data in chunked_data:
            future = e.submit(test, write_to_file, file_name, output_dir, data, chunk_number)
            future.add_done_callback(callback_function)
            futures.append(future)
            chunk_number = chunk_number + 1

    results = [f.result() for f in futures]
    # collect results
    # total_lns = np.sum(results)#[count(col) for col in results]
    # print("total lines {}".format(total_lns))
    print(file_name + "all threads finished")

def parse_lines2(lines, num_workers):
    parameters = [1, 2, 3]
    et = ProcessPoolExecutor(max_workers=num_workers)

    futures = []

    for line in lines:
        for params in parameters:
            future = et.submit(split_data_by_year, write_to_file, params, line)
            futures.append(future)

def start_process():
    global parser, args
    parser = argparse.ArgumentParser(description='Parallel text processor')
    parser.add_argument('--num_workers', '-n', default=8, type=int)
    parser.add_argument('--chunk_size', '-c', default=8, type=int)
    parser.add_argument('--no_of_file_in_parallel', '-f', default=8, type=int)
    parser.add_argument('--output_dir', type=str, required=True)
    parser.add_argument('--input_dir', type=str, required=True)
    args = parser.parse_args()
    main(args.num_workers, args.chunk_size, args.no_of_file_in_parallel, args.input_dir, args.output_dir)

def get_file_size(file_path):
    size = os.path.getsize(file_path)
    return size

def convert_bytes(size, unit=None):
    if unit == "KB":
        return ('File size: ' + str(round(size / 1024, 3)) + ' Kilobytes')
    elif unit == "MB":
        return ('File size: ' + str(round(size / (1024 * 1024), 3)) + ' Megabytes')
    elif unit == "GB":
        return ('File size: ' + str(round(size / (1024 * 1024 * 1024), 3)) + ' Gigabytes')
    else:
        return ('File size: ' + str(size) + ' bytes')

def main(num_workers, chunk_size, no_of_file_in_parallel, input_dir, output_dir):
    print("{} workers".format(num_workers))
    output_dir = os.path.join(output_dir, "1995_2005_ds")
    if not os.path.exists(output_dir):
        print("output will be here {}", output_dir)
        os.makedirs(output_dir)
    # for each input file
    files = os.listdir(input_dir)
    file_worker = no_of_file_in_parallel

    file_futures = []
    with ProcessPoolExecutor(max_workers=file_worker) as fe:
        for file_name in files:
            file_future = fe.submit(process_file, chunk_size, file_name, input_dir, num_workers, output_dir)
            file_futures.append(file_future)
        file_results = [f.result() for f in file_futures]
        print(file_results)
        # process_file(chunk_size, file_name, input_dir, num_workers, output_dir)

def process_file(chunk_size, file_name, input_dir, num_workers, output_dir):
    futures = []
    try:
        file_name = os.path.join(input_dir, file_name)
        size = get_file_size(file_name)
        print("processing file {}, size {} <=> size{}, started at {}".format(file_name, convert_bytes(size, "MB"),
                                                                             convert_bytes(size, "GB"),
                                                                             datetime.utcnow()))
        chunk_number = 0
        with open(file_name) as file:
            while True:
                lines = list(islice(file, chunk_size))
                if not lines:
                    break
                batch_size = int(chunk_size / num_workers)
                chunked_data = chunks(lines, batch_size)
                with ProcessPoolExecutor(max_workers=num_workers) as e:
                    for data in chunked_data:
                        future = e.submit(split_data_by_year, file_name, output_dir, data,
                                          chunk_number, ("started at {}".format(current_time()), chunk_number))
                        future.add_done_callback(callback_function)
                        futures.append(future)
                        chunk_number = chunk_number + 1

        results = [f.result() for f in futures]
        # total_lns = np.sum(results)  # [count(col) for col in results]
        # print("total lines {}".format(total_lns))
        print("{} finished at {}".format(file_name, datetime.utcnow()))
    except Exception as e:
        traceback.print_exc()

# main(1,500000, 'E:/download/test/output',"E:/download/1995_2005_ds")

if __name__ == '__main__':
    # test_batch_execution('one_file', "E:/download/output_files")
    start_process()
    print("in main")

max_stars_repo_name: numberonewastefellow/projects

max_issues_repo_name: numberonewastefellow/projects

max_forks_repo_name: numberonewastefellow/projects


text:
import megengine as mge
import megengine.functional as F
import numpy as np

def two_layer_conv(x):
    # (8, 3, 3, 3) 代表（输出信道数，输入信道数，卷积核高度，卷积核宽度）
    conv_weight = mge.Parameter(np.random.randn(8, 3, 3, 3).astype(np.float32))
    # 对于 8 个卷积核，提供 8 个 bias
    conv_bias = mge.Parameter(np.zeros((1, 8, 1, 1), dtype=np.float32))
    x = F.conv2d(x, conv_weight, conv_bias)
    x = F.relu(x)
    conv_weight = mge.Parameter(np.random.randn(16, 8, 3, 3).astype(np.float32))
    conv_bias = mge.Parameter(np.zeros((1, 16, 1, 1), dtype=np.float32))
    x = F.conv2d(x, conv_weight, conv_bias)
    x = F.relu(x)
    return x

# 输入形状为 (2, 3, 32, 32) 的张量
x = mge.tensor(np.random.randn(2, 3, 32, 32).astype(np.float32))
out = two_layer_conv(x)
print(out.shape)  # 输出： (2, 16, 28, 28)

max_stars_repo_name: GG-yuki/bugs

max_issues_repo_name: GG-yuki/bugs

max_forks_repo_name: GG-yuki/bugs