text:
''' Baisc packages
'''
import os
import glob
import tqdm
import copy
import random
import importlib
import numpy as np

''' Configuration packages
'''
import yaml
import argparse
from easydict import EasyDict as edict

''' PyTorch packages
'''
import torch
import torch.nn as nn
import torch.autograd as autograd
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
from torch.nn.functional import cosine_similarity

from torch.utils.data import DataLoader
from utils.utils_check import check_var
from utils.utils_loss import VGGLoss

def train(args, logger=None, vis_logger=None):
    logger.info('=> Import libs')
    Datalib     = importlib.import_module('.{}'.format(args.DATASET_CONFIG.dataset.name), package=args.DATASET_CONFIG.package)
    Model       = importlib.import_module('.{}'.format(args.MODEL.name[0]), package=args.MODEL.package)
    Model_PGGAN = importlib.import_module('.{}'.format(args.MODEL.name[1]), package=args.MODEL.package)

    logger.info('=> Set dataloader')
    trainset = Datalib.inverse(args.DATASET_CONFIG)
    train_loader = DataLoader(trainset, batch_size=args.TRAIN.batch_size, num_workers=args.TRAIN.num_workers, drop_last=True, pin_memory=True, shuffle=True)

    logger.info('=> Make models and load checkpoint')
    encoder_net = Model.Vgg16(pre_train=args.MODEL.vgg.pre_train,
                              requires_grad=args.MODEL.vgg.requires_grad,
                              vae_encoder=args.MODEL.vgg.use_vae,
                              global_pooling=args.MODEL.vgg.global_pooling,
                              if_downsample=args.MODEL.vgg.if_downsample)
    G_net = Model_PGGAN.PGGANGenerator(model_name=args.MODEL.pggan.model_name, logger=logger)
    D_net = Model.Discriminator(size=args.MODEL.D.imgsize,
                                input_channel=args.MODEL.D.input_channel,
                                ndf=args.MODEL.D.ndf,
                                channel_multiplier=args.MODEL.D.channel_multiplier,
                                use_sigmoid=args.MODEL.D.use_sigmoid,
                                use_sn=args.MODEL.D.use_sn)
    if args.USE_GPU:
        logger.info('===> Use {} GPUs'.format(args.NUM_GPU))
        assert torch.cuda.is_available()
        device = torch.device('cuda')
        GPU_list = [i for i in range(args.NUM_GPU)]
        encoder_net = nn.DataParallel(encoder_net, device_ids=GPU_list)
        D_net = nn.DataParallel(D_net, device_ids=GPU_list)
        encoder_net.to(device)
        D_net.to(device)
    else:
        ValueError('Unsupported mode!')
    if args.MODEL.checkpoint.ckp_flag:
        logger.info('===> Load ckp for <vgg_encoder> from {}'.format(args.MODEL.checkpoint.ckp_path))
        statedict = torch.load(args.MODEL.checkpoint.ckp_path)
        encoder_net.module.load_state_dict(statedict)

    logger.info('=> Set loss')
    MSE_loss = nn.MSELoss()
   # MSE_loss = nn.L1Loss()
    D_loss = nn.BCELoss()
    VGG_loss = VGGLoss(weight_vgg=1.)
    if args.USE_GPU:
        MSE_loss.to(device)
        VGG_loss.to(device)
        D_loss.to(device)

    logger.info('=> Set optimizer')
    optimizer_encoder = torch.optim.Adam(encoder_net.module.parameters(), \
                        lr=args.OPTIMIZER.lr_encoder, \
                        betas=(args.OPTIMIZER.Adam.beta1, args.OPTIMIZER.Adam.beta2), \
                        weight_decay=1e-4)
    optimizer_D       = torch.optim.Adam(D_net.module.parameters(), \
                        lr=args.OPTIMIZER.lr_D, \
                        betas=(args.OPTIMIZER.Adam.beta1, args.OPTIMIZER.Adam.beta2), \
                        weight_decay=1e-4)

    """ Intrisic function"""
    def cor_square_error(x, y, eps=1e-12):
        return (1.0 - cosine_similarity(x, y, eps=eps)).mean()
    def save_checkpoint(net, name, dir_path, epoch, bs, gap=1):
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        if epoch % gap == 0:
            path = os.path.join(dir_path, '{}_Batchsize_{}_Epoch_{}.pth'.format(name, bs, epoch))
            torch.save(net.module.state_dict(), path)
            logger.info('===> Save checkpoint at {}'.format(path))

    logger.info('=> Begin to train the model')
    encoder_net.train()
    D_net.train()
    Tensor = torch.cuda.FloatTensor if args.USE_GPU else torch.FloatTensor

    for epoch in range(0, args.TRAIN.epochs):
        bar = tqdm.tqdm(train_loader)
        dis_loss_list = []
        gan_loss_list = []
        latent_loss_list = []
        en_mse_loss_list = []
        en_vgg_loss_list = []
        encoder_loss_list = []
       ####################################################
       #       gt_img
       #         |
       #     inverse_net
       #         |
       #       lat_code  + label +  uniform
       #                    |
       #                  pggan
       #                    |
       #                recon_img
       #                    |
       #                inverse_net
       #                    |
       #                 rec_code
       ####################################################
        for idx, (input_img, input_label) in enumerate(bar):
            input_img = input_img.cuda()
            input_label = input_label.cuda()
            uniform_code = torch.Tensor(input_img.shape[0], args.TRAIN.uniform_dim).uniform_().type_as(input_img)
            uniform_code = uniform_code.cuda()

            latent_code4, skip = encoder_net(input_img)  #输入为gt，得到其inverse后的code
            latent_code = torch.cat((latent_code4, input_label, uniform_code), dim=1)
            recon_img = G_net.synthesize(latent_code)  #输入为pred的code，通过pggan得到recon_img，希望它可以和gt尽可能一样

            #Update D_net
            optimizer_D.zero_grad()
            pred_real = D_net(input_img)
            pred_fake = D_net(recon_img.detach())
            dis_loss = 0

            for idx_real, idx_fake in zip(pred_real, pred_fake):
                global_real_label = Tensor(idx_real.size()).fill_(1.).cuda()
                global_fake_label = Tensor(idx_fake.size()).fill_(0.).cuda()
                dis_loss += (D_loss(idx_real, global_real_label) + D_loss(idx_fake, global_fake_label))
            dis_loss.backward()
            optimizer_D.step()
            dis_loss_list.append(dis_loss.item())

            #Update encoder_net
            optimizer_encoder.zero_grad()
            pred_fake = D_net(recon_img)
            recon_latent_code, _ = encoder_net(recon_img)

            gan_loss = 0  #gan loss
            for idx_fake in pred_fake:
                global_real_label = Tensor(idx_fake.size()).fill_(1.).cuda()
                gan_loss += D_loss(idx_fake, global_real_label)
            latent_loss = 1. * cor_square_error(recon_latent_code, latent_code4) +\
                          1. * MSE_loss(recon_latent_code, latent_code4)
            mse_loss = MSE_loss(recon_img, input_img)
            vgg_loss = VGG_loss(recon_img, input_img)
            encoder_loss = (args.TRAIN.weight_mse * mse_loss +\
                           args.TRAIN.weight_vgg * vgg_loss +\
                           args.TRAIN.weight_latent * latent_loss +\
                           args.TRAIN.weight_gan * gan_loss).mean()
            encoder_loss.backward()
            optimizer_encoder.step()
            gan_loss_list.append(gan_loss.item())
            latent_loss_list.append(latent_loss.item())
            en_mse_loss_list.append(mse_loss.item())
            en_vgg_loss_list.append(vgg_loss.item())
            encoder_loss_list.append(encoder_loss.item())

            if idx % args.TRAIN.save_iter == 0:
                #dump images to tensorboard
                vis_logger.visual_image(
                    {'input_image': input_img,
                     'recon_image': recon_img},
                    (idx+1), normalize=True
                )

        vis_logger.visual_text(
            {'D_net': np.mean(dis_loss_list)},
            epoch,
            main_tag='D_loss'
        )
        vis_logger.visual_text(
            {'EncoderNet': np.mean(encoder_loss_list),
             'gan_loss': np.mean(gan_loss_list),
             'latent_loss': np.mean(latent_loss_list),
             'recon_loss': np.mean(en_mse_loss_list),
             'perceptual_loss': np.mean(en_vgg_loss_list)},
            epoch,
            main_tag='Encoder_loss'
        )
        logger.info('[Epoch: {0}/{1}] | [D_net: {2:.5f}] [EncoderNet: {3:.5f}] [gan_loss: {4:.5f}] [latent_loss: {5:.5f}] [recon_loss: {6:.5f}] [perceptual_loss: {7:.5f}]'.format(
            epoch,
            args.TRAIN.epochs,
            np.mean(dis_loss_list),
            np.mean(encoder_loss_list),
            np.mean(gan_loss_list),
            np.mean(latent_loss_list),
            np.mean(en_mse_loss_list),
            np.mean(en_vgg_loss_list)
        ))

        save_checkpoint(encoder_net, 'EncoderNet', args.MODEL.checkpoint.save_path[0], epoch, args.TRAIN.batch_size, gap=2)
        save_checkpoint(encoder_net, 'DNet', args.MODEL.checkpoint.save_path[1], epoch, args.TRAIN.batch_size, gap=2)

max_stars_repo_name: zhaoyuzhi/ChildPredictor

max_issues_repo_name: zhaoyuzhi/ChildPredictor

max_forks_repo_name: zhaoyuzhi/ChildPredictor


text:
from os import listdir
from os.path import isfile, join
from image import Image
import numpy as np

class DataManager:
    def __init__(self):
        self.training_set = np.empty(10, Image)
        self.test_set = np.empty(10, Image)

    def load_db_from_path(self, folder):
        mnistImages = []
        imageCount = 0

        for digit in range(10):
            print('Loading digit: ' + str(digit))
            path = folder + '/d' + str(digit) + '/'
            files = [f for f in listdir(path) if isfile(join(path, f))]
            digit_images = []

            for image in files:
                img = Image(path+image)
                img.image_data = Image.flatten(img.image_data)
                digit_images.append(img)
                imageCount += 1

            mnistImages.append(digit_images)
            digit_images = []

        print('Loaded ' + str(imageCount) + ' images.')
        return mnistImages

    def split_data(self, data, percentage):
        size_data = len(data)
        for i in range(size_data):
            total = len(data[i])
            n_training = total - (total - (total * percentage / 100))
            n_training = int(n_training)
            digit_images = np.empty(n_training, Image)

            for j in range(n_training): # Images for training
                digit_images[j] = data[i][j]

            self.training_set[i] = digit_images
            digit_images = np.empty(total - n_training, Image)

            for j in range(n_training, total): # Images for test
                digit_images[j - n_training] = data[i][j]

            self.test_set[i] = (digit_images)

    def generate_tags_01(self, data, validDigit): #data is expected to be a 2D array
        totalData = 0
        index = 0
        size_data_valid = len(data[validDigit])

        for i in data:
            totalData += len(i)

        for i in range(validDigit):
            index += len(data[i])

        tags = np.empty(totalData)
        for i in range(index):
            tags[i] = -1

        for i in range(index, index+size_data_valid):
            tags[i] = 1

        for i in range(index+size_data_valid, totalData):
            tags[i] = -1

        return tags

    def generate_tags_09(self, data): #data is expected to be a 2D array
        totalData = 0
        for i in data:
            totalData += len(i)

        tags = np.empty(totalData)
        index = 0
        for digit in range(10):
            for image in data[digit]:
                tags[index] = digit
                index += 1

        return tags

    def flatten(self, array_2D):
        size_array_2D = len(array_2D)
        total = 0
        for i in array_2D:
            total += len(i)

        array_1D = np.empty(total, Image)
        index = 0
        for i in array_2D:
            for img in i:
                array_1D[index] = img
                index += 1

        return array_1D

ext: py

reason: import numpy


text:
import ML.regression as regression
import data
import numpy as np

def test_linear_regression():
    x, y = data.continuous_data()
    lr = regression.LinearRegression()
    lr.fit(x, y)
    predicted_y = lr.predict(x)
    np.testing.assert_array_almost_equal(predicted_y, y)

def test_linear_regression_descent():
    x, y = data.continuous_data()
    lr = regression.LinearRegression()
    lr.fit(x, y, gradient=True)
    predicted_y = lr.predict(x)
    np.testing.assert_array_almost_equal(predicted_y, y)

def test_linear_regression_regularized():
    x, y = data.continuous_data()
    lr = regression.LinearRegression()
    lr.fit(x, y, reg_parameter=0.1)
    predicted_y = lr.predict(x)
    target_y = np.array([0.018688,  1.009157,  1.999626,  2.990095,
                          3.980564])
    np.testing.assert_array_almost_equal(predicted_y, target_y)

def test_linear_regression_descent_regularized():
    x, y = data.continuous_data()
    lr = regression.LinearRegression()
    lr.fit(x, y, gradient=True, reg_parameter=0.1)
    predicted_y = lr.predict(x)
    target_y = np.array([0.018688,  1.009157,  1.999626,  2.990095,
                          3.980564])
    np.testing.assert_array_almost_equal(predicted_y, target_y)

def test_logistic_regression():
    x, y = data.categorical_data()
    lr = regression.LogisticRegression()
    lr.fit(x, y)
    predicted_y = lr.predict(x)
    np.testing.assert_array_almost_equal(predicted_y, y)

def test_logistic_regression_regularized():
    x, y = data.categorical_data()
    lr = regression.LogisticRegression()
    lr.fit(x, y, reg_parameter=0.1)
    predicted_y = lr.predict(x)
    np.testing.assert_array_almost_equal(predicted_y, y)

max_stars_repo_name: christopherjenness/ML-lib

max_forks_repo_name: christopherjenness/ML-lib


text:
'''
Reads the CAESAR .lmrk file format.

'''

from __future__ import absolute_import
__all__ = ['load', 'loads']

def load(f, *args, **kwargs):
    from baiji.serialization.util.openlib import ensure_file_open_and_call
    return ensure_file_open_and_call(f, _load, 'rb', *args, **kwargs)

def _load(f, *args, **kwargs): # pylint: disable=unused-argument
    return loads(f.read())

def loads(s, ret_commands=False):
    import numpy as np

    landmarks = {}
    commands = {}

    for line in s.splitlines():
        if not line.strip():
            continue

        key = line.split()[0]
        data = [float(x) for x in line.split()[1:]]

        if key == '_scale':
            commands['scale'] = np.matrix(data)
        elif key == '_translate':
            commands['translate'] = np.matrix(data)
        elif key == '_rotation':
            commands['rotation'] = np.matrix(data).reshape(3, 3)
        else:
            landmarks[key] = [data[1], data[2], data[0]]

    return (landmarks, commands) if ret_commands else landmarks

max_stars_repo_name: bodylabs/lace

max_issues_repo_name: lace/lace

max_forks_repo_name: metabolize/lace


text:
"""
This module is used to analyze indents. Right now this includes ,indent area and volume,  pile-up area and volume and
z-min/max. Right now it is still early work in progress, and major changes are likely.
@author: Nathanael Jöhrmann
"""
from typing import List

import matplotlib.pyplot as plt
import numpy as np
from matplotlib.axes import Axes

from gdef_reader.gdef_measurement import GDEFMeasurement

class GDEFIndentAnalyzer:
    """
    Class to analyze a GDEFMeasurment with an indent.

    :InstanceAttributes:
    measurement: GDEFMeasurement with the indent to analyze.
    :EndInstanceAttributes:
    """
    # speed optimization to reduce calculationtime of _is_pixel_in_radius()
    pixel_radius_distance_matrix = {}
    max_pixel_radius_value = 0

    def __init__(self, measurement: GDEFMeasurement):
        """
        :param measurement: GDEFMeasurement with the indent to analyze.
        """
        self.measurement = measurement
        self.radius = 0
        self.pileup = []
        self.indent = []
        self.surface = []

        self.below_surface_limit = 0
        self.above_surface_limit = 0

    @classmethod
    def _check_pixel_radius_dict(cls):
        if cls.pixel_radius_distance_matrix == {}:  # todo: or GDEFMeasurement.max_pixel_radius_value < self.settings...
            for x in range(-255, 256):
                for y in range(-255, 256):  # todo: symmetry???
                    cls.pixel_radius_distance_matrix[x, y] = (x ** 2 + y ** 2) ** 0.5

    def _is_pixel_in_radius(self, position, center, radius):
        """Radius in [m]"""
        # optimized for speed; do not introduce help variables, if not necessary
        if self.measurement.settings._pixel_width * GDEFIndentAnalyzer.pixel_radius_distance_matrix[
                (position[0] - center[0]), (position[1] - center[1])] <= radius:
            return True
        else:
            return False

    def _calc_volume_with_radius(self):
        self._check_pixel_radius_dict()
        minimum = np.min(self.measurement.values)
        if minimum is None:
            return 0
        self.radius = abs(7 * minimum)
        minimum_position = self._get_minimum_position()
        pixel_area = self.measurement.settings.pixel_area()
        result = 0
        for index, value in np.ndenumerate(self.measurement.values):
            if self._is_pixel_in_radius(index, minimum_position, self.radius):
                result += value * pixel_area
        return result

    def _get_indent_pile_up_area_mask(self, roughness_part=0.05):
        self._check_pixel_radius_dict()
        minimum = np.min(self.measurement.values)
        self.radius = abs(7 * minimum)
        minimum_position = self._get_minimum_position()

        self.below_surface_limit = roughness_part * minimum
        self.above_surface_limit = abs(self.below_surface_limit)

        result = np.zeros((self.measurement.values.shape[0], self.measurement.values.shape[1], 4))

        for index, _ in np.ndenumerate(self.measurement.values):
            if self._is_pixel_in_radius(index, minimum_position, self.radius):
                if self.measurement.values[index] < self.below_surface_limit:
                    result[index] = (0, 0, 1, 0.6)
                    self.indent.append(index)
                elif self.measurement.values[index] > self.above_surface_limit:
                    result[index] = (0, 1, 0, 0.6)
                    self.pileup.append(index)
                    self.surface.append(index)
                else:
                    result[index] = (0, 0, 0, 0.1)
        return result

    def add_map_with_indent_pile_up_mask_to_axes(self, ax: Axes, roughness_part=0.05) -> Axes:
        """
        Add a topography map with a color mask for pile-up to the given ax. Pile-up is determined as all pixels with
        z>0 + roughness_part \* z_max
        :param ax: Axes object, to whitch the masked map should be added
        :param roughness_part:
        :return: Axes
        """
        data = self._get_indent_pile_up_area_mask(roughness_part=roughness_part)
        extent = self.measurement.settings.size_in_um_for_plot()
        ax.imshow(data, cmap=plt.cm.Reds_r, interpolation='none', extent=extent)
        return ax

    # def analyze_indent(self, roughness_part=0.05):
        pass

    def _calc_area_and_volume(self, index_list):
        area = 0
        volume = 0
        for index in index_list:
            area += self.measurement.settings.pixel_area()
            volume += self.measurement.settings.pixel_area() * abs(self.measurement.values[index])
        return area, volume

    def get_summary_table_data(self) -> List[list]:  # todo: consider move method to utils.py
        """
        Returns a table (list of lists) with data of the indent. The result can be used directly to fill a pptx-table
        with `python-ppxt-interface <https://github.com/natter1/python_pptx_interface/>`_.
        :return:
        """
        indent_area, indent_volume = self._calc_area_and_volume(self.indent)
        pileup_area, pileup_volume = self._calc_area_and_volume(self.pileup)

        result = [["z min / max [m]", f"{self.measurement.values.min():.2e} / {self.measurement.values.max():.2e}"]]
        # result.append(["maximum [m]", f"{self.measurement.values.max():.2e}"])
        result.append(["radius [m]", f"{self.radius:.2e}"])
        result.append(["surface limit [m]", f"+/- {self.above_surface_limit:.2e}"])

        result.append(["indent area [m^2]", f"{indent_area:.2e}"])
        result.append(["indent volume [m^3]", f"{indent_volume:.2e}"])

        result.append(["pileup area [m^2]", f"{pileup_area:.2e}"])
        result.append(["pileup volume [m^3]", f"{pileup_volume:.2e}"])

        return result

    def _get_minimum_position(self):
        # ---------------------------------------------------------------------------------------------------
        # this makes code actually slower (not this method, but other code parts ^^); so do not use np.where:
        # delme = np.where(self.values == np.amin(self.values))
        # return delme[0][0], delme[1][0]
        # ---------------------------------------------------------------------------------------------------
        minimum = np.min(self.measurement.values)
        minimum_position = (0, 0)
        for index, value in np.ndenumerate(self.measurement.values):
            if value == minimum:
                minimum_position = index
                break
        return minimum_position


text:
#-----------------------------------------------------------------------------
# Title      : PyRogue FileIO - File Reader
#-----------------------------------------------------------------------------
# Description:
# Module for reading file data.
# This is a standalone module and can be used outside of a pyrogue installation.
#-----------------------------------------------------------------------------
# This file is part of the rogue software platform. It is subject to
# the license terms in the LICENSE.txt file found in the top-level directory
# of this distribution and at:
#    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html.
# No part of the rogue software platform, including this file, may be
# copied, modified, propagated, or distributed except according to the terms
# contained in the LICENSE.txt file.
#-----------------------------------------------------------------------------
from dataclasses import dataclass
import os
import struct
import numpy
import yaml
import logging

RogueHeaderSize  = 8
RogueHeaderPack  = 'IHBB'

# Default header as a named tuple
@dataclass
class RogueHeader:
    size : int     # 4 Bytes, uint32, I
    flags: int     # 2 bytes, uint16, H
    error: int     # 1 bytes, uint8,  B
    channel: int   # 1 bytes, uint8,  B

""" Rogue Header data

    Attributes
    ----------
    size : int
        Size of the record, minus the header

    flags : int
        Frame flags for the record

    error : int
        Error status for the record

    channel : int
        Channel id for the record

"""

BatchHeaderSize  = 8
BatchHeaderPack  = 'IBBBB'

# Batcher Header
@dataclass
class BatchHeader:
    size: int   # 4 Bytes, uint32, I
    tdest: int  # 1 Byte, uint8, B
    fUser: int  # 1 Byte, uint8, B
    lUser: int  # 1 Byte, uint8, B
    width: int  # 1 Byte, uint8, B

""" Batcher Header data

    Attributes
    ----------
    size: int
        Sub frame size in bytes

    tdest: int
        TDEST value from AXI Stream Frame

    fUser: int
        First USER value from AXI Stream Frame

    lUser: int
        Last USER value from AXI Stream Frame

    width : int
        Width of the batched data in bytes

"""

class FileReaderException(Exception):
    """ File reader exception """
    pass

class FileReader(object):
    """
    A lightweight file reader for Rogue.

    Parameters
    ----------
    files : str or list
        Filename or list of filenams to read data from

    configChan : int
        Channel id of configuration/status stream in the data file. Set to None to disable processing configuration.

    log : obj
        Logging object to use. If set to None a new logger will be created with the name "pyrogue.FileReader".

    batched : bool
        Flag to indicate if data file contains batched data

    Attributes
    ----------
    currCount: int
        Number of data records processed from current file

    totCount: int
        Total number of data records processed from

    configDig: obj
        Current configuration/status dictionary
    """

    def __init__(self, files, configChan=None, log=None, batched=False):
        self._configChan = configChan
        self._currFile   = None
        self._fileSize   = 0
        self._header     = None
        self._config     = {}
        self._currFName  = ""
        self._currCount  = 0
        self._totCount   = 0
        self._batched    = batched

        if log is None:
            self._log = logging.getLogger('pyrogue.FileReader')
        else:
            self._log = log

        if isinstance(files,list):
            self._fileList = files
        else:
            self._fileList = [files]

        # Check to make sure all the files are readable
        for fn in self._fileList:
            if not os.access(fn,os.R_OK):
                raise FileReaderException("Failed to read file {}".format(fn))

    def _nextRecord(self):
        while True:

            # Hit end of file
            if self._currFile.tell() == self._fileSize:
                return False

            # Not enough data left in the file
            if (self._fileSize - self._currFile.tell()) < RogueHeaderSize:
                self._log.warning(f'File under run reading {self._currFName}')
                return False

            self._header = RogueHeader(*struct.unpack(RogueHeaderPack, self._currFile.read(RogueHeaderSize)))
            self._header.size -= 4

            # Set next frame position
            recEnd = self._currFile.tell() + self._header.size

            # Sanity check
            if recEnd > self._fileSize:
                self._log.warning(f"File under run reading {self._currFName}")
                return False

            # Process meta data
            if self._configChan is not None and self._header.channel == self._configChan:
                self._updateConfig(yaml.load(self._currFile.read(self._header.size).decode('utf-8')))

            # This is a data channel
            else:

                self._currCount += 1
                self._totCount += 1
                return True

    def records(self):
        """
        Generator which returns (header, data) tuples

        Returns
        -------
        RogueHeader, bytearray
            (header, data) tuple where header is a dictionary and data is a byte array, for batched = False.

        RogueHeader, BatchHeader, bytearray
            (header, header, data) tuple where header are dictionaried and data is a byte array, for batched = True.

        """
        self._config = {}
        self._currCount = 0
        self._totCount  = 0

        for fn in self._fileList:
            self._fileSize = os.path.getsize(fn)
            self._currFName = fn
            self._currCount = 0

            self._log.debug(f"Processing data records from {self._currFName}")
            with open(fn,'rb') as f:
                self._currFile = f

                while self._nextRecord():

                    # Batch Mode
                    if self._batched:

                        curIdx = 0

                        while True:

                            # Check header size
                            if curIdx + 8 > self._header.size:
                                raise FileReaderException(f'Batch frame header underrun in {self._currFName}')

                            # Read in header data
                            bHead = BatchHeader(*struct.unpack(BatchHeaderPack, self._currFile.read(BatchHeaderSize)))
                            curIdx += 8

                            # Fix header width
                            bHead.width = [2, 4, 8, 16][bHead.width]

                            # Skip rest of header if more than 64-bits
                            if bHead.width > 8:

                                if curIdx + (bHead.width-8) > self._header.size:
                                    raise FileReaderException(f'Batch frame header underrun in {self._currFName}')

                                self._currFile.seek(bHead.width-8)

                            # Check payload size
                            if curIdx + bHead.size > self._header.size:
                                raise FileReaderException(f'Batch frame data underrun in {self._currFName}')

                            # Get data
                            data = numpy.fromfile(self._currFile, dtype=numpy.int8, count=bHead.size)
                            curIdx += bHead.size

                            yield (self._header, bHead, data)

                            if self._header.size == curIdx:
                                break

                    else:
                        try:
                            data = numpy.fromfile(self._currFile, dtype=numpy.int8, count=self._header.size)
                        except Exception:
                            raise FileReaderException(f'Failed to read data from {self._currFName}')

                        yield (self._header, data)

            self._log.debug(f"Processed {self._currCount} data records from {self._currFName}")

        self._log.debug(f"Processed a total of {self._totCount} data records")

    @property
    def currCount(self):
        return self._currCount

    @property
    def totCount(self):
        return self._totCount

    @property
    def configDict(self):
        return self._config

    def configValue(self, path):
        """
        Get a configuration or status value

        Parameters
        ----------
        path : str
            Path of the config/status value to return

        Returns
        -------
        obj
            Requested configuration or status value

        """

        obj = self._config

        if '.' in path:
            lst = path.split('.')
        else:
            lst = [path]

        for a in lst:
            if a in obj:
                obj = obj[a]
            else:
                raise FileReaderException("Failed to find path {}".format(path))

        return obj

    def _updateConfig(self, new):
        # Combination of dictUpdate and keyValueUpdate from pyrogue helpers

        for k,v in new.items():
            if '.' in k:
                d = self._config
                parts = k.split('.')
                for part in parts[:-1]:
                    if part not in d:
                        d[part] = {}
                    d = d.get(part)
                d[parts[-1]] = v
            elif k in self._config:
                self._config[k].update(v)
            else:
                self._config[k] = v

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        pass

max_stars_repo_name: slaclab/rouge

max_issues_repo_name: slaclab/rouge

max_forks_repo_name: slaclab/rouge


text:
from nltk.corpus import stopwords
import nltk
nltk.download('stopwords')
stop=set(stopwords.words('english'))
from keras.preprocessing.sequence import pad_sequences
from transformers import BertTokenizer
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from model import BertForSequenceClassification
import torch
device = torch.device("cuda")
model=BertForSequenceClassification(3)
model.cuda()
import numpy as numpy
import json
import os
import os.path
import math
import torch.nn.functional as F
import numpy as np
import pandas as pd

# string=r"#$%&'!()*+,-.:;<=>?@[\]^_\"`{|}~"

# def remove_URL(text):
#     return re.sub(r"https?://\S+|www\.\S+", "", text)

# def remove_html(text):
#     html=re.compile(r'<.*?>')
#     return html.sub(r'',text)

# def remove_punct(text):
#     table=str.maketrans('\t',' ',string)
#     return text.translate(table)

# def fix_Plan(location):
#     letters_only = re.sub("[^a-zA-Z]",  # Search for all non-letters
#                           " ",          # Replace all non-letters with spaces
#                           str(location))     # Column and row to search

#     words = letters_only.lower().split()
#     stops = set(stopwords.words("english"))
#     meaningful_words = [w for w in words]
#     return (" ".join(meaningful_words))

# class Prediction():

#     def processed(text):

#         text=text.apply(str)
#         text=text.apply(lambda x : x.lower())
#         text=text.apply(lambda x : remove_URL(x))
#         text=text.apply(lambda x : remove_html(x))
#         text=text.apply(lambda x : remove_punct(x))
#         num_responses = len(text)
#         clean_Plan_responses = []

#         for i in range(0,num_responses):
#             clean_Plan_responses.append(fix_Plan(text[i]))

#         text_predict=[]
#         for i in range(len(clean_Plan_responses)):
#             result = re.sub(r"^\s+", "", clean_Plan_responses[i])
#             text_predict.append(re.sub(r"\s+$", "", result))

#     def data_tensor():
#         input_ids = []
#         for sent in text_predict:
#             encoded_sent = tokenizer.encode(sent,add_special_tokens = True)
#             input_ids.append(encoded_sent)
#         MAX_LEN = 32
#         input_ids = pad_sequences(input_ids_train, maxlen=MAX_LEN, dtype="long",value=0, truncating="post", padding="post")
#         attention_masks = []
#         for sent in input_ids:
#             att_mask = [int(token_id > 0) for token_id in sent]
#             attention_masks.append(att_mask)

#         inputs = torch.tensor(input_ids)
#         masks = torch.tensor(attention_masks)
#         data = TensorDataset(inputs,masks)
#         sampler = SequentialSampler(data)
#         loader = DataLoader(data, sampler=sampler, batch_size=inputs.shape[0])

#     resume_weights = './checkpoint_4.pth.tar'
#     if os.path.isfile(resume_weights):
#         if device:
#             checkpoint = torch.load(resume_weights)

#         start_epoch = checkpoint['epoch']
#         best_accuracy = checkpoint['best_accuracy']
#         model.load_state_dict(checkpoint['state_dict'])

#     model.eval()
#     for batch in loader:
#         batch = tuple(t.to(device) for t in batch)
#         b_input_ids, b_input_mask, b_labels = batch
#         with torch.no_grad():
#             outputs = model(b_input_ids,token_type_ids=None,attention_mask=b_input_mask)
#         logits_val = outputs
#         logits_val=F.softmax(logits_val,dim=-1)
#         logits_val = logits_val.detach().cpu().numpy()
#         label_ids = b_labels.to('cpu').numpy()
#         pred_flat = np.argmax(logits_val.reshape(logits_val.shape[0],3), axis=1).flatten()

#     for i in range(len(pred_flat)):
#         if pred_flat[i]==0:
#             pred_flat[i]="neutral"
#         if pred_flat[i]==1:
#             pred_flat[i]="positive"
#         if pred_flat[i]==2:
#             pred_flat[i]=="negative"

#     predictions=json.dumps(pred_flat)
# print(predictons)

def prediction(text):

  string=r"#$%&'!()*+,-.:;<=>?@[\]^_\"`{|}~"
  #text = input().split(':')
  text_df=pd.DataFrame()
  text_df["text"]=text
  print(text_df)
  def remove_URL(text):
      return re.sub(r"https?://\S+|www\.\S+", "", text)

  def remove_html(text):
      html=re.compile(r'<.*?>')
      return html.sub(r'',text)

  def remove_punct(text):
      table=str.maketrans('\t',' ',string)
      return text.translate(table)

  def fix_Plan(location):
      letters_only = re.sub("[^a-zA-Z]",  # Search for all non-letters
                            " ",          # Replace all non-letters with spaces
                            str(location))     # Column and row to search

      words = letters_only.lower().split()
      stops = set(stopwords.words("english"))
      meaningful_words = [w for w in words]
      return (" ".join(meaningful_words))

  resume_weights = './checkpoint_4.pth.tar'
  if os.path.isfile(resume_weights):
    if device:
      checkpoint = torch.load(resume_weights)

    start_epoch = checkpoint['epoch']
    best_accuracy = checkpoint['best_accuracy']
    model.load_state_dict(checkpoint['state_dict'])
  #print("=> loaded checkpoint '{}' (trained for {} epochs)",checkpoint['epoch'],best_accuracy,start_epoch)

  text_df["text"]=text_df["text"].apply(str)
  text_df["text"]=text_df["text"].apply(lambda x : x.lower())
  text_df["text"]=text_df["text"].apply(lambda x : remove_URL(x))
  text_df["text"]=text_df["text"].apply(lambda x : remove_html(x))
  text_df["text"]=text_df["text"].apply(lambda x : remove_punct(x))
  num_responses = text_df.shape[0]
  clean_Plan_responses = []

  for i in range(0,num_responses):
      clean_Plan_responses.append(fix_Plan(text[i]))

  text_predict=[]
  for i in range(len(clean_Plan_responses)):
      result = re.sub(r"^\s+", "", clean_Plan_responses[i])
      text_predict.append(re.sub(r"\s+$", "", result))

  input_ids = []
  for sent in text_predict:
      encoded_sent = tokenizer.encode(sent,add_special_tokens = True,max_length=512)
      input_ids.append(encoded_sent)
  MAX_LEN = 32
  input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long",value=0, truncating="post", padding="post")
  attention_masks = []
  for sent in input_ids:
      att_mask = [int(token_id > 0) for token_id in sent]
      attention_masks.append(att_mask)

  inputs = torch.tensor(input_ids)
  masks = torch.tensor(attention_masks)
  data = TensorDataset(inputs,masks)
  sampler = SequentialSampler(data)
  loader = DataLoader(data, sampler=sampler, batch_size=inputs.shape[0])
  #print("aaa")
  results=[]
  model.eval()
  for batch in loader:
      batch = tuple(t.to(device) for t in batch)
      b_input_ids, b_input_mask = batch
      with torch.no_grad():
          outputs = model(b_input_ids,token_type_ids=None,attention_mask=b_input_mask)
      logits_val = outputs
      logits_val=F.softmax(logits_val,dim=-1)
      logits_val = logits_val.detach().cpu().numpy()

      results.append(logits_val)
  results=np.array(results)
  print(results.shape)
  temp=[]
  for i in range(len(text)):
    temp.append(results[0][i])
  results=np.array(temp)

  return results


text:
#
# Copyright 2021 Lars Pastewka
#           2021 Antoine Sanner
#
# ### MIT license
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

"""
Scale-dependent slope
"""

import numpy as np

from ..HeightContainer import NonuniformLineScanInterface, UniformTopographyInterface

def scale_dependent_slope_from_profile(topography, **kwargs):
    r"""
    Compute the one-dimensional scale-dependent slope.

    The scale-dependent slope is given by

       .. math::
         :nowrap:

         \begin{equation}
         h_\text{rms}^\prime(\lambda) = \left[2A(\lambda)\right]^{1/2}/\lambda
         \end{equation}

    where :math:`A(\lambda)` is the autocorrelation function.

    Parameters
    ----------
    topography : :obj:`SurfaceTopography` or :obj:`UniformLineScan`
        Container storing the uniform topography map
    **kwargs : dict
        Additional keyword parameters are passed on to `autocorrelation_1D`

    Returns
    -------
    r : array
        Distances. (Units: length)
    slope : array
        Slope. (Units: dimensionless)
    """  # noqa: E501
    r, A = topography.autocorrelation_from_profile(**kwargs)
    return r[1:], np.sqrt(2 * A[1:]) / r[1:]

def scale_dependent_slope_from_area(topography, **kwargs):
    r"""
    Compute the two-dimensional, radially averaged scale-dependent slope.

    The scale-dependent slope is given by

       .. math::
         :nowrap:

         \begin{equation}
         h_\text{rms}^\prime(\lambda) = \left[2A(\lambda)\right]^{1/2}/\lambda
         \end{equation}

    where :math:`A(\lambda)` is the autocorrelation function.

    Parameters
    ----------
    topography : :obj:`SurfaceTopography` or :obj:`UniformLineScan`
        Container storing the uniform topography map
    **kwargs : dict
        Additional keyword parameters are passed on to `autocorrelation_2D`

    Returns
    -------
    r : array
        Distances. (Units: length)
    slope : array
        Slope. (Units: dimensionless)
    """  # noqa: E501
    r, A = topography.autocorrelation_from_area(**kwargs)
    return r[1:], np.sqrt(2 * A[1:]) / r[1:]

# Register analysis functions from this module
UniformTopographyInterface.register_function('scale_dependent_slope_from_profile',
                                             scale_dependent_slope_from_profile)
NonuniformLineScanInterface.register_function('scale_dependent_slope_from_profile',
                                              scale_dependent_slope_from_profile)
UniformTopographyInterface.register_function('scale_dependent_slope_from_area',
                                             scale_dependent_slope_from_area)
NonuniformLineScanInterface.register_function('scale_dependent_slope_from_area',
                                              scale_dependent_slope_from_area)

max_stars_repo_name: wgnoehring/SurfaceTopography

max_issues_repo_name: wgnoehring/SurfaceTopography

max_forks_repo_name: wgnoehring/SurfaceTopography


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Layer
from tensorflow.keras.callbacks import Callback

from deep_recommenders.keras.models.nlp import MultiHeadAttention

@tf.keras.utils.register_keras_serializable()
class PositionEncoding(Layer):

    def __init__(self, model_dim, **kwargs):
        self._model_dim = model_dim
        super(PositionEncoding, self).__init__(**kwargs)

    def call(self, inputs, **kwargs):
        seq_length = inputs.shape[1]
        position_encodings = np.zeros((seq_length, self._model_dim))
        for pos in range(seq_length):
            for i in range(self._model_dim):
                position_encodings[pos, i] = pos / np.power(10000, (i-i%2) / self._model_dim)
        position_encodings[:, 0::2] = np.sin(position_encodings[:, 0::2]) # 2i
        position_encodings[:, 1::2] = np.cos(position_encodings[:, 1::2]) # 2i+1
        position_encodings = K.cast(position_encodings, 'float32')
        return position_encodings

    def compute_output_shape(self, input_shape):
        return input_shape

@tf.keras.utils.register_keras_serializable()
class Add(Layer):

    def __init__(self, **kwargs):
        super(Add, self).__init__(**kwargs)

    def call(self, inputs, **kwargs):
        input_a, input_b = inputs
        return input_a + input_b

    def compute_output_shape(self, input_shape):
        return input_shape[0]

@tf.keras.utils.register_keras_serializable()
class PositionWiseFeedForward(Layer):

    def __init__(self, model_dim, inner_dim, trainable=True, **kwargs):
        self._model_dim = model_dim
        self._inner_dim = inner_dim
        self._trainable = trainable
        super(PositionWiseFeedForward, self).__init__(**kwargs)

    def build(self, input_shape):
        self.weights_inner = self.add_weight(
            shape=(input_shape[-1], self._inner_dim),
            initializer='glorot_uniform',
            trainable=self._trainable,
            name="weights_inner")
        self.weights_out = self.add_weight(
            shape=(self._inner_dim, self._model_dim),
            initializer='glorot_uniform',
            trainable=self._trainable,
            name="weights_out")
        self.bias_inner = self.add_weight(
            shape=(self._inner_dim,),
            initializer='uniform',
            trainable=self._trainable,
            name="bias_inner")
        self.bias_out = self.add_weight(
            shape=(self._model_dim,),
            initializer='uniform',
            trainable=self._trainable,
            name="bias_out")
        super(PositionWiseFeedForward, self).build(input_shape)

    def call(self, inputs, **kwargs):
        if K.dtype(inputs) != 'float32':
            inputs = K.cast(inputs, 'float32')
        inner_out = K.relu(K.dot(inputs, self.weights_inner) + self.bias_inner)
        outputs = K.dot(inner_out, self.weights_out) + self.bias_out
        return outputs

    def compute_output_shape(self, input_shape):
        return self._model_dim

@tf.keras.utils.register_keras_serializable()
class LayerNormalization(Layer):

    def __init__(self, epsilon=1e-8, **kwargs):
        self._epsilon = epsilon
        super(LayerNormalization, self).__init__(**kwargs)

    def build(self, input_shape):
        self.beta = self.add_weight(
            shape=(input_shape[-1],),
            initializer='zero',
            name='beta')
        self.gamma = self.add_weight(
            shape=(input_shape[-1],),
            initializer='one',
            name='gamma')
        super(LayerNormalization, self).build(input_shape)

    def call(self, inputs, **kwargs):
        mean, variance = tf.nn.moments(inputs, [-1], keepdims=True)
        normalized = (inputs - mean) / ((variance + self._epsilon) ** 0.5)
        outputs = self.gamma * normalized + self.beta
        return outputs

    def compute_output_shape(self, input_shape):
        return input_shape

@tf.keras.utils.register_keras_serializable()
class Transformer(Layer):

    def __init__(self,
                 vocab_size,
                 model_dim,
                 n_heads=8,
                 encoder_stack=6,
                 decoder_stack=6,
                 feed_forward_size=2048,
                 dropout_rate=0.1,
                 **kwargs):

        self._vocab_size = vocab_size
        self._model_dim = model_dim
        self._n_heads = n_heads
        self._encoder_stack = encoder_stack
        self._decoder_stack = decoder_stack
        self._feed_forward_size = feed_forward_size
        self._dropout_rate = dropout_rate
        super(Transformer, self).__init__(**kwargs)

    def build(self, input_shape):
        self.embeddings = self.add_weight(
            shape=(self._vocab_size, self._model_dim),
            initializer='glorot_uniform',
            trainable=True,
            name="embeddings")
        self.EncoderPositionEncoding = PositionEncoding(self._model_dim)
        self.EncoderMultiHeadAttentions = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads)
            for _ in range(self._encoder_stack)
        ]
        self.EncoderLayerNorms0 = [
            LayerNormalization()
            for _ in range(self._encoder_stack)
        ]
        self.EncoderPositionWiseFeedForwards = [
            PositionWiseFeedForward(self._model_dim, self._feed_forward_size)
            for _ in range(self._encoder_stack)
        ]
        self.EncoderLayerNorms1 = [
            LayerNormalization()
            for _ in range(self._encoder_stack)
        ]
        self.DecoderPositionEncoding = PositionEncoding(self._model_dim)
        self.DecoderMultiHeadAttentions0 = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads, future=True)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms0 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        self.DecoderMultiHeadAttentions1 = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms1 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        self.DecoderPositionWiseFeedForwards = [
            PositionWiseFeedForward(self._model_dim, self._feed_forward_size)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms2 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        super(Transformer, self).build(input_shape)

    def encoder(self, inputs):
        if K.dtype(inputs) != 'int32':
            inputs = K.cast(inputs, 'int32')

        masks = K.equal(inputs, 0)
        # Embeddings
        embeddings = K.gather(self.embeddings, inputs)
        embeddings *= self._model_dim ** 0.5 # Scale
        # Position Encodings
        position_encodings = self.EncoderPositionEncoding(embeddings)
        # Embeddings + Position-encodings
        encodings = embeddings + position_encodings
        # Dropout
        encodings = K.dropout(encodings, self._dropout_rate)

        for i in range(self._encoder_stack):
            # Multi-head-Attention
            attention = self.EncoderMultiHeadAttentions[i]
            attention_input = [encodings, encodings, encodings, masks]
            attention_out = attention(attention_input)
            # Add & Norm
            attention_out += encodings
            attention_out = self.EncoderLayerNorms0[i](attention_out)
            # Feed-Forward
            ff = self.EncoderPositionWiseFeedForwards[i]
            ff_out = ff(attention_out)
            # Add & Norm
            ff_out += attention_out
            encodings = self.EncoderLayerNorms1[i](ff_out)

        return encodings, masks

    def decoder(self, inputs):
        decoder_inputs, encoder_encodings, encoder_masks = inputs
        if K.dtype(decoder_inputs) != 'int32':
            decoder_inputs = K.cast(decoder_inputs, 'int32')

        decoder_masks = K.equal(decoder_inputs, 0)
        # Embeddings
        embeddings = K.gather(self.embeddings, decoder_inputs)
        embeddings *= self._model_dim ** 0.5 # Scale
        # Position Encodings
        position_encodings = self.DecoderPositionEncoding(embeddings)
        # Embeddings + Position-encodings
        encodings = embeddings + position_encodings
        # Dropout
        encodings = K.dropout(encodings, self._dropout_rate)

        for i in range(self._decoder_stack):
            # Masked-Multi-head-Attention
            masked_attention = self.DecoderMultiHeadAttentions0[i]
            masked_attention_input = [encodings, encodings, encodings, decoder_masks]
            masked_attention_out = masked_attention(masked_attention_input)
            # Add & Norm
            masked_attention_out += encodings
            masked_attention_out = self.DecoderLayerNorms0[i](masked_attention_out)

            # Multi-head-Attention
            attention = self.DecoderMultiHeadAttentions1[i]
            attention_input = [masked_attention_out, encoder_encodings, encoder_encodings, encoder_masks]
            attention_out = attention(attention_input)
            # Add & Norm
            attention_out += masked_attention_out
            attention_out = self.DecoderLayerNorms1[i](attention_out)

            # Feed-Forward
            ff = self.DecoderPositionWiseFeedForwards[i]
            ff_out = ff(attention_out)
            # Add & Norm
            ff_out += attention_out
            encodings = self.DecoderLayerNorms2[i](ff_out)

        # Pre-SoftMax 与 Embeddings 共享参数
        linear_projection = K.dot(encodings, K.transpose(self.embeddings))
        outputs = K.softmax(linear_projection)
        return outputs

    def call(self, encoder_inputs, decoder_inputs, **kwargs):
        encoder_encodings, encoder_masks = self.encoder(encoder_inputs)
        encoder_outputs = self.decoder([decoder_inputs, encoder_encodings, encoder_masks])
        return encoder_outputs

    def compute_output_shape(self, input_shape):
        return input_shape[0][0], input_shape[0][1], self._vocab_size

    def get_config(self):
        config = {
            "vocab_size": self._vocab_size,
            "model_dim": self._model_dim,
            "n_heads": self._n_heads,
            "encoder_stack": self._encoder_stack,
            "decoder_stack": self._decoder_stack,
            "feed_forward_size": self._feed_forward_size,
            "dropout_rate": self._dropout_rate
        }
        base_config = super(Transformer, self).get_config()
        return {**base_config, **config}

class Noam(Callback):

    def __init__(self, model_dim, step_num=0, warmup_steps=4000, verbose=False):
        self._model_dim = model_dim
        self._step_num = step_num
        self._warmup_steps = warmup_steps
        self.verbose = verbose
        super(Noam, self).__init__()

    def on_train_begin(self, logs=None):
        logs = logs or {}
        init_lr = self._model_dim ** -.5 * self._warmup_steps ** -1.5
        K.set_value(self.model.optimizer.lr, init_lr)

    def on_batch_end(self, epoch, logs=None):
        logs = logs or {}
        self._step_num += 1
        lrate = self._model_dim ** -.5 * K.minimum(self._step_num ** -.5, self._step_num * self._warmup_steps ** -1.5)
        K.set_value(self.model.optimizer.lr, lrate)

    def on_epoch_begin(self, epoch, logs=None):
        if self.verbose:
            lrate = K.get_value(self.model.optimizer.lr)
            print(f"epoch {epoch} lr: {lrate}")

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        logs['lr'] = K.get_value(self.model.optimizer.lr)

def label_smoothing(inputs, epsilon=0.1):
    """目标平滑"""
    output_dim = inputs.shape[-1]
    smooth_label = (1 - epsilon) * inputs + (epsilon / output_dim)
    return smooth_label

max_issues_repo_name: LongmaoTeamTf/Deep-NLP


text:
# ------------------------------------------------------------------------
# MIT License
#
# Copyright (c) [2021] [Avinash Ranganath]
#
# This code is part of the library PyDL <https://github.com/nash911/PyDL>
# This code is licensed under MIT license (see LICENSE.txt for details)
# ------------------------------------------------------------------------

import numpy as np
import matplotlib.pyplot as plt
import time
from collections import OrderedDict

from pydl.training.training import Training

class PlainTraining(Training):
    """Class for training non-recurrent NN architectures like FC and CNN.

    Args:
        name (str): Name of the training algorithm.
    """

    def __init__(self, nn=None, step_size=1e-2, reg_lambda=0, train_size=70, test_size=30,
                 activatin_type=None, regression=False, name=None):
        super().__init__(nn=nn, step_size=step_size, reg_lambda=reg_lambda, train_size=train_size,
                         test_size=test_size, activatin_type=activatin_type, regression=regression,
                         name=name)

    def prepare_data(self, X, y, normalize=None, dims=None, shuffle=True, batch_size=256,
                     y_onehot=False):
        # Shuffle and split data into train and test sets
        self._train_X, self._train_y, self._test_X, self._test_y = \
            self.shuffle_split_data(X, y, shuffle=shuffle, y_onehot=y_onehot)

        if normalize is not None and any(str in normalize.lower() for str in ['mean', 'zero',
                                                                              'center']):
            # Mean Normalize data
            mean, std, self._train_X = self.mean_normalize(self._train_X)
            _, _, self._test_X = self.mean_normalize(self._test_X, mean, std)
        elif normalize is not None and 'pca' in normalize.lower():
            # Project data on to principle componnts using SVD
            mean, U, S, dims, self._train_X = self.reduce_data_dimensions(self._train_X, dims=dims)
            _, _, _, _, self._test_X = \
                self.reduce_data_dimensions(self._test_X, dims=dims, mean=mean, U=U, S=S,
                                            N=self._train_X.shape[0])
            self._nn.layers[0].reinitialize_weights(inputs=self._train_X)

        print("Training Data:\n", self._train_X.shape)
        print("Test Data:\n", self._test_X.shape)

    def train(self, batch_size=256, epochs=1000, plot=None, log_freq=100):
        start_time = time.time()

        if plot is not None:
            fig, axs = plt.subplots(2, sharey=False, sharex=True)
            fig.suptitle(plot, fontsize=20)
        else:
            fig = axs = None

        epochs_list = list()
        train_loss = list()
        test_loss = list()
        train_accuracy = list()
        test_accuracy = list()
        num_batches = int(np.ceil(self._train_X.shape[0] / batch_size))

        init_train_l = self.batch_loss(self._train_X, self._train_y, batch_size, inference=False,
                                       log_freq=log_freq)
        self.print_log(0, plot, fig, axs, batch_size, init_train_l, epochs_list, train_loss,
                       test_loss, train_accuracy, test_accuracy, log_freq)

        for e in range(epochs):
            for i in range(num_batches):
                if log_freq < 0:
                    print("Epoch: %d -- Batch: %d" % (e, i))
                start = int(batch_size * i)
                if i == num_batches - 1:
                    end = self._train_X.shape[0]
                else:
                    end = start + batch_size

                train_l = self.loss(self._train_X[start:end], self._train_y[start:end],
                                    inference=False)
                loss_grad = self.loss_gradient(self._train_X[start:end], self._train_y[start:end])
                _ = self._nn.backward(loss_grad, self._lambda)
                self.update_network(e + 1)

            if (e + 1) % np.abs(log_freq) == 0:
                self.print_log((e + 1), plot, fig, axs, batch_size, train_l, epochs_list,
                               train_loss, test_loss, train_accuracy, test_accuracy, log_freq)

        training_logs_dict = OrderedDict()
        training_logs_dict['epochs'] = epochs_list
        training_logs_dict['train_loss'] = train_loss
        training_logs_dict['test_loss'] = test_loss
        training_logs_dict['train_accuracy'] = train_accuracy
        training_logs_dict['test_accuracy'] = test_accuracy

        end_time = time.time()
        print("\nTraining Time: %.2f(s)" % (end_time - start_time))

        return training_logs_dict


text:
##############################################################################################################################################################
##############################################################################################################################################################
"""
Helperfunctions used by other scripts.
"""
##############################################################################################################################################################
##############################################################################################################################################################

import toml
import time
import torch
import datetime
import dateutil

import numpy as np

from torchvision.utils import save_image as save_grid
from torchvision.utils import make_grid

from matplotlib import pyplot as plt
plt.style.use(['seaborn-white', 'seaborn-paper'])
plt.rc('font', family='serif')

##############################################################################################################################################################
##############################################################################################################################################################

class Tee(object):
    """
    Class to make it possible to print text to the console and also write the
    output to a file.
    """

    def __init__(self, original_stdout, file):

        # keep the original stdout
        self.original_stdout = original_stdout

        # the file to write to
        self.log_file_handler= open(file, 'w')

        # all the files the print statement should be saved to
        self.files = [self.original_stdout, self.log_file_handler]

    def write(self, obj):

        # for each file
        for f in self.files:

            # write to the file
            f.write(obj)

            # If you want the output to be visible immediately
            f.flush()

    def flush(self):

        # for each file
        for f in self.files:

            # If you want the output to be visible immediately
            f.flush()

    def end(self):

        # close the file
        self.log_file_handler.close()

        # return the original stdout
        return self.original_stdout

##############################################################################################################################################################

def write_toml_to_file(cfg, file_path):
    """
    Write the parser to a file.
    """

    with open(file_path, 'w') as output_file:
        toml.dump(cfg, output_file)

    print('=' * 57)
    print("Config file saved: ", file_path)
    print('=' * 57)

##############################################################################################################################################################

def append_accuracy(save_folder, score):

    with save_folder["accuracy"].open('a') as file:
        file.write(str(score)+"\n")

def save_accuracy(save_path, score):

    with save_path.open('w') as file:
        file.write(str(score))

##############################################################################################################################################################

class TrainingTimer():
    """
    Keep track of training times.
    """

    def __init__(self):

        # get the current time
        self.start = datetime.datetime.fromtimestamp(time.time())
        self.eval_start = datetime.datetime.fromtimestamp(time.time())

        # print it human readable
        print("Training start: ", self.start)
        print('=' * 57)

    def print_end_time(self):

        # get datetimes for simplicity
        datetime_now = datetime.datetime.fromtimestamp(time.time())

        print("Training finish: ", datetime_now)

        # compute the time different between now and the input
        rd = dateutil.relativedelta.relativedelta(datetime_now, self.start)

        # print the duration in human readable
        print(f"Training duration: {rd.hours} hours, {rd.minutes} minutes, {rd.seconds} seconds")

    def print_time_delta(self):

        # get datetimes for simplicity
        datetime_now = datetime.datetime.fromtimestamp(time.time())

        # compute the time different between now and the input
        rd = dateutil.relativedelta.relativedelta(datetime_now, self.eval_start)

        # print the duration in human readable
        print(f"Duration since last evaluation: {rd.hours} hours, {rd.minutes} minutes, {rd.seconds} seconds")
        print('=' * 57)

        # update starting time
        self.eval_start = datetime.datetime.fromtimestamp(time.time())

##############################################################################################################################################################

def plot_progress(input_images, recon_images, save_folder, nbr_epoch, text, max_nbr_samples=16, **kwargs):

    # check which number is smaller and use it to extract the correct amount of images
    nbr_samples = min(input_images.shape[0], max_nbr_samples)

    # collect what to plot
    gather_sample_results = []
    gather_sample_results.append(input_images[:nbr_samples].detach())
    gather_sample_results.append(recon_images[:nbr_samples].detach())

    # make it a torch tensor
    gather_sample_results = torch.cat(gather_sample_results, dim=0)

    # create the filename
    save_name = save_folder["images"] / (text + "_" + str(nbr_epoch) + ".pdf")

    # save images as grid
    # each element in the batch has its own column
    save_grid(gather_sample_results, save_name, nrow=nbr_samples, **kwargs)

##############################################################################################################################################################

def plot_recon_and_lable(input_images, labels, recon_images, predictions, save_folder, nbr_epoch, text, max_nbr_samples=16, **kwargs):

    # check which number is smaller and use it to extract the correct amount of images
    nbr_samples = min(input_images.shape[0], max_nbr_samples)

    predictions = predictions[:nbr_samples]
    labels = labels[:nbr_samples]

    # collect what to plot
    gather_sample_results = []
    gather_sample_results.append(input_images[:nbr_samples].detach())
    gather_sample_results.append(recon_images[:nbr_samples].detach())

    # make it a torch tensor
    gather_sample_results = torch.cat(gather_sample_results, dim=0)

    # create the filename
    save_name = save_folder["images"] / (text + "_" + str(nbr_epoch) + ".png")

    # save images as grid
    # each element in the batch has its own column
    grid = make_grid(gather_sample_results, nrow=nbr_samples, **kwargs).cpu().numpy()

    fig, ax = plt.subplots(figsize=(16*4, 4))
    plt.imshow(np.transpose(grid, (1,2,0)), interpolation='nearest')
    for index, (pred, labels) in enumerate(zip(predictions, labels)):
        ax.text(x=index*130+64, y=-13, s=str(labels), fontsize=11, horizontalalignment="center", verticalalignment="center")
        ax.text(x=index*130+64, y=278, s=str(pred), fontsize=11, horizontalalignment="center", verticalalignment="center", color="green" if pred==labels else "red")
    plt.axis("off")
    fig.savefig(save_name, dpi=fig.dpi, bbox_inches='tight')
    plt.close(fig=fig)

##############################################################################################################################################################

def stringify(x):
    # get the labels from each dimension and combine them to a string
    return "_".join(str(y) for y in x)

##############################################################################################################################################################


text:
import scipy, scipy.stats, numpy, itertools

# Chi Square Test of independence (returns probability that attributes are dependant)
def chiSqTest(attr1,attr2):
    # consistency check
    if len(attr1)!=len(attr2):
        raise ValueError('atribute lists are not the same size')
    # build class list
    class1 = list(set(attr1)-set(['']))
    class2 = list(set(attr2)-set(['']))
    # build contingency table
    cont_table = numpy.zeros((len(class1),len(class2)),numpy.int)
    # fill contingency table
    for val1,val2 in itertools.izip(attr1,attr2):
        #print val1, val2
        if len(val1)>0 and len(val2)>0:
            i = class1.index(val1)
            j = class2.index(val2)
            cont_table[i,j]+=1
    # perform chi square test
    try:
        chi2, p, dof, exp_frequencies = scipy.stats.chi2_contingency(cont_table)
    except ValueError:
        # print 'chi_sq_test_failed'
        return 0
    # validity check for expected frequencies table
    total_count = float(len(class1)*len(class2))
    invalid_count = 0.0
    for val in numpy.nditer(exp_frequencies):
        # test should be used only if expected frequency in each cell is at least 5
        if val<5:
            invalid_count+=1;
    # if test is invalid (more than 20% cells are below 5), return 0
    if invalid_count/total_count>0.2:
        return 0
    # if test is valid, return inverted probability
    return 1-p

# Pearson correlation coeficient for numeric attributes
def pearsonTest(attr1,attr2):
    # consistency check
    if len(attr1)!=len(attr2):
        raise ValueError('atribute lists are not the same size')
    # create attribute arrays
    attr1_list = list()
    attr2_list = list()
    for val1,val2 in itertools.izip(attr1,attr2):
        if len(val1)!=0 and len(val2)!=0:
            attr1_list.append(float(val1))
            attr2_list.append(float(val2))
    attr1_array = numpy.array(attr1_list,numpy.float)
    attr2_array = numpy.array(attr2_list,numpy.float)
    # calculate correlation
    return abs(scipy.stats.pearsonr(attr1_array, attr2_array)[0])

max_stars_repo_path: University - Team Projects/Technicom APG/modules/tests.py

max_stars_repo_name: mpuheim/Various

max_issues_repo_name: mpuheim/Various

max_forks_repo_name: mpuheim/Various

reason: import scipy


text:
import numpy as np

import autogalaxy as ag

# MockProfiles #

class MockLightProfile(ag.lp.LightProfile):
    def __init__(
        self,
        image_2d=None,
        image_2d_value=None,
        image_2d_first_value=None,
        value=None,
        value1=None,
    ):

        super().__init__()

        self.image_2d = image_2d
        self.image_2d_value = image_2d_value
        self.image_2d_first_value = image_2d_first_value

        self.value = value
        self.value1 = value1

    def image_2d_from(self, grid):

        if self.image_2d is not None:
            return self.image_2d

        image_2d = np.ones(shape=(grid.shape[0]))

        if self.image_2d_first_value is not None:

            image_2d[0] = self.image_2d_first_value

        return image_2d


text:
# #########################################################################
# Copyright (c) 2020, UChicago Argonne, LLC. All rights reserved.         #
#                                                                         #
# Copyright 2020. UChicago Argonne, LLC. This software was produced       #
# under U.S. Government contract DE-AC02-06CH11357 for Argonne National   #
# Laboratory (ANL), which is operated by UChicago Argonne, LLC for the    #
# U.S. Department of Energy. The U.S. Government has rights to use,       #
# reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR    #
# UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR        #
# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is     #
# modified to produce derivative works, such modified software should     #
# be clearly marked, so as not to confuse it with the version available   #
# from ANL.                                                               #
#                                                                         #
# Additionally, redistribution and use in source and binary forms, with   #
# or without modification, are permitted provided that the following      #
# conditions are met:                                                     #
#                                                                         #
#     * Redistributions of source code must retain the above copyright    #
#       notice, this list of conditions and the following disclaimer.     #
#                                                                         #
#     * Redistributions in binary form must reproduce the above copyright #
#       notice, this list of conditions and the following disclaimer in   #
#       the documentation and/or other materials provided with the        #
#       distribution.                                                     #
#                                                                         #
#     * Neither the name of UChicago Argonne, LLC, Argonne National       #
#       Laboratory, ANL, the U.S. Government, nor the names of its        #
#       contributors may be used to endorse or promote products derived   #
#       from this software without specific prior written permission.     #
#                                                                         #
# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS     #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT       #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS       #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago     #
# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,        #
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,    #
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;        #
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER        #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT      #
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN       #
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE         #
# POSSIBILITY OF SUCH DAMAGE.                                             #
# #########################################################################

import numpy as np
import pywt
import os
import sys
import time
from PIL import Image
import glob
import scipy.constants as sc
from func import prColor, frankotchellappa, image_roi, Wavelet_transform, write_h5, write_json, find_disp
from euclidean_dist import dist_numba
import matplotlib
# matplotlib.use('Agg')
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
import multiprocessing as ms
import concurrent.futures
import scipy.interpolate as sfit
import copy

def load_image(file_path):
    if os.path.exists(file_path):
        img = np.array(Image.open(file_path))
    else:
        prColor('Error: wrong data path. No data is loaded.', 'red')
        sys.exit()
    return np.array(img)

def image_preprocess(image, have_dark, dark_img, have_flat, flat_img):
    '''
        do the flat or dark correction for the images
        img:            image to be corrected
        have_dark:      if there is dark
        dark_img:           dark image
        have_flat:      if there is flat
        flat_img:           flat image
    '''
    if (have_flat != 0 and have_dark != 0):
        numerator = (flat_img - dark_img).clip(0.00000001)
        # numerator = numerator / np.amax(numerator)
        image = ((image - dark_img) / numerator) * np.amax(image)
    elif (have_dark != 0):
        image = (image - dark_img).clip(0.00000001)
    elif (have_flat != 0):
        flat_img[flat_img == 0] = 0.00000001
        # flat_img = flat_img / np.amax(flat_img)
        image = (image / flat_img) * np.amax(image)

    return image

class WXST:
    def __init__(self,
                 img,
                 ref,
                 M_image=512,
                 N_s=5,
                 cal_half_window=20,
                 N_s_extend=4,
                 n_cores=4,
                 n_group=4,
                 energy=14e3,
                 p_x=0.65e-6,
                 z=500e-3,
                 wavelet_level_cut=2,
                 pyramid_level=2,
                 n_iter=1,
                 use_wavelet=True):
        self.img_data = img
        self.ref_data = ref
        # roi of the images
        self.M_image = M_image
        # template window, the N_s nearby pixels used to represent the local pixel, 2*N_s+1
        self.N_s = N_s
        # the number of the area to calculate for each pixel, 2*cal_half_window X 2*cal_half_window
        self.cal_half_window = cal_half_window
        # the calculation window for high order pyramid
        self.N_s_extend = N_s_extend

        # process number for parallel
        self.n_cores = n_cores
        # number to reduce the each memory use
        self.n_group = n_group

        # energy, 10kev
        self.energy = energy
        self.wavelength = sc.value(
            'inverse meter-electron volt relationship') / energy
        # pixel size [m]
        self.p_x = p_x
        # distance [m]
        self.z = z
        self.wavelet_level_cut = wavelet_level_cut
        # pyramid level to wrap the images
        self.pyramid_level = pyramid_level
        # iterations for the calculation
        self.n_iter = n_iter
        # if use wavelet transform or not
        self.use_wavelet = use_wavelet

    def template_stack(self, img):
        '''
            stack the nearby pixels in 2*N_s+1
        '''
        img_stack = []
        axis_Nw = np.arange(-self.N_s, self.N_s + 1)
        for x in axis_Nw:
            for y in axis_Nw:
                img_stack.append(np.roll(np.roll(img, x, axis=0), y, axis=1))

        return np.array(img_stack)

    def pyramid_data(self):
        # get the pyramid data
        # method 1, pyramid then stack the image, which means the template window size is increasing for different pyramid level
        # get the pyramid wrapping images
        ref_pyramid = []
        img_pyramid = []
        prColor(
            'obtain pyramid image and stack the window with pyramid level: {}'.
            format(self.pyramid_level), 'green')
        ref_pyramid.append(self.ref_data)
        img_pyramid.append(self.img_data)

        for kk in range(self.pyramid_level):
            ref_pyramid.append(
                pywt.dwtn(ref_pyramid[kk], 'db3', mode='zero',
                          axes=(-2, -1))['aa'])
            img_pyramid.append(
                pywt.dwtn(img_pyramid[kk], 'db3', mode='zero',
                          axes=(-2, -1))['aa'])

        normlize_std = lambda img: (
            (img - np.ndarray.mean(img, axis=0)) / np.ndarray.std(img, axis=0))

        ref_pyramid = [
            normlize_std(self.template_stack(img_data))
            for img_data in ref_pyramid
        ]
        img_pyramid = [
            normlize_std(self.template_stack(img_data))
            for img_data in img_pyramid
        ]

        return ref_pyramid, img_pyramid

    def resampling_spline(self, img, s):
        # img: original
        # s: size of the sampling, (row, col)
        m, n = img.shape
        x_axis = np.arange(n)
        y_axis = np.arange(m)
        fit = sfit.RectBivariateSpline(y_axis, x_axis, img)

        x_new = np.linspace(0, n - 1, s[1])
        y_new = np.linspace(0, m - 1, s[0])

        return fit(y_new, x_new)

    def wavelet_data(self):
        # process the data to get the wavelet transform
        ref_pyramid, img_pyramid = self.pyramid_data()
        if self.use_wavelet:
            prColor('obtain wavelet data...', 'green')
            wavelet_method = 'db2'
            # wavelet_method = 'bior1.3'
            # wavelet wrapping level. 2 is half, 3 is 1/3 of the size
            max_wavelet_level = pywt.dwt_max_level(ref_pyramid[0].shape[0],
                                                   wavelet_method)
            prColor('max wavelet level: {}'.format(max_wavelet_level), 'green')
            self.wavelet_level = max_wavelet_level
            coefs_level = self.wavelet_level + 1 - self.wavelet_level_cut

            if ref_pyramid[0].shape[0] > 150:
                self.wavelet_add_list = [0, 0, 0, 0, 0, 0]
            elif ref_pyramid[0].shape[0] > 50:
                self.wavelet_add_list = [0, 0, 1, 2, 2, 2]
            else:
                self.wavelet_add_list = [2, 2, 2, 2, 2, 2]

            # wavelet transform and cut for the pyramid images
            start_time = time.time()
            for p_level in range(len(img_pyramid)):
                if p_level > len(self.wavelet_add_list):
                    wavelevel_add = 2
                else:
                    wavelevel_add = self.wavelet_add_list[p_level]

                img_wa, level_name = Wavelet_transform(
                    img_pyramid[p_level],
                    wavelet_method,
                    w_level=self.wavelet_level,
                    return_level=coefs_level + wavelevel_add)
                img_pyramid[p_level] = img_wa

                ref_wa, level_name = Wavelet_transform(
                    ref_pyramid[p_level],
                    wavelet_method,
                    w_level=self.wavelet_level,
                    return_level=coefs_level + wavelevel_add)
                ref_pyramid[p_level] = ref_wa

                prColor(
                    'pyramid level: {}\nvector length: {}\nUse wavelet coef: {}'
                    .format(p_level, ref_wa.shape[2], level_name), 'green')

            end_time = time.time()
            print('wavelet time: {}'.format(end_time - start_time))
        else:
            img_pyramid = [
                np.moveaxis(img_data, 0, -1) for img_data in img_pyramid
            ]
            ref_pyramid = [
                np.moveaxis(img_data, 0, -1) for img_data in ref_pyramid
            ]
            self.wavelet_level = None
            self.wavelet_add_list = None
            self.wavelet_level_cut = None

        return ref_pyramid, img_pyramid

    def displace_wavelet(self, y_list, img_wa_stack, ref_wa_stack,
                         displace_pyramid, cal_half_window, n_pad):
        '''
            calculate the coefficient of each pixel
        '''
        dim = img_wa_stack.shape
        disp_x = np.zeros((dim[0], dim[1]))
        disp_y = np.zeros((dim[0], dim[1]))

        # the axis for the peak position finding
        window_size = 2 * cal_half_window + 1
        y_axis = np.arange(window_size) - cal_half_window
        x_axis = np.arange(window_size) - cal_half_window
        XX, YY = np.meshgrid(x_axis, y_axis)

        for yy in range(dim[0]):
            for xx in range(dim[1]):
                img_wa_line = img_wa_stack[yy, xx, :]
                ref_wa_data = ref_wa_stack[
                    n_pad + yy + int(displace_pyramid[0][yy, xx]):n_pad + yy +
                    int(displace_pyramid[0][yy, xx]) + window_size,
                    n_pad + xx + int(displace_pyramid[1][yy, xx]):n_pad + xx +
                    int(displace_pyramid[1][yy, xx]) + window_size, :]

                Corr_img = dist_numba(img_wa_line, ref_wa_data)
                '''
                    use gradient to find the peak
                '''
                disp_y[yy, xx], disp_x[yy, xx] = find_disp(
                    Corr_img, XX, YY, sub_resolution=True)

        disp_add_y = displace_pyramid[0] + disp_y
        disp_add_x = displace_pyramid[1] + disp_x
        return disp_add_y, disp_add_x, y_list

    def solver(self):

        ref_pyramid, img_pyramid = self.wavelet_data()
        transmission = self.img_data / self.ref_data
        for attr in ('img_data', 'ref_data'):
            self.__dict__.pop(attr, None)

        cores = ms.cpu_count()
        prColor('Computer available cores: {}'.format(cores), 'green')

        if cores > self.n_cores:
            cores = self.n_cores
        else:
            cores = ms.cpu_count()
        prColor('Use {} cores'.format(cores), 'light_purple')
        prColor('Process group number: {}'.format(self.n_group),
                'light_purple')

        if cores * self.n_group > self.M_image:
            n_tasks = 4
        else:
            n_tasks = cores * self.n_group

        start_time = time.time()
        # use pyramid wrapping
        max_pyramid_searching_window = int(
            np.ceil(self.cal_half_window / 2**self.pyramid_level))
        searching_window_pyramid_list = [self.N_s_extend
                                         ] * self.pyramid_level + [
                                             int(max_pyramid_searching_window)
                                         ]
        m, n, c = img_pyramid[0].shape
        displace = [np.zeros((m, n)), np.zeros((m, n))]

        for k_iter in range(self.n_iter):
            # iteration to approximating the results
            displace = [img / 2**self.pyramid_level for img in displace]

            m, n, c = img_pyramid[-1].shape
            displace[0] = self.resampling_spline(displace[0], (m, n))
            displace[1] = self.resampling_spline(displace[1], (m, n))

            prColor(
                'down sampling the dispalce to size: {}'.format(
                    displace[0].shape), 'green')

            displace = [
                np.fmax(
                    np.fmin(displace[0],
                            self.cal_half_window / 2**self.pyramid_level),
                    -self.cal_half_window / 2**self.pyramid_level),
                np.fmax(
                    np.fmin(displace[1],
                            self.cal_half_window / 2**self.pyramid_level),
                    -self.cal_half_window / 2**self.pyramid_level)
            ]

            for p_level in range(self.pyramid_level, -1, -1):
                # first pyramid, searching the window. Then search nearby
                if p_level == self.pyramid_level:
                    pyramid_seaching_window = searching_window_pyramid_list[
                        p_level]
                    m, n, c = img_pyramid[p_level].shape
                    displace_pyramid = [np.round(img) for img in displace]

                    n_pad = int(np.ceil(self.cal_half_window / 2**p_level))

                else:
                    pyramid_seaching_window = searching_window_pyramid_list[
                        p_level]

                    m, n, c = img_pyramid[p_level].shape
                    displace_pyramid = [
                        np.round(self.resampling_spline(img * 2, (m, n)))
                        for img in displace
                    ]

                    displace_pyramid = [
                        np.fmax(
                            np.fmin(displace_pyramid[0],
                                    self.cal_half_window / 2**p_level),
                            -self.cal_half_window / 2**p_level),
                        np.fmax(
                            np.fmin(displace_pyramid[1],
                                    self.cal_half_window / 2**p_level),
                            -self.cal_half_window / 2**p_level)
                    ]

                    n_pad = int(np.ceil(self.cal_half_window / 2**p_level))

                prColor(
                    'pyramid level: {}\nImage size: {}\nsearching window:{}'.
                    format(p_level, ref_pyramid[p_level].shape,
                           pyramid_seaching_window), 'cyan')

                y_axis = np.arange(ref_pyramid[p_level].shape[0])
                chunks_idx_y = np.array_split(y_axis, n_tasks)

                dim = img_pyramid[p_level].shape

                ref_wa_pad = np.pad(ref_pyramid[p_level],
                                    ((n_pad + pyramid_seaching_window,
                                      n_pad + pyramid_seaching_window),
                                     (n_pad + pyramid_seaching_window,
                                      n_pad + pyramid_seaching_window),
                                     (0, 0)),
                                    'constant',
                                    constant_values=(0, 0))

                # use CPU parallel to calculate
                result_list = []
                '''
                    calculate the pixel displacement for the pyramid images
                '''
                with concurrent.futures.ProcessPoolExecutor(
                        max_workers=cores) as executor:

                    futures = []
                    for y_list in chunks_idx_y:
                        # get the stack data
                        img_wa_stack = img_pyramid[p_level][y_list, :, :]
                        ref_wa_stack = ref_wa_pad[
                            y_list[0]:y_list[-1] + 2 *
                            (n_pad + pyramid_seaching_window) + 1, :, :]

                        # start the jobs
                        futures.append(
                            executor.submit(self.displace_wavelet, y_list,
                                            img_wa_stack, ref_wa_stack,
                                            (displace_pyramid[0][y_list, :],
                                             displace_pyramid[1][y_list, :]),
                                            pyramid_seaching_window, n_pad))

                    for future in concurrent.futures.as_completed(futures):

                        try:
                            result_list.append(future.result())
                            # display the status of the program
                            Total_iter = cores * self.n_group
                            Current_iter = len(result_list)
                            percent_iter = Current_iter / Total_iter * 100
                            str_bar = '>' * (int(np.ceil(
                                percent_iter / 2))) + ' ' * (int(
                                    (100 - percent_iter) // 2))
                            prColor(
                                '\r' + str_bar + 'processing: [%3.1f%%] ' %
                                (percent_iter), 'purple')

                        except:
                            prColor('Error in the parallel calculation', 'red')

                disp_y_list = [item[0] for item in result_list]
                disp_x_list = [item[1] for item in result_list]
                y_list = [item[2] for item in result_list]

                displace_y = np.zeros((dim[0], dim[1]))
                displace_x = np.zeros((dim[0], dim[1]))

                for y, disp_x, disp_y in zip(y_list, disp_x_list, disp_y_list):
                    displace_x[y, :] = disp_x
                    displace_y[y, :] = disp_y

                displace = [
                    np.fmax(
                        np.fmin(displace_y, self.cal_half_window / 2**p_level),
                        -self.cal_half_window / 2**p_level),
                    np.fmax(
                        np.fmin(displace_x, self.cal_half_window / 2**p_level),
                        -self.cal_half_window / 2**p_level)
                ]
                prColor('displace map wrapping: {}'.format(displace[0].shape),
                        'green')
                print('max of displace: {}, min of displace: {}'.format(
                    np.amax(displace[0]), np.amin(displace[1])))

        end_time = time.time()
        prColor(
            '\r' + 'Processing time: {:0.3f} s'.format(end_time - start_time),
            'light_purple')

        displace[0] = -displace[0][self.cal_half_window:-self.cal_half_window,
                                   self.cal_half_window:-self.cal_half_window]
        displace[1] = -displace[1][self.cal_half_window:-self.cal_half_window,
                                   self.cal_half_window:-self.cal_half_window]

        DPC_y = (displace[0] - np.mean(displace[0])) * p_x / z
        DPC_x = (displace[1] - np.mean(displace[1])) * p_x / z

        phase = -frankotchellappa(
            DPC_x, DPC_y) * self.p_x * 2 * np.pi / self.wavelength
        self.time_cost = end_time - start_time

        return displace, [DPC_y, DPC_x], phase, transmission

    def run(self, result_path=None):
        self.displace, self.DPC, self.phase, self.transmission = self.solver()

        if result_path is not None:
            '''
            save the calculation results
            '''
            if not os.path.exists(result_path):
                os.makedirs(result_path)

            self.result_filename = 'WXST' + str(self.M_image) + '_px_' + str(
                self.wavelet_level_cut) + 'wavelet_Cutlevel_' + str(
                    self.pyramid_level) + 'pyramid_level'

            kk = 1
            while os.path.exists(
                    os.path.join(
                        result_path,
                        self.result_filename + '.hdf5')) or os.path.exists(
                            os.path.join(result_path,
                                         self.result_filename + '.json')):
                self.result_filename = 'WXST' + str(
                    self.M_image) + '_px_' + str(
                        self.wavelet_level_cut) + 'wavelet_Cutlevel_' + str(
                            self.pyramid_level
                        ) + 'pyramid_level' + '_{}'.format(kk)
                kk += 1
            write_h5(
                result_path, self.result_filename, {
                    'displace_x': self.displace[1],
                    'displace_y': self.displace[0],
                    'DPC_x': self.DPC[1],
                    'DPC_y': self.DPC[0],
                    'phase': self.phase,
                    'transmission_image': self.transmission
                })

            parameter_dict = {
                'M_image': self.M_image,
                'template_window': self.N_s,
                'N_s extend': self.N_s_extend,
                'half_window': self.cal_half_window,
                'energy': self.energy,
                'wavelength': self.wavelength,
                'pixel_size': self.p_x,
                'z_distance': self.z,
                'cpu_cores': self.n_cores,
                'n_group': self.n_group,
                'wavelet_level': self.wavelet_level,
                'pyramid_level': self.pyramid_level,
                'n_iter': self.n_iter,
                'time_cost': self.time_cost,
                'use_wavelet': self.use_wavelet,
                'wavelet_level_cut': self.wavelet_level_cut,
                'wavelet_add': self.wavelet_add_list
            }

            write_json(result_path, self.result_filename, parameter_dict)

if __name__ == "__main__":
    if len(sys.argv) == 1:

        Folder_path = 'Data_folder_path'
        File_ref = os.path.join(Folder_path, 'ref_001.tif')
        File_img = os.path.join(Folder_path, 'sample_001.tif')

        Folder_result = os.path.join(Folder_path, 'WXST_test')
        # [image_size, template_window, cal_half_window, n_group, n_cores, energy, pixel_size, distance, use_wavelet, wavelet_ct, pyramid level, n_iteration]
        parameter_wavelet = [
            500, 5, 10, 4, 4, 14e3, 0.65e-6, 310e-3, 1, 2, 2, 1
        ]

    elif len(sys.argv) == 4:
        File_img = sys.argv[1]
        File_ref = sys.argv[2]
        Folder_result = sys.argv[3]
        # [image_size, template_window, cal_half_window, n_group, n_cores, energy, pixel_size, distance, wavelet_ct, pyramid level, n_iteration]
        parameter_wavelet = [
            1500, 5, 20, 4, 4, 14e3, 0.65e-6, 500e-3, 1, 2, 2, 1
        ]
    elif len(sys.argv) == 16:
        File_img = sys.argv[1]
        File_ref = sys.argv[2]
        Folder_result = sys.argv[3]
        parameter_wavelet = sys.argv[4:]
    else:
        prColor('Wrong parameters! should be: sample, ref, result', 'red')

    prColor('folder: {}'.format(Folder_result), 'green')
    # roi of the images
    M_image = int(parameter_wavelet[0])
    # template window, the N_s nearby pixels used to represent the local pixel, 2*N_s+1
    N_s = int(parameter_wavelet[1])
    # the number of the area to calculate for each pixel, 2*cal_half_window X 2*cal_half_window
    cal_half_window = int(parameter_wavelet[2])
    # the calculation window for high order pyramid
    N_s_extend = 4

    # process number for parallel
    n_cores = int(parameter_wavelet[3])
    # number to reduce the each memory use
    n_group = int(parameter_wavelet[4])

    # energy, 10kev
    energy = float(parameter_wavelet[5])
    wavelength = sc.value('inverse meter-electron volt relationship') / energy
    p_x = float(parameter_wavelet[6])
    z = float(parameter_wavelet[7])
    use_wavelet = int(parameter_wavelet[8])
    wavelet_level_cut = int(parameter_wavelet[9])
    # pyramid level to wrap the images
    pyramid_level = int(parameter_wavelet[10])
    n_iter = int(parameter_wavelet[11])

    ref_data = load_image(File_ref)
    img_data = load_image(File_img)

    # radio_flat = load_images(Folder_radio, 'flat*.tif')
    # radio_dark = load_images(Folder_radio, 'dark*.tif')

    # ref_data = image_preprocess(ref_data, have_dark=1, dark_img=radio_dark, have_flat=1, flat_img=radio_flat)
    # img_data = image_preprocess(img_data, have_dark=1, dark_img=radio_dark, have_flat=1, flat_img=radio_flat)

    # take out the roi
    ref_data = image_roi(ref_data, M_image)
    img_data = image_roi(img_data, M_image)

    WXST_solver = WXST(img_data,
                       ref_data,
                       M_image=M_image,
                       N_s=N_s,
                       cal_half_window=cal_half_window,
                       N_s_extend=N_s_extend,
                       n_cores=n_cores,
                       n_group=n_group,
                       energy=energy,
                       p_x=p_x,
                       z=z,
                       wavelet_level_cut=wavelet_level_cut,
                       pyramid_level=pyramid_level,
                       n_iter=n_iter,
                       use_wavelet=use_wavelet)

    if not os.path.exists(Folder_result):
        os.makedirs(Folder_result)
    sample_transmission = img_data / ref_data
    plt.imsave(os.path.join(Folder_result, 'transmission.png'),
               sample_transmission)

    WXST_solver.run(result_path=Folder_result)

    displace = WXST_solver.displace
    DPC_x = WXST_solver.DPC[1]
    DPC_y = WXST_solver.DPC[0]
    phase = WXST_solver.phase
    result_filename = WXST_solver.result_filename

    plt.imsave(os.path.join(Folder_result, 'displace_x.png'), displace[1])
    plt.imsave(os.path.join(Folder_result, 'displace_y.png'), displace[0])
    plt.imsave(os.path.join(Folder_result, 'dpc_x.png'), DPC_x)
    plt.imsave(os.path.join(Folder_result, 'dpc_y.png'), DPC_y)
    plt.imsave(os.path.join(Folder_result, 'phase.png'), phase)

    plt.figure()
    plt.imshow(displace[0])
    cbar = plt.colorbar()
    cbar.set_label('[pixels]', rotation=90)
    plt.savefig(os.path.join(Folder_result, 'displace_y_colorbar.png'))
    plt.figure()
    plt.imshow(displace[1])
    cbar = plt.colorbar()
    cbar.set_label('[pixels]', rotation=90)
    plt.savefig(os.path.join(Folder_result, 'displace_x_colorbar.png'))
    plt.figure()
    plt.imshow(phase)
    cbar = plt.colorbar()
    cbar.set_label('[rad]', rotation=90)
    plt.savefig(os.path.join(Folder_result, 'phase_colorbar.png'))

    fig = plt.figure()
    ax1 = fig.add_subplot(111, projection='3d')
    XX, YY = np.meshgrid(
        np.arange(phase.shape[1]) * p_x * 1e6,
        np.arange(phase.shape[0]) * p_x * 1e6)
    ax1.plot_surface(XX, YY, phase, cmap=cm.get_cmap('hot'))
    ax1.set_xlabel('x [$\mu m$]')
    ax1.set_ylabel('y [$\mu m$]')
    ax1.set_zlabel('phase [rad]')
    plt.savefig(os.path.join(Folder_result, 'Phase_3d.png'))

    plt.close()

max_stars_repo_name: APS-XSD-OPT-Group/XSVT

max_issues_repo_name: APS-XSD-OPT-Group/XSVT

max_forks_repo_name: APS-XSD-OPT-Group/XSVT

reason: import numpy,import scipy