text:
"""
This module contains a class for estimating longitude, latitude, tilt and azimuth from an input signal. This code
accepts an input signal data in the form of a `solar-data-tools` `DataHandler` object, which is used to standardize
and pre-process the data. The provided class will then estimate  longitude, latitude, tilt and azimuth using the
 'estimate_longitude', 'estimate_latitude' and 'estimate_orientation' methods, respectively. Alternatively, all four
 parameters can be estimated at once using the 'estimate_all' method.
"""
# Standard Imports
import numpy as np
# Solar Data Tools Imports
from solardatatools.solar_noon import energy_com, avg_sunrise_sunset
# Module Imports
from pvsystemprofiler.utilities.equation_of_time import eot_da_rosa, eot_duffie
from solardatatools.algorithms import SunriseSunset
from pvsystemprofiler.algorithms.latitude.hours_daylight import calculate_hours_daylight
from pvsystemprofiler.utilities.hour_angle_equation import calculate_omega
from pvsystemprofiler.utilities.declination_equation import delta_cooper
from pvsystemprofiler.algorithms.angle_of_incidence.curve_fitting import run_curve_fit
from pvsystemprofiler.algorithms.performance_model_estimation import find_fit_costheta
from pvsystemprofiler.algorithms.angle_of_incidence.lambda_functions import select_function
from pvsystemprofiler.algorithms.angle_of_incidence.dynamic_value_functions import determine_keys
from pvsystemprofiler.algorithms.angle_of_incidence.dynamic_value_functions import select_init_values
from pvsystemprofiler.algorithms.tilt_azimuth.daytime_threshold_quantile import filter_data
from pvsystemprofiler.utilities.tools import random_initial_values
from pvsystemprofiler.algorithms.longitude.estimation import estimate_longitude
from pvsystemprofiler.algorithms.latitude.estimation import estimate_latitude

class ConfigurationEstimator():
    def __init__(self, data_handler, gmt_offset, day_selection_method='all', solar_noon_method='optimized_estimates',
                 daylight_method='optimized_estimates', data_matrix='filled', daytime_threshold=None):
        if not data_handler._ran_pipeline:
            data_handler.run_pipeline()
        self.data_handler = data_handler
        if data_matrix == 'raw':
            self.data_matrix = data_handler.raw_data_matrix
        elif data_matrix == 'filled':
            self.data_matrix = data_handler.filled_data_matrix
        # Parameters to be estimated
        self.longitude = None
        self.latitude = None
        self.tilt = None
        self.azimuth = None
        # Attributes used for all calculations
        self.gmt_offset = gmt_offset
        self.hours_daylight = None
        self.daily_meas = self.data_handler.filled_data_matrix.shape[0]
        self.daytime_threshold = None
        self.day_interval = None
        self.x1 = None
        self.x2 = None
        self.data_sampling = self.data_handler.data_sampling
        self.num_days = self.data_handler.num_days
        self.day_of_year = self.data_handler.day_index.dayofyear
        self.eot_duffie = eot_duffie(self.day_of_year)
        self.eot_da_rosa = eot_da_rosa(self.day_of_year)
        self.delta = delta_cooper(self.day_of_year, self.daily_meas)
        self.omega = None
        if day_selection_method == 'all':
            self.days = self.data_handler.daily_flags.no_errors
        elif day_selection_method == 'clear':
            self.days = self.days.daily_flags.clear
        elif day_selection_method == 'cloudy':
            self.days = self.data_handler.daily_flags.cloudy

        if solar_noon_method == 'optimized_estimates' or daylight_method == 'optimized_estimates':
            ss = SunriseSunset()

        if solar_noon_method == 'rise_set_average':
            self.solarnoon = avg_sunrise_sunset(self.data_matrix)
        elif solar_noon_method == 'energy_com':
            self.solarnoon = energy_com(self.data_matrix)
        if solar_noon_method == 'optimized_estimates':
            ss.run_optimizer(data=self.data_matrix)
            self.solarnoon = np.nanmean([ss.sunrise_estimates, ss.sunset_estimates], axis=0)

        if daylight_method in ('sunrise-sunset', 'sunrise sunset'):
            self.hours_daylight = calculate_hours_daylight(self.data_matrix, daytime_threshold)
        elif daylight_method == 'optimized_estimates':
            self.hours_daylight = ss.sunset_estimates - ss.sunrise_estimates

    def estimate_longitude(self, estimator='fit_l1', eot_calculation='duffie'):
        """
        :param estimator: 'calculated', 'fit_l1', 'fit_l2' or 'fit_huber'
        :param eot_calculation: 'rise_set_average', 'energy_com', 'optimized_estimates',
        or 'optimized_measurements'
        :return: None
        """
        if eot_calculation in ('duffie', 'd', 'duf'):
            eot = self.eot_duffie
        elif eot_calculation in ('da_rosa', 'dr', 'rosa'):
            eot = self.eot_da_rosa
        self.longitude = estimate_longitude(estimator, eot, self.solarnoon, self.days, self.gmt_offset)
        return

    def estimate_latitude(self):
        hours_daylight_filtered, delta_filtered = self._prepare_lat_input_data()
        self.latitude = estimate_latitude(hours_daylight_filtered, delta_filtered)
        return

    def _prepare_lat_input_data(self):
        if np.any(np.isnan(self.hours_daylight)):
            hours_mask = np.isnan(self.hours_daylight)
            full_mask = ~hours_mask & self.days
            hours_daylight_filtered = self.hours_daylight[full_mask]
            delta = self.delta[:, full_mask]
        else:
            hours_daylight_filtered = self.hours_daylight[self.days]
            delta = self.delta[:, self.days]
        return hours_daylight_filtered, delta

    # estimate tilt and azimuth with or without longitude and latitude input values
    def estimate_orientation(self, longitude=None, latitude=None, tilt=None, azimuth=None, day_interval=None, x1=0.9,
                             x2=0.9):
        """
        Estimates tilt and azimuth. The intended use is to estimate tilt and azimuth given longitude and latitude.
        However, the algorithm will estimate any of longitude, latitude, tilt and azimuth depending on the input values.
        If a parameter is not provided as an input value, it will be estimated. Any other use than the stated above as
        the intended use was found to yield inaccurate results.

        :param longitude: optional. Longitude value to be used in parameter estimation.
        :param latitude: optional. Latitude value to be used in parameter estimation.
        :param tilt: optional. Tilt value to be used in parameter estimation.
        :param azimuth: optional. Azimuth value to be used in parameter estimation.
        :param day_interval: 'all', 'clear' or 'cloudy'.
        :param x1: cvx parameter. Factor used in signal decomposition for estimation of daytime threshold.
        :param x2: Quantile of data used in estimation of daytime threshold.
        :return: None
        """

        if longitude is None:
            est_lon = ConfigurationEstimator(self.data_handler, self.gmt_offset)
            est_lon.estimate_longitude()
            self.longitude = est_lon.longitude
        else:
            self.longitude = longitude
        if latitude is None:
            est_lat = ConfigurationEstimator(self.data_handler, self.gmt_offset)
            est_lat.estimate_latitude()
            self.latitude = est_lat.latitude
        else:
            self.latitude = latitude

        self.tilt = tilt
        self.azimuth = azimuth
        self.day_interval = day_interval
        self.x1 = x1
        self.x2 = x2
        dh = self.data_handler
        self.data_matrix = dh.filled_data_matrix
        self.num_days = dh.num_days
        self.omega = calculate_omega(self.data_sampling, self.num_days, self.longitude, self.day_of_year,
                                     self.gmt_offset)

        self.tilt, self.azimuth = self._cal_orientation_helper()

    def estimate_all(self, day_interval=None, x1=0.9, x2=0.9):
        """
        Estimate latitude, longitude, tilt and azimuth all at once.
        :param day_interval: 'all', 'clear', 'cloudy'.
        :param x1: cvx parameter. Factor used in signal decomposition for estimation of daytime threshold.
        :param x2: Quantile of data used in estimation of daytime threshold.
        """

        self.tilt = None
        self.azimuth = None
        self.day_interval = day_interval
        self.x1 = x1
        self.x2 = x2
        dh = self.data_handler
        self.data_matrix = dh.filled_data_matrix
        self.days = dh.daily_flags.clear
        self.num_days = dh.num_days
        self.delta = delta_cooper(self.day_of_year, self.daily_meas)
        est_lon = ConfigurationEstimator(self.data_handler, self.gmt_offset)
        est_lon.estimate_longitude()
        self.longitude = est_lon.longitude
        est_lat = ConfigurationEstimator(self.data_handler, self.gmt_offset)
        est_lat.estimate_latitude()
        self.latitude = est_lat.latitude
        self.omega = calculate_omega(self.data_sampling, self.num_days, self.longitude, self.day_of_year,
                                     self.gmt_offset)

        self.tilt, self.azimuth = self._cal_orientation_helper()

    def _cal_orientation_helper(self):
        if self.day_interval is not None:
            day_range = (self.day_of_year > self.day_interval[0]) & (self.day_of_year < self.day_interval[1])
        else:
            day_range = np.ones(self.day_of_year.shape, dtype=bool)

        scale_factor_costheta, costheta_fit = find_fit_costheta(self.data_matrix, self.days)

        boolean_filter = filter_data(self.data_matrix, self.daytime_threshold, self.x1, self.x2)

        boolean_filter = boolean_filter * self.days * day_range

        delta_f = self.delta[boolean_filter]
        omega_f = self.omega[boolean_filter]
        if ~np.any(boolean_filter):
            print('No data made it through filters')

        lat_initial, tilt_initial, azim_initial = random_initial_values(1)

        func_customized, bounds = select_function(self.latitude, self.tilt, self.azimuth)
        dict_keys = determine_keys(latitude=self.latitude, tilt=self.tilt, azimuth=self.azimuth)

        init_values_dict = {'latitude': lat_initial[0], 'tilt': tilt_initial[0], 'azimuth': azim_initial[0]}
        init_values, ivr = select_init_values(init_values_dict, dict_keys)

        estimates = run_curve_fit(func=func_customized, keys=dict_keys, delta=delta_f, omega=omega_f,
                                  costheta=costheta_fit, boolean_filter=boolean_filter, init_values=init_values,
                                  fit_bounds=bounds)

        for i, estimate in enumerate(dict_keys):
            if estimate == 'latitude_estimate':
                lat_estimate = estimates[i]
            if estimate == 'tilt_estimate':
                tilt_estimate = estimates[i]
            if estimate == 'azimuth_estimate':
                azimuth_estimate = estimates[i]

        if 'tilt_estimate' not in dict_keys:
            tilt_estimate = None
        if 'azimuth_estimate' not in dict_keys:
            azimuth_estimate = None
        return tilt_estimate, azimuth_estimate

max_stars_repo_name: slacgismo/pv-system-profiler

max_issues_repo_name: slacgismo/pv-system-profiler

max_forks_repo_name: slacgismo/pv-system-profiler


text:
from __future__ import absolute_import
import numpy as np
import warnings
from pyti import catch_errors
from pyti.function_helper import fill_for_noncomputable_vals
from pyti.typical_price import typical_price
from pyti.money_flow import money_flow
from six.moves import range

def money_flow_index(close_data, high_data, low_data, volume, period):
    """
    Money Flow Index.

    Formula:
    MFI = 100 - (100 / (1 + PMF / NMF))
    """
    catch_errors.check_for_input_len_diff(
        close_data, high_data, low_data, volume
        )
    catch_errors.check_for_period_error(close_data, period)

    mf = money_flow(close_data, high_data, low_data, volume)
    tp = typical_price(close_data, high_data, low_data)

    flow = [tp[idx] > tp[idx-1] for idx in range(1, len(tp))]
    pf = [mf[idx] if flow[idx] else 0 for idx in range(0, len(flow))]
    nf = [mf[idx] if not flow[idx] else 0 for idx in range(0, len(flow))]

    pmf = [sum(pf[idx+1-period:idx+1]) for idx in range(period-1, len(pf))]
    nmf = [sum(nf[idx+1-period:idx+1]) for idx in range(period-1, len(nf))]

    # Dividing by 0 is not an issue, it turns the value into NaN which we would
    # want in that case
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category=RuntimeWarning)
        money_ratio = np.array(pmf) / np.array(nmf)

    mfi = 100 - (100 / (1 + money_ratio))

    mfi = fill_for_noncomputable_vals(close_data, mfi)

    return mfi

ext: py

reason: import numpy


text:
import warnings

import cupy

from cupy_backends.cuda.api import runtime
from cupyx.jit import _cuda_types
from cupyx.jit._internal_types import BuiltinFunc
from cupyx.jit._internal_types import Data
from cupyx.jit._internal_types import Constant
from cupyx.jit._internal_types import Range
from cupyx.jit import _compile

from functools import reduce

class RangeFunc(BuiltinFunc):

    def call(self, env, *args, **kwargs):
        if len(args) == 0:
            raise TypeError('range expected at least 1 argument, got 0')
        elif len(args) == 1:
            start, stop, step = Constant(0), args[0], Constant(1)
        elif len(args) == 2:
            start, stop, step = args[0], args[1], Constant(1)
        elif len(args) == 3:
            start, stop, step = args
        else:
            raise TypeError(
                f'range expected at most 3 argument, got {len(args)}')

        if isinstance(step, Constant):
            step_is_positive = step.obj >= 0
        elif step.ctype.dtype.kind == 'u':
            step_is_positive = True
        else:
            step_is_positive = None

        stop = Data.init(stop, env)
        start = Data.init(start, env)
        step = Data.init(step, env)

        if start.ctype.dtype.kind not in 'iu':
            raise TypeError('range supports only for integer type.')
        if stop.ctype.dtype.kind not in 'iu':
            raise TypeError('range supports only for integer type.')
        if step.ctype.dtype.kind not in 'iu':
            raise TypeError('range supports only for integer type.')

        if env.mode == 'numpy':
            ctype = _cuda_types.Scalar(int)
        elif env.mode == 'cuda':
            ctype = stop.ctype
        else:
            assert False

        return Range(start, stop, step, ctype, step_is_positive)

class LenFunc(BuiltinFunc):
    def call(self, env, *args, **kwds):
        if len(args) != 1:
            raise TypeError(f'len() expects only 1 argument, got {len(args)}')
        if kwds:
            raise TypeError('keyword arguments are not supported')
        arg = args[0]
        if not isinstance(arg.ctype, _cuda_types.CArray):
            raise TypeError('len() supports only array type')
        if not arg.ctype.ndim:
            raise TypeError('len() of unsized array')
        return Data(f'static_cast<long long>({arg.code}.shape()[0])',
                    _cuda_types.Scalar('q'))

class MinFunc(BuiltinFunc):
    def call(self, env, *args, **kwds):
        if len(args) < 2:
            raise TypeError(
                f'min() expects at least 2 arguments, got {len(args)}')
        if kwds:
            raise TypeError('keyword arguments are not supported')
        return reduce(lambda a, b: _compile._call_ufunc(
            cupy.minimum, (a, b), None, env), args)

class MaxFunc(BuiltinFunc):
    def call(self, env, *args, **kwds):
        if len(args) < 2:
            raise TypeError(
                f'max() expects at least 2 arguments, got {len(args)}')
        if kwds:
            raise TypeError('keyword arguments are not supported')
        return reduce(lambda a, b: _compile._call_ufunc(
            cupy.maximum, (a, b), None, env), args)

class SyncThreads(BuiltinFunc):

    def __call__(self):
        """Calls ``__syncthreads()``.

        .. seealso:: `Synchronization functions`_

        .. _Synchronization functions:
            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#synchronization-functions
        """
        super.__call__(self)

    def call_const(self, env):
        return Data('__syncthreads()', _cuda_types.void)

class SyncWarp(BuiltinFunc):

    def __call__(self, *, mask=0xffffffff):
        """Calls ``__syncwarp()``.

        Args:
            mask (int): Active threads in a warp. Default is 0xffffffff.

        .. seealso:: `Synchronization functions`_

        .. _Synchronization functions:
            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#synchronization-functions
        """
        super().__call__()

    def call(self, env, *, mask=None):
        if runtime.is_hip:
            warnings.warn(f'mask {mask} is ignored on HIP', RuntimeWarning)
            mask = None

        if mask:
            if isinstance(mask, Constant):
                if not (0x0 <= mask.obj <= 0xffffffff):
                    raise ValueError('mask is out of range')
            mask = _compile._astype_scalar(
                mask, _cuda_types.int32, 'same_kind', env)
            mask = Data.init(mask, env)
            code = f'__syncwarp({mask.code})'
        else:
            code = '__syncwarp()'
        return Data(code, _cuda_types.void)

class SharedMemory(BuiltinFunc):

    def __call__(self, dtype, size):
        """Allocates shared memory and returns the 1-dim array.

        Args:
            dtype (dtype):
                The dtype of the returned array.
            size (int or None):
                If ``int`` type, the size of static shared memory.
                If ``None``, declares the shared memory with extern specifier.
        """
        super.__call__(self)

    def call_const(self, env, dtype, size):
        name = env.get_fresh_variable_name(prefix='_smem')
        child_type = _cuda_types.Scalar(dtype)
        while env[name] is not None:
            name = env.get_fresh_variable_name(prefix='_smem')  # retry
        env[name] = Data(name, _cuda_types.SharedMem(child_type, size))
        return Data(name, _cuda_types.Ptr(child_type))

class AtomicOp(BuiltinFunc):

    def __init__(self, op, dtypes):
        self._op = op
        self._name = 'atomic' + op
        self._dtypes = dtypes
        super().__init__()

    def call(self, env, array, index, value):
        array = Data.init(array, env)
        if not isinstance(array.ctype, (_cuda_types.CArray, _cuda_types.Ptr)):
            raise TypeError('The first argument must be of array type.')
        target = _compile._indexing(array, index, env)
        ctype = target.ctype
        value = _compile._astype_scalar(value, ctype, 'same_kind', env)
        name = self._name
        value = Data.init(value, env)
        if ctype.dtype.char not in self._dtypes:
            raise TypeError(f'`{name}` does not support {ctype.dtype} input.')
        if ctype.dtype.char == 'e' and runtime.runtimeGetVersion() < 10000:
            raise RuntimeError(
                'float16 atomic operation is not supported this CUDA version.')
        return Data(f'{name}(&{target.code}, {value.code})', ctype)

class Grid(BuiltinFunc):

    def __call__(self, ndim):
        """Compute the thread index in the grid.

        Computation of the first integer is as follows::

            jit.threadIdx.x + jit.blockIdx.x * jit.blockDim.x

        and for the other two integers the ``y`` and ``z`` attributes are used.

        Args:
            ndim (int): The dimension of the grid. Only 1, 2, or 3 is allowed.

        Returns:
            int or tuple:
                If ``ndim`` is 1, an integer is returned, otherwise a tuple.

        .. note::
            This function follows the convention of Numba's `numba.cuda.grid`_.

        .. _numba.cuda.grid:
            https://numba.readthedocs.io/en/stable/cuda/kernels.html#absolute-positions

        """
        super.__call__(self)

    def call_const(self, env, ndim):
        if not isinstance(ndim, int):
            raise TypeError('ndim must be an integer')

        # Numba convention: for 1D we return a single variable,
        # otherwise a tuple
        code = 'threadIdx.{n} + blockIdx.{n} * blockDim.{n}'
        if ndim == 1:
            return Data(code.format(n='x'), _cuda_types.uint32)
        elif ndim == 2:
            dims = ('x', 'y')
        elif ndim == 3:
            dims = ('x', 'y', 'z')
        else:
            raise ValueError('Only ndim=1,2,3 are supported')

        elts_code = ', '.join(code.format(n=n) for n in dims)
        ctype = _cuda_types.Tuple([_cuda_types.uint32]*ndim)
        return Data(f'thrust::make_tuple({elts_code})', ctype)

class WarpShuffleOp(BuiltinFunc):

    def __init__(self, op, dtypes):
        self._op = op
        self._name = '__shfl_' + (op + '_' if op else '') + 'sync'
        self._dtypes = dtypes
        doc = f"""Call the {self._name} function. Please refer to
        `Warp Shuffle Functions`_ for detail explanation.

        .. _Warp Shuffle Functions:
            https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#warp-shuffle-functions
        """
        self.__doc__ = doc

    def __call__(self, mask, var, val_id, *, width=32):
        super().__call__()

    def call(self, env, mask, var, val_id, *, width=None):
        name = self._name

        var = Data.init(var, env)
        ctype = var.ctype
        if ctype.dtype.char not in self._dtypes:
            raise TypeError(f'`{name}` does not support {ctype.dtype} input.')

        try:
            mask = mask.obj
        except Exception:
            raise TypeError('mask must be an integer')
        if runtime.is_hip:
            warnings.warn(f'mask {mask} is ignored on HIP', RuntimeWarning)
        elif not (0x0 <= mask <= 0xffffffff):
            raise ValueError('mask is out of range')

        # val_id refers to "delta" for shfl_{up, down}, "srcLane" for shfl, and
        # "laneMask" for shfl_xor
        if self._op in ('up', 'down'):
            val_id_t = _cuda_types.uint32
        else:
            val_id_t = _cuda_types.int32
        val_id = _compile._astype_scalar(val_id, val_id_t, 'same_kind', env)
        val_id = Data.init(val_id, env)

        if width:
            if isinstance(width, Constant):
                if width.obj not in (2, 4, 8, 16, 32):
                    raise ValueError('width needs to be power of 2')
        else:
            width = Constant(64) if runtime.is_hip else Constant(32)
        width = _compile._astype_scalar(
            width, _cuda_types.int32, 'same_kind', env)
        width = Data.init(width, env)

        code = f'{name}({hex(mask)}, {var.code}, {val_id.code}'
        code += f', {width.code})'
        return Data(code, ctype)

builtin_functions_dict = {
    range: RangeFunc(),
    len: LenFunc(),
    min: MinFunc(),
    max: MaxFunc(),
}

syncthreads = SyncThreads()
syncwarp = SyncWarp()
shared_memory = SharedMemory()
grid = Grid()

# TODO: Add more atomic functions.
atomic_add = AtomicOp('Add', 'iILQefd')

# warp-shuffle functions
shfl_sync = WarpShuffleOp('', 'iIlqfd' if runtime.is_hip else 'iIlLqQefd')
shfl_up_sync = WarpShuffleOp('up', 'iIlqfd' if runtime.is_hip else 'iIlLqQefd')
shfl_down_sync = WarpShuffleOp('down',
                               'iIlqfd' if runtime.is_hip else 'iIlLqQefd')
shfl_xor_sync = WarpShuffleOp('xor',
                              'iIlqfd' if runtime.is_hip else 'iIlLqQefd')

max_stars_repo_name: eternalphane/cupy

max_issues_repo_name: eternalphane/cupy

max_forks_repo_name: eternalphane/cupy

reason: import cupy,from cupy


text:
import numpy as np
import torch
import os

MY_DATASET_PATH = "/home/apoorva/TaxiNet-Release-Five-Without-Trained-Models/ProcessedData"
DATASET_PATH = "/home/apoorva/datasets/taxinet"

class MyTaxiNet(torch.utils.data.TensorDataset):
    def __init__(self, split='train'):
        folder = os.path.join(DATASET_PATH,  split)
        if not os.path.exists(folder):
            print(folder + " does not exist, check split argument")
            raise AttributeError

        X = np.load(os.path.join(folder, "X.npy"), allow_pickle=True)
        y = np.load(os.path.join(folder, "Y.npy"), allow_pickle=True)[:,0]
        # Y.npy has 2 dim entries that correspond to the CTE and the heading error
        # we train the model to just output CTE

        idx = np.nonzero( [x is not None for x in X] )
        x_filt = np.stack(X[idx])*1./256
        y_filt = np.stack(y[idx])[:,None]

        x_t = torch.from_numpy(x_filt).float().permute(0,3,1,2)
        y_t = torch.from_numpy(y_filt).float()

        super().__init__(x_t, y_t)

class TaxiNetData(torch.utils.data.Dataset):
    def __init__(self, split='train',N=1000, shuffle=True):
        N = N
        idx = None
        if split == 'train':
            split = 'exp1_train'
            start_idx = 0
            end_idx = 30000
            idx = np.arange(start_idx, end_idx)
        elif split == 'val':
            split = 'exp1_train'
            start_idx = 30000
            end_idx = 37051
            idx = np.arange(start_idx, end_idx)

        folder = os.path.join(DATASET_PATH,  split)
        if not os.path.exists(folder):
            print(folder + " does not exist, check split argument")
            raise AttributeError

        y = np.load(os.path.join(folder, "Y.npy"))
        n = y.shape[0]

        if idx is None:
            if shuffle:
                idx = np.random.choice(n, N, replace=False)
            else:
                idx = range(N)

        self.X = np.load(os.path.join(folder, "X.npy"))[idx]
        self.y = y[idx,0]

    def __len__(self):
        return self.X.shape[0]

    def __getitem__(self, idx):
        x = torch.from_numpy(self.X[idx,:,:,:]*1./256).float().permute(2,0,1)
        y = torch.from_numpy(self.y[idx,None]).float()

        return x,y

class TaxiNetFull(torch.utils.data.Dataset):
    def __init__(self, split='train',N=1000, shuffle=True):
        N = N
        idx = None
        if split == 'train':
            split = 'exp1_train'
            start_idx = 0
            end_idx = 30000
            idx = np.arange(start_idx, end_idx)
        elif split == 'val':
            split = 'exp1_train'
            start_idx = 30000
            end_idx = 37051
            idx = np.arange(start_idx, end_idx)

        folder = os.path.join(DATASET_PATH,  split)
        if not os.path.exists(folder):
            print(folder + " does not exist, check split argument")
            raise AttributeError

        y = np.load(os.path.join(folder, "Y.npy"))
        n = y.shape[0]

        if idx is None:
            if shuffle:
                idx = np.random.choice(n, N, replace=False)
            else:
                idx = range(N)

        self.X = np.load(os.path.join(folder, "X.npy"))[idx]
        self.y = y[idx]

    def __len__(self):
        return self.X.shape[0]

    def __getitem__(self, idx):
        x = torch.from_numpy(self.X[idx,:,:,:]*1./256).float().permute(2,0,1)
        y = torch.from_numpy(self.y[idx,:]).float()

        return x,y

max_stars_repo_name: apoorva-sharma/SCOD

max_issues_repo_name: apoorva-sharma/SCOD

max_forks_repo_name: apoorva-sharma/SCOD


text:
import numpy as np
import matplotlib.pyplot as plot

def fill_array_of_results(filename):
    yarray = [0];
    i = 1;
    tmp_file = open(filename, "r");
    for line in tmp_file:
        yarray.append(float(line));
        i += 1;
    tmp_file.close();
    return np.array(yarray);

def draw_graphic(xarray, yarrays, message):
    figure = plot.figure();
    plot.plot(xarray, yarrays[0], label = 'Сортировка минимум/максимум');
    plot.plot(xarray, yarrays[1], label = 'Сортировка пузырьком');
    plot.plot(xarray, yarrays[2], label = 'Сортировка шейкером');
    plot.legend();
    plot.title('Скорость работы алгоритма ' + message);
    plot.ylabel('Процессорное время, тики');
    plot.xlabel('Длина массива, количество элементов');
    plot.grid(True);
    plot.show();

if __name__ == "__main__":
    yarrays = list();
    yarrays.append(fill_array_of_results("calculations/sort_min_max_random.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_bubble_random.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_shaker_random.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_min_max_sorted.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_bubble_sorted.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_shaker_sorted.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_min_max_unsorted.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_bubble_unsorted.txt"));
    yarrays.append(fill_array_of_results("calculations/sort_shaker_unsorted.txt"));
    xarray = [0];
    for i in range(len(yarrays[0]) - 1):
        xarray.append(i + 1);
    draw_graphic(np.array(xarray), yarrays, '(случайный массив)');
    draw_graphic(np.array(xarray), yarrays[3:], '(отсортированный по возрастанию массив)');
    draw_graphic(np.array(xarray), yarrays[6:], '(отсортированный по убыванию массив)');

max_stars_repo_name: rkolyan/Algorithm-s-Analyses

max_issues_repo_name: rkolyan/Algorithm-s-Analyses

max_forks_repo_name: rkolyan/Algorithm-s-Analyses


text:
"""
Construct event function from a DD experiment.
"""

import numpy as np

from .velocity import VelocityFunction
from .angle_average import angle_average, v_earth_frame
from .xenon import XENON1T
from .memoize import LookUpTable

DEFAULT_SHIFT_TO_EARTH = 235.  # km/s
SPEED_OF_LIGHT = 3.E5  # km/s
KM_OVER_CM = 1E5
M_PROTON = 938.27231E-3  # GeV

class EventsAtVelocity(VelocityFunction):
    """
    Expected number of signal events as a function of velocity in the galactic
    frame.
    """
    SIGMA = 1E-46  # cm^2
    RHO = 0.3  # GeV/cm^2

    def __init__(self, mass, experiment=None, shift_to_earth=None):
        """
        @param mass Mass of DM in GeV
        @param shift_to_earth Shift to earth frame in km/s
        """
        self.mass = mass
        self.experiment = experiment or XENON1T
        self.shift_to_earth = shift_to_earth or DEFAULT_SHIFT_TO_EARTH
        # Velocity and component independent factor in unit GeV * km/s
        self.factor = (0.5 * KM_OVER_CM * SPEED_OF_LIGHT**2 *
                       self.experiment.exposure * self.RHO * self.SIGMA
                       / (self.mass * self._reduced_mass(M_PROTON)**2))

    def _reduced_mass(self, mass):
        """
        @returns Reduced mass of DM particle and other in GeV
        """
        return mass * self.mass / (mass + self.mass)

    def _max_energy(self, velocity, mass):
        """
        @param mass Mass of component of detector
        @returns Maximum energy from kinematics in GeV
        """
        return 2. * (self._reduced_mass(mass) * velocity
                     / SPEED_OF_LIGHT)**2 / mass

    @LookUpTable
    def events_earth_frame(self, velocity):
        """
        @returns Expected number of events as function of velocity in earth
        frame
        """
        max_energies = np.array([self._max_energy(velocity, c.mass)
                                 for c in self.experiment.components])
        sum_ = sum(c.contribution(e)
                   for e, c in zip(max_energies, self.experiment.components))
        return sum_ * self.factor / velocity

    @LookUpTable
    def events_galactic_frame(self, velocity, cos_theta):
        """
        @returns Expected number of events as function of velocity in galactic
        frame
        """
        v_earth_frame_ = v_earth_frame(velocity, cos_theta, self.shift_to_earth)
        return self.events_earth_frame(v_earth_frame_)

    @LookUpTable
    def averaged_events_galactic_frame(self, velocity):
        """
        @returns Expected number of events as function of velocity in galactic
        frame, angle-averaged
        """
        angle_averaged = angle_average(self.events_galactic_frame)
        return angle_averaged(velocity)

    def __call__(self, velocity, cos_theta=None):
        if cos_theta is None:
            return self.averaged_events_galactic_frame(velocity)
        return self.events_galactic_frame(velocity, cos_theta)

if __name__ == "__main__":

    MASS = 60.
    E = EventsAtVelocity(MASS)
    E.plot()

max_stars_repo_name: andrewfowlie/veltropy

max_issues_repo_name: andrewfowlie/veltropy

max_forks_repo_name: andrewfowlie/veltropy


text:
import os
import matplotlib.image as mpimg
import random
from flask import Flask, render_template, request, flash, redirect, jsonify, send_file
from flask_bootstrap import Bootstrap
import networkx as nx
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pandas as pd
from werkzeug.utils import secure_filename
from database import *
import randomNetCreation as randomNet
import fileNetCreation as fileNet
from function import *

app = Flask(__name__)
bootstrap = Bootstrap(app)

schyear_class = nx.Graph()
matrix_siblings = []

@app.route('/')
def index():
    title = "Sire"
    user = {'nombre': 'Maria'}

    return render_template('plantillabase.html', title=title, user=user)

@app.route('/login')
def login():
    return render_template('login.html')

@app.route('/aboutapp')
def aboutapp():
    return render_template('aboutapp.html')

@app.route('/login/LogInDB', methods=['POST', 'GET'])
def logInDB():
	exist = logIn()
	if exist:
		return render_template('user_home.html')
	return jsonify({'message': 'El usuario especificado no existe, compruebe su mail y contrasena'})

@app.route('/showSignUp')
def showSignUp():
    return render_template('register.html')

@app.route('/showSingup/register', methods=['POST','GET'])
def register():
	user_created = signUp()
	if user_created == json.dumps({'message':'User created successfully !'}):
		return render_template('login.html')
	return jsonify({'NOT': x})

@app.route('/user_home')
def show_user_home():
	return render_template('user_home.html')

@app.route('/selection')
def show_selection_page():
	return render_template('selection.html')

@app.route('/upload')
def upload_form():
    return render_template('upload.html')

@app.route('/upload', methods=['POST'])
def upload_file():
	path = os.getcwd()
	UPLOAD_FOLDER = os.path.join(path, 'uploads')
	global schyear_class
	global matrix_siblings

	if not os.path.isdir(UPLOAD_FOLDER):
		os.mkdir(UPLOAD_FOLDER)

	app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

	ALLOWED_EXTENSIONS = set(['gexf', 'csv'])

	def allowed_file(filename):
		return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

	if request.method == 'POST':
		if 'files[]' not in request.files:
			flash('No file part')
			return redirect(request.url)

		files = request.files.getlist('files[]')

		if len(files) != 2:
			flash('Debe introducir dos archivos.')
			return render_template('upload.html')

		name1 = files[0].filename
		extension1 = files[0].filename.rsplit('.', 1)[1].lower()
		name2 = files[1].filename
		extension2 = files[1].filename.rsplit('.', 1)[1].lower()

		if (extension1 == 'gexf' and extension2 == 'csv') or (extension2 == 'gexf' and extension1 == 'csv'):
			for file in files:
				if file and allowed_file(file.filename):
					filename = secure_filename(file.filename)
					file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
				else:
					flash('Los archivos permitidos son .gexf y .csv')
					return render_template('upload.html')
		else:
			flash('Debe introducir un .csv y un grafo.')
			return render_template('upload.html')

		file1 = addFile(name1, os.path.join(app.config['UPLOAD_FOLDER'], name1))
		file2 = addFile(name2, os.path.join(app.config['UPLOAD_FOLDER'], name2))

		if file1 == json.dumps({'message':'File added successfully!'}) and file2 == json.dumps({'message':'File added successfully!'}):
			file1_name, file1_directory = get_last_file()
			file2_name, file2_directory = get_penultimate_file()

			n1, ext1 = os.path.splitext(file1_name)
			n2, ext2 = os.path.splitext(file2_name)

			if ext1 == '.gexf':
				G = nx.read_gexf(file1_directory)
			elif ext1 == '.csv':
				df = pd.read_csv(file1_directory, index_col=0)
			if ext2 == '.gexf':
				G = nx.read_gexf(file2_directory)
			elif ext2 == '.csv':
				df = pd.read_csv(file2_directory, index_col=0)

			schoolyear_class = fileNet.create_schoolyear_class_network(df, G)
			schyear_class = schoolyear_class
			matrix_siblings = df.values

			_totalStudents = len(G.nodes())
			_numberSiblings = len(df)

			addNet(_totalStudents, _numberSiblings)

		return render_template('random_advanced.html')

@app.route("/downloadfile", methods=['GET'])
def show_download():
	return render_template('downloads.html')

@app.route("/download")
def download():
	path = os.getcwd()
	UPLOAD_FOLDER = os.path.join(path, 'uploads')
	app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
	p = "/randomGraphuploaded.gexf"
	return send_file(app.config['UPLOAD_FOLDER']+ p, as_attachment=True)

@app.route('/showData')
def show_introduce_data():
    return render_template('introduce_data.html')

@app.route('/showData/data', methods=['POST', 'GET'])
def addNetwork():
	global schyear_class
	global matrix_siblings

	clean_variables()

	_totalStudents = int(request.form['totalStudents'])
	_numberSiblings = int(request.form['numberSiblings'])

	addNet(_totalStudents, _numberSiblings)

	randomNet.create_initial_network(_totalStudents,_numberSiblings)
	schoolyear_class = randomNet.create_schoolyear_class_network()
	schyear_class = schoolyear_class
	pos=nx.kamada_kawai_layout(schyear_class)
	nx.draw(schyear_class, pos)
	node_labels = nx.get_node_attributes(schyear_class,'Nombre')
	nx.draw_networkx_labels(schyear_class, pos, labels = node_labels)

	plt.savefig('Net_images/schyear_class_initial.png')
	plt.close('all')

	randomNet.create_siblings_matrix()
	matrix_siblings = randomNet.__siblingsMatrix

	return render_template('random_advanced.html')

def clean_variables():
	global schyear_class
	global matrix_siblings
	randomNet.__initial_network = nx.Graph()
	randomNet.__schoolyear_class = nx.Graph()
	randomNet.__siblings = []
	randomNet.__siblingsMatrix = []
	schyear_class = nx.Graph()
	matrix_siblings = []

@app.route('/showSelection')
def show_random_advanced():
    return render_template('random_advanced.html')

@app.route('/showSelection/randomAdvanced', methods=['POST', 'GET'])
def pick_option():
	selected_option = None
	selected_option = request.form.get('options')
	values = []

	if selected_option == 'default':
		tf = random.uniform(0.05, 0.01)
		alpha = random.uniform(0.8, 0.99)
		l = random.randint(10,50)
		option_cooling = 6
		seed_value = random.randint(0,10)
		percentage_component = 60

		cooling_sequence = change_number_to_cooling_sequence(option_cooling)

		net_id, totalStudents, numberSiblings = returnNet()

		initial_t, initial_neighbour, ini_fmax, best_neighbour, current_fmax, G = solve_simulated_annealing(schyear_class,matrix_siblings,numberSiblings,totalStudents,l,tf, alpha ,int(option_cooling), seed_value, percentage_component)

		pos=nx.kamada_kawai_layout(G)
		nx.draw(G, pos)
		node_labels = nx.get_node_attributes(G,'Nombre')
		nx.draw_networkx_labels(G, pos, labels = node_labels)

		plt.savefig('Net_images/schyear_class_default_processed.png')
		plt.close('all')

		introduce_net_data(net_id, l, initial_t, tf, alpha, cooling_sequence, seed_value)

		diff_ini = list(set(initial_neighbour) - set(best_neighbour))
		diff_proc = list(set(best_neighbour) - set(initial_neighbour))

		return render_template('results.html', initial_neighbour = initial_neighbour, ini_fmax = ini_fmax, best_neighbour = best_neighbour, current_fmax = current_fmax, diff_ini = diff_ini, diff_proc = diff_proc)
	elif selected_option == 'advanced':
		return render_template('advanced.html')

	if selected_option == None:
		flash('Debe seleccionar una opción')
		return render_template('random_advanced.html')

@app.route('/advancedOptions', methods=['POST', 'GET'])
def advanced_option():
	alpha = 0
	seed_value = random.randint(0,10)

	_l = request.form['l']
	_tf = request.form['tf']
	_coolingSeq = request.form['cooling_sequence']
	_percentage = request.form['percentage_component']

	l = int(_l)
	tf = float(_tf)
	option_cooling = int(_coolingSeq)
	percentage_component = int(_percentage)

	if l < 0:
		flash('L debe ser entero positivo')
		return render_template('advanced.html')

	if tf == 0:
		flash('tf no puede ser 0')
		return render_template('advanced.html')

	if percentage_component > 100:
		flash('El porcentaje no puede ser mayor de 100')
		return render_template('advanced.html')

	if option_cooling == 3:
		alpha = 20
	elif option_cooling == 6:
		alpha = random.uniform(0.8, 0.99)

	cooling_sequence = change_number_to_cooling_sequence(option_cooling)
	net_id, totalStudents, numberSiblings = returnNet()

	initial_t, initial_neighbour, ini_fmax, best_neighbour, current_fmax, G = solve_simulated_annealing(schyear_class,matrix_siblings,numberSiblings,totalStudents,l,tf, alpha ,int(option_cooling), seed_value, percentage_component)

	introduce_net_data(net_id, l, initial_t, tf, alpha, cooling_sequence, seed_value)

	pos=nx.kamada_kawai_layout(G)
	nx.draw(G, pos)
	node_labels = nx.get_node_attributes(G,'Nombre')
	nx.draw_networkx_labels(G, pos, labels = node_labels)

	plt.savefig('Net_images/schyear_class_advanced_processed.png')
	plt.close('all')

	diff_ini = list(set(initial_neighbour) - set(best_neighbour))
	diff_proc = list(set(best_neighbour) - set(initial_neighbour))

	return render_template('results.html', initial_neighbour = initial_neighbour, ini_fmax = ini_fmax, best_neighbour = best_neighbour, current_fmax = current_fmax, diff_ini=diff_ini, diff_proc=diff_proc)

def change_number_to_cooling_sequence(number):
	if number == 1:
		cooling_sequence = 'linear_cooling_sequence'
	elif number == 3:
		cooling_sequence = 'logarithmic_cooling_sequence'
	elif number == 4:
		cooling_sequence = 'cauchy_cooling_sequence'
	elif number == 5:
		cooling_sequence = 'modified_cauchy_cooling_sequence'
	elif number == 6:
		cooling_sequence = 'geometrical_cooling_sequence'

	return cooling_sequence

@app.route('/show_previous')
def show_previous():
	return render_template('show_previous.html')

@app.route('/show_examples')
def show_exaples():
	return render_template('examples.html')

@app.route('/logout')
def logout():
	session.clear()
	return render_template('login.html')

if __name__ == '__main__':
	app.secret_key = os.urandom(24)
	app.run()

max_stars_repo_name: Mariaojruiz/Sibling-Rewiring

max_issues_repo_name: Mariaojruiz/Sibling-Rewiring

max_forks_repo_name: Mariaojruiz/Sibling-Rewiring

reason: import networkx


text:
#!/usr/bin/env python3
from obspy.core import read
from numpy import sqrt
t1_hamming = read('t100.hamming.taper.sac')
t1_hanning = read('t100.hanning.taper.sac')
t1_cosine  = read('t100.cosine.taper.sac')

t2_hamming = read('t101.hamming.taper.sac')
t2_hanning = read('t101.hanning.taper.sac')
t2_cosine  = read('t101.cosine.taper.sac')

cline1 = ''
for i in range(len(t1_hamming[0].data)):
    cline1 = cline1 + '%.8lf, %.8lf, %.8lf\n'%(t1_hamming[0].data[i], t1_hanning[0].data[i], pow(t1_cosine[0].data[i],1))
print(cline1)
of = open('../taper100.all.txt', 'w')
of.write(cline1)
of.close()

cline2 = ''
for i in range(len(t2_hamming[0].data)):
    cline2 = cline2 + '%.8lf, %.8lf, %.8lf\n'%(t2_hamming[0].data[i], t2_hanning[0].data[i], pow(t2_cosine[0].data[i],1))
print(cline2)
of = open('../taper101.all.txt', 'w')
of.write(cline2)
of.close()

"""
of = open('../taper100.hanning.txt', 'w')
of.write(cline1)
of.close()

cline2 = ''
for d in t2[0].data:
    cline2 = cline2 + '%lf\n'%d
print(cline2)
of = open('../taper101.hamming.txt', 'w')
of.write(cline2)
of.close()
"""

max_issues_repo_name: uofuseismo/rtseis

reason: from numpy


text:
import numpy as np
from pegasusio import MultimodalData
from typing import List, Dict, Union

import logging
logger = logging.getLogger(__name__)

def parse_subset_selections(subset_selections: List[str]) -> Dict[str, List[str]]:
    """ Take a list of selection criteria strings and return a parsed selection dictionary.

    Parameters
    ----------
    subset_selections: ``List[str]``
        A list of subset selection criteria. Each criterion takes the format of 'attribute:value,value,...,value'.

    Returns
    -------
    ``Dict[str, List[str]]``
    A dictionary of subsetting criteria
    """
    subsets_dict = {}
    for subset_str in subset_selections:
        attr, value_str = subset_str.split(":")
        if attr in subsets_dict:
            subsets_dict[attr].extend(value_str.split(","))
        else:
            subsets_dict[attr] = value_str.split(",")
    return subsets_dict

def clone_subset(data: MultimodalData, subset_selections: Union[List[bool], Union[str, List[str]]]) -> MultimodalData:
    """ Clone a subset of data as a new MultimodalData object for subclustering purpose.

    Parameters
    ----------
    data: ``pegasusio.MultimodalData``
        Annotated data matrix with rows for cells and columns for genes.
    subset_selections: ``List[bool]`` or ``str`` or ``List[str]``
        This parameter can take three forms. First, it can be an array/list of boolean values (List[bool]) indicating which cells are selected. Second, it can be a string with the format of 'attribute:value,value,...,value'. In this case, Pegasus will search for 'attribute' in the 'obs' field and select all cells with 'attribute' in the specified values. Lastly, it can be a list of strings as criteria and Pegasus will only select cells that satisfy all criteria (AND logic).

    Returns
    -------
    ``pegasusio.MultimodalData``
        Return a new subsetted data
    """
    if isinstance(subset_selections, str):
        subset_selections = [subset_selections]

    if len(subset_selections) == 0:
        raise ValueError(f"No entry in subset_selections!")

    val = subset_selections[0]
    if isinstance(val, bool) or isinstance(val, np.bool_): # if bool list
        obs_index = subset_selections
    else:
        obs_index = np.full(data.shape[0], True)
        subsets_dict = parse_subset_selections(subset_selections)
        for key, value in subsets_dict.items():
            logger.info(f"Select {key} in {str(value)}")
            obs_index = obs_index & np.isin(data.obs[key], value)

    newdata = data[obs_index, :].copy()
    # Add _par suffix to clustering labels
    mapper = {}
    for attr in newdata.obs.columns:
        if newdata.get_attr_type(attr) == "cluster":
            new_attr = f"{attr}_par"
            newdata.register_attr(attr)
            newdata.register_attr(new_attr, "cluster")
            mapper[attr] = new_attr
    if len(mapper) > 0:
        newdata.obs.rename(columns = mapper, inplace = True)
    # Add _par suffix to basis and also drop knn
    for attr in list(newdata.obsm.keys()):
        attr_type = newdata.get_attr_type(attr)
        if attr_type == "basis":
            new_attr = f"{attr}_par"
            newdata.register_attr(attr)
            newdata.register_attr(new_attr, "basis")
            newdata.obsm[new_attr] = newdata.obsm.pop(attr)
        elif attr_type == "knn":
            newdata.register_attr(attr) # pop up
            del newdata.obsm[attr]
    # For var, update 'n_cells', mark genes with n_cells == 0 as unrobust and make hvg same as robust by default
    newdata.var["n_cells"] = newdata.X.getnnz(axis=0)
    values = newdata.var["robust"].values if ("robust" in newdata.var) else np.full(newdata.shape[1], True)
    values &= (newdata.var["n_cells"] > 0).values
    newdata.var["robust"] = values
    newdata.var["highly_variable_features"] = newdata.var["robust"]  # default all robust genes are "highly" variable

    logger.info(f"{newdata.shape[0]} cells are selected.")

    return newdata

max_stars_repo_name: klarman-cell-observatory/pegasus

max_issues_repo_name: klarman-cell-observatory/pegasus

max_forks_repo_name: klarman-cell-observatory/pegasus


text:
#!/usr/bin/env python
# coding: utf-8

# # CCKS 2020: 基于本体的金融知识图谱自动化构建技术评测
#
# 竞赛背景
# 金融研报是各类金融研究结构对宏观经济、金融、行业、产业链以及公司的研究报告。报告通常是有专业人员撰写，对宏观、行业和公司的数据信息搜集全面、研究深入，质量高，内容可靠。报告内容往往包含产业、经济、金融、政策、社会等多领域的数据与知识，是构建行业知识图谱非常关键的数据来源。另一方面，由于研报本身所容纳的数据与知识涉及面广泛，专业知识众多，不同的研究结构和专业认识对相同的内容的表达方式也会略有差异。这些特点导致了从研报自动化构建知识图谱困难重重，解决这些问题则能够极大促进自动化构建知识图谱方面的技术进步。
#
# 本评测任务参考 TAC KBP 中的 Cold Start 评测任务的方案，围绕金融研报知识图谱的自动化图谱构建所展开。评测从预定义图谱模式（Schema）和少量的种子知识图谱开始，从非结构化的文本数据中构建知识图谱。其中图谱模式包括 10 种实体类型，如机构、产品、业务、风险等；19 个实体间的关系，如(机构，生产销售，产品)、(机构，投资，机构)等；以及若干实体类型带有属性，如（机构，英文名）、（研报，评级）等。在给定图谱模式和种子知识图谱的条件下，评测内容为自动地从研报文本中抽取出符合图谱模式的实体、关系和属性值，实现金融知识图谱的自动化构建。所构建的图谱在大金融行业、监管部门、政府、行业研究机构和行业公司等应用非常广泛，如风险监测、智能投研、智能监管、智能风控等，具有巨大的学术价值和产业价值。
#
# 评测本身不限制各参赛队伍使用的模型、算法和技术。希望各参赛队伍发挥聪明才智，构建各类无监督、弱监督、远程监督、半监督等系统，迭代的实现知识图谱的自动化构建，共同促进知识图谱技术的进步。
#
# 竞赛任务
# 本评测任务参考 TAC KBP 中的 Cold Start 评测任务的方案，围绕金融研报知识图谱的自动化图谱构建所展开。评测从预定义图谱模式（Schema）和少量的种子知识图谱开始，从非结构化的文本数据中构建知识图谱。评测本身不限制各参赛队伍使用的模型、算法和技术。希望各参赛队伍发挥聪明才智，构建各类无监督、弱监督、远程监督、半监督等系统，迭代的实现知识图谱的自动化构建，共同促进知识图谱技术的进步。
#
# 主办方邮箱  wangwenguang@datagrand.com kdd.wang@gmail.com
#
#
# 参考：https://www.biendata.com/competition/ccks_2020_5/

# In[41]:

import json
import logging
import os
import random
import re
import base64
from collections import defaultdict
from pathlib import Path

# import attr
import tqdm
import hanlp
import numpy as np
import torch
import torch.optim
import torch.utils.data
from torch.nn import functional as F
from torchcrf import CRF
from pytorch_transformers import BertModel, BertTokenizer
import jieba
from jieba.analyse.tfidf import TFIDF
from jieba.posseg import POSTokenizer
import jieba.posseg as pseg
from itertools import product
from IPython.display import HTML

# In[ ]:

# # 预处理函数
#
# 对文章进行预处理，切分句子和子句等

# In[2]:

def split_to_sents(content, filter_length=(2, 1000)):
    content = re.sub(r"\s*", "", content)
    content = re.sub("([。！…？?!；;])", "\\1\1", content)
    sents = content.split("\1")
    sents = [_[: filter_length[1]] for _ in sents]
    return [_ for _ in sents
            if filter_length[0] <= len(_) <= filter_length[1]]

def split_to_subsents(content, filter_length=(2, 1000)):
    content = re.sub(r"\s*", "", content)
    content = re.sub("([。！…？?!；;,，])", "\\1\1", content)
    sents = content.split("\1")
    sents = [_[: filter_length[1]] for _ in sents]
    return [_ for _ in sents
            if filter_length[0] <= len(_) <= filter_length[1]]

# In[3]:

def read_json(file_path):
    with open(file_path, mode='r', encoding='utf8') as f:
        return json.load(f)

# # 预训练模型配置
#
# 参考 https://github.com/huggingface/pytorch-transformers 下载预训练模型，并配置下面参数为相关路径
#
# ```python
# PRETRAINED_BERT_MODEL_DIR = '/you/path/to/bert-base-chinese/'
# ```

# In[4]:

PRETRAINED_BERT_MODEL_DIR = './model/'

# # 一些参数

# In[5]:

DATA_DIR = './data'  # 输入数据文件夹
OUT_DIR = './output'  # 输出文件夹

Path(OUT_DIR).mkdir(exist_ok=True)

BATCH_SIZE = 32
TOTAL_EPOCH_NUMS = 10
if torch.cuda.is_available():
    DEVICE = 'cuda:0'
else:
    DEVICE = 'cpu'
YANBAO_DIR_PATH = str(Path(DATA_DIR, 'yanbao_txt'))
SAVE_MODEL_DIR = str(OUT_DIR)
print(DEVICE, flush=True)

# ## 读入原始数据
#
# - 读入：所有研报内容
# - 读入：原始训练实体数据

# In[6]:

yanbao_texts = []
for yanbao_file_path in Path(YANBAO_DIR_PATH).glob('*.txt'):
    with open(yanbao_file_path) as f:
        yanbao_texts.append(f.read())
#         if len(yanbao_texts) == 10:
#             break

# 来做官方的实体训练集，后续会混合来自第三方工具，规则，训练数据来扩充模型训练数据
to_be_trained_entities = read_json(Path(DATA_DIR, 'entities.json'))

# # 用hanlp进行实体识别
#
# hanlp支持对人物、机构的实体识别，可以使用它来对其中的两个实体类型进行识别：人物、机构。
#
# hanlp见[https://github.com/hankcs/HanLP](https://github.com/hankcs/HanLP)

# In[7]:

## NER by third party tool
class HanlpNER:
    def __init__(self):
        self.recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH)
        self.max_sent_len = 126
        self.ent_type_map = {
            'NR': '人物',
            'NT': '机构'
        }
        self.black_list = {'公司'}

    def recognize(self, sent):
        entities_dict = {}
        for result in self.recognizer.predict([list(sent)]):
            for entity, hanlp_ent_type, _, _ in result:
                if not re.findall(r'^[\.\s\da-zA-Z]{1,2}$', entity) and                         len(entity) > 1 and entity not in self.black_list                         and hanlp_ent_type in self.ent_type_map:
                    entities_dict.setdefault(self.ent_type_map[hanlp_ent_type], []).append(entity)
        return entities_dict

# In[8]:

def aug_entities_by_third_party_tool():
    hanlpner = HanlpNER()
    entities_by_third_party_tool = defaultdict(list)
    for file in tqdm.tqdm(list(Path(DATA_DIR, 'yanbao_txt').glob('*.txt'))[:]):
        with open(file, encoding='utf-8') as f:
            sents = [[]]
            cur_sent_len = 0
            for line in f:
                for sent in split_to_subsents(line):
                    sent = sent[:hanlpner.max_sent_len]
                    if cur_sent_len + len(sent) > hanlpner.max_sent_len:
                        sents.append([sent])
                        cur_sent_len = len(sent)
                    else:
                        sents[-1].append(sent)
                        cur_sent_len += len(sent)
            sents = [''.join(_) for _ in sents]
            sents = [_ for _ in sents if _]
            for sent in sents:
                entities_dict = hanlpner.recognize(sent)
                for ent_type, ents in entities_dict.items():
                    entities_by_third_party_tool[ent_type] += ents

    for ent_type, ents in entities_by_third_party_tool.items():
        entities_by_third_party_tool[ent_type] = list([ent for ent in set(ents) if len(ent) > 1])
    return entities_by_third_party_tool

# In[10]:

# 此任务十分慢, 但是只需要运行一次
entities_by_third_party_tool = aug_entities_by_third_party_tool()
for ent_type, ents in entities_by_third_party_tool.items():
    to_be_trained_entities[ent_type] = list(set(to_be_trained_entities[ent_type] + ents))

# In[11]:

for k, v in entities_by_third_party_tool.items():
    print(k)
    print(set(v))

# ## 通过规则抽取实体
#
# - 机构
# - 研报
# - 文章
# - 风险

# In[12]:

def aug_entities_by_rules(yanbao_dir):
    entities_by_rule = defaultdict(list)
    for file in list(yanbao_dir.glob('*.txt'))[:]:
        with open(file, encoding='utf-8') as f:
            found_yanbao = False
            found_fengxian = False
            for lidx, line in enumerate(f):
                # 公司的标题
                ret = re.findall('^[\(（]*[\d一二三四五六七八九十①②③④⑤]*[\)）\.\s]*(.*有限公司)$', line)
                if ret:
                    entities_by_rule['机构'].append(ret[0])

                # 研报
                if not found_yanbao and lidx <= 5 and len(line) > 10:
                    may_be_yanbao = line.strip()
                    if not re.findall(r'\d{4}\s*[年-]\s*\d{1,2}\s*[月-]\s*\d{1,2}\s*日?', may_be_yanbao)                             and not re.findall('^[\d一二三四五六七八九十]+\s*[\.、]\s*.*$', may_be_yanbao)                             and not re.findall('[\(（]\d+\.*[A-Z]*[\)）]', may_be_yanbao)                             and len(may_be_yanbao) > 5:
                        entities_by_rule['研报'].append(may_be_yanbao)
                        found_yanbao = True

                # 文章
                for sent in split_to_sents(line):
                    results = re.findall('《(.*?)》', sent)
                    for result in results:
                        entities_by_rule['文章'].append(result)

                # 风险
                for sent in split_to_sents(line):
                    if found_fengxian:
                        sent = sent.split('：')[0]
                        fengxian_entities = re.split('以及|、|，|；|。', sent)
                        fengxian_entities = [re.sub('^[■]+[\d一二三四五六七八九十①②③④⑤]+', '', ent) for ent in fengxian_entities]
                        fengxian_entities = [re.sub('^[\(（]*[\d一二三四五六七八九十①②③④⑤]+[\)）\.\s]+', '', ent) for ent in fengxian_entities]
                        fengxian_entities = [_ for _ in fengxian_entities if len(_) >=4]
                        entities_by_rule['风险'] += fengxian_entities
                        found_fengxian = False
                    if not found_fengxian and re.findall('^\s*[\d一二三四五六七八九十]*\s*[\.、]*\s*风险提示[:：]*$', sent):
                        found_fengxian = True

                    results = re.findall('^\s*[\d一二三四五六七八九十]*\s*[\.、]*\s*风险提示[:：]*(.{5,})$', sent)
                    if results:
                        fengxian_entities = re.split('以及|、|，|；|。', results[0])
                        fengxian_entities = [re.sub('^[■]+[\d一二三四五六七八九十①②③④⑤]+', '', ent) for ent in fengxian_entities]
                        fengxian_entities = [re.sub('^[\(（]*[\d一二三四五六七八九十①②③④⑤]+[\)）\.\s]+', '', ent) for ent in fengxian_entities]
                        fengxian_entities = [_ for _ in fengxian_entities if len(_) >=4]
                        entities_by_rule['风险'] += fengxian_entities

    for ent_type, ents in entities_by_rule.items():
        entities_by_rule[ent_type] = list(set(ents))
    return entities_by_rule

# In[13]:

# 通过规则来寻找新的实体
entities_by_rule = aug_entities_by_rules(Path(DATA_DIR, 'yanbao_txt'))
for ent_type, ents in entities_by_rule.items():
    to_be_trained_entities[ent_type] = list(set(to_be_trained_entities[ent_type] + ents))

# In[14]:

for k, v in entities_by_rule.items():
    print(k)
    print(set(v))

# # 定义NER模型
#

# In[15]:

class BertCRF(torch.nn.Module):
    def __init__(self, pretrained_bert_model_file_path, num_tags: int, batch_first: bool = False, hidden_size=768):
        super(BertCRF, self).__init__()
        self.bert_module = BertModel.from_pretrained(pretrained_bert_model_file_path)
        self.tag_linear = torch.nn.Linear(hidden_size, num_tags)
        self.crf_module = CRF(num_tags, batch_first)

    def forward(self,
                inputs_ids,
                tags,
                mask = None,
                token_type_ids=None,
                reduction = 'mean'
                ) -> torch.Tensor:
        bert_outputs = self.bert_module.forward(inputs_ids, attention_mask=mask, token_type_ids=token_type_ids)[0]
        bert_outputs = F.dropout(bert_outputs, p=0.2, training=self.training)
        bert_outputs = self.tag_linear(bert_outputs)
        score = -self.crf_module.forward(bert_outputs, tags=tags, mask=mask, reduction=reduction)
        return score

    def decode(self,
               input_ids,
               attention_mask=None,
               token_type_ids=None
               ):
        bert_outputs = self.bert_module.forward(
            input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids
        )[0]
        bert_outputs = self.tag_linear(bert_outputs)
        best_tags_list = self.crf_module.decode(bert_outputs, mask=attention_mask)
        return best_tags_list

# ## 定义NER标签

# In[16]:

chinese_entity_type_vs_english_entity_type = {
    '人物': 'People',
    '行业': 'Industry',
    '业务': 'Business',
    '研报': 'Report',
    '机构': 'Organization',
    '风险': 'Risk',
    '文章': 'Article',
    '指标': 'Indicator',
    '品牌': 'Brand',
    '产品': 'Product'
}
english_entity_type_vs_chinese_entity_type = {v: k for k, v in chinese_entity_type_vs_english_entity_type.items()}

START_TAG = "[CLS]"
END_TAG = "[SEP]"
O = "O"

BPeople = "B-People"
IPeople = "I-People"
BIndustry = "B-Industry"
IIndustry = "I-Industry"
BBusiness = 'B-Business'
IBusiness = 'I-Business'
BProduct = 'B-Product'
IProduct = 'I-Product'
BReport = 'B-Report'
IReport = 'I-Report'
BOrganization = 'B-Organization'
IOrganization = 'I-Organization'
BRisk = 'B-Risk'
IRisk = 'I-Risk'
BArticle = 'B-Article'
IArticle = 'I-Article'
BIndicator = 'B-Indicator'
IIndicator = 'I-Indicator'
BBrand = 'B-Brand'
IBrand = 'I-Brand'

PAD = "[PAD]"
UNK = "[UNK]"
tag2idx = {
    START_TAG: 0,
    END_TAG: 1,
    O: 2,
    BPeople: 3,
    IPeople: 4,
    BIndustry: 5,
    IIndustry: 6,
    BBusiness: 7,
    IBusiness: 8,
    BProduct: 9,
    IProduct: 10,
    BReport: 11,
    IReport: 12,
    BOrganization: 13,
    IOrganization: 14,
    BRisk: 15,
    IRisk: 16,
    BArticle: 17,
    IArticle: 18,
    BIndicator: 19,
    IIndicator: 20,
    BBrand: 21,
    IBrand: 22,
}
tag2id = tag2idx
idx2tag = {v: k for k, v in tag2idx.items()}

# ## 预处理数据函数
#
# `preprocess_data` 函数中的 `for_train` 参数比较重要，指示是否是训练集
#
# 由于给定的训练数据实体部分没有给定出现的位置，这里需要自行查找到实体出现的位置
#
# - 如果是训练集, 按照`entities_json`中的内容在文章中寻找位置并标注, 并将训练数据处理成bio形式
# - 测试数据仅仅做了分句并转化成token id

# In[17]:

class Article:
    def __init__(self, text):
        self._text = text
        self.para_texts = self.split_into_paras(self._text)
        self.sent_texts = [self.split_into_sentence(para) for para in self.para_texts]

    def fix_text(self, text: str) -> str:
        paras = text.split('\n')
        paras = list(filter(lambda para: len(para.strip()) != 0, paras))
        return '\n'.join(paras)

    def split_into_paras(self, text: str):
        paras = list(filter(lambda para: len(para.strip()) != 0, text.split('\n')))
        return paras

    def split_into_sentence(self, one_para_text: str, splited_puncs = None):
        if splited_puncs is None:
            splited_puncs = ['。', '？', '！']
        splited_re_pattern = '[' + ''.join(splited_puncs) + ']'

        para = one_para_text
        sentences = re.split(splited_re_pattern, para)
        sentences = list(filter(lambda sent: len(sent) != 0, sentences))

        return sentences

    def find_sents_by_entity_name(self, entity_text):
        ret_sents = []
        if entity_text not in self._text:
            return []
        else:
            for para in self.split_into_paras(self._text):
                if entity_text not in para:
                    continue
                else:
                    for sent in self.split_into_sentence(para):
                        if entity_text in sent:
                            ret_sents.append(sent)
        return ret_sents

# In[18]:

def _find_all_start_end(source, target):
    if not target:
        return []
    occurs = []
    offset = 0
    while offset < len(source):
        found = source[offset:].find(target)
        if found == -1:
            break
        else:
            occurs.append([offset + found, offset + found + len(target) - 1])
        offset += (found + len(target))
    return occurs

def preprocess_data(entities_json,
                    article_texts,
                    tokenizer: BertTokenizer,
                    for_train: bool = True):
    """
    [{
        'sent': xxx, 'entity_name': yyy, 'entity_type': zzz, 'start_token_id': 0, 'end_token_id': 5,
        'start_index': 0, 'end_index': 2,
            'sent_tokens': ['token1', 'token2'], 'entity_tokens': ['token3', 'token4']
    }]
    """

    preprocessed_datas = []

    all_sents = []
    for article in tqdm.tqdm([Article(t) for t in article_texts]):
        for para_text in article.para_texts:
            for sent in article.split_into_sentence(para_text):
                sent_tokens = list(sent)
                entity_labels = []
                for entity_type, entities in entities_json.items():
                    for entity_name in entities:
                        if entity_name not in sent:
                            continue
                        all_sents.append(sent)
                        start_end_indexes = _find_all_start_end(sent, entity_name)
                        assert len(start_end_indexes) >= 1
                        for str_start_index, str_end_index in start_end_indexes:
                            entity_tokens = list(entity_name)

                            one_entity_label = {
                                'entity_type': entity_type,
                                'start_token_id': str_start_index,
                                'end_token_id': str_end_index,
                                'start_index': str_start_index,
                                'end_index': str_end_index,
                                'entity_tokens': entity_tokens,
                                'entity_name': entity_name
                            }
                            entity_labels.append(one_entity_label)

                if not entity_labels:
                    tags = [O for _ in range(len(sent_tokens))]
                    tag_ids = [tag2idx[O] for _ in range(len(sent_tokens))]
                    if for_train:
                        continue
                else:
                    tags = []
                    tag_ids = []
                    for sent_token_index in range(len(sent_tokens)):
                        tag = O
                        for entity_label in entity_labels:
                            if sent_token_index == entity_label['start_token_id']:
                                tag = f'B-{chinese_entity_type_vs_english_entity_type[entity_label["entity_type"]]}'
                            elif entity_label['start_token_id'] < sent_token_index < entity_label["end_token_id"]:
                                tag = f'I-{chinese_entity_type_vs_english_entity_type[entity_label["entity_type"]]}'
                        tag_id = tag2idx[tag]
                        tags.append(tag)
                        tag_ids.append(tag_id)
                assert len(sent_tokens) == len(tags) == len(tag_ids)
                not_o_indexes = [index for index, tag in enumerate(tags) if tag != O]
                all_entities = [sent_tokens[index] for index in not_o_indexes]
                all_entities2 = entity_labels

                preprocessed_datas.append({
                    'sent': sent,
                    'sent_tokens': sent_tokens,
                    'sent_token_ids': tokenizer.convert_tokens_to_ids(sent_tokens),
                    'entity_labels': entity_labels,
                    'tags': tags,
                    'tag_ids': tag_ids
                })
    return preprocessed_datas

# # 定义dataset 以及 dataloader

# In[19]:

class MyDataset(torch.utils.data.Dataset):
    def __init__(self, preprocessed_datas, tokenizer: BertTokenizer, max_length=512 ):
        self.preprocessed_datas = preprocessed_datas
        self.tokenizer = tokenizer
        self.max_length = max_length

    def pad_sent_ids(self, sent_ids, max_length, padded_token_id):
        mask = [1] * (min(len(sent_ids), max_length)) + [0] * (max_length - len(sent_ids))
        sent_ids = sent_ids[:max_length] + [padded_token_id] * (max_length - len(sent_ids))
        return sent_ids, mask

    def process_one_preprocessed_data(self, preprocessed_data):
        import copy
        preprocessed_data = copy.deepcopy(preprocessed_data)

        sent_token_ids = self.tokenizer.convert_tokens_to_ids(
            [START_TAG]) + preprocessed_data['sent_token_ids'] + self.tokenizer.convert_tokens_to_ids([END_TAG])

        sent_token_ids, mask = self.pad_sent_ids(
            sent_token_ids, max_length=self.max_length, padded_token_id=self.tokenizer.pad_token_id)

        sent_token_ids = np.array(sent_token_ids)
        mask = np.array(mask)

        preprocessed_data['sent'] = '^' + preprocessed_data['sent'] + '$'
        preprocessed_data['sent_tokens'] = [START_TAG] + preprocessed_data['sent_tokens'] + [END_TAG]
        preprocessed_data['sent_token_ids'] = sent_token_ids

        tags = [START_TAG] + preprocessed_data['tags'] + [END_TAG]
        tag_ids = [tag2idx[START_TAG]] + preprocessed_data['tag_ids'] + [tag2idx[END_TAG]]
        tag_ids, _ = self.pad_sent_ids(tag_ids, max_length=self.max_length, padded_token_id=tag2idx[O])
        tag_ids = np.array(tag_ids)

        for entity_label in preprocessed_data['entity_labels']:
            entity_label['start_token_id'] += 1
            entity_label['end_token_id'] += 1
            entity_label['start_index'] += 1
            entity_label['end_index'] += 1

        preprocessed_data['tags'] = tags
        preprocessed_data['tag_ids'] = tag_ids

        not_o_indexes = [index for index, tag in enumerate(preprocessed_data['tags']) if tag != O]

        not_o_indexes_str = not_o_indexes
        all_entities = [preprocessed_data['sent_tokens'][index] for index in not_o_indexes]
        all_entities2 = preprocessed_data['entity_labels']
        all_entities3 = [preprocessed_data['sent'][index] for index in not_o_indexes_str]

        preprocessed_data.update({'mask': mask})

        return preprocessed_data

    def __getitem__(self, item):
        return self.process_one_preprocessed_data(
            self.preprocessed_datas[item]
        )

    def __len__(self):
        return len(self.preprocessed_datas)

def custom_collate_fn(data):
    # copy from torch official，无需深究
    from torch._six import container_abcs, string_classes

    r"""Converts each NumPy array data field into a tensor"""
    np_str_obj_array_pattern = re.compile(r'[SaUO]')
    elem_type = type(data)
    if isinstance(data, torch.Tensor):
        return data
    elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' and elem_type.__name__ != 'string_':
        # array of string classes and object
        if elem_type.__name__ == 'ndarray' and np_str_obj_array_pattern.search(data.dtype.str) is not None:
            return data
        return torch.as_tensor(data)
    elif isinstance(data, container_abcs.Mapping):
        tmp_dict = {}
        for key in data:
            if key in ['sent_token_ids', 'tag_ids', 'mask']:
                tmp_dict[key] = custom_collate_fn(data[key])
                if key == 'mask':
                    tmp_dict[key] = tmp_dict[key].byte()
            else:
                tmp_dict[key] = data[key]
        return tmp_dict
    elif isinstance(data, tuple) and hasattr(data, '_fields'):  # namedtuple
        return elem_type(*(custom_collate_fn(d) for d in data))
    elif isinstance(data, container_abcs.Sequence) and not isinstance(data, string_classes):
        return [custom_collate_fn(d) for d in data]
    else:
        return data

def build_dataloader(preprocessed_datas, tokenizer: BertTokenizer, batch_size=32, shuffle=True):
    dataset = MyDataset(preprocessed_datas, tokenizer)
    import torch.utils.data
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, collate_fn=custom_collate_fn, shuffle=shuffle)
    return dataloader

# # 定义训练时评价指标
#
# 仅供训练时参考, 包含实体的precision，recall以及f1。
#
# 只有和标注的数据完全相同才算是1，否则为0

# In[20]:

# 训练时指标
class EvaluateScores:
    def __init__(self, entities_json, predict_entities_json):
        self.entities_json = entities_json
        self.predict_entities_json = predict_entities_json

    def compute_entities_score(self):
        return evaluate_entities(self.entities_json, self.predict_entities_json, list(set(self.entities_json.keys())))

def _compute_metrics(ytrue, ypred):
    ytrue = set(ytrue)
    ypred = set(ypred)
    tr = len(ytrue)
    pr = len(ypred)
    hit = len(ypred.intersection(ytrue))
    p = hit / pr if pr!=0 else 0
    r = hit / tr if tr!=0 else 0
    f1 = 2 * p * r / (p + r) if (p+r)!=0 else 0
    return {
        'p': p,
        'r': r,
        'f': f1,
    }

def evaluate_entities(true_entities, pred_entities, entity_types):
    scores = []

    ps2 = []
    rs2 = []
    fs2 = []

    for ent_type in entity_types:

        true_entities_list = true_entities.get(ent_type, [])
        pred_entities_list = pred_entities.get(ent_type, [])
        s = _compute_metrics(true_entities_list, pred_entities_list)
        scores.append(s)
    ps = [i['p'] for i in scores]
    rs = [i['r'] for i in scores]
    fs = [i['f'] for i in scores]
    s = {
        'p': sum(ps) / len(ps),
        'r': sum(rs) / len(rs),
        'f': sum(fs) / len(fs),
    }
    return s

# ## 定义ner train loop， evaluate loop ，test loop

# In[21]:

def train(model: BertCRF, optimizer, data_loader: torch.utils.data.DataLoader, logger: logging.Logger, epoch_id,
          device='cpu'):
    pbar = tqdm.tqdm(data_loader)
    for batch_id, one_data in enumerate(pbar):
        model.train()

        sent_token_ids = torch.stack([d['sent_token_ids'] for d in one_data]).to(device)
        tag_ids = torch.stack([d['tag_ids'] for d in one_data]).to(device)
        mask = torch.stack([d['mask'] for d in one_data]).to(device)

        loss = model.forward(sent_token_ids, tag_ids, mask)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        pbar.set_description('epoch: {}, loss: {:.3f}'.format(epoch_id, loss.item()))

def evaluate(
        model, data_loader: torch.utils.data.DataLoader, logger: logging.Logger,
        tokenizer, device='cpu',
):
    founded_entities_json = defaultdict(set)
    golden_entities_json = defaultdict(set)
    for batch_id, one_data in enumerate(data_loader):
        model.eval()
        sent_token_ids = torch.stack([d['sent_token_ids'] for d in one_data]).to(device)
        tag_ids = torch.stack([d['tag_ids'] for d in one_data]).to(device)
        mask = torch.stack([d['mask'] for d in one_data]).to(device)

        best_tag_ids_list = model.decode(sent_token_ids, attention_mask=mask)
        best_tags_list = [[idx2tag[idx] for idx in idxs] for idxs in best_tag_ids_list]

        for data, best_tags in zip(one_data, best_tag_ids_list):

            for entity_label in data['entity_labels']:
                golden_entities_json[entity_label['entity_type']].add(entity_label['entity_name'])

            record = False
            for token_index, tag_id in enumerate(best_tags):
                tag = idx2tag[tag_id]
                if tag.startswith('B'):
                    start_token_index = token_index
                    entity_type = tag[2:]
                    record = True
                elif record and tag == O:
                    end_token_index = token_index

                    str_start_index = start_token_index
                    str_end_index = end_token_index

                    entity_name = data['sent'][str_start_index: str_end_index]

                    entity_type = english_entity_type_vs_chinese_entity_type[entity_type]
                    founded_entities_json[entity_type].add(entity_name)
                    record = False
    evaluate_tool = EvaluateScores(golden_entities_json, founded_entities_json)
    scores = evaluate_tool.compute_entities_score()
    return scores['f']

def test(model, data_loader: torch.utils.data.DataLoader, logger: logging.Logger, device):
    founded_entities = []
    for batch_id, one_data in enumerate(tqdm.tqdm(data_loader)):
        model.eval()
        sent_token_ids = torch.stack([d['sent_token_ids'] for d in one_data]).to(device)
        mask = torch.stack([d['mask'] for d in one_data]).to(device)

        with torch.no_grad():
            best_tag_ids_list = model.decode(sent_token_ids, attention_mask=mask, token_type_ids=None)

        for data, best_tags in zip(one_data, best_tag_ids_list):
            record = False
            for token_index, tag_id in enumerate(best_tags):
                tag = idx2tag[tag_id]
                if tag.startswith('B'):
                    start_token_index = token_index
                    entity_type = tag[2:]
                    record = True
                elif record and tag == O:
                    end_token_index = token_index
                    entity_name = data['sent_tokens'][start_token_index: end_token_index + 1]
                    founded_entities.append((entity_name, entity_type, data['sent']))
                    record = False
    result = defaultdict(list)
    for entity_name, entity_type, sent in founded_entities:
        entity = ''.join(entity_name).replace('##', '')
        entity = entity.replace('[CLS]', '')
        entity = entity.replace('[UNK]', '')
        entity = entity.replace('[SEP]', '')
        if len(entity) > 1:
            result[english_entity_type_vs_chinese_entity_type[entity_type]].append((entity, sent))

    for ent_type, ents in result.items():
        result[ent_type] = list(set(ents))
    return result

# # ner主要训练流程
#
# - 分隔训练集验证集，并处理成dataset dataloader
# - 训练，验证，保存模型

# In[22]:

def main_train(logger, tokenizer, model, to_be_trained_entities, yanbao_texts):
    entities_json = to_be_trained_entities
    train_entities_json = {k: [] for k in entities_json}
    dev_entities_json = {k: [] for k in entities_json}

    train_proportion = 0.9
    for entity_type, entities in entities_json.items():
        entities = entities.copy()
        random.shuffle(entities)

        train_entities_json[entity_type] = entities[: int(len(entities) * train_proportion)]
        dev_entities_json[entity_type] = entities[int(len(entities) * train_proportion):]

    train_preprocessed_datas = preprocess_data(train_entities_json, yanbao_texts, tokenizer)
    train_dataloader = build_dataloader(train_preprocessed_datas, tokenizer, batch_size=BATCH_SIZE)

    dev_preprocessed_datas = preprocess_data(dev_entities_json, yanbao_texts, tokenizer)
    dev_dataloader = build_dataloader(dev_preprocessed_datas, tokenizer, batch_size=BATCH_SIZE)

    model = model.to(DEVICE)
    for name, param in model.named_parameters():
        if "bert_module" in name:
            param.requires_grad = False
        else:
            param.requires_grad = True
    optimizer = torch.optim.Adam([para for para in model.parameters() if para.requires_grad],
                                 lr=0.001,
                                 weight_decay=0.0005)
    best_evaluate_score = 0
    for epoch in range(TOTAL_EPOCH_NUMS):
        train(model, optimizer, train_dataloader, logger=logger, epoch_id=epoch, device=DEVICE)
        evaluate_score = evaluate(model, dev_dataloader, logger=logger, tokenizer=tokenizer, device=DEVICE)
        print('评估分数：', evaluate_score)
        if evaluate_score >= best_evaluate_score:
            best_evaluate_score = evaluate_score
            save_model_path = os.path.join(SAVE_MODEL_DIR, 'finnal_ccks_model.pth')
            logger.info('saving model to {}'.format(save_model_path))
            torch.save(model.cpu().state_dict(), save_model_path)
            model.to(DEVICE)

# ## 准备训练ner模型

# In[47]:

logger = logging.getLogger(__name__)

tokenizer = BertTokenizer.from_pretrained(
    os.path.join(PRETRAINED_BERT_MODEL_DIR, 'vocab.txt'))

model = BertCRF(pretrained_bert_model_file_path=PRETRAINED_BERT_MODEL_DIR,
                num_tags=len(tag2id),
                batch_first=True)

save_model_path = os.path.join(SAVE_MODEL_DIR, 'finnal_ccks_model.pth')
if Path(save_model_path).exists():
    model_state_dict = torch.load(save_model_path, map_location='cpu')
    model.load_state_dict(model_state_dict)

# In[46]:

# In[48]:

# 训练数据在main_train函数中处理并生成dataset dataloader，此处无需生成

# 测试数据在此处处理并生成dataset dataloader
test_preprocessed_datas = preprocess_data({},
                                          yanbao_texts,
                                          tokenizer,
                                          for_train=False)
test_dataloader = build_dataloader(test_preprocessed_datas,
                                   tokenizer,
                                   batch_size=BATCH_SIZE)

# ## 整个训练流程是：
#
# - 使用数据集增强得到更多的实体
# - 使用增强过后的实体来指导训练
#
#
# - 训练后的模型重新对所有文档中进行预测，得到新的实体，加入到实体数据集中
# - 使用扩增后的实体数据集来进行二次训练，再得到新的实体，再增强实体数据集
# - (模型预测出来的数据需要`review_model_predict_entities`后处理形成提交格式)
#
#
# - 如果提交结果，需要`extract_entities`函数删除提交数据中那些出现在训练数据中的实体

# ### 模型预测结果后处理函数
#
# - `review_model_predict_entities`函数将模型预测结果后处理，从而生成提交文件格式

# In[49]:

def review_model_predict_entities(model_predict_entities):
    word_tag_map = POSTokenizer().word_tag_tab
    idf_freq = TFIDF().idf_freq
    reviewed_entities = defaultdict(list)
    for ent_type, ent_and_sent_list in model_predict_entities.items():
        for ent, sent in ent_and_sent_list:
            start = sent.lower().find(ent)
            if start == -1:
                continue
            start += 1
            end = start + len(ent) - 1
            tokens = jieba.lcut(sent)
            offset = 0
            selected_tokens = []
            for token in tokens:
                offset += len(token)
                if offset >= start:
                    selected_tokens.append(token)
                if offset >= end:
                    break

            fixed_entity = ''.join(selected_tokens)
            fixed_entity = re.sub(r'\d*\.?\d+%$', '', fixed_entity)
            if ent_type == '人物':
                if len(fixed_entity) >= 10:
                    continue
            if len(fixed_entity) <= 1:
                continue
            if re.findall(r'^\d+$', fixed_entity):
                continue
            if word_tag_map.get(fixed_entity,
                                '') == 'v' and idf_freq[fixed_entity] < 7:
                continue
            reviewed_entities[ent_type].append(fixed_entity)
    return reviewed_entities

# - `extract_entities` 删除与训练集中重复的实体

# In[50]:

def extract_entities(to_be_trained_entities):
    test_entities = to_be_trained_entities
    train_entities = read_json(Path(DATA_DIR, 'entities.json'))

    for ent_type, ents in test_entities.items():
        test_entities[ent_type] = list(
            set(ents) - set(train_entities[ent_type]))

    for ent_type in train_entities.keys():
        if ent_type not in test_entities:
            test_entities[ent_type] = []
    return test_entities

# In[ ]:

# 循环轮次数目
nums_round = 1
for i in range(nums_round):
    # train
    main_train(logger, tokenizer, model, to_be_trained_entities, yanbao_texts)

    model = model.to(DEVICE)
    model_predict_entities = test(model, test_dataloader, logger=logger, device=DEVICE)

    # 修复训练预测结果
    reviewed_entities = review_model_predict_entities(model_predict_entities)

    # 将训练预测结果再次放入训练集中， 重新训练或者直接出结果
    for ent_type, ents in reviewed_entities.items():
        to_be_trained_entities[ent_type] = list(set(to_be_trained_entities[ent_type] + ents))

# 创造出提交结果
submit_entities = extract_entities(to_be_trained_entities)

# # 属性抽取
#
# 通过规则抽取属性
#
# - 研报时间
# - 研报评级
# - 文章时间

# In[ ]:

def find_article_time(yanbao_txt, entity):
    str_start_index = yanbao_txt.index(entity)
    str_end_index = str_start_index + len(entity)
    para_start_index = yanbao_txt.rindex('\n', 0, str_start_index)
    para_end_index = yanbao_txt.index('\n', str_end_index)

    para = yanbao_txt[para_start_index + 1: para_end_index].strip()
    if len(entity) > 5:
        ret = re.findall(r'(\d{4})\s*[年-]\s*(\d{1,2})\s*[月-]\s*(\d{1,2})\s*日?', para)
        if ret:
            year, month, day = ret[0]
            time = '{}/{}/{}'.format(year, month.lstrip(), day.lstrip())
            return time

    start_index = 0
    time = None
    min_gap = float('inf')
    for word, poseg in pseg.cut(para):
        if poseg in ['t', 'TIME'] and str_start_index <= start_index < str_end_index:
            gap = abs(start_index - (str_start_index + str_end_index) // 2)
            if gap < min_gap:
                min_gap = gap
                time = word
        start_index += len(word)
    return time

def find_yanbao_time(yanbao_txt, entity):
    paras = [para.strip() for para in yanbao_txt.split('\n') if para.strip()][:5]
    for para in paras:
        ret = re.findall(r'(\d{4})\s*[\./年-]\s*(\d{1,2})\s*[\./月-]\s*(\d{1,2})\s*日?', para)
        if ret:
            year, month, day = ret[0]
            time = '{}/{}/{}'.format(year, month.lstrip(), day.lstrip())
            return time
    return None

# In[ ]:

def extract_attrs(entities_json):
    train_attrs = read_json(Path(DATA_DIR, 'attrs.json'))['attrs']

    seen_pingjis = []
    for attr in train_attrs:
        if attr[1] == '评级':
            seen_pingjis.append(attr[2])
    article_entities = entities_json.get('文章', [])
    yanbao_entities = entities_json.get('研报', [])

    attrs_json = []
    for file_path in tqdm.tqdm(list(Path(DATA_DIR, 'yanbao_txt').glob('*.txt'))):
        yanbao_txt = '\n' + Path(file_path).open().read() + '\n'
        for entity in article_entities:
            if entity not in yanbao_txt:
                continue
            time = find_article_time(yanbao_txt, entity)
            if time:
                attrs_json.append([entity, '发布时间', time])

        yanbao_txt = '\n'.join(
            [para.strip() for para in yanbao_txt.split('\n') if
             len(para.strip()) != 0])
        for entity in yanbao_entities:
            if entity not in yanbao_txt:
                continue

            paras = yanbao_txt.split('\n')
            for para_id, para in enumerate(paras):
                if entity in para:
                    break

            paras = paras[: para_id + 5]
            for para in paras:
                for pingji in seen_pingjis:
                    if pingji in para:
                        if '上次' in para:
                            attrs_json.append([entity, '上次评级', pingji])
                            continue
                        elif '维持' in para:
                            attrs_json.append([entity, '上次评级', pingji])
                        attrs_json.append([entity, '评级', pingji])

            time = find_yanbao_time(yanbao_txt, entity)
            if time:
                attrs_json.append([entity, '发布时间', time])
    attrs_json = list(set(tuple(_) for _ in attrs_json) - set(tuple(_) for _ in train_attrs))

    return attrs_json

# In[ ]:

train_attrs = read_json(Path(DATA_DIR, 'attrs.json'))['attrs']
submit_attrs = extract_attrs(submit_entities)

# In[ ]:

submit_attrs

# # 关系抽取
#
# - 对于研报实体，整个文档抽取特定类型(行业，机构，指标)的关系实体
# - 其他的实体仅考虑与其出现在同一句话中的其他实体组织成特定关系

# In[ ]:

def extract_relations(schema, entities_json):
    relation_by_rules = []
    relation_schema = schema['relationships']
    unique_s_o_types = []
    so_type_cnt = defaultdict(int)
    for s_type, p, o_type in schema['relationships']:
        so_type_cnt[(s_type, o_type)] += 1
    for (s_type, o_type), cnt in so_type_cnt.items():
        if cnt == 1 and s_type != o_type:
            unique_s_o_types.append((s_type, o_type))

    for path in tqdm.tqdm(list(Path(DATA_DIR, 'yanbao_txt').glob('*.txt'))):
        with open(path) as f:
            entity_dict_in_file = defaultdict(lambda: defaultdict(list))
            main_org = None
            for line_idx, line in enumerate(f.readlines()):
                for sent_idx, sent in enumerate(split_to_sents(line)):
                    for ent_type, ents in entities_json.items():
                        for ent in ents:
                            if ent in sent:
                                if ent_type == '机构' and len(line) - len(ent) < 3 or                                         re.findall('[\(（]\d+\.*[A-Z]*[\)）]', line):
                                    main_org = ent
                                else:
                                    if main_org and '客户' in sent:
                                        relation_by_rules.append([ent, '客户', main_org])
                                entity_dict_in_file[ent_type][
                                    ('test', ent)].append(
                                    [line_idx, sent_idx, sent,
                                     sent.find(ent)]
                                )

            for s_type, p, o_type in relation_schema:
                s_ents = entity_dict_in_file[s_type]
                o_ents = entity_dict_in_file[o_type]
                if o_type == '业务' and not '业务' in line:
                    continue
                if o_type == '行业' and not '行业' in line:
                    continue
                if o_type == '文章' and not ('《' in line or not '》' in line):
                    continue
                if s_ents and o_ents:
                    for (s_ent_src, s_ent), (o_ent_src, o_ent) in product(s_ents, o_ents):
                        if s_ent != o_ent:
                            s_occs = [tuple(_[:2]) for _ in
                                      s_ents[(s_ent_src, s_ent)]]
                            o_occs = [tuple(_[:2]) for _ in
                                      o_ents[(o_ent_src, o_ent)]]
                            intersection = set(s_occs) & set(o_occs)
                            if s_type == '研报' and s_ent_src == 'test':
                                relation_by_rules.append([s_ent, p, o_ent])
                                continue
                            if not intersection:
                                continue
                            if (s_type, o_type) in unique_s_o_types and s_ent_src == 'test':
                                relation_by_rules.append([s_ent, p, o_ent])

    train_relations = read_json(Path(DATA_DIR, 'relationships.json'))['relationships']
    result_relations_set = list(set(tuple(_) for _ in relation_by_rules) - set(tuple(_) for _ in train_relations))
    return result_relations_set

# In[ ]:

schema = read_json(Path(DATA_DIR, 'schema.json'))
submit_relations  = extract_relations(schema, submit_entities)

# In[ ]:

submit_relations

# ## 生成提交文件
#
# 根据biendata的要求生成提交文件
#
# 参考：https://www.biendata.com/competition/ccks_2020_5/make-submission/

# In[ ]:

final_answer = {'attrs': submit_attrs,
                'entities': submit_entities,
                'relationships': submit_relations,
                }

with open('output/answers.json', mode='w') as fw:
        json.dump(final_answer, fw, ensure_ascii=False, indent=4)

# In[ ]:

with open('output/answers.json', 'rb') as fb:
    data = fb.read()

b64 = base64.b64encode(data)
payload = b64.decode()
html = '<a download="{filename}" href="data:text/json;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title='answers.json',filename='answers.json')
HTML(html)

max_stars_repo_name: YerbaPage/KnowledgeGraph

max_issues_repo_name: YerbaPage/KnowledgeGraph

max_forks_repo_name: YerbaPage/KnowledgeGraph


text:
from bisect import bisect
import numpy as np
from sklearn.linear_model import Ridge
from sklearn.preprocessing import normalize, MinMaxScaler
from sklearn.utils import check_random_state

class LIMEExplainer(object):
  def __init__(self, training_data, scale=True, distance_kernel=None, sample_size=5000, categorical_features=None, random_state=None):
    """Instantiates the explainer, and creates hypersphere samples for estimation

    Parameters
    ----------
    training_data: numpy.array (2d)
      Training data for the reference model. Categorical variables should be encoded as integers.

    distance_kernel: Callable
      PDF function defining the distance kernel. Should at least take one argument.

    sample_size: Int
      Number of points to sample within the hypersphere.

    categorical_features: [[int]]
      List indexes of categorical features in training_data.

    random_state: Int
      Optional seed for random choices.

    """
    self.random_state = check_random_state(random_state)
    np.random.seed(random_state)

    # allow pandas dataframe, or np array with one-hot encoded categorical features.
    self.training_data = np.asanyarray(training_data)
    self.categorical_features = categorical_features if categorical_features is not None else []
    self.sample_size = sample_size

    if scale:
      #self.scaler = StandardScaler(with_mean=True)
      self.scaler = MinMaxScaler()
      self.scaler.fit(self.training_data)

      for index in self.categorical_features:
        self.scaler.min_[index] = 0
        self.scaler.scale_[index] = 1
    else:
      self.scaler = None

    # Create hypersphere samples.
    self.n_categorical = len(self.categorical_features)
    self.n_numerical = training_data.shape[1] - self.n_categorical
    self.cat_idx = list(self.categorical_features.keys())

    if distance_kernel is None:
      self.distance_kernel = np.vectorize(lambda x: x ** (1 / self.n_numerical))
    else:
      accounted_kernel = lambda x: distance_kernel(x) * (x ** (self.n_numerical - 1))
      self.distance_kernel = np.vectorize(self._transform(accounted_kernel, self.n_numerical))

  @property
  def surrogate(self):
    """ Yield the surrogate model """
    try:
      return self._surrogate
    except AttributeError:
      self._surrogate = Ridge(alpha=1, fit_intercept=True, random_state=self.random_state)
      return self._surrogate

  @property
  def distance_kernel(self):
    return self._distance_kernel

  @distance_kernel.setter
  def distance_kernel(self, value):
    """ Set distance kernel and generate transfer dataset (sphere) as a side effect. """
    accounted_kernel = lambda x: value(x) * (x ** (self.n_numerical - 1))
    self._distance_kernel = np.vectorize(self._transform(accounted_kernel, self.n_numerical))

    if self.n_numerical > 0:
      sphere = np.random.normal(size=(self.sample_size, self.n_numerical))
      sphere = normalize(sphere)
      sphere *= self._distance_kernel(np.random.uniform(size=self.sample_size)).reshape(-1,1) / 2
    else:
      sphere = np.zeros(shape=(0, self.sample_size))

    for index in self.categorical_features:
      categories, frequencies = np.unique(self.training_data[:, index], return_counts=True)
      values = self.random_state.choice(categories, size=self.sample_size, replace=True, p=frequencies / float(sum(frequencies)))
      sphere = np.insert(sphere, index, values, axis=1)

    self.sphere = sphere

  def explain_instance(self, instance, predict_fn, kernel_width=0.5, labels=(0,), surrogate=None):
    """Explain the instance.

    Parameters
    ----------
    instance: numpy.array (2d)
      Instance to be explained. Should match the columns of the provided training data (so categorical variables
      one-hot encoded)

    predict_fn: Callable
      Predict function of the model to approximate.

    kernel_width: float
      Size of the region of interest, as a fraction of the normalized feature space width. 1 means a spherical region
      that touches min and max of every feature.

    labels: tuple
      Indexes of classes to show explanation from. Defaults to the first class.

    surrogate: (sklearn.base.BaseEstimator, sklearn.base.RegressorMixin)
      Surrogate model to train. Should at least conform to BaseEstimator and RegressorMixin.

    Returns
    -------
    [([float], float)]
      Returns an array of results (one for each class). Every results is a tuple of feature contributions (one for each
      feature) and a R^2 score for the fit.

    """
    surrogate = surrogate or self.surrogate

    # Create transfer dataset by permuting the original instance with the hypersphere samples
    if self.scaler is not None:
      instance = self.scaler.transform(instance)

    scalar = [1 if i in self.cat_idx else kernel_width for i in range(instance.shape[1])]
    X_transfer = scalar * self.sphere
    X_transfer += [0 if i in self.cat_idx else val for i, val in enumerate(instance[0])]

    for index in self.categorical_features:
      X_transfer[:,index] = (X_transfer[:,index] == instance[:,index]).astype(int)

    y_transfer = predict_fn(self.scaler.inverse_transform(X_transfer) if self.scaler is not None else X_transfer)

    def explain_label(label):
      surrogate.fit(X_transfer, y_transfer[:, label])
      score = surrogate.score(X_transfer, y_transfer[:, label])
      return surrogate.coef_, score

    return [explain_label(label) for label in labels]

  """
  Transforms a PDF into an inverse CDF for sampling.
  """
  def _transform(self, pdf, dimensions, sample_size = 1000):
    # Approximate cumulative distribution function
    cdf_samples = np.array([pdf(x) for x in np.linspace(0, 1, sample_size)])
    cdf_samples = np.cumsum(cdf_samples)
    cdf_samples /= cdf_samples[-1]

    # Return inverse of cdf
    return lambda y: bisect(cdf_samples, y) / sample_size

def trapezoid_pdf(x, a, b):
  if 0 <= x and  x <= a:
    return (2 / (a + b))
  elif a <= x and x <= b:
    return (2 / (a + b)) * ((b - x) / (b - a))
  else:
    return 0

def exponential_pdf(x, kernel_width):
  return np.sqrt(np.exp(-(x ** 2) / kernel_width ** 2))

max_stars_repo_name: rajeshkalakoti/cvplot

max_issues_repo_name: rajeshkalakoti/cvplot

max_forks_repo_name: rajeshkalakoti/cvplot