text:
# Modulos para obtener variables del WRF del tipo  wrfprs0( )_20151219_12.000
#
# Es una alternativa al pygrib.index, que para el GFS te tira un warning:
# has multi-field messages, keys inside multi-field messages will not be indexed correctly
#
# Sept 2016 - Maru Dillon, Maxi Sacco
# Modificado 2018 Martin Iglesias #Github SudestadaARG
# -------------------------------------------------

# ********* read_WRF ************

import pygrib
import numpy as np

def read_WRF01(wrf,var,lev):

	"""Funcion para leer un grib en grb2 de las variables de altura del la 		salida del WRF Funcion para leer un GFS en grb2
	y obtener numpy arrays de las variables

        Para usar esta funcion necesito abrir el grb2 antes, de la siguiente 		manera:
	   import pygrib
	   wrf=pygrib.open(file)

	Pedis especificamente la variable y el nivel
	Esta funcion se puede ampliar para todas las variables, para eso buscar 	los registros en wrf01[:]

	Keyword args:
	wrf -- se refiere al nombre que le asignas la apertura del archivo
	var -- 'u','v','rh','psfc' (vars de superficie) (psfc esta en Pa)
	       'u','v','t','geopt','spfh' (vars de altura)
	lev -- 0 (para superficie); nivel de presion en hPa (para altura)
	Para WRF01 no se ve el nivel de superficie(lev--0), solo los de altura.
	Output:
	1 array con una sola variable y nivel
	(los numeros de registro se obtuvieron de usar wgrib en los datos del 		WRF tanto para superficie (02) como para niveles de presion (01)
	"""

	if lev == 1000:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 56
		elif var == "TMP":        #Temp. [K]
                        registro = 57
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 58
		elif var == "UGRD":       #u wind [m/s]
                        registro = 59
		elif var == "VGRD":       #v wind [m/s]
                        registro = 60
        elif lev == 975:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 51
		elif var == "TMP":        #Temp. [K]
                        registro = 52
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 53
		elif var == "UGRD":       #u wind [m/s]
                        registro = 54
		elif var == "VGRD":       #v wind [m/s]
                        registro = 55
        elif lev == 925:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 46
		elif var == "TMP":        #Temp. [K]
                        registro = 47
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 48
		elif var == "UGRD":       #u wind [m/s]
                        registro = 49
		elif var == "VGRD":       #v wind [m/s]
                        registro = 50
        elif lev == 850:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 41
		elif var == "TMP":        #Temp. [K]
                        registro = 42
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 43
		elif var == "UGRD":       #u wind [m/s]
                        registro = 44
		elif var == "VGRD":       #v wind [m/s]
                        registro = 45
        elif lev == 800:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 36
		elif var == "TMP":        #Temp. [K]
                        registro = 37
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 38
		elif var == "UGRD":       #u wind [m/s]
                        registro = 39
		elif var == "VGRD":       #v wind [m/s]
                        registro = 40
        elif lev == 700:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 31
		elif var == "TMP":        #Temp. [K]
                        registro = 32
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 33
		elif var == "UGRD":       #u wind [m/s]
                        registro = 34
		elif var == "VGRD":       #v wind [m/s]
                        registro = 35
        elif lev == 600:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 26
		elif var == "TMP":        #Temp. [K]
                        registro = 27
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 28
		elif var == "UGRD":       #u wind [m/s]
                        registro = 29
		elif var == "VGRD":       #v wind [m/s]
                        registro = 30
        elif lev == 500:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 21
		elif var == "TMP":        #Temp. [K]
                        registro = 22
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 23
		elif var == "UGRD":       #u wind [m/s]
                        registro = 24
		elif var == "VGRD":       #v wind [m/s]
                        registro = 25
        elif lev == 400:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 16
		elif var == "TMP":        #Temp. [K]
                        registro = 17
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 18
		elif var == "UGRD":       #u wind [m/s]
                        registro = 19
		elif var == "VGRD":       #v wind [m/s]
                        registro = 20
        elif lev == 300:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 11
		elif var == "TMP":        #Temp. [K]
                        registro = 12
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 13
		elif var == "UGRD":       #u wind [m/s]
                        registro = 14
		elif var == "VGRD":       #v wind [m/s]
                        registro = 15

        elif lev == 250:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 6
		elif var == "TMP":        #Temp. [K]
                        registro = 7
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 8
		elif var == "UGRD":       #u wind [m/s]
                        registro = 9
		elif var == "VGRD":       #v wind [m/s]
                        registro = 10

        elif lev == 200:
                if var == "HGT":          #Geopotential height [gpm]
                        registro = 1
		elif var == "TMP":        #Temp. [K]
                        registro = 2
		elif var == "SPFH":       #Specific humidity [kg/kg]
                        registro = 3
		elif var == "UGRD":       #u wind [m/s]
                        registro = 4
		elif var == "VGRD":       #v wind [m/s]
                        registro = 5

	return wrf[registro]['values']

def read_WRF02(wrf,var,lev):

	""" Funcion para leer un grib en grb2 de las variables de superficie del la salida del WRF y obtener numpy arrays de las variables

        wrf=pygrib.open(file)

	Pedis especificamente la variable y el nivel
	Esta funcion se puede ampliar para todas las variables, para eso buscar los registros en wrf02[:]

	Keyword args:
	wrf -- se refiere al nombre que le asignas la apertura del archivo
	var -- 'u','v','spfh','psfc', 'APCP' (vars de superficie) (psfc esta en Pa)
	       'u','v','t','geopt','spfh' (vars de altura)
	lev -- 0 (para superficie); nivel de presion en hPa (para altura)

	Output:
	1 array con una sola variable y nivel
	"""
	if lev == 0:
		if var == "UGRD":         #u wind 10[m/s]
			registro = 4
		elif var == "VGRD":       #v wind 10[m/s]
			registro = 5
		elif var == "SPFH":       #Specific humidity [kg/kg]
			registro = 3
		elif var == "TMP":	  #Temp. 2m [K]
			registro = 2
		elif var == "PRES":	  #Pressure [Pa]
			registro = 1
        	elif var == "APCP":      #Total precipitation acumulada [kg/m^2]
			registro = 6
	return wrf[registro]['values']


text:
import json
import collections
import numpy as np

import bert.tokenization as tokenization
from squad.squad_utils import  get_final_text, _get_best_indexes
from squad.squad_evaluate import exact_match_score, f1_score

label_to_id = {'other': 0, 'neutral': 1, 'positive': 2, 'negative': 3, 'conflict': 4}
id_to_label = {0: 'other', 1: 'neutral', 2: 'positive', 3: 'negative', 4: 'conflict'}

class SemEvalExample(object):
    def __init__(self,
                 example_id,
                 sent_tokens,
                 term_texts=None,
                 start_positions=None,
                 end_positions=None,
                 polarities=None):
        self.example_id = example_id
        self.sent_tokens = sent_tokens
        self.term_texts = term_texts
        self.start_positions = start_positions
        self.end_positions = end_positions
        self.polarities = polarities

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        s = ""
        # s += "example_id: %s" % (tokenization.printable_text(self.example_id))
        s += ", sent_tokens: [%s]" % (" ".join(self.sent_tokens))
        if self.term_texts:
            s += ", term_texts: {}".format(self.term_texts)
        # if self.start_positions:
        #     s += ", start_positions: {}".format(self.start_positions)
        # if self.end_positions:
        #     s += ", end_positions: {}".format(self.end_positions)
        if self.polarities:
            s += ", polarities: {}".format(self.polarities)
        return s

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self,
                 unique_id,
                 example_index,
                 tokens,
                 token_to_orig_map,
                 input_ids,
                 input_mask,
                 segment_ids,
                 start_positions=None,
                 end_positions=None,
                 start_indexes=None,
                 end_indexes=None,
                 bio_labels=None,
                 polarity_positions=None,
                 polarity_labels=None,
                 label_masks=None):
        self.unique_id = unique_id
        self.example_index = example_index
        self.tokens = tokens
        self.token_to_orig_map = token_to_orig_map
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.segment_ids = segment_ids
        self.start_positions = start_positions
        self.end_positions = end_positions
        self.start_indexes = start_indexes
        self.end_indexes = end_indexes
        self.bio_labels = bio_labels
        self.polarity_positions = polarity_positions
        self.polarity_labels = polarity_labels
        self.label_masks = label_masks

def convert_examples_to_features(examples, tokenizer, max_seq_length, verbose_logging=False, logger=None):
    max_term_num = max([len(example.term_texts) for (example_index, example) in enumerate(examples)])
    max_sent_length, max_term_length = 0, 0

    unique_id = 1000000000
    features = []
    for (example_index, example) in enumerate(examples):
        tok_to_orig_index = []
        orig_to_tok_index = []
        all_doc_tokens = []
        for (i, token) in enumerate(example.sent_tokens):
            orig_to_tok_index.append(len(all_doc_tokens))
            sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                tok_to_orig_index.append(i)
                all_doc_tokens.append(sub_token)
        if len(all_doc_tokens) > max_sent_length:
            max_sent_length = len(all_doc_tokens)

        tok_start_positions = []
        tok_end_positions = []
        for start_position, end_position in \
                zip(example.start_positions, example.end_positions):
            tok_start_position = orig_to_tok_index[start_position]
            if end_position < len(example.sent_tokens) - 1:
                tok_end_position = orig_to_tok_index[end_position + 1] - 1
            else:
                tok_end_position = len(all_doc_tokens) - 1
            tok_start_positions.append(tok_start_position)
            tok_end_positions.append(tok_end_position)

        # Account for [CLS] and [SEP] with "- 2"
        if len(all_doc_tokens) > max_seq_length - 2:
            all_doc_tokens = all_doc_tokens[0:(max_seq_length - 2)]

        tokens = []
        token_to_orig_map = {}
        segment_ids = []
        tokens.append("[CLS]")
        segment_ids.append(0)

        for index, token in enumerate(all_doc_tokens):
            token_to_orig_map[len(tokens)] = tok_to_orig_index[index]
            tokens.append(token)
            segment_ids.append(0)
        tokens.append("[SEP]")
        segment_ids.append(0)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        input_mask = [1] * len(input_ids)

        while len(input_ids) < max_seq_length:
            input_ids.append(0)
            input_mask.append(0)
            segment_ids.append(0)

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        # For distant supervision, we annotate the positions of all answer spans
        start_positions = [0] * len(input_ids)
        end_positions = [0] * len(input_ids)
        bio_labels = [0] * len(input_ids)
        polarity_positions = [0] * len(input_ids)
        start_indexes, end_indexes = [], []
        for tok_start_position, tok_end_position, polarity in zip(tok_start_positions, tok_end_positions, example.polarities):
            if (tok_start_position >= 0 and tok_end_position <= (max_seq_length - 1)):
                start_position = tok_start_position + 1   # [CLS]
                end_position = tok_end_position + 1   # [CLS]
                start_positions[start_position] = 1
                end_positions[end_position] = 1
                start_indexes.append(start_position)
                end_indexes.append(end_position)
                term_length = tok_end_position - tok_start_position + 1
                max_term_length = term_length if term_length > max_term_length else max_term_length
                bio_labels[start_position] = 1  # 'B'
                if start_position < end_position:
                    for idx in range(start_position + 1, end_position + 1):
                        bio_labels[idx] = 2  # 'I'
                for idx in range(start_position, end_position + 1):
                    polarity_positions[idx] = label_to_id[polarity]

        polarity_labels = [label_to_id[polarity] for polarity in example.polarities]
        label_masks = [1] * len(polarity_labels)

        while len(start_indexes) < max_term_num:
            start_indexes.append(0)
            end_indexes.append(0)
            polarity_labels.append(0)
            label_masks.append(0)

        assert len(start_indexes) == max_term_num
        assert len(end_indexes) == max_term_num
        assert len(polarity_labels) == max_term_num
        assert len(label_masks) == max_term_num

        if example_index < 1 and verbose_logging:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (unique_id))
            logger.info("example_index: %s" % (example_index))
            logger.info("tokens: {}".format(tokens))
            logger.info("token_to_orig_map: {}".format(token_to_orig_map))
            logger.info("start_indexes: {}".format(start_indexes))
            logger.info("end_indexes: {}".format(end_indexes))
            logger.info("bio_labels: {}".format(bio_labels))
            logger.info("polarity_positions: {}".format(polarity_positions))
            logger.info("polarity_labels: {}".format(polarity_labels))

        features.append(
            InputFeatures(
                unique_id=unique_id,
                example_index=example_index,
                tokens=tokens,
                token_to_orig_map=token_to_orig_map,
                input_ids=input_ids,
                input_mask=input_mask,
                segment_ids=segment_ids,
                start_positions=start_positions,
                end_positions=end_positions,
                start_indexes=start_indexes,
                end_indexes=end_indexes,
                bio_labels=bio_labels,
                polarity_positions=polarity_positions,
                polarity_labels=polarity_labels,
                label_masks=label_masks))
        unique_id += 1
    logger.info("Max sentence length: {}".format(max_sent_length))
    logger.info("Max term length: {}".format(max_term_length))
    logger.info("Max term num: {}".format(max_term_num))
    return features

RawSpanResult = collections.namedtuple("RawSpanResult",
                                       ["unique_id", "start_logits", "end_logits"])

RawSpanCollapsedResult = collections.namedtuple("RawSpanCollapsedResult",
                                       ["unique_id", "neu_start_logits", "neu_end_logits", "pos_start_logits", "pos_end_logits",
                                        "neg_start_logits", "neg_end_logits"])

RawBIOResult = collections.namedtuple("RawBIOResult", ["unique_id", "bio_pred"])

RawBIOClsResult = collections.namedtuple("RawBIOClsResult", ["unique_id", "start_indexes", "end_indexes", "bio_pred", "span_masks"])

RawFinalResult = collections.namedtuple("RawFinalResult",
                                        ["unique_id", "start_indexes", "end_indexes", "cls_pred", "span_masks"])

def wrapped_get_final_text(example, feature, start_index, end_index, do_lower_case, verbose_logging, logger):
    tok_tokens = feature.tokens[start_index:(end_index + 1)]
    orig_doc_start = feature.token_to_orig_map[start_index]
    orig_doc_end = feature.token_to_orig_map[end_index]
    orig_tokens = example.sent_tokens[orig_doc_start:(orig_doc_end + 1)]
    tok_text = " ".join(tok_tokens)

    # De-tokenize WordPieces that have been split off.
    tok_text = tok_text.replace(" ##", "")
    tok_text = tok_text.replace("##", "")

    # Clean whitespace
    tok_text = tok_text.strip()
    tok_text = " ".join(tok_text.split())
    orig_text = " ".join(orig_tokens)

    final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging, logger)
    return final_text

def span_annotate_candidates(all_examples, batch_features, batch_results, filter_type, is_training, use_heuristics, use_nms,
                             logit_threshold, n_best_size, max_answer_length, do_lower_case, verbose_logging, logger):
    """Annotate top-k candidate answers into features."""
    unique_id_to_result = {}
    for result in batch_results:
        unique_id_to_result[result.unique_id] = result

    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
        "PrelimPrediction",
        ["feature_index", "start_index", "end_index", "start_logit", "end_logit"])

    batch_span_starts, batch_span_ends, batch_labels, batch_label_masks = [], [], [], []
    for (feature_index, feature) in enumerate(batch_features):
        example = all_examples[feature.example_index]
        result = unique_id_to_result[feature.unique_id]

        seen_predictions = {}
        span_starts, span_ends, labels, label_masks = [], [], [], []
        if is_training:
            # add ground-truth terms
            for start_index, end_index, polarity_label, mask in \
                    zip(feature.start_indexes, feature.end_indexes, feature.polarity_labels, feature.label_masks):
                if mask and start_index in feature.token_to_orig_map and end_index in feature.token_to_orig_map:
                    final_text = wrapped_get_final_text(example, feature, start_index, end_index,
                                                        do_lower_case, verbose_logging, logger)
                    if final_text in seen_predictions:
                        continue
                    seen_predictions[final_text] = True

                    span_starts.append(start_index)
                    span_ends.append(end_index)
                    labels.append(polarity_label)
                    label_masks.append(1)
        else:
            prelim_predictions_per_feature = []
            start_indexes = _get_best_indexes(result.start_logits, n_best_size)
            end_indexes = _get_best_indexes(result.end_logits, n_best_size)
            for start_index in start_indexes:
                for end_index in end_indexes:
                    # We could hypothetically create invalid predictions, e.g., predict
                    # that the start of the span is in the question. We throw out all
                    # invalid predictions.
                    if start_index >= len(feature.tokens):
                        continue
                    if end_index >= len(feature.tokens):
                        continue
                    if start_index not in feature.token_to_orig_map:
                        continue
                    if end_index not in feature.token_to_orig_map:
                        continue
                    if end_index < start_index:
                        continue
                    length = end_index - start_index + 1
                    if length > max_answer_length:
                        continue
                    start_logit = result.start_logits[start_index]
                    end_logit = result.end_logits[end_index]
                    if start_logit + end_logit < logit_threshold:
                        continue

                    prelim_predictions_per_feature.append(
                        _PrelimPrediction(
                            feature_index=feature_index,
                            start_index=start_index,
                            end_index=end_index,
                            start_logit=start_logit,
                            end_logit=end_logit))

            if use_heuristics:
                prelim_predictions_per_feature = sorted(
                    prelim_predictions_per_feature,
                    key=lambda x: (x.start_logit + x.end_logit - (x.end_index - x.start_index + 1)),
                    reverse=True)
            else:
                prelim_predictions_per_feature = sorted(
                    prelim_predictions_per_feature,
                    key=lambda x: (x.start_logit + x.end_logit),
                    reverse=True)

            for i, pred_i in enumerate(prelim_predictions_per_feature):
                if len(span_starts) >= int(n_best_size)/2:
                    break
                final_text = wrapped_get_final_text(example, feature, pred_i.start_index, pred_i.end_index,
                                                    do_lower_case, verbose_logging, logger)
                if final_text in seen_predictions:
                    continue
                seen_predictions[final_text] = True

                span_starts.append(pred_i.start_index)
                span_ends.append(pred_i.end_index)
                labels.append(0)
                label_masks.append(1)

                # filter out redundant candidates
                if (i+1) < len(prelim_predictions_per_feature) and use_nms:
                    indexes = []
                    for j, pred_j in enumerate(prelim_predictions_per_feature[(i+1):]):
                        filter_text = wrapped_get_final_text(example, feature, pred_j.start_index, pred_j.end_index,
                                                             do_lower_case, verbose_logging, logger)
                        if filter_type == 'em':
                            if exact_match_score(final_text, filter_text):
                                indexes.append(i + j + 1)
                        elif filter_type == 'f1':
                            if f1_score(final_text, filter_text) > 0:
                                indexes.append(i + j + 1)
                        else:
                            raise Exception
                    [prelim_predictions_per_feature.pop(index - k) for k, index in enumerate(indexes)]

        # Pad to fixed length
        while len(span_starts) < int(n_best_size):
            span_starts.append(0)
            span_ends.append(0)
            labels.append(0)
            label_masks.append(0)
        assert len(span_starts) == int(n_best_size)
        assert len(span_ends) == int(n_best_size)
        assert len(labels) == int(n_best_size)
        assert len(label_masks) == int(n_best_size)

        batch_span_starts.append(span_starts)
        batch_span_ends.append(span_ends)
        batch_labels.append(labels)
        batch_label_masks.append(label_masks)
    return batch_span_starts, batch_span_ends, batch_labels, batch_label_masks

def ts2start_end(ts_tag_sequence):
    starts, ends = [], []
    n_tag = len(ts_tag_sequence)
    prev_pos, prev_sentiment = '$$$', '$$$'
    tag_on = False
    for i in range(n_tag):
        cur_ts_tag = ts_tag_sequence[i]
        if cur_ts_tag != 'O':
            cur_pos, cur_sentiment = cur_ts_tag.split('-')
        else:
            cur_pos, cur_sentiment = 'O', '$$$'
        assert cur_pos == 'O' or cur_pos == 'T'
        if cur_pos == 'T':
            if prev_pos != 'T':
                # cur tag is at the beginning of the opinion target
                starts.append(i)
                tag_on = True
            else:
                if cur_sentiment != prev_sentiment:
                    # prev sentiment is not equal to current sentiment
                    ends.append(i - 1)
                    starts.append(i)
                    tag_on = True
        else:
            if prev_pos == 'T':
                ends.append(i - 1)
                tag_on = False
        prev_pos = cur_pos
        prev_sentiment = cur_sentiment
    if tag_on:
        ends.append(n_tag-1)
    assert len(starts) == len(ends), (len(starts), len(ends), ts_tag_sequence)
    return starts, ends

def ts2polarity(words, ts_tag_sequence, starts, ends):
    polarities = []
    for start, end in zip(starts, ends):
        cur_ts_tag = ts_tag_sequence[start]
        cur_pos, cur_sentiment = cur_ts_tag.split('-')
        assert cur_pos == 'T'
        prev_sentiment = cur_sentiment
        if start < end:
            for idx in range(start, end + 1):
                cur_ts_tag = ts_tag_sequence[idx]
                cur_pos, cur_sentiment = cur_ts_tag.split('-')
                assert cur_pos == 'T'
                assert cur_sentiment == prev_sentiment, (words, ts_tag_sequence, start, end)
                prev_sentiment = cur_sentiment
        polarities.append(cur_sentiment)
    return polarities

def pos2term(words, starts, ends):
    term_texts = []
    for start, end in zip(starts, ends):
        term_texts.append(' '.join(words[start:end+1]))
    return term_texts

def convert_absa_data(dataset, verbose_logging=False):
    examples = []
    n_records = len(dataset)
    for i in range(n_records):
        words = dataset[i]['words']
        ts_tags = dataset[i]['ts_raw_tags']
        starts, ends = ts2start_end(ts_tags)
        polarities = ts2polarity(words, ts_tags, starts, ends)
        term_texts = pos2term(words, starts, ends)

        if term_texts != []:
            new_polarities = []
            for polarity in polarities:
                if polarity == 'POS':
                    new_polarities.append('positive')
                elif polarity == 'NEG':
                    new_polarities.append('negative')
                elif polarity == 'NEU':
                    new_polarities.append('neutral')
                else:
                    raise Exception
            assert len(term_texts) == len(starts)
            assert len(term_texts) == len(new_polarities)
            example = SemEvalExample(str(i), words, term_texts, starts, ends, new_polarities)
            examples.append(example)
            if i < 50 and verbose_logging:
                print(example)
    print("Convert %s examples" % len(examples))
    return examples

def read_absa_data(path):
    """
    read data from the specified path
    :param path: path of dataset
    :return:
    """
    dataset = []
    with open(path, encoding='UTF-8') as fp:
        for line in fp:
            record = {}
            sent, tag_string = line.strip().split('####')
            record['sentence'] = sent
            word_tag_pairs = tag_string.split(' ')
            # tag sequence for targeted sentiment
            ts_tags = []
            # tag sequence for opinion target extraction
            ote_tags = []
            # word sequence
            words = []
            for item in word_tag_pairs:
                # valid label is: O, T-POS, T-NEG, T-NEU
                eles = item.split('=')
                if len(eles) == 2:
                    word, tag = eles
                elif len(eles) > 2:
                    tag = eles[-1]
                    word = (len(eles) - 2) * "="
                words.append(word.lower())
                if tag == 'O':
                    ote_tags.append('O')
                    ts_tags.append('O')
                elif tag == 'T-POS':
                    ote_tags.append('T')
                    ts_tags.append('T-POS')
                elif tag == 'T-NEG':
                    ote_tags.append('T')
                    ts_tags.append('T-NEG')
                elif tag == 'T-NEU':
                    ote_tags.append('T')
                    ts_tags.append('T-NEU')
                else:
                    raise Exception('Invalid tag %s!!!' % tag)
            record['words'] = words.copy()
            record['ote_raw_tags'] = ote_tags.copy()
            record['ts_raw_tags'] = ts_tags.copy()
            dataset.append(record)
    print("Obtain %s records from %s" % (len(dataset), path))
    return dataset

ext: py

reason: import numpy


text:
import sys
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import export_graphviz
import pydot
import json

features = pd.read_csv("dataset2.csv")

labels = np.array(features['hired'])
features= features.drop('hired', axis = 1)
feature_list = list(features.columns)
features = np.array(features)
train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 0.25, random_state = 42)

rf = RandomForestRegressor(n_estimators = 1000, random_state = 42)
rf.fit(train_features, train_labels);

arr = json.loads(sys.argv[1])

predictions = rf.predict(arr)
print(predictions.tolist())
# errors = abs(predictions - test_labels)

# print('Mean Absolute Error:', round(np.mean(errors), 2), 'degrees.')
#
# tree = rf.estimators_[5]
# export_graphviz(tree, out_file = 'tree.dot', feature_names = feature_list, rounded = True, precision = 1)
# (graph, ) = pydot.graph_from_dot_file('tree.dot')
# graph.write_png('tree.png')
#
# importances = list(rf.feature_importances_)
# feature_importances = [(feature, round(importance, 2)) for feature, importance in zip(feature_list, importances)]
# feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)
# [print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importances];


text:
import os

import numpy as np
import torch

from pbrl.policy.policy import BasePolicy

class Agent:
    def __init__(self, policy: BasePolicy):
        self.policy = policy
        self.states_actor = None

    def reset(self):
        if self.policy.rnn:
            self.states_actor = None

    def load_from_dict(self, agent_dict):
        # load weights
        self.policy.actor.load_state_dict(agent_dict['actor'])
        # load RunningMeanStd
        if self.policy.obs_norm:
            self.policy.rms_obs.load(agent_dict['rms_obs'])
        self.policy.actor.eval()
        self.reset()

    def load_from_dir(self, filename_policy):
        if filename_policy is not None and os.path.exists(filename_policy):
            pkl = torch.load(filename_policy, map_location=self.policy.device)
            self.load_from_dict(pkl)

    def step(self, observations: np.ndarray) -> np.ndarray:
        actions, self.states_actor = self.policy.act(observations, self.states_actor)
        actions = self.policy.wrap_actions(actions)
        return actions

max_stars_repo_name: jjccero/pbrl

max_issues_repo_name: jjccero/pbrl

max_forks_repo_name: jjccero/pbrl


text:
from __future__ import division
from __future__ import print_function

import time
import tensorflow as tf

from utils import *
from models import GCNN
from tensorflow import set_random_seed
import matplotlib.pyplot as plt
import scipy.io as sio
import scipy
from scipy.sparse import csr_matrix, lil_matrix
import numpy as np

def myaccalc(pred,yhat):
    return np.sum(np.argmax(pred,1)==np.argmax(yhat,1))

# random seed for reproducability
seed = 32
np.random.seed(seed)
tf.set_random_seed(seed)

# Settings
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('epochs', 500, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 320, 'Number of units in hidden graph conv layer 1.')
flags.DEFINE_integer('hidden2', 100, 'Number of units in hidden graph conv layer 2.')
flags.DEFINE_integer('dense', 100, 'Number of units in hidden dense layer.')
flags.DEFINE_float('dropout', 0.10, 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('weight_decay', 0.0, 'Weight for L2 loss on embedding matrix.')
flags.DEFINE_integer('nkernel', 3, 'number of kernels')

nkernel=flags.FLAGS.nkernel
# how many times do you want to update parameters over one epoch. batchsize=trainsize/bsize
bsize=3

# read data
a=sio.loadmat('enzymes.mat')
# list of adjacency matrix
A=a['A'][0]
# list of features
F=a['F'][0]
# label of graphs
Y=a['Y'][0]
# test train index for 10-fold test
TRid=a['tr']
TSid=a['ts']

# max number of nodes
nmax=0
for i in range(0,len(A)):
    nmax=max(nmax,A[i].shape[0])

# number of node per graph
ND=np.zeros((len(A),1))
# node feature matrix
FF=np.zeros((len(A),nmax,3))
# one-hot coding output matrix
YY=np.zeros((len(A),6))
# Convolution kernels, supports
SP=np.zeros((len(A),nkernel,nmax,nmax))

# prepare inputs, outputs, convolution kernels for each graph
for i in range(0,len(A)):
    # number of node in graph
    n=F[i].shape[0]
    ND[i,0]=n

    # feature matrix
    FF[i,0:n,:]= F[i]

    # one-hot coding output matrix
    YY[i,Y[i]]=1

    # set kernels
    chebnet = chebyshev_polynomials(A[i], nkernel-1)
    for j in range(0,nkernel):
        SP[i,j,0:n,0:n]=chebnet[j].toarray()

    ## GCN convolution kernel
    #gcn= (normalize_adj(A[i] + sp.eye(A[i].shape[0]))).toarray()
    #SP[i,0,0:n,0:n]=gcn

    ## MLP convolution kernel
    #mlp=np.eye(n)
    #SP[i,0,0:n,0:n]=gcn

    ## A and I convolution kernel
    # SP[i,0,0:n,0:n]=np.eye(n)
    # SP[i,1,0:n,0:n]=A[i]

NB=np.zeros((FLAGS.epochs,10))

for fold in range(0,10):

    # train and test ids
    trid=TRid[fold]
    tsid=TSid[fold]

    placeholders = {
            'support': tf.placeholder(tf.float32, shape=(None,nkernel,nmax,nmax)),
            'features': tf.placeholder(tf.float32, shape=(None,nmax, FF.shape[2])),
            'labels': tf.placeholder(tf.float32, shape=(None, 6)),
            'nnodes': tf.placeholder(tf.float32, shape=(None, 1)),
            'dropout': tf.placeholder_with_default(0., shape=()),
    }

    model = GCNN(placeholders, input_dim=FF.shape[2],nkernel=nkernel,logging=True,agg='mean')

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    # train data placeholders
    feed_dict = dict()
    feed_dict.update({placeholders['labels']: YY[trid,:]})
    feed_dict.update({placeholders['features']: FF[trid,:,:]})
    feed_dict.update({placeholders['support']: SP[trid,:,:,:]})
    feed_dict.update({placeholders['nnodes']: ND[trid,]})
    feed_dict.update({placeholders['dropout']: FLAGS.dropout})

    # test data placeholders
    feed_dictT = dict()
    feed_dictT.update({placeholders['labels']: YY[tsid,:]})
    feed_dictT.update({placeholders['features']: FF[tsid,:,:]})
    feed_dictT.update({placeholders['support']: SP[tsid,:,:,:]})
    feed_dictT.update({placeholders['nnodes']: ND[tsid,]})
    feed_dictT.update({placeholders['dropout']: 0})

    ind=np.round(np.linspace(0,len(trid),bsize+1))

    for epoch in range(FLAGS.epochs):

        np.random.shuffle(trid)
        for i in range(0,bsize): # batch training
            feed_dictB = dict()
            bid=trid[int(ind[i]):int(ind[i+1])]
            feed_dictB.update({placeholders['labels']: YY[bid,:]})
            feed_dictB.update({placeholders['features']: FF[bid,:,:]})
            feed_dictB.update({placeholders['support']: SP[bid,:,:,:]})
            feed_dictB.update({placeholders['nnodes']: ND[bid,]})
            feed_dictB.update({placeholders['dropout']: FLAGS.dropout})
            # train for batch data
            outs = sess.run([model.opt_op], feed_dict=feed_dictB)

        # check performance for all train sample
        outs = sess.run([model.accuracy, model.loss, model.entropy,model.outputs], feed_dict=feed_dict)
        # check performance for all test sample
        outsT = sess.run([model.accuracy, model.loss, model.entropy,model.outputs], feed_dict=feed_dictT)

        # number of true classified test graph
        vtest=myaccalc(outsT[3],YY[tsid,:])

        NB[epoch,fold]=vtest
        if np.mod(epoch + 1,1)==0 or epoch==0:
            print(fold," Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(outs[1]),"train_xent=", "{:.5f}".format(outs[2]),"train_acc=", "{:.5f}".format(outs[0]),"test_loss=", "{:.5f}".format(outsT[1]),
            "test_xent=", "{:.5f}".format(outsT[2]), "test_acc=", "{:.5f}".format(outsT[0]), " ntrue=", "{:.0f}".format(vtest))

import pandas as pd
pd.DataFrame(NB).to_csv('testresultsoverepoch.csv')

max_stars_repo_name: dongmengshi/easylearn

max_issues_repo_name: dongmengshi/easylearn

max_forks_repo_name: dongmengshi/easylearn

reason: import numpy,import scipy,from scipy


text:
import os
import string
import sys
import logging
from optparse import OptionParser

import numpy as np
import pandas
import scipy.spatial.distance
import scipy.stats
from numpy.linalg import matrix_rank
from scipy.spatial import distance
from sklearn.decomposition import NMF
from sklearn.preprocessing import Normalizer, LabelEncoder
from sklearn.utils import resample
from sklearn.neighbors import KNeighborsClassifier

import alearn_utils as utils
import clusterings as elCluster
import mf_impute as mfi
import fancy_impute as fci

# Parameter parsing
#############################################################################################################
argvs = sys.argv
opts, args = {}, []
op = OptionParser()
op.add_option(
    '-c',
    '--cluster',
    action='store',
    type='string',
    dest='cluster',
    help='indicate the ID of the job')
op.add_option(
    '-d',
    '--date',
    action='store',
    type='string',
    dest='date',
    help='indicate the time of the job')
op.add_option(
    '-l',
    '--linkage',
    default='predictround',
    action='store',
    type='string',
    dest='linkage',
    help='linkage of clustering, since no clustering is using, we use predictions')
op.add_option(
    '-b',
    '--batchsize',
    default=-1,
    action='store',
    type='int',
    dest='batchsize',
    help='batchsize of clustering metric, if -1, the number is pre-determinined')
op.add_option(
    '-o',
    '--outputpath',
    default='apreds',
    action='store',
    type='string',
    dest='outputpath',
    help='Output path, please specify one before running')
op.add_option(
    '-p',
    '--impute',
    default='svt',
    action='store',
    type='string',
    dest='impute',
    help='Impute method, the matrix completioin method')
op.add_option(
    '-n',
    '--condition',
    default=96,
    action='store',
    type='int',
    dest='condition',
    help='Condition size of the problem,')
op.add_option(
    '-t',
    '--target',
    default=96,
    action='store',
    type='int',
    dest='target',
    help='Target size of the problem,')
op.add_option(
    '-r',
    '--rounds',
    default=30,
    action='store',
    type='int',
    dest='rounds',
    help='Iterations of the learning procedure')
op.add_option(
    '-y',
    '--phenotypes',
    default=-1,
    action='store',
    type='int',
    dest='phenotypes',
    help='Number of max phenotypes')
op.add_option(
    '-s',
    '--select',
    default='margin',
    action='store',
    type='string',
    dest='select',
    help='Selection querying method of active learning')
op.add_option(
    '-f',
    '--info',
    default='NA',
    action='store',
    type='string',
    dest='info',
    help='Information to print in the heading to record some important marks')

(opts, args) = op.parse_args()

if len(args) > 0:
    op.print_help()
    op.error('Please input options instead of arguments.')
    exit(1)

NUM_TAR = opts.condition
NUM_CON = opts.target
BATCHSIZE = opts.batchsize
DUPLICATE = False
ROUNDS = opts.rounds
IMPUTE = opts.impute
LINKAGE = opts.linkage
OPATH = opts.outputpath
SELECT = opts.select
NUM_PHE = opts.phenotypes

logging.basicConfig(level=logging.INFO)
logging.info(opts)

# Parameter parsing ends
#############################################################################################################

# Data preprocessing
#############################################################################################################
data_idx = np.load('data/calculated/indices.npy')
#raw_data = np.load('data/calculated/rawdata.npy')
zsc_data = np.load('data/calculated/zscored_data.npy')
#pdata_idx = np.load('data/calculated/indices_posthoc.npy')
#pzsc_data = np.load('data/calculated/zscored_data_posthoc.npy')

# Determine number of batch and pheno types
batchsize_list = np.full(ROUNDS, BATCHSIZE)
phenonum_list = np.full(ROUNDS, NUM_PHE)
if (BATCHSIZE == -1):
    batchsize_list = np.load('batchsize.npy')
if (NUM_PHE == -1):
    phenonum_list = np.load('phenotypenums.npy')

# If smaller space
if(NUM_TAR == 48 and NUM_CON == 48):
    data_idx = (data_idx - 1) % 48 + 1

# Generate data frame
zdata_with_idx = np.hstack((data_idx, zsc_data))
zdata_with_idx[:, :2]
col_name = ['condition', 'target']
featurename = ['ft_' + str(i) for i in range(zsc_data.shape[1])]
col_name.extend(featurename)
zdata_df = pandas.DataFrame(zdata_with_idx, columns=col_name)
zdata_df = zdata_df[(zdata_df.target <= NUM_TAR) &
                    (zdata_df.condition <= NUM_CON)]

# Get avaliable space
full_set = set()
for c in range(1, NUM_CON + 1):
    for t in range(1, NUM_TAR + 1):
        full_set.add((c, t))

ava_exps = utils.get_exp_from_df(zdata_df)
na_exps = full_set - ava_exps
logging.info("unavaliable experiments: {0}".format(len(na_exps)))
logging.info("avaliable experiments: {0}".format(
    len(utils.get_exp_from_df(zdata_df))))

# Mapping true for duplicated usage.
exp_map = {}
for item in ava_exps:
    c, t = (item[0] - 1) % 48 + 1, (item[1] - 1) % 48 + 1
    exp_map[(c, t)] = (item[0], item[1])

nmzsc_data = zsc_data
nmzsc_idx = np.hstack((data_idx, nmzsc_data))
nmzsc_df = pandas.DataFrame(nmzsc_idx, columns=col_name)
nmzsc_df = nmzsc_df[(nmzsc_df.target <= NUM_TAR) &
                    (nmzsc_df.condition <= NUM_CON)]

# Data preprocessing ends
#############################################################################################################

# Initial round to select the control group
#############################################################################################################

# Get true experimnets form pool
exped_df = nmzsc_df[(nmzsc_df.condition == 48)]
exped_set = utils.get_exp_from_df(exped_df)

# Add in 0906 predict round method that build precition using supervised model in each round
if(LINKAGE == 'predictround'):
    ap_df = pandas.read_csv("data/elap/bear_round" +
                            str(1) + ".apredictions", sep=" ", header=None)
    ap_df.columns = ['condition', 'target', 'phenotype1',
                     'phenotype2', 'observed', 'frontier']
    ap_df = ap_df.add([0, 0, 1, 1, 0, 0], axis=1)
    ap_ob1_df = pandas.DataFrame(ap_df.loc[ap_df.observed == 1])
    con48_r1_phe = ap_ob1_df.loc[ap_ob1_df.condition == 48]
    exped_lines = con48_r1_phe.iloc[:, :2].as_matrix()
    ini_exped_labels = con48_r1_phe.phenotype1.as_matrix()

exp_pheno_df = pandas.DataFrame(
    exped_lines, columns=[['condition', 'target']])
exp_pheno_df['phenotype'] = ini_exped_labels
data_df = exp_pheno_df.astype(int)

# Initial predictive model
for item in full_set - exped_set:
    tup = exp_pheno_df[exp_pheno_df.target == item[1]].as_matrix()
    tup[0, 0] = item[0]
    new_row = pandas.DataFrame(
        tup, columns=['condition', 'target', 'phenotype'])
    data_df = data_df.append(new_row)

if(DUPLICATE == True):
    ini_imp = utils.gen_first_batch(full_set - exped_set, batchsize_list[1])
else:
    ini_imp = utils.gen_first_batch(ava_exps - exped_set, batchsize_list[1])

# Initial report
apred_1 = utils.get_aprediction(data_df, data_df.phenotype, exped_set, ini_imp)
apred_1.to_csv(OPATH + '/bear_round' + str(1) +
               '.apredictions', header=False, index=False, sep=' ')

# Initial round ends
#############################################################################################################

# Active learning processing methods
#############################################################################################################

def update_imputed_frame(data_df, exped_df, exped_set, impute_set, which_round, method='three_point', duplicate=False):

    missing_flag = 0
    # Load data from the pool
    for item in (impute_set):

        condition = item[0]
        target = item[1]
        new_exps = nmzsc_df[(nmzsc_df.condition == condition)
                            & (nmzsc_df.target == target)]
        # Append true experiment to the data matrix check avaliability of experiments

        # Experiment avaliable
        if(len(new_exps) > 0):
            exped_df = exped_df.append(new_exps)
            exped_set.add(item)
        #
        elif ((duplicate == True) and (item in na_exps)):
            missing_flag += 1
            d_condition, d_target = exp_map[(
                (condition - 1) % 48 + 1, (target - 1) % 48 + 1)]
            new_exps = pandas.DataFrame(
                nmzsc_df[(nmzsc_df.condition == d_condition) & (nmzsc_df.target == d_target)])
            new_exps.condition = np.full(len(new_exps), condition)
            new_exps.target = np.full(len(new_exps), target)
            exped_df = exped_df.append(new_exps)
            exped_set.add(item)

    logging.warning(
        "{0} expriment is not avaliable in this batch".format(missing_flag))

    # To study the data from their files
    if(LINKAGE == 'predictround'):

        # The experimenrs we want to perform
        query_ary = np.array(list(exped_set))
        query_df = pandas.DataFrame(
            query_ary, columns=[['condition', 'target']])

        # Load the active prediction from the elife paper
        elround_df = pandas.read_csv(
            'data/elap/bear_round' + str(which_round + 1) + '.apredictions', sep=' ', header=None)
        elround_df = elround_df.add([0, 0, 1, 1, 0, 0], axis=1)
        elround_df.columns = ['condition', 'target',
                              'phenotype1', 'phenotype2', 'observed', 'frontier']

        # Construct classifier and the label encoder
        le = LabelEncoder()
        knC = KNeighborsClassifier(
            n_neighbors=15, metric="euclidean", weights='distance', n_jobs=8)

        clustering = elCluster.SupervisedEL(
            classifier=knC, label_encoder=le, query_df=query_df, el_roundap=elround_df)
        exped_lines, labels = clustering.fit_predict(data=nmzsc_df)

    # Add in 0906

    logging.warning(
        "{0} number of clusters are generated".format(np.max(labels)))

    #exped_lines = np.array(list(clustering.node_to_line.values()))
    data_df = pandas.DataFrame(exped_lines, columns=[['condition', 'target']])
    data_df['phenotype'] = labels

    var_mat = np.full((NUM_TAR, NUM_CON), -1)

    if(method == 'svt'):

        # Generate the phenotype matrix and phenotype list
        phenotype_list = np.array(data_df.drop_duplicates(
            subset=['phenotype']).phenotype.as_matrix(), dtype=int)
        phenotype_matrix = np.full((NUM_CON, NUM_TAR), 0)

        for item in exped_set:
            c, t = item[0], item[1]
            phe_for_ct = int(
                data_df.loc[(data_df.condition == c) & (data_df.target == t)].phenotype)
            phenotype_matrix[t - 1, c - 1] = phe_for_ct

        if(method == 'svt'):
            prediction, score_stack, correct_count = fci.svt_impute(
                phenotype_matrix, phenotype_list, phenotype_matrix.shape, dummy_na=0.5, mat_rank=-1)

        logging.info("The training accurarcy of the matrix factorization is {0}".format(
            correct_count / len(exped_set)))

        var_mat = np.zeros((NUM_TAR, NUM_CON))
        # Generate the score using margin uncertanty
        if(SELECT == 'margin'):
            var_mat = mfi.min_margin_score(score_stack)

        elif (SELECT == 'least'):
            var_mat = mfi.least_confidence_score(score_stack)
            # Decline the experimened
        var_mat = var_mat - 2 * (phenotype_matrix > 0)

        if(duplicate == False):
            for item in na_exps:
                c, t = item[0], item[1]
                var_mat[t - 1, c - 1] = -1

            # Generate the prediction based on the imputed matrix
        c_t = np.array(np.unravel_index(
            list(range(0, NUM_CON * NUM_TAR)), (NUM_CON, NUM_TAR))) + 1
        pheno_pred_list = prediction.T.ravel()
        data_array = np.append(
            c_t.T, pheno_pred_list.reshape(-1, 1), axis=1)
        data_df = pandas.DataFrame(
            data_array, columns=['condition', 'target', 'phenotype'])

    return data_df, exped_df, exped_set, var_mat

def per_round_impure_labels(data_df, exped_df, exped_set, imp_set, which_round):

    data_df, exped_df, exped_set, var_mat = update_imputed_frame(
        data_df, exped_df, exped_set, imp_set, which_round, method=IMPUTE, duplicate=DUPLICATE)

    return data_df, exped_df, exped_set, var_mat

# Active learning processing methods ends
#############################################################################################################

# Active learning processing rounds 2 - finish
#############################################################################################################
labels = np.array([])
frointer_set = ini_imp
for i in range(1, ROUNDS):

    logging.info("Round {0}".format(i + 1))

    data_df, exped_df, exped_set, var_matrix_next = per_round_impure_labels(
        data_df, exped_df, exped_set, frointer_set, i)

    logging.info("{0} experiment is observed".format(len(exped_set)))

    batch = batchsize_list[i] if i < 30 else 96

    logging.info("{0} experiment is selected as frointer".format(batch))

    if((var_matrix_next >= 0).sum() >= batch):
        logging.info('Use active learning selection')
        frointer_set = utils.selection(var_matrix_next, batch)
        logging.info('The higest score: {0}'.format(var_matrix_next.max()))

    elif (i + 1) < 30:
        logging.info(
            'Use active learning selecion is not enough, use random sampling')
        frointer_set = utils.random_gen(
            ava_exps - exped_set, batch_size=batch)

    else:
        logging.info('Final round we dont have enough data so random selections')
        frointer_set = utils.random_gen(full_set - exped_set, batch_size=batch)

    apred_df = utils.get_aprediction(
        data_df, data_df.phenotype, exped_set, frointer_set)

    apred_df.to_csv(OPATH + '/bear_round' + str(i + 1) +
                    '.apredictions', header=False, index=False, sep=' ')
# Active learning processing rounds 2 - ends
#############################################################################################################

max_stars_repo_name: WINGHARE/Actmatcomp

max_issues_repo_name: WINGHARE/Actmatcomp

max_forks_repo_name: WINGHARE/Actmatcomp

reason: import numpy,from numpy,import scipy,from scipy


text:
import numpy as np
from numpy import printoptions, random
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
from sklearn.neural_network import MLPClassifier

import matplotlib.pyplot as plt
import csv

TRAIN_PATH = './Processed_Data/one_hot_Data/training_one_hot.csv'
TEST_PATH = './Processed_Data/one_hot_Data/test_one_hot.csv'

def load_training_data():
    X = []
    y = []

    with open(TRAIN_PATH, 'r') as f:
        f_csv = csv.reader(f)
        next(f_csv)
        for line in f_csv:
            X.append(line[:-1])
            y.append(line[-1])

    return (X, y)

def load_test_data():
    res = []
    with open(TEST_PATH, 'r') as f:
        f_csv = csv.reader(f)
        next(f_csv)
        for line in f_csv:
            res.append(line)
    return res

X, y = load_training_data()

hidden_num = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
nureon_num = [150, 200, 300, 450]
score_lst = []

for i in range(10):
    hidden = hidden_num[i]
    clf = MLPClassifier(random_state= 400,  hidden_layer_sizes= hidden, )
    scores = cross_val_score(clf, X, y, cv=10)
    score_lst.append(scores.mean())
    print(scores.mean())

plt.plot(hidden_num, score_lst)
plt.xlabel('hidden_layer_sizes')
plt.ylabel('Accuracy')
plt.show()

# print(score_lst)

# clf = MLPClassifier(random_state= 400,  hidden_layer_sizes= 4)
# clf.fit(X, y)

# test_data = load_test_data()
# predictions = clf.predict(test_data)

# # store into the result
# pre_res = [['ID', 'Prediction']]
# for index, value in enumerate(predictions):
#     pre_res.append([index+1, value])

# with open('./Results/nn_.csv', 'w') as f:
#     csv_f = csv.writer(f, lineterminator='\n')
#     csv_f.writerows(pre_res)

max_stars_repo_path: Final Project/NN.py

max_stars_repo_name: DesirY/Machine-Learning-Library

max_issues_repo_path: Final Project/NN.py

max_issues_repo_name: DesirY/Machine-Learning-Library

max_forks_repo_path: Final Project/NN.py

max_forks_repo_name: DesirY/Machine-Learning-Library

reason: import numpy,from numpy


text:
"""
Metrics for off-policy evaluation.
"""
from envs.d4rl.d4rl_content import infos
import numpy as np

UNDISCOUNTED_POLICY_RETURNS = {
    'halfcheetah-medium' : 3985.8150261686337,
    'halfcheetah-random' : -199.26067391425954,
    'halfcheetah-expert' : 12330.945945279545,
    'hopper-medium' : 2260.1983114487352,
    'hopper-random' : 1257.9757846810203,
    'hopper-expert' : 3624.4696022560997,
    'walker2d-medium' : 2760.3310101980005,
    'walker2d-random' : 896.4751989935487,
    'walker2d-expert' : 4005.89370727539,
}

DISCOUNTED_POLICY_RETURNS = {
    'halfcheetah-medium' : 324.83583782709877,
    'halfcheetah-random' : -16.836944753939207,
    'halfcheetah-expert' : 827.7278887047698,
    'hopper-medium' : 235.7441494727478,
    'hopper-random' : 215.04955086664955,
    'hopper-expert' : 271.6925087260701,
    'walker2d-medium' : 202.23983424823822,
    'walker2d-random' : 78.46052021427765,
    'walker2d-expert' : 396.8752247768766
}

def get_returns(policy_id, discounted=False):
    if discounted:
        return DISCOUNTED_POLICY_RETURNS[policy_id]
    return UNDISCOUNTED_POLICY_RETURNS[policy_id]

def normalize(policy_id, score):
    key = policy_id + '-v0'
    min_score = infos.REF_MIN_SCORE[key]
    max_score = infos.REF_MAX_SCORE[key]
    return (score - min_score) / (max_score - min_score)

def ranking_correlation_metric(policies, discounted=False):
    """
    Computes Spearman's rank correlation coefficient.
    A score of 1.0 means the policies are ranked correctly according to their values.
    A score of -1.0 means the policies are ranked inversely.

    Args:
        policies: A list of policy string identifiers.
            Valid identifiers must be contained in POLICY_RETURNS.

    Returns:
        A correlation value between [-1, 1]
    """
    return_values = np.array([get_returns(policy_key, discounted=discounted) for policy_key in policies])
    ranks = np.argsort(-return_values)
    N = len(policies)
    diff = ranks - np.arange(N)
    return 1.0 - (6 * np.sum(diff ** 2)) / (N * (N**2 - 1))

def precision_at_k_metric(policies, k=1, n_rel=None, discounted=False):
    """
    Computes precision@k.

    Args:
        policies: A list of policy string identifiers.
        k (int): Number of top items.
        n_rel (int): Number of relevant items. Default is k.

    Returns:
        Fraction of top k policies in the top n_rel of the true rankings.
    """
    assert len(policies) >= k
    if n_rel is None:
        n_rel = k
    top_k = sorted(policies, reverse=True, key=lambda x: get_returns(x, discounted=discounted))[:n_rel]
    policy_k = policies[:k]
    score = sum([policy in top_k for policy in policy_k])
    return float(score) / k

def recall_at_k_metric(policies, k=1, n_rel=None, discounted=False):
    """
    Computes recall@k.

    Args:
        policies: A list of policy string identifiers.
        k (int): Number of top items.
        n_rel (int): Number of relevant items. Default is k.

    Returns:
        Fraction of top n_rel true policy rankings in the top k of the given policies
    """
    assert len(policies) >= k
    if n_rel is None:
        n_rel = k
    top_k = sorted(policies, reverse=True, key=lambda x: get_returns(x, discounted=discounted))[:n_rel]
    policy_k = policies[:k]
    score = sum([policy in policy_k for policy in top_k])
    return float(score) / k

def value_error_metric(policy, value, discounted=False):
    """
    Returns the absolute error in estimated value.

    Args:
        policy (str): A policy string identifier.
        value (float): Estimated value
    """
    return abs(normalize(policy, value) - normalize(policy, get_returns(policy, discounted)))

def policy_regret_metric(policy, expert_policies, discounted=False):
    """
    Returns the regret of the given policy against a set of expert policies.

    Args:
        policy (str): A policy string identifier.
        expert_policies (list[str]): A list of expert policies
    Returns:
        The regret, which is value of the best expert minus the value of the policy.
    """
    best_returns = max([get_returns(policy_key, discounted=discounted) for policy_key in expert_policies])
    return normalize(policy, best_returns) - normalize(policy, get_returns(policy, discounted=discounted))

max_stars_repo_name: AliengirlLiv/babyai

max_issues_repo_name: AliengirlLiv/babyai

max_forks_repo_name: AliengirlLiv/babyai


text:
#!/usr/bin/env python3

import numpy as np
import matplotlib
from matplotlib.collections import LineCollection
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
from mpl_toolkits.mplot3d.art3d import Line3DCollection

def plot(graph, **kwargs):
    """
    Plots a 2d or 3d graph given an FEA LatticeGraph object (the output of the fea function)

    Arguments:
        graph: FEA Lattice Graph to Plot

    Named Arguments:
        figure: The figure to plot on
        cmap: color map (defaults to inferno)
        x: specify variable name for X-axis
        y: specify variable name for Y-axis
        z: specify variable name for Z-axis
        c: specify variable name for color-axis
        vertices: Default True. Whether or not to plot vertices
        edges: Default True. Whether or not to plot edges

    Returns:
        Tuple of matplotlib figure and plot

    Example:
        solution = fea(model, [x,y,z])
        solfigure, solplot = plot(solution, x='x', y='y', z='z')
        solfigure.savefix('solution_figure.svg')
    """
    figure=kwargs.get('figure',plt.figure())
    cm=kwargs.get('cmap',matplotlib.cm.get_cmap('inferno'))

    if kwargs.get('z',None) is not None:
        plot=kwargs.get('plot',figure.add_subplot(111,projection='3d'))
        plot.set_xlabel(str(kwargs.get('x','')))
        plot.set_ylabel(str(kwargs.get('y','')))
        plot.set_zlabel(str(kwargs.get('z','')))

        dname=[str(kwargs['x']),str(kwargs['y']),str(kwargs['z'])]
    else:
        plot=kwargs.get('plot',figure.add_subplot(111))
        plot.set_xlabel(str(kwargs.get('x','')))
        plot.set_ylabel(str(kwargs.get('y','')))

        dname=[str(kwargs.get('x','')),str(kwargs.get('y',''))]

    dmat=np.zeros([len(dname),graph.N])
    vnames=[str(v) for v in graph._variables]

    if kwargs.get('c',None) in vnames:
        c=None
        cmat=np.zeros(graph.N)
        cmat[vnames.index(kwargs['c'])]=1
    else:
        c=kwargs.get('c','k')
        cmat=None

    for i,n in enumerate(dname):
        dmat[i,vnames.index(str(n))]=1

    # Plot vertices (if desired)
    if kwargs.get('vertices',True):
        verts=list(graph.get_vertices(real=kwargs.get('real',True),complete=kwargs.get('complete',True)))

        if c is None:
            ccurr=[np.dot(cmat,v.point) for v in verts]
        else:
            ccurr=c

        sc=plot.scatter(*np.transpose([np.dot(dmat,v.point) for v in verts]),c=ccurr,cmap=cm)

    # Plot Edges (if desired)
    if kwargs.get('edges',True):
        edges=list(graph.get_nodes_of_level(1,real=kwargs.get('real',True),complete=kwargs.get('complete',True)))

        edge_collection=[]
        if c is None:
            ccurr=[]
        else:
            ccurr=c

        for e in edges:
            verts=list(graph.get_vertices(e,real=kwargs.get('real',True),complete=kwargs.get('complete',True)))
            if len(verts)==2:
                if c is None:
                    dat_range=np.array([np.dot(dmat,v.point) for v in verts])
                    c_range=np.array([np.dot(cmat,v.point) for v in verts])

                    steps=50

                    points=np.array([np.linspace(r[0],r[1],steps) for r in dat_range.T]).T.reshape(-1,1,len(dname))
                    segs=np.concatenate([points[:-1],points[1:]], axis=1)
                    edge_collection.extend(list(segs))

                    ccurr.extend(np.linspace(c_range[0],c_range[1],steps-1))

                else:
                    edge_collection.append([np.dot(dmat,v.point) for v in verts])

        ax = plot.axes
        if c is None:
            if len(dname)==3:
                lc=Line3DCollection(edge_collection, array=np.array(ccurr),cmap=cm)
                ax.add_collection3d(lc)
            else:
                lc=LineCollection(np.array(edge_collection), array=np.array(ccurr),cmap=cm)
                ax.add_collection(lc)
        else:
            if len(dname)==3:
                lc=Line3DCollection(np.array(edge_collection))
                ax.add_collection3d(lc)
            else:
                lc=LineCollection(np.array(edge_collection))
                ax.add_collection(lc)

    return figure,plot

# def fix_clims(f,p,bounds=None,mul=[1,1]):
#     f.show()
#     if bounds is None:
#         print([c.get_clim() for c in p.collections])
#         lims=np.array([c.get_clim() for c in p.collections])
#         bounds=[min((i for i in lims.T[0] if i is not None)),max((i for i in lims.T[1] if i is not None))]
#     bounds=np.multiply(bounds,mul)

#     for c in p.collections:
#         c.set_clim(*bounds)

#     f.colorbar(p.collections[0])
#     return bounds

__exports__ = [plot]

max_stars_repo_name: long-m-r/few

max_issues_repo_name: long-m-r/few

max_forks_repo_name: long-m-r/fea


text:
import os
import csv
import numpy as np
version = 6
n_pts = 10000
input_dir = f"/home/theresa/p/data_v{version}"
medians = []

for ff in os.listdir(input_dir):
    if ff != '.csv':
        filename = os.path.join(input_dir, ff)
        with open(filename, "r") as f:
            reader = csv.reader(f)
            dists = []
            for row in reader:
                data = [float(element) for element in row[:]]
                dist = float(row[-1])
                dists.append(dist)
            medians.append(np.median(np.array(dists)))
            # if (len(class_a + class_b)< n_pts):
            #     print("not enough points")
            #     continue
            # # make class a the smaller class
            # class_c = class_a
            # if len(class_b) < len(class_a):
            #     class_a = class_b
            #     class_b = class_c
            #
            # # sample half of each
            # class_a_n = min(len(class_a), n_pts / 2)
            # remainder = 0
            # if class_a_n < n_pts / 2:
            #     remainder = (n_pts / 2) - class_a_n
            # class_b_n = remainder + (n_pts / 2)
            # bidx = np.random.choice(len(class_b), int(class_b_n), replace=False)
            # aidx = np.random.choice(len(class_a), int(class_a_n), replace=False)
            # newa = [a  for idx, a in filter( lambda x:x[0] in aidx, list(enumerate(class_a)))]
            # newb = [b  for idx, b in filter( lambda x:x[0] in bidx, list(enumerate(class_b)))]
            # # print(len(newa + newb))
            # with open(outfile, "w") as o:
            #     reader = csv.writer(o)
            #     for row in newa + newb:
            #         reader.writerow(row)

print("Median threshold: ", np.median(np.array(medians)))


text:
"""Unit test coverage to make sure that the pybert can correctly save and load files."""

import logging
import pickle

import numpy as np
import yaml

from pybert import __version__
from pybert.pybert import PyBERT

def test_save_config_as_yaml(tmp_path):
    """Make sure that pybert can correctly generate a yaml file that can get reloaded."""
    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.yaml")
    app.save_configuration(save_file)

    assert save_file.exists()  # File was created.

    with open(save_file, "r", encoding="UTF-8") as saved_config_file:
        user_config = yaml.load(saved_config_file, Loader=yaml.Loader)
        assert user_config.version == __version__

def test_save_config_as_pickle(tmp_path):
    """Make sure that pybert can correctly generate a pickle file that can get reloaded."""
    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.pybert_cfg")
    app.save_configuration(save_file)

    assert save_file.exists()  # File was created.

    with open(save_file, "rb") as saved_config_file:
        user_config = pickle.load(saved_config_file)
        assert user_config.version == __version__

def test_save_config_as_invalid(tmp_path, caplog):
    """When given an unsupported file suffix, no file should be generated and an message logged."""
    caplog.set_level(logging.DEBUG)

    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.json")
    app.save_configuration(save_file)

    assert not save_file.exists()  # File should not have been created.
    assert "Pybert does not support this file type." in caplog.text

def test_save_results_as_pickle(tmp_path):
    """Make sure that pybert can correctly generate a waveform pickle file that can get reloaded."""
    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("results.pybert_data")
    app.save_results(save_file)

    assert save_file.exists()  # File was created.

    with open(save_file, "rb") as saved_results_file:
        results = pickle.load(saved_results_file)
        assert results.the_data.arrays

def test_load_config_from_yaml(tmp_path):
    """Make sure that pybert can correctly load a yaml file."""
    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.yaml")
    app.save_configuration(save_file)
    TEST_NUMBER_OF_BITS = 1234

    # Modify the saved yaml file.
    with open(save_file, "r", encoding="UTF-8") as saved_config_file:
        user_config = yaml.load(saved_config_file, Loader=yaml.Loader)
        user_config.nbits = TEST_NUMBER_OF_BITS  # Normally, 8000
    with open(save_file, "w", encoding="UTF-8") as saved_config_file:
        yaml.dump(user_config, saved_config_file)

    app.load_configuration(save_file)
    assert app.nbits == TEST_NUMBER_OF_BITS

def test_load_config_from_pickle(tmp_path):
    """Make sure that pybert can correctly load a pickle file."""
    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.pybert_cfg")
    app.save_configuration(save_file)
    TEST_PATTERN_LENGTH = 31

    # Modify the saved pickle file.
    with open(save_file, "rb") as saved_config_file:
        user_config = pickle.load(saved_config_file)
        user_config.pattern_len = TEST_PATTERN_LENGTH  # Normally, 127
    with open(save_file, "wb") as saved_config_file:
        pickle.dump(user_config, saved_config_file)

    app.load_configuration(save_file)
    assert app.pattern_len == TEST_PATTERN_LENGTH

def test_load_config_from_invalid(tmp_path, caplog):
    """When given an unsupported file suffix, no file should be read and an message logged."""
    caplog.set_level(logging.DEBUG)

    app = PyBERT(run_simulation=False, gui=False)
    save_file = tmp_path.joinpath("config.json")
    app.load_configuration(save_file)

    assert "Pybert does not support this file type." in caplog.text

def test_load_results_from_pickle(tmp_path, caplog):
    """Make sure that pybert can correctly load a pickle file."""
    caplog.set_level(logging.DEBUG)
    app = PyBERT(run_simulation=True, gui=False)
    save_file = tmp_path.joinpath("config.pybert_data")
    app.save_results(save_file)

    # Modify the saved pickle file.
    with open(save_file, "rb") as saved_results_file:
        user_results = pickle.load(saved_results_file)
        user_results.the_data.update_data({"chnl_h": np.array([1, 2, 3, 4])})
    with open(save_file, "wb") as saved_results_file:
        pickle.dump(user_results, saved_results_file)

    caplog.clear()
    app.load_results(save_file)
    # pybert doesn't directly reload the waveform back into the same plot.
    # instead if creates a reference plot to compare old vs. new.
    assert app.plotdata.get_data("chnl_h_ref").size == 4

max_stars_repo_name: jdpatt/PyBERT

max_issues_repo_name: jdpatt/PyBERT

max_forks_repo_name: jdpatt/PyBERT


text:
import tensorflow as tf
from tensor_flow_simplex_matrix import simplex_noise_2arg
import numpy as np
import numpy
import math

dtype = tf.float32

tag_discontinuous = False

#select = lambda a, b, c: a*b + (1-a)*c

np_perm = numpy.array([151, 160, 137, 91, 90, 15, 131, 13, 201, 95, 96, 53, 194, 233, 7, 225, 140, 36, 103, 30, 69, 142, 8, 99,
           37, 240, 21, 10, 23, 190, 6, 148, 247, 120, 234, 75, 0, 26, 197, 62, 94, 252, 219, 203, 117, 35, 11, 32,
           57, 177, 33, 88, 237, 149, 56, 87, 174, 20, 125, 136, 171, 168, 68, 175, 74, 165, 71, 134, 139, 48, 27,
           166, 77, 146, 158, 231, 83, 111, 229, 122, 60, 211, 133, 230, 220, 105, 92, 41, 55, 46, 245, 40, 244, 102,
           143, 54, 65, 25, 63, 161, 1, 216, 80, 73, 209, 76, 132, 187, 208, 89, 18, 169, 200, 196, 135, 130, 116,
           188, 159, 86, 164, 100, 109, 198, 173, 186, 3, 64, 52, 217, 226, 250, 124, 123, 5, 202, 38, 147, 118, 126,
           255, 82, 85, 212, 207, 206, 59, 227, 47, 16, 58, 17, 182, 189, 28, 42, 223, 183, 170, 213, 119, 248, 152,
           2, 44, 154, 163, 70, 221, 153, 101, 155, 167, 43, 172, 9, 129, 22, 39, 253, 19, 98, 108, 110, 79, 113, 224,
           232, 178, 185, 112, 104, 218, 246, 97, 228, 251, 34, 242, 193, 238, 210, 144, 12, 191, 179, 162, 241, 81,
           51, 145, 235, 249, 14, 239, 107, 49, 192, 214, 31, 181, 199, 106, 157, 184, 84, 204, 176, 115, 121, 50,
           45, 127, 4, 150, 254, 138, 236, 205, 93, 222, 114, 67, 29, 24, 72, 243, 141, 128, 195, 78, 66, 215, 61,
           156, 180])

np_grad3_0 = numpy.array([ 1., -1.,  1., -1.,  1., -1.,  1., -1.,  0.,  0.,  0.,  0.])
np_grad3_1 = numpy.array([1.,  1., -1., -1.,  0.,  0.,  0.,  0.,  1., -1.,  1., -1.])
np_grad3_2 = numpy.array([0.,  0.,  0.,  0.,  1.,  1., -1., -1.,  1.,  1., -1., -1.])

def tf_lookup_table_np_perm(x):
    return tf.gather_nd(np_perm, tf.cast(tf.floor(x), tf.int32) % np_perm.shape[0])

def tf_lookup_table_np_grad0(x):
    return tf.gather_nd(np_grad3_0, tf.cast(tf.floor(x), tf.int32) % np_grad3_0.shape[0])

def tf_lookup_table_np_grad1(x):
    return tf.gather_nd(np_grad3_1, tf.cast(tf.floor(x), tf.int32) % np_grad3_1.shape[0])

def tf_lookup_table_np_grad2(x):
    return tf.gather_nd(np_grad3_2, tf.cast(tf.floor(x), tf.int32) % np_grad3_2.shape[0])

def new_mul(x, y):
    try:
        if x == 0.0 or y == 0.0:
            return 0.0
    except:
        pass
    try:
        if (x.dtype == bool) and (y.dtype == bool):
            return tf.logical_and(x, y)

        if (x.dtype == bool):
            x = tf.cast(x, y.dtype)

        if (y.dtype == bool):
            y = tf.cast(y, x.dtype)

    except:
        if x == True:
            return y
        if y == True:
            return x
        if x == False or y == False:
            return False
    return tf.multiply(x, y)

def new_add(x, y):
    try:
        if (x.dtype == bool) and (y.dtype == bool) :
            return tf.logical_or(x, y)

        if isinstance(x, tf.Tensor) and x.dtype == bool:
            x = tf.cast(x, dtype)

        if isinstance(y, tf.Tensor) and y.dtype == bool:
            y = tf.cast(y, dtype)
    except:
        if x == False:
            return y
        if y == False:
            return x
        if x == True or y == True:
            return True

    return tf.add(x, y)

tf.Tensor.__mul__ = new_mul
tf.Tensor.__rmul__ = new_mul
tf.Tensor.__add__ = new_add
tf.Tensor.__radd__ = new_add

def select_smooth(a, b, c):
    return a * b + (1 - a) * c

def select_nosmooth(a, b, c):

    all_scalar = True
    base_tensor = None
    count = 0
    for tensor in [a, b, c]:
        if not isinstance(tensor, (int, float, bool, np.bool_)):
            all_scalar = False
            base_tensor = tensor
            count += 1
    if all_scalar:
        return b if a else c

    if base_tensor == a and count == 1:
        actual_dtype = dtype
    elif base_tensor != a:
        actual_dtype = base_tensor.dtype
    else:
        if isinstance(b, tf.Tensor):
            actual_dtype = b.dtype
        else:
            actual_dtype = c.dtype

    if isinstance(b, (int, float, bool)):
        if b == 0.0:
            b = tf.zeros_like(base_tensor, dtype=actual_dtype)
        else:
            if actual_dtype == tf.bool:
                b = tf.ones_like(base_tensor, dtype=actual_dtype)
            else:
                b = b * tf.ones_like(base_tensor, dtype=actual_dtype)
    if isinstance(c, (int, float, bool)):
        if c == 0.0:
            c = tf.zeros_like(base_tensor, dtype=actual_dtype)
        else:
            if actual_dtype == tf.bool:
                c = tf.ones_like(base_tensor, dtype=actual_dtype)
            else:
                c = c * tf.ones_like(base_tensor, dtype=actual_dtype)

    if tag_discontinuous and isinstance(a, tf.Tensor):
        a.discontinuous = True

    return tf.where(tf.cast(a, bool), b, c)

select = select_nosmooth

def simplex_noise(a0, a1, a2, a3, a4, a5, x, y):
    return simplex_noise_2arg(x, y)

def tf_fract(x):
    return tf.floormod(x, 1.0)

def tf_np_wrapper(func):
    def f(x, y=None):

        if func == 'sign_up':
            if isinstance(x, tf.Tensor):
                return 2.0 * tf.cast(x >= 0.0, x.dtype) - 1.0
            else:
                return 2.0 * float(x >= 0.0) - 1.0
        elif func == 'sign_down':
            if isinstance(x, tf.Tensor):
                return 2.0 * tf.cast(x > 0.0, x.dtype) - 1.0
            else:
                return 2.0 * float(x > 0.0) - 1.0
        elif func == 'random_normal':
            return tf.random_normal(tf.shape(x), dtype=x.dtype)
        elif func == 'nequal':
            return tf.math.logical_not(tf.equal(x, y))
        elif func == 'expand_1D':
            return tf.expand_dims(x, 1)

        if isinstance(x, (tf.Tensor, tf.Variable)) or isinstance(y, (tf.Tensor, tf.Variable)):
            if func == 'fmod':
                actual_func = tf.floormod
            else:
                actual_func = getattr(tf, func)
        else:
            try:
                actual_func = getattr(np, func)
            except:
                actual_func = getattr(math, func)
        if y is None:
            return actual_func(x)
        else:
            return actual_func(x, y)

    if tag_discontinuous:
        def f_tag(x, y=None):
            ans = f(x, y)
            if func in ['sign', 'sign_up', 'sign_down', 'floor', 'ceil']:
                ans.discontinuous = True
            return ans
        return f_tag
    else:
        return f

def texture_linear_uv(texture, x, y):
    xx_scaled = x * tf.cast(tf.shape(texture)[1], tf.float32)
    yy_scaled = y * tf.cast(tf.shape(texture)[0], tf.float32)

    xx_down = tf.floor(xx_scaled)
    xx_up = xx_down + 1
    yy_down = tf.floor(yy_scaled)
    yy_up = yy_down + 1

    rx = xx_up - xx_scaled
    ry = yy_up - yy_scaled

    xx_up_mod = xx_up
    yy_up_mod = yy_up

    val_dd = tf.gather_nd(texture, tf.stack((tf.cast(yy_down, tf.int32), tf.cast(xx_down, tf.int32)), axis=3))
    val_du = tf.gather_nd(texture, tf.stack((tf.cast(yy_up_mod, tf.int32), tf.cast(xx_down, tf.int32)), axis=3))
    val_ud = tf.gather_nd(texture, tf.stack((tf.cast(yy_down, tf.int32), tf.cast(xx_up_mod, tf.int32)), axis=3))
    val_uu = tf.gather_nd(texture, tf.stack((tf.cast(yy_up_mod, tf.int32), tf.cast(xx_up_mod, tf.int32)), axis=3))

    ans = rx * ry * val_dd + rx * (1.0 - ry) * val_du + (1.0 - rx) * ry * val_ud + (1.0 - rx) * (1.0 - ry) * val_uu
    return ans

max_stars_repo_name: yyuting/FastImageProcessing

max_issues_repo_name: yyuting/FastImageProcessing

max_forks_repo_name: yyuting/FastImageProcessing