text:
import logging
import datetime as dt

# Scientific
import numpy as np
import pandas as pd

# Local imports
from utilities.paths import *

# ##################################
# PARAMS
# ##################################

# Maximum number of rows to be read by default
MAX_ROWS: int = int(1e6)

# Column suffix for candles: e.g. 'index_close'
OHLC = ["open", "high", "low", "close"]

# ##################################
# TRAIN & TEST SETS
# ##################################

def load_test_set(currency: str, max_rows=MAX_ROWS):
    """
    Load the test set for a given currency (3-letter) symbol.

    :param currency: 3-letter string symbol (e.g. BTC)
    :param max_rows: Maximum number of rows to be read (int)
    :return: pandas DataFrame
    """
    logging.info(f"Loading test dataset in {currency} with a maximum of {max_rows} rows.")
    p = os.path.join(get_data_folder_path(), f"{str(currency).upper()}_test.csv")
    return load_dataset(path=p, max_rows=max_rows)

def load_train_set(currency: str, max_rows=MAX_ROWS):
    """
    Load the train set for a given currency (3-letter) symbol.

    :param currency: 3-letter string symbol (e.g. BTC)
    :param max_rows: Maximum number of rows to be read (int)
    :return: pandas DataFrame
    """
    logging.info(f"Loading training dataset in {currency} with a maximum of {max_rows} rows.")
    p = os.path.join(get_data_folder_path(), f"{str(currency).upper()}_train.csv")
    return load_dataset(path=p, max_rows=max_rows)

# ##################################
# BASIC LOADER USING PANDAS
# ##################################

def load_dataset(path: str, max_rows: int = None):
    """
    Load a dataset for a given currency (3-letter) symbol.

    :param path: Local dir with CSV file in (absolute).
    :param max_rows: Maximum number of rows to be read (int, optional)
    :return: pandas DataFrame
    """
    # Load the raw dataframe using pandas
    df = load_raw_dataframe(max_rows, path)

    # Clean-up the index and resample
    df = reindex_dataframe(df=df, resample=True)

    logging.info("Dataset loaded.")

    # Will fwd-fill spot price data (OHLC)
    # then back-fill the first row
    df = back_and_forward_fill_ohlc_index_columns(df.copy())
    df = clean_iv_columns(df)

    # This is added to prevent mistakes...
    # should be done more thoroughly
    assert "perpetual" in df.columns, "Required column 'perpetual' missing from dataset."
    if "spot" in df.columns:
        if np.mean(df.spot) > 10.0:
            logging.warning(f"Rescaling 'spot' time series.")
            df.spot = df.spot / df.perpetual - 1.0

    return df

# ##################################
# ANCILLARY METHODS: CLEANING
# ##################################

def back_and_forward_fill_ohlc_index_columns(df: pd.DataFrame):
    """
    Fill the DataFrame column forward, then backward.

    :param df: pandas DataFrame
    :return: pandas DataFrame (filled)
    """

    index_ohlc_cols = [f"index_{ohlc}" for ohlc in OHLC]
    for col in index_ohlc_cols:

        if col not in df.columns:
            continue

        logging.info(f"Filling dataframe column {col} forward and back.")
        df[col] = df[col].ffill().bfill()

    return df

def clean_iv_columns(df: pd.DataFrame):
    """
    Cleans the DataFrame columns representing implied vol data by
    replacing infinities by NaN, then filling it.

    :param df: pandas DataFrame
    :return: pandas DataFrame (filled)
    """
    for c in df.columns:
        if "iv_" not in c:
            continue
        df[c] = df[c].replace(np.inf, np.nan, inplace=False)
        df[c] = df[c].ffill().bfill()
    return df

# ##################################
# ANCILLARY METHODS: PANDAS WRAPPER
# ##################################

def load_raw_dataframe(max_rows: int, path: str):
    """
    Load a raw dataframe from a CSV file, basically a Pandas wrapper.

    :param max_rows: maximum number of rows to be read, as int
    :param path: path as string
    :return: pandas DataFrame
    """
    # Load data using pandas
    if max_rows is not None:
        logging.debug(f"Loading dataset with a maximum of {max_rows}.")
        df = pd.read_csv(path, nrows=max_rows)
    else:
        logging.debug(f"Loading dataset without a maximum row count.")
        df = pd.read_csv(path)
    logging.info(f"Dataset loaded with shape: {df.shape}.")
    return df

# ##################################
# ANCILLARY METHODS: INDEX CLEANING
# ##################################

def reindex_dataframe(df: pd.DataFrame, resample: bool = True):
    """
    Re-index the dataframe

    :param df: pandas dataframe
    :param resample: Bool, to make sure the interval is OK.
    :return: pandas DataFrame
    """

    logging.debug(f"Creating index.")
    df["timestamp"] /= 1e9
    idx = pd.DatetimeIndex([dt.datetime.utcfromtimestamp(d) for d in df["timestamp"].to_numpy()])

    # Set index
    df.set_index(idx, inplace=True)
    df.drop(columns=["timestamp", ], inplace=True)

    # Sort the index in place
    df.sort_index(inplace=True)
    logging.debug(f"Spot index sorted.")

    # Avoid potential duplicates
    df.index.drop_duplicates(keep="last")
    logging.debug(f"Spot (potential) duplicates dropped.")

    if resample:
        df = df.resample("1min").last()
        logging.info(f"Spot re-sampled at 1 minute interval.")

    return df


text:
#! /usr/bin/env python
"""
Combine taxonomy information and "togetherness" into one mishmash of data
structures.
"""
import argparse
import pprint
import scipy.cluster.hierarchy as sch
from pickle import load, dump

from sourmash.lca import lca_utils

from . import utils

def do_cluster(mat, hashes_to_tax):
    """
    Use scipy.cluster.hierarchy to cluster the matrix.
    """
    n_hashes = mat.shape[1]
    assert len(hashes_to_tax) == n_hashes

    # do the clustering...
    Y = sch.linkage(mat, method='complete')
    rootnode, nodelist = sch.to_tree(Y, rd=True)

    # now, track the taxonomy <-> hashval <-> cluster node.
    hashlist = list(sorted(hashes_to_tax))
    node_id_to_tax = {}

    # here we are assuming that the nodelist is in the same order as the
    # original labels. Is that right? Yes, it is. Just make sure the
    # hashes are sorted (same as when we created the matrix).
    for pos in range(n_hashes):
        node = nodelist[pos]
        assert node.get_id() == pos
        hashval = hashlist[pos]
        tax_info = hashes_to_tax[hashval]

        x = set()
        if tax_info:
            x.add(tax_info)
        node_id_to_tax[pos] = x

    ###

    def traverse_and_label(node):
        "do recursive traversal from node and label with taxonomy LCA"
        if node.is_leaf():
            tax_id = node_id_to_tax[node.get_id()]
            x = set()
            if tax_id:
                x.update(tax_id)
            return x
        else:
            l = traverse_and_label(node.get_left())
            r = traverse_and_label(node.get_right())

            # taxonomy of a node is union of taxonomies below node.
            x = l.union(r)
            node_id_to_tax[node.get_id()] = x

            return x

    traverse_and_label(rootnode)

    ###

    return rootnode, nodelist, node_id_to_tax

def main():
    p = argparse.ArgumentParser()
    p.add_argument('--load-matrix-pickle', help='output of match_metagenomes',
                   required=True)
    p.add_argument('--load-tax-hashes', help='output of genome_shred_to_tax',
                   required=True)
    p.add_argument('--pickle-tree', default=None)
    args = p.parse_args()

    # output of match_metagenomes
    print('calculating distance matrix from', args.load_matrix_pickle)
    with open(args.load_matrix_pickle, 'rb') as fp:
        matrix_obj = load(fp)
    mat, n_orig_hashes = utils.make_distance_matrix(matrix_obj.mat)

    # output of genome_shred_to_tax
    with open(args.load_tax_hashes, 'rb') as fp:
        hashes_to_tax = load(fp)

    # some basic validation
    assert matrix_obj.ksize == hashes_to_tax.ksize
    print(matrix_obj.genome_file, hashes_to_tax.genome_file)
    assert matrix_obj.genome_file == hashes_to_tax.genome_file
    assert matrix_obj.query_hashlist == list(sorted(hashes_to_tax))
    assert matrix_obj.query_fragment_size == hashes_to_tax.fragment_size
    assert n_orig_hashes == len(hashes_to_tax), "mismatch! was same --scaled used to compute these?"
    assert mat.shape[1] == len(hashes_to_tax)

    print('distance matrix is {} x {}; found {} matching hashes.'.format(mat.shape[0], mat.shape[1], len(hashes_to_tax)))

    print('clustering by togetherness & assigning taxonomy!')
    rootnode, nodelist, node_id_to_tax = do_cluster(mat, hashes_to_tax)

    if args.pickle_tree:
        print('pickling tree & taxonomy to file', args.pickle_tree)
        with open(args.pickle_tree, 'wb') as fp:
            dump((rootnode, nodelist, node_id_to_tax), fp)

if __name__ == '__main__':
    main()

ext: py

reason: import scipy


text:
import haiku as hk
import jax
import jax.numpy as jnp
import numpy as np
from einops import rearrange, repeat

from mesh_transformer.util import f_psum, g_psum, maybe_shard, head_print
from jax.experimental import PartitionSpec as P
from jax.experimental.maps import thread_resources

class ReplicatedLayerNorm(hk.Module):
    def __init__(self, offset=True):
        super().__init__()
        self.offset = offset

    def __call__(self, inputs: jnp.ndarray) -> jnp.ndarray:
        mean = jnp.mean(inputs, axis=-1, keepdims=True)
        variance = jnp.var(inputs, axis=-1, keepdims=True)

        param_shape = inputs.shape[-1:]
        scale = hk.get_parameter("scale", param_shape, inputs.dtype, init=jnp.ones)
        scale = jax.lax.all_gather(scale, "shard")[0]

        offset = hk.get_parameter("offset", param_shape, inputs.dtype, init=jnp.zeros)
        offset = jax.lax.all_gather(offset, "shard")[0]

        scale = jnp.broadcast_to(scale, inputs.shape)
        offset = jnp.broadcast_to(offset, inputs.shape)
        mean = jnp.broadcast_to(mean, inputs.shape)

        inv = scale * jax.lax.rsqrt(variance + 1e-5)
        if self.offset:
            return inv * (inputs - mean) + offset
        else:
            return inv * (inputs - mean)

class RMSNorm(hk.Module):
    def __init__(self, offset, elementwise):
        super().__init__()
        self.offset = offset
        self.elementwise = elementwise

    def __call__(self, x):
        param_shape = (x.shape[-1],) if self.elementwise else ()
        normed = x / (jnp.linalg.norm(x, axis=-1, keepdims=True) + 1e-5)

        scale = hk.get_parameter('scale', param_shape, init=hk.initializers.Constant(x.shape[-1] ** 0.5))
        scale = jax.lax.pmean(scale, "shard")
        normed = normed * scale

        if self.offset:
            offset = hk.get_parameter('offset', param_shape, init=jnp.zeros)
            offset = jax.lax.pmean(offset, "shard")
            normed = normed + offset

        return normed

def getnorm(type):
    if type == "layernorm":
        return ReplicatedLayerNorm()
    if type == "layernorm-desync":
        return hk.LayerNorm(-1, True, True)
    elif type == "layernorm-nobias":
        return ReplicatedLayerNorm(offset=False)
    elif type == "rmsnorm":
        return RMSNorm(False, True)
    elif type == "scalenorm":
        return RMSNorm(False, False)
    elif type == "rmsnorm-bias":
        return RMSNorm(True, True)
    elif type == "scalenorm-bias":
        return RMSNorm(True, False)
    else:
        raise Exception("Not implemented")

class RelativePositionEmbs(hk.Module):
    @staticmethod
    def _relative_position_bucket(relative_position,
                                  num_buckets=32,
                                  max_distance=128):
        ret = 0
        n = -relative_position
        n = np.maximum(n, 0)
        # now n is in the range [0, inf)
        max_exact = num_buckets // 2
        is_small = (n < max_exact)
        val_if_large = max_exact + (
                np.log(n.astype(np.float32) / max_exact + np.finfo(np.float32).eps) /
                np.log(max_distance / max_exact) *
                (num_buckets - max_exact)).astype(np.int32)
        val_if_large = np.minimum(val_if_large, num_buckets - 1)
        ret += np.where(is_small, n, val_if_large)
        return ret

    def __call__(self, qlen, klen, heads, num_buckets):
        """Produce relative position embedding attention biases.
        Returns:
          output: `(heads, q_len, k_len)` attention bias
        """
        context_position = np.arange(qlen, dtype=jnp.int32)[:, None]
        memory_position = np.arange(klen, dtype=jnp.int32)[None, :]
        relative_position = memory_position - context_position  # shape (qlen, klen)
        rp_bucket = self._relative_position_bucket(relative_position)
        relative_attention_bias = hk.get_parameter('rel_embedding', [heads, num_buckets],
                                                   init=hk.initializers.TruncatedNormal(stddev=0.02))
        # Instead of using a slow gather, we create a leading-dimension one-hot
        # array from rp_bucket and use it to perform the gather-equivalent via a
        # contraction, i.e.:
        # (num_head, num_buckets) x (num_buckets one-hot, qlen, klen).
        # This is equivalent to relative_attention_bias[:, rp_bucket]
        bcast_iota = jax.lax.broadcasted_iota(jnp.int32, (num_buckets, 1, 1), 0)
        rp_bucket_one_hot = jnp.array(rp_bucket[jnp.newaxis, Ellipsis] == bcast_iota).astype(
            relative_attention_bias.dtype)
        # --> shape (qlen, klen, num_heads)
        values = jax.lax.dot_general(
            relative_attention_bias,
            rp_bucket_one_hot,
            (
                ((1,), (0,)),  # rhs, lhs contracting dims
                ((), ())))  # no batched dims
        return values

def fixed_pos_embedding(x, seq_dim=0):
    dim = x.shape[-1]
    inv_freq = 1. / (10000 ** (np.arange(0, dim, 2) / dim))

    sinusoid_inp = np.einsum('i , j -> i j', np.arange(x.shape[seq_dim]), inv_freq)

    return np.sin(sinusoid_inp), np.cos(sinusoid_inp)

def rotate_every_two(x):
    x1 = x[:, :, ::2]
    x2 = x[:, :, 1::2]

    x = jnp.stack((-x2, x1), axis=-1)

    return rearrange(x, '... d j -> ... (d j)')

def apply_rotary_pos_emb(x, sincos):
    sin, cos = map(lambda t: repeat(t, 'b n -> b (n j)', j=2)[-x.shape[0]:, None, :], sincos)
    return (x * cos) + (rotate_every_two(x) * sin)

def rotate_every_two_v2(x):
    x1 = x[:, :, :, ::2]
    x2 = x[:, :, :, 1::2]

    x = jnp.stack((-x2, x1), axis=-1)

    return rearrange(x, '... d j -> ... (d j)')

def apply_rotary_pos_emb_v2(x, sincos):
    sin, cos = map(lambda t: repeat(t, '... b n -> ... b (n j)', j=2)[-x.shape[-3]:, None, :], sincos)
    return (x * cos) + (rotate_every_two_v2(x) * sin)

class EmbeddingShard(hk.Module):
    def __init__(self, config, name=None):
        super().__init__(name=name)
        in_dim = config["n_vocab"]
        out_dim = config["d_model"]
        shards = config["cores_per_replica"]

        assert in_dim % shards == 0

        self.in_dim = in_dim
        self.out_dim = out_dim
        self.in_dim_per_shard = in_dim // shards
        self.out_dim_per_shard = out_dim // shards

        if config["pe"] == "fixed":
            embed_init = hk.initializers.TruncatedNormal(stddev=0.02)
            self.positional_embeddings = hk.get_parameter('pos_embs', [config["seq"], self.out_dim_per_shard], init=embed_init)
        else:
            self.positional_embeddings = None

        self.proj = hk.Linear(self.out_dim, w_init=hk.initializers.TruncatedNormal(stddev=1 / np.sqrt(in_dim)))

    def __call__(self, x, dtype=jnp.bfloat16):
        shard_start_index = jax.lax.axis_index('shard') * self.in_dim_per_shard

        input_onehot = jax.nn.one_hot(x - shard_start_index, self.in_dim_per_shard)
        proj_out = self.proj(input_onehot)

        proj_out = g_psum(proj_out)

        if self.positional_embeddings is not None:
            all_pos_embed = jax.lax.all_gather(self.positional_embeddings, 'shard')

            all_pos_embed = hk.Flatten()(jnp.transpose(all_pos_embed, (1, 0, 2)))

            proj_out += all_pos_embed

        return proj_out

class EmbeddingShardV2(hk.Module):
    def __init__(self, config, name=None):
        super().__init__(name=name)
        in_dim = config["n_vocab"]
        out_dim = config["d_model"]
        shards = config["cores_per_replica"]

        assert in_dim % shards == 0

        self.in_dim = in_dim
        self.out_dim = out_dim

        self.proj = hk.Linear(self.out_dim, w_init=hk.initializers.TruncatedNormal(stddev=1 / np.sqrt(in_dim)))

    def __call__(self, x, dtype=jnp.bfloat16):
        input_onehot = jax.nn.one_hot(x, self.in_dim)
        input_onehot = maybe_shard(input_onehot, P("dp", None, "mp"))

        proj_out = self.proj(input_onehot)

        return proj_out

# We actually combine the FF and dense in one layer (i.e. compute in parallel) to minimize all reduces
class TransformerLayerShard(hk.Module):
    def __init__(self, config, name=None, init_scale=1.):
        super().__init__(name=name)
        heads = config["n_heads"]
        dim = config["d_model"]
        shards = config["cores_per_replica"]
        norm = getnorm(config["norm"])
        self.is_rotary = config["pe"] == "rotary"

        assert dim % heads == 0
        assert heads % shards == 0

        self.dim = dim
        self.dim_per_head = dim // heads
        self.heads_per_shard = heads // shards
        self.dim_per_shard = dim // shards
        self.pe_rotary_dims = config.get("pe_rotary_dims", self.dim_per_head)

        self.norm = norm

        self.q = hk.Linear(self.dim_per_shard, with_bias=False)
        self.v = hk.Linear(self.dim_per_shard, with_bias=False)
        self.k = hk.Linear(self.dim_per_shard, with_bias=False)

        self.o = hk.Linear(self.dim, with_bias=False,
                           w_init=hk.initializers.TruncatedNormal(stddev=init_scale / np.sqrt(self.dim)))

        self.dense_proj = hk.Linear(self.dim_per_shard * 4)
        self.dense_proj_o = hk.Linear(self.dim,
                                      w_init=hk.initializers.TruncatedNormal(stddev=init_scale / np.sqrt(self.dim)))

    def self_attn(self, q, v, k, attn_bias):
        if self.is_rotary:
            k_rot = k[:, :, :self.pe_rotary_dims]
            k_pass = k[:, :, self.pe_rotary_dims:]

            q_rot = q[:, :, :self.pe_rotary_dims]
            q_pass = q[:, :, self.pe_rotary_dims:]

            sincos = fixed_pos_embedding(k_rot)
            q_rot = apply_rotary_pos_emb(q_rot, sincos)
            k_rot = apply_rotary_pos_emb(k_rot, sincos)

            k = jnp.concatenate([k_rot, k_pass], axis=-1)
            q = jnp.concatenate([q_rot, q_pass], axis=-1)

        attention_logits = jnp.einsum("thd,Thd->htT", q, k)

        sqrt_key_size = np.sqrt(self.dim_per_head).astype(k.dtype)
        attention_logits = attention_logits / sqrt_key_size

        attention_logits += attn_bias

        attention_weights = jax.nn.softmax(attention_logits)
        attention_vec = jnp.einsum("htT,Thd->thd", attention_weights, v).reshape((-1, self.dim_per_shard))

        return self.o(attention_vec)

    def ff(self, x):
        dense_proj = self.dense_proj(x)
        dense_proj = jax.nn.gelu(dense_proj)
        return self.dense_proj_o(dense_proj)

    def qvk_proj(self, x):
        q = self.q(x).reshape(x.shape[:-1] + (self.heads_per_shard, self.dim_per_head))
        v = self.v(x).reshape(x.shape[:-1] + (self.heads_per_shard, self.dim_per_head))
        k = self.k(x).reshape(x.shape[:-1] + (self.heads_per_shard, self.dim_per_head))

        return q, v, k

    def __call__(self, x, attn_bias):
        x = f_psum(x)
        x = self.norm(x)

        q, v, k = self.qvk_proj(x)

        seq_len = x.shape[0]
        causal_mask = np.tril(np.ones((seq_len, seq_len)))
        bias = -1e10 * (1. - causal_mask)
        bias += attn_bias

        attn_out = self.self_attn(q, v, k, bias)
        dense_out = self.ff(x)

        return g_psum(attn_out + dense_out)

    # iterate the decoding process by a single token
    def decode_once(self, decode_state, x, attn_bias):
        x = f_psum(x)
        x = self.norm(x)

        assert x.shape[0] == 1

        q, v, k = self.qvk_proj(x)

        # add new kv to end
        v = jnp.concatenate((decode_state["v"], v), axis=0)[1:]
        k = jnp.concatenate((decode_state["k"], k), axis=0)[1:]

        tokens_decoded = decode_state["tokens_decoded"] + 1
        length = v.shape[0]

        masked_tokens = length - tokens_decoded

        attention_mask = jnp.arange(0, length) < masked_tokens
        bias = (-1e10 * attention_mask)
        bias += attn_bias

        attn_out = self.self_attn(q, v, k, bias)
        dense_out = self.ff(x)

        return g_psum(attn_out + dense_out), {
            "tokens_decoded": tokens_decoded,
            "k": k,
            "v": v
        }

    # take in right aligned context tokens and generate an initial state
    def get_init_decode_state(self, x, given_length, attn_bias):
        x = f_psum(x)
        x = self.norm(x)

        q, v, k = self.qvk_proj(x)

        full_length = x.shape[0]
        masked_tokens = full_length - given_length

        seq_len = x.shape[0]
        causal_mask = np.tril(np.ones((seq_len, seq_len)))

        bias = -1e10 * (1. - causal_mask)  # regular AR masking
        bias -= 1e10 * (jnp.arange(0, full_length) < masked_tokens)  # mask out zero tokens before context starts
        bias += attn_bias  # finally add attn bias for rpe

        attn_out = self.self_attn(q, v, k, bias)
        dense_out = self.ff(x)

        return g_psum(attn_out + dense_out), {"k": k, "v": v, "tokens_decoded": given_length.astype(jnp.uint32)}

# This new class combines the input and output projection into one matmul for better efficiency
class TransformerLayerShardV2(hk.Module):
    def __init__(self, config, name=None, init_scale=1.):
        super().__init__(name=name)
        self.dim = config["d_model"]
        self.n_head = config["n_heads"]
        self.d_head = config["d_head"]
        self.d_rotary = config["pe_rotary_dims"]
        self.mp_num = thread_resources.env.shape['mp']

        self.norm = hk.LayerNorm(-1, True, True)
        self.input_proj = hk.Linear(self.d_head * self.n_head * 3 + self.dim * 4)
        self.output_proj = hk.Linear(self.dim,
                                     w_init=hk.initializers.TruncatedNormal(stddev=init_scale / jnp.sqrt(self.dim)))

    def self_attn(self, q, v, k, attn_bias):
        k_rot = k[:, :, :, :self.d_rotary]
        k_pass = k[:, :, :, self.d_rotary:]

        q_rot = q[:, :, :, :self.d_rotary]
        q_pass = q[:, :, :, self.d_rotary:]

        sincos = fixed_pos_embedding(k_rot, seq_dim=1)
        q_rot = apply_rotary_pos_emb_v2(q_rot, sincos)
        k_rot = apply_rotary_pos_emb_v2(k_rot, sincos)
        q_rot = maybe_shard(q_rot, P("dp", None, "mp", None))
        k_rot = maybe_shard(k_rot, P("dp", None, "mp", None))

        k = jnp.concatenate([k_rot, k_pass], axis=-1)
        q = jnp.concatenate([q_rot, q_pass], axis=-1)

        k = maybe_shard(k, P("dp", None, "mp", None))
        q = maybe_shard(q, P("dp", None, "mp", None))

        attention_logits = jnp.einsum("bthd,bThd->bhtT", q, k)

        attention_logits = maybe_shard(attention_logits, P("dp", "mp", None, None))

        sqrt_key_size = np.sqrt(self.d_head).astype(k.dtype)
        attention_logits = attention_logits / sqrt_key_size

        attention_logits += attn_bias
        attention_logits = maybe_shard(attention_logits, P("dp", "mp", None, None))

        attention_weights = jax.nn.softmax(attention_logits)
        attention_weights = maybe_shard(attention_weights, P("dp", "mp", None, None))

        attention_vec = jnp.einsum("bhtT,bThd->bthd", attention_weights, v)

        attention_vec = maybe_shard(attention_vec, P("dp", None, "mp", None))
        sharded_attn_vec = attention_vec.reshape(attention_vec.shape[:2] + (self.mp_num, self.n_head//self.mp_num, -1))
        sharded_attn_vec = maybe_shard(sharded_attn_vec, P("dp", None, "mp", None, None))

        attention_vec = attention_vec.reshape(sharded_attn_vec.shape[:2] + (self.mp_num, -1))
        return maybe_shard(attention_vec, P("dp", None, "mp", None))

    # input: [batch, seq, dim]
    # output: [batch, seq, n_head, d_head]
    def head_split(self, x):
        reshaped = x.reshape(x.shape[:-1] + (self.n_head//self.mp_num, self.d_head))
        reshaped = reshaped.reshape(x.shape[:-2] + (-1, ) + x.shape[-1:])

        # return reshaped
        return maybe_shard(reshaped, P("dp", None, "mp", None))

    def input(self, x):
        # [batch, seq, dim]
        projected = self.input_proj(x)

        # [batch, seq, mp, dim//mp]
        projected = maybe_shard(projected, P("dp", None, "mp"))
        mp_split = jnp.reshape(projected, projected.shape[:-1] + (self.mp_num, -1))
        mp_split = maybe_shard(mp_split, P("dp", None, "mp", None))

        local_dim = self.d_head * self.n_head // self.mp_num

        q, v, k, ff = jnp.split(mp_split, [local_dim, local_dim * 2, local_dim * 3], axis=-1)

        q = self.head_split(q)
        v = self.head_split(v)
        k = self.head_split(k)

        return q, v, k, ff

    def output(self, *x):
        out = jnp.concatenate(x, axis=-1)
        out = maybe_shard(out, P("dp", None, "mp", None))

        out = out.reshape(x[0].shape[:-2] + (-1,))
        out_shard = maybe_shard(out, P("dp", None, "mp"))

        return self.output_proj(out_shard)

    def __call__(self, x, attn_bias):

        x = self.norm(x)

        q, v, k, ff = self.input(x)

        # head_print("x.shape", x.shape)
        # head_print("attn_bias.shape", attn_bias.shape)

        seq_len = x.shape[1]
        causal_mask = np.tril(np.ones((seq_len, seq_len)))[None, :, :]
        bias = -1e10 * (1. - causal_mask)

        # head_print("bias.shape", bias.shape)

        bias += attn_bias

        attn_out = self.self_attn(q, v, k, bias)
        ff_out = jax.nn.gelu(ff)

        return self.output(attn_out, ff_out)

    # iterate the decoding process by a single token
    def decode_once(self, decode_state, x, attn_bias):
        x = self.norm(x)

        assert x.shape[0] == 1

        q, v, k, ff = self.input(x)

        # add new kv to end
        v = jnp.concatenate((decode_state["v"], v), axis=1)[1:]
        k = jnp.concatenate((decode_state["k"], k), axis=1)[1:]

        tokens_decoded = decode_state["tokens_decoded"] + 1
        length = v.shape[1]

        masked_tokens = length - tokens_decoded

        attention_mask = jnp.arange(0, length) < masked_tokens
        bias = (-1e10 * attention_mask)
        bias += attn_bias

        attn_out = self.self_attn(q, v, k, bias)
        ff_out = jax.nn.gelu(ff)

        return self.output(attn_out, ff_out), {
            "tokens_decoded": tokens_decoded,
            "k": k,
            "v": v
        }

    # take in right aligned context tokens and generate an initial state
    def get_init_decode_state(self, x, given_length, attn_bias):
        x = f_psum(x)
        x = self.norm(x)

        q, v, k, ff = self.input(x)

        full_length = x.shape[1]
        masked_tokens = full_length - given_length

        causal_mask = np.tril(np.ones((full_length, full_length)))

        bias = -1e10 * (1. - causal_mask)  # regular AR masking
        bias -= 1e10 * (jnp.arange(0, full_length) < masked_tokens)  # mask out zero tokens before context starts
        bias += attn_bias  # finally add attn bias for rpe

        attn_out = self.self_attn(q, v, k, bias)
        ff_out = jax.nn.gelu(ff)

        return self.output(attn_out, ff_out),\
               {"k": k, "v": v, "tokens_decoded": given_length.astype(jnp.uint32)}

class ProjectionShard(hk.Module):
    def __init__(self, config, name=None):
        super().__init__(name=name)
        out_dim = config["n_vocab"]
        shards = config["cores_per_replica"]
        norm = getnorm(config["norm"])

        assert out_dim % shards == 0

        self.dim = out_dim
        self.dim_per_shard = out_dim // shards

        self.norm = norm

        self.proj = hk.Linear(self.dim_per_shard)

    def __call__(self, x):
        x = self.norm(x)
        proj = self.proj(x)

        all_proj = jax.lax.all_gather(proj, 'shard')

        return hk.Flatten()(jnp.transpose(all_proj, (1, 0, 2)))

    def loss(self, x, targets, z_loss=1):
        x = f_psum(x)
        x = self.norm(x)
        logits = self.proj(x)

        shard_start_index = jax.lax.axis_index('shard') * self.dim_per_shard
        global_max = jax.lax.pmax(jax.lax.stop_gradient(logits.max(-1, keepdims=True)), "shard")
        logits -= jax.lax.stop_gradient(global_max)

        gt_onehot = jax.nn.one_hot(targets - shard_start_index, self.dim_per_shard)
        predicted_logits = jnp.sum(jnp.multiply(gt_onehot, logits), axis=-1)
        predicted_logits = g_psum(predicted_logits)

        exp_logits = jnp.exp(logits)

        sum_exp_logits = exp_logits.sum(axis=-1)
        sum_exp_logits = g_psum(sum_exp_logits)

        loss = jnp.log(sum_exp_logits) - predicted_logits

        loss += (1e-4 * jnp.square(jnp.log(sum_exp_logits)) * z_loss).mean()

        correct = (0.0 == predicted_logits)

        return loss, correct

class Projection(hk.Module):
    def __init__(self, config, name=None):
        super().__init__(name=name)
        out_dim = config["n_vocab"]

        self.dim = out_dim
        self.norm = hk.LayerNorm(-1, True, True)

        self.proj = hk.Linear(self.dim)

    def __call__(self, x):
        x = self.norm(x)
        return self.proj(x)

    def loss(self, x, targets, z_loss=1):
        x = self.norm(x)
        logits = self.proj(x)

        logits -= logits.max(-1, keepdims=True)

        gt_onehot = jax.nn.one_hot(targets, self.dim)
        predicted_logits = jnp.sum(jnp.multiply(gt_onehot, logits), axis=-1)
        exp_logits = jnp.exp(logits)

        sum_exp_logits = exp_logits.sum(axis=-1)

        loss = jnp.log(sum_exp_logits) - predicted_logits

        loss += (1e-4 * jnp.square(jnp.log(sum_exp_logits)) * z_loss).mean()
        correct = (0.0 == predicted_logits)
        return loss, correct

max_stars_repo_name: narphorium/mesh-transformer-jax

max_issues_repo_name: narphorium/mesh-transformer-jax

max_forks_repo_name: narphorium/mesh-transformer-jax

reason: import numpy,import jax,from jax


text:
import numpy as np
from scipy.optimize import linprog

c = np.array([6, 8]) * 100000
A_ub = [
    [1, 1],
    [-40, -50]
]
b_ub = [9, -400]

bounds = ((0, 8), (0, 10))

def callback(xk, **kwargs):
    np.set_printoptions(precision=3)
    np.set_printoptions(suppress=True)
    print(kwargs)

res = linprog(c, A_ub=A_ub, b_ub=b_ub, bounds=bounds, callback=callback)
print(res)

reason: import numpy,from scipy


text:
from wandb import save
from team_code import image_agent
import torch
import torchvision
import numpy as np
import carla
import os
import cv2
import json

DEBUG = int(os.environ.get('HAS_DISPLAY', 0))

def get_entry_point():
    return 'myImageAgent'

def debug_display(tick_data, target_cam, out, steer, throttle, brake, desired_speed, step):
    return image_agent.debug_display(tick_data, target_cam, out, steer, throttle, brake, desired_speed, step)

def mkdir(step):
    os.mkdir(os.path.join('tmp_record', str(step*0.05)))

def save_info(step, data):
    with open(os.path.join('tmp_record', str(step*0.05), 'info.json'),'w') as f:
        json.dump(data, f, indent=4)

class myImageAgent(image_agent.ImageAgent):

    def setup(self, path_to_conf_file):
        super().setup(path_to_conf_file)
        self.save_attr=['gps','speed','compass','target']
        self.save_img = ['rgb','rgb_left','rgb_right']

    @torch.no_grad()
    def run_step(self, input_data, timestamp):
        if not self.initialized:
            self._init()

        tick_data = self.tick(input_data)

        img = torchvision.transforms.functional.to_tensor(tick_data['image'])
        img = img[None].cuda()

        target = torch.from_numpy(tick_data['target'])
        target = target[None].cuda()

        points, (target_cam, _) = self.net.forward(img, target)
        points_cam = points.clone().cpu()
        points_cam[..., 0] = (points_cam[..., 0] + 1) / 2 * img.shape[-1]
        points_cam[..., 1] = (points_cam[..., 1] + 1) / 2 * img.shape[-2]
        points_cam = points_cam.squeeze()
        points_world = self.converter.cam_to_world(points_cam).numpy()

        aim = (points_world[1] + points_world[0]) / 2.0
        angle = np.degrees(np.pi / 2 - np.arctan2(aim[1], aim[0])) / 90
        steer = self._turn_controller.step(angle)
        steer = np.clip(steer, -1.0, 1.0)

        desired_speed = np.linalg.norm(points_world[0] - points_world[1]) * 2.0
        # desired_speed *= (1 - abs(angle)) ** 2

        speed = tick_data['speed']

        brake = desired_speed < 0.4 or (speed / desired_speed) > 1.1

        delta = np.clip(desired_speed - speed, 0.0, 0.25)
        throttle = self._speed_controller.step(delta)
        throttle = np.clip(throttle, 0.0, 0.75)
        throttle = throttle if not brake else 0.0

        control = carla.VehicleControl()
        control.steer = steer
        control.throttle = throttle
        control.brake = float(brake)

        if self.step%20==0:
            mkdir(self.step)
            for im in self.save_img:
                cv2.imwrite(os.path.join('tmp_record', str(self.step*0.05), im+'.png'), tick_data[im])
            save_data = {}
            for attr in self.save_attr:
                save_data[attr] = tick_data[attr]
            save_data['steer'] = steer
            save_data['brake'] = 'True' if brake else 'False'
            save_data['throttle'] = throttle
            save_data['gps'] = save_data['gps'].tolist()
            save_data['target'] = save_data['target'].tolist()
            save_info(self.step, save_data)
            print(f'log success with step {self.step}')

        if DEBUG:
            debug_display(
                    tick_data, target_cam.squeeze(), points.cpu().squeeze(),
                    steer, throttle, brake, desired_speed,
                    self.step)

        return control

reason: import numpy


text:
import random
import argparse
import pprint
from pathlib import Path
import numpy as np
from vocabulary import Vocabulary
import imdb

def read_data(filename):
    with np.load(filename) as data:
        X = data['X']
        y = data['y']
    return X, y

def read_model(filename):
    with np.load(filename) as data:
        w = data['w']
        b = data['b']
    return w, b

def binary_classification_report(y_true, y_pred):
    # count true positives, false positives, true negatives, and false negatives
    tp = fp = tn = fn = 0
    for gold, pred in zip(y_true, y_pred):
        if pred == True:
            if gold == True:
                tp += 1
            else:
                fp += 1
        else:
            if gold == False:
                tn += 1
            else:
                fn += 1
    # calculate precision and recall
    precision = tp / (tp + fp)
    recall    = tp / (tp + fn)
    # calculate f1 score
    fscore = 2 * precision * recall / (precision + recall)
    # calculate accuracy
    accuracy  = (tp + tn) / len(y_true)
    # number of positive labels in y_true
    support   = sum(y_true)
    return {
        'precision': precision,
        'recall': recall,
        'f1-score': fscore,
        'support': support,
        'accuracy': accuracy,
    }

if __name__ == '__main__':

    # parse command-line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('data_file', type=Path, help='path to the data file')
    parser.add_argument('model_file', type=Path, help='path to model file')
    args = parser.parse_args()

    print('loading model ...')
    w, b = read_model(args.model_file)
    print('loading data ...')
    X, y_true = read_data(args.data_file)
    print('applying model ...')
    y_pred = np.where(X @ w + b > 0, 1, 0)
    print('classification report:')
    report = binary_classification_report(y_true, y_pred)
    pprint.pprint(report)

max_stars_repo_name: clulab/gentlenlp

max_issues_repo_name: clulab/gentlenlp

max_forks_repo_name: clulab/gentlenlp


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 26 19:03:07 2021

@author: Xabi
"""

from functions import read_from_csv_net, read_populations
import matplotlib.pyplot as plt
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error

if __name__ == '__main__':
    df = read_from_csv_net()
    cities = df['municipios'].columns[:-1].to_list()
    sizes = read_populations()

    # Filter only more than 5000 ppl cities

    sizes = sizes[sizes['inhabitants'] > 5000]

    cities = sizes.index.to_list()

    # Prepare the data and plot if needed
    # Aim is to input a ([t-5, t] days x n_cities) array
    # and output a ([t + 1, t + 3 x 1]) array for a choosen city
    # By Keras standard reshape input to be 3D [samples, timesteps, features]

    dataX = np.zeros(shape=(len(df['municipios']), len(cities)))
    dataY = np.zeros(shape = (len(df['municipios']), 1))

    plot = True
    i = 0
    pred_city = "Tolosa"
    pred_i = cities.index(pred_city)
    predict_days = 5 # Try to forecast 5 days in future

    for city in cities:
        incidencia = df['municipios'][city].rolling(14).sum() * 100000 / int(sizes[sizes.index == city]['inhabitants'])

        if plot:
            plt.plot(incidencia)
        # Normalize by 3000 (never an incidence higher than 3000)
        incidencia = incidencia / 3000

        # Fill na values with 0
        incidencia = incidencia.fillna(0)

        # Turn into a numpy array
        incidencia = incidencia.to_numpy()

        # Add to array
        dataX[:, i] = incidencia

        # Add predicted value
        if city == pred_city:
            dataY = np.roll(incidencia, -predict_days)
        i += 1

    if plot:
        plt.grid(True)
        plt.legend(cities)

    # Reshape the arrays for Keras input
    dataX = dataX.reshape(dataX.shape[0], 1, dataX.shape[1])
    dataY = dataY.reshape(dataY.shape[0], 1, 1)

    # Divide data
    train_days = 150
    trainX = dataX[:train_days, :, :]
    trainY = dataY[:train_days, :, :]

    # This is my network
    # create and fit the LSTM network
    model = Sequential()
    model.add(LSTM(50, input_shape=(trainX.shape[1], trainX.shape[2])))
    model.add(Dense(1))
    model.compile(loss='mean_squared_error', optimizer='adam')
    model.fit(trainX, trainY, epochs=100, batch_size=1, verbose=2)

    predY = model.predict(dataX)

    # Plot results
    plt.plot(dataX[:, :, pred_i].reshape((dataX.shape[0])))
    plt.plot(dataY.reshape((dataY.shape[0])))
    plt.plot(predY.reshape((predY.shape[0])))
    plt.plot([train_days, train_days], [0,np.max(dataX[:,:, pred_i])],  'r--')

    plt.legend(["Input data", "Real future data", "Predicted future data", "Train boundary"])
    plt.grid(True)
    plt.ylabel("Incidencia 14 dias por 100k habitantes (normalizado /3000)")
    plt.xlabel("Days since data begin")
    plt.title(f"{predict_days} in the future forecast for {pred_city}, LSTM 4 neurons, trained for {train_days} days")


text:
import tensorflow as tf
import numpy as np
import os.path
import Checkpoint
import Features
import Normalization

class AllPolicy:
    def suggest_moves(self, board):
        moves = []
        for x in xrange(board.N):
            for y in xrange(board.N):
                if board.play_is_legal(x, y, board.color_to_play):
                    moves.append((x,y))
        return moves

def softmax(E, temp):
    #print "E =\n", E
    expE = np.exp(temp * (E - max(E))) # subtract max to avoid overflow
    return expE / np.sum(expE)

class TFPolicy:
    def __init__(self, model, threshold_prob, softmax_temp):
        self.model = model
        self.threshold_prob = threshold_prob
        self.softmax_temp = softmax_temp

        # build the graph
        with tf.Graph().as_default():
            with tf.device('/cpu:0'):
                self.feature_planes = tf.placeholder(tf.float32, shape=[None, self.model.N, self.model.N, self.model.Nfeat], name='feature_planes')
                self.logits = model.inference(self.feature_planes, self.model.N, self.model.Nfeat)
                saver = tf.train.Saver(tf.trainable_variables())
                init = tf.initialize_all_variables()
                self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
                self.sess.run(init)
                checkpoint_dir = os.path.join(model.train_dir, 'checkpoints')
                Checkpoint.restore_from_checkpoint(self.sess, saver, checkpoint_dir)

    def suggest_moves(self, board):
        board_feature_planes = Features.make_feature_planes_stones_4liberties_4history_ko_4captures(board, board.color_to_play).astype(np.float32)
        Normalization.apply_featurewise_normalization_C(board_feature_planes)
        feed_dict = {self.feature_planes: board_feature_planes.reshape(1,self.model.N,self.model.N,self.model.Nfeat)}
        move_logits = self.sess.run(self.logits, feed_dict).ravel() # ravel flattens to 1D
        # zero out illegal moves
        for x in xrange(self.model.N):
            for y in xrange(self.model.N):
                ind = self.model.N * x + y
                if not board.play_is_legal(x, y, board.color_to_play):
                    move_logits[ind] = -1e99
        move_probs = softmax(move_logits, self.softmax_temp)
        sum_probs = np.sum(move_probs)
        if sum_probs == 0: return [] # no legal moves
        move_probs /= sum_probs # re-normalize probabilities

        good_moves = []
        cum_prob = 0.0
        while cum_prob < self.threshold_prob:
            ind = np.argmax(move_probs)
            x,y = ind / self.model.N, ind % self.model.N
            good_moves.append((x,y))
            prob = move_probs[ind]
            cum_prob += prob
            move_probs[ind] = 0

        return good_moves


text:
import time
import scipy.sparse as sp
import numpy as np
import tensorflow as tf
import argparse
import os
from models import GAT
from models import SpGAT
from utils import process
import json

dataset = 'dblp'
checkpt_file = 'pre_trained/cora/model_'+dataset+'cora.ckpt'

# training params
batch_size = 1
nb_epochs = 100
patience = 50
lr = 0.002  # learning rate
l2_coef = 0.0005  # weight decay
hid_units = [8] # numbers of hidden units per each attention head in each layer
n_heads = [8, 1] # additional entry for the output layer
residual = False
nonlinearity = tf.nn.elu
# model = GAT
model = SpGAT

print('Dataset: ' + dataset)
print('----- Opt. hyperparams -----')
print('lr: ' + str(lr))
print('l2_coef: ' + str(l2_coef))
print('----- Archi. hyperparams -----')
print('nb. layers: ' + str(len(hid_units)))
print('nb. units per layer: ' + str(hid_units))
print('nb. attention heads: ' + str(n_heads))
print('residual: ' + str(residual))
print('nonlinearity: ' + str(nonlinearity))
print('model: ' + str(model))

sparse = True

for per_class in [200]:
    result_path = 'result/' + dataset + str(per_class) + '/'
    if not os.path.exists('result'):
        os.mkdir('result')
    if not os.path.exists(result_path):
        os.mkdir(result_path)

    metapaths, metapaths_name, adjs, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = process.load_data(
        dataset,per_class)
    features, spars = process.preprocess_features(features)

    nb_nodes = features.shape[0]
    ft_size = features.shape[1]
    nb_classes = y_train.shape[1]

    features = features[np.newaxis]
    y_train = y_train[np.newaxis]
    y_val = y_val[np.newaxis]
    y_test = y_test[np.newaxis]
    train_mask = train_mask[np.newaxis]
    val_mask = val_mask[np.newaxis]
    test_mask = test_mask[np.newaxis]

    all_metapaths_best = (0, 0, 0, 0)
    for metapath_id in range(len(metapaths)):
        best = (0, 0, 0, '')
        # 只测试APCPA
        if metapaths_name[metapath_id]!= ['AP','PC','CP','PA']:
            continue
        # if not(metapaths_name[metapath_id]!=[] and metapaths_name[metapath_id][0][0]==metapaths_name[metapath_id][-1][-1] and metapaths_name[metapath_id][0][0]=='P'):
        #     continue

        print("metapath=", metapaths_name[metapath_id], "-----------------------------")
        adj = adjs[metapath_id]
        if sparse:
            biases = process.preprocess_adj_bias(adj)
        else:
            adj = adj.todense()
            adj = adj[np.newaxis]
            biases = process.adj_to_bias(adj, [nb_nodes], nhood=1)

        with tf.Graph().as_default():
            with tf.name_scope('input'):
                ftr_in = tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, ft_size))
                if sparse:
                    # bias_idx = tf.placeholder(tf.int64)
                    # bias_val = tf.placeholder(tf.float32)
                    # bias_shape = tf.placeholder(tf.int64)
                    bias_in = tf.sparse_placeholder(dtype=tf.float32)
                else:
                    bias_in = tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, nb_nodes))
                lbl_in = tf.placeholder(dtype=tf.int32, shape=(batch_size, nb_nodes, nb_classes))
                msk_in = tf.placeholder(dtype=tf.int32, shape=(batch_size, nb_nodes))
                attn_drop = tf.placeholder(dtype=tf.float32, shape=())
                ffd_drop = tf.placeholder(dtype=tf.float32, shape=())
                is_train = tf.placeholder(dtype=tf.bool, shape=())

            logits = model.inference(ftr_in, nb_classes, nb_nodes, is_train,
                                     attn_drop, ffd_drop,
                                     bias_mat=bias_in,
                                     hid_units=hid_units, n_heads=n_heads,
                                     residual=residual, activation=nonlinearity)
            log_resh = tf.reshape(logits, [-1, nb_classes])
            lab_resh = tf.reshape(lbl_in, [-1, nb_classes])
            msk_resh = tf.reshape(msk_in, [-1])
            loss = model.masked_softmax_cross_entropy(log_resh, lab_resh, msk_resh)
            accuracy = model.masked_accuracy(log_resh, lab_resh, msk_resh)

            train_op = model.training(loss, lr, l2_coef)

            saver = tf.train.Saver()

            init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

            vlss_mn = np.inf
            vacc_mx = 0.0
            curr_step = 0

            with tf.Session() as sess:
                sess.run(init_op)

                train_loss_avg = 0
                train_acc_avg = 0
                val_loss_avg = 0
                val_acc_avg = 0

                for epoch in range(nb_epochs):
                    tr_step = 0
                    tr_size = features.shape[0]

                    while tr_step * batch_size < tr_size:
                        if sparse:
                            bbias = biases
                        else:
                            bbias = biases[tr_step * batch_size:(tr_step + 1) * batch_size]

                        _, loss_value_tr, acc_tr = sess.run([train_op, loss, accuracy],
                                                            feed_dict={
                                                                ftr_in: features[
                                                                        tr_step * batch_size:(
                                                                                tr_step + 1) * batch_size],
                                                                bias_in: bbias,
                                                                lbl_in: y_train[
                                                                        tr_step * batch_size:(
                                                                                tr_step + 1) * batch_size],
                                                                msk_in: train_mask[
                                                                        tr_step * batch_size:(
                                                                                tr_step + 1) * batch_size],
                                                                is_train: True,
                                                                attn_drop: 0.6, ffd_drop: 0.6})
                        train_loss_avg += loss_value_tr
                        train_acc_avg += acc_tr
                        tr_step += 1

                    vl_step = 0
                    vl_size = features.shape[0]

                    while vl_step * batch_size < vl_size:
                        if sparse:
                            bbias = biases
                        else:
                            bbias = biases[vl_step * batch_size:(vl_step + 1) * batch_size]
                        loss_value_vl, acc_vl = sess.run([loss, accuracy],
                                                         feed_dict={
                                                             ftr_in: features[
                                                                     vl_step * batch_size:(vl_step + 1) * batch_size],
                                                             bias_in: bbias,
                                                             lbl_in: y_val[
                                                                     vl_step * batch_size:(vl_step + 1) * batch_size],
                                                             msk_in: val_mask[
                                                                     vl_step * batch_size:(vl_step + 1) * batch_size],
                                                             is_train: False,
                                                             attn_drop: 0.0, ffd_drop: 0.0})
                        val_loss_avg += loss_value_vl
                        val_acc_avg += acc_vl
                        vl_step += 1

                    print('Epoch = %d, Training: loss = %.5f, acc = %.5f | Val: loss = %.5f, acc = %.5f' %
                          (epoch, train_loss_avg / tr_step, train_acc_avg / tr_step,
                           val_loss_avg / vl_step, val_acc_avg / vl_step))

                    # 保存最好的valid acc
                    if val_acc_avg > best[0]:
                        ts_size = features.shape[0]
                        ts_step = 0
                        ts_loss = 0.0
                        ts_acc = 0.0

                        show_feas=[]
                        test_y=[]

                        while ts_step * batch_size < ts_size:
                            if sparse:
                                bbias = biases
                            else:
                                bbias = biases[ts_step * batch_size:(ts_step + 1) * batch_size]
                            loss_value_ts, acc_ts = sess.run([loss, accuracy],
                                                             feed_dict={
                                                                 ftr_in: features[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 bias_in: bbias,
                                                                 lbl_in: y_test[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 msk_in: test_mask[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 is_train: False,
                                                                 attn_drop: 0.0, ffd_drop: 0.0})

                            tensor_dict = tf.get_default_graph().get_tensor_by_name('concat:0')
                            output = sess.run(tensor_dict, feed_dict={
                                                                 ftr_in: features[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 bias_in: bbias,
                                                                 lbl_in: y_test[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 msk_in: test_mask[
                                                                         ts_step * batch_size:(
                                                                                ts_step + 1) * batch_size],
                                                                 is_train: False,
                                                                 attn_drop: 0.0, ffd_drop: 0.0})

                            output=output[0]
                            print('batch_feature---------------', output.shape,test_mask[0].shape)
                            output = [z for z, mask in zip(output, test_mask[0]) if mask]
                            np.save("gat_show.npy", output)
                            output = [z for z, mask in zip(y_test[0], test_mask[0]) if mask]
                            np.save("gat_show_label.npy", np.array(output).argmax(axis=1))

                            ts_loss += loss_value_ts
                            ts_acc += acc_ts
                            ts_step += 1

                        print('Test loss:', ts_loss / ts_step, '; Test accuracy:', ts_acc / ts_step)
                        best = (val_acc_avg, ts_acc / ts_step, epoch, str(metapaths_name[metapath_id]))
                        result = "temp---------at_best_validate: valid accuracy=%.5f, test accuracy=%.5f, epoch=%d, metapath=%s" % (
                            best[0], best[1], best[2], best[3])
                        print(result)
                        with open(result_path + "result_%s.txt" % str(metapaths_name[metapath_id]), 'w') as fout:
                            fout.write(json.dumps(result))

                    if val_acc_avg / vl_step >= vacc_mx or val_loss_avg / vl_step <= vlss_mn:
                        if val_acc_avg / vl_step >= vacc_mx and val_loss_avg / vl_step <= vlss_mn:
                            vacc_early_model = val_acc_avg / vl_step
                            vlss_early_model = val_loss_avg / vl_step
                            saver.save(sess, checkpt_file)
                        vacc_mx = np.max((val_acc_avg / vl_step, vacc_mx))
                        vlss_mn = np.min((val_loss_avg / vl_step, vlss_mn))
                        curr_step = 0
                    else:
                        curr_step += 1
                        if curr_step == patience:
                            print('Early stop! Min loss: ', vlss_mn, ', Max accuracy: ', vacc_mx)
                            print('Early stop model validation loss: ', vlss_early_model, ', accuracy: ',
                                  vacc_early_model)
                            break

                    train_loss_avg = 0
                    train_acc_avg = 0
                    val_loss_avg = 0
                    val_acc_avg = 0

                saver.restore(sess, checkpt_file)

                ts_size = features.shape[0]
                ts_step = 0
                ts_loss = 0.0
                ts_acc = 0.0

                while ts_step * batch_size < ts_size:
                    if sparse:
                        bbias = biases
                    else:
                        bbias = biases[ts_step * batch_size:(ts_step + 1) * batch_size]
                    loss_value_ts, acc_ts = sess.run([loss, accuracy],
                                                     feed_dict={
                                                         ftr_in: features[
                                                                 ts_step * batch_size:(ts_step + 1) * batch_size],
                                                         bias_in: bbias,
                                                         lbl_in: y_test[
                                                                 ts_step * batch_size:(ts_step + 1) * batch_size],
                                                         msk_in: test_mask[
                                                                 ts_step * batch_size:(ts_step + 1) * batch_size],
                                                         is_train: False,
                                                         attn_drop: 0.0, ffd_drop: 0.0})
                    ts_loss += loss_value_ts
                    ts_acc += acc_ts
                    ts_step += 1

                print('Test loss:', ts_loss / ts_step, '; Test accuracy:', ts_acc / ts_step)

                sess.close()

                result = "at_best_validate: valid accuracy=%.5f, test accuracy=%.5f, epoch=%d, metapath=%s" % (
                    best[0], best[1], best[2], best[3])
                print(result)
                with open(result_path + "result_%s.txt" % str(metapaths_name[metapath_id]), 'w') as fout:
                    fout.write(json.dumps(result))

                if best[1] > all_metapaths_best[1]:
                    all_metapaths_best = best

        result = "best result: valid accuracy=%.5f, test accuracy=%.5f, epoch=%d, metapath=%s" % (
            all_metapaths_best[0], all_metapaths_best[1], all_metapaths_best[2], all_metapaths_best[3])
        print(result)
        with open(result_path + "best_result.txt", 'w') as fout:
            fout.write(json.dumps(result))

reason: import numpy,import scipy


text:
import os
import numpy as np
from torch import nn
import torch
from torchvision import models, transforms, datasets
import torch.nn.functional as F

import pdb

class SSLModel(nn.Module):
    def __init__(self, config):
        super().__init__()

        self.dim = config.bank_dim
        self.num_classes = config.numcls + 1

        self.mapping_module = nn.Sequential(
                                       nn.Conv2d(2048, 1024, 3, stride=1, padding=1),
                                       nn.ReLU(),
                                       nn.Conv2d(1024, self.dim, 3, stride=1, padding=1),
                                       nn.ReLU(),)
        self.mapping_cls = nn.Sequential(
                                     nn.Conv2d(self.dim, self.dim, 3, stride=2),
                                     nn.ReLU(),
                                     nn.Conv2d(self.dim, self.num_classes, 3, stride=2),
                                     nn.ReLU(),)

    def forward(self, feat):
        map_feat = self.mapping_module(feat)
        map_cls = self.mapping_cls(map_feat)
        map_cls = map_cls.view(map_cls.size(0), self.num_classes, -1)
        map_cls, _ = map_cls.max(2)
        return map_feat, map_cls


text:
import sys
sys.path.insert(0,'/home/csunix/schtmt/NewFolder/ConvLSTM_Caffe_master/caffe_master/python')
import numpy as np
import matplotlib.pyplot as plt
import caffe

caffe.set_device(0)
caffe.set_mode_gpu()

solver = caffe.SGDSolver('lstm_solver.prototxt')
solver.net.params['lstm1'][2].data[15:30]=5
a = np.arange(0,32,0.01)
d = 0.5*np.sin(2*a) - 0.05 * np.cos( 17*a + 0.8  ) + 0.05 * np.sin( 25 * a + 10 ) - 0.02 * np.cos( 45 * a + 0.3)
d = d / max(np.max(d), -np.min(d))
d = d - np.mean(d)
niter=5000
train_loss = np.zeros(niter)
solver.net.params['lstm1'][2].data[15:30]=5
solver.net.blobs['clip'].data[...] = 1
for i in range(niter) :
    seq_idx = i % (len(d) / 320)
    solver.net.blobs['clip'].data[0] = seq_idx > 0
    solver.net.blobs['label'].data[:,0] = d[ seq_idx * 320 : (seq_idx+1) * 320 ]
    solver.step(1)
    train_loss[i] = solver.net.blobs['loss'].data

#% matplotlib inline
from IPython import get_ipython
get_ipython().run_line_magic('matplotlib', 'inline')
plt.plot(np.arange(niter), train_loss)


text:
import pandas as pd
import numpy as np
import sys

def read_ply(fp, newline=None):

    if (sys.version_info > (3, 0)):
        open_file = open(fp, encoding='ISO-8859-1', newline=newline)
    else:
        open_file = open(fp)

    with open_file as ply:

        length = 0
        prop = []
        dtype_map = {'double':'d', 'float64':'f8', 'float32':'f4', 'float': 'f4', 'uchar': 'B', 'int':'i'}
        dtype = []
        fmt = 'binary'

        for i, line in enumerate(ply.readlines()):
            length += len(line)
            if i == 1:
                if 'ascii' in line:
                    fmt = 'ascii'
            if 'element vertex' in line: N = int(line.split()[2])
            if 'property' in line:
                dtype.append(dtype_map[line.split()[1]])
                prop.append(line.split()[2])
            if 'element face' in line:
                raise Exception('.ply appears to be a mesh')
            if 'end_header' in line: break

        ply.seek(length)

        if fmt == 'binary':
            arr = np.fromfile(ply, dtype=','.join(dtype))
        else:
            arr = np.loadtxt(ply)
        df = pd.DataFrame(data=arr)
        df.columns = prop

    return df

def write_ply(output_name, pc, comments=[]):

    cols = ['x', 'y', 'z']
    pc[['x', 'y', 'z']] = pc[['x', 'y', 'z']].astype('f4')

    with open(output_name, 'w') as ply:

        ply.write("ply\n")
        ply.write('format binary_little_endian 1.0\n')
        ply.write("comment Author: Phil Wilkes\n")
        for comment in comments:
            ply.write("comment {}\n".format(comment))
        ply.write("obj_info generated with pcd2ply.py\n")
        ply.write("element vertex {}\n".format(len(pc)))
        ply.write("property float x\n")
        ply.write("property float y\n")
        ply.write("property float z\n")
        if 'red' in pc.columns:
            cols += ['red', 'green', 'blue']
            pc[['red', 'green', 'blue']] = pc[['red', 'green', 'blue']].astype('i')
            ply.write("property int red\n")
            ply.write("property int green\n")
            ply.write("property int blue\n")
        for col in pc.columns:
            if col in cols: continue
            cols += [col]
            pc[col] = pc[col].astype('f4')
            ply.write("property float {}\n".format(col))
        ply.write("end_header\n")

    with open(output_name, 'ab') as ply:
        ply.write(pc[cols].to_records(index=False).tobytes())

max_stars_repo_name: philwilkes/branch-extraction

max_issues_repo_name: philwilkes/branch-extraction

max_forks_repo_name: philwilkes/branch-extraction


text:
import nltk
import numpy as np
import re
import copy

#text_listにリストとして読み込む
with open('book2.txt', 'r') as f:
    #改行("\n")を""に変換
    #text_list = f.read().splitlines()
    text = f.read()

#正規表現で"を削除
text = re.sub('"', '', text)
text = text.lower()

morph = nltk.word_tokenize(text)
pos = nltk.pos_tag(morph)
#print(pos)

kazu=0
hinsi=[]#品詞の名前
hinsi_kosuu=[]#品詞の個数．配列は品詞の名前と対応している．
list_bangou=0

be=["is","was","are","ware","am","been","'m","'re","not","n't"]
be_sinkou=["is","are","am","'m","'re"]
be_kanryou=["was","ware","been"]

kigou=0
kigou_reigai=["=","+","'"]

zisei_kazu=0 #変数
zisei=0 #時制，アスペクトの数
zisei_kurikaesi=0 #繰り返されている数

while kazu < len(pos):
    #未来形
    if pos[kazu][1] == "VBG":
        #アスペクト
        #未来形で進行
        if pos[kazu-1][0] in be_sinkou:
            if zisei_kazu==0:
                zisei_1 ="VBG_s"
            else:
                zisei_2 ="VBG_s"
            zisei+=1
        elif pos[kazu-1][0] in be_kanryou:
            if zisei_kazu==0:
                zisei_1 ="VBG_k"
            else:
                zisei_2 ="VBG_k"
            zisei_kazu+=1
            zisei+=1
    #過去形
    elif pos[kazu][1] == "VBD":
        #アスペクト
        if pos[kazu-1][0] in be_sinkou:
            if zisei_kazu==0:
                zisei_1 ="VBD_s"
            else:
                zisei_2 ="VBD_s"
            zisei+=1
        elif pos[kazu-1][0] in be_kanryou:
            if zisei_kazu==0:
                zisei_1 ="VBD_k"
            else:
                zisei_2 ="VBD_k"
            zisei_kazu+=1
            zisei+=1
    #原形
    elif pos[kazu][1] == "VB":
        if zisei_kazu==0:
            zisei_1 ="VB"
            zisei+=1
        else:
            zisei_2 ="VB"
            zisei+=1
        zisei_kazu+=1

    #時制＋アスペクトが繰り返されているか確認
    if zisei_kazu == 2:
        if zisei_1 == zisei_2:
            zisei_kurikaesi+=1
        zisei_1 = zisei_2
        zisei_kazu =1

    #いらない記号は排除
    if (re.match("\W", pos[kazu][1].lower())) and (pos[kazu][0].lower() not in kigou_reigai) :
        kigou+=1
    #品詞をリストに入れる
    #もう出ている品詞なら，hinsi_kosuuの数を１増やす
    elif pos[kazu][1] in hinsi:
        list_bangou=hinsi.index(pos[kazu][1])
        hinsi_kosuu[list_bangou]=hinsi_kosuu[list_bangou]+1

    #新しい品詞が出てきたら，hinsiリストに品詞を追加して，hinsi_kosuuリストを１にする．
    else:
        hinsi.append(pos[kazu][1])
        hinsi_kosuu.append(1)

    kazu+=1

#print(hinsi)
#print(hinsi_kosuu)

print("時制の繰り返し",zisei_kurikaesi)
print("時制",zisei)
zisei_sukoa=zisei_kurikaesi/zisei
print(zisei_sukoa)
#zentai = sum(hinsi_kosuu)
#print("総単語数",zentai)
#0.834	0.836 0.851
#0.8645465253239105 0.894649751792609 0.8840952994204765 0.9359415305245056


text:
import json
import numpy as np
import pandas as pd

floc = "./pyEnSySim/BoundaryConditions/Electrical/SLP/"
fname = ["PHH", "G0", "L0"]

for name in fname:
    with open(floc + name + ".json", 'r') as data_file:
        data = json.load(data_file)

    keys_level1 = list(data[0].keys())
    keys_level2 = []

    for key in keys_level1:
        keys_level2.append(list(data[0][key][0].keys()))

    colIdx = []

    nKeys = len(keys_level1)

    if nKeys != len(keys_level2):
        raise("Error length of level1 and level2 keys must be equal")

    for idx in range(nKeys):
        for sub_idx in range(len(keys_level2[idx])):
            colIdx.append((keys_level1[idx], keys_level2[idx][sub_idx]))

    colIdx = pd.MultiIndex.from_tuples(colIdx)

    df = pd.DataFrame(0, index=np.arange(len(data)), columns=colIdx)

    for lNr, line in enumerate(data):
        for idx in colIdx:
            df.loc[lNr, idx] = line[idx[0]][0][idx[1]]

    df.to_hdf(floc + name + ".h5", name, 'w')

max_stars_repo_name: HSZittauGoerlitz/EnSySim

max_issues_repo_name: HSZittauGoerlitz/EnSySim

max_forks_repo_name: HSZittauGoerlitz/EnSySim


text:
import math

import numpy as np
from pytest_cases import THIS_MODULE, parametrize_with_cases

from eddington.print_util import to_digit_string, to_relevant_precision_string

def case_positive_int_add_zeros():
    return 31, 4, "31.000"

def case_big_positive_int_reduce_digits():
    return 31415, 3, "3.142e+4"

def case_small_positive_int_reduce_digits():
    return 25.5033557, 4, "25.503"

def case_negative_int_add_zeros():
    return -31, 3, "-31.00"

def case_negative_int_reduce_digits():
    return -31415, 3, "-3.142e+4"

def case_one_adds_zeros():
    return 1, 3, "1.000"

def case_negative_one_add_zeros():
    return -1, 3, "-1.000"

def case_zero_add_zeros():
    return 0.0, 3, "0.000"

def case_minus_zero_add_zeros():
    return -0.0, 3, "0.000"

def case_float_bigger_than_one_reduce_digits():
    return np.pi, 2, "3.14"

def case_float_bigger_than_one_add_zeroes():
    return np.pi, 5, "3.14159"

def case_float_smaller_than_one_reduce_digits():
    return 0.712, 1, "0.71"

def case_float_smaller_than_one_add_zeroes():
    return 0.712, 4, "0.71200"

def case_small_float_reduce_digits():
    return 3.289e-5, 1, "3.3e-5"

def case_small_float_add_zeroes():
    return 3.289e-5, 4, "3.2890e-5"

def case_numpy_infinity():
    return np.inf, 3, "inf"

def case_python_infinity():
    return math.inf, 3, "inf"

def case_numpy_negative_infinity():
    return -np.inf, 3, "-inf"

def case_python_negative_infinity():
    return -math.inf, 3, "-inf"

def case_numpy_nan():
    return np.nan, 3, "nan"

def case_python_nan():
    return math.nan, 3, "nan"

@parametrize_with_cases(argnames=["a", "n", "string"], cases=THIS_MODULE)
def test_precise_string(a, n, string):
    assert (
        to_relevant_precision_string(a, relevant_digits=n) == string
    ), "Relevant precision is different than expected"

def test_inf_to_digit_string():
    assert to_digit_string(np.inf, 5) == "inf"


text:
"""Unwrap and geocode Sentinel-1 INSAR products from GAMMA"""

import argparse
import logging
import os
import subprocess
from tempfile import TemporaryDirectory

import numpy as np
from PIL import Image
from hyp3lib.execute import execute
from hyp3lib.getParameter import getParameter
from osgeo import gdal

from hyp3_gamma.water_mask import create_water_mask

log = logging.getLogger(__name__)

def get_ref_point_info(log_text: str):
    log_lines = log_text.splitlines()

    ref = [line for line in log_lines if "phase at reference point:" in line]
    ref_offset = float(ref[0].split(" ")[-2])

    init = [line for line in log_lines if "phase initialization flag:" in line]
    glb_offset = float(init[0].split(" ")[-2])
    init_flg = int(init[0].split(" ")[3])

    return {"initflg": init_flg, "refoffset": ref_offset, "glboffset": glb_offset}

def coords_from_sapix_coord(in_mli_par: str, ref_azlin: int, ref_rpix: int, in_dem_par: str) -> list:
    """
    Will return list of 6 coordinates if in_dem_par file is provided:
        row_s, col_s, row_m, col_m, y, x
    otherwise will return 4 coordinates:
        row_s, col_s, lat, lon
    with (row_s,col_s)in SAR space, and the rest of the coordinates in MAP space
    """
    cmd = ['sarpix_coord', in_mli_par, '-', in_dem_par, str(ref_azlin), str(ref_rpix)]
    log.info(f'Running command: {" ".join(cmd)}')
    result = subprocess.run(cmd, capture_output=True, text=True)
    coord_log_lines = result.stdout.splitlines()

    selected_coords = [line for line in coord_log_lines if "selected" in line]
    log.info(f'Selected sarpix coordinates: {selected_coords[0]}')
    coord_lst = selected_coords[0].split()[:-1]
    coord_lst = [float(s) for s in coord_lst]
    return coord_lst

def get_coords(in_mli_par: str, ref_azlin: int = 0, ref_rpix: int = 0, in_dem_par: str = None) -> dict:
    """
    Args:
        in_mli_par: GAMMA MLI par file
        ref_azlin: Reference azimuth line
        ref_rpix:  Reference range pixel
        in_dem_par: GAMMA DEM par file

    Returns:
        coordinates dictionary with row_s, col_s, lat, lon coordinates. Additionally, if
        in_dem_par is provided, coords will have row_m, col_m, y, and x.
    """
    row_s, col_s, lat, lon = coords_from_sapix_coord(in_mli_par, ref_azlin, ref_rpix, '-')
    coords = {"row_s": int(row_s), "col_s": int(col_s), "lat": lat, "lon": lon}

    if in_dem_par:
        _, _, _, _, y, x = coords_from_sapix_coord(in_mli_par, ref_azlin, ref_rpix, in_dem_par)
        coords["y"] = y
        coords["x"] = x

    return coords

def read_bin(file, lines: int, samples: int):
    data = np.fromfile(file, dtype='>f4')
    values = np.reshape(data, (lines, samples))
    return values

def read_bmp(file):
    im = Image.open(file)
    data = np.asarray(im)
    return data

def ref_point_with_max_cc(fcc: str, fmask: str, mlines: int, mwidth: int):
    data_mk = read_bmp(fmask)
    data_mk_max = data_mk.max()
    data_mk = np.ma.masked_values(data_mk, data_mk_max)

    data_cc = read_bin(fcc, mlines, mwidth)
    data_cc_max = data_cc[data_mk.mask].max()
    idx = np.where(data_cc == data_cc_max)
    if idx:
        ref_i = idx[0][0]
        ref_j = idx[1][0]
        return ref_i, ref_j

    return 0, 0

def geocode_back(inname, outname, width, lt, demw, demn, type_):
    execute(f"geocode_back {inname} {width} {lt} {outname} {demw} {demn} 0 {type_}", uselogging=True)

def geocode(inname, outname, inwidth, lt, outwidth, outlines, type_):
    execute(f"geocode {lt} {inname} {inwidth} {outname} {outwidth} {outlines} - {type_}", uselogging=True)

def data2geotiff(inname, outname, dempar, type_):
    execute(f"data2geotiff {dempar} {inname} {type_} {outname} ", uselogging=True)

def create_phase_from_complex(incpx, outfloat, width):
    execute(f"cpx_to_real {incpx} {outfloat} {width} 4", uselogging=True)

def get_water_mask(cc_mask_file, mwidth, lt, demw, demn, dempar):
    """
    create water_mask based on the cc_mask_file
    """
    with TemporaryDirectory() as temp_dir:
        os.system(f'cp {cc_mask_file} {temp_dir}/tmp_mask.bmp')
        # 2--bmp
        geocode_back(f'{temp_dir}/tmp_mask.bmp', f'{temp_dir}/tmp_mask_geo.bmp', mwidth, lt, demw, demn, 2)
        # 0--bmp
        data2geotiff(f'{temp_dir}/tmp_mask_geo.bmp', f'{temp_dir}/tmp_mask_geo.tif', dempar, 0)
        # create water_mask.tif file
        create_water_mask(f'{temp_dir}/tmp_mask_geo.tif', 'water_mask.tif')
    ds = gdal.Open('water_mask.tif')
    band = ds.GetRasterBand(1)
    mask = band.ReadAsArray()
    del ds
    return mask

def convert_from_sar_2_map(in_file, out_geotiff, width, lt, dempar, demw, demn, type_):

    geocode_back(in_file, "tmp.bmp", width, lt, demw, demn, type_)

    data2geotiff("tmp.bmp", out_geotiff, dempar, type_)

def combine_water_mask(cc_mask_file, mwidth, mlines, lt, demw, demn, dempar):
    """combine cc_mask with water_mask
    """
    # read the original mask file
    in_im = Image.open(cc_mask_file)
    in_data = np.array(in_im)
    in_palette = in_im.getpalette()

    with TemporaryDirectory() as temp_dir:
        # get mask data and save it into the water_mask.bmp file
        mask = get_water_mask(cc_mask_file, mwidth, lt, demw, demn, dempar)
        water_im = Image.fromarray(mask)
        water_im.putpalette(in_palette)
        water_bmp_file = f'{temp_dir}/water_mask.bmp'
        water_im.save(water_bmp_file)

        # map water_mask.bmp file to SAR coordinators
        water_mask_bmp_sar_file = f'{temp_dir}/water_mask_sar.bmp'
        geocode(water_bmp_file, water_mask_bmp_sar_file, demw, lt, mwidth, mlines, 2)

        # read water_mask_bmp_sar_file
        water_mask_sar_im = Image.open(water_mask_bmp_sar_file)
        water_mask_sar_data = np.array(water_mask_sar_im)

        # combine two masks and output combined_mask.bmp
        in_data[water_mask_sar_data == 0] = 0
        out_im = Image.fromarray(in_data)
        out_im.putpalette(in_palette)
        out_file = os.path.join(os.path.dirname(cc_mask_file), "combined_mask.bmp")
        out_im.save(out_file)

    return out_file

def unwrapping_geocoding(reference, secondary, step="man", rlooks=10, alooks=2, trimode=0,
                         npatr=1, npata=1, alpha=0.6, apply_water_mask=False):

    dem = "./DEM/demseg"
    dempar = "./DEM/demseg.par"
    lt = "./DEM/MAP2RDC"
    ifgname = "{}_{}".format(reference, secondary)
    offit = "{}.off.it".format(ifgname)
    mmli = reference + ".mli"
    smli = secondary + ".mli"

    if not os.path.isfile(dempar):
        log.error("ERROR: Unable to find dem par file {}".format(dempar))

    if not os.path.isfile(lt):
        log.error("ERROR: Unable to find look up table file {}".format(lt))

    if not os.path.isfile(offit):
        log.error("ERROR: Unable to find offset file {}".format(offit))

    width = getParameter(offit, "interferogram_width")
    mwidth = getParameter(mmli + ".par", "range_samples")
    mlines = getParameter(mmli + ".par", "azimuth_lines")
    swidth = getParameter(smli + ".par", "range_samples")
    demw = getParameter(dempar, "width")
    demn = getParameter(dempar, "nlines")

    ifgf = "{}.diff0.{}".format(ifgname, step)

    log.info("{} will be used for unwrapping and geocoding".format(ifgf))

    log.info("-------------------------------------------------")
    log.info("            Start unwrapping")
    log.info("-------------------------------------------------")

    execute(f"cc_wave {ifgf} {mmli} - {ifgname}.cc {width}", uselogging=True)

    execute(f"adf {ifgf} {ifgf}.adf {ifgname}.adf.cc {width} {alpha} - 5", uselogging=True)

    execute(f"rasmph_pwr {ifgf}.adf {mmli} {width}", uselogging=True)

    execute(f"rascc_mask {ifgname}.adf.cc {mmli} {width} 1 1 0 1 1 0.10 0.20 ", uselogging=True)

    if apply_water_mask:
        # create and apply water mask
        out_file = combine_water_mask(f"{ifgname}.adf.cc_mask.bmp", mwidth, mlines, lt, demw, demn, dempar)
    else:
        # create water mask only
        _ = get_water_mask(f"{ifgname}.adf.cc_mask.bmp", mwidth, lt, demw, demn, dempar)
        out_file = f"{ifgname}.adf.cc_mask.bmp"

    ref_azlin, ref_rpix = ref_point_with_max_cc(f"{ifgname}.cc", out_file, int(mlines), int(mwidth))

    mcf_log = execute(f"mcf {ifgf}.adf {ifgname}.adf.cc {out_file} {ifgname}.adf.unw {width} {trimode} 0 0"
                      f" - - {npatr} {npata} - {ref_rpix} {ref_azlin} 1", uselogging=True)

    ref_point_info = get_ref_point_info(mcf_log)

    coords = get_coords(f"{mmli}.par", ref_azlin=ref_azlin, ref_rpix=ref_rpix, in_dem_par=dempar)

    execute(f"rasdt_pwr {ifgname}.adf.unw {mmli} {width} - - - - - {6 * np.pi} 1 rmg.cm {ifgname}.adf.unw.ras",
            uselogging=True)

    execute(f"dispmap {ifgname}.adf.unw DEM/HGT_SAR_{rlooks}_{alooks} {mmli}.par"
            f" - {ifgname}.vert.disp 1", uselogging=True)

    execute(f"dispmap {ifgname}.adf.unw DEM/HGT_SAR_{rlooks}_{alooks} {mmli}.par"
            f" - {ifgname}.los.disp 0", uselogging=True)

    execute(f"gc_map2 {mmli}.par DEM/demseg.par 0 - - - - - - - inc_ell")

    log.info("-------------------------------------------------")
    log.info("            End unwrapping")
    log.info("-------------------------------------------------")

    log.info("-------------------------------------------------")
    log.info("            Start geocoding")
    log.info("-------------------------------------------------")

    geocode_back(mmli, mmli + ".geo", mwidth, lt, demw, demn, 0)
    geocode_back(smli, smli + ".geo", swidth, lt, demw, demn, 0)
    geocode_back("{}.sim_unw".format(ifgname), "{}.sim_unw.geo".format(ifgname), width, lt, demw, demn, 0)
    geocode_back("{}.adf.unw".format(ifgname), "{}.adf.unw.geo".format(ifgname), width, lt, demw, demn, 0)
    geocode_back("{}.adf".format(ifgf), "{}.adf.geo".format(ifgf), width, lt, demw, demn, 1)
    geocode_back("{}.adf.unw.ras".format(ifgname), "{}.adf.unw.geo.bmp".format(ifgname), width, lt, demw, demn, 2)
    geocode_back("{}.adf.bmp".format(ifgf), "{}.adf.bmp.geo".format(ifgf), width, lt, demw, demn, 2)
    geocode_back("{}.cc".format(ifgname), "{}.cc.geo".format(ifgname), width, lt, demw, demn, 0)
    geocode_back("{}.adf.cc".format(ifgname), "{}.adf.cc.geo".format(ifgname), width, lt, demw, demn, 0)
    geocode_back("{}.vert.disp".format(ifgname), "{}.vert.disp.geo".format(ifgname), width, lt, demw, demn, 0)
    geocode_back("{}.los.disp".format(ifgname), "{}.los.disp.geo".format(ifgname), width, lt, demw, demn, 0)

    create_phase_from_complex("{}.adf.geo".format(ifgf), "{}.adf.geo.phase".format(ifgf), width)

    data2geotiff(mmli + ".geo", mmli + ".geo.tif", dempar, 2)
    data2geotiff(smli + ".geo", smli + ".geo.tif", dempar, 2)
    data2geotiff("{}.sim_unw.geo".format(ifgname), "{}.sim_unw.geo.tif".format(ifgname), dempar, 2)
    data2geotiff("{}.adf.unw.geo".format(ifgname), "{}.adf.unw.geo.tif".format(ifgname), dempar, 2)
    data2geotiff("{}.adf.unw.geo.bmp".format(ifgname), "{}.adf.unw.geo.bmp.tif".format(ifgname), dempar, 0)
    data2geotiff("{}.adf.geo.phase".format(ifgf), "{}.adf.geo.tif".format(ifgf), dempar, 2)
    data2geotiff("{}.adf.bmp.geo".format(ifgf), "{}.adf.bmp.geo.tif".format(ifgf), dempar, 0)
    data2geotiff("{}.cc.geo".format(ifgname), "{}.cc.geo.tif".format(ifgname), dempar, 2)
    data2geotiff("{}.adf.cc.geo".format(ifgname), "{}.adf.cc.geo.tif".format(ifgname), dempar, 2)
    data2geotiff("DEM/demseg", "{}.dem.tif".format(ifgname), dempar, 2)
    data2geotiff("{}.vert.disp.geo".format(ifgname), "{}.vert.disp.geo.org.tif".format(ifgname), dempar, 2)
    data2geotiff("{}.los.disp.geo".format(ifgname), "{}.los.disp.geo.org.tif".format(ifgname), dempar, 2)
    data2geotiff("DEM/inc", "{}.inc.tif".format(ifgname), dempar, 2)
    data2geotiff("inc_ell", "{}.inc_ell.tif".format(ifgname), dempar, 2)
    execute(f"look_vector {mmli}.par {offit} {dempar} {dem} lv_theta lv_phi", uselogging=True)

    data2geotiff("lv_theta", "{}.lv_theta.tif".format(ifgname), dempar, 2)
    data2geotiff("lv_phi", "{}.lv_phi.tif".format(ifgname), dempar, 2)

    log.info("-------------------------------------------------")
    log.info("            End geocoding")
    log.info("-------------------------------------------------")

    return coords, ref_point_info

def main():
    """Main entrypoint"""
    parser = argparse.ArgumentParser(
        prog='unwrapping_geocoding.py',
        description=__doc__,
    )
    parser.add_argument("reference", help='Reference scene identifier')
    parser.add_argument("secondary", help='Secondary scene identifier')
    parser.add_argument("-s", "--step", default='man', help='Level of interferogram for unwrapping (def=man)')
    parser.add_argument("-r", "--rlooks", default=10, help="Number of range looks (def=10)")
    parser.add_argument("-a", "--alooks", default=2, help="Number of azimuth looks (def=2)")
    parser.add_argument("-t", "--tri", default=0,
                        help="Triangulation method for mcf unwrapper: "
                             "0) filled traingular mesh (default); 1) Delaunay triangulation")
    parser.add_argument("--alpha", default=0.6, type=float, help="adf filter alpha value (def=0.6)")
    parser.add_argument("--npatr", default=1, help="Number of patches in range (def=1)")
    parser.add_argument("--npata", default=1, help="Number of patches in azimuth (def=1)")
    args = parser.parse_args()

    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                        datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO)

    unwrapping_geocoding(args.reference, args.secondary, step=args.step, rlooks=args.rlooks, alooks=args.alooks,
                         trimode=args.tri, npatr=args.npatr, npata=args.npata, alpha=args.alpha)

if __name__ == "__main__":
    main()


text:
#-------------------------------------------------------------------------------
# Name:        Area.py
# Purpose:     Green Scale Tool Area Module (Calculates surface area based on any set of coordinates)
#
# Author:      Holly Tina Ferguson
#
# Created:     15/09/2013
# Copyright:   (c) Holly Tina Ferguson 2013
# Licence:     The University of Notre Dame
#-------------------------------------------------------------------------------
import math
from objects.BaseElement import BaseElement
from objects.Shadow import Shadow
from numpy import *
import numpy

class Area(BaseElement):
    # Cartesian point
    cartesian_point = None

    # Elements
    openings = list()
    constructions = list()
    layers = list()
    materials = list()

    def surfaceArea(self, coordinates, az, tilt, surface):
        """
        Determines the area of an object from a set of coordinate points (window, surface, etc.)
        """
        azi_surf_rad = az*(math.pi/180)
        tilt_surf_rad = tilt*(math.pi/180)
        X_rec = coordinates  #coordinates == surface.cps  # [4][3] set of coordinates of surface

        #z = zeros((len(coordinates), 1))
        #row = 0
        #while row < len(z):
        #    col = 0
        #    z[row] = surface.cps[row][2]
        #    row += 1

        #print X_rec
        #print X_rec[0]
        self.shadow = Shadow()
        X1, u1, v1, w1 = self.shadow.wc2rc(X_rec, X_rec[0], azi_surf_rad, tilt_surf_rad)
        area = self.shadow.polyarea2(X1)

        return area

    def createAreaDictionary(self):
        global areaDictionary
        areaDictionary = dict()

    def addArea(self, index, value):
        areaDictionary[index] = value
        #print index, value

    def getArea(self, index):
        return areaDictionary.get(index)

    def getDictionary(self):
        return areaDictionary

    def createWinAreaDictionary(self):
        global areaWinDictionary
        areaWinDictionary = dict()

    def addWinArea(self, index, value):
        areaWinDictionary[index] = value
        #print index, value

    def getWinArea(self, index):
        return areaWinDictionary.get(index)

    def getWinDictionary(self):
        return areaWinDictionary

max_stars_repo_name: Green-Resilience/wfRunner

max_issues_repo_name: Green-Resilience/wfRunner

max_forks_repo_name: Green-Resilience/wfRunner

reason: import numpy,from numpy


text:
import numpy as np

BLUE = 0
WHITE = 1
YELLOW = 2
GREEN = 3

colors = {
            "blue": BLUE,
            "white": WHITE,
            "yellow": YELLOW,
            "green": GREEN
         }

def get_can_color_mapping(cans):
    can_to_colors = {}
    for can in cans:
        color = np.random.choice(list(colors.keys()))
        can_to_colors[can.lower()] = (color, colors[color])

    return can_to_colors

max_issues_repo_name: Algorithmic-Alignment-Lab/OpenTAMP

max_forks_repo_name: Algorithmic-Alignment-Lab/OpenTAMP


text:
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.

import logging

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as init
from ml.rl import types as rlt
from ml.rl.models.base import ModelBase
from ml.rl.models.fully_connected_network import gaussian_fill_w_gain
from ml.rl.tensorboardX import SummaryWriterContext

logger = logging.getLogger(__name__)

class DuelingQNetwork(ModelBase):
    def __init__(self, layers, activations, use_batch_norm=False, action_dim=0) -> None:
        """
        Dueling Q-Network Architecture: https://arxiv.org/abs/1511.06581

        :param layers: List of layer dimensions
        :param activations: List of layer activations
        :param use_batch_norm: bool indicating whether to apply batch normalization
        :param action_dim: if !=0 use parametric dueling DQN, else standard dueling DQN
        """
        super().__init__()
        self.layers: nn.ModuleList = nn.ModuleList()
        self.batch_norm_ops: nn.ModuleList = nn.ModuleList()
        self.activations = activations
        self.use_batch_norm = use_batch_norm

        assert len(layers) >= 3, "Invalid layer schema {} for network".format(layers)
        assert (
            len(layers) == len(activations) + 1
        ), "Invalid activation schema {} for network".format(activations)
        assert (
            layers[-2] % 2 == 0
        ), """Last shared layer in dueling architecture should be
        divisible by 2."""

        self.state_dim = layers[0]
        self.action_dim = action_dim

        for i, layer in enumerate(layers[1:-1]):
            self.layers.append(nn.Linear(layers[i], layer))
            self.batch_norm_ops.append(nn.BatchNorm1d(layers[i]))
            gaussian_fill_w_gain(self.layers[i].weight, self.activations[i], layers[i])
            init.constant_(self.layers[i].bias, 0)

        self.parametric_action = action_dim > 0
        # Split last layer into a value & advantage stream
        self.advantage = nn.Sequential(
            nn.Linear(int(layers[-2] + action_dim), int(layers[-2] / 2)),
            nn.ReLU(),
            nn.Linear(int(layers[-2] / 2), layers[-1]),
        )
        self.value = nn.Sequential(
            nn.Linear(int(layers[-2]), int(layers[-2] / 2)),
            nn.ReLU(),
            nn.Linear(int(layers[-2] / 2), 1),
        )
        self._name = "unnamed"

    def input_prototype(self):
        if self.parametric_action:
            return rlt.StateAction(
                state=rlt.FeatureVector(float_features=torch.randn(1, self.state_dim)),
                action=rlt.FeatureVector(
                    float_features=torch.randn(1, self.action_dim)
                ),
            )
        else:
            return rlt.StateInput(
                state=rlt.FeatureVector(float_features=torch.randn(1, self.state_dim))
            )

    def forward(self, input) -> torch.FloatTensor:
        output_tensor = False
        if isinstance(input, torch.Tensor):
            # Maintaining backward compatibility for a bit
            state_dim = self.layers[0].in_features
            state = input[:, :state_dim]
            action = input[:, state_dim:]
            output_tensor = True
        elif self.parametric_action:
            state = input.state.float_features
            action = input.action.float_features
        else:
            state = input.state.float_features
            action = None

        x = state
        for i, activation in enumerate(self.activations[:-1]):
            if self.use_batch_norm:
                x = self.batch_norm_ops[i](x)

            x = self.layers[i](x)
            if activation == "linear":
                continue
            elif activation == "tanh":
                activation_func = torch.tanh
            else:
                activation_func = getattr(F, activation)
            x = activation_func(x)

        value = self.value(x)
        if action is not None:
            x = torch.cat((x, action), dim=1)
        raw_advantage = self.advantage(x)
        if self.parametric_action:
            advantage = raw_advantage
        else:
            advantage = raw_advantage - raw_advantage.mean(dim=1, keepdim=True)

        q_value = value + advantage

        if SummaryWriterContext._global_step % 1000 == 0:
            SummaryWriterContext.add_histogram(
                "dueling_network/{}/value".format(self._name), value.detach().cpu()
            )
            SummaryWriterContext.add_scalar(
                "dueling_network/{}/mean_value".format(self._name),
                value.detach().mean().cpu(),
            )
            SummaryWriterContext.add_histogram(
                "dueling_network/{}/q_value".format(self._name), q_value.detach().cpu()
            )
            SummaryWriterContext.add_scalar(
                "dueling_network/{}/mean_q_value".format(self._name),
                q_value.detach().mean().cpu(),
            )
            SummaryWriterContext.add_histogram(
                "dueling_network/{}/raw_advantage".format(self._name),
                raw_advantage.detach().cpu(),
            )
            SummaryWriterContext.add_scalar(
                "dueling_network/{}/mean_raw_advantage".format(self._name),
                raw_advantage.detach().mean().cpu(),
            )
            if not self.parametric_action:
                for i in range(advantage.shape[1]):
                    a = advantage.detach()[:, i]
                    SummaryWriterContext.add_histogram(
                        "dueling_network/{}/advatage/{}".format(self._name, i), a.cpu()
                    )
                    SummaryWriterContext.add_scalar(
                        "dueling_network/{}/mean_advatage/{}".format(self._name, i),
                        a.mean().cpu(),
                    )

        if output_tensor:
            return q_value
        elif self.parametric_action:
            return rlt.SingleQValue(q_value=q_value)
        else:
            return rlt.AllActionQValues(q_values=q_value)

max_stars_repo_name: ccphillippi/Horizon

max_issues_repo_name: ccphillippi/Horizon

max_forks_repo_name: ccphillippi/Horizon


text:
import numpy as np
import matplotlib.pyplot as plt
from tools import get_array, dags, names

data = dags[0]

find_or_add_index = get_array(data, "find_or_add index")
find_or_add_time = get_array(data, "find_or_add time")
find_or_add_add = get_array(data, "find_or_add add")
find_or_add_level = get_array(data, "find_or_add level")

kwargs = {"linestyle": "None", "marker": "o", "markersize": 3}

plt.xlabel("index")
plt.ylabel("time")

indices_add = find_or_add_add == 1
indices_noadd = find_or_add_add == 0

print("Num add: ", np.sum(indices_add))
print("Num no add: ", np.sum(indices_noadd))

plt.title(names[0])

assert np.sum(indices_add) + np.sum(indices_noadd) == len(find_or_add_index)

plot_index = 0
for level in range(32):
    level_indices = level == find_or_add_level
    if np.sum(level_indices) == 0:
        continue

    plot_index += 1
    plt.subplot(2, 3, plot_index)
    plt.title("level = " + str(level))
    plt.plot(find_or_add_index[np.logical_and(indices_add, level_indices)],
             find_or_add_time[np.logical_and(indices_add, level_indices)], color="red", **kwargs)
    plt.plot(find_or_add_index[np.logical_and(indices_noadd, level_indices)],
             find_or_add_time[np.logical_and(indices_noadd, level_indices)], color="blue", **kwargs)
plt.show()

max_stars_repo_name: MaximeDup/HashDAG

max_issues_repo_name: MaximeDup/HashDAG

max_forks_repo_name: MaximeDup/HashDAG