text:
'''
This script is for testing the curve fitting of the perturber mass
'''

import numpy as np; import pandas as pd; import matplotlib.pyplot as plt
import os; from random import choices
pd.options.mode.chained_assignment = None

def gauss(x, h, mu, sigma):
    return h*np.exp(-((x-mu)**2)/(2*sigma**2))

G = 1.908e5 # R_sol*(M_sol)^-1*km^2*s^-2
cwd = os.getcwd()

COMPAS_Results_path = "/COMPAS_Output_1%solar_metallicity"
SN = pd.read_csv(cwd + COMPAS_Results_path + "/BSE_Supernovae.csv", skiprows=2)
SP = pd.read_csv(cwd + COMPAS_Results_path + "/BSE_System_Parameters.csv", skiprows=2)

invalidVals = SN.loc[(SN["SystemicSpeed "] == "          -nan")|(SN["SystemicSpeed "] == "          -nan")|(SN["SystemicSpeed "] == "          -nan")]
if len(invalidVals)>0:
    print("{} systems dropped".format(len(invalidVals)))
    SN.drop(invalidVals.index, inplace=True)

SN = SN.astype({"SystemicSpeed ":"float64",
                "ComponentSpeed(SN)":"float64",
                "ComponentSpeed(CP)":"float64",
                "SemiMajorAxis ":"float64",
                " Eccentricity ":"float64"})

# systems undergoing a second SN
SN_dup_1 = SN.loc[SN.duplicated(subset="    SEED    ", keep = "last")]

# BHBH systems that remain bound
BHB = SN.loc[(SN["Unbound"]==0)&(SN["Stellar_Type(SN)"]==14)&(SN["Stellar_Type(CP)"]==14)]
BHB.reset_index(drop=True, inplace=True)

# BHBH systems which become unbound
BHB_unbound = SN.loc[(SN["Unbound"]==1)&(SN["Stellar_Type(SN)"]==14)&(SN["Stellar_Type(CP)"]==14)]
BHB_unbound.reset_index(drop=True, inplace=True)

# Black holes formed in the first supernovae which break the binary
BH1_unbound = SN.loc[(SN["Unbound"]==1)&(SN["Stellar_Type(SN)"]==14)&(~SN["    SEED    "].isin(SN_dup_1["    SEED    "]))]
BH1_unbound.reset_index(drop=True, inplace=True)

v_esc = np.array([20]) # Cluster escape velocity kms^-1
i=0

# Ejected on first SN?
retained_from_first = SN_dup_1.loc[SN_dup_1["SystemicSpeed "]<v_esc[i]]

# Number of bound BHBH systems that are retained
retained_bound = BHB.loc[(BHB["SystemicSpeed "]<v_esc[i])&(BHB["    SEED    "].isin(retained_from_first["    SEED    "]))]

# Now look at the number of retained lone BHs
retained_unbound_first_mass = BH1_unbound["   Mass(SN)   "].loc[BH1_unbound["ComponentSpeed(SN)"]<v_esc[i]]
retained_unbound_second_mass1 = BHB_unbound["   Mass(SN)   "].loc[(BHB_unbound["ComponentSpeed(SN)"]<v_esc[i])&(BHB_unbound["    SEED    "].isin(retained_from_first["    SEED    "]))]
retained_unbound_second_mass2 = BHB_unbound["   Mass(CP)   "].loc[(BHB_unbound["ComponentSpeed(CP)"]<v_esc[i])&(BHB_unbound["    SEED    "].isin(retained_from_first["    SEED    "]))]

lone_mass = np.append(retained_unbound_first_mass.values, retained_unbound_second_mass1.values)
lone_mass = np.append(lone_mass, retained_unbound_second_mass2)

values, bins = np.histogram(lone_mass, bins = range(0, round(max(lone_mass)), 1), density = True)
bin_mid = np.array([(bins[i+1]+bins[i])/2 for i in range(len(bins)-1)])

m_perturb = choices(bin_mid, weights=values, k=len(retained_bound))

plt.hist(m_perturb, bins = range(0, round(max(lone_mass)), 1), histtype = "step")

plt.show()

max_stars_repo_path: Curvefitting perturber mass.py

max_issues_repo_path: Curvefitting perturber mass.py

max_forks_repo_path: Curvefitting perturber mass.py


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
.. _vta-get-started:

Get Started with VTA
====================
**Author**: `Thierry Moreau <https://homes.cs.washington.edu/~moreau/>`_

This is an introduction tutorial on how to use TVM to program the VTA design.

In this tutorial, we will demonstrate the basic TVM workflow to implement
a vector addition on the VTA design's vector ALU.
This process includes specific scheduling transformations necessary to lower
computation down to low-level accelerator operations.

To begin, we need to import TVM which is our deep learning optimizing compiler.
We also need to import the VTA python package which contains VTA specific
extensions for TVM to target the VTA design.
"""
from __future__ import absolute_import, print_function

import os
import tvm
from tvm import te
import vta
import numpy as np

######################################################################
# Loading in VTA Parameters
# ~~~~~~~~~~~~~~~~~~~~~~~~~
# VTA is a modular and customizable design. Consequently, the user
# is free to modify high-level hardware parameters that affect
# the hardware design layout.
# These parameters are specified in the :code:`vta_config.json` file by their
# :code:`log2` values.
# These VTA parameters can be loaded with the :code:`vta.get_env`
# function.
#
# Finally, the TVM target is also specified in the :code:`vta_config.json` file.
# When set to *sim*, execution will take place inside of a behavioral
# VTA simulator.
# If you want to run this tutorial on the Pynq FPGA development platform,
# follow the *VTA Pynq-Based Testing Setup* guide.

env = vta.get_env()

######################################################################
# FPGA Programming
# ----------------
# When targeting the Pynq FPGA development board, we need to configure
# the board with a VTA bitstream.

# We'll need the TVM RPC module and the VTA simulator module
from tvm import rpc
from tvm.contrib import utils
from vta.testing import simulator

# We read the Pynq RPC host IP address and port number from the OS environment
host = os.environ.get("VTA_RPC_HOST", "192.168.2.99")
port = int(os.environ.get("VTA_RPC_PORT", "9091"))

# We configure both the bitstream and the runtime system on the Pynq
# to match the VTA configuration specified by the vta_config.json file.
if env.TARGET == "pynq" or env.TARGET == "de10nano":

    # Make sure that TVM was compiled with RPC=1
    assert tvm.runtime.enabled("rpc")
    remote = rpc.connect(host, port)

    # Reconfigure the JIT runtime
    vta.reconfig_runtime(remote)

    # Program the FPGA with a pre-compiled VTA bitstream.
    # You can program the FPGA with your own custom bitstream
    # by passing the path to the bitstream file instead of None.
    vta.program_fpga(remote, bitstream=None)

# In simulation mode, host the RPC server locally.
elif env.TARGET == "sim":
    remote = rpc.LocalSession()

######################################################################
# Computation Declaration
# -----------------------
# As a first step, we need to describe our computation.
# TVM adopts tensor semantics, with each intermediate result
# represented as multi-dimensional array. The user needs to describe
# the computation rule that generates the output tensors.
#
# In this example we describe a vector addition, which requires multiple
# computation stages, as shown in the dataflow diagram below.
# First we describe the input tensors :code:`A` and :code:`B` that are living
# in main memory.
# Second, we need to declare intermediate tensors :code:`A_buf` and
# :code:`B_buf`, which will live in VTA's on-chip buffers.
# Having this extra computational stage allows us to explicitly
# stage cached reads and writes.
# Third, we describe the vector addition computation which will
# add :code:`A_buf` to :code:`B_buf` to produce :code:`C_buf`.
# The last operation is a cast and copy back to DRAM, into results tensor
# :code:`C`.
#
# .. image:: https://raw.githubusercontent.com/uwsaml/web-data/main/vta/tutorial/vadd_dataflow.png
#      :align: center

######################################################################
# Input Placeholders
# ~~~~~~~~~~~~~~~~~~
# We describe the placeholder tensors :code:`A`, and :code:`B` in a tiled data
# format to match the data layout requirements imposed by the VTA vector ALU.
#
# For VTA's general purpose operations such as vector adds, the tile size is
# :code:`(env.BATCH, env.BLOCK_OUT)`.
# The dimensions are specified in
# the :code:`vta_config.json` configuration file and are set by default to
# a (1, 16) vector.
#
# In addition, A and B's data types also needs to match the :code:`env.acc_dtype`
# which is set by the :code:`vta_config.json` file to be a 32-bit integer.

# Output channel factor m - total 64 x 16 = 1024 output channels
m = 64
# Batch factor o - total 1 x 1 = 1
o = 1
# A placeholder tensor in tiled data format
A = te.placeholder((o, m, env.BATCH, env.BLOCK_OUT), name="A", dtype=env.acc_dtype)
# B placeholder tensor in tiled data format
B = te.placeholder((o, m, env.BATCH, env.BLOCK_OUT), name="B", dtype=env.acc_dtype)

######################################################################
# Copy Buffers
# ~~~~~~~~~~~~
# One specificity of hardware accelerators, is that on-chip memory has to be
# explicitly managed.
# This means that we'll need to describe intermediate tensors :code:`A_buf`
# and :code:`B_buf` that can have a different memory scope than the original
# placeholder tensors :code:`A` and :code:`B`.
#
# Later in the scheduling phase, we can tell the compiler that :code:`A_buf`
# and :code:`B_buf` will live in the VTA's on-chip buffers (SRAM), while
# :code:`A` and :code:`B` will live in main memory (DRAM).
# We describe A_buf and B_buf as the result of a compute
# operation that is the identity function.
# This can later be interpreted by the compiler as a cached read operation.

# A copy buffer
A_buf = te.compute((o, m, env.BATCH, env.BLOCK_OUT), lambda *i: A(*i), "A_buf")
# B copy buffer
B_buf = te.compute((o, m, env.BATCH, env.BLOCK_OUT), lambda *i: B(*i), "B_buf")

######################################################################
# Vector Addition
# ~~~~~~~~~~~~~~~
# Now we're ready to describe the vector addition result tensor :code:`C`,
# with another compute operation.
# The compute function takes the shape of the tensor, as well as a lambda
# function that describes the computation rule for each position of the tensor.
#
# No computation happens during this phase, as we are only declaring how
# the computation should be done.

# Describe the in-VTA vector addition
C_buf = te.compute(
    (o, m, env.BATCH, env.BLOCK_OUT),
    lambda *i: A_buf(*i).astype(env.acc_dtype) + B_buf(*i).astype(env.acc_dtype),
    name="C_buf",
)

######################################################################
# Casting the Results
# ~~~~~~~~~~~~~~~~~~~
# After the computation is done, we'll need to send the results computed by VTA
# back to main memory.

######################################################################
# .. note::
#
#   **Memory Store Restrictions**
#
#   One specificity of VTA is that it only supports DRAM stores in the narrow
#   :code:`env.inp_dtype` data type format.
#   This lets us reduce the data footprint for memory transfers (more on this
#   in the basic matrix multiply example).
#
# We perform one last typecast operation to the narrow
# input activation data format.

# Cast to output type, and send to main memory
C = te.compute(
    (o, m, env.BATCH, env.BLOCK_OUT), lambda *i: C_buf(*i).astype(env.inp_dtype), name="C"
)

######################################################################
# This concludes the computation declaration part of this tutorial.

######################################################################
# Scheduling the Computation
# --------------------------
# While the above lines describes the computation rule, we can obtain
# :code:`C` in many ways.
# TVM asks the user to provide an implementation of the computation called
# *schedule*.
#
# A schedule is a set of transformations to an original computation that
# transforms the implementation of the computation without affecting
# correctness.
# This simple VTA programming tutorial aims to demonstrate basic schedule
# transformations that will map the original schedule down to VTA hardware
# primitives.

######################################################################
# Default Schedule
# ~~~~~~~~~~~~~~~~
# After we construct the schedule, by default the schedule computes
# :code:`C` in the following way:

# Let's take a look at the generated schedule
s = te.create_schedule(C.op)

print(tvm.lower(s, [A, B, C], simple_mode=True))

######################################################################
# Although this schedule makes sense, it won't compile to VTA.
# In order to obtain correct code generation, we need to apply scheduling
# primitives and code annotation that will transform the schedule into
# one that can be directly lowered onto VTA hardware intrinsics.
# Those include:
#
#  - DMA copy operations which will take globally-scoped tensors and copy
#    those into locally-scoped tensors.
#  - Vector ALU operations that will perform the vector add.

######################################################################
# Buffer Scopes
# ~~~~~~~~~~~~~
# First, we set the scope of the copy buffers to indicate to TVM that these
# intermediate tensors will be stored in the VTA's on-chip SRAM buffers.
# Below, we tell TVM that :code:`A_buf`, :code:`B_buf`, :code:`C_buf`
# will live in VTA's on-chip *accumulator buffer* which serves as
# VTA's general purpose register file.
#
# Set the intermediate tensors' scope to VTA's on-chip accumulator buffer
s[A_buf].set_scope(env.acc_scope)
s[B_buf].set_scope(env.acc_scope)
s[C_buf].set_scope(env.acc_scope)

######################################################################
# DMA Transfers
# ~~~~~~~~~~~~~
# We need to schedule DMA transfers to move data living in DRAM to
# and from the VTA on-chip buffers.
# We insert :code:`dma_copy` pragmas to indicate to the compiler
# that the copy operations will be performed in bulk via DMA,
# which is common in hardware accelerators.

# Tag the buffer copies with the DMA pragma to map a copy loop to a
# DMA transfer operation
s[A_buf].pragma(s[A_buf].op.axis[0], env.dma_copy)
s[B_buf].pragma(s[B_buf].op.axis[0], env.dma_copy)
s[C].pragma(s[C].op.axis[0], env.dma_copy)

######################################################################
# ALU Operations
# ~~~~~~~~~~~~~~
# VTA has a vector ALU that can perform vector operations on tensors
# in the accumulator buffer.
# In order to tell TVM that a given operation needs to be mapped to the
# VTA's vector ALU, we need to explicitly tag the vector addition loop
# with an :code:`env.alu` pragma.

# Tell TVM that the computation needs to be performed
# on VTA's vector ALU
s[C_buf].pragma(C_buf.op.axis[0], env.alu)

# Let's take a look at the finalized schedule
print(vta.lower(s, [A, B, C], simple_mode=True))

######################################################################
# This concludes the scheduling portion of this tutorial.

######################################################################
# TVM Compilation
# ---------------
# After we have finished specifying the schedule, we can compile it
# into a TVM function. By default TVM compiles into a type-erased
# function that can be directly called from python side.
#
# In the following line, we use :code:`tvm.build` to create a function.
# The build function takes the schedule, the desired signature of the
# function(including the inputs and outputs) as well as target language
# we want to compile to.
#
my_vadd = vta.build(s, [A, B, C], "ext_dev", env.target_host, name="my_vadd")

######################################################################
# Saving the Module
# ~~~~~~~~~~~~~~~~~
# TVM lets us save our module into a file so it can loaded back later. This
# is called ahead-of-time compilation and allows us to save some compilation
# time.
# More importantly, this allows us to cross-compile the executable on our
# development machine and send it over to the Pynq FPGA board over RPC for
# execution.

# Write the compiled module into an object file.
temp = utils.tempdir()
my_vadd.save(temp.relpath("vadd.o"))

# Send the executable over RPC
remote.upload(temp.relpath("vadd.o"))

######################################################################
# Loading the Module
# ~~~~~~~~~~~~~~~~~~
# We can load the compiled module from the file system to run the code.

f = remote.load_module("vadd.o")

######################################################################
# Running the Function
# --------------------
# The compiled TVM function uses a concise C API and can be invoked from
# any language.
#
# TVM provides an array API in python to aid quick testing and prototyping.
# The array API is based on `DLPack <https://github.com/dmlc/dlpack>`_ standard.
#
# - We first create a remote context (for remote execution on the Pynq).
# - Then :code:`tvm.nd.array` formats the data accordingly.
# - :code:`f()` runs the actual computation.
# - :code:`asnumpy()` copies the result array back in a format that can be
#   interpreted.
#

# Get the remote device context
ctx = remote.ext_dev(0)

# Initialize the A and B arrays randomly in the int range of (-128, 128]
A_orig = np.random.randint(-128, 128, size=(o * env.BATCH, m * env.BLOCK_OUT)).astype(A.dtype)
B_orig = np.random.randint(-128, 128, size=(o * env.BATCH, m * env.BLOCK_OUT)).astype(B.dtype)

# Apply packing to the A and B arrays from a 2D to a 4D packed layout
A_packed = A_orig.reshape(o, env.BATCH, m, env.BLOCK_OUT).transpose((0, 2, 1, 3))
B_packed = B_orig.reshape(o, env.BATCH, m, env.BLOCK_OUT).transpose((0, 2, 1, 3))

# Format the input/output arrays with tvm.nd.array to the DLPack standard
A_nd = tvm.nd.array(A_packed, ctx)
B_nd = tvm.nd.array(B_packed, ctx)
C_nd = tvm.nd.array(np.zeros((o, m, env.BATCH, env.BLOCK_OUT)).astype(C.dtype), ctx)

# Invoke the module to perform the computation
f(A_nd, B_nd, C_nd)

######################################################################
# Verifying Correctness
# ---------------------
# Compute the reference result with numpy and assert that the output of the
# matrix multiplication indeed is correct

# Compute reference result with numpy
C_ref = (A_orig.astype(env.acc_dtype) + B_orig.astype(env.acc_dtype)).astype(C.dtype)
C_ref = C_ref.reshape(o, env.BATCH, m, env.BLOCK_OUT).transpose((0, 2, 1, 3))
np.testing.assert_equal(C_ref, C_nd.asnumpy())
print("Successful vector add test!")

######################################################################
# Summary
# -------
# This tutorial provides a walk-through of TVM for programming the
# deep learning accelerator VTA with a simple vector addition example.
# The general workflow includes:
#
# - Programming the FPGA with the VTA bitstream over RPC.
# - Describing the vector add computation via a series of computations.
# - Describing how we want to perform the computation using schedule primitives.
# - Compiling the function to the VTA target.
# - Running the compiled module and verifying it against a numpy implementation.
#
# You are more than welcome to check other examples out and tutorials
# to learn more about the supported operations, schedule primitives
# and other features supported by TVM to program VTA.
#

ext: py

max_stars_repo_name: adelbertc/incubator-tvm

max_issues_repo_name: adelbertc/incubator-tvm

max_forks_repo_name: adelbertc/incubator-tvm

reason: import numpy


text:
"""data management piece
22.09.2020 - @yashbonde"""

import re
import json
import numpy as np
import pandas as pd
import networkx as nx
from tabulate import tabulate
import sentencepiece as spm

import torch
from torch.utils.data import Dataset

# ====== Helper functions ======= #
def parse_db_to_networkx(db):
    """convert the db to a networkx graph with proper attributes

    nodes have features:
      - id: {table}.{name}
      - primary: True/False
      - type: type
    edges have features:
      -
    """
    columns = db["column_names"][1:] # ignore the attritbute
    table_names = db["table_names"]
    column_types = db["column_types"]
    foreign_keys = db["foreign_keys"]
    primary_keys = db["primary_keys"]

    if len(set([x[0] for x in columns])) != len(table_names):
        raise ValueError("More tables given in ")

    # make graph
    g = nx.Graph()

    # add nodes and data
    for i, c in enumerate(columns):
        name = c[1].replace(" ", "_")
        table = table_names[c[0]]
        g.add_node(
            i, id = f"{table}.{name}", name = name, table = table,
            primary = True if (i+1) in primary_keys else False,
            type = column_types[i]
        )

    # for edges first foriegn keys because simpler
    for (s,t) in foreign_keys:
        g.add_edge(s-1, t-1, foreign = True)

    # then those within the table
    for i in range(len(table_names)):
        cols = list(filter(
            lambda c: c[1][0] == i, enumerate(columns)
        ))
        cols = [x[0] for x in cols]
        for i,c in enumerate(cols):
            for cc in cols[i+1:]:
                g.add_edge(c, cc, foreign = False)
    return g

def get_db_attention_mask(g, size, device = "cpu", inf = 1e6):
    A = nx.adjacency_matrix(g).todense()
    A = 1 - (A + np.eye(len(A)))  # add self loops
    if size == -1:
        m = A
    else:
        m = np.zeros((size, size))
        m[:len(A), :len(A)] = A  # add to big loop
    m = m * inf
    return m, len(A)

def get_tokenised_attention_mask(g, t, size = None, inf = 1e6):
    """In method get_db_attention_mask() we do not consider that the tokens
    will have a subword splitting and so the final attention mask will look
    a bit different. This takes care of that by creating mask on subwords
    as well.
    :param g: graph
    :param t: sentencepiece tokenizer
    :param size: dimension of the output attention_mask
    :param inf: what will be the negative infinity value

    NOTE: 11th November, 2020 @yashbonde you ass don't do anymore complicated
    engineering, what you need is more compute not more engineering you ass.
    """
    # att = get_db_attention_mask(g, size = -1)
    ts = []
    sizes = []
    for x in g.nodes().data():
        # we directly call the internal wordpiece_tokenizer, helps in debugging
        tokens = t.encode(" ".join([x[1].get("table"), x[1].get("name")]))
        sizes.append(len(tokens))
        ts.extend(tokens)

    # get adjacency matrix
    mat = nx.adjacency_matrix(g).todense()
    mat = mat + np.eye(len(mat)) # add self loops
    mat = mat.tolist()

    # now the code to expand the matrix in place
    tmat = np.zeros((sum(sizes),sum(sizes)))
    tid = 0
    for i in range(len(mat)):
        idx = np.arange(len(mat))[np.asarray(mat[i]) == 1]
        for s in range(sizes[i]):
            for j in idx:
                start = sum(sizes[:j])
                end = sum(sizes[:j+1])
                tmat[tid, start:end] = 1
            tid += 1
    tmat = tmat + tmat.T
    tmat[tmat > 1] = 1
    tmat = tmat.astype(int)

    if size is not None:
        # convert to required shapes and put in masking values
        fmat = np.zeros((size, size)).astype(int)
        if size < len(tmat):
            tmat = tmat[:size, :size]
        fmat[:tmat.shape[0], :tmat.shape[0]] = tmat
        fmat = 1 - fmat
        fmat = fmat * -inf
    else:
        fmat = tmat

    return fmat, ts, sum(sizes)

def format_sql(in_str):
    in_str = in_str.lower()
    for p in re.findall(r"\w+\s+as\s+t\d+", in_str):
        # print(p)
        try:
            table, id = [x.strip() for x in p.split("as")]
        except:
            table, id = [x.strip() for x in p.split(" as ")]
        # replace phrase that contains "<table_name> AS <id>"
        in_str = in_str.replace(p, table)

        # replace the table
        in_str = in_str.replace(id, table)
        in_str = in_str.replace(id.lower(), table)
        in_str = in_str.replace(id.upper(), table)

    # basic cleaning
    in_str = re.sub(r"\s+", " ", in_str)
    in_str = re.sub(r"\"+", '"', in_str)
    return in_str

# ====== Main Class Object ====== #
class T2SDataset(Dataset):
    def __init__(self, config, mode, method = "m1"):
        if method not in ["m1", "m2"]:
            raise ValueError("Invalid method, please check documentation")

        self.config = config
        self.method = method

        with open(config.schema_file) as f:
            self.schemas = {k:parse_db_to_networkx(v) for k,v in json.load(f).items()}

        with open(config.questions_file) as f:
            df = pd.read_csv(config.questions_file, sep="\t")
            mode = 1 if mode == "train" else 0
            df = df[df.train == mode]

        self.questions = df.question.values
        self.queries = df["query"].values # I didn't know .query was a function
        self.db_ids = df.db_id.values

    def __len__(self):
        return len(self.questions)

    def _get_question(self, index, config, t):
        # prepare the questions
        question = self.questions[index]
        question = [t.bos_id()] + t.encode(question) + [t.eos_id()]
        sent_len = len(question)
        if config.maxlen > len(question):
            question = question + [t.pad_id() for _ in range(config.maxlen - len(question))]
        else:
            question = question[:config.maxlen]
        sent_attn = np.zeros((config.maxlen, config.maxlen)).astype(np.int32)
        sent_attn[:sent_len, :sent_len] = 1
        sent_attn = 1 - sent_attn
        sent_attn = sent_attn * -1e6

        return question, sent_attn

    def _get_sql(self, index, config, t):
        # prepare the sql query
        sql = self.queries[index]
        sql = [t.bos_id()] + t.encode(sql) + [t.eos_id()]
        sql_len = len(sql)
        if config.maxlen > len(sql) + 1:
            sql = sql + [t.pad_id()
                         for _ in range(config.maxlen - len(sql) + 1)]
        else:
            sql = sql[:config.maxlen + 1]
        sql_attn = np.zeros((config.maxlen, config.maxlen)).astype(np.int32)
        sql_attn[:sql_len, :sql_len] = 1
        sql_attn = sql_attn - np.triu(sql_attn, k=1)  # casual masking
        sql_attn = 1 - sql_attn
        sql_attn = sql_attn * -1e6

        # create labels
        labels = torch.from_numpy(np.asarray(sql[1:])).long()
        labels[sql_len-1:] = -100  # minus 1 because already shifted

        # create input ids
        sql_ids = torch.from_numpy(np.asarray(sql[:-1])).long()

        return sql_ids, sql_attn, labels

    # === functions based on methods === #
    def _getitem_m1(self, index):
        config = self.config
        t = self.config.tokenizer

        # prepare the questions
        question, sent_attn = self._get_question(index, config, t)

        # get sql
        sql_ids, sql_attn, labels = self._get_sql(question, config, t)

        # prepare the DB sequence
        g = self.schemas[self.db_ids[index]]
        db_attn_mat, db_tokens, len_db = get_tokenised_attention_mask(g, t, size=config.maxlen)
        db_tokens = [t.bos_id()] + db_tokens + [t.eos_id()]
        if config.maxlen > len(db_tokens):
            db_tokens = db_tokens + [t.pad_id() for _ in range(config.maxlen - len(db_tokens))]
        else:
            db_tokens = db_tokens[:config.maxlen]

        # return the output dictionary
        return {
            "sql_ids": sql_ids,
            "labels": labels,
            "sent": torch.from_numpy(np.asarray(question)).long(),
            "db": torch.from_numpy(np.asarray(db_tokens)).long(),
            "sql_attn": torch.from_numpy(np.asarray([sql_attn]).astype(np.float32)),
            "sent_attn": torch.from_numpy(np.asarray([sent_attn]).astype(np.float32)),
            "db_attn": torch.from_numpy(np.asarray([db_attn_mat]).astype(np.float32))
        }

    def _getitem_m2(self, index):
        config = self.config
        t = self.config.tokenizer

        # prepare the questions
        question, sent_attn = self._get_question(index, config, t)

        # get sql
        sql_ids, sql_attn, labels = self._get_sql(index, config, t)

        # prepare the DB sequence
        g = self.schemas[self.db_ids[index]]
        db_attn_mat, len_db = get_db_attention_mask(g, size=-1)
        db_tokens = []
        sizes = []
        for x in g.nodes().data():
            # we directly call the internal wordpiece_tokenizer, helps in debugging
            tokens = t.encode(" ".join([x[1].get("table"), x[1].get("name")]))
            sizes.append(len(tokens))
            db_tokens.extend(tokens)

        # creating a merging_index list as [0,0,0,1,1,2,3,4,4,5,6,6,7,8,8,8,9,10,10,10]
        merging_index = [0] # bos
        for i,s in enumerate(sizes):
            merging_index.extend([i+1 for _ in range(s)])
        merging_index = merging_index + [len(sizes) + 1] # eos

        if config.maxlen > len(merging_index):
            merging_index = merging_index + [len(sizes)+2 for _ in range(config.maxlen - len(merging_index))]
        else:
            merging_index = merging_index[:config.maxlen]

        # final tokens
        db_tokens = [t.bos_id()] + db_tokens + [t.eos_id()]
        if config.maxlen > len(db_tokens):
            db_tokens = db_tokens + [t.pad_id() for _ in range(config.maxlen - len(db_tokens))]
        else:
            db_tokens = db_tokens[:config.maxlen]

        print(len(db_tokens), len(merging_index))
        print(db_tokens, merging_index)

        assert len(db_tokens) == len(merging_index)

        return {
            "sql_ids": sql_ids,
            "labels": labels,
            "sent": torch.from_numpy(np.asarray(question)).long(),
            "db": torch.from_numpy(np.asarray(db_tokens)).long(),
            "merging_index": torch.from_numpy(np.asarray(merging_index)).long(),

            "sql_attn": torch.from_numpy(np.asarray([sql_attn]).astype(np.float32)),
            "sent_attn": torch.from_numpy(np.asarray([sent_attn]).astype(np.float32)),
            "db_attn": torch.from_numpy(np.asarray([db_attn_mat]).astype(np.float32)),
        }

    def __getitem__(self, index):
        if self.method == "m1":
            return self._getitem_m1(index)
        elif self.method == "m2":
            return self._getitem_m2(index)

class T2SDatasetConfig:
    schema_file = None # json file with schema dump
    questions_file = None # TSV file with questions-sql dump
    maxlen = 150 # maximum length for all is same for simplicity
    # also same size helps fit in the encoder mask as well as the
    # cross attention mask
    tokenizer_path = None

    maxlen_db = 1900 # maximum length of DB string to support

    def __init__(self, **kwargs):
        self.attrs = ["schema_file", "questions_file", "maxlen"]
        for k, v in kwargs.items():
            setattr(self, k, v)
            self.attrs.append(k)
        self.tokenizer = spm.SentencePieceProcessor()
        self.tokenizer.load(self.tokenizer_path)

        print(f"Loaded tokenizer from: {self.tokenizer_path}. (vocab_size: {self.tokenizer.vocab_size()})")

    def __repr__(self):
        kvs = [(k, f"{getattr(self, k)}") for k in sorted(list(set(self.attrs)))]
        return tabulate(kvs, ["argument", "value"], tablefmt="psql")

if __name__ == "__main__":
    config = T2SDatasetConfig(
        schema_file="/Users/yashbonde/Desktop/AI/text2sql/fdata/all_schema.json",
        questions_file="/Users/yashbonde/Desktop/AI/text2sql/fdata/all_questions.tsv",
        maxlen = 150,
        tokenizer_path="/Users/yashbonde/Desktop/AI/text2sql/data/model.model",
    )
    print(config)
    ds = T2SDataset(config=config, mode="train", method = "m2")

    print(ds[123])

reason: import numpy,import networkx


text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import argparse
import random

import paddle
import numpy as np

from paddleseg.cvlibs import manager
from paddleseg.utils import get_sys_env, logger, config_check
from datasets.humanseg import HumanSeg
from paddleseg.datasets import Dataset
from datasets.mixed_dataset import MixedDataset
from scripts.config import Config
from scripts.unstructured_prune_mixed_utils import train

# python unstructured_prune_mixed_train.py --ratio=0.75 --prune_params_type=conv1x1_only \
# --local_sparsity=True --pruning_strategy=gmp \
# --pruning_iters 4500 --tunning_iters 4500 \
# --config ${cfg} --do_eval --num_workers 4 \
# --save_interval 30

# assume inital lr=0.1, lr piecewise_decay, pruning iter lr=0.01, tunning iter lr=[0.001,0.0001]

def parse_args():
    parser = argparse.ArgumentParser(description='Model training')
    # params of pruning
    parser.add_argument(
        '--pruning_mode',
        dest='pruning_mode',
        help=
        'the pruning mode: whether by ratio or by threshold. Default: ratio',
        type=str,
        default='ratio')
    parser.add_argument(
        '--ratio',
        dest='ratio',
        help=
        'The ratio to set zeros, the smaller part bounded by the ratio will be zeros. Default: 0.55',
        type=float,
        default=0.55)
    parser.add_argument(
        '--threshold',
        dest='threshold',
        help='The threshold to set zeros. Default: 0.01',
        type=float,
        default=0.01)
    parser.add_argument(
        '--prune_params_type',
        dest='prune_params_type',
        help=
        'Which kind of params should be pruned, we only support None (all but norms) and conv1x1_only for now. Default: None',
        type=str,
        default=None)
    parser.add_argument(
        '--local_sparsity',
        dest='local_sparsity',
        help=
        'Whether to prune all the parameter matrix at the same ratio or not. Default: False',
        type=bool,
        default=False)

    parser.add_argument(
        '--pruning_strategy',
        dest='pruning_strategy',
        help=
        'Which training strategy to use in pruning, we only support base and gmp for now. Default: base',
        type=str,
        default='base')
    parser.add_argument(
        '--stable_iters',
        dest='stable_iters',
        help=
        "The iter numbers used to stablize the model before pruning. Default: 0",
        type=int,
        default=0)
    parser.add_argument(
        '--pruning_iters',
        dest='pruning_iters',
        help=
        'The iter numbers used to prune the model by a ratio step. Default: 60',
        type=int)
    parser.add_argument(
        '--tunning_iters',
        dest='tunning_iters',
        help=
        'The iter numbers used to tune the after-pruned models. Default: 60',
        type=int)
    parser.add_argument(
        '--resume_iter',
        dest='resume_iter',
        help="The iteration we'll resume training from. Default: 0",
        type=int,
        default=0)
    parser.add_argument(
        '--pruning_steps',
        dest='pruning_steps',
        help=
        'How many times you want to increase your ratio during training. Default: 100',
        type=int,
        default=100)
    parser.add_argument(
        '--initial_ratio',
        dest='initial_ratio',
        help=
        'The initial pruning ratio used at the start of pruning stage. Default: 0.15',
        type=float,
        default=0.15)

    # params of training
    parser.add_argument(
        "--config", dest="cfg", help="The config file.", default=None, type=str)
    parser.add_argument(
        '--iters',
        dest='iters',
        help='iters for training',
        type=int,
        default=None)
    parser.add_argument(
        '--batch_size',
        dest='batch_size',
        help='Mini batch size of one gpu or cpu',
        type=int,
        default=None)
    parser.add_argument(
        '--learning_rate',
        dest='learning_rate',
        help='Learning rate',
        type=float,
        default=None)
    parser.add_argument(
        '--save_interval',
        dest='save_interval',
        help='How many iters to save a model snapshot once during training.',
        type=int,
        default=None)
    parser.add_argument(
        '--resume_model',
        dest='resume_model',
        help='The path of resume model',
        type=str,
        default=None)
    parser.add_argument(
        '--save_dir',
        dest='save_dir',
        help='The directory for saving the model snapshot',
        type=str,
        default='./output')
    parser.add_argument(
        '--keep_checkpoint_max',
        dest='keep_checkpoint_max',
        help='Maximum number of checkpoints to save',
        type=int,
        default=5)
    parser.add_argument(
        '--num_workers',
        dest='num_workers',
        help='Num workers for data loader',
        type=int,
        default=0)
    parser.add_argument(
        '--do_eval',
        dest='do_eval',
        help='Eval while training',
        action='store_true')
    parser.add_argument(
        '--log_iters',
        dest='log_iters',
        help='Display logging information at every log_iters',
        default=10,
        type=int)
    parser.add_argument(
        '--use_vdl',
        dest='use_vdl',
        help='Whether to record the data to VisualDL during training',
        action='store_true')
    parser.add_argument(
        '--seed',
        dest='seed',
        help='Set the random seed during training.',
        default=None,
        type=int)
    parser.add_argument(
        '--fp16', dest='fp16', help='Whther to use amp', action='store_true')
    parser.add_argument(
        '--data_format',
        dest='data_format',
        help=
        'Data format that specifies the layout of input. It can be "NCHW" or "NHWC". Default: "NCHW".',
        type=str,
        default='NCHW')
    parser.add_argument(
        '--profiler_options',
        type=str,
        default=None,
        help='The option of train profiler. If profiler_options is not None, the train ' \
            'profiler is enabled. Refer to the paddleseg/utils/train_profiler.py for details.'
    )

    return parser.parse_args()

def main(args):

    if args.seed is not None:
        paddle.seed(args.seed)
        np.random.seed(args.seed)
        random.seed(args.seed)

    env_info = get_sys_env()
    info = ['{}: {}'.format(k, v) for k, v in env_info.items()]
    info = '\n'.join(['', format('Environment Information', '-^48s')] + info +
                     ['-' * 48])
    logger.info(info)

    place = 'gpu' if env_info['Paddle compiled with cuda'] and env_info[
        'GPUs used'] else 'cpu'

    paddle.set_device(place)
    if not args.cfg:
        raise RuntimeError('No configuration file specified.')

    cfg = Config(
        args.cfg,
        learning_rate=args.learning_rate,
        iters=args.iters,
        batch_size=args.batch_size,
        save_interval=args.save_interval)

    train_data_ratio = cfg.train_data_ratio
    val_roots = cfg.val_roots
    dataset_weights = cfg.dataset_weights
    class_weights = cfg.class_weights
    train_file_lists = cfg.train_file_lists
    train_transforms = cfg.train_transforms
    val_transforms = cfg.val_transforms
    losses = cfg.loss

    msg = '\n---------------Config Information---------------\n'
    msg += str(cfg)
    msg += '------------------------------------------------'
    logger.info(msg)

    train_datasets = []
    for i, train_root in enumerate(cfg.train_roots):
        train_datasets.append(
            Dataset(
                train_transforms,
                train_root,
                2,
                train_path=os.path.join(train_root, train_file_lists[i])))
    if len(train_datasets) == 0:
        raise ValueError(
            'The length of train_datasets is 0. Please check if your dataset is valid'
        )
    mixed_train_dataset = MixedDataset(
        train_datasets,
        train_data_ratio,
        transforms=train_transforms,
        num_classes=2)

    if args.do_eval:
        val_dataset0 = Dataset(
            val_transforms,
            val_roots[0],
            2,
            mode='val',
            val_path=os.path.join(val_roots[0], 'val.txt'))
        val_dataset1 = Dataset(
            val_transforms,
            val_roots[1],
            2,
            mode='val',
            val_path=os.path.join(val_roots[1], 'val.txt'))
        val_dataset2 = Dataset(
            val_transforms,
            val_roots[2],
            2,
            mode='val',
            val_path=os.path.join(val_roots[2], 'val.txt'))
        val_datasets = [val_dataset0, val_dataset1, val_dataset2]
        # val_datasets = val_dataset0
    else:
        val_datasets = None

    train(
        args,
        cfg.model,
        mixed_train_dataset,
        val_datasets=val_datasets,
        class_weights=class_weights,
        dataset_weights=dataset_weights,
        optimizer=cfg.optimizer,
        save_dir=args.save_dir,
        iters=cfg.iters,
        batch_size=cfg.batch_size,
        resume_model=args.resume_model,
        save_interval=cfg.save_interval,
        log_iters=args.log_iters,
        num_workers=args.num_workers,
        use_vdl=args.use_vdl,
        losses=losses,
        keep_checkpoint_max=args.keep_checkpoint_max)

if __name__ == '__main__':
    args = parse_args()
    main(args)

max_stars_repo_name: DaoDaoer/PaddleSeg

max_issues_repo_name: DaoDaoer/PaddleSeg

max_forks_repo_name: DaoDaoer/PaddleSeg


text:
import tensorflow as tf
import numpy as np
import pandas as pd

def model_fn_wrapper():
    def model_fn(features,labels,mode):
        x = tf.cast(features['x'],dtype=tf.float64)

        print("x",x.shape.as_list())
        w = tf.get_variable(name='W',shape = [1],dtype=tf.float64)
        b = tf.get_variable(name='b',shape = [1],dtype=tf.float64)
        pre = w*x+b
        train_op = None
        loss = None
        if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:
            y = tf.cast(features['y'], dtype=tf.float64)
            loss = tf.reduce_sum(tf.square(y-pre))
        if mode == tf.estimator.ModeKeys.TRAIN:
            global_steps  = tf.train.get_global_step()
            train_op = tf.train.AdamOptimizer(0.01).minimize(loss,global_step=global_steps)

        return tf.estimator.EstimatorSpec(mode=mode,
        predictions=pre,
        loss=loss,
        train_op=train_op,)
    return model_fn

# define our data sets
x_train = np.arange(0,40)
y_train = x_train*2+1.01

x_eval = np.arange(40,80)
y_eval = x_eval*2+1.01
def m(record):

    record['x'] = 2*record['x']
    return record

def create_fn(x ,y=None,batch_size = 1,epoch = None):
    def fn():

        dict = {}
        dict['x'] = x
        if y is not None:
            dict['y']=y
        dataset=tf.data.Dataset.from_tensor_slices(dict)
        dataset = dataset.map(m)
        dataset = dataset.batch(batch_size)

        dataset  = dataset.repeat(epoch) if epoch is  not None else dataset.repeat()
        iterator = dataset.make_one_shot_iterator()

        featurs= iterator.get_next()
        return featurs
    return fn

# input_fn = tf.estimator.inputs.numpy_input_fn(
#     {"x": x_train,"y":y_train}, None, batch_size=4, num_epochs=None, shuffle=True)
# train_input_fn = tf.estimator.inputs.numpy_input_fn(
#     {"x": x_train,"y":y_train}, None, batch_size=4, num_epochs=1000, shuffle=False)
# eval_input_fn = tf.estimator.inputs.numpy_input_fn(
#     {"x": x_eval,"y":y_eval}, None, batch_size=4, num_epochs=1000, shuffle=False)

def train():
    # train
    estimator = tf.estimator.Estimator(model_fn=model_fn_wrapper(), model_dir='./d2')
    estimator.train(input_fn=create_fn(x_train,y_train,8), steps=3000)
    # Here we evaluate how well our model did.

max_stars_repo_name: Huggable/bert

max_issues_repo_name: Huggable/bert

max_forks_repo_name: Huggable/bert


text:
from __future__ import annotations

from typing import Dict, Optional, Protocol, Union

import numpy.typing as npt
import ophyd

Number = Union[int, float]
PrimitiveType = Union[str, int, bool, float]

OphydBaseType = Union[
    str,
    int,
    bool,
    float,
]
OphydDataType = Union[
    OphydBaseType,
    list[OphydBaseType],
    npt.NDArray[OphydBaseType]
]

class OphydCallback(Protocol):
    def __call__(**kwargs) -> None:
        ...

SignalToValue = Dict[ophyd.Signal, OphydDataType]

class MdsCalculateFunction(Protocol):
    """Calculation handler for MultiDerivedSignal."""
    def __call__(self, signal_to_value: SignalToValue) -> OphydDataType:
        ...

class MdsOnPutFunction(Protocol):
    """Put handler for MultiDerivedSignal."""
    def __call__(self, value: OphydDataType) -> Optional[SignalToValue]:
        ...

max_stars_repo_name: mkestra/pcdsdevices

max_issues_repo_name: mkestra/pcdsdevices

max_forks_repo_name: mkestra/pcdsdevices


text:
from collections import Counter, defaultdict
import operator
import math
import pandas as pd
import scipy
from tqdm import tqdm_notebook

def add_time_rank_to_dataframe(df):

    # Set mails with absurd dates to date before any real mail
    min_date_string = '1998-12-20 00:00:01'

    correct_prefix = [chars[0:2] == '00' for chars in list(df['date'])]
    df.loc[correct_prefix, 'date'] = min_date_string

    # Convert dates to timestamps
    df['parsed_date'] = pd.to_datetime(df['date'], yearfirst=True)

    # Add time rank column and sort dataframe
    df['time_rank'] = df['parsed_date'].rank(ascending=False)
    df = df.sort_values('parsed_date')
    return df

def recency_predictions_to_standard(recency_predictions):
    standard_prediction = {}
    for sender, predictions in recency_predictions.items():
        mids = predictions[0]
        if(mids):  # at least one mid for sender
            recipients = predictions[1][0]
            for mid in mids:
                standard_prediction[mid] = recipients
    return standard_prediction

def get_frequency_address_books(emails_ids_per_sender, email_df):
    """
    Create address book with frequency information for each user
    @return address_books with sender as key and recipients as ranked list
    """
    address_books = {}
    idx_sender = 0

    pbar_senders = tqdm_notebook(emails_ids_per_sender.items())
    for sender, ids in pbar_senders:
        recs_temp = []
        for my_id in ids:
            recipients = email_df[email_df['mid'] == int(my_id)][
                'recipients'].tolist()
            recipients = recipients[0].split(' ')
            # keep only legitimate email addresses
            recipients = [rec for rec in recipients if '@' in rec]
            recs_temp.append(recipients)
        # flatten
        recs_temp = [elt for sublist in recs_temp for elt in sublist]
        # compute recipient counts
        rec_occ = dict(Counter(recs_temp))
        # order by frequency
        sorted_rec_occ = sorted(
            rec_occ.items(), key=operator.itemgetter(1), reverse=True)
        # save
        address_books[sender] = sorted_rec_occ

        if idx_sender % 10 == 0:
            print('processed {nb_sender} senders'.format(nb_sender=idx_sender))
        idx_sender += 1
    return address_books

def get_recency_address_books(emails_ids_per_sender, email_df,
                              beta, use_rank=False):
    """
    Create address book with recency information for each user
    @return address_books with sender as key and recipients as ranked list
    """
    address_books = {}

    # get time ranks if @use_rank, else get datetimes
    if(use_rank):
        email_recency_rank_dic = get_recency_rank_dic(email_df)
    else:
        email_recency_datetime_dic = get_rencency_datetime_dic(email_df)
        latest_email_date = email_df.parsed_date.max()

    pbar_senders = tqdm_notebook(emails_ids_per_sender.items())
    for sender, ids in pbar_senders:
        recipients_scores = defaultdict(lambda: 0)
        for my_id in ids:
            my_id = int(my_id)
            if(use_rank):
                # Compute rank scores
                recency_score = get_email_recency_rank_score(
                    email_recency_rank_dic, my_id, beta)
            else:
                # Compute datetime scores
                recency_score = get_email_recency_time_score(
                    email_recency_datetime_dic, my_id, beta, latest_email_date)

            recipients = email_df[email_df['mid'] == int(my_id)][
                'recipients'].tolist()
            recipients = recipients[0].split(' ')
            # keep only legitimate email addresses
            recipients = [rec for rec in recipients if '@' in rec]
            for recipient in recipients:
                recipients_scores[recipient] += recency_score
        # flatten
        rec_occ = dict(Counter(recipients_scores))
        # order by frequency
        sorted_rec_occ = sorted(
            rec_occ.items(), key=operator.itemgetter(1), reverse=True)
        # save
        address_books[sender] = sorted_rec_occ

    return address_books

def predictions_from_addressbook(test_dic, address_books,
                                 keep_all=False, k=10):
    """
    Writes results to csv file for kaggle submission
    for text-independent models (frequency, recency)
    @address_books must be a dict with sender as key and recpient as value
    with recipients ordered according to rank
    @test_dic is a dictionnary with email address as key and mid list as value
    as returned by get_restricted_email_ids_per_sender or
    get_email_ids_per_sender from preprocess.py
    """
    predictions_per_sender = {}
    for sender, mids_predict in test_dic.items():
        recency_preds = []
        if(keep_all):
            k_most = [elt[0] for elt in address_books[sender]]
        else:
            # select k most frequent recipients for the user
            k_most = [elt[0] for elt in address_books[sender][:k]]
        for id_predict in mids_predict:
            # for recency baselines, the predictions are always the same
            recency_preds.append(k_most)
        predictions_per_sender[sender] = [mids_predict, recency_preds]
    return predictions_per_sender

def get_recency_rank_dic(email_df):
    """
    Creates dictionnary that links mid to time rank
    (1 for most recent mail)
    """
    recency_dic = dict(zip(email_df['mid'], email_df['time_rank']))
    return recency_dic

def get_rencency_datetime_dic(email_df):
    recency_dic = dict(zip(email_df['mid'], email_df['parsed_date']))
    return recency_dic

def get_email_recency_rank_score(mid_rank_dic, mid, beta):
    """
    Compute the recency score for a given mid
    """
    mid_rank = mid_rank_dic[mid]
    recency_score = math.exp(- mid_rank / beta)
    return recency_score

def get_email_recency_time_score(mid_datetime_dic, mid, beta, last_email_date):
    """
    Compute the recency score for a given mid
    """
    mid_date_time = mid_datetime_dic[mid]
    recency_score = math.pow(0.5, (last_email_date - mid_date_time
                                   ).total_seconds() / beta)
    return recency_score

def get_sparse_date_info(idx_to_mids, df_info):
    """
    gets time info as one-hot encoding in sparse matrix
    idx matching mids as in @idx_to_mids in rows and days in columns
    """
    df_info = add_time_rank_to_dataframe(df_info)
    nb_samples = len(idx_to_mids)
    timestamp_dic = get_rencency_datetime_dic(df_info)

    date_features = scipy.sparse.lil_matrix((nb_samples, 7))
    for idx, mid in idx_to_mids.items():
        # Get time info for given mid
        mid_timestamp = timestamp_dic[mid]
        # mid_hour = mid_timestamp.hour
        mid_day = mid_timestamp.weekday()
        date_features[idx, mid_day] = 1
    return date_features

def get_sender_sparse_date_info(email_ids_per_sender,
                                senders_idx_to_mid_dic,
                                df_info):
    """
    gets time info as one-hot encoding in sparse matrix
    idx matching mids as in idx_to_mids in rows and days in columns
    idx_to_mids for each sender is stored in
    @param senders_mid_features_dic['sender']
    """
    # Get time info for all emails
    df_info = add_time_rank_to_dataframe(df_info)
    timestamp_dic = get_rencency_datetime_dic(df_info)

    # Create dictionnary {sender: time_feature}
    date_features_dic = {}
    for sender, sender_idx_to_mids in senders_idx_to_mid_dic.items():
        nb_samples = len(sender_idx_to_mids)
        date_features = scipy.sparse.lil_matrix((nb_samples, 7))
        for idx, mid in sender_idx_to_mids.items():
            # Get time info for given mid
            mid_timestamp = timestamp_dic[mid]
            # mid_hour = mid_timestamp.hour
            mid_day = mid_timestamp.weekday()
            date_features[idx, mid_day] = 1
        date_features_dic[sender] = date_features
    return date_features_dic

max_stars_repo_name: GeoffNN/ALTEGRADEnron

max_issues_repo_name: GeoffNN/ALTEGRADEnron

max_forks_repo_name: GeoffNN/ALTEGRADEnron

reason: import scipy


text:
# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: replicate
   :platform: Unix
   :synopsis: A class to replicate the slice list of a dataset \
       (not the data itself!)

.. moduleauthor:: Nicola Wadeson <scientificsoftware@diamond.ac.uk>

"""

import numpy as np
import copy

from savu.data.data_structures.data_types.base_type import BaseType

class Replicate(BaseType):
    """ Class to replicate the slice list of a dataset (not the data itself!)
    """

    def __init__(self, data_obj, reps):
        self.data_obj = data_obj
        self.reps = reps
        super(Replicate, self).__init__()

        self.rep_dim = len(data_obj.get_shape()) + 1
        self.shape = data_obj.get_shape() + (reps,)
        self.data = data_obj.data
        self.original_patterns = data_obj.get_data_patterns()
        self.__set_patterns(copy.deepcopy(self.original_patterns))

    def clone_data_args(self, args, kwargs, extras):
        args = ['self', 'reps']
        return args, kwargs, extras

    def __getitem__(self, idx):
        return np.expand_dims(self.data[idx[:-1]], self.rep_dim)

    def get_shape(self):
        return self.shape

    def __set_patterns(self, data_obj, patterns):
        for p in patterns:
            patterns[p]['slice_dir'] += (3,)
        data_obj.data_info.set('data_patterns', patterns)

    def _reset(self):
        self.data_obj.data_info.set('data_patterns', self.original_patterns)
        return self.data

max_stars_repo_name: malte-storm/Savu

max_issues_repo_name: malte-storm/Savu

max_forks_repo_name: malte-storm/Savu


text:
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ragged_tensor.convert_to_tensor_or_ragged."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized
import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.ops.ragged import ragged_tensor
from tensorflow.python.ops.ragged import ragged_test_util
from tensorflow.python.platform import googletest

@test_util.run_all_in_graph_and_eager_modes
class RaggedConvertToTensorOrRaggedTensorTest(
    ragged_test_util.RaggedTensorTestCase, parameterized.TestCase):

  #=============================================================================
  # Tests where the 'value' param is a RaggedTensor
  #=============================================================================
  @parameterized.parameters([
      dict(pylist=[[1, 2], [3]]),
      dict(pylist=[[1, 2], [3]], preferred_dtype=dtypes.float32),
      dict(pylist=[[1, 2], [3]], preferred_dtype=dtypes.string),
  ])
  def testConvertRaggedTensor(self, pylist, dtype=None, preferred_dtype=None):
    rt = ragged_factory_ops.constant(pylist)
    converted = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        rt, dtype, preferred_dtype)
    self.assertIs(converted, rt)

  @parameterized.parameters([
      dict(
          pylist=[[1, 2], [3, 4]],
          dtype=dtypes.float32,
          message=('Tensor conversion requested dtype float32 for '
                   'RaggedTensor with dtype int32')),
      dict(
          pylist=[[1, 2], [3, 4]],
          dtype=dtypes.string,
          message=('Tensor conversion requested dtype string for '
                   'RaggedTensor with dtype .*')),
  ])
  def testConvertRaggedTensorError(self,
                                   pylist,
                                   message,
                                   dtype=None,
                                   preferred_dtype=None):
    rt = ragged_factory_ops.constant(pylist)

    with self.assertRaisesRegexp(ValueError, message):
      ragged_tensor.convert_to_tensor_or_ragged_tensor(rt, dtype,
                                                       preferred_dtype)

  #=============================================================================
  # Tests where the 'value' param is a RaggedTensorValue
  #=============================================================================
  @parameterized.parameters(
      [
          dict(
              value=ragged_factory_ops.constant_value([[1, 2], [3]],
                                                      dtype=np.int32),
              expected_dtype=dtypes.int32),
          dict(
              value=ragged_factory_ops.constant_value([[b'a', b'b'], [b'c']]),
              expected_dtype=dtypes.string),
          dict(
              value=ragged_factory_ops.constant_value([[1, 2], [3]],
                                                      dtype=np.int32),
              dtype=dtypes.float32,
              expected_dtype=dtypes.float32),
          dict(
              value=ragged_factory_ops.constant_value([[1, 2], [3]],
                                                      dtype=np.int32),
              preferred_dtype=dtypes.float32,
              expected_dtype=dtypes.float32),
          dict(
              value=ragged_factory_ops.constant_value([[1, 2], [3]],
                                                      dtype=np.int32),
              preferred_dtype=dtypes.string,
              expected_dtype=dtypes.int32),
      ])
  def testConvertRaggedTensorValue(self,
                                   value,
                                   dtype=None,
                                   preferred_dtype=None,
                                   expected_dtype=None):
    if expected_dtype is None:
      expected_dtype = value.dtype if dtype is None else dtype
    converted = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        value, dtype, preferred_dtype)
    self.assertEqual(value.ragged_rank, converted.ragged_rank)
    self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype)
    self.assertEqual(value.to_list(), self.eval_to_list(converted))

  @parameterized.parameters([
      dict(
          value=ragged_factory_ops.constant_value([['a', 'b'], ['c']],
                                                  dtype=str),
          dtype=dtypes.int32,
          message=r"invalid literal for int\(\) with base 10: 'a'"),
  ])
  def testConvertRaggedTensorValueError(self,
                                        value,
                                        message,
                                        dtype=None,
                                        preferred_dtype=None):
    with self.assertRaisesRegexp(ValueError, message):
      ragged_tensor.convert_to_tensor_or_ragged_tensor(value, dtype,
                                                       preferred_dtype)

  #=============================================================================
  # Tests where the 'value' param is a Tensor
  #=============================================================================
  @parameterized.parameters([
      dict(pylist=[[1, 2], [3, 4]]),
      dict(pylist=[[1, 2], [3, 4]], preferred_dtype=dtypes.float32),
      dict(pylist=[[1, 2], [3, 4]], preferred_dtype=dtypes.string),
  ])
  def testConvertTensor(self, pylist, dtype=None, preferred_dtype=None):
    tensor = constant_op.constant(pylist)
    converted = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        tensor, dtype, preferred_dtype)
    self.assertIs(tensor, converted)

  @parameterized.parameters([
      dict(
          pylist=[[1, 2], [3, 4]],
          dtype=dtypes.float32,
          message=('Tensor conversion requested dtype float32 for '
                   'Tensor with dtype int32')),
      dict(
          pylist=[[1, 2], [3, 4]],
          dtype=dtypes.string,
          message=('Tensor conversion requested dtype string for '
                   'Tensor with dtype int32')),
  ])
  def testConvertTensorError(self,
                             pylist,
                             message,
                             dtype=None,
                             preferred_dtype=None):
    tensor = constant_op.constant(pylist)
    with self.assertRaisesRegexp(ValueError, message):
      ragged_tensor.convert_to_tensor_or_ragged_tensor(tensor, dtype,
                                                       preferred_dtype)

  #=============================================================================
  # Tests where the 'value' param is a np.array
  #=============================================================================
  @parameterized.parameters([
      dict(
          value=np.array([[1, 2], [3, 4]], dtype=np.int32),
          expected_dtype=dtypes.int32),
      dict(
          value=np.array([[b'a', b'b'], [b'c', b'd']]),
          expected_dtype=dtypes.string),
      dict(
          value=np.array([[1, 2], [3, 4]], dtype=np.int32),
          dtype=dtypes.float32,
          expected_dtype=dtypes.float32),
      dict(
          value=np.array([[1, 2], [3, 4]], dtype=np.int32),
          preferred_dtype=dtypes.float32,
          expected_dtype=dtypes.float32),
      dict(
          value=np.array([[1, 2], [3, 4]], dtype=np.int32),
          preferred_dtype=dtypes.string,
          expected_dtype=dtypes.int32),
  ])
  def testConvertNumpyArray(self,
                            value,
                            dtype=None,
                            preferred_dtype=None,
                            expected_dtype=None):
    if expected_dtype is None:
      expected_dtype = value.dtype if dtype is None else dtype
    converted = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        value, dtype, preferred_dtype)
    self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype)
    self.assertAllEqual(value, converted)

  @parameterized.parameters([
      dict(
          value=np.array([['a', 'b'], ['c', 'd']], dtype=str),
          dtype=dtypes.int32,
          message=r"invalid literal for int\(\) with base 10: 'a'"),
  ])
  def testConvertNumpyArrayError(self,
                                 value,
                                 message,
                                 dtype=None,
                                 preferred_dtype=None):
    with self.assertRaisesRegexp(ValueError, message):
      ragged_tensor.convert_to_tensor_or_ragged_tensor(value, dtype,
                                                       preferred_dtype)

if __name__ == '__main__':
  googletest.main()

max_stars_repo_name: owennewo/kfserving

max_issues_repo_name: shekharpalit/tensorflow

max_forks_repo_name: shekharpalit/tensorflow


text:
import numpy as np

from BitVector import BitVector
import WAH

class Bitmap:
    def __init__(self, data):
        """
        Initialize a new Bitmap.
        Args:
            attrs : dict
                A dictionary mapping attribute names with their domains.
        """
        unq_vals = set(data)
        self.size = len(data)
        self.UB = {}
        self.VB = {}
        self.attribute = data

        for i in unq_vals:
            self.UB[i] = BitVector(size=self.size)
            self.VB[i] = BitVector(size=self.size)

        for index, value in enumerate(self.attribute):
            self.VB[value][index] = 1

    def insert(self, value):
        """
        Insert a new row into the bitmap.
        Args:
            value : int
                new value
        """
        self.attribute.append(value)
        hasPadding = 1

        # Checking if any of the last values of either VB or UB are 1
        # to see if there is padding space
        for val in self.VB:
            if self.VB[val][-1] == 1 or self.UB[val][-1] == 1:
                hasPadding = 0
                break

        if hasPadding == 0:  #Add padding space to each vb and ub
            for val in self.VB:
                self.VB[val].pad_from_right(1)
                self.UB[val].pad_from_right(1)
            self.size += 1
        #Set last value (if padded space was added or not?) to 1
        if value not in self.UB.keys():
            self.VB[value] = BitVector(size = self.size)
            self.UB[value] = BitVector(size = self.size)
        self.UB[value][-1] = 0
        self.VB[value][-1] = 1

    def update(self, rid, value):
        """
        Update a row in the bitmap.
        Args:
            rid : int
                The id corresponding to the row being updated.
            value : int
                new value
        """

        old_val = 0
        # Negate the old value
        for val in self.VB:
            if self.VB[val][rid] == 1:      #If old vb is 1, change ub to 1
                old_val = val
                self.UB[val][rid] = 1
            elif self.VB[val][rid] == 0 and self.UB[val][rid] == 1: #If ub is 1 and vb is 0, change ub to 0
                self.UB[val][rid] = 0

        # Negate the new value
        if self.UB[value][rid] == 0:    #If ub at rid is 0, then change to 1, otherwise change to 0
            self.UB[value][rid] = 1
        else:
            self.UB[value][rid] = 0

    def delete(self, rid):
        """
        Delete a row from the bitmap.
        Args:
            rid : int
                The id corresponding to the row being deleted.
        """
        # Find the value of rid
        # For each value, look through VB at the rid
        for val in self.VB:
            # If value at rid is 1 and UB is untouched, update UB
            if self.VB[val][rid] == 1 and self.UB[val][rid] == 0:
                self.UB[val][rid] = 1
            # But if this rid has been switched to this value and hasn't been merged yet,
            # then, update the UB
            elif self.VB[val][rid] == 0 and self.UB[val][rid] == 1:
                self.UB[val][rid] = 0
        pass

    def query(self, value):
        return [i for i,x in enumerate(self.VB[value].__xor__(self.UB[value])) if x]
        """
        # If the UB is all zeros, return the VB
        made = ""
        if self.UB[value].intValue() == 0:
            made = self.VB[value]
        # Else, return the VB XOR UB
        else:
            made = self.VB[value] ^ self.UB[value]
        comp, fence = WAH.compress(made)
        que = BitVector(bitstring = comp)
        counter = 0
        table_index = []
        for i in fence:
            if que[counter] == 0:
                for x in range(i, min(i+31, len(self.attribute))):
                    if self.attribute[x] == value:
                        #table_index.append(self.attribute[x])
                        table_index.append(x)
            elif que[counter] == 1 and que[counter+1] == 1:
                count = BitVector(bitstring = que[counter+2:counter+31]).intValue()
                for q in range(i, i + (31 * count)):
                    #table_index.append(self.attribute[q])
                    table_index.append(q)
            counter += 31
        return table_index
        """


text:
#!/usr/bin/env python3
# coding=utf-8
# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)
"""

import argparse
import json
import logging
import os
from glob import glob

import numpy as np
import torch
from tqdm import tqdm
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.sampler import SequentialSampler

from transformers import (
    GPT2LMHeadModel,
    GPT2Tokenizer,
    PreTrainedTokenizer
)

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,
)
logger = logging.getLogger(__name__)

MAX_LENGTH = int(10000)  # Hardcoded max length to avoid infinite loop

MODEL_CLASSES = {
    "gpt2": (GPT2LMHeadModel, GPT2Tokenizer)
}

def set_seed(args):
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

def adjust_length_to_model(length, max_sequence_length):
    if length < 0 and max_sequence_length > 0:
        length = max_sequence_length
    elif 0 < max_sequence_length < length:
        length = max_sequence_length  # No generation bigger than model size
    elif length < 0:
        length = MAX_LENGTH  # avoid infinite loop
    return length

class LoadDataset(Dataset):
    def __init__(self, tokenizer: PreTrainedTokenizer, args):
        assert os.path.isdir(args.eval_data_file)

        # add special tokens which shouldn't be split
        special_tokens_dict = {'cls_token': '<TLDR>', 'eos_token': '<EOD>'} #, 'additional_special_tokens': ['<EOT>']}
        tokenizer.add_special_tokens(special_tokens_dict)

        self.examples = []
        # load all json files in the data directory
        data_path = os.path.join(args.eval_data_file, "*.json")
        precomputed = [f.split("/")[-1] for f in glob(os.path.join(args.output_dir, "*.json"))]
        print(precomputed)
        self.inputs =[]
        skipped = 0
        for f_path in glob(data_path):
            f_name = f_path.split('/')[-1]
            # print(f_name)
            if f_name not in precomputed:
                with open(f_path) as f:
                    line = json.load(f)
                    article = " ".join(line['article']) if type(line['article']) is list else line['article']
                    tokenized = tokenizer.encode(article + ' <TLDR> ')
                    if len(tokenized) <= args.block_size:
                        line['id'] = f_name
                        self.examples.append(line)
                        self.inputs.append(tokenized)
            else:
                skipped += 1
        print("Skipped {} files which have already been precomputed in the output_dir.".format(skipped))

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, i):
        return torch.tensor(self.inputs[i], dtype=torch.long)

    def get_summary(self, i):
        line = self.examples[i]['abstract']
        abstract = " ".join(line) if type(line) is list else line
        return abstract

    def get_raw_article(self, i):
        line = self.examples[i]['article']
        article = " ".join(line) if type(line) is list else line
        return article

    def get_id(self, i):
        return self.examples[i]['id']

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--model_type",
        default=None,
        type=str,
        required=True,
        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
        default=None,
        type=str,
        required=True,
        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--eval_data_file",
        default=None,
        type=str,
        help="An optional input evaluation data file to evaluate the perplexity on (a text file).",
    )
    parser.add_argument(
        "--output_dir",
        type=str,
        required=True,
        help="The output directory where the model predictions will be written.",
    )
    parser.add_argument(
        "--block_size",
        default=-1,
        type=int,
        help="Optional input sequence length after tokenization."
             "The training dataset will be truncated in block of this size for training."
             "Default to the model max input length for single sentence inputs (take into account special tokens).",
    )
    parser.add_argument(
        "--per_gpu_eval_batch_size", default=4, type=int, help="Batch size per GPU/CPU for evaluation."
    )
    parser.add_argument("--length", type=int, default=20)
    parser.add_argument("--stop_token", type=str, default=None, help="Token at which text generation is stopped")

    parser.add_argument(
        "--temperature",
        type=float,
        default=1.0,
        help="temperature of 1.0 has no effect, lower tend toward greedy sampling",
    )
    parser.add_argument(
        "--repetition_penalty", type=float, default=1.0, help="primarily useful for CTRL model; in that case, use 1.2"
    )
    parser.add_argument("--k", type=int, default=0)
    parser.add_argument("--p", type=float, default=0.9)

    parser.add_argument("--padding_text", type=str, default="", help="Padding text for Transfo-XL and XLNet.")
    parser.add_argument("--xlm_language", type=str, default="", help="Optional language when used with the XLM model.")

    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
    parser.add_argument("--num_return_sequences", type=int, default=1, help="The number of samples to generate.")
    args = parser.parse_args()

    args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()

    set_seed(args)

    # Initialize the model and tokenizer
    try:
        args.model_type = args.model_type.lower()
        model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
    except KeyError:
        raise KeyError("the model {} you specified is not supported. You are welcome to add it and open a PR :)")

    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)

    # load all json files in the data directory
    eval_dataset = LoadDataset(tokenizer, args)

    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)

    def collate(examples):
        return pad_sequence(examples, batch_first=True)

    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(
        eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate
    )

    model = model_class.from_pretrained(args.model_name_or_path)

    model.to(args.device)
    model.resize_token_embeddings(len(tokenizer))

    args.length = adjust_length_to_model(args.length, max_sequence_length=model.config.max_position_embeddings)

    iterator = tqdm(eval_dataloader, desc="Evaluating")
    if not os.path.isdir(args.output_dir):
        os.makedirs(args.output_dir, exist_ok=True)

    for step, batch in enumerate(iterator):

        batch = batch.to(args.device)

        output_sequence = model.generate(
            input_ids=batch,
            max_length=args.length + len(batch[0]),
            temperature=args.temperature,
            top_k=args.k,
            top_p=args.p,
            repetition_penalty=args.repetition_penalty,
            do_sample=False,
            num_return_sequences=args.num_return_sequences,
        )

        example = output_sequence.squeeze_().tolist()

        text = tokenizer.decode(example, clean_up_tokenization_spaces=True)
        text = text[: text.find(args.stop_token) if args.stop_token else None]
        text = text[text.find(tokenizer.cls_token) + len(tokenizer.cls_token):]

        total_dict = {'abstract': eval_dataset.get_summary(step),
                      'article': eval_dataset.get_raw_article(step),
                      'output': text}

        out_path = os.path.join(args.output_dir, eval_dataset.get_id(step))

        with open(out_path, 'w') as f:
            json.dump(total_dict, f)

    # print("FINISHED EVALUATION...")
    # for i, example in enumerate(outputs):
    #     os.makedirs(args.output_dir, exist_ok=True)
    #
    #     text = tokenizer.decode(example, clean_up_tokenization_spaces=True)
    #     text = text[: text.find(args.stop_token) if args.stop_token else None]
    #     text = text[text.find(tokenizer.cls_token) + len(tokenizer.cls_token):]
    #
    #     total_dict = {'abstract': eval_dataset.get_summary(i),
    #                   'article': eval_dataset.get_raw_article(i),
    #                   'output': text}
    #
    #     out_path = os.path.join(args.output_dir, eval_dataset.get_id(i))
    #
    #     with open(out_path, 'w') as f:
    #         json.dump(total_dict, f)

if __name__ == "__main__":
    main()

max_stars_repo_name: Kmeco/transformers

max_issues_repo_name: Kmeco/transformers

max_forks_repo_name: Kmeco/transformers