text:
""" full training (train rnn-ext + abs + RL) """
import argparse
import json
import pickle as pkl
import os
from os.path import join, exists
from itertools import cycle
import sys

from toolz.sandbox.core import unzip
from cytoolz import identity, concat

import torch
from torch import optim
from torch.optim.lr_scheduler import ReduceLROnPlateau
from torch.utils.data import DataLoader, Dataset
from utils import count_data
import pickle

from data.data import CnnDmDataset
from data.batcher import tokenize

from model.rl import ActorCritic, ActorCriticCand, ActorCriticSentBertCand, ActorCriticSentWordBertCand
from model.extract import PtrExtractSumm, PtrExtractRewrittenSumm, PtrExtractRewrittenBertSumm, PtrExtractRewrittenSentBertSumm, PtrExtractRewrittenSentWordBertSumm

from training import BasicTrainer
from rl import get_grad_fn
from rl import A2CPipeline
from decoding import load_best_ckpt
from decoding import Abstractor, ArticleBatcher, ConditionalAbstractor
from metric import compute_rouge_l, compute_rouge_l_summ, compute_rouge_n, compute_weighted_rouge_1_2
import random
import numpy as np

MAX_ABS_LEN = 30

try:
    DATA_DIR = os.environ['DATA']
except KeyError:
    print('please use environment variable to specify data directories')

class RLDataset(CnnDmDataset):
    """ get the article sentences only (for decoding use)"""
    def __init__(self, split):
        super().__init__(split, DATA_DIR)

    def __getitem__(self, i):
        js_data = super().__getitem__(i)
        art_sents = js_data['article']
        abs_sents = js_data['abstract']
        return art_sents, abs_sents

class RLDataset_backup(Dataset):
    def __init__(self, split):
        split_dir = os.path.join(DATA_DIR, split)
        cached_features_file = os.path.join(DATA_DIR, 'cached_' + split)

        if os.path.exists(cached_features_file):
            print("Loading features from cached file %s", cached_features_file)
            with open(cached_features_file, 'rb') as handle:
                self.examples = pickle.load(handle)
        else:
            print("Creating features from dataset file at %s", DATA_DIR)
            self.examples = []
            n_data = count_data(split_dir)
            for i in range(n_data):
                js = json.load(open(join(split_dir, '{}.json'.format(i))))
                if js['article'] and js['abstract']:
                    doc_sent_list = js['article']
                    summary_sent_list = js['abstract']
                    self.examples.append( (doc_sent_list, summary_sent_list) )

            print("Saving features into cached file %s", cached_features_file)
            with open(cached_features_file, 'wb') as handle:
                pickle.dump(self.examples, handle, protocol=4)

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, item):
        return self.examples[item]

def load_ext_net(ext_dir, ext_type):
    ext_meta = json.load(open(join(ext_dir, 'meta.json')))
    #assert ext_meta['net'] == 'ml_rnn_extractor'
    assert 'ml_{}_extractor'.format(ext_type) == ext_meta['net']
    ext_ckpt = load_best_ckpt(ext_dir)
    print("finish load chkpt")
    #if '_extractor._stop' not in ext_ckpt:
    #    ext_ckpt['_extractor._stop'] = torch.zeros_like(ext_ckpt['_extractor._init_i'])
    ext_args = ext_meta['net_args']
    vocab = pkl.load(open(join(ext_dir, 'vocab.pkl'), 'rb'))
    print("finish load vocab")
    if ext_type == "rewritten_rnn":
        ext = PtrExtractRewrittenSumm(**ext_args)
    elif ext_type == "rewritten_bert_rnn":
        ext = PtrExtractRewrittenBertSumm(**ext_args)
    elif ext_type == "rewritten_sent_bert_rnn":
        ext = PtrExtractRewrittenSentBertSumm(**ext_args)
    elif ext_type == "rewritten_sent_word_bert_rnn":
        ext = PtrExtractRewrittenSentWordBertSumm(**ext_args)
    else:
        ext = PtrExtractSumm(**ext_args)
    ext.load_state_dict(ext_ckpt)
    print("loaded extractor")
    return ext, vocab

def configure_net(abs_dir, ext_dir, ext_type, emb_type, cuda, num_candidates=1, is_conditional_abs=False):
    """ load pretrained sub-modules and build the actor-critic network"""
    # load pretrained abstractor model
    """
    if abs_dir is not None:
        abstractor = Abstractor(abs_dir, MAX_ABS_LEN, cuda)
    else:
        abstractor = identity
    """
    if abs_dir is None or "rewritten" in ext_type:
        abstractor = identity
    elif is_conditional_abs:
        abstractor = ConditionalAbstractor(abs_dir, MAX_ABS_LEN, cuda)
    else:
        abstractor = Abstractor(abs_dir, MAX_ABS_LEN, cuda)

    # load ML trained extractor net and buiild RL agent
    extractor, agent_vocab = load_ext_net(ext_dir, ext_type)

    if ext_type == "rewritten_rnn":
        assert num_candidates == extractor.num_candidates
        agent = ActorCriticCand(extractor._candidate_sent_enc,
                        extractor._candidate_agg,
                        extractor._art_enc,
                        extractor._extractor,
                        ArticleBatcher(agent_vocab, emb_type, cuda),
                                num_candidates)
    elif ext_type == "rewritten_bert_rnn":
        raise ValueError
    elif ext_type == "rewritten_sent_bert_rnn":
        assert num_candidates == extractor.num_candidates
        agent = ActorCriticSentBertCand(extractor._sentence_encoder,
                                extractor._bert_w,
                                extractor._candidate_agg,
                                extractor._art_enc,
                                extractor._extractor,
                                ArticleBatcher(agent_vocab, emb_type, cuda),
                                num_candidates)
    elif ext_type == "rewritten_sent_word_bert_rnn":
        assert num_candidates == extractor.num_candidates
        agent = ActorCriticSentWordBertCand(extractor._sentence_encoder,
                                extractor._bert_w,
                                extractor._candidate_sent_enc,
                                extractor._candidate_agg,
                                extractor._art_enc,
                                extractor._extractor,
                                ArticleBatcher(agent_vocab, emb_type, cuda),
                                num_candidates)
    else:
        agent = ActorCritic(extractor._sent_enc,
                        extractor._art_enc,
                        extractor._extractor,
                        ArticleBatcher(agent_vocab, emb_type, cuda))
    if cuda:
        agent = agent.cuda()

    net_args = {}
    net_args['abstractor'] = (None if abs_dir is None
                              else json.load(open(join(abs_dir, 'meta.json'))))
    net_args['extractor'] = json.load(open(join(ext_dir, 'meta.json')))

    return agent, agent_vocab, abstractor, net_args

def configure_training(opt, lr, clip_grad, lr_decay, batch_size,
                       gamma, reward, stop_coeff, stop_reward):
    assert opt in ['adam']
    opt_kwargs = {}
    opt_kwargs['lr'] = lr

    train_params = {}
    train_params['optimizer']      = (opt, opt_kwargs)
    train_params['clip_grad_norm'] = clip_grad
    train_params['batch_size']     = batch_size
    train_params['lr_decay']       = lr_decay
    train_params['gamma']          = gamma
    train_params['reward']         = reward
    train_params['stop_coeff']     = stop_coeff
    train_params['stop_reward']    = stop_reward

    return train_params

def build_batchers(batch_size, train_set_folder, valid_set_folder, emb_type, num_candidates, max_word):
    def coll(batch):
        art_batch, abs_batch = unzip(batch)
        """
        # debug
        art_batch = list(art_batch)
        print("raw batch:")
        print(list(art_batch)[0][0])
        print(list(art_batch)[0][1])
        print(list(art_batch)[0][2])
        print(list(art_batch)[0][3])
        """
        #art_sents = list(filter(bool, map(tokenize(max_word, emb_type, num_candidates), art_batch)))
        #abs_sents = list(filter(bool, map(tokenize(max_word, emb_type, num_candidates), abs_batch)))

        art_sents = []
        #abs_sents = []
        raw_art_sents = []
        raw_abs_sents = []
        for art, abs in zip(art_batch, abs_batch):
            tokenized_art = tokenize(max_word, emb_type, num_candidates, art)[:args.max_sent]
            # tokenized_abs = tokenize(max_word, emb_type, num_candidates, abs)
            raw_art = [sent.split(" ")[:max_word] for sent in art]
            raw_abs = [sent.split(" ")[:max_word] for sent in abs]
            if tokenized_art and raw_abs:
                art_sents.append(tokenized_art)
                #abs_sents.append(tokenized_abs)
                raw_art_sents.append(raw_art)
                raw_abs_sents.append(raw_abs)

        return art_sents, raw_art_sents, raw_abs_sents
    loader = DataLoader(
        RLDataset(train_set_folder), batch_size=batch_size,
        shuffle=True, num_workers=4, pin_memory=True,
        collate_fn=coll
    )
    val_loader = DataLoader(
        RLDataset(valid_set_folder), batch_size=batch_size,
        shuffle=False, num_workers=4, pin_memory=True,
        collate_fn=coll
    )
    return cycle(loader), val_loader

def weighted_sum_rouge_12l(prediction_sent_list, abstract_sent_list):
    concated_prediction = list(concat(prediction_sent_list))
    concated_abstract = list(concat(abstract_sent_list))
    rouge_1 = compute_rouge_n(concated_prediction, concated_abstract, n=1, mode='f')
    rouge_2 = compute_rouge_n(concated_prediction, concated_abstract, n=2, mode='f')
    rouge_l = compute_rouge_l_summ(prediction_sent_list, abstract_sent_list, mode='f')
    weight = 1.0/3
    return weight * rouge_1 + weight * rouge_2 + weight * rouge_l

def train(args):
    if not exists(args.path):
        os.makedirs(args.path)

    if "bert" in args.ext_type:
        args.emb_type = "bert"
    else:
        args.emb_type = "w2v"

    print("emb_type")
    print(args.emb_type)

    # make net
    agent, agent_vocab, abstractor, net_args = configure_net(
        args.abs_dir, args.ext_dir, args.ext_type, args.emb_type, args.cuda, args.num_candidates, args.is_conditional_abs)

    print("network config")

    # configure training setting
    assert args.stop > 0
    train_params = configure_training(
        'adam', args.lr, args.clip, args.decay, args.batch,
        args.gamma, args.reward, args.stop, 'rouge-1'
    )
    print("train config")
    train_batcher, val_batcher = build_batchers(args.batch, args.train_set_folder, args.valid_set_folder, args.emb_type, args.num_candidates, args.max_word)
    print("build batcher")
    if args.reward_type == 0:
        reward_fn = compute_rouge_l
        stop_reward_fn = compute_rouge_n(n=1)
    elif args.reward_type == 1 or args.reward_type == 2:
        reward_fn = compute_rouge_l_summ
        stop_reward_fn = compute_rouge_n(n=1)
    elif args.reward_type == 3:
        reward_fn = compute_rouge_l_summ(mode='r')
        stop_reward_fn = compute_rouge_n(n=1)
    elif args.reward_type == 4:
        reward_fn = compute_rouge_l_summ
        stop_reward_fn = compute_weighted_rouge_1_2(rouge_1_weight=0.5, mode='f')
    else:
        raise ValueError

    # save abstractor binary
    if args.abs_dir is not None:
        abs_ckpt = {}
        abs_ckpt['state_dict'] = load_best_ckpt(args.abs_dir)
        abs_vocab = pkl.load(open(join(args.abs_dir, 'vocab.pkl'), 'rb'))
        abs_dir = join(args.path, 'abstractor')
        os.makedirs(join(abs_dir, 'ckpt'))
        with open(join(abs_dir, 'meta.json'), 'w') as f:
            json.dump(net_args['abstractor'], f, indent=4)
        torch.save(abs_ckpt, join(abs_dir, 'ckpt/ckpt-0-0'))
        with open(join(abs_dir, 'vocab.pkl'), 'wb') as f:
            pkl.dump(abs_vocab, f, protocol=4)
    # save configuration
    meta = {}
    meta['net']           = '{}-ext_abs_rl'.format(args.ext_type)  # 'ml_{}_extractor'
    meta['net_args']      = net_args
    meta['train_params']  = train_params
    with open(join(args.path, 'meta.json'), 'w') as f:
        json.dump(meta, f, indent=4)
    with open(join(args.path, 'agent_vocab.pkl'), 'wb') as f:
        pkl.dump(agent_vocab, f, protocol=4)

    # prepare trainer
    grad_fn = get_grad_fn(agent, args.clip)
    optimizer = optim.Adam(agent.parameters(), **train_params['optimizer'][1])
    if args.no_lr_decay:
        scheduler = None
    else:
        scheduler = ReduceLROnPlateau(optimizer, 'max', verbose=True,
                                      factor=args.decay, min_lr=args.min_lr,
                                      patience=args.lr_p)

    pipeline = A2CPipeline(meta['net'], agent, abstractor,
                           train_batcher, val_batcher,
                           optimizer, grad_fn,
                           reward_fn, args.gamma,
                           stop_reward_fn, args.stop, args.reward_type, args.disable_selected_mask, args.is_conditional_abs, args.debug)
    trainer = BasicTrainer(pipeline, args.path,
                           args.ckpt_freq, args.patience, scheduler,
                           val_mode='score')

    print('start training with the following hyper-parameters:')
    print(meta)
    sys.stdout.flush()
    trainer.train()

if __name__ == '__main__':
    print("Go")
    parser = argparse.ArgumentParser(
        description='program to demo a Seq2Seq model'
    )
    parser.add_argument('--path', required=True, help='root of the model')

    # model options
    parser.add_argument('--abs_dir', action='store',
                        help='pretrained summarizer model root path')
    parser.add_argument('--ext_dir', action='store',
                        help='root of the extractor model')
    parser.add_argument('--ckpt', type=int, action='store', default=None,
                        help='ckeckpoint used decode')

    # training options
    parser.add_argument('--reward', action='store', default='rouge-l',
                        help='reward function for RL')
    parser.add_argument('--lr', type=float, action='store', default=1e-4,
                        help='learning rate')
    parser.add_argument('--decay', type=float, action='store', default=0.5,
                        help='learning rate decay ratio')
    parser.add_argument('--lr_p', type=int, action='store', default=0,
                        help='patience for learning rate decay')
    parser.add_argument('--gamma', type=float, action='store', default=0.95,
                        help='discount factor of RL')
    parser.add_argument('--stop', type=float, action='store', default=1.0,
                        help='stop coefficient for rouge-1')
    parser.add_argument('--clip', type=float, action='store', default=2.0,
                        help='gradient clipping')
    parser.add_argument('--batch', type=int, action='store', default=32,
                        help='the training batch size')
    parser.add_argument('--max_word', type=int, action='store', default=100,
                        help='maximun words in a single article sentence')
    parser.add_argument('--max_sent', type=int, action='store', default=60,
                        help='maximun sentences in an article article')
    parser.add_argument(
        '--ckpt_freq', type=int, action='store', default=1000,
        help='number of update steps for checkpoint and validation'
    )
    parser.add_argument('--patience', type=int, action='store', default=3,
                        help='patience for early stopping')
    parser.add_argument('--no-cuda', action='store_true',
                        help='disable GPU training')
    parser.add_argument('--disable_selected_mask', action='store_true',
                        help='disable the selection mask in the ptr network')

    parser.add_argument('--num_candidates', type=int, action='store', default=1,
                        help='number of candidates for each sentence.')
    parser.add_argument('--ext_type', type=str, action='store', default="rnn", choices=["rnn", "rewritten_rnn", "rewritten_bert_rnn", "rewritten_sent_bert_rnn", "rewritten_sent_word_bert_rnn"],
                        help='model type of the extractor (rnn, rewritten_rnn).')

    parser.add_argument('--train_set_folder', type=str, action='store', default="train",
                        help='The name of training set folder')
    parser.add_argument('--valid_set_folder', type=str, action='store', default="val",
                        help='The name of validation set folder')
    parser.add_argument('--debug', action='store_true',
                        help='use debug mode')

    parser.add_argument('--no_lr_decay', action='store_true',
                        help='do not use lr decay')
    parser.add_argument('--seed', type=int, action='store', default=9527,
                        help='disable GPU training')

    parser.add_argument('--min_lr', type=float, action='store', default=0,
                        help='minimum learning rate')
    parser.add_argument('--is_conditional_abs', action='store_true',
                        help='use conditional abstractor')

    parser.add_argument('--reward_type', type=int, action='store', default=0, choices=[0, 1, 2, 3, 4],
                        help='0: fast-rl, 1: weighted sum of Rouge 1, 2 and L + reward shaping')

    args = parser.parse_args()
    args.cuda = torch.cuda.is_available() and not args.no_cuda
    print("cuda is available:")
    print(args.cuda)
    print()

    torch.manual_seed(args.seed)
    np.random.seed(args.seed)
    random.seed(args.seed)

    train(args)


text:
import sys

import rasterio as rio
import numpy as np

def his(src_rst, out_csv, mask_rst):
    """ computes image histogram by segments.

    Args :
        src_rst (str) : path to the source raster
        out_csv (str) : path to the output csv file
        mask_rst (str, optional) : path to the mask raster (a labeled raster 0 being the background)

    Return :
        out_csv
    """
    # identify features in the mask
    features = None
    if mask_rst:
        with rio.open(mask_rst) as f:
            mask_flat = f.read(1).flatten()

        # count all the possible classes
        count = np.bincount(mask_flat)

        # create a list of features that only keep the not null one
        features = [i in range(np.amax(mask_flat) + 1)]
        features = features[np.nonzero(count)]

        # remove 0 (its the background)
        features = features[1:]

        # release memory
        del mask_flat

    # open mask and image
    with rio.open(mask_rst) as mask_f, rio.open(src_rst) as src_f:
        mask = mask_f.read()
        src = src_f.read()

    # count the number of bands
    # reshape src to have at least 1 band
    if src.ndim == 2:
        src = src[np.newaxis, :, :]
    nb_bands = src.shape[0]

    # create the dataset content
    mask_id = []
    frequencies = []
    bands = []
    values = []
    hists = []

    # loop in each feature of the mask
    for feat in features:

        for band in range(nb_bands):

            # get the array of the masked value from src image
            # feat == 0 means that there is no mask
            if feat == 0:
                slice_ = src[i][mask != feat]
            else:
                slice_ = src[i][mask == feat]

            # get the size
            size = slice_.shape[0]

            # count all the possible classes
            loc_count = np.bincount(slice_)

            # create a list of value that only keep the not null one
            loc_features = [i in range(np.amax(slice_) + 1)]
            loc_features = features[np.nonzero(loc_count)]

            # do the same for the loc_count
            loc_count = loc_count[np.nonzero(loc_count)]

            # add the value to the output
            mask_id += [feat for i in range(loc_count.shape[0])]
            frequencies += [size for i in range(loc_count.shape[0])]
            bands += [band for i in range(loc_count.shape[0])]
            values += loc_features
            hist += loc_count

    # create the final dataframe
    df = pd.DataFrame({'mask_id': mask_id, 'frequency': frequencies, 'band': bands, 'value': values, 'histogram': hists})

    # export
    df.to_csv(out_csv, index=False)

    return df

if __name__ == "__main__":

    # read arguments
    out_csv = sys.argv[sys.argv.index('-o') + 1]
    src_rst = sys.argv[sys.argv.index('-i') + 1]
    mask_rst = sys.argv[index('-s') + 1]

    # launch the function
    res = his(src_rst, out_csv, mask_rst)

    # dispay result
    if res:
        print(f'The segemented histogram have been created : {out_csv}')

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

import scipy.misc
import scipy.stats

from booltest import common
from booltest.booltest_main import *

logger = logging.getLogger(__name__)
coloredlogs.install(level=logging.DEBUG)

# Main - argument parsing + processing
class RandVerif(Booltest):
    def __init__(self, *args, **kwargs):
        super(RandVerif, self).__init__(*args, **kwargs)
        self.args = None
        self.tester = None
        self.blocklen = None
        self.input_poly = []

    # noinspection PyBroadException
    def work(self):
        """
        Main entry point - data processing.

        RandVerif is used to benchmark particular data sources, with different seeds.
        It takes e.g., AES-CTR(SHA256(random)), runs it 1000 times and stores the results. This particular
        computation was used to determine reference z-scores of the test.

        Another usual scenario is to take Java util.Random, seed it 1000 times with a different random seed
        and analyze the results.

        RandVerif supports supplying custom distinguishers so the whole space is not searched. This
        setup is used to assess found distinguishers on more data / independent data streams with different seeds.

        RandVerif produces results on STDOUT, it contains multiple sections, separated by -----BEGIN SECTION-----
        Sections contain stats (avg. z-score), all z-scores, the best distinguishers for further analysis...
        :return:
        """
        self.blocklen = int(self.defset(self.args.blocklen, 128))
        deg = int(self.defset(self.args.degree, 3))
        tvsize_orig = int(self.defset(self.process_size(self.args.tvsize), 1024*256))
        zscore_thresh = float(self.args.conf)
        rounds = int(self.args.rounds) if self.args.rounds is not None else None
        top_k = int(self.args.topk) if self.args.topk is not None else None
        top_comb = int(self.defset(self.args.combdeg, 2))
        reffile = self.defset(self.args.reffile)
        all_deg = self.args.alldeg
        tvsize = tvsize_orig

        top_distinguishers = []

        # Load input polynomials
        self.load_input_poly()
        script_path = common.get_script_path()

        logger.info('Basic settings, deg: %s, blocklen: %s, TV size: %s, rounds: %s'
                    % (deg, self.blocklen, tvsize_orig, rounds))

        total_terms = int(common.comb(self.blocklen, deg, True))
        hwanalysis = HWAnalysis()
        hwanalysis.deg = deg
        hwanalysis.blocklen = self.blocklen
        hwanalysis.top_comb = top_comb
        hwanalysis.comb_random = self.args.comb_random
        hwanalysis.top_k = top_k
        hwanalysis.combine_all_deg = all_deg
        hwanalysis.zscore_thresh = zscore_thresh
        hwanalysis.do_ref = reffile is not None
        hwanalysis.skip_print_res = True
        hwanalysis.input_poly = self.input_poly
        hwanalysis.no_comb_and = self.args.no_comb_and
        hwanalysis.no_comb_xor = self.args.no_comb_xor
        hwanalysis.prob_comb = self.args.prob_comb
        hwanalysis.all_deg_compute = len(self.input_poly) == 0
        hwanalysis.do_only_top_comb = self.args.only_top_comb
        hwanalysis.do_only_top_deg = self.args.only_top_deg
        hwanalysis.no_term_map = self.args.no_term_map
        hwanalysis.use_zscore_heap = self.args.topterm_heap
        hwanalysis.sort_best_zscores = max(common.replace_none([self.args.topterm_heap_k, top_k, 100]))
        hwanalysis.best_x_combinations = self.args.best_x_combinations
        logger.info('Initializing test')
        hwanalysis.init()

        dist_result_map = {}
        for idx, poly in enumerate(self.input_poly):
            dist_result_map[idx] = []

        for test_idx in range(self.args.tests):
            seed = random.randint(0, 2**32-1)
            iobj = None
            if self.args.test_randc:
                path = os.path.realpath(os.path.join(script_path, '../assets/rndgen-c/rand'))
                cmd = '%s %s' % (path, seed)
                iobj = common.CommandStdoutInputObject(cmd=cmd, seed=seed, desc='randc-%s' % seed)

            elif self.args.test_randc_small:
                path = os.path.realpath(os.path.join(script_path, '../assets/rndgen-c-small/rand'))
                cmd = '%s %s' % (path, seed)
                iobj = common.CommandStdoutInputObject(cmd=cmd, seed=seed, desc='randc-small-%s' % seed)

            elif self.args.test_java:
                path = os.path.realpath(os.path.join(script_path, '../assets/rndgen-java/'))
                cmd = 'java -cp %s Main %s' % (path, seed)
                iobj = common.CommandStdoutInputObject(cmd=cmd, seed=seed, desc='randjava-%s' % seed)

            elif self.args.test_aes:
                iobj = common.AESInputObject(seed=seed)

            else:
                raise ValueError('No generator to test')

            size = iobj.size()
            logger.info('Testing input object: %s, size: %d kB, iteration: %d' % (iobj, size/1024.0, test_idx))

            # size smaller than TV? Adapt tv then
            if size >= 0 and size < tvsize:
                logger.info('File size is smaller than TV, updating TV to %d' % size)
                tvsize = size

            if tvsize*8 % self.blocklen != 0:
                rem = tvsize*8 % self.blocklen
                logger.warning('Input data size not aligned to the block size. '
                               'Input bytes: %d, block bits: %d, rem: %d' % (tvsize, self.blocklen, rem))
                tvsize -= rem//8
                logger.info('Updating TV to %d' % tvsize)

            hwanalysis.reset()
            logger.info('BlockLength: %d, deg: %d, terms: %d' % (self.blocklen, deg, total_terms))
            with iobj:
                data_read = 0
                cur_round = 0

                while size < 0 or data_read < size:
                    if rounds is not None and cur_round > rounds:
                        break

                    data = iobj.read(tvsize)
                    bits = common.to_bitarray(data)
                    if len(bits) == 0:
                        logger.info('File read completely')
                        break

                    logger.info('Pre-computing with TV, deg: %d, blocklen: %04d, tvsize: %08d = %8.2f kB = %8.2f MB, '
                                'round: %d, avail: %d' %
                                (deg, self.blocklen, tvsize, tvsize/1024.0, tvsize/1024.0/1024.0, cur_round, len(bits)))

                    hwanalysis.process_chunk(bits, None)
                    cur_round += 1
                pass

            res = hwanalysis.input_poly_last_res
            if res is not None and len(res) > 0:
                res_top = res[0]
                top_distinguishers.append((res_top, seed))

                for cur in res:
                    dist_result_map[cur.idx].append(cur.zscore)

            elif hwanalysis.last_res is not None and len(hwanalysis.last_res) > 0:
                res_top = hwanalysis.last_res[0]
                top_distinguishers.append((res_top, seed))

            else:
                raise ValueError('No data from the analysis')

            logger.info('Finished processing %s ' % iobj)
            logger.info('Data read %s ' % iobj.data_read)
            logger.info('Read data hash %s ' % iobj.sha1.hexdigest())

        all_zscores = []
        print('-----BEGIN JSON-----')
        js = []
        for dist in top_distinguishers:
            cr = collections.OrderedDict()
            cr['z'] = dist[0].zscore
            try:
                cr['d'] = dist[0].idx
            except:
                pass

            cr['seed'] = dist[1]
            js.append(cr)
            all_zscores.append(dist[0].zscore)
        print(json.dumps(js, indent=2))

        print('-----BEGIN JSON-STATS-----')
        js = []
        for idx in dist_result_map:
            cur = dist_result_map[idx]
            cr = collections.OrderedDict()
            cr['idx'] = idx
            cr['poly'] = common.poly2str(self.input_poly[idx])
            cr['avg'] = sum([abs(x) for x in cur])/float(len(cur))
            cr['cnt'] = len(cur)
            cr['zscores'] = cur
            js.append(cr)
        print(json.dumps(js, indent=2))

        print('-----BEGIN RUN-CONFIG-----')
        js = collections.OrderedDict()
        js['block'] = self.blocklen
        js['deg'] = deg
        js['top_comb'] = top_comb
        js['top_k'] = top_k
        js['tvsize'] = tvsize
        js['tests'] = self.args.tests
        js['prob_comb'] = self.args.prob_comb
        js['all_deg'] = all_deg
        print(json.dumps(js))

        print('-----BEGIN Z-SCORES-NORM-----')
        print(all_zscores)
        print('-----BEGIN Z-SCORES-ABS-----')
        print([abs(x) for x in all_zscores])
        print('-----BEGIN Z-SCORES-AVG-----')
        print(sum([abs(x) for x in all_zscores])/float(len(all_zscores)))
        print('-----BEGIN Z-SCORES-NAVG-----')
        print(sum([x for x in all_zscores])/float(len(all_zscores)))

        if self.args.csv_zscore:
            print('-----BEGIN Z-SCORES-CSV-----')
            print('zscore')
            for x in [abs(x) for x in all_zscores]:
                print(x)

        logger.info('Processing finished')

    def main(self):
        logger.debug('App started')

        parser = argparse.ArgumentParser(description='RandVerif benchmarks particular data sources, with different seeds.')
        parser.add_argument('-t', '--threads', dest='threads', type=int, default=None,
                            help='Number of threads to use')

        parser.add_argument('--debug', dest='debug', action='store_const', const=True,
                            help='enables debug mode')

        parser.add_argument('--verbose', dest='verbose', action='store_const', const=True,
                            help='enables verbose mode')

        parser.add_argument('--ref', dest='reffile',
                            help='reference file with random data')

        parser.add_argument('--block', dest='blocklen',
                            help='block size in bits')

        parser.add_argument('--degree', dest='degree',
                            help='maximum degree of computation')

        parser.add_argument('--tv', dest='tvsize',
                            help='Size of one test vector, in this interpretation = number of bytes to read from file. '
                                 'Has to be aligned on block size')

        parser.add_argument('-r', '--rounds', dest='rounds',
                            help='Maximal number of rounds')

        parser.add_argument('--top', dest='topk', default=30, type=int,
                            help='top K number of best distinguishers to combine together')

        parser.add_argument('--comb-rand', dest='comb_random', default=0, type=int,
                            help='number of terms to add randomly to the combination set')

        parser.add_argument('--combine-deg', dest='combdeg', default=2, type=int,
                            help='Degree of combination')

        parser.add_argument('--conf', dest='conf', type=float, default=1.96,
                            help='Zscore failing threshold')

        parser.add_argument('--alldeg', dest='alldeg', action='store_const', const=True, default=False,
                            help='Add top K best terms to the combination group also for lower degree, not just top one')

        parser.add_argument('--poly', dest='polynomials', nargs=argparse.ZERO_OR_MORE, default=[],
                            help='input polynomial to evaluate on the input data instead of generated one')

        parser.add_argument('--poly-file', dest='poly_file', nargs=argparse.ZERO_OR_MORE, default=[],
                            help='input file with polynomials to test, one polynomial per line, in json array notation')

        parser.add_argument('--poly-ignore', dest='poly_ignore', action='store_const', const=True, default=False,
                            help='Ignore input polynomial variables out of range')

        parser.add_argument('--poly-mod', dest='poly_mod', action='store_const', const=True, default=False,
                            help='Mod input polynomial variables out of range')

        parser.add_argument('--no-comb-xor', dest='no_comb_xor', action='store_const', const=True, default=False,
                            help='Disables XOR combinations')

        parser.add_argument('--no-comb-and', dest='no_comb_and', action='store_const', const=True, default=False,
                            help='Disables AND combinations')

        parser.add_argument('--only-top-comb', dest='only_top_comb', action='store_const', const=True, default=False,
                            help='If set only the top combination is performed, otherwise all up to given combination degree')

        parser.add_argument('--only-top-deg', dest='only_top_deg', action='store_const', const=True, default=False,
                            help='If set only the top degree if base polynomials combinations are considered, otherwise '
                                 'also lower degrees are input to the topk for next state - combinations')

        parser.add_argument('--no-term-map', dest='no_term_map', action='store_const', const=True, default=False,
                            help='Disables term map precomputation, uses unranking algorithm instead')

        parser.add_argument('--prob-comb', dest='prob_comb', type=float, default=1.0,
                            help='Probability the given combination is going to be chosen.')

        parser.add_argument('--topterm-heap', dest='topterm_heap', action='store_const', const=True, default=False,
                            help='Use heap to compute best X terms for stats & input to the combinations')

        parser.add_argument('--topterm-heap-k', dest='topterm_heap_k', default=None, type=int,
                            help='Number of terms to keep in the heap')

        parser.add_argument('--best-x-combs', dest='best_x_combinations', default=None, type=int,
                            help='Number of best combinations to return. If defined, heap is used')

        parser.add_argument('--csv-zscore', dest='csv_zscore', action='store_const', const=True, default=False,
                            help='CSV output with zscores')

        parser.add_argument('--test-randc', dest='test_randc', action='store_const', const=True, default=False,
                            help='Test randc generator')

        parser.add_argument('--test-randc-small', dest='test_randc_small', action='store_const', const=True, default=False,
                            help='Test randc_small generator')

        parser.add_argument('--test-java', dest='test_java', action='store_const', const=True, default=False,
                            help='Test java generator')

        parser.add_argument('--test-aes', dest='test_aes', action='store_const', const=True, default=False,
                            help='AES test')

        parser.add_argument('--tests', dest='tests', type=int, default=100,
                            help='Number of tests to do')

        self.args = parser.parse_args()
        self.work()

# Launcher
app = None
if __name__ == "__main__":
    app = RandVerif()
    app.main()

max_stars_repo_name: sobuch/polynomial-distinguishers

max_issues_repo_name: sobuch/polynomial-distinguishers

max_forks_repo_name: sobuch/polynomial-distinguishers

reason: import scipy


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from PIL import ImageStat
import PIL.Image as Image
import random

import os
import numpy as np
import pandas as pd
import cv2
import albumentations as A
from albumentations.pytorch import ToTensorV2
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold, StratifiedKFold

import torchvision.transforms as transforms
from PIL import Image
from models.DCL import *
from config import *
from utils.dataset_DCL import collate_fn4train, collate_fn4val, collate_fn4test, collate_fn4backbone
from models.EffNet import EfficientNet

# args
DIR_INPUT = '/data1/yanglichen'
# DIR_INPUT = '/nvme/yangzeyu/plant-pathology-2020-fgvc7/images_mixup/'
# TRAIN_CSV_PATH = '/nvme/yangzeyu/plant-pathology-2020-fgvc7/train_XL_mixup_plus.csv'
PRETRAINED_FILE = "./resnet18-5c106cde.pth"
MODELPATH = "./models_dcleff512_XLData"
LOGFILE = "dcleff.log"
BATCH_SIZE = 8
SIZE = 512
N_EPOCH = 40
LR = 5e-4
N_FOLDS = 5
TTA_FLAG = True
TTA_TIMES = 8
NUM_CLS = 4
resize_resolution=512
crop_resolution=448
swap_num=[7, 7]
SUBMITNAME = "submission_dcleff512_XLData"

device_ids = [0, 1, 2, 3]
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True

setup_seed(1)

class PlantDataset(Dataset):
    def __init__(self, df, swap_size=(7, 7), num_classes=4, common_aug=None, swap=None, totensor=None, train=False,
                 train_val=False, test=False, transforms=None):
        self.df = df
        self.swap = swap
        self.common_aug = common_aug
        self.train = train
        self.test = test
        self.swap_size = swap_size
        self.totensor = totensor
        self.transforms = transforms
        self.numcls = num_classes

    def __len__(self):
        return self.df.shape[0]

    def crop_image(self, image, cropnum):
        width, high = image.size
        crop_x = [int((width / cropnum[0]) * i) for i in range(cropnum[0] + 1)]
        crop_y = [int((high / cropnum[1]) * i) for i in range(cropnum[1] + 1)]
        im_list = []
        for j in range(len(crop_y) - 1):
            for i in range(len(crop_x) - 1):
                im_list.append(image.crop((crop_x[i], crop_y[j], min(crop_x[i + 1], width), min(crop_y[j + 1], high))))
        return im_list

    def pil_loader(self,imgpath):
        with open(imgpath, 'rb') as f:
            with Image.open(f) as img:
                return img.convert('RGB')

    def __getitem__(self, idx):
        image_src = DIR_INPUT + '/images/' + self.df.loc[idx, 'image_id'] + '.jpg'
        # image_src = DIR_INPUT + self.df.loc[idx, 'image_id'] + '.jpg'
        image = self.pil_loader(image_src)
        label = self.df.loc[idx, ['healthy', 'multiple_diseases', 'rust', 'scab']].values
        label = label.astype(np.uint8)

        img_unswap = self.common_aug(image) if not self.common_aug is None else image
        image_unswap_list = self.crop_image(img_unswap, self.swap_size)
        swap_range = self.swap_size[0] * self.swap_size[1]
        swap_law1 = [(i - (swap_range // 2)) / swap_range for i in range(swap_range)]
        if self.train:
            img_swap = self.swap(img_unswap)
            image_swap_list = self.crop_image(img_swap, self.swap_size)
            unswap_stats = [sum(ImageStat.Stat(im).mean) for im in image_unswap_list]
            swap_stats = [sum(ImageStat.Stat(im).mean) for im in image_swap_list]
            swap_law2 = []
            for swap_im in swap_stats:
                distance = [abs(swap_im - unswap_im) for unswap_im in unswap_stats]
                index = distance.index(min(distance))
                swap_law2.append((index - (swap_range // 2)) / swap_range)
            img_swap = self.totensor(img_swap)
            label_swap = np.append(np.array([0]*self.numcls),label)
            img_unswap = self.totensor(img_unswap)
            return img_unswap, img_swap, label, label_swap, swap_law1, swap_law2, image_src
        else:
            swap_law2 = [(i - (swap_range // 2)) / swap_range for i in range(swap_range)]
            label_swap = label
            img_unswap = self.totensor(img_unswap)
            return img_unswap, label, label_swap, swap_law1, swap_law2, image_src

class Efficientb7(nn.Module):
    def __init__(self, num_classes=4):
        super().__init__()
        self.model = EfficientNet.from_pretrained('efficientnet-b7')
        in_features = self.model._fc.in_features
        self.model._fc = nn.Linear(in_features, num_classes)

    def forward(self, x):
        x = self.model(x)
        return x

class DenseCrossEntropy(nn.Module):

    def __init__(self):
        super(DenseCrossEntropy, self).__init__()

    def forward(self, logits, labels):
        logits = logits.float()
        labels = labels.float()

        logprobs = F.log_softmax(logits, dim=-1)

        loss = -labels * logprobs
        loss = loss.sum(-1)

        return loss.mean()

class DenseCrossEntropy_smooth(nn.Module):
    def __init__(self, num_classes=4, label_smoothing=0.05):
        super(DenseCrossEntropy_smooth, self).__init__()
        self.smoothing = label_smoothing
        self.criterion = DenseCrossEntropy()
        self.num_classes = num_classes

    def forward(self, x, target):
        x = x.float()
        traget = target.float()
        assert x.size(1) == self.num_classes
        target_smooth = ((1 - self.smoothing) * target) + (self.smoothing / self.num_classes)

        return self.criterion(x, target_smooth)

transformers = load_data_transformers(resize_resolution, crop_resolution, swap_num)

#submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
submission_df = pd.read_csv('./sample_submission.csv')
submission_df.iloc[:, 1:] = 0

dataset_test = PlantDataset(df=submission_df,
                            num_classes=NUM_CLS,
                            common_aug=transformers["None"],
                            swap=transformers["None"],
                            totensor=transformers["test_totensor"],
                            test=True)
dataloader_test = DataLoader(dataset_test,
                             batch_size=BATCH_SIZE,
                             shuffle=False,
                             num_workers=8,
                             collate_fn=collate_fn4test,
                             drop_last=False,
                             pin_memory=True)

setattr(dataloader_test, 'total_item_len', len(dataset_test))
setattr(dataloader_test, 'num_cls', NUM_CLS)

labeled_df = pd.read_csv(DIR_INPUT + '/train_XL.csv')
# labeled_df = pd.read_csv(TRAIN_CSV_PATH)
train_df = labeled_df
train_labels = train_df.iloc[:, 1:].values
train_y = train_labels[:, 2] + train_labels[:, 3] * 2 + train_labels[:, 1] * 3
dataset_train = PlantDataset(df=train_df,
                             num_classes=NUM_CLS,
                             common_aug=transformers["common_aug"],
                             swap=transformers["swap"],
                             totensor=transformers["train_totensor"],
                             train=True)
dataloader_train = DataLoader(dataset_train,
                              batch_size=BATCH_SIZE,
                              shuffle=True,
                              num_workers=8,
                              collate_fn=collate_fn4train,
                              drop_last=False,
                              pin_memory=True)
setattr(dataloader_train, 'total_item_len', len(dataset_train))

valid_df = labeled_df.loc[:220]
valid_df.index = pd.Series(range(221))
valid_labels = valid_df.iloc[:, 1:].values
dataset_valid = PlantDataset(df=valid_df,
                             num_classes=NUM_CLS,
                             common_aug=transformers["None"],
                             swap=transformers["None"],
                             totensor=transformers["val_totensor"],
                             train=False,
                             train_val=True)
dataloader_valid = DataLoader(dataset_valid,
                              batch_size=BATCH_SIZE,
                              shuffle=False,
                              num_workers=8,
                              collate_fn=collate_fn4test,
                              drop_last=False,
                              pin_memory=True)
setattr(dataloader_valid, 'total_item_len', len(dataset_valid))
setattr(dataloader_valid, 'num_cls', NUM_CLS)

criterion4 = DenseCrossEntropy_smooth(num_classes=4)
criterion8 = DenseCrossEntropy_smooth(num_classes=8)

def train_fold(i_fold, model, optimizer, dataloader_train, dataloader_valid, scheduler, logf):
    print("start_training\n")
    bestscore = 0
    bestpred = None
    train_batch_size = dataloader_train.batch_size
    add_loss = nn.L1Loss()
    for epoch in range(N_EPOCH):
        model.train(True)
        tr_loss = 0
        tr_preds = None
        tr_labels = None
        stepscnt = 0
        for step, batch in enumerate(dataloader_train):
            loss = 0.0
            model.train(True)

            inputs, labels, labels_swap, swap_law, img_names = batch
            labels = torch.from_numpy(np.array(labels))

            if tr_labels is None:
                tr_labels = labels.clone().squeeze(-1)
            else:
                tr_labels = torch.cat((tr_labels, labels.clone().squeeze(-1)), dim=0)

            inputs = Variable(inputs.to(device, dtype=torch.float))
            labels = Variable(labels.to(device, dtype=torch.long))
            labels_swap = Variable(torch.from_numpy(np.array(labels_swap)).to(device, dtype=torch.float))
            swap_law = Variable(torch.from_numpy(np.array(swap_law)).float().to(device, dtype=torch.float))

            optimizer.zero_grad()

            '''
            if inputs.size(0) < 2 * train_batch_size:
                outputs = model(inputs, inputs[0:-1:2])
            else:
                outputs = model(inputs, None)
            '''

            outputs = model(inputs, None)

            ce_loss = criterion4(outputs[0], labels)
            loss += ce_loss

            preds = torch.softmax(outputs[0], dim=1).data.cpu()
            if tr_preds is None:
                tr_preds = preds
            else:
                tr_preds = torch.cat((tr_preds, preds), dim=0)

            alpha_ = 1.0
            beta_ = 1.0
            gamma_ = 1.0

            swap_loss = criterion8(outputs[1], labels_swap) * beta_
            loss += swap_loss
            law_loss = add_loss(outputs[2], swap_law) * gamma_
            loss += law_loss

            tr_loss += loss

            loss.backward()
            torch.cuda.synchronize()
            print("train step : {}, loss : {}".format(step, loss))
            stepscnt += 1
            optimizer.step()
            torch.cuda.synchronize()

        scheduler.step()
        train_score = roc_auc_score(tr_labels, tr_preds, average='macro')
        print("train EPOCH : {},loss : {}".format(epoch, tr_loss / stepscnt))
        logf.write("train EPOCH : {},loss : {}\n".format(epoch, tr_loss / stepscnt))
        print("train EPOCH : {},train roc_auc: {}".format(epoch, roc_auc_score(tr_labels, tr_preds, average='macro')))
        logf.write(
            "train EPOCH : {},train roc_auc: {}\n".format(epoch, roc_auc_score(tr_labels, tr_preds, average='macro')))
        logf.flush()

        model.train(False)
        val_preds = None
        val_labels = None
        with torch.no_grad():
            for step, batch in enumerate(dataloader_valid):
                inputs = batch[0]
                labels = torch.from_numpy(np.array(batch[1]))
                if val_labels is None:
                    val_labels = labels.clone().squeeze(-1)
                else:
                    val_labels = torch.cat((val_labels, labels.clone().squeeze(-1)), dim=0)
                inputs = Variable(inputs.to(device, dtype=torch.float))
                labels = Variable(labels.to(device, dtype=torch.long))
                outputs = model(inputs)
                preds = torch.softmax(outputs[0], dim=1).data.cpu()
                if val_preds is None:
                    val_preds = preds
                else:
                    val_preds = torch.cat((val_preds, preds), dim=0)
        val_score = roc_auc_score(val_labels, val_preds, average='macro')
        score = val_score
        if (train_score < val_score):
            score = train_score
        if (score > bestscore):
            bestscore = score
            torch.save(model.state_dict(), '%s/dcleffnet_%03d.pth' % (MODELPATH, i_fold + 1))
            print("score improved")
            logf.write("score improved\n")
            logf.flush()
            bestpred = val_preds

        print("train EPOCH : {},roc_auc: {}".format(epoch, val_score))
        logf.write("train EPOCH : {},roc_auc: {}\n".format(epoch, val_score))
        logf.flush()

    return bestpred

def train(logf):
    train_results = []
    submissions = None
    folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=0)
    oof_preds = np.zeros((train_df.shape[0], 4))

    for i_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df, train_y)):
        print("Fold {}/{}".format(i_fold + 1, N_FOLDS))
        logf.write("Fold {}/{}".format(i_fold + 1, N_FOLDS))
        logf.flush()
        valid = train_df.iloc[valid_idx]
        valid.reset_index(drop=True, inplace=True)

        train = train_df.iloc[train_idx]
        train.reset_index(drop=True, inplace=True)

        dataset_train = PlantDataset(df=train,
                                     num_classes=NUM_CLS,
                                     common_aug=transformers["common_aug"],
                                     swap=transformers["swap"],
                                     totensor=transformers["train_totensor"],
                                     train=True)
        dataset_valid = PlantDataset(df=valid,
                                     num_classes=NUM_CLS,
                                     common_aug=transformers["None"],
                                     swap=transformers["None"],
                                     totensor=transformers["val_totensor"],
                                     train=False,
                                     train_val=True)
        dataloader_train = DataLoader(dataset_train,
                                      batch_size=BATCH_SIZE,
                                      shuffle=True,
                                      num_workers=8,
                                      collate_fn=collate_fn4train,
                                      drop_last=False,
                                      pin_memory=True)
        setattr(dataloader_train, 'total_item_len', len(dataset_train))
        dataloader_valid = DataLoader(dataset_valid,
                                      batch_size=BATCH_SIZE,
                                      shuffle=False,
                                      num_workers=8,
                                      collate_fn=collate_fn4test,
                                      drop_last=False,
                                      pin_memory=True)
        setattr(dataloader_valid, 'total_item_len', len(dataset_valid))
        setattr(dataloader_valid, 'num_cls', NUM_CLS)

        backbone = Efficientb7(num_classes=NUM_CLS)
        #backbone.load_state_dict(torch.load('./models_eff512_XLData/effnet_%03d.pth' % (i_fold + 1)))
        model = nn.DataParallel(MainModel(backbone=backbone, numcls=NUM_CLS), device_ids=device_ids)
        model.to(device)
        optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=5e-4)
        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10, 20, 30], gamma=0.3)

        val_preds = train_fold(i_fold, model, optimizer, dataloader_train, dataloader_valid, scheduler,logf=logf)
        oof_preds[valid_idx, :] = val_preds.numpy()

        model.eval()
        test_preds = eval(i_fold, model)
        if submissions is None:
            submissions = test_preds / N_FOLDS
        else:
            submissions += test_preds / N_FOLDS
    print("5-Folds CV score: {:.4f}".format(roc_auc_score(train_labels, oof_preds, average='macro')))
    logf.write("5-Folds CV score: {:.4f}\n".format(roc_auc_score(train_labels, oof_preds, average='macro')))
    logf.close()
    submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = torch.softmax(submissions, dim=1)
    submission_df.to_csv("{}.csv".format(SUBMITNAME), index=False)

def eval(i_fold, model):
    test_preds = None

    for step, batch in enumerate(dataloader_test):
        images = batch[0]

        images = images.to(device, dtype=torch.float)
        with torch.no_grad():
            outputs = model(images)
            preds = outputs[0] + outputs[1][:,0:NUM_CLS] + outputs[1][:,NUM_CLS:2*NUM_CLS]
            preds = preds.data.cpu()
            if test_preds is None:
                test_preds = preds
            else:
                test_preds = torch.cat((test_preds, preds), dim=0)
        # print(step)
    submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = torch.softmax(test_preds, dim=1)
    submission_df.to_csv('{}_fold{}.csv'.format(SUBMITNAME, i_fold), index=False)
    return test_preds

def TTA_eval():
    submissions = None
    backbone = Efficientb7(num_classes=NUM_CLS)
    model = nn.DataParallel(MainModel(backbone=backbone, numcls=NUM_CLS), device_ids=device_ids)
    model.to(device)
    for i_fold in range(5):
        #model.load_state_dict(torch.load('%s/effnet_%03d.pth' % (MODELPATH, i_fold + 1)))
        model.load_state_dict(torch.load('%s/dcleffnet_%03d.pth' % (MODELPATH, i_fold + 1)))
        model.eval()
        for j in range(TTA_TIMES):
            test_preds = eval(i_fold, model)
            if submissions is None:
                submissions = test_preds / (N_FOLDS * TTA_TIMES)
            else:
                submissions += test_preds / (N_FOLDS * TTA_TIMES)
        print("fold{} done".format(i_fold));

    submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = torch.softmax(submissions, dim=1)
    submission_df.to_csv("{}_TTA.csv".format(SUBMITNAME), index=False)

if __name__ == '__main__':
    logf = open(LOGFILE, "w")
    train(logf)
    TTA_eval()
    logf.close()


text:
"""Module with implementation of the Data class."""

import os
import string
import random
import shutil

import numpy
import dask.array as dsarray
import pyarrow
import zarr

import cbox.lib.boost as cbox

class Data(cbox.create.Data):
    """Default class to store data"""

    type_ = "default"

    def __init__(self, **attributes):
        """Initialize the class object

        Parameters
        ----------
        attributes : dictionary
                     { 'nblocks'   : total number of blocks,
                       'nxb'       : number of grid points per block in x dir,
                       'nyb'       : number of grid points per block in y dir,
                       'nzb'       : number of grid points per block in z dir,
                       'xguard'    : number of guard cells in x dir,
                       'yguard'    : number of guard cells in y dir,
                       'zguard'    : number of guard cells in z dir,
                       'inputfile' : hdf5 inputfile default (None),
                       'variables' : dictionary of variables default ({})
                       'storage'   : 'numpy', 'zarr', 'dask', 'pyarrow' }

        """
        super().__init__()
        self._set_attributes(attributes)
        self._set_data()

    def __repr__(self):
        """
        Return a representation of the object
        """
        return ("Data:\n" +
                " - type   : {}\n".format(type(self)) +
                " - keys   : {}\n".format(self.varlist))

    def __getitem__(self,varkey):
        """
        Get variable data
        """
        return self.variables[varkey]

    def __setitem__(self,varkey,value):
        """
        Set variable data
        """
        self.variables[varkey] = value

    def _set_attributes(self,attributes):
        """
        Private method for intialization
        """

        default_attributes = {'nblocks'   : 1,
                              'inputfile' : None,
                              'boxmem'    : None,
                              'variables' : {},
                                    'nxb' : 1, 'nyb'    : 1, 'nzb'    : 1,
                                 'xguard' : 0, 'yguard' : 0, 'zguard' : 0,
                                'storage' : 'numpy-memmap'}

        for key in attributes:
            if key in default_attributes:
                default_attributes[key] = attributes[key]
            else:
                raise ValueError('[bubblebox.library.create.Data] '+
                                 'Attribute "{}" not present in class Data'.format(key))

        for key, value in default_attributes.items(): setattr(self, key, value)

    def _set_data(self):
        """
        Private method for setting new data
        """
        self.varlist = list(self.variables.keys())

        if self.storage == 'numpy':
            self._create_numpy_arrays()
        elif self.storage == 'numpy-memmap':
            self._create_numpy_memmap()
        elif self.storage == 'zarr':
            self._create_zarr_objects()
        elif self.storage == 'dask':
            self._create_numpy_memmap()
            self._create_dask_objects()
        else:
            raise NotImplementedError('[bubblebox.library.create.Data] '+
                                      'Storage format "{}" not implemented'.format(self.storage))

    def _create_numpy_memmap(self):
        """
        Create numpy memory maps for empty keys in variables dictionary
        """
        emptykeys = [key for key,value in self.variables.items() if type(value) is type(None)]
        if not emptykeys: return

        if not self.boxmem:
            namerandom  = ''.join(random.choice(string.ascii_lowercase) for i in range(5))
            self.boxmem = "".join(['./boxmem_',namerandom])
        try:
            os.mkdir(self.boxmem)
        except FileExistsError:
            pass

        for varkey in emptykeys:
            outputfile = os.path.join(self.boxmem,varkey)
            outputshape = (self.nblocks, self.nzb+2*self.zguard, self.nyb+2*self.yguard, self.nxb+2*self.xguard)
            self.variables[varkey] = numpy.memmap(outputfile, dtype=float, shape=outputshape, mode='w+')

    def _create_zarr_objects(self):
        """
        Create zarr objects
        """
        emptykeys = [key for key,value in self.variables.items() if type(value) is type(None)]
        if not emptykeys: return

        if not self.boxmem:
            namerandom  = ''.join(random.choice(string.ascii_lowercase) for i in range(5))
            self.boxmem = "".join(['./boxmem_',namerandom])
        try:
            os.mkdir(self.boxmem)
        except FileExistsError:
            pass

        for varkey in emptykeys:
            outputfile = os.path.join(self.boxmem,varkey)
            outputshape = (self.nblocks, self.nzb+2*self.zguard, self.nyb+2*self.yguard, self.nxb+2*self.xguard)
            self.variables[varkey] = zarr.open(outputfile, mode='w', shape=outputshape,
                                         chunks=(1, self.nzb+2*self.zguard,
                                                    self.nyb+2*self.yguard, self.nxb+2*self.xguard),dtype=float)

    def _create_numpy_arrays(self):
        """
        Create numpy arrays for empty keys in variables dictionary
        """
        emptykeys = [key for key,value in self.variables.items() if type(value) is type(None)]
        if not emptykeys: return

        for varkey in emptykeys:
            outputshape = (self.nblocks, self.nzb+2*self.zguard, self.nyb+2*self.yguard, self.nxb+2*self.xguard)
            self.variables[varkey] = numpy.ndarray(dtype=float, shape=outputshape)

    def _create_dask_objects(self):
        """
        Create dask array representation of data
        """
        emptykeys = [key for key,value in self.variables.items() if type(value) is type(None)]
        if not emptykeys: return

        for varkey in emptykeys:
            if type(self.variables[varkey]) is not dsarray.core.Array:
                self.variables[varkey] = dsarray.from_array(self.variables[varkey],
                                             chunks=(1, self.nzb+2*self.zguard,
                                                        self.nyb+2*self.yguard, self.nxb+2*self.xguard))

    def _create_pyarrow_objects(self):
        """
        Create a pyarrow tensor objects
        """
        emptykeys = [key for key,value in self.variables.items() if type(value) is type(None)]
        if not emptykeys: return

        for varkey in emptykeys:
            if type(self.variables[varkey]) is not pyarrow.lib.Tensor:
                templist = []
                for lblock in range(self.nblocks):
                    templist.append(pyarrow.Tensor.from_numpy(self.variables[varkey][lblock]))
                self.variables[varkey] = templist

    def purge(self,purgeflag='all'):
        """
        Clean up data and close it
        """
        if self.boxmem and (purgeflag == 'all' or purgeflag == 'boxmem'):
            try:
                shutil.rmtree(self.boxmem)
            except:
                pass

        if self.inputfile and (purgeflag == 'all' or purgeflag == 'inputfile'):
            self.inputfile.close()

    def addvar(self,varkey):
        """
        Add a variables to data
        """
        self.variables[varkey] = None
        self._set_data()

    def delvar(self,varkey):
        """
        Delete a variable
        """
        del self.variables[varkey]

        if self.boxmem:
            outputfile  = os.path.join(self.boxmem,varkey)
            try:
                shutil.rmtree(outputfile)
            except:
                pass

        self.varlist = list(self.variables.keys())

max_stars_repo_name: akashdhruv/BubbleBox

max_issues_repo_name: akashdhruv/BubbleBox

max_forks_repo_name: akashdhruv/BubbleBox


text:
import copy
import os
import sys
import time
from collections import OrderedDict, defaultdict

import torch

import numpy as np
from hyperopt import hp
import ray
import gorilla
from ray.tune.trial import Trial
from ray.tune.trial_runner import TrialRunner
from ray.tune.suggest.hyperopt import HyperOptSearch
from ray.tune import register_trainable, run_experiments
from tqdm import tqdm

from pathlib import Path
lib_dir = (Path("__file__").parent).resolve()
if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
from FastAutoAugment.archive import remove_deplicates, policy_decoder
from FastAutoAugment.augmentations import augment_list
from FastAutoAugment.common import get_logger, add_filehandler
from FastAutoAugment.data import get_dataloaders, get_custom_dataloaders
from FastAutoAugment.metrics import Accumulator
from FastAutoAugment.networks import get_model, num_class
from FastAutoAugment.train import train_and_eval, train_controller, batch_policy_decoder, train_and_eval_ctl
from theconf import Config as C, ConfigArgumentParser
from FastAutoAugment.controller import Controller, RandAug

top1_valid_by_cv = defaultdict(lambda: list)

@ray.remote(num_gpus=1)
def eval_controller(config, controller, dataroot, cv_ratio=0., cv_fold=0, save_path=None, skip_exist=False):
    """
    training with augmented data and test with pure data
    """
    C.get()
    C.get().conf = config
    result = train_and_eval_ctl(None, controller, dataroot, test_ratio=cv_ratio, cv_fold=cv_fold, save_path=save_path, only_eval=skip_exist)
    return C.get()['model']['type'], cv_fold, result

def step_w_log(self):
    original = gorilla.get_original_attribute(ray.tune.trial_runner.TrialRunner, 'step')

    # log
    cnts = OrderedDict()
    for status in [Trial.RUNNING, Trial.TERMINATED, Trial.PENDING, Trial.PAUSED, Trial.ERROR]:
        cnt = len(list(filter(lambda x: x.status == status, self._trials)))
        cnts[status] = cnt
    best_top1_acc = 0.
    for trial in filter(lambda x: x.status == Trial.TERMINATED, self._trials):
        if not trial.last_result:
            continue
        best_top1_acc = max(best_top1_acc, trial.last_result['top1_valid'])
    print('iter', self._iteration, 'top1_acc=%.3f' % best_top1_acc, cnts, end='\r')
    return original(self)

patch = gorilla.Patch(ray.tune.trial_runner.TrialRunner, 'step', step_w_log, settings=gorilla.Settings(allow_hit=True))
gorilla.apply(patch)

logger = get_logger('Fast AutoAugment')

def _get_path(dataset, model, tag, basemodel=True):
    base_path = "models" if basemodel else f"models/{C.get()['exp_name']}"
    os.makedirs(base_path, exist_ok=True)
    return os.path.join(os.path.dirname(os.path.realpath(__file__)), '%s/%s_%s_%s.model' % (base_path, dataset, model, tag))     # TODO

@ray.remote(num_gpus=1)
def train_model(config, dataloaders, dataroot, augment, cv_ratio_test, cv_fold, save_path=None, skip_exist=False, reduced=False):
    C.get()
    C.get().conf = config
    C.get()['aug'] = augment

    result = train_and_eval(None, dataloaders, dataroot, cv_ratio_test, cv_fold, save_path=save_path, only_eval=skip_exist, reduced=reduced)
    return C.get()['model']['type'], cv_fold, result

if __name__ == '__main__':
    import json
    from pystopwatch2 import PyStopwatch
    w = PyStopwatch()

    parser = ConfigArgumentParser(conflict_handler='resolve')
    parser.add_argument('--dataroot', type=str, default='/mnt/ssd/data/', help='torchvision data folder')
    parser.add_argument('--until', type=int, default=5)
    parser.add_argument('--num-op', type=int, default=2)
    parser.add_argument('--num-policy', type=int, default=5)
    parser.add_argument('--num-search', type=int, default=200)
    parser.add_argument('--cv-ratio', type=float, default=0.40)
    parser.add_argument('--decay', type=float, default=-1)
    parser.add_argument('--redis', type=str)
    parser.add_argument('--per-class', action='store_true')
    parser.add_argument('--resume', action='store_true')
    parser.add_argument('--smoke-test', action='store_true')
    parser.add_argument('--cv-num', type=int, default=1)
    parser.add_argument('--exp_name', type=str)

    parser.add_argument('--lstm-size', type=int, default=100)
    parser.add_argument('--num-group', type=int, default=0)
    parser.add_argument('--gr-prob-weight', type=float, default=1e-3)
    parser.add_argument('--random-group', action='store_true', default=False)
    parser.add_argument('--random-aug', action='store_true')

    args = parser.parse_args()
    C.get()['exp_name'] = args.exp_name
    if args.decay > 0:
        logger.info('decay=%.4f' % args.decay)
        C.get()['optimizer']['decay'] = args.decay

    add_filehandler(logger, os.path.join('models', '%s_%s_cv%.1f.log' % (C.get()['dataset'], C.get()['model']['type'], args.cv_ratio)))
    logger.info('configuration...')
    logger.info(json.dumps(C.get().conf, sort_keys=True, indent=4))
    logger.info('initialize ray...')
    ray.init(address=args.redis, num_gpus=4)

    num_result_per_cv = 10
    cv_num = args.cv_num
    copied_c = copy.deepcopy(C.get().conf)

    dataloaders = get_custom_dataloaders(C.get()['dataset'], C.get()['batch'], C.get()['dataroot'], args.cv_ratio)
    logger.info('search augmentation policies, dataset=%s model=%s' % (C.get()['dataset'], C.get()['model']['type']))
    logger.info('----- Train without Augmentations cv=%d ratio(test)=%.1f -----' % (cv_num, args.cv_ratio))
    w.start(tag='train_no_aug')
    paths = [_get_path(C.get()['dataset'], C.get()['model']['type'], 'ratio%.1f_fold%d' % (args.cv_ratio, i)) for i in range(cv_num)]
    print(paths)
    reqs = [ # model training
        train_model.remote(copy.deepcopy(copied_c), dataloaders, args.dataroot, C.get()['aug'], args.cv_ratio, i, save_path=paths[i], skip_exist=True)
        for i in range(cv_num)]

    tqdm_epoch = tqdm(range(C.get()['epoch']))
    is_done = False
    for epoch in tqdm_epoch:
        while True:
            epochs_per_cv = OrderedDict()
            for cv_idx in range(cv_num):
                try:
                    latest_ckpt = torch.load(paths[cv_idx])
                    if 'epoch' not in latest_ckpt:
                        epochs_per_cv['cv%d' % (cv_idx + 1)] = C.get()['epoch']
                        continue
                    epochs_per_cv['cv%d' % (cv_idx+1)] = latest_ckpt['epoch']
                except Exception as e:
                    continue
            tqdm_epoch.set_postfix(epochs_per_cv)
            if len(epochs_per_cv) == cv_num and min(epochs_per_cv.values()) >= C.get()['epoch']:
                is_done = True
            if len(epochs_per_cv) == cv_num and min(epochs_per_cv.values()) >= epoch:
                break
            time.sleep(10)
        if is_done:
            break

    logger.info('getting results...')
    pretrain_results = ray.get(reqs)
    for r_model, r_cv, r_dict in pretrain_results:
        logger.info('model=%s cv=%d top1_train=%.4f top1_valid=%.4f' % (r_model, r_cv+1, r_dict['top1_train'], r_dict['top1_valid']))
    logger.info('processed in %.4f secs' % w.pause('train_no_aug'))

    if args.until == 1:
        sys.exit(0)
    if not args.random_aug:
        logger.info('----- Search Test-Time Augmentation Policies -----')
        w.start(tag='search')
        ctl_save_path = f"./models/{args.exp_name}"
        os.makedirs(ctl_save_path, exist_ok=True)
        ctl_save_path += "/controller.pt"
        # for _ in range(0):  # run multiple times.
        #     for cv_fold in range(cv_num):
                # TODO: training controller -> search policy using lstm controller
        controller = Controller(n_subpolicy=args.num_policy, lstm_size=args.lstm_size, n_group=args.num_group, gr_prob_weight=args.gr_prob_weight,\
                                img_input=not args.random_group).cuda()
        metrics, baseline = train_controller(controller, dataloaders, paths[0], ctl_save_path)
    else:
        controller = RandAug(n_subpolicy=args.num_policy)

    logger.info('----- Train with Augmentations model=%s dataset=%s aug=%s ratio(test)=%.1f -----' % (C.get()['model']['type'], C.get()['dataset'], C.get()['aug'], args.cv_ratio))
    w.start(tag='train_aug')

    num_experiments = 4
    # default_path = [_get_path(C.get()['dataset'], C.get()['model']['type'], 'ratio%.1f_default%d' % (args.cv_ratio, _), False) for _ in range(num_experiments)]
    augment_path = [_get_path(C.get()['dataset'], C.get()['model']['type'], 'ratio%.1f_augment%d' % (args.cv_ratio, _), False) for _ in range(num_experiments)]
    # reqs = [train_model.remote(copy.deepcopy(copied_c), dataloaders, args.dataroot, C.get()['aug'], 0.0, 0, save_path=default_path[_], skip_exist=True) for _ in range(num_experiments)] + \
    reqs = [eval_controller.remote(copy.deepcopy(copied_c), controller, args.dataroot, 0.0, 0, save_path=augment_path[_]) for _ in range(num_experiments)]
            # [train_model.remote(copy.deepcopy(copied_c), dataloaders, args.dataroot, "fa_reduced_cifar10", 0.0, 0, save_path=augment_path[_]) for _ in range(num_experiments)] + \

    tqdm_epoch = tqdm(range(C.get()['epoch']))
    is_done = False
    for epoch in tqdm_epoch:
        while True:
            epochs = OrderedDict()
            for exp_idx in range(num_experiments):
                # try:
                #     if os.path.exists(default_path[exp_idx]):
                #         latest_ckpt = torch.load(default_path[exp_idx])
                #         epochs['default_exp%d' % (exp_idx + 1)] = latest_ckpt['epoch']
                # except:
                #     pass
                try:
                    if os.path.exists(augment_path[exp_idx]):
                        latest_ckpt = torch.load(augment_path[exp_idx])
                        epochs['augment_exp%d' % (exp_idx + 1)] = latest_ckpt['epoch']
                except:
                    pass

            tqdm_epoch.set_postfix(epochs)
            if len(epochs) == num_experiments and min(epochs.values()) >= C.get()['epoch']:
                is_done = True
            if len(epochs) == num_experiments and min(epochs.values()) >= epoch:
                break
            time.sleep(10)
        if is_done:
            break

    logger.info('getting results...')
    final_results = ray.get(reqs)
    print(final_results)
    print(final_results[0])
    print(final_results[0][-1]['top1_test'])
    for train_mode in ['augment']: # ['default', 'augment']
        avg = 0.
        for _ in range(num_experiments):
            r_model, r_cv, r_dict = final_results.pop(0)
            logger.info('[%s] top1_train=%.4f top1_test=%.4f' % (train_mode, r_dict['top1_train'], r_dict['top1_test']))
            avg += r_dict['top1_test']
        avg /= num_experiments
        logger.info('[%s] top1_test average=%.4f (#experiments=%d)' % (train_mode, avg, num_experiments))
    logger.info('processed in %.4f secs' % w.pause('train_aug'))

    logger.info(w)

max_stars_repo_name: yongsu-bloo/fast-autoaugment

max_issues_repo_name: yongsu-bloo/fast-autoaugment

max_forks_repo_name: yongsu-bloo/fast-autoaugment


text:
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')

from gensim.models import KeyedVectors
import jsonlines
import sys
import math
import numpy as np
import argparse
from sklearn.linear_model import LogisticRegression

def get_wv_model(google_model):
    try:
        if not google_model:
            model = KeyedVectors.load('text8.model')
            model = model.wv
        else:
            # model available at: https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit
            model = KeyedVectors.load_word2vec_format('model\GoogleNews-vectors-negative300.bin', binary=True)
        return model
    except IOError:
        sys.stderr.write('ERROR: Cannot open word2vec model.\n')
        sys.exit(1)

def get_json(file):
    try:
        return jsonlines.open(file)
    except IOError:
        sys.stderr.write('ERROR: Cannot open %s.\n' % file)
        sys.exit(1)

def product_similarity(sent1, sent2):
    sent1 = sent1.strip().split(" ")
    sent2 = sent2.strip().split(" ")

    hypothesis_dict = {}
    sent22 = [word2.lower() for word2 in sent2 if word2.lower() in model.vocab]
    sent11 = [word1.lower() for word1 in sent1 if word1.lower() in model.vocab]

    for word2 in sent22:
        sim = 0
        for word1 in sent11:
            currsim = model.similarity(word2, word1)
            if currsim > sim:
                sim = currsim
                hypothesis_dict[word2] = (word1, currsim)

    product = 1
    for k, (v1, v2) in hypothesis_dict.items():
        product *= v2
    return product

def mean_similarity(sent1, sent2):
    sent1 = sent1.strip().split(" ")
    sent2 = sent2.strip().split(" ")

    hypothesis_dict = {}
    sent22 = [word2.lower() for word2 in sent2 if word2.lower() in model.vocab]
    sent11 = [word1.lower() for word1 in sent1 if word1.lower() in model.vocab]

    for word2 in sent22:
        sim = 0
        for word1 in sent11:
            currsim = model.similarity(word2, word1)
            if currsim > sim:
                sim = currsim
                hypothesis_dict[word2] = (word1, currsim)

    sum = 0
    for k, (v1, v2) in hypothesis_dict.items():
        sum += v2
    if len(hypothesis_dict.items()) > 0:
        return sum/len(hypothesis_dict.items()) # normalize sum
    else:
        return 0

def train_model(similarity_function):
    reader_train = get_json(file_train)

    element = 0
    feat = []
    y = []
    for obj in reader_train:
        if element < TRAIN:
            sent1 = obj["sentence1"]
            sent2 = obj["sentence2"]
            product = similarity_function(sent1, sent2)
            logprob = math.log(product + 1)
            if obj["gold_label"] == "neutral":
                y.append(1.0)
            else:
                y.append(0.0)

            feat.append(logprob)
            element += 1
        else:
            break
    one_vec = [1.0 for i in feat]
    X = np.matrix([one_vec, feat])
    X = X.transpose()
    logisticRegr = LogisticRegression()
    logisticRegr.fit(X, y)
    score = logisticRegr.score(X, y)
    print("Logreg score on train: ", score)
    return logisticRegr

def predict(logisticRegr,similarity_function):
    reader_dev = get_json(file_dev)
    feat_test = []
    y_test = []
    element = 0
    for obj in reader_dev:
        if element < DEV:
            sent1 = obj["sentence1"]
            sent2 = obj["sentence2"]
            product = similarity_function(sent1, sent2)
            logprob = math.log(product + 1)
            if obj["gold_label"] == "neutral":
                y_test.append(1.0)
            else:
                y_test.append(0.0)
            feat_test.append(logprob)
            element += 1
    one_vec = [1.0 for i in feat_test]
    X_test = np.matrix([one_vec, feat_test])
    X_test = X_test.transpose()
    predictions = logisticRegr.predict(X_test)
    score = logisticRegr.score(X_test, y_test)
    print("Logreg score on dev: ", score)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='word2vec baseline for nli-task made by Zsaknadrag')
    parser.add_argument('-googlemodel', action='store_true', help = 'use (larger) google model, default value: False')
    parser.add_argument('-number', metavar='number', type=int, help='number of analyzed (train + dev together) sentence pairs, default value: 1000')
    parser.add_argument('-train', metavar='train', type=str, help='input train file in jsonl format')
    parser.add_argument('-dev', metavar='dev', type=str, help='input dev file in jsonl format')
    args = parser.parse_args()

    MAX_PEEK = 1000
    if args.number:
        MAX_PEEK = args.number
    TRAIN = 0.75 * MAX_PEEK
    DEV = MAX_PEEK - TRAIN

    GOOGLE_MODEL = False
    if args.googlemodel:
        GOOGLE_MODEL = True

    file_train = "multinli/multinli_1.0_train.jsonl"
    if args.train:
        file_train = args.train

    file_dev = "multinli/multinli_1.0_dev_mismatched.jsonl"
    if args.dev:
        file_dev = args.dev

    model = get_wv_model(GOOGLE_MODEL)
    print("Product Similarity results")
    logreg_model = train_model(product_similarity)
    predict(logreg_model, product_similarity)
    print("Mean Similarity results")
    logreg_model = train_model(mean_similarity)
    predict(logreg_model, mean_similarity)


text:
import unittest
from pyapprox.quantile_regression import *
from pyapprox.risk_measures import value_at_risk
from pyapprox.optimization import check_gradients
from scipy.special import erf, erfinv, factorial
from scipy import stats

try:
    import cvxopt
    cvxopt_missing = False
except:
    cvxopt_missing = True
skipcvxopttest = unittest.skipIf(
    cvxopt_missing, reason="cvxopt package missing")

class TestQuantileRegression(unittest.TestCase):

    def setUp(self):
        np.random.seed(1)

    @skipcvxopttest
    def test_quantile_regression(self):
        np.random.seed(1)
        nbasis = 20

        def func(x):
            return (1+x-x**2+x**3).T
        samples = np.random.uniform(-1, 1, (1, 201))
        values = func(samples)

        def eval_basis_matrix(x):
            return (x**np.arange(nbasis)[:, None]).T
        tau = 0.75
        quantile_coef = solve_quantile_regression(
            tau, samples, values, eval_basis_matrix)
        true_coef = np.zeros((nbasis))
        true_coef[:4] = [1, 1, -1, 1]
        assert np.allclose(quantile_coef[:, 0], true_coef)

if __name__ == "__main__":
    quantile_regression_test_suite = \
        unittest.TestLoader().loadTestsFromTestCase(TestQuantileRegression)
    unittest.TextTestRunner(verbosity=2).run(quantile_regression_test_suite)

max_stars_repo_name: ConnectedSystems/pyapprox

max_issues_repo_name: ConnectedSystems/pyapprox

max_forks_repo_name: ConnectedSystems/pyapprox

reason: from scipy


text:
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import os

import tensorflow as tf
import tqdm
from absl import app
from absl import flags

from data.common import MODEL_DIR, get_pretrained_word_embedding
from data.mscoco import load_mscoco_data, DATA_DIR
from membership.utils import compute_adversarial_advantage
from text_encoder import ImageTextEncoder

flags.DEFINE_integer('print_every', 100, 'Print loss every N iterations')
flags.DEFINE_integer('batch_size', 512, 'Batch size')
flags.DEFINE_integer('encoder_dim', 512, 'encoder dim')
flags.DEFINE_integer('epoch', 100, 'Epochs of training')
flags.DEFINE_boolean('contrastive', False, 'Use vse++ contrastive loss')
flags.DEFINE_float('train_size', 0.1, 'Temp for sharpening')
flags.DEFINE_string('metric', 'cosine', 'Metric to use for membership')
flags.DEFINE_string('model', 'linear', 'Attack model')
flags.DEFINE_string('encoder_type', 'rnn', 'Text encoder type')
flags.DEFINE_string('image_model_name', 'resnet',
                    'Image feature extractor name')
flags.DEFINE_string('save_dir', os.path.join(MODEL_DIR, 'imgtxt'),
                    'Model directory for embedding model')
FLAGS = flags.FLAGS

def collect_metrics(img_embs, text_embs, img_id_text_ids):
  metrics = []
  for img_id in tqdm.tqdm(img_id_text_ids):
    text_ids = img_id_text_ids[img_id]
    i_emb = img_embs[img_id]
    t_embs = text_embs[text_ids]
    if FLAGS.metric == 'l2':
      scores = [-np.linalg.norm(i_emb - t_emb) for t_emb in t_embs]
    elif FLAGS.metric == 'cosine':
      scores = [np.dot(i_emb, t_emb) for t_emb in t_embs]
    else:
      raise ValueError(FLAGS.metrics)

    metrics.append(scores)
  return metrics

def encode_image_text_data():
  train_data, val_data, vocab = load_mscoco_data(FLAGS.image_model_name)
  word_embedding = get_pretrained_word_embedding(
      vocab, glove=False, vocab_include_pad=True,
      save_path=DATA_DIR + 'w2v.npz')
  encoder = ImageTextEncoder(word_embedding, FLAGS.encoder_dim,
                             FLAGS.encoder_type)

  model_name = 'coco_e{}_b{}_{}'.format(
    FLAGS.epoch, FLAGS.batch_size, FLAGS.encoder_type)
  if FLAGS.contrastive:
    model_name += '_vse'

  save_path = os.path.join(FLAGS.save_dir, model_name)

  encoder.load_weights(save_path)
  train_img_embs, train_text_embs, train_img_id_text_ids = \
      encoder.encode(train_data)
  test_img_embs, test_text_embs, test_img_id_text_ids = \
      encoder.encode(val_data)

  train_data = train_img_embs, train_text_embs, train_img_id_text_ids
  test_data = test_img_embs, test_text_embs, test_img_id_text_ids
  tf.keras.backend.clear_session()
  return train_data, test_data

def main(_):
  train_data, test_data = encode_image_text_data()
  train_metrics = collect_metrics(*train_data)
  test_metrics = collect_metrics(*test_data)
  compute_adversarial_advantage([np.mean(m) for m in train_metrics],
                                [np.mean(m) for m in test_metrics])

if __name__ == '__main__':
  app.run(main)

max_stars_repo_name: dbogatov/embedding-tests

max_issues_repo_name: dbogatov/embedding-tests

max_forks_repo_name: dbogatov/embedding-tests


text:
"""
 Copyright (c) 2020 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import argparse
from os.path import exists, join, abspath, isfile
from os import listdir, walk, makedirs
from shutil import rmtree

import cv2
import numpy as np
from tqdm import tqdm

def create_dirs(dir_path, override=False):
    if override:
        if exists(dir_path):
            rmtree(dir_path)
        makedirs(dir_path)
    elif not exists(dir_path):
        makedirs(dir_path)

def parse_relative_paths(data_dir):
    data_dir = abspath(data_dir)
    skip_size = len(data_dir) + 1

    relative_paths = []
    for root, sub_dirs, files in walk(data_dir):
        if len(sub_dirs) == 0 and len(files) > 0:
            relative_paths.append(root[skip_size:])

    return relative_paths

def prepare_tasks(relative_paths, in_images, in_masks, out_images, out_masks):
    out_data = []
    for relative_path in relative_paths:
        in_images_dir = join(in_images, relative_path)
        assert exists(in_images_dir)

        in_masks_dir = join(in_masks, relative_path)
        assert exists(in_masks_dir)

        image_names = [f.split('.')[0] for f in listdir(in_images_dir) if isfile(join(in_images_dir, f))]
        mask_names = [f.split('.')[0] for f in listdir(in_masks_dir) if isfile(join(in_masks_dir, f))]
        valid_names = list(set(image_names) & set(mask_names))

        if len(valid_names) == 0:
            continue

        out_images_dir = join(out_images, relative_path)
        if not exists(out_images_dir):
            makedirs(out_images_dir)

        out_masks_dir = join(out_masks, relative_path)
        if not exists(out_masks_dir):
            makedirs(out_masks_dir)

        for name in valid_names:
            in_image_path = join(in_images_dir, '{}.jpg'.format(name))
            in_mask_path = join(in_masks_dir, '{}.png'.format(name))
            out_image_path = join(out_images_dir, '{}.jpg'.format(name))
            out_mask_path = join(out_masks_dir, '{}.png'.format(name))

            out_data.append((in_image_path, in_mask_path, out_image_path, out_mask_path))

    return out_data

def estimate_bbox(mask, scale):
    pts = cv2.findNonZero(mask)
    x, y, w, h = cv2.boundingRect(pts)

    crop_center_x = float(x) + 0.5 * w
    crop_center_y = float(y) + 0.5 * h

    new_crop_width_half = 0.5 * scale * float(w)
    new_crop_height_half = 0.5 * scale * float(h)

    image_height, image_width = mask.shape
    xmin = max(0, int(crop_center_x - new_crop_width_half))
    ymin = max(0, int(crop_center_y - new_crop_height_half))
    xmax = min(image_width, int(crop_center_x + new_crop_width_half))
    ymax = min(image_height, int(crop_center_y + new_crop_height_half))

    return xmin, ymin, xmax, ymax

def make_crop(img, bbox):
    xmin, ymin, xmax, ymax = bbox
    return img[ymin:ymax, xmin:xmax]

def valid(img, invalid_size=50):
    return img is not None and img.shape[0] >= invalid_size and img.shape[1] >= invalid_size

def resize_image(image, mask, min_side_size):
    if min_side_size <= 0:
        return image, mask

    src_height, src_width = image.shape[:2]

    ar = float(src_height) / float(src_width)
    if ar > 1.0:
        trg_width = min_side_size
        trg_height = int(trg_width * ar)
    else:
        trg_height = min_side_size
        trg_width = int(trg_height / ar)

    image = cv2.resize(image, (trg_width, trg_height), interpolation=cv2.INTER_LINEAR)
    mask = cv2.resize(mask, (trg_width, trg_height), interpolation=cv2.INTER_NEAREST)

    return image, mask

def process_tasks(tasks, min_non_zero_fraction, scale, min_side_size):
    num_valid_tasks = 0
    for in_image_path, in_mask_path, out_image_path, out_mask_path in tqdm(tasks):
        mask = cv2.imread(in_mask_path, cv2.IMREAD_GRAYSCALE)
        if not valid(mask):
            continue

        mask_area = np.count_nonzero(mask)
        non_zero_fraction = float(mask_area) / float(mask.shape[0] * mask.shape[1])
        if non_zero_fraction < min_non_zero_fraction:
            continue

        image = cv2.imread(in_image_path, cv2.IMREAD_COLOR)
        if not valid(image):
            continue

        bbox = estimate_bbox(mask, scale)
        image = make_crop(image, bbox)
        mask = make_crop(mask, bbox)
        if not valid(image) or not valid(mask):
            continue

        image, mask = resize_image(image, mask, min_side_size)

        cv2.imwrite(out_image_path, image)
        cv2.imwrite(out_mask_path, mask)

        num_valid_tasks += 1

    num_invalid_tasks = len(tasks) - num_valid_tasks
    print('Final num valid images: {} ({} invalid)'.format(num_valid_tasks, num_invalid_tasks))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--in-images', '-ii', type=str, required=True)
    parser.add_argument('--in-masks', '-im', type=str, required=True)
    parser.add_argument('--out-images', '-oi', type=str, required=True)
    parser.add_argument('--out-masks', '-om', type=str, required=True)
    parser.add_argument('--non-zero-fraction', '-nz', type=float, required=False, default=0.2)
    parser.add_argument('--crop-scale', '-cs', type=float, required=False, default=1.08)
    parser.add_argument('--min-side-size', '-ms', type=int, required=False, default=240)
    args = parser.parse_args()

    assert exists(args.in_images)
    assert exists(args.in_masks)

    create_dirs(args.out_images, override=True)
    create_dirs(args.out_masks, override=True)

    print('Preparing tasks ...')
    relative_paths = parse_relative_paths(args.in_images)
    tasks = prepare_tasks(relative_paths, args.in_images, args.in_masks, args.out_images, args.out_masks)

    print('Processing tasks ...')
    process_tasks(tasks, args.non_zero_fraction, args.crop_scale, args.min_side_size)

if __name__ == '__main__':
    main()

max_stars_repo_name: dhenisdj/deep-object-reid

max_issues_repo_name: dhenisdj/deep-object-reid

max_forks_repo_name: dhenisdj/deep-object-reid


text:
"""The input / output functions related to file system."""
import fabio
from functools import wraps
import numpy as np
from diffpy.srfit.fitbase import Profile
from diffpy.structure import loadStructure
from diffpy.utils.parsers.loaddata import loadData
from pyobjcryst import loadCrystal

from pdffitx import modeling as md
from pdffitx.modeling.fitobjs import MyParser

@wraps(loadCrystal)
def load_crystal(*args, **kwargs):
    return loadCrystal(*args, **kwargs)

@wraps(loadStructure)
def load_structure(*args, **kwargs):
    return loadStructure(*args, **kwargs)

def load_parser(filename: str, meta: dict) -> MyParser:
    """Load data and metadata from the filename into a parser.

    Parameters
    ----------
    filename :
        The name of the data file.

    meta :
        The dictionary of the meta data, like {'qdamp': 0.04, 'qbroad': 0.02}.

    Returns
    -------
    parser :
        The parser that contains the data and metadata.
    """
    parser = MyParser()
    parser.parseFile(filename, meta=meta)
    return parser

def load_profile(filename: str, metadata: dict = None) -> Profile:
    profile = Profile()
    parser = md.MyParser()
    parser.parseFile(filename, metadata)
    profile.loadParsedData(parser)
    return profile

def load_img(img_file: str) -> np.ndarray:
    """Load the img data from the img_file."""
    img = fabio.open(img_file).data
    return img

def load_array(data_file: str, minrows=10, **kwargs) -> np.ndarray:
    """Load data columns from the .txt file and turn columns to rows and return the numpy array."""
    return loadData(data_file, minrows=minrows, **kwargs).T

max_stars_repo_name: xpdAcq/pdffitx

max_issues_repo_name: xpdAcq/pdffitx

max_forks_repo_name: xpdAcq/pdffitx


text:
from sklearn.metrics import average_precision_score
import pandas as pd
import numpy as np

def mAP_score(preds: pd.DataFrame, trues: pd.DataFrame):
    """ Funkcja obliczająca wynik modelu sieci neuronowej

        :param preds: Wartości przewidywane przez model
        :type preds: pd.DataFrame
        :param trues: Wartości prawdziwe
        :type trues: pd.DataFrame
        :return: Ogólny wynik modelu oraz wyniki dla każdej z cech w postaci listy
        :rtype: float, list
        """
    labels_score = []
    trues = trues.replace(-1,0)
    score = 0.0
    for p, t in zip(preds, trues):
        ap = 0.0
        if np.sum(trues[t]) != 0:
            ap = average_precision_score(trues[t], preds[p])
        labels_score.append(ap)
        score += ap

    score /= preds.shape[1]

    return score, labels_score


text:
import numpy as np

# def extract_sentence_containing_word(paragraph, word):
#     """ Document and test me"""
#     sentences = paragraph.split(".")
#     selected = [s.strip() for s in sentences if word in s]
#     return selected

def integrate_trapz(xs, ys):
    """ Document and test me"""
    widths = np.diff(xs)
    midpoints = 0.5 * (ys[1:] + ys[:-1])
    area = np.sum(widths * midpoints)
    return area