text:
import numpy as np
import argparse
import sys
import tensorflow as tf
import os
from threading import Thread
import time

from patroller_cnn import Patroller_CNN
from patroller_rule import Patroller_Rule as Patroller_h
from poacher_cnn import Poacher
from poacher_rule import Poacher as Poacher_h
from env import Env
from replay_buffer import ReplayBuffer
from DeDOL_util import simulate_payoff
from DeDOL_util import calc_pa_best_response_PER as calc_pa_best_response
from DeDOL_util import extend_payoff
from DeDOL_util import calc_NE,calc_NE_zero
from DeDOL_util import calc_po_best_response_PER as calc_po_best_response
from DeDOL_util import tf_copy
from DeDOL_util import PRDsolver
from patroller_randomsweeping import RandomSweepingPatroller
from maps import Mountainmap,  generate_map
from GUI_util import test_gui

global eps_pa
global eps_po
global pa_start_num
global po_strat_num
eps_pa, eps_po = [], []
pa_strat_num, po_strat_num = 0, 0

argparser = argparse.ArgumentParser()
########################################################################################
### Environment
argparser.add_argument('--row_num', type=int, default=7)
argparser.add_argument('--column_num', type=int, default=7)
argparser.add_argument('--ani_den_seed', type=int, default=66)

### Patroller
argparser.add_argument('--pa_state_size', type=int, default=20)
argparser.add_argument('--pa_num_actions', type=int, default=5)

### Poacher CNN
argparser.add_argument('--snare_num', type=int, default=6)
argparser.add_argument('--po_state_size', type=int, default=22) #yf: add self footprint to poacher
argparser.add_argument('--po_num_actions', type=int, default=10)

### Poacher Rule Base
argparser.add_argument('--po_act_den_w', type=float, default=3.)
argparser.add_argument('--po_act_enter_w', type=float, default=0.3)
argparser.add_argument('--po_act_leave_w', type=float, default=-1.0)
argparser.add_argument('--po_act_temp', type=float, default=5.0)
argparser.add_argument('--po_home_dir_w', type=float, default=3.0)

### Training
argparser.add_argument('--map_type', type = str, default = 'random')
argparser.add_argument('--save_path', type=str, default='./Results_77_random_collecting/')
argparser.add_argument('--load_path', type=str, default='./Results77Advanced_random_mode')
argparser.add_argument('--load_num', type=int, default=1)

argparser.add_argument('--naive', type = bool, default = False)
argparser.add_argument('--advanced_training', type = bool, default = True,
            help = 'whether using dueling double DQN with graident clipping')
argparser.add_argument('--po_location', type = int, default = None)
argparser.add_argument('--pa_episode_num', type=int, default=300000)
argparser.add_argument('--po_episode_num', type=int, default=300000)
argparser.add_argument('--epi_num_incr', type=int, default=0)
argparser.add_argument('--final_incr_iter', type = int, default = 10)
argparser.add_argument('--pa_replay_buffer_size', type=int, default=200000)
argparser.add_argument('--po_replay_buffer_size', type=int, default=100000)
argparser.add_argument('--test_episode_num', type=int, default=5000)
argparser.add_argument('--iter_num', type=int, default=2) #new added
argparser.add_argument('--pa_initial_lr', type=float, default=1e-4)
argparser.add_argument('--po_initial_lr', type=float, default=5e-5)

argparser.add_argument('--print_every', type=int, default=50)
argparser.add_argument('--zero_sum', type=int, default=1)
argparser.add_argument('--batch_size', type=int, default=32)
argparser.add_argument('--target_update_every', type=int, default=2000)
argparser.add_argument('--reward_gamma', type=float, default=0.95)
argparser.add_argument('--save_every_episode', type=int, default= 5000)
argparser.add_argument('--test_every_episode', type=int, default= 10000)
argparser.add_argument('--gui_every_episode', type=int, default=500)
argparser.add_argument('--gui_test_num', type = int, default = 20)
argparser.add_argument('--gui', type = int, default = 0)
argparser.add_argument('--Delta', type = float, default = 0)
argparser.add_argument('--mix_every_episode', type=int, default=250) #new added
argparser.add_argument('--epsilon_decrease', type=float, default=0.05) #new added
argparser.add_argument('--PER', type = bool, default = False)
argparser.add_argument('--reward_shaping', type = bool, default = False)
#########################################################################################
args = argparser.parse_args()

if args.row_num == 7:
    args.column_num = 7
    args.max_time = 75
    args.pa_initial_lr = 1e-4
    args.po_initial_lr = 5e-5
    args.pa_replay_buffer_size = 200000
    args.po_replay_buffer_size = 100000
    if args.po_location is not None:
        args.pa_episode_num = 200000
        args.po_episode_num = 200000

elif args.row_num == 5:
    args.column_num = 5
    args.max_time = 25
    args.pa_episode_num = 300000
    args.po_episode_num = 300000
    args.pa_initial_lr = 1e-4
    args.po_initial_lr = 5e-5
    args.pa_replay_buffer_size = 50000
    args.po_replay_buffer_size = 40000
    if args.po_location is not None:
        args.pa_episode_num = 200000

elif args.row_num == 3:
    args.column_num = 3
    args.max_time = 4
    args.snare_num = 3
    args.pa_episode_num = 100000
    args.po_episode_num = 100000
    args.pa_initial_lr = 1e-4
    args.po_initial_lr = 5e-5
    args.pa_replay_buffer_size = 10000
    args.po_replay_buffer_size = 8000
    if args.po_location is not None:
        args.pa_episode_num = 80000
        args.po_episode_num = 80000

if args.naive:
    args.Delta = 0.0
    args.po_location = None
else:
    args.Delta = 0.15

if args.save_path and (not os.path.exists(args.save_path)):
    os.makedirs(args.save_path)

paralog = open(args.save_path + 'paralog.txt', 'w')
paralog.write('row_num {0} \n'.format(args.row_num))
paralog.write('snare_num {0} \n'.format(args.snare_num))
paralog.write('max_time {0} \n'.format(args.max_time))
paralog.write('animal density seed {0} \n'.format(args.ani_den_seed))
paralog.write('pa_initial_episode_num {0} \n'.format(args.pa_episode_num))
paralog.write('po_initial_episode_num {0} \n'.format(args.po_episode_num))
paralog.write('epi_num_incr {0} \n'.format(args.epi_num_incr))
paralog.write('final_incr_iter {0} \n'.format(args.final_incr_iter))
paralog.write('pa_replay_buffer_size {0} \n'.format(args.pa_replay_buffer_size))
paralog.write('po_replay_buffer_size {0} \n'.format(args.po_replay_buffer_size))
paralog.write('pa_initial_lr {0} \n'.format(args.pa_initial_lr))
paralog.write('po_initial_lr {0} \n'.format(args.po_initial_lr))
paralog.write('test_episode_num {0} \n'.format(args.test_episode_num))
paralog.write('Delta {0} \n'.format(args.Delta))
paralog.write('po_location {0} \n'.format(str(args.po_location)))
paralog.write('map_type {0} \n'.format(str(args.map_type)))
paralog.write('naive {0} \n'.format(str(args.naive)))
paralog.flush()
paralog.close()

################## for initialization ###########################
global log_file

log_file = open(args.save_path + 'log.txt', 'w')

animal_density = generate_map(args)
env_pa = Env(args, animal_density, cell_length=None, canvas=None, gui=False)
env_po = Env(args, animal_density, cell_length=None, canvas=None, gui=False)

load_num = args.load_num
pa_number = 4 * load_num + 4 # 0: RS mode0; 1 -- 4*load_num: DQNs; 4*load_num + 1 -- 4 * load_num + 3: RS mode 1,2,3
po_number = 4 * load_num + 4 # 0: heuristic; 1 -- 4*load_num: DQNs; 4*load_num + 1 -- 4 * load_num + 3: heuristic 1,2,3
patrollers = [Patroller_CNN(args, 'pa_model' + str(i)) for i in range(load_num * 4 + 5 + args.iter_num)]
poachers = [Poacher(args, 'po_model' + str(i)) for i in range(load_num * 4 + 5 + args.iter_num)]
po_locations = [0]
for mode in range(4):
    for _ in range(load_num):
        po_locations.append(mode)
po_locations += [1,2,3]

### initialize poachers needed for training a separate best-response poacher DQN
br_poacher = Poacher(args, 'br_poacher')
br_target_poacher = Poacher(args, 'br_target_poacher')
br_good_poacher = Poacher(args, 'br_good_poacher')
br_utility = np.zeros(2)

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())

### copy ops needed for training a separate best-response poacher DQN
br_po_copy_ops = tf_copy(br_target_poacher, br_poacher, sess)
br_po_good_copy_ops = tf_copy(br_good_poacher, br_poacher, sess)

patrollers[0] = RandomSweepingPatroller(args, mode = 0)
poachers[0] = Poacher_h(args, animal_density)

### load pretrained models
if args.load_path is not None and args.load_num > 0:
    for i in range(1, 4):
        load_path = args.load_path + str(i) + '/'  ##### load_path needs to be checked
        for j in range(1, load_num + 1):
            patrollers[j].load(sess, load_path + 'iteration_{0}_pa_model.ckpt'.format(j))
            poachers[j].load(sess, load_path + 'iteration_{0}_po_model.ckpt'.format(j))
            pa_copy_op = tf_copy(patrollers[i * load_num + j], patrollers[j], sess)
            po_copy_op = tf_copy(poachers[i * load_num + j], poachers[j], sess)
            sess.run(pa_copy_op)
            sess.run(po_copy_op)
    load_path = args.load_path + '0/'
    for j in range(1, load_num + 1):
        patrollers[j].load(sess, load_path + 'iteration_{0}_pa_model.ckpt'.format(j))
        poachers[j].load(sess, load_path + 'iteration_{0}_po_model.ckpt'.format(j))

### add the rest mode patroller random sweeping models
for i in range(3):
    patrollers[1 + 4 * load_num + i] = RandomSweepingPatroller(args, i + 1)
    poachers[1 + 4 * load_num + i] = Poacher_h(args, animal_density)

print('----------------Load Models Succeed-----------')
print('----------------Load Models Succeed-----------')
print('----------------Load Models Succeed-----------')

### finding the NE among these strategies
pa_payoff = np.zeros((pa_number,po_number))
po_payoff = np.zeros((pa_number,po_number))
pa_type = ['RS']
po_type = ['PARAM']
pa_type += ['DQN' for i in range(4 * load_num)]
po_type += ['DQN' for j in range(4 * load_num)]
pa_type += ['RS' for i in range(3)]
po_type += ['PARAM' for i in range(3)]

for i in range(pa_number):
    for j in range(po_number):
        pa_payoff[i, j], po_payoff[i, j], _ = simulate_payoff(patrollers, poachers, i, j, env_pa, sess,
            args, pa_type = pa_type[i], po_type = po_type[j], po_location=po_locations[j])

pa_strategy, po_strategy = calc_NE_zero(pa_payoff, po_payoff, 0.0)
log_file = open(args.save_path + 'log.txt', 'w')
log_file.write('pa_payoff: ' + '\n' + str(pa_payoff) + '\n')
log_file.write('pa_strategy:' + str(pa_strategy) + '\n')
log_file.write('po_strategy:' + str(po_strategy) + '\n')

np.save(file = args.save_path + 'pa_strategy_iter_0', arr = pa_strategy)
np.save(file = args.save_path + 'po_strategy_iter_0', arr = po_strategy)

np.save(file = args.save_path + 'pa_payoff_iter_0', arr = pa_payoff)
np.save(file = args.save_path + 'po_payoff_iter_0', arr = po_payoff)

#################### keep training, in global modes ####################################
iteration = 1
pa_pointer, po_pointer = pa_number, po_number ### the pointer counting the number of strategies for pa and po.
args.po_location = None ### make sure its gloal mode training
length= np.zeros((pa_number,po_number))

while(1):
    time_begin = time.time()

    pa_payoff, po_payoff, length = extend_payoff(pa_payoff, po_payoff, length,
            pa_pointer + 1, po_pointer + 1)
    po_type.append('DQN')
    pa_type.append('DQN')

    log_file.flush()

    print('\n' +  'NEW_ITERATION: ' + str(iteration) + '\n')
    log_file.write('\n' + 'NEW_ITERATION: ' + str(iteration) + '\n')

    ### compute the NE utility for both sides
    po_ne_utility = 0
    pa_ne_utility = 0
    for pa_strat in range(pa_pointer):
        for po_strat in range(po_pointer):
            po_ne_utility += pa_strategy[pa_strat] * po_strategy[po_strat] * po_payoff[pa_strat, po_strat]
            pa_ne_utility += pa_strategy[pa_strat] * po_strategy[po_strat] * pa_payoff[pa_strat, po_strat]

    log_file.write('last_pa_ne_utility:' + str(pa_ne_utility) + '\n')
    log_file.write('last_po_ne_utility:' + str(po_ne_utility) + '\n')
    pre_pa_strategy = pa_strategy
    pre_po_strategy = po_strategy

    ### compute the best response poacher utility
    ### 1. train a best response poacher DQN against the current pa strategy
    # calc_po_best_response(br_poacher, br_target_poacher, br_po_copy_ops, br_po_good_copy_ops, patrollers,
    #         pa_strategy, pa_type, iteration, sess, env_pa, args, br_utility, 0, train_episode_num=args.br_po_DQN_episode_num)
    # br_DQN_utility = br_utility[1]

    ### 2. test against the heuristic poacher
    # br_heuristic_utility = 0.
    # for i in range(pa_pointer):
    #     _, po_utility, _ = simulate_payoff(patrollers, poachers, i, 0, env_pa, sess, args,
    #         pa_type=pa_type[i], po_type = po_type[0])
    #     br_heuristic_utility += po_utility * pa_strategy[i]

    ### choose the better one
    # better = 'DQN' if br_DQN_utility >= br_heuristic_utility else 'heuristic'
    # br_poacher_utility = max(br_DQN_utility, br_heuristic_utility)
    # log_file.write('Iteration {0} poacher best response utility {1} poacher best response type {2} \n'.format(
    #         iteration, br_poacher_utility, better))
    # print('Iteration {0} poacher best response utility {1} poacher best response type {2}'.format(
    #         iteration, br_poacher_utility, better))

    ###train the best response agent
    ### using threading to accelerate the training
    good_patrollers = []
    good_poachers = []
    final_utility = [0.0, 0.0]
    target_patroller = Patroller_CNN(args, 'target_patroller' + str(iteration))
    good_patroller = Patroller_CNN(args, 'good_patroller' + str(iteration))
    pa_copy_ops = tf_copy(target_patroller, patrollers[pa_pointer], sess)
    pa_good_copy_ops = tf_copy(good_patroller, patrollers[pa_pointer], sess)
    pa_inverse_ops = tf_copy(patrollers[pa_pointer], good_patroller, sess)

    target_poacher = Poacher(args, 'target_poacher' + str(iteration))
    good_poacher = Poacher(args, 'good_poacher' + str(iteration))
    po_copy_ops = tf_copy(target_poacher, poachers[po_pointer], sess)
    po_good_copy_ops = tf_copy(good_poacher, poachers[po_pointer], sess)
    po_inverse_ops = tf_copy(poachers[po_pointer], good_poacher, sess)

    funcs = [calc_pa_best_response, calc_po_best_response]
    ### computing the patroller best response, needs to pass the poacher DQN local modes
    params = [[patrollers[pa_pointer], target_patroller, pa_copy_ops, pa_good_copy_ops, poachers,
                po_strategy, po_type, iteration, sess, env_pa, args, final_utility,0, args.pa_episode_num, po_locations],
              [poachers[po_pointer], target_poacher, po_copy_ops, po_good_copy_ops, patrollers,
                pa_strategy, pa_type, iteration, sess, env_po, args, final_utility,0]]

    ### if the maximum iteration number is achieved
    if args.iter_num == iteration:
        log_file.write('\n DO reaches terminating iteration {0}'.format(iteration) + '\n')
        log_file.write('Final Pa-payoff: \n' + str(pa_payoff) + '\n')
        log_file.write('Final Po-payoff: \n'+ str(po_payoff) + '\n')
        log_file.write('Final pa_strat:\n' + str(pa_strategy) + '\n')
        log_file.write('Final po_strat:\n'+ str(po_strategy) + '\n')
        log_file.write('Final pa_ne_utility:' + str(pa_ne_utility) + '\n')
        log_file.write('Final po_ne_utility:' + str(po_ne_utility) + '\n')
        log_file.flush()

        threads = []
        for i in range(2):
            process = Thread(target=funcs[i], args=params[i])
            process.start()
            threads.append(process)
        ### We now pause execution on the main thread by 'joining' all of our started threads.
        for process in threads:
            process.join()

        pa_exploit = final_utility[0] - pa_ne_utility
        po_exploit = final_utility[1] - po_ne_utility
        log_file.write('Final pa_best_response_utility:' + str(final_utility[0]) + '\n')
        log_file.write('Final po_best_response_utility:' + str(final_utility[1]) + '\n')
        log_file.write('Final pa exploitibility:' + str(pa_exploit) + '\n')
        log_file.write('Final po exploitibility:' + str(po_exploit) + '\n')
        break

    ### not the final iteration
    threads = []

    for i in range(2):
        process = Thread(target=funcs[i], args=params[i])
        process.start()
        threads.append(process)
    for process in threads:
        process.join()

    # calc_pa_best_response(patrollers[pa_pointer], target_patroller, pa_copy_ops, pa_good_copy_ops, poachers,
    #         po_strategy, iteration, sess, env_pa, args, final_utility,0)

    sess.run(pa_inverse_ops)
    sess.run(po_inverse_ops)

    ### add a new poacher DQN, global mode
    po_locations.append(None)
    params[0][-1] = po_locations

    for pa_strat in range(pa_pointer):
        pa_payoff[pa_strat, po_pointer ],po_payoff[pa_strat, po_pointer], _  = \
            simulate_payoff(patrollers, poachers, pa_strat, po_pointer, env_pa, sess, args,
                pa_type=pa_type[pa_strat], po_type=po_type[po_pointer]) # new poacher DQN must be global

    for po_strat in range(po_pointer):
        pa_payoff[pa_pointer, po_strat],po_payoff[pa_pointer, po_strat],_  = \
            simulate_payoff(patrollers, poachers, pa_pointer, po_strat, env_pa, sess, args,
            pa_type=pa_type[pa_pointer], po_type = po_type[po_strat], po_location=po_locations[po_strat])
            ### past poacher DQN could be local

    pa_payoff[pa_pointer, po_pointer],po_payoff[pa_pointer, po_pointer],_  = \
        simulate_payoff(patrollers, poachers, pa_pointer, po_pointer, env_pa, sess, args,
        pa_type=pa_type[pa_pointer], po_type = po_type[po_pointer]) ### new poacher DQN must be global

    pa_strategy, po_strategy = calc_NE_zero(pa_payoff, po_payoff, args.Delta)
    # pa_strategy, po_strategy = calc_NE(pa_payoff, po_payoff)
    # pa_strategy, po_strategy = np.ones(iteration + 1) / (iteration + 1), np.ones(iteration + 1) / (iteration + 1)

    params[0][5] = po_strategy
    params[1][5] = pa_strategy

    po_best_response = final_utility[1]
    pa_best_response = final_utility[0]
    # for pa_strat in range(pa_pointer):
    #     po_best_response += pre_pa_strategy[pa_strat] * po_payoff[pa_strat, po_pointer]
    # for po_strat in range(po_pointer):
    #     pa_best_response += pre_po_strategy[po_strat] * pa_payoff[pa_pointer, po_strat]

    # eps_po.append(po_best_response - po_ne_utility)
    # eps_pa.append(pa_best_response - pa_ne_utility)

    log_file.write('In DO pa_best_utility:' + str(pa_best_response) + '\n')
    log_file.write('In DO po_best_utility:' + str(po_best_response) + '\n')
    # log_file.write('eps_pa: ' + str(eps_pa) + '\n')
    # log_file.write('eps_po: ' + str(eps_po) + '\n')

    ######### save models for this iteration #############
    save_name = args.save_path + 'iteration_' + str(iteration) + '_pa_model.ckpt'
    patrollers[pa_pointer].save(sess =sess, filename = save_name)
    save_name = args.save_path + 'iteration_' + str(iteration) + '_po_model.ckpt'
    poachers[po_pointer].save(sess =sess, filename = save_name)

    ### save payoff matrix and ne strategies
    np.save(file = args.save_path + 'pa_payoff_iter_' + str(iteration), arr = pa_payoff)
    np.save(file = args.save_path + 'po_payoff_iter_' + str(iteration), arr = po_payoff)
    np.save(file = args.save_path + 'pa_strategy_iter_' + str(iteration), arr = pa_strategy)
    np.save(file = args.save_path + 'po_strategy_iter_' + str(iteration), arr = po_strategy)

    log_file.write('pa_payoff:\n' + str(pa_payoff) + '\n')
    log_file.write('po_payoff:\n' + str(po_payoff) + '\n')
    log_file.write('pa_strategy:\n' + str(pa_strategy) + '\n')
    log_file.write('po_strategy:\n' + str(po_strategy) + '\n')

    iteration += 1
    pa_pointer += 1
    po_pointer += 1

    time_end = time.time()

    log_file.write('Using time: \n' + str(time_end - time_begin) + '\n')
    log_file.flush()

log_file.close()

max_stars_repo_name: nimisis/DeDOL

max_issues_repo_name: nimisis/DeDOL

max_forks_repo_name: nimisis/DeDOL


text:
import os
from glob import glob

import numpy as np
import argparse

def parse_args():
    parser = argparse.ArgumentParser(description='Test ')
    parser.add_argument('--data-dir', default='/mnt/hdd02/shibuya_scramble',
                        help='original data directory')
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = parse_args()

    ## Random Train-Val split

    im_list = sorted(glob(os.path.join(args.data_dir, '*.jpg')))
    im_list = [im_name for im_name in im_list]

    tr_im_list = list(np.random.choice(im_list, size=int(len(im_list)*0.8), replace=False))
    vl_im_list = list(set(im_list) - set(tr_im_list))

    for phase in ['train', 'val']:
        with open(os.path.join(args.data_dir, './{}.txt'.format(phase)), mode='w') as f:
            if phase == 'train':
                f.write('\n'.join(tr_im_list))
            elif phase == 'val':
                f.write('\n'.join(vl_im_list))

ext: py

max_stars_repo_name: junprog/contrastive-baseline

max_issues_repo_name: junprog/contrastive-baseline

max_forks_repo_name: junprog/contrastive-baseline

reason: import numpy


text:
import os
import sys
import time
import glob
import numpy as np
import torch
import utils
import logging
import argparse
import torch.nn as nn
import torch.utils
import torch.nn.functional as F
import torchvision.datasets as dset
import torch.backends.cudnn as cudnn

from torch.autograd import Variable
from model_search import Network, NetworkGalaxyZoo
from architect import Architect
from datasets import load_dataset, DSET_NAME_TBL
from sklearn.metrics import r2_score

parser = argparse.ArgumentParser("darts")

# NON-PERFORMANCE RELATED
parser.add_argument('--dataset', type=str, default='cifar', help='name of the dataset to use (e.g. cifar, mnist, graphene)')
parser.add_argument('--data', type=str, default='../data', help='location of the data corpus')
parser.add_argument('--folder_name', type=str, default=None, help='Name of top-level folder containing the data, e.g. galaxy_zoo')
parser.add_argument('--use_xarray', action='store_true', default=True, help='use xarray package for data loading')
parser.add_argument('--report_freq', type=float, default=50, help='report frequency')
parser.add_argument('--gpu', type=int, default=0, help='gpu device id')
parser.add_argument('--model_path', type=str, default='saved_models', help='path to save the model')
parser.add_argument('--save', type=str, default='EXP', help='experiment name')
parser.add_argument('--seed', type=int, default=2, help='random seed')

# PERFORMANCE-RELATED
### Highly important
parser.add_argument('--init_channels', type=int, default=16, help='num of init channels')
parser.add_argument('--layers', type=int, default=8, help='total number of layers')
parser.add_argument('--learning_rate', type=float, default=1e-3, help='init learning rate')
parser.add_argument('--learning_rate_min', type=float, default=1e-4, help='min learning rate')
parser.add_argument('--weight_decay', type=float, default=1e-6, help='weight decay')
parser.add_argument('--L1_lambda', type=float, default=None, help='multiplier on the architecture weight L1 penalty')
parser.add_argument('--arch_learning_rate', type=float, default=1e-2, help='learning rate for arch encoding')
parser.add_argument('--arch_weight_decay', type=float, default=1e-6, help='weight decay for arch encoding')
parser.add_argument('--cell_steps', type=int, default=4, help='number of steps in a cell (see model_search.Network)')
parser.add_argument('--cell_multiplier', type=int, default=4, help='multiplier for a cell (see model_search.Network)')
### Somewhat important
parser.add_argument('--epochs', type=int, default=50, help='num of training epochs')
parser.add_argument('--batch_size', type=int, default=16, help='batch size')
parser.add_argument('--optimizer', type=str, default='Adam', help='optimizer; one of SGD or Adam')
parser.add_argument('--momentum', type=float, default=0.9, help='momentum')
parser.add_argument('--gz_dtree', action='store_true', default=False,
                    help='run GalaxyZoo with decision tree structure (default False: use simple regression)')
parser.add_argument('--fc1_size', type=int, default=1024, help='for gz_dtree: number of units in fully connected layer 1')
parser.add_argument('--fc2_size', type=int, default=1024, help='for gz_dtree: number of units in fully connected layer 2')
parser.add_argument('--primitives', type=str, default='Default',
                    help='set of primitive operations for arch search; defined in genotypes.py')
### Not very important
parser.add_argument('--train_portion', type=float, default=0.5, help='portion of training data')
parser.add_argument('--grad_clip', type=float, default=5, help='gradient clipping')
parser.add_argument('--unrolled', action='store_true', default=True, help='use one-step unrolled validation loss')
parser.add_argument('--cutout', action='store_true', default=False, help='use cutout')
parser.add_argument('--cutout_length', type=int, default=16, help='cutout length')
# the below arg is not used anywhere in this script, nor is it used in any of the loaded modules
# parser.add_argument('--drop_path_prob', type=float, default=0.3, help='drop path probability')
args = parser.parse_args()

args.save = 'search-{}-{}'.format(args.save, time.strftime("%Y%m%d-%H%M%S"))
utils.create_exp_dir(args.save, scripts_to_save=glob.glob('*.py'))

log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
    format=log_format, datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(args.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)

# Get normalized dataset name
dataset = DSET_NAME_TBL[args.dataset.lower().strip()]

# If the default set of primitives is requested, use the normalized name of the dataset
primitives_name = dataset if args.primitives == 'Default' else args.primitives

def main():
  if not torch.cuda.is_available():
    logging.info('no gpu device available')
    sys.exit(1)

  np.random.seed(args.seed)
  torch.cuda.set_device(args.gpu)
  cudnn.benchmark = True
  torch.manual_seed(args.seed)
  cudnn.enabled=True
  torch.cuda.manual_seed(args.seed)
  logging.info('gpu device = %d' % args.gpu)
  logging.info("args = %s", args)

  train_data, OUTPUT_DIM, IN_CHANNELS, inference_type = load_dataset(args, train=True)

  num_train = len(train_data)
  indices = list(range(num_train))
  split = int(np.floor(args.train_portion * num_train))

  train_queue = torch.utils.data.DataLoader(
      train_data, batch_size=args.batch_size,
      sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
      pin_memory=True, num_workers=2)

  valid_queue = torch.utils.data.DataLoader(
      train_data, batch_size=args.batch_size,
      sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
      pin_memory=True, num_workers=2)

  criterion = utils.loss_criterion(inferency_type)

  # Special network for Galaxy Zoo regression
  if dataset == 'GalaxyZoo' and args.gz_dtree:
      model = NetworkGalaxyZoo(C=args.init_channels, num_classes=OUTPUT_DIM, primitives_name=primitives_name,
                             layers=args.layers, criterion=criterion,
                             fc1_size=args.fc1_size, fc2_size=args.fc2_size,
                             num_channels=IN_CHANNELS, steps=args.cell_steps,
                             multiplier=args.cell_multiplier)
  else:
      model = Network(C=args.init_channels, num_classes=OUTPUT_DIM, primitives_name=primitives_name,
                      layers=args.layers, criterion=criterion, num_channels=IN_CHANNELS,
                      steps=args.cell_steps, multiplier=args.cell_multiplier)

  model = model.cuda()
  logging.info("param size = %fMB", utils.count_parameters_in_MB(model))

  # build optimizer based on optimizer input; one of SGD or Adam
  if args.optimizer == 'SGD':
    optimizer = torch.optim.SGD(
      model.parameters(),
      args.learning_rate,
      momentum=args.momentum,
      weight_decay=args.weight_decay)
  elif args.optimizer == 'Adam':
    optimizer= torch.optim.Adam(
      params=model.parameters(),
      lr=args.learning_rate,
      betas=(0.90, 0.999),
      weight_decay=args.weight_decay)
  else:
    raise ValueError(f"Bad optimizer; got {args.optimizer}, must be one of 'SGD' or 'Adam'.")

  scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
        optimizer, float(args.epochs), eta_min=args.learning_rate_min)

  architect = Architect(model, args, L1_lambda=args.L1_lambda)

  # history of training and validation loss; 2 columns for loss and accuracy / R2
  hist_trn = np.zeros((args.epochs, 2))
  hist_val = np.zeros((args.epochs, 2))
  metric_name = 'accuracy' if inference_type == 'classification' else 'R2'

  for epoch in range(args.epochs):
    scheduler.step()
    lr = scheduler.get_lr()[0]
    logging.info('epoch %d lr %e', epoch, lr)

    genotype = model.genotype()
    logging.info('genotype = %s', genotype)

    # saving & logging weights for architecture
    normal_weights = F.softmax(model.alphas_normal, dim=-1)
    reduce_weights = F.softmax(model.alphas_reduce, dim=-1)
    logging.info('\nalphas_normal = {}'.format(normal_weights))
    logging.info('\nalphas_reduce = {}'.format(reduce_weights))
    np.save(os.path.join(args.save, 'normal_%03d' % epoch), normal_weights.data.cpu().numpy())
    np.save(os.path.join(args.save, 'reduce_%03d' % epoch), reduce_weights.data.cpu().numpy())

    # training
    train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, inference_type=inference_type)
    logging.info(f'training loss; {metric_name}: {train_obj:e} {train_acc:f}')
    # save history to numpy arrays
    hist_trn[epoch] = [train_acc, train_obj]
    np.save(os.path.join(args.save, 'hist_trn'), hist_trn)

    # validation
    valid_acc, valid_obj = infer(valid_queue, model, criterion, inference_type=inference_type)
    logging.info(f'validation loss; {metric_name}: {valid_obj:e} {valid_acc:f}')
    # save history to numpy arrays
    hist_val[epoch] = [valid_acc, valid_obj]
    np.save(os.path.join(args.save, 'hist_val'), hist_val)

    # save weights
    utils.save(model, os.path.join(args.save, 'weights.pt'))

    # save loss history

def train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, inference_type='classification'):
  objs = utils.AverageMeter()
  top1 = utils.AverageMeter()
  top5 = utils.AverageMeter()

  for step, (input, target) in enumerate(train_queue):
    model.train()
    n = input.size(0)

    input = Variable(input, requires_grad=False).cuda()
    target = Variable(target, requires_grad=False).cuda(async=True)

    # get a random minibatch from the search queue with replacement
    input_search, target_search = next(iter(valid_queue))
    input_search = Variable(input_search, requires_grad=False).cuda()
    target_search = Variable(target_search, requires_grad=False).cuda(async=True)

    architect.step(input, target, input_search, target_search, lr, optimizer, unrolled=args.unrolled)

    optimizer.zero_grad()
    logits = model(input)
    loss = criterion(logits, target)

    loss.backward()
    nn.utils.clip_grad_norm(model.parameters(), args.grad_clip)
    optimizer.step()

    if inference_type == 'classification':
        prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5))
        objs.update(loss.data[0], n)
        top1.update(prec1.data[0], n)
        top5.update(prec5.data[0], n)

        if step % args.report_freq == 0:
          logging.info('train %03d %e %f %f', step, objs.avg, top1.avg, top5.avg)
    elif inference_type in ['regression', 'multi_binary']:
        r2 = r2_score(target.data.cpu().numpy(), logits.data.cpu().numpy())
        objs.update(loss.data[0], n)
        top1.update(r2, n) # "top1" for regression is the R^2

        if step % args.report_freq == 0:
          logging.info('train %03d %e %f', step, objs.avg, top1.avg)
    else:
        ValueError

  return top1.avg, objs.avg

def infer(valid_queue, model, criterion, inference_type='classification'):
  objs = utils.AverageMeter()
  top1 = utils.AverageMeter()
  top5 = utils.AverageMeter()
  model.eval()

  for step, (input, target) in enumerate(valid_queue):
    input = Variable(input, volatile=True).cuda()
    target = Variable(target, volatile=True).cuda(async=True)

    n = input.size(0)

    logits = model(input)
    loss = criterion(logits, target)

    if inference_type == 'classification':
        prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5))
        objs.update(loss.data[0], n)
        top1.update(prec1.data[0], n)
        top5.update(prec5.data[0], n)

        if step % args.report_freq == 0:
          logging.info('valid %03d %e %f %f', step, objs.avg, top1.avg, top5.avg)
    elif inference_type in ['regression', 'multi_binary']:
        # use the same metric for both cases for now
        r2 = r2_score(target.data.cpu().numpy(), logits.data.cpu().numpy())
        objs.update(loss.data[0], n)
        top1.update(r2, n) # "top1" for regression is the R^2

        if step % args.report_freq == 0:
          logging.info('valid %03d %e %f', step, objs.avg, top1.avg)
    else:
        ValueError

  return top1.avg, objs.avg

if __name__ == '__main__':
  main()


text:
""" Package Imports  """
import numpy as np

""" Python File Imports"""
from src.pngs import PNGS
from src.panel import Panel
from src.blosum import BLOSUM
from src.weights import Weights
from src.epitope_dist import get_epitope_distance
from src.ic50 import IC50

""" Relative Python Paths """
rel_panel_path = './files/seap2020/136_panel_with_4lts.fa'
rel_weight_path = './files/seap2020/vrc01_wts.4lts.txt'
rel_blosum_path = './files/seap2020/BLOSUM62.txt'
rel_ic50_path = './files/seap2020/vrc01_ic50.txt'

""" Instantiating Each Class """
panel = Panel(rel_panel_path)
blosum = BLOSUM(rel_blosum_path)
weights = Weights(rel_weight_path)
weight_array_modified = np.zeros(panel.get_seq_length())
ic50 = IC50(rel_ic50_path, (panel.get_number_of_seq() - 2))

class Main:

    def get_consensus_sequence():
        return panel.get_consensus_sequence()

    def get_blosum_dict():
        return blosum.get_blosum_dict()

    def get_ic50_weights():
        return ic50.get_ic50_arr()

""" Turn weights -> weight_array that is usable for epitope distance """
def init_weight_array_modified():
    VRC_seq = panel.get_seq_from_name('#4lst_G(VRC01)')
    AA_order = weights.get_aa_at_an_order()
    counter = 0

    for i in range(0, panel.get_seq_length() - 1):
        # print('i', i)
        if (VRC_seq[i] != '-'):
            weight_array_modified[i] = weights.get_weight_by_order(counter)
            # print('counter', counter)
            # print('VRC_seq[i]', VRC_seq[i], 'AA_order[counter]', AA_order[counter])
            counter += 1

def epitope_distance(consensus_sequence, blosum_dict, ic50_weights):
    init_weight_array_modified()
    panel_ic50 = np.empty((2, (panel.get_number_of_seq() - 2)))
    for i in range(1, panel.get_number_of_seq() - 1):
        panel_ic50[0][i - 1] = get_epitope_distance(panel.get_seq(i), consensus_sequence, blosum_dict, weight_array_modified)
        panel_ic50[1][i - 1] = ic50_weights[i - 1]
        # print(panel_ic50[0][i - 1], panel_ic50[1][i - 1])

    return panel_ic50

""" Testing out the Functions """

"""
print(weight_array_modified)

print('Number of PNGS: ' + str(PNGS('PPNNSNNSNPSN')))

print('Consensus sequence:', panel.get_consensus_sequence())
print('Test:', panel.get_seq('HXB2.DG'))

print('blosum.get_value("A", "R")', blosum.get_value('A', 'R'))
print('Panda BLOSUM Matrix')
print(blosum.get_panda_matrix())

print('4lst_G(VRC01) Sequence', panel.get_seq('#4lst_G(VRC01)'))

print('weights.get_weight_by_site("G.44")', weights.get_weight_by_site('G.44'))
print('weights.get_weight_by_order(53)', weights.get_weight_by_order(53))
"""

max_stars_repo_path: Week 3/main.py

max_issues_repo_path: Week 3/main.py

max_forks_repo_path: Week 3/main.py


text:
import pytest
import numpy as np
import math

from pressio4py.apps.advection_diffusion1d import AdvDiff1d

def test1():
  fomObj = AdvDiff1d(nGrid=4, adv_coef=2.0)
  x = fomObj.xGrid
  print(x)
  assert(math.isclose(x[0], 0.))
  assert(math.isclose(fomObj.adv_coef, 2.))

# todo: add more


text:
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2016 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Trains an LSTM from sentences in the vectorized corpus.
"""

import argparse

import numpy as np
from path import Path

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop

from vocabulary import vocabulary
from training_utils import LoopBatchesEndlessly, Sentences

# Based on White et al. 2015
SENTENCE_LENGTH = 20
SIGMOID_ACTIVATIONS = 300

# This is arbitrary, but it should be fairly small.
BATCH_SIZE = 512
FOLD = 0

if __name__ == '__main__':
    filename = Path('/dev/shm/tiny-corpus.sqlite3')

    # Get the tokens from the 9 training folds.
    training_batches = LoopBatchesEndlessly\
        .for_training(filename, FOLD,
                      batch_size=BATCH_SIZE,
                      sentence_length=SENTENCE_LENGTH,
                      backwards=True)
    eval_batches = LoopBatchesEndlessly\
        .for_evaluation(filename, FOLD,
                      batch_size=BATCH_SIZE,
                      sentence_length=SENTENCE_LENGTH,
                      backwards=True)
    print("Will train on", training_batches.samples_per_epoch, "samples")

    # Defining the model:
    model = Sequential()
    model.add(LSTM(SIGMOID_ACTIVATIONS,
                   input_shape=(SENTENCE_LENGTH, len(vocabulary))))
    model.add(Dense(len(vocabulary)))
    model.add(Activation('softmax'))

    print("Compiling the model...")
    model.compile(loss='categorical_crossentropy',
                  optimizer=RMSprop(lr=0.001),
                  metrics=['categorical_accuracy'])
    print("Done")

    #model.load_weights("javascript.2.h5")

    print("Training for one epoch...")
    model.fit_generator(iter(training_batches),
                        samples_per_epoch=training_batches.samples_per_epoch,
                        validation_data=iter(eval_batches),
                        nb_val_samples=eval_batches.samples_per_epoch // BATCH_SIZE,
                        verbose=1,
                        pickle_safe=True,
                        nb_epoch=2)

    print("Saving model.")
    model.save_weights('javascript-backwards.2.h5')

max_stars_repo_name: onethirdzero/training-grammar-guru

max_issues_repo_name: onethirdzero/training-grammar-guru

max_forks_repo_name: onethirdzero/training-grammar-guru


text:
from __future__ import annotations

import numpy as np

from bqskit.ir.circuit import Circuit
from bqskit.ir.opt.cost import HilbertSchmidtCostGenerator

def test_hilbert_schmidt_cost(r3_qubit_circuit: Circuit) -> None:
    x0 = np.random.random((r3_qubit_circuit.num_params,))
    cost = HilbertSchmidtCostGenerator().gen_cost(
        r3_qubit_circuit, r3_qubit_circuit.get_unitary(x0),
    )
    assert cost.get_cost(x0) < 1e-10


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 18 13:13:26 2022

@author: annabramslow
"""
import numpy as np
import pandas as pd
import os
import geopandas as gpd
from shapely.geometry import Point, LineString, Polygon
from shapely.ops import transform, nearest_points
from geopy import distance
from scipy.integrate import simpson

from road_features import count_traffic_lights, count_road_signs, oneway, get_speed_limit, count_pedestrian_crossings, count_speed_bumps, bike_facilities

# pd.set_option('mode.chained_assignment', None)

# user_path = os.path.split(os.getcwd())[0][:-3]
# filename = '13743_0_'
# sensor_filepath = os.path.join(user_path, 'data','interpolated',filename+'sensor.pickle')
# gps_filepath = os.path.join(user_path, 'data','interpolated_gps',filename+'gps.pickle')

# sensor_data = pd.read_pickle(sensor_filepath)
# gps = pd.read_pickle(gps_filepath)

def process_trip(sensor_data, gps, task_id):
    """ Assuming mapmatched sensor data """

    sensor_data['kW'] = sensor_data['obd.trac_cons.value'] - 160
    sensor_data['W'] = sensor_data['kW']*1000
    sensor_data['s'] = sensor_data['Time'].apply(lambda t: t.hour*60*60 + t.minute*60 + t.second + t.microsecond * 10**(-6))

    #Add unique id for each passed road (give same id if street is same as prior)
    # sensor_data["street_id"] = sensor_data.apply(lambda x: )
    # gps["street_id"] = gps.apply(lambda x:)

    df_final = pd.DataFrame()
    # run through all segments
    for street in gps.street_name.unique():
        print(f"Processing street {street} in trip {task_id}")

        road_features = {}
        road_features['task_id'] = task_id
        road_features['street_name'] = street

        #go through all road feature
        df_street = gps.loc[gps.street_name == street]
        road_features['traffic_lights'] = count_traffic_lights(df_street)
        road_features['road_signs'] = count_road_signs(df_street)
        road_features['one_way'] = oneway(df_street)
        road_features['speed_limit'] = get_speed_limit(df_street)
        road_features['ped_walks'] = count_pedestrian_crossings(df_street)
        road_features['speed_bumps'] = count_speed_bumps(df_street)
        road_features['bike_lane'] = min(len(bike_facilities(df_street)),1)
        road_features['start_hour'] = df_street['TS_or_Distance'].min().hour

        # compute energy
        df_energy = sensor_data[sensor_data['T']=='obd.trac_cons']
        df_energy = df_energy.loc[df_energy.street_name_start == street]
        df_energy = df_energy.drop_duplicates(subset='s')
        ws = simpson(df_energy.W, df_energy.s)
        road_features['kWh'] = ws / 3600000

        # compute distance and duration
        df_dist = sensor_data[sensor_data['T']=='obd.odo']
        df_dist = df_dist.loc[df_dist.street_name_start == street]
        road_features['distance'] = df_dist['obd.odo.value'].max()-df_dist['obd.odo.value'].min()
        road_features['duration'] = df_dist['s'].max()-df_dist['s'].min()

        # compute average speed
        df_speed = sensor_data[sensor_data['T']=='obd.spd']
        df_speed = df_speed.loc[df_speed.street_name_start == street]
        road_features['avg_speed'] = df_speed['obd.spd.value'].mean()

        # compute window wiper activity
        df_wiper = sensor_data[sensor_data['T']=='obd.ww_f_stat']
        df_wiper = df_wiper.loc[df_wiper.street_name_start == street]
        road_features['ww_active_pct'] = df_wiper['obd.ww_f_stat.value'].sum()/len(df_wiper)

        # compute longitudinal acceleration
        acc = sensor_data.loc[(sensor_data['T'] == 'acc.xyz') & (sensor_data["street_name_start"] == street) ]['acc.xyz.x'].dropna()
        road_features['avg_acc'] = acc.mean()
        road_features['neg_acc_percent'] = np.sum(acc.lt(0)) / len(acc)
        road_features['pos_acc_percent'] = np.sum(acc.gt(0)) / len(acc)
        road_features['min_acc'] = acc.min()
        road_features['max_acc'] = acc.max()

        # Append extracted features to df
        df_final = pd.concat((df_final, pd.DataFrame(road_features, index=[0])))

    return df_final

reason: import numpy,from scipy


text:
# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/master/LICENSE

from __future__ import absolute_import

import sys

import pytest
import numpy

import awkward1

@pytest.mark.skipif(awkward1._util.py27 and awkward1._util.win, reason="Windows Python 2.7 inserts 'L' after numbers in repr")
def test():
    assert str(awkward1.Array([
        {"a": 1, "a b": 2}, {"a": 1, "a b": 2}, {"a": 1, "a b": 2}, {"a": 1, "a b": 2}
    ])) in (
        "[{a: 1, 'a b': 2}, {a: 1, 'a b': 2}, {a: 1, 'a b': 2}, {a: 1, 'a b': 2}]",
        "[{'a b': 2, a: 1}, {'a b': 2, a: 1}, {'a b': 2, a: 1}, {'a b': 2, a: 1}]"
    )
    assert repr(awkward1.Array([
        {"a": 1, "a b": 2}, {"a": 1, "a b": 2}, {"a": 1, "a b": 2}
    ])) in (
        "<Array [{a: 1, 'a b': 2}, ... {a: 1, 'a b': 2}] type='3 * {\"a\": int64, \"a b\": in...'>"
        "<Array [{'a b': 2, a: 1}, ... {'a b': 2, a: 1}] type='3 * {\"a b\": int64, \"a\": in...'>"
    )


text:
"""
Random surfaces.

The code in this module was substantially adapted from Pierre Vigier's code at
https://github.com/pvigier/perlin-numpy

Author: Pierre Vigier
Licence: MIT

Copyright 2022 Pierre Vigier

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import numpy as np
import xarray as xr

def interpolant(t):
    """
    By Pierre Vigier https://github.com/pvigier/perlin-numpy
    Licence: MIT
    """
    return t*t*t*(t*(t*6 - 15) + 10)

def perlin_noise(shape, res,
                 rng=None,
                 tileable=(False, False),
                 interpolant=interpolant):
    """
    Generate a 2D numpy array of perlin noise.

    By Pierre Vigier https://github.com/pvigier/perlin-numpy
    Licence: MIT

    Args:
        shape: The shape of the generated array (tuple of two ints).
            This must be a multple of res.
        res: The number of periods of noise to generate along each
            axis (tuple of two ints). Note shape must be a multiple of
            res.
        tileable: If the noise should be tileable along each axis
            (tuple of two bools). Defaults to (False, False).
        interpolant: The interpolation function, defaults to
            t*t*t*(t*(t*6 - 15) + 10).

    Returns:
        A numpy array of shape shape with the generated noise.

    Raises:
        ValueError: If shape is not a multiple of res.
    """
    delta = (res[0] / shape[0], res[1] / shape[1])
    d = (shape[0] // res[0], shape[1] // res[1])
    grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1]].transpose(1, 2, 0) % 1

    # Gradients.
    angles = 2 * np.pi * rng.random(size=(res[0]+1, res[1]+1))
    gradients = np.dstack((np.cos(angles), np.sin(angles)))
    if tileable[0]:
        gradients[-1, :] = gradients[0, :]
    if tileable[1]:
        gradients[:,-1] = gradients[:,0]
    gradients = gradients.repeat(d[0], 0).repeat(d[1], 1)
    g00 = gradients[    :-d[0],    :-d[1]]
    g10 = gradients[d[0]:     ,    :-d[1]]
    g01 = gradients[    :-d[0],d[1]:     ]
    g11 = gradients[d[0]:     ,d[1]:     ]

    # Ramps.
    n00 = np.sum(np.dstack((grid[:,:,0]  , grid[:,:,1]  )) * g00, 2)
    n10 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1]  )) * g10, 2)
    n01 = np.sum(np.dstack((grid[:,:,0]  , grid[:,:,1]-1)) * g01, 2)
    n11 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1]-1)) * g11, 2)

    # Interpolation.
    t = interpolant(grid)
    n0 = n00*(1-t[:,:,0]) + t[:,:,0]*n10
    n1 = n01*(1-t[:,:,0]) + t[:,:,0]*n11
    return np.sqrt(2)*((1-t[:,:,1])*n0 + t[:,:,1]*n1)

def fractal_noise(shape,
                  res,
                  rng=None,
                  octaves=1,
                  persistence=0.5,
                  lacunarity=2,
                  tileable=(False, False),
                  interpolant=interpolant,
                  ):
    """
    Generate a 2D numpy array of fractal noise.

    By Pierre Vigier https://github.com/pvigier/perlin-numpy
    Licence: MIT

    Args:
        shape: The shape of the generated array (tuple of two ints).
            This must be a multiple of lacunarity**(octaves-1)*res.
        res: The number of periods of noise to generate along each
            axis (tuple of two ints). Note shape must be a multiple of
            (lacunarity**(octaves-1)*res).
        octaves: The number of octaves in the noise. Defaults to 1.
        persistence: The scaling factor between two octaves.
        lacunarity: The frequency factor between two octaves.
        tileable: If the noise should be tileable along each axis
            (tuple of two bools). Defaults to (False, False).
        interpolant: The, interpolation function, defaults to
            t*t*t*(t*(t*6 - 15) + 10).

    Returns:
        A numpy array of fractal noise and of shape shape generated by
        combining several octaves of perlin noise.

    Raises:
        ValueError: If shape is not a multiple of lacunarity**(octaves-1)*res.
    """
    noise = np.zeros(shape)
    f, a = 1, 1
    for _ in range(octaves):
        this_res = (f*res[0], f*res[1])
        noise += a * perlin_noise(shape,
                                  this_res,
                                  tileable=tileable,
                                  interpolant=interpolant,
                                  rng=rng
                                  )
        f *= lacunarity
        a *= persistence

    return noise

def generate_random_surface(size,
                            res=2,
                            octaves=2,
                            lacunarity=2,
                            persistence=0.5,
                            random_seed=None
                            ):
    """
    Make a square grid of **at least** the given size (length of side), with
    the given parameters. Size must be a multiple of lacunarity**(octaves-1)*res
    so the smallest possible size greater than the one provided is used.

    Args:
        size (int or tuple): minimum length of side of the grid.
        res (int or tuple): number of periods of noise to generate along
            each axis.
        octaves (int): number of octaves of noise (bandwidth).
        lacunarity (float): lacunarity of the noise (frequency factor between
            octaves). Default: 2.
        persistence (float): persistence of the noise (amplitude factor between
            octaves). Default: 0.5.
        random_seed (int): random seed.

    Returns:
        xarray.DataArray: the grid.
    """
    # `res` and `size` can be ints or tuples.
    if isinstance(res, int):
        res = (res, res)
    if isinstance(size, int):
        size = (size, size)

    # Note shape must be a multiple of (lacunarity**(octaves-1)*res).
    # Not sure what happens if res has different values for each axis.
    size0, size1 = size
    while size0 % (lacunarity**(octaves - 1) * res[0]) > 0:
        size0 += 1
        if size0 > 35_500:
            raise RuntimeError("Smallest compatible surface is >10GB.")
    while size1 % (lacunarity**(octaves - 1) * res[1]) > 0:
        size1 += 1
        if size1 > 35_500:
            raise RuntimeError("Smallest compatible surface is >10GB.")

    rng = np.random.default_rng(seed=random_seed)
    noise = fractal_noise((size0, size1),
                          res=res,
                          octaves=octaves,
                          lacunarity=lacunarity,
                          persistence=persistence,
                          rng=rng
                          )
    i, j = map(np.arange, noise.shape)
    return xr.DataArray(noise,
                        dims=('i', 'j'),
                        coords={'i': i,
                                'j': j},
                        attrs={'source': 'gio.random',})

max_stars_repo_name: agilescientific/gio

max_issues_repo_name: agilescientific/gio

max_forks_repo_name: agilescientific/gio


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import itertools as it
import numpy as np
from graphdot.codegen import Template
from .typetool import can_cast, _dtype_util

def cpptype(decls=[], **kwdecls):
    """
    cpptype is a class decorator that simplifies the translation of python
    objects to corresponding C++ structures.
    """
    ctype = np.dtype(decls + list(kwdecls.items()), align=True)

    def decor(cls):
        class CppType(type(cls)):
            @property
            def dtype(self):
                '''Enables numpy.dtype(cls) to return the layout of the C++
                counterpart.'''
                return ctype

            def __repr__(self):
                return f'@cpptype({repr(ctype)})\n{repr(cls)}'

        class Class(cls, metaclass=CppType):

            @property
            def dtype(self):
                '''Useful in microkernel composition.'''
                return Class.dtype

            @property
            def state(self):
                state = []
                for key, (field, _) in Class.dtype.fields.items():
                    if _dtype_util.is_object(field):
                        state.append(getattr(self, key).state)
                    elif _dtype_util.is_array(field):
                        states = getattr(self, key)
                        if not isinstance(states, np.ndarray):
                            raise TypeError(
                                f'attribute {key} declared as an array'
                                f'but the actual type is not a numpy'
                                f'ndarray.'
                            )
                        if states.shape != field.shape:
                            raise ValueError(
                                f'attribute {key}: '
                                f'actual shape of {states.shape} does not '
                                f'match declared shape of {field.shape}.'
                            )
                        if _dtype_util.is_object(field.base):
                            states = [
                                states[coord].state for coord
                                in it.product(*map(range, field.shape))
                            ]
                        state.append(
                            np.array(states, dtype=field.base)
                              .reshape(field.shape).tolist())
                    else:
                        state.append(field.type(getattr(self, key)))
                return tuple(state)

            def __setattr__(self, name, value):
                if name in Class.dtype.names:
                    lt = Class.dtype.fields[name][0]
                    if _dtype_util.is_array(lt):
                        if not isinstance(value, np.ndarray):
                            value = np.array(value)
                        if value.shape != lt.shape:
                            raise ValueError(
                                f"Cannot set array attribute '{name}' with "
                                f"value of mismatching shape:\n"
                                f"{value}"
                            )
                        for t in map(np.dtype, value.ravel()):
                            if not can_cast(t, lt.base):
                                raise TypeError(
                                    f"Cannot set attribute '{name}' "
                                    f"(C++ type {decltype(lt)}) "
                                    f"with values of {value.dtype}"
                                )
                    else:
                        rt = np.dtype(type(value))
                        if not can_cast(rt, lt):
                            raise TypeError(
                                f"Cannot set attribute '{name}' "
                                f"(C++ type {decltype(lt)}) "
                                f"with value {value} of {type(value)}"
                            )
                super().__setattr__(name, value)

            # TODO: need a nicer __repr__

        Class.__name__ = cls.__name__

        return Class

    return decor

def _assert_is_identifier(name):
    if name and not name.isidentifier():
        raise ValueError(
            f'Name {name} is not a valid Python/C++ identifier.'
        )

def decltype(t, name=''):
    '''Generate C++ source code for structures corresponding to a `cpptype`
    class.'''
    t = np.dtype(t, align=True)  # convert np.float32 etc. to dtype
    if name.startswith('$'):
        ''' template class types '''
        n, t, *Ts = name[1:].split('::')
        _assert_is_identifier(n)
        return Template(r'${template}<${arguments,}>${name}').render(
            template=t,
            arguments=[decltype(T) for T in Ts],
            name=n
        )
    else:
        _assert_is_identifier(name)
        if _dtype_util.is_object(t):
            ''' structs '''
            if len(t.names):
                return Template(r'struct{${members;};}${name}').render(
                    name=name,
                    members=[decltype(t.fields[v][0], v) for v in t.names])
            else:
                return f'constexpr static _empty {name} {{}}'
        elif _dtype_util.is_array(t):
            ''' C-style arrays '''
            return Template(r'${t} ${name}[${shape][}]').render(
                t=decltype(t.base),
                name=name,
                shape=t.shape
            )
        else:
            ''' scalar types and C-strings '''
            if t.kind == 'S':
                return f'char {name}[{t.itemsize}]'
            else:
                return f'{str(t.name)} {name}'.strip()

max_stars_repo_name: yhtang/GraphDot

max_issues_repo_name: yhtang/graphdot

max_forks_repo_name: yhtang/graphdot


text:
import matplotlib.pyplot as plt
import numpy as np

import sys
# data = np.loadtxt("res/ptot_1d_ 0.dat")
# a = data[1, 0::3]
data = np.loadtxt("../../res/ptot_1d_19.dat")
# t = data[:,0]
# d = data[:,5]
ptot = data[2::3]

#print np.shape(ptot)

#sys.exit(-1)

font_title = {'family' : 'serif',
        'weight' : 'normal',
        'size'   : 18,}
font_label = {
        'weight' : 'normal',
        'size'   : 16,}

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
plt.plot(ptot)
ax.grid(True)
plt.xlabel(r"Time (fs)", fontdict=font_label)
plt.ylabel(r"Polarization (arb.)", fontdict=font_label)
plt.title(r"Polarization for different directions", fontdict=font_title)
plt.savefig("fig/ppar.svg")
plt.show()


text:
from email import policy
import os
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam, SGD
import numpy as np

abs_path = os.getcwd()

class Actor(keras.Model):
    """Defines a Actor Deep Learning Network"""

    def __init__(self, n_actions, density=1024, name='actor'):
        super(Actor, self).__init__()

        self.model_name = name
        self.checkpoint = self.model_name + '.h5'

        self.H1 = tf.keras.layers.Dense(density, activation='relu')
        self.H2 = tf.keras.layers.Dense(density, activation='relu')
        self.drop = tf.keras.layers.Dropout(0.1)
        self.H3 = tf.keras.layers.Dense(density, activation='relu')
        self.H4 = tf.keras.layers.Dense(density, activation='relu')
        self.mu = tf.keras.layers.Dense(n_actions, activation='softmax')

    @tf.function()
    def call(self, state):
        state = self.H1(state)
        state = self.H2(state)
        state = self.drop(state)
        state = self.H3(state)
        state = self.H4(state)
        mu = self.mu(state)
        return mu

class Discriminator(keras.Model):
    """Defines a Critic Deep Learning Network"""

    def __init__(self, density=512, name='critic'):
        super(Discriminator, self).__init__()

        self.model_name = name
        self.checkpoint = self.model_name + '.h5'

        self.H1 = tf.keras.layers.Dense(density, activation='relu')
        self.H2 = tf.keras.layers.Dense(density, activation='relu')
        self.drop = tf.keras.layers.Dropout(0.1)
        self.H3 = tf.keras.layers.Dense(density, activation='relu')
        self.H4 = tf.keras.layers.Dense(density, activation='relu')
        self.Q = tf.keras.layers.Dense(1)

    @tf.function()
    def call(self, state, action):
        state = tf.cast(state, tf.float64)
        action = tf.cast(action, tf.float64)
        action = self.H1(tf.concat([state, action], axis=1))
        action = self.H2(action)
        action = self.drop(action)
        action = self.H3(action)
        action = self.H4(action)
        Q = self.Q(action)
        return Q

class Buffer():
    def __init__(self, max_size, state_shape, action_shape):
        self.mem_size = max_size
        self.mem_cntr = 0

        self.state_memory = np.zeros(
            (self.mem_size, *state_shape), dtype=np.float32)
        self.action_memory = np.zeros(
            (self.mem_size, *action_shape), dtype=np.float32)

    def store_transition(self, state, action):
        index = self.mem_cntr % self.mem_size

        self.state_memory[index] = state
        self.action_memory[index] = action

        self.mem_cntr += 1

    def memorize_expert(self, state, action):
        self.state_memory = state
        self.action_memory = action
        self.mem_cntr = len(state)

    def sample_buffer(self, batch_size):
        max_mem = 2500
        batch = np.random.choice(max_mem, batch_size, replace=False)

        states = self.state_memory[batch]
        actions = self.action_memory[batch]

        return states, actions

class Agent:
    def __init__(self, expert_obs, expert_actions, env, batch_size):
        self.env = env

        # Action Space Augmentation
        expert_actions = expert_actions.tolist()
        for i in range(len(expert_actions)):
            if expert_actions[i] == 0:
                expert_actions[i] = np.array([1, 0], dtype=np.float32)
            elif expert_actions[i] == 1:
                expert_actions[i] = np.array([0, 1], dtype=np.float32)
        expert_actions = np.vstack(expert_actions)

        self.batch_size = batch_size
        self.exp_obs = expert_obs
        self.exp_actions = expert_actions
        self.mem_size = len(self.exp_obs)
        self.expert_memory = Buffer(
            self.mem_size, self.exp_obs[0].shape, self.exp_actions[0].shape)
        self.replay_memory = Buffer(
            self.mem_size, self.exp_obs[0].shape, self.exp_actions[0].shape)
        n_actions = env.action_space.n

        self.actor = Actor(n_actions)
        self.discriminator = Discriminator()
        self.actor.compile(optimizer=Adam(1e-3))
        self.discriminator.compile(optimizer=Adam(2e-3))

        self.d_loss = []
        self.a_loss = []

        self.cross_entropy = tf.keras.losses.BinaryCrossentropy(
            from_logits=True)

    def choose_action(self, observation):
        state = np.array([observation])
        actions = self.actor(state)
        return actions

    def memorize_expert(self):
        print(f'Memorizing Expert Data')
        self.expert_memory.memorize_expert(self.exp_obs, self.exp_actions)
        print('Done!')

    def discriminator_loss(self, real_output, fake_output):
        real_loss = self.cross_entropy(tf.ones_like(real_output), real_output)
        fake_loss = self.cross_entropy(tf.zeros_like(fake_output), fake_output)
        total_loss = real_loss + fake_loss
        return total_loss

    def generator_loss(self, fake_output):
        return self.cross_entropy(tf.ones_like(fake_output), fake_output)

    def optimize(self, steps):

        for i in range(steps):
            # Sample Expert datasets & Policy Dataset
            exp_states, exp_actions = self.expert_memory.sample_buffer(
                self.batch_size)
            states = exp_states

            # Optimize the Descriminator and Actor
            with tf.GradientTape() as desc_tape, tf.GradientTape() as actor_tape:
                actions = tf.squeeze(self.choose_action(states))

                expert_probs = self.discriminator(exp_states, exp_actions)
                policy_probs = self.discriminator(states, actions)

                # Loss Functions implemented as per the GAIL research paper
                # Optimize the Descriminator
                desc_loss = tf.nn.sigmoid_cross_entropy_with_logits(
                    logits=policy_probs, labels=tf.ones_like(policy_probs)) + tf.nn.sigmoid_cross_entropy_with_logits(
                    logits=expert_probs, labels=tf.zeros_like(expert_probs))
                desc_gradients = desc_tape.gradient(
                    desc_loss, self.discriminator.trainable_weights)
                self.discriminator.optimizer.apply_gradients(
                    zip(desc_gradients, self.discriminator.trainable_weights))

                # Optimize the Actor
                Q = tf.nn.sigmoid_cross_entropy_with_logits(
                    logits=policy_probs, labels=tf.zeros_like(policy_probs))
                actor_loss = tf.math.reduce_mean(tf.multiply(actions, Q)) - (1e-3 * -tf.math.reduce_sum(
                    tf.multiply(actions, tf.math.log(actions + 1e-11))/tf.math.reduce_sum(actions)))
                actor_gradients = actor_tape.gradient(
                    actor_loss, self.actor.trainable_weights)
                self.actor.optimizer.apply_gradients(
                    zip(actor_gradients, self.actor.trainable_weights))

        print(
            f'\tDesc. Loss: {np.mean(self.d_loss):0.4f}, Actor Loss: {np.mean(self.a_loss):0.4f}')
        self.d_loss.append(np.mean(desc_loss))
        self.a_loss.append(np.mean(actor_loss))
        np.save(os.getcwd()+'/GAIL/data/d_loss',
                self.d_loss, allow_pickle=False)
        np.save(os.getcwd()+'/GAIL/data/a_loss',
                self.a_loss, allow_pickle=False)

    def save_model(self):
        self.actor.save_weights(abs_path+'/GAIL/data/actor.h5')
        self.discriminator.save_weights(abs_path+'/GAIL/data/discriminator.h5')


text:
# coding=utf-8
import sys
caffe_root = '/home/luojh2/Software/caffe-master/'
sys.path.insert(0, caffe_root + 'python')
import caffe
from net_generator import solver_and_prototxt
import numpy as np
from PIL import Image
import random
import time
import os

def im_resize(im, height=224, width=224):
    d_type = im.dtype
    im = Image.fromarray(im)
    im = im.resize([height, width], Image.BICUBIC)
    im = np.array(im, d_type)
    return im

def convert2rgb(im):
    if len(im.shape) == 2:
        im = im.reshape((im.shape[0], im.shape[1], 1))
        im = np.concatenate((im, im, im), axis=2)
    if im.shape[2] == 4:
        im = np.array(Image.fromarray(im).convert('RGB'))
    return im

def get_index(compress_rate, compress_layer, gpu):
    # set parameters
    candidate = ['conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv4_1',
                 'conv4_2', 'conv4_3', 'conv5_1', 'conv5_2', 'conv5_3', 'fc6', 'fc7', 'fc8']
    relu_candidate = ['relu1_1', 'pool1', 'relu2_1', 'pool2', 'relu3_1', 'relu3_2', 'pool3', 'relu4_1',
                      'relu4_2', 'pool4', 'relu5_1', 'relu5_2', 'pool5', 'relu6', 'relu7']

    # load net
    model_def = candidate[compress_layer] + '/deploy.prototxt'
    model_weights = 'model.caffemodel'
    caffe.set_device(gpu)
    caffe.set_mode_gpu()
    net = caffe.Net(model_def, model_weights, caffe.TEST)

    mean_value = np.array([104, 117, 123], dtype=np.float32)
    mean_value = mean_value.reshape([3, 1, 1])

    # extract feature
    count = 0
    sample_num = 10  # for each category, we sample 10 images
    channel_num = 10  # for each image, we sample 10 locations
    padding = 1
    kernel_size = 3 # for vgg-16, the padding and kernel_size are same in each layer
    for foldername in os.listdir(r'/opt/luojh/Dataset/ImageNet/images/train'):
        img_list = os.listdir(r'/opt/luojh/Dataset/ImageNet/images/train/' + foldername)
        img_index = random.sample(range(len(img_list)), sample_num)
        for file_index in img_index:
            time_start = time.time()

            file_path = '/opt/luojh/Dataset/ImageNet/images/train/' + foldername + '/' + img_list[file_index]
            im = Image.open(file_path)
            im = convert2rgb(np.array(im))
            im = im_resize(im, 256, 256)
            im = np.array(im, np.float64)
            im = im[:, :, ::-1]  # convert RGB to BGR
            im = im.transpose((2, 0, 1))  # convert to 3x256x256
            im -= mean_value

            # shape for input (data blob is N x C x H x W), set data
            # center crop
            im = im[:, 16:240, 16:240]
            net.blobs['data'].reshape(1, *im.shape)
            net.blobs['data'].data[...] = im
            # run net and take argmax for prediction
            net.forward()

            Activation = net.blobs[candidate[compress_layer + 1]].data
            Input = net.blobs[relu_candidate[compress_layer]].data
            Filters = net.params[candidate[compress_layer + 1]][0].data
            Bias = net.params[candidate[compress_layer + 1]][1].data

            if compress_layer < 12:
                # conv1_1 to conv 5_2
                if count == 0:
                    X = np.zeros(
                        [channel_num * 1000 * sample_num, Filters.shape[1]])
                    Y = np.zeros([channel_num * 1000 * sample_num, 1])

                padded = np.zeros((Input.shape[0], Input.shape[1], Input.shape[
                                  2] + 2 * padding, Input.shape[3] + 2 * padding), dtype=np.float32)
                padded[:, :, padding:-padding, padding:-padding] = Input
                Input = padded

                for tmp in range(channel_num):
                    filter_num = random.randint(0, Filters.shape[0] - 1)
                    i = random.randint(0, Input.shape[2] - kernel_size)
                    j = random.randint(0, Input.shape[3] - kernel_size)
                    In_ = Input[:, :, i:i + kernel_size, j:j + kernel_size]
                    In_ = In_.reshape([In_.shape[1], -1])
                    F_ = Filters[filter_num, :, :, :]
                    F_ = F_.reshape([F_.shape[0], -1])
                    Out_ = Activation[0, filter_num, i, j] - Bias[filter_num]
                    X[count, :] = np.reshape(np.sum(F_ * In_, axis=1), [1, -1])
                    Y[count, 0] = np.reshape(Out_, [1, -1])
                    count += 1
            elif compress_layer == 12:
                # conv 5_3
                if count == 0:
                    X = np.zeros(
                        [channel_num * 1000 * sample_num, 512])
                    Y = np.zeros([channel_num * 1000 * sample_num, 1])

                for tmp in range(channel_num):
                    filter_num = random.randint(0, Filters.shape[0] - 1)
                    In_ = Input.reshape([Input.shape[1], -1])
                    F_ = Filters[filter_num, :]
                    F_ = F_.reshape([512, -1])
                    Out_ = Activation[:, filter_num] - Bias[filter_num]
                    X[count, :] = np.reshape(np.sum(F_ * In_, axis=1), [1, -1])
                    Y[count, 0] = np.reshape(Out_, [1, -1])
                    count += 1
            else:
                # fc6 fc7
                if count == 0:
                    X = np.zeros(
                        [channel_num * 1000 * sample_num, Filters.shape[1]])
                    Y = np.zeros([channel_num * 1000 * sample_num, 1])

                for tmp in range(channel_num):
                    filter_num = random.randint(0, Filters.shape[0] - 1)
                    In_ = Input.reshape(-1)
                    F_ = Filters[filter_num, :]
                    Out_ = Activation[:, filter_num] - Bias[filter_num]
                    X[count, :] = np.reshape(F_ * In_, [1, -1])
                    Y[count, 0] = np.reshape(Out_, [1, -1])
                    count += 1

            time_end = time.time()
            print 'Done! use %f second, %d image' % (time_end - time_start, count / channel_num)

    # select filters
    return value_sum(X, Y, compress_rate)

# use greedy method to select filters
# x:N*64 matrix, N is the instance number, 64 is channel number
def value_sum(x, y, compress_rate):
    # 1. set parameters
    time_s = time.time()
    x = np.mat(x)
    y = np.mat(y)
    goal_num = int(x.shape[1] * compress_rate)
    discard_num = int(x.shape[1]) - goal_num
    discard_index = []

    # 2. select
    x_tmp = 0
    for i in range(discard_num):
        min_value = float("inf")
        for j in range(x.shape[1]):
            if j not in discard_index:
                tmp_value = np.linalg.norm(x_tmp + x[:, j])
                if tmp_value < min_value:
                    min_value = tmp_value
                    min_index = j
        discard_index.append(min_index)
        x_tmp += x[:, min_index]
        print('discard num={0}, channel num={1}, i={2}, loss={3:.3f}'.format(discard_num, x.shape[1], i, min_value))

    # 3. return index
    index = set(range(x.shape[1])) - set(discard_index)
    index = np.array(list(index))
    index = np.sort(index)

    # 4.least square
    selected_x = x[:, index]
    w = (selected_x.T * selected_x).I * (selected_x.T * y)
    w = np.array(w)

    loss = np.linalg.norm(y - selected_x*w)
    print('loss before w={0:.3f}, loss with w={1:.3f}'.format(np.linalg.norm(y - np.sum(selected_x, 1)), loss))
    print 'Time used: %f s' % (time.time()-time_s)
    return index, w

def compress_net(index, w, compress_layer):
    # other layers
    candidate = ['conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv4_1',
                 'conv4_2', 'conv4_3', 'conv5_1', 'conv5_2', 'conv5_3', 'fc6', 'fc7', 'fc8']

    model_def = candidate[compress_layer] + '/deploy.prototxt'
    model_weights = 'model.caffemodel'
    net = caffe.Net(model_def,  # defines the structure of the matrix
                    model_weights,  # contains the trained weights
                    caffe.TEST)  # use test mode (e.g., don't perform dropout)

    model_def = candidate[compress_layer] + '/trainval.prototxt'
    # use test mode (e.g., don't perform dropout)
    net_new = caffe.Net(model_def, caffe.TEST)

    current_layer = candidate[compress_layer]
    next_layer = candidate[compress_layer + 1]
    for i in range(0, len(candidate)):
        if candidate[i] != current_layer and candidate[i] != next_layer:
            net_new.params[candidate[i]][
                0].data[...] = net.params[candidate[i]][0].data
            net_new.params[candidate[i]][
                1].data[...] = net.params[candidate[i]][1].data

    weight = net.params[current_layer][0].data
    bias = net.params[current_layer][1].data
    if len(weight.shape) == 4:
        # conv layer
        weight = weight[index, :, :, :]
    else:
        # fc layer
        weight = weight[index, :]

    bias = bias[index]
    net_new.params[current_layer][0].data[...] = weight
    net_new.params[current_layer][1].data[...] = bias
    # next
    weight = net.params[next_layer][0].data
    bias = net.params[next_layer][1].data
    if len(weight.shape) == 4:
        # conv layer
        weight = weight[:, index, :, :]
        for i in range(weight.shape[1]):
            weight[:, i, :, :] *= w[i]
    elif next_layer == 'fc6':
        # fc6
        weight = weight.reshape([4096, 512, 7, 7])
        weight = weight[:, index, :, :]
        for i in range(weight.shape[1]):
            weight[:, i, :, :] *= w[i]
        weight = weight.reshape([4096, -1])
    else:
        # fc7 or fc8 layer
        weight = weight[:, index]
        for i in range(weight.shape[1]):
            weight[:, i] *= w[i]

    net_new.params[next_layer][0].data[...] = weight
    net_new.params[next_layer][1].data[...] = bias

    net_new.save('model.caffemodel')
    print 'OK!'

if __name__ == '__main__':
    compress_layer = int(sys.argv[1])
    gpu = int(sys.argv[2])
    compress_rate = float(sys.argv[3])

    solver_and_prototxt(compress_layer, compress_rate)
    index, w = get_index(compress_rate, compress_layer, gpu)

    compress_net(index, w, compress_layer)


text:
import numpy as np
import os
import cv2
import sys
pycaffe = os.path.split(os.path.realpath(__file__))[0] + '/../python'
sys.path.insert(0, pycaffe)
import caffe
from caffe.proto import caffe_pb2
import google.protobuf.text_format as txtf

def first_conv_u8_input(quantized_prototxt, weights, clx = False, force_u8_prototxt = '', new_weights_path = ''):

    if new_weights_path == '':
        new_weights_path = weights.split(".caffemodel")[0] + "_new.caffemodel"
    if force_u8_prototxt == '':
        force_u8_prototxt = quantized_prototxt.split('.prototxt')[0] + "_force_u8.prototxt"

    if os.path.isfile(quantized_prototxt):
        print ('caffeNet found.')
    else:
        print ('caffenet not found')

    net = caffe.Net(quantized_prototxt, weights, caffe.TEST)

    proto_net = caffe_pb2.NetParameter()
    with open(quantized_prototxt) as f:
        s = f.read()
        txtf.Merge(s, proto_net)

    for index, layer in enumerate(proto_net.layer):
        if layer.type == "Data":
            data_layer = layer
        if layer.type == "Convolution":
            first_conv_layer = layer
            break

    first_conv_weight = net.params[first_conv_layer.name][0].data
    first_conv_bias = net.params[first_conv_layer.name][1].data

    scale_in = first_conv_layer.quantization_param.scale_in[0]
    bias_scale = 128/scale_in
    new_bias = np.zeros(first_conv_bias.shape)
    for i in range(len(first_conv_bias)):
        new_bias[i] = first_conv_bias[i] - bias_scale * np.sum(first_conv_weight[i])

    net.params[first_conv_layer.name][1].data[...] = new_bias

    net.save(new_weights_path)

    first_conv_layer.quantization_param.is_negative_input = False
    old_pad = first_conv_layer.convolution_param.pad[0]
    first_conv_layer.convolution_param.pad[0] = 0L
    first_conv_layer.quantization_param.force_u8_input = True
    if not clx:
        for i in range(len(first_conv_layer.quantization_param.scale_params)):
            first_conv_layer.quantization_param.scale_params[i] = float(first_conv_layer.quantization_param.scale_params[i]) * 0.5

    data_layer.transform_param.pad = long(old_pad)
    mean_value = np.array(data_layer.transform_param.mean_value).astype(float)
    scale = data_layer.transform_param.scale
    new_scale = scale_in * scale
    new_mean_value = mean_value - 128/new_scale
    data_layer.transform_param.mean_value[0] = new_mean_value[0]
    data_layer.transform_param.mean_value[1] = new_mean_value[1]
    data_layer.transform_param.mean_value[2] = new_mean_value[2]

    data_layer.transform_param.scale = new_scale

    with open(force_u8_prototxt, 'w') as f:
        f.write(str(proto_net))

    print("Please use new prototxt in {0}, caffemodel in {1}".format(force_u8_prototxt, new_weights_path))

if __name__=='__main__':
    if len(sys.argv) < 2:
        print "Usgae: python first_conv_force_u8.py $prototxt $weights"
        sys.exit(0)
    first_conv_u8_input(sys.argv[1], sys.argv[2])

max_stars_repo_name: vimers/caffe

max_issues_repo_name: vimers/caffe

max_forks_repo_name: vimers/caffe


text:
import os

import numpy as np
import torch

from socket import socket, AF_INET, SOCK_STREAM
from torch import nn

class NetWork(nn.Module):
    def __init__(self, name, x_shape, num_class):
        super(NetWork, self).__init__()
        self.name = name
        self.x_shape = x_shape
        self.num_class = num_class
        self.first_layer = nn.Sequential(
            nn.Conv2d(in_channels=x_shape[1], out_channels=256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=True),
            nn.BatchNorm2d(256),
            nn.ReLU(),
        )

        self.blocks = nn.ModuleList()
        for i in range(19):
            self.blocks.append(
                nn.Sequential(
                    nn.Conv2d(in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)),
                    nn.BatchNorm2d(256),
                    nn.ReLU(),
                    nn.Conv2d(in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)),
                    nn.BatchNorm2d(256)
                )
            )
        self.relu = nn.ReLU()
        self.classify_layer = nn.Conv2d(in_channels=256, out_channels=1, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0))

    def forward(self, feature):
        x = self.first_layer(feature)

        for i in range(len(self.blocks)):
            x = x + self.blocks[i](x)

        x = self.relu(x)

        x = self.classify_layer(x)
        out = x.reshape(-1, 361)

        return out

    def get_weights(self):
        return self.parameters()

def play():
    files = os.listdir()
    model_file = ''
    for file in files:
        if file == 'model.pt':
            model_file = file
    if model_file == '':
        return

    model = NetWork('forward', [1, 31, 19, 19], 361)
    model.load_state_dict(torch.load(model_file))

    if torch.cuda.is_available():
        model = model.cuda()

    server_name = '127.0.0.1'
    server_port = 23334
    buf_size = 120000
    ADDR = (server_name, server_port)

    server_socket = socket(AF_INET, SOCK_STREAM)
    server_socket.bind(ADDR)

    server_socket.listen(1)

    client_socket, client_addr = server_socket.accept()
    while True:
        try:
            data_byte = client_socket.recv(buf_size)
        except IOError:
            client_socket.close()
            break
        if not data_byte:
            break
        length = int.from_bytes(data_byte, signed=False, byteorder='big')
        data_src = []
        for i in range(length):
            data_byte = client_socket.recv(1806)
            data_src.append(data_byte.decode('utf8').strip())
        prob = forward(data_src, model)
        for one_state_res in prob:
            client_socket.send(one_state_res)

def forward(data_src, model):
    layer_base = [0, 3, 11, 27, 19]
    input_data = torch.zeros((len(data_src), 31, 19, 19), dtype=torch.float32)
    for k in range(len(data_src)):
        data = data_src[k]
        color = int(data[-1])

        feature = torch.tensor(np.array(list(map(int, data))[:-1], dtype=np.int8).reshape(5, 19, 19))

        train_feature = torch.zeros((31, 19, 19), dtype=torch.float32)

        for i in range(19):
            for j in range(19):
                if feature[0][i][j] == 0:
                    train_feature[2][i][j] = 1
                else:
                    train_feature[1 - (int(feature[0][i][j]) == color)][i][j] = 1

        train_feature[3] = torch.ones((19, 19))

        for layer_num in range(1, 5):
            layer = feature[layer_num]
            for i in range(19):
                for j in range(19):
                    if layer[i][j] != 0:
                        train_feature[layer_base[layer_num] + int(layer[i][j])][i][j] = 1

        if color == 1:
            train_feature[30] = torch.ones((19, 19))

        input_data[k] = train_feature

    if torch.cuda.is_available():
        input_data = input_data.cuda()

    out = model.forward(input_data)
    out = out.cpu().detach().numpy()

    pro = np.exp(out) / np.sum(np.exp(out))
    res = []
    for i in pro:
        one_state_res = b''
        for j in i:
            float_str = "{:.6f}".format(j)[::-1]
            one_state_res += float_str[0:-2].encode('utf8')
        res.append(one_state_res)
    return res

if __name__ == '__main__':
    play()

max_stars_repo_name: MartrixG/GraduationProject

max_issues_repo_name: MartrixG/GraduationProject

max_forks_repo_name: MartrixG/GraduationProject