text:
import os
import numpy as np

import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader
from torch.utils.data.sampler import SubsetRandomSampler, WeightedRandomSampler

from torchlib.datasets.fersynthetic  import SyntheticFaceDataset
from torchlib.datasets.factory import FactoryDataset
from torchlib.attentionnet import AttentionNeuralNet, AttentionGMMNeuralNet
from torchlib.classnet import ClassNeuralNet
from torchlib.datasets.datasets import Dataset

import datetime
from argparse import ArgumentParser
from aug import get_transforms_aug, get_transforms_det

def arg_parser():
    """Arg parser"""
    parser = ArgumentParser()
    parser.add_argument('data', metavar='DIR',
                        help='path to dataset')
    parser.add_argument('--databack', default=None,
                        help='path to background dataset')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='enables CUDA training')
    parser.add_argument('-g', '--gpu', default=0, type=int, metavar='N',
                        help='divice number (default: 0)')
    parser.add_argument('--seed', type=int, default=1,
                        help='random seed (default: 1)')
    parser.add_argument('-j', '--workers', default=1, type=int, metavar='N',
                        help='number of data loading workers (default: 1)')
    parser.add_argument('--epochs', default=90, type=int, metavar='N',
                        help='number of total epochs to run')
    parser.add_argument('--kfold', default=0, type=int, metavar='N',
                        help='k fold')
    parser.add_argument('--nactor', default=0, type=int, metavar='N',
                        help='number of the actores')
    parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                        help='manual epoch number (useful on restarts)')
    parser.add_argument('-b', '--batch-size', default=256, type=int, metavar='N',
                        help='mini-batch size (default: 256)')
    parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float, metavar='LR',
                        help='initial learning rate')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--print-freq', '-p', default=10, type=int, metavar='N',
                        help='print frequency (default: 10)')
    parser.add_argument('--snapshot', '-sh', default=10, type=int, metavar='N',
                        help='snapshot (default: 10)')
    parser.add_argument('--project', default='./runs', type=str, metavar='PATH',
                        help='path to project (default: ./runs)')
    parser.add_argument('--name', default='exp', type=str,
                        help='name of experiment')
    parser.add_argument('--resume', default=None, type=str, metavar='NAME',
                        help='name to latest checkpoint (default: none)')
    parser.add_argument('--arch', default='simplenet', type=str,
                        help='architecture')
    parser.add_argument('--finetuning', action='store_true', default=False,
                        help='Finetuning')
    parser.add_argument('--loss', default='cross', type=str,
                        help='loss function')
    parser.add_argument('--opt', default='adam', type=str,
                        help='optimize function')
    parser.add_argument('--scheduler', default='fixed', type=str,
                        help='scheduler function for learning rate')
    parser.add_argument('--image-size', default=128, type=int, metavar='N',
                        help='image size')
    parser.add_argument('--channels', default=1, type=int, metavar='N',
                        help='input channel (default: 1)')
    parser.add_argument('--dim', default=64, type=int, metavar='N',
                        help='code size (default: 64)')
    parser.add_argument('--num-classes', '-c', default=10, type=int, metavar='N',
                        help='num classes (default: 10)')
    parser.add_argument('--name-dataset', default='mnist', type=str,
                        help='name dataset')
    parser.add_argument('--name-method', default='attnet', type=str,
                        help='name method')
    parser.add_argument('--parallel', action='store_true', default=False,
                        help='Parallel')
    parser.add_argument('--balance', action='store_true', default=False,
                        help='balance data')
    parser.add_argument('--backbone', default='preactresnet', type=str,
                        help='backbone for classification and representation')
    parser.add_argument('--trainiteration', default=None, type=int, metavar='N',
                        help='train iteration to increase data')
    parser.add_argument('--testiteration', default=None, type=int, metavar='N',
                        help='train iteration to increase data')
    parser.add_argument('--breal', type=str, default='real', help='dataset is real or synthetic')
    parser.add_argument('--num_filters', type=int, default=32, help='reconstruction size')

    parser.add_argument('--alpha', default=2.0, type=float, help='The first parameter of the beta distribution')
    parser.add_argument('--beta', default=2.0, type=float, help="the second parameter of the beta distribution")

    return parser

def main():
    # This model has a lot of variabilty, so it needs a lot of parameters.
    # We use an arg parser to get all the arguments we need.
    # See above for the default values, definitions and information on the datatypes.
    parser = arg_parser()
    args = parser.parse_args()
    imsize = args.image_size
    parallel = args.parallel
    num_classes = args.num_classes
    num_channels = args.channels
    num_filters = args.num_filters
    dim = args.dim
    view_freq = 1
    trainiteration = args.trainiteration
    testiteration = args.testiteration
    alpha = args.alpha
    beta = args.beta

    # Which network do we want to use? Initialize it.
    fname = args.name_method
    fnet = {
        'attnet': AttentionNeuralNet, # network for
        'attgmmnet': AttentionGMMNeuralNet,
        'classnet': ClassNeuralNet,
    }
    network = fnet[fname](
        patchproject=args.project,
        nameproject=args.name,
        no_cuda=args.no_cuda,
        parallel=parallel,
        seed=args.seed,
        print_freq=args.print_freq,
        gpu=args.gpu,
        view_freq=view_freq,
    )
    network.create(
        arch=args.arch,
        num_output_channels=dim,
        num_input_channels=num_channels,
        loss=args.loss,
        lr=args.lr,
        momentum=args.momentum,
        optimizer=args.opt,
        lrsch=args.scheduler,
        pretrained=args.finetuning,
        size_input=imsize,
        num_classes=num_classes,
        breal=args.breal,
        alpha=alpha,
        beta=beta
    )

    # resume if a model can be loaded
    if args.resume is not None:
        network.resume(os.path.join(network.pathmodels, args.resume))
    cudnn.benchmark = True

    kfold = args.kfold
    nactores = args.nactor
    idenselect = np.arange(nactores) + kfold * nactores

    # choose and load our datasets
    # training dataset
    if args.breal == 'real':
        train_data = Dataset(
            data=FactoryDataset.factory(
                pathname=args.data,
                name=args.name_dataset,
                subset=FactoryDataset.training,
                idenselect=idenselect,
                download=True
            ),
            num_channels=3,
            transform=get_transforms_det(args.image_size),
        )

        val_data = Dataset(
            data=FactoryDataset.factory(
                pathname=args.data,
                name=args.name_dataset,
                subset=FactoryDataset.validation,
                idenselect=idenselect,
                download=True
            ),
            num_channels=3,
            transform=get_transforms_det(args.image_size),
        )
    else:
        # SyntheticFaceDataset
        train_data = SyntheticFaceDataset(
            data=FactoryDataset.factory(
                pathname=args.data,
                name=args.name_dataset,
                subset=FactoryDataset.training,
                idenselect=idenselect,
                download=True
            ),
            pathnameback=args.databack,
            ext='jpg',
            count=trainiteration,
            num_channels=num_channels,
            iluminate=True, angle=30, translation=0.2, warp=0.1, factor=0.2,
            transform_data=get_transforms_aug(imsize),
            transform_image=get_transforms_det(imsize),
        )

        # validate dataset
        val_data = SyntheticFaceDataset(
            data=FactoryDataset.factory(
                pathname=args.data,
                name=args.name_dataset,
                idenselect=idenselect,
                subset=FactoryDataset.validation,
                download=True
            ),
            pathnameback=args.databack,
            ext='jpg',
            count=testiteration,
            num_channels=num_channels,
            iluminate=True, angle=30, translation=0.2, warp=0.1, factor=0.2,
            transform_data=get_transforms_aug(imsize),
            transform_image=get_transforms_det(imsize),
        )

    num_train = len(train_data)
    # sample to balance the dataset
    if args.balance:
        labels, counts = np.unique(train_data.labels, return_counts=True)
        weights = 1 / (counts / counts.sum())
        samples_weights = np.array([weights[x] for x in train_data.labels])
        sampler = WeightedRandomSampler(weights=samples_weights, num_samples=len(samples_weights), replacement=True)
    else:
        sampler = SubsetRandomSampler(np.random.permutation(num_train))

    # Now that we have our dataset, make loaders to facilitate training and validation
    train_loader = DataLoader(train_data, batch_size=args.batch_size,
                              num_workers=args.workers, pin_memory=network.cuda, drop_last=True,
                              sampler=sampler)  # shuffle=True,

    val_loader = DataLoader(val_data, batch_size=args.batch_size, shuffle=True,
                            num_workers=args.workers, pin_memory=network.cuda, drop_last=False)

    print(f"Train size: {len(train_data)}, test size: {len(val_data)}")

    # print neural net class
    start = datetime.datetime.now()
    print('SEG-Torch: {}'.format(start))
    print(network)

    # fit the neural net - this function performs both training and validation,
    # printing loss values to the screen and saving the best model for use later.
    network.fit(train_loader, val_loader, args.epochs, args.snapshot)

    print("Optimization Finished!")
    end = datetime.datetime.now()
    print("End time is", end)
    print("Time duration is:", round((end - start).total_seconds()/60, 2))
    print("DONE!!!")

if __name__ == '__main__':
   main()

max_stars_repo_name: HelenGuohx/cv-ferattn-code

max_issues_repo_name: HelenGuohx/cv-ferattn-code

max_forks_repo_name: HelenGuohx/cv-ferattn-code


text:
# Attempts at vectorising the lensing process

# import modules
import numpy as np
import matplotlib.pyplot as plt
import timeit

# %%

# start the timer
start = timeit.default_timer()

# set up some initial parameters
rc = 0.7
eps = 0
dom = 1  # abs() of domain of r values (normally -1, 1 --> 1)
size = 21  # odd for test

# set up an image of the source. For simplest test, a single pixel at centre
# As for the initial test
image_s = np.zeros([size, size, 3])  # allow for RGB from the start
image_s[int((size-1)/2), int((size-1)/2), 0] = 1

# plot that image
plt.imshow(image_s)

# set up an empty array to store lensed image
image_l = np.zeros([size, size, 3])

# find the width of each pixel given the domain and size
p_width = 2*dom/(size)

# Vectorising the mapping of pixels to source plane and copying data:

# set up an array of index numbers in the lens image array
i_arr = np.arange(0, size, 1)
j_arr = np.arange(0, size, 1)

# based on pixel position get reduced coords. r_1 and r_2 as arrays
r1 = 2*dom*i_arr/(size-1) - dom
r2 = 2*dom*j_arr/(size-1) - dom

# grid them
r1g, r2g = np.meshgrid(r1, r2)

# use lens equation to get array of positions on image_s for each pixel in lens image
s1 = r1g - ((1 - eps)*r1g)/np.sqrt(rc**2 + (1 - eps)*r1g**2 + (1 + eps)*r2g**2)
s2 = r2g - ((1 + eps)*r2g)/np.sqrt(rc**2 + (1 - eps)*r1g**2 + (1 + eps)*r2g**2)

# s coordinates dont have to be pixel positions in reduced
# need to find which pixel they lie in in the original image for all pixels
# therefore for all array values
index_1 = np.floor((s1 + dom)/p_width)
index_2 = np.floor((s2 + dom)/p_width)

# change them to integers
index_1 = index_1.astype(int).transpose()
index_2 = index_2.astype(int).transpose()

# copy the data from source image at these indexes over to lens image array
image_l[:, :, :] = image_s[index_1, index_2, :]

plt.figure()
# plot the resulting image
plt.imshow(image_l)

# return time to run
stop = timeit.default_timer()
print('Time to run was: {:.4f}'.format(stop - start) + ' s')

ext: py

max_stars_repo_path: attempts/stage 2/optimising attempt 1.py

reason: import numpy


text:
from misc import util
from misc import array

from collections import namedtuple
import numpy as np
import pickle
import re
from skimage.measure import block_reduce

SIZE = 7
WINDOW_SIZE = 5

#WOOD_VARIANTS = ["oak", "pine", "birch"]
#ORE_VARIANTS = ["copper", "iron", "nickel"]
#STONE_VARIANTS = ["granite", "quartz", "slate"]
WOOD_VARIANTS = ["oak", "pine"]
ORE_VARIANTS = ["copper", "iron"]
STONE_VARIANTS = ["granite", "quartz"]
#WOOD_VARIANTS = ["oak"]
#ORE_VARIANTS = ["copper"]
#STONE_VARIANTS = ["granite"]

INGREDIENTS = (
    ["wood_%s" % v for v in WOOD_VARIANTS]
    + ["ore_%s" % v for v in ORE_VARIANTS]
    + ["grass"]
    + ["stone_%s" % v for v in STONE_VARIANTS]
)

CRAFTS = {
    #"stick_?": ["stick_%s" % v for v in WOOD_VARIANTS],
    #"metal_?": ["metal_%s" % v for v in ORE_VARIANTS],
    #"rope": ["rope"],
    "shovel_?_?": ["shovel_%s_%s" % (v1, v2) for v1 in WOOD_VARIANTS for v2 in ORE_VARIANTS],
    "ladder_?": ["ladder_%s" % v for v in WOOD_VARIANTS],
    "axe_?_?": ["axe_%s_%s" % (v1, v2) for v1 in WOOD_VARIANTS for v2 in STONE_VARIANTS],
    "trap_?": ["trap_%s" % v for v in ORE_VARIANTS],
    "sword_?_?": ["sword_%s_%s" % (v1, v2) for v1 in ORE_VARIANTS for v2 in STONE_VARIANTS],
    "bridge_?": ["bridge_%s" % v for v in STONE_VARIANTS]
}

TEMPLATES = {
    #"stick_?": ["wood_?"],
    #"metal_?": ["ore_?"],
    #"rope": ["grass"],
    #"axe_?_?": ["stick_?", "stone_?"],
    #"trap_?": ["metal_?", "rope"],
    #"sword_?_?": ["metal_?", "stone_?"],
    #"bridge_?": ["rope", "stone_?"]

    "shovel_?_?": ["wood_?", "ore_?"],
    "ladder_?": ["wood_?", "grass"],
    "axe_?_?": ["wood_?", "stone_?"],
    "trap_?": ["ore_?", "grass"],
    "sword_?_?": ["ore_?", "stone_?"],
    "bridge_?": ["grass", "stone_?"]
}

START = "<s>"
STOP = "</s>"

RECIPES = {}
HINTS = []
for group in CRAFTS.values():
    for goal in group:
        variants = goal.split("_")[1:]
        abstracted = re.sub(r"_[a-z]+", "_?", goal)
        ingredients = TEMPLATES[abstracted]

        spec_parents = []
        var_parents = list(variants)
        for ingredient in ingredients:
            if "?" in ingredient:
                spec_parents.append(ingredient.replace("?", var_parents.pop(0)))
            else:
                spec_parents.append(ingredient)
        RECIPES[goal] = spec_parents

        base = False
        while not base:
            base = True
            new_ingredients = []
            for ing in ingredients:
                if ing in TEMPLATES:
                    new_ingredients += TEMPLATES[ing]
                    base = False
                else:
                    new_ingredients.append(ing)
            ingredients = new_ingredients
        specialized = []
        for ingredient in ingredients:
            if "?" in ingredient:
                specialized.append(ingredient.replace("?", variants.pop(0)))
            else:
                specialized.append(ingredient)
        #print goal, specialized
        HINTS.append((goal, [START] + specialized + [STOP]))

for ingredient in INGREDIENTS:
    HINTS.append((ingredient, [START, ingredient, STOP]))

util.next_random().shuffle(HINTS)

#for x in RECIPES.items():
#    print x
#
#print

#for x in HINTS:
#    print x

#for i, (k, v) in enumerate(HINTS):
#    print i, k, v

#HINTS = [
#    ("wood", ["wood"]),
#    ("ore", ["ore"]),
#    ("grass", ["grass"]),
#    ("stone", ["stone"]),
#    ("stick", ["wood", "craft"]),
#    ("metal", ["ore", "craft"]),
#    ("rope", ["grass", "craft"]),
#    ("shovel", ["wood", "metal", "craft"]),
#    ("ladder", ["wood", "grass", "craft"]),
#    ("axe", ["wood", "stone", "craft"]),
#    ("trap", ["ore", "grass", "craft"]),
#    ("sword", ["ore", "stone", "craft"]),
#    ("bridge", ["grass", "stone", "craft"]),
#    #("shovel", ["stick", "metal", "craft"]),
#    #("ladder", ["stick", "rope", "craft"]),
#    #("axe", ["stick", "stone", "craft"]),
#    #("trap", ["metal", "rope", "craft"]),
#    #("sword", ["metal", "stone", "craft"]),
#    #("bridge", ["rope", "stone", "craft"]),
#]

TEST_IDS = list(range(len(HINTS))[::4])
#TEST_IDS = list(range(len(HINTS))[::2])
TRAIN_IDS = [i for i in range(len(HINTS)) if i not in TEST_IDS]

#TRAIN_IDS = [TEST_IDS[3]]
#TEST_IDS = [TEST_IDS[0]]

#TEST_IDS = list(range(len(HINTS)))

#TEST_IDS = [len(HINTS)-1]
#TRAIN_IDS = [len(HINTS)-1]

N_ACTIONS = 6
UP, DOWN, LEFT, RIGHT, USE, CRAFT = range(N_ACTIONS)

Minicraft2Task = namedtuple("Minicraft2Task", ["id", "goal", "hint"])

def random_free(grid, random):
    pos = None
    while pos is None:
        (x, y) = (random.randint(SIZE), random.randint(SIZE))
        if grid[x, y, :].any():
            continue
        pos = (x, y)
    return pos

def neighbors(pos, dir=None):
    x, y = pos
    neighbors = []
    if x > 0 and (dir is None or dir == LEFT):
        neighbors.append((x-1, y))
    if y > 0 and (dir is None or dir == DOWN):
        neighbors.append((x, y-1))
    if x < SIZE - 1 and (dir is None or dir == RIGHT):
        neighbors.append((x+1, y))
    if y < SIZE - 1 and (dir is None or dir == UP):
        neighbors.append((x, y+1))
    return neighbors

class Minicraft2Instance(object):
    def __init__(self, task, init_state, demo=None):
        self.task = task
        self.state = init_state
        self.demo = demo
        self.instruction = task.hint

class Minicraft2World(object):
    def __init__(self):
        self.tasks = []
        self.index = util.Index()
        self.vocab = util.Index()
        self.ingredients = [self.index.index(k) for k in INGREDIENTS]
        self.recipes = {
            self.index.index(k): set(self.index.index(vv) for vv in v)
            for k, v in RECIPES.items()
        }
        #print self.recipes
        #self.hints = [
        #    (self.index.index(k), tuple(self.vocab.index(vv) for vv in v))
        #    for k, v in HINTS
        #]
        self.hints = []
        for k, v in HINTS:
            self.hints.append((self.index.index(k), v))
            for w in v:
                self.vocab.index(w)

        self.kind_to_obs = {}
        self.obs_to_kind = {}
        for k in self.ingredients:
            self.kind_to_obs[k] = len(self.kind_to_obs)
            self.obs_to_kind[self.kind_to_obs[k]] = k

        self.n_obs = (
            2 * WINDOW_SIZE * WINDOW_SIZE * len(self.kind_to_obs)
            + len(self.index)
            + 4)
        self.n_act = N_ACTIONS
        self.n_actions = self.n_act
        self.n_features = self.n_obs
        self.is_discrete = True

        self.max_hint_len = 3
        self.n_vocab = len(self.vocab)
        self.random = util.next_random()

        self.START = START
        self.STOP = STOP

        self.tasks = []
        for i, (goal, steps) in enumerate(self.hints):
            self.tasks.append(Minicraft2Task(i, goal, None))
        self.n_tasks = len(self.tasks)
        self.n_train = len(TRAIN_IDS)
        self.n_val = 0
        self.n_test = len(TEST_IDS)

        self.demos = {}
        #for i in range(len(TRAIN_IDS)):
        #    with open("data/minicraft/paths.%d.pkl" % i, "rb") as pf:
        #        paths = pickle.load(pf)[-1000:]
        #        task = paths[0][0]._replace(hint=None)
        #        assert task == self.tasks[TRAIN_IDS[i]]
        #        self.demos[TRAIN_IDS[i]] = [v for k, v in paths]

    def sample_instance(self, task_id):
        task = self.tasks[task_id]
        _, steps = self.hints[task_id]
        indexed_steps = [self.vocab[w] for w in steps]
        task = task._replace(hint=tuple(indexed_steps))
        demo = None
        if task_id in self.demos:
            episodes = self.demos[task_id]
            episode = episodes[self.random.randint(len(episodes))]
            demo = []
            for transition in episode:
                demo.append((
                    FakeMinicraft2State(transition.s1),
                    transition.a[0],
                    FakeMinicraft2State(transition.s2)))
        state = self.sample_state(task)
        return Minicraft2Instance(task, state, demo)

    def sample_state(self, task):
        grid = np.zeros((SIZE, SIZE, len(self.kind_to_obs)))
        for k in self.ingredients:
            obs = self.kind_to_obs[k]
            for _ in range(1):
                x, y = random_free(grid, self.random)
                grid[x, y, obs] = 1

        init_pos = random_free(grid, self.random)
        init_dir = self.random.randint(4)
        return Minicraft2State(self, grid, init_pos, init_dir,
                np.zeros(len(self.index)), task, task.hint)

    #def sample_train(self, p=None):
    #    return self.sample_instance(self.random.choice(TRAIN_IDS, p=p))

    #def sample_val(self, p=None):
    #    assert False
    #    return self.sample_instance(self.random.choice(TRAIN_IDS, p=p))

    #def sample_test(self, p=None):
    #    return self.sample_instance(self.random.choice(TEST_IDS, p=p))

    def sample_train(self):
        return self.sample_instance(self.random.choice(TRAIN_IDS)).state

    def sample_test(self):
        return self.sample_instance(self.random.choice(TEST_IDS)).state

    def reset(self, insts):
        return [inst.state.features() for inst in insts]

    def step(self, actions, insts):
        features, rewards, stops = [], [], []
        for action, inst in zip(actions, insts):
            reward, new_state, stop = inst.state.step(action)
            inst.state = new_state
            features.append(new_state.features())
            rewards.append(reward)
            stops.append(stop)
        return features, rewards, stops

class FakeMinicraft2State(object):
    def __init__(self, features):
        self._features = features

    def features(self):
        return self._features

    def step(self, action):
        assert False

class Minicraft2State(object):
    def __init__(self, world, grid, pos, dir, inventory, task, instruction):
        self.world = world
        self.grid = grid
        self.pos = pos
        self.dir = dir
        self.inventory = inventory
        self.task = task
        self._cached_features = None
        self.instruction = instruction
        self.task_id = task.id
        self.features = self._features()

    def _features(self):
        if self._cached_features is not None:
            return self._cached_features

        x, y = self.pos
        hs = WINDOW_SIZE / 2
        bhs = WINDOW_SIZE * WINDOW_SIZE / 2

        grid_feats = array.pad_slice(self.grid, (x-hs, x+hs+1), (y-hs, y+hs+1))
        grid_feats_big = array.pad_slice(self.grid, (x-bhs, x+bhs+1), (y-bhs, y+bhs+1))
        grid_feats_red = block_reduce(grid_feats_big, (WINDOW_SIZE, WINDOW_SIZE, 1), func=np.max)

        pos_feats = np.asarray(self.pos, dtype=np.float32) / SIZE
        dir_feats = np.zeros(4)
        dir_feats[self.dir] = 1

        features = np.concatenate((
            grid_feats.ravel(),
            grid_feats_red.ravel(),
            self.inventory,
            dir_feats))
        self._cached_features = features
        return features

    def step(self, action):
        x, y = self.pos
        new_dir = self.dir
        new_inventory = self.inventory
        new_grid = self.grid
        reward = 0
        stop = False

        dx, dy = 0, 0
        if action == UP:
            dx, dy = 0, 1
            new_dir = UP
        elif action == DOWN:
            dx, dy = 0, -1
            new_dir = DOWN
        elif action == LEFT:
            dx, dy = -1, 0
            new_dir = LEFT
        elif action == RIGHT:
            dx, dy = 1, 0
            new_dir = RIGHT
        elif action == USE:
            for nx, ny in neighbors(self.pos, self.dir):
                here = self.grid[nx, ny, :]
                if not here.any():
                    continue
                if here.sum() > 1:
                    assert False
                assert here.sum() == 1
                obs = here.argmax()
                new_inventory = self.inventory.copy()
                new_grid = self.grid.copy()
                new_inventory[self.world.obs_to_kind[obs]] += 1
                new_grid[nx, ny, obs] = 0
        elif action == CRAFT:
            new_inventory = self.inventory.copy()
            more = True
            while more:
                more = False
                for product, ingredients in self.world.recipes.items():
                    if all(new_inventory[ing] > 0 for ing in ingredients):
                        new_inventory[product] += 1
                        for ing in ingredients:
                            new_inventory[ing] -= 1
                        #more = True
                        break

        n_x = x + dx
        n_y = y + dy
        if n_x < 0 or n_x >= SIZE:
            n_x = x
        if n_y < 0 or n_y >= SIZE:
            n_y = y
        if self.grid[n_x, n_y, :].any():
            n_x, n_y = x, y

        if new_inventory[self.task.goal] > 0:
            reward = 1
            stop = True

        new_state = Minicraft2State(self.world, new_grid, (n_x, n_y), new_dir, new_inventory, self.task, self.instruction)
        #return reward, new_state, stop
        return new_state, reward, stop


text:
import loader as ld
import fun_basicas as fun
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize as opt
from scipy.optimize import minimize

def coste(theta1, theta2, X, Y, num_etiquetas):  # Y preparada

    A1, A2, h = forward_prop(X, theta1, theta2)

    sum1 = Y * np.log(h)
    sum2 = (1 - Y) * np.log(1 - h + 1e-6)

    return (-1 / X.shape[0]) * np.sum(sum1 + sum2)

def coste_reg(theta1, theta2, X, Y, num_etiquetas, Lambda):
    c = coste(theta1, theta2, X, Y, num_etiquetas)
    m = X.shape[0]

    e = sum(sum(theta1[:, 1:] ** 2)) + sum(sum(theta2[:, 1:] ** 2))

    return c + (Lambda / (2 * m)) * e

def forward_prop(X, theta1, theta2):
    n = X.shape[0]

    # Se añade una fila de unos a la matriz inicial
    X = np.hstack([np.ones([n, 1]), X])

    # La capa oculta utiliza la primera matriz de pesos para crear sus neuronas y le añade una fila de unos
    Oculta = fun.sigmoide(np.dot(X, theta1.T))
    Oculta = np.hstack([np.ones([n, 1]), Oculta])

    # El resultado se calcula pasando por la segunda matriz de pesos todas las neuronas de la capa oculta
    Resultado = fun.sigmoide(np.dot(Oculta, theta2.T))

    return X, Oculta, Resultado

def gradiente(theta1, theta2, X, y):
    # Creamos los Delta con la forma de theta pero inicializados a cero
    Delta1 = np.zeros(np.shape(theta1))
    Delta2 = np.zeros(np.shape(theta2))

    m = len(y)

    # Se realiza la propagación hacia delante
    A1, A2, h = forward_prop(X, theta1, theta2)

    # Se realiza la propagación hacia atras para cada
    # elemento para comprobar el fallo
    for k in range(m):
        a1k = A1[k, :]
        a2k = A2[k, :]
        a3k = h[k, :]
        yk = y[k, :]

        d3 = a3k - yk
        g_prima = (a2k * (1 - a2k))
        d2 = np.dot(theta2.T, d3) * g_prima

        Delta1 = Delta1 + np.dot(d2[1:, np.newaxis], a1k[np.newaxis, :])
        Delta2 = Delta2 + np.dot(d3[:, np.newaxis], a2k[np.newaxis, :])

    # Se devuelven los Deltas que corresponden al gradiente
    return Delta1 / m, Delta2 / m

def gradiente_reg(theta1, theta2, X, y, Lambda):
    m = len(y)
    Delta1, Delta2 = gradiente(theta1, theta2, X, y)

    # A cada elemento del gradiente (menos la primera columna) se le añade el termino de regularización Lambda
    # multiplicado por cada elemento de las matriz theta 1 y theta2
    Delta1[:, 1:] = Delta1[:, 1:] + (Lambda / m) * theta1[:, 1:]
    Delta2[:, 1:] = Delta2[:, 1:] + (Lambda / m) * theta2[:, 1:]

    return Delta1, Delta2

def backprop(params_rn, num_entradas, num_ocultas, num_etiquetas, X, y, reg):
    # backprop devuelve una tupla (coste, gradiente) con el coste y el gradiente de
    # una red neuronal de tres capas , con num_entradas , num_ocultas nodos en la capa
    # oculta y num_etiquetas nodos en la capa de salida. Si m es el numero de ejemplos
    # de entrenamiento, la dimensión de ’X’ es (m, num_entradas) y la de ’y’ es
    # (m, num_etiquetas)

    theta1 = np.reshape(params_rn[:num_ocultas * (num_entradas + 1)], (num_ocultas, (num_entradas + 1)))
    theta2 = np.reshape(params_rn[num_ocultas * (num_entradas + 1):], (num_etiquetas, (num_ocultas + 1)))

    m = len(y)

    D1, D2 = gradiente_reg(theta1, theta2, X, y, reg)

    coste = coste_reg(theta1, theta2, X, y, num_etiquetas, reg)

    gradiente = np.concatenate((np.ravel(D1), np.ravel(D2)))

    return coste, gradiente

def prueba_neurona(X, y, theta1, theta2):
    """función que devuelve el porcentaje de acierto de una red neuronal utilizando unas matrices de pesos dadas"""
    n = len(y)

    y = np.ravel(y)

    _, _, result = forward_prop(X, theta1, theta2)

    result = np.argmax(result, axis=1)

    return (sum((result + 1)%4 == y) / n * 100)

def validacion_redes(random_state, num_labels, iteraciones, hiddens, lambdas, colores = ['r', 'b' , 'g', 'm']):

    Ex, Ey, Vx, Vy, Px, Py = ld.carga_Numpy(random_state)

    y_onehot = fun.one_hot(Ey, 4)

    input_size = Ex.shape[1]

    INIT_EPSILON = 0.12

    for hidden_size in hiddens:

        theta1 = np.random.random((hidden_size,(input_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
        theta2 = np.random.random((num_labels,(hidden_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON

        params = np.concatenate((np.ravel(theta1), np.ravel(theta2)))

        plt.figure()

        i = 0

        for reg in lambdas:
            percent = []
            for iters in iteraciones:
                fmin = minimize(fun=backprop, x0=params,
                        args=(input_size, hidden_size,
                        num_labels, Ex, y_onehot, reg),
                        method='TNC', jac=True,
                        options={'maxiter': iters})

                theta1 = np.reshape(fmin.x[:hidden_size*(input_size + 1)],(hidden_size,(input_size + 1)))
                theta2 = np.reshape(fmin.x[hidden_size * (input_size+1):],(num_labels,(hidden_size + 1)))

                p = prueba_neurona(Vx, Vy, theta1, theta2)
                print(p)
                percent.append(p)
            plt.plot(iteraciones, percent, c = colores[i] , label = ' lambda = {} '.format(reg))
            i = i+1

        plt.legend()
        plt.title("hidden sizes: {}".format(hidden_size))
        plt.show()

def prueba_redes(random_state, num_labels, iteraciones, hiddens, lambdas, colores = ['r', 'b' , 'g', 'm']):

    Ex, Ey, Vx, Vy, Px, Py = ld.carga_Numpy(random_state)

    y_onehot = fun.one_hot(Ey, 4)

    input_size = Ex.shape[1]

    INIT_EPSILON = 0.12

    for hidden_size in hiddens:

        theta1 = np.random.random((hidden_size,(input_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
        theta2 = np.random.random((num_labels,(hidden_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON

        params = np.concatenate((np.ravel(theta1), np.ravel(theta2)))

        plt.figure()

        i = 0

        for reg in lambdas:
            percent1 = []
            percent2 = []
            for iters in iteraciones:
                fmin = minimize(fun=backprop, x0=params,
                        args=(input_size, hidden_size,
                        num_labels, Ex, y_onehot, reg),
                        method='TNC', jac=True,
                        options={'maxiter': iters})

                theta1 = np.reshape(fmin.x[:hidden_size*(input_size + 1)],(hidden_size,(input_size + 1)))
                theta2 = np.reshape(fmin.x[hidden_size * (input_size+1):],(num_labels,(hidden_size + 1)))

                p1 = prueba_neurona(Vx, Vy, theta1, theta2)
                print("validación = {}".format(p1))
                p2 = prueba_neurona(Px, Py, theta1, theta2)
                print("prueba = {}".format(p2))
                percent1.append(p1)
                percent2.append(p2)

            plt.plot(iteraciones, percent1, c = colores[i] , label = 'validación')
            plt.plot(iteraciones, percent2, c = colores[i + 1] , label = 'prueba')
            i = i+1

        plt.legend()
        plt.title("hidden sizes: {}".format(hidden_size))
        plt.show()

#### Para redes de dos capas ocultas(luego hare una para un numero de capas ocultas cualquiera)

def coste2(theta1, theta2, theta3, X, Y, num_etiquetas):  # Y preparada

    A1, A2, A3, h = forward_prop2(X, theta1, theta2, theta3)

    sum1 = Y * np.log(h)
    sum2 = (1 - Y) * np.log(1 - h + 1e-6)

    return (-1 / X.shape[0]) * np.sum(sum1 + sum2)

def coste_reg2(theta1, theta2, theta3, X, Y, num_etiquetas, Lambda):
    c = coste2(theta1, theta2, theta3, X, Y, num_etiquetas)
    m = X.shape[0]

    e = sum(sum(theta1[:, 1:] ** 2)) + sum(sum(theta2[:, 1:] ** 2)) + sum(sum(theta3[:, 1:] ** 2))

    return c + (Lambda / (2 * m)) * e

def forward_prop2(X, theta1, theta2, theta3):
    n = X.shape[0]

    # Se añade una fila de unos a la matriz inicial
    X = np.hstack([np.ones([n, 1]), X])

    # Las capas ocultas utilizan la primera y segunda matrices de pesos para crear sus neuronas y les añaden una fila de unos
    Oculta1 = fun.sigmoide(np.dot(X, theta1.T))
    Oculta1 = np.hstack([np.ones([n, 1]), Oculta1])

    Oculta2 = fun.sigmoide(np.dot(Oculta1, theta2.T))
    Oculta2 = np.hstack([np.ones([n, 1]), Oculta2])

    # El resultado se calcula pasando por la segunda matriz de pesos todas las neuronas de la capa oculta
    Resultado = fun.sigmoide(np.dot(Oculta2, theta3.T))

    return X, Oculta1, Oculta2, Resultado

def gradiente2(theta1, theta2, theta3, X, y):
    # Creamos los Delta con la forma de theta pero inicializados a cero
    Delta1 = np.zeros(np.shape(theta1))
    Delta2 = np.zeros(np.shape(theta2))
    Delta3 = np.zeros(np.shape(theta3))

    m = len(y)

    # Se realiza la propagación hacia delante
    A1, A2, A3, h = forward_prop2(X, theta1, theta2, theta3)

    # Se realiza la propagación hacia atras para cada
    # elemento para comprobar el fallo
    for k in range(m):
        a1k = A1[k, :]
        a2k = A2[k, :]
        a3k = A3[k, :]
        a4k = h[k, :]
        yk = y[k, :]

        d4 = a4k - yk
        g_prima = (a3k * (1 - a3k))
        d3 = np.dot(theta3.T, d4) * g_prima

        g_prima = (a2k * (1 - a2k))
        d2 = np.dot(theta2.T, d3[1:]) * g_prima

        Delta1 = Delta1 + np.dot(d2[1:, np.newaxis], a1k[np.newaxis, :])
        Delta2 = Delta2 + np.dot(d3[1:, np.newaxis], a2k[np.newaxis, :])
        Delta3 = Delta3 + np.dot(d4[:, np.newaxis], a3k[np.newaxis, :])

    # Se devuelven los Deltas que corresponden al gradiente
    return Delta1 / m, Delta2 / m, Delta3 / m

def gradiente_reg2(theta1, theta2, theta3, X, y, Lambda):
    m = len(y)
    Delta1, Delta2, Delta3 = gradiente2(theta1, theta2, theta3, X, y)

    # A cada elemento del gradiente (menos la primera columna) se le añade el termino de regularización Lambda
    # multiplicado por cada elemento de las matriz theta 1 y theta2
    Delta1[:, 1:] = Delta1[:, 1:] + (Lambda / m) * theta1[:, 1:]
    Delta2[:, 1:] = Delta2[:, 1:] + (Lambda / m) * theta2[:, 1:]
    Delta3[:, 1:] = Delta3[:, 1:] + (Lambda / m) * theta3[:, 1:]

    return Delta1, Delta2, Delta3

def backprop2(params_rn, num_entradas, num_ocultas1, num_ocultas2, num_etiquetas, X, y, reg):
    # backprop pero con dos capas ocultas

    pos = (num_ocultas1 * (num_entradas + 1)) + (num_ocultas2 * (num_ocultas1 + 1))

    theta1 = np.reshape(params_rn[:num_ocultas1 * (num_entradas + 1)], (num_ocultas1, (num_entradas + 1)))
    theta2 = np.reshape(params_rn[num_ocultas1 * (num_entradas + 1): pos ], (num_ocultas2, (num_ocultas1 + 1)))
    theta3 = np.reshape(params_rn[pos :], (num_etiquetas, (num_ocultas2 + 1)))

    m = len(y)

    D1, D2, D3 = gradiente_reg2(theta1, theta2, theta3, X, y, reg)

    coste = coste_reg2(theta1, theta2, theta3, X, y, num_etiquetas, reg)

    gradiente = np.concatenate((np.ravel(D1), np.ravel(D2), np.ravel(D3)))

    return coste, gradiente

def prueba_neurona2(X, y, theta1, theta2, theta3):
    """función que devuelve el porcentaje de acierto de una red neuronal utilizando unas matrices de pesos dadas"""
    n = len(y)

    y = np.ravel(y)

    _, _, _, result = forward_prop2(X, theta1, theta2, theta3)

    result = np.argmax(result, axis=1)

    return (sum((result + 1)%4 == y) / n * 100)

def validacion_redes2(random_state, num_labels, iteraciones, hiddens1, hiddens2, lambdas, colores = ['r', 'b' , 'g', 'm']):

    Ex, Ey, Vx, Vy, Px, Py = ld.carga_Numpy(random_state)

    y_onehot = fun.one_hot(Ey, 4)

    input_size = Ex.shape[1]

    INIT_EPSILON = 0.12

    for hidden_size1 in hiddens1:

        for hidden_size2 in hiddens2:

            if hidden_size1 >= hidden_size2:

                theta1 = np.random.random((hidden_size1,(input_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
                theta2 = np.random.random((hidden_size2,(hidden_size1 + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
                theta3 = np.random.random((num_labels,(hidden_size2 + 1)))*(2*INIT_EPSILON) - INIT_EPSILON

                params = np.concatenate((np.ravel(theta1), np.ravel(theta2), np.ravel(theta3)))

                plt.figure()

                i = 0

                for reg in lambdas:

                    percent = []

                    for iters in iteraciones:

                        fmin = minimize(fun=backprop2, x0=params,
                                args=(input_size, hidden_size1, hidden_size2,
                                num_labels, Ex, y_onehot, reg),
                                method='TNC', jac=True,
                                options={'maxiter': iters})

                        pos = (hidden_size1 * (input_size + 1)) + (hidden_size2 * (hidden_size1 + 1))

                        theta1 = np.reshape(fmin.x[:hidden_size1 * (input_size + 1)], (hidden_size1, (input_size + 1)))
                        theta2 = np.reshape(fmin.x[hidden_size1 * (input_size + 1): pos ], (hidden_size2, (hidden_size1 + 1)))
                        theta3 = np.reshape(fmin.x[pos :], (num_labels, (hidden_size2 + 1)))

                        p = prueba_neurona2(Vx, Vy, theta1, theta2, theta3)
                        print(p)
                        percent.append(p)

                    plt.plot(iteraciones, percent, c = colores[i] , label = ' lambda = {} '.format(reg))
                    i = i+1
                plt.title("hidden sizes: {}, {}".format(hidden_size1, hidden_size2))
                plt.legend()
                plt.show()

def prueba_redes2(random_state, num_labels, iteraciones, hiddens1, hiddens2, lambdas, colores = ['r', 'b' , 'g', 'm']):

    Ex, Ey, Vx, Vy, Px, Py = ld.carga_Numpy(random_state)

    y_onehot = fun.one_hot(Ey, 4)

    input_size = Ex.shape[1]

    INIT_EPSILON = 0.12

    for hidden_size1 in hiddens1:

        for hidden_size2 in hiddens2:

            if hidden_size1 >= hidden_size2:

                theta1 = np.random.random((hidden_size1,(input_size + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
                theta2 = np.random.random((hidden_size2,(hidden_size1 + 1)))*(2*INIT_EPSILON) - INIT_EPSILON
                theta3 = np.random.random((num_labels,(hidden_size2 + 1)))*(2*INIT_EPSILON) - INIT_EPSILON

                params = np.concatenate((np.ravel(theta1), np.ravel(theta2), np.ravel(theta3)))

                plt.figure()

                i = 0

                for reg in lambdas:

                    percent1 = []
                    percent2 = []

                    for iters in iteraciones:

                        fmin = minimize(fun=backprop2, x0=params,
                                args=(input_size, hidden_size1, hidden_size2,
                                num_labels, Ex, y_onehot, reg),
                                method='TNC', jac=True,
                                options={'maxiter': iters})

                        pos = (hidden_size1 * (input_size + 1)) + (hidden_size2 * (hidden_size1 + 1))

                        theta1 = np.reshape(fmin.x[:hidden_size1 * (input_size + 1)], (hidden_size1, (input_size + 1)))
                        theta2 = np.reshape(fmin.x[hidden_size1 * (input_size + 1): pos ], (hidden_size2, (hidden_size1 + 1)))
                        theta3 = np.reshape(fmin.x[pos :], (num_labels, (hidden_size2 + 1)))

                        p1 = prueba_neurona2(Vx, Vy, theta1, theta2, theta3)
                        print("validación = {}".format(p1))
                        p2 = prueba_neurona2(Px, Py, theta1, theta2, theta3)
                        print("prueba = {}".format(p2))
                        percent1.append(p1)
                        percent2.append(p2)

                    plt.plot(iteraciones, percent1, c = colores[i] , label = 'validación')
                    plt.plot(iteraciones, percent2, c = colores[i + 1] , label = 'prueba')
                    i = i+1
                plt.title("hidden sizes: {}, {}".format(hidden_size1, hidden_size2))
                plt.legend()
                plt.show()

reason: import numpy,import scipy,from scipy


text:
import cv2
import numpy as np

#Step Background Extraction
cap = cv2.VideoCapture('highway.mp4')
_,f=cap.read()
avg = np.float32(f)
i=400 #Variable used for no. of frames
#We got clear Background for highway.mp4 video by taking 400 frames,but this can vary video to video.
while(i):
    ret, img = cap.read()
    cv2.accumulateWeighted(img,avg,0.01)
    final = cv2.convertScaleAbs(avg)
    i-=1

cv2.imshow('Final', final) #final output
cv2.imwrite('BG1.jpg',final) #saving output image
cap.release()
cv2.waitKey(0)
cv2.destroyAllWindows()


text:
import math
pi = math.pi
##from math import pi

##Constants:
h=6.626*10**(-34)
c=3*10**8
Kb=1.38*10**(-23)

##Session where I define all the function needed
def Planck(x,e,T):
    a=np.expm1(0.0144043/(x*10**(-9))/T)
    P=e/(x*10**(-9))**5*3.74691*10**(-16)*1/(a-1)
    return P
#Defined Wien function
def Wien(Int,x):
    W=Kb/h/c*np.log((x*10**(-9))**5*Int/2/pi/h/c**2)
    return W
#Defined two-colour function
def TwoCol(Int,x):
    count=len(x)
    delta=200
    k=count-delta
    TTwo=[]*count
    #while (i<k):
    for i in range (0,k):#(0,count-1):

        f1=1/(x[i]*10**(-9))
        f2=1/(x[i+delta]*10**(-9))
        i1=np.log(Int[i]/2/pi/h/c**2/f1**5)*Kb/h/c
        i2=np.log(Int[i+delta]/2/pi/h/c**2/f2**5)*Kb/h/c
        TTwo.append(abs((f2-f1)/(i2-i1)))
    for i in range (k,count):
        a = float('nan')
        TTwo.append(a)
    return TTwo
#Defined linear fit for Wien function
def FWien(x,e,T):
    a=1/T
    b=Kb/h/c*np.log(e)
    W=b-a*x
    return W
#Defined Gauss fit
def gaus(x, a, x0, sigma):
    return np.real(a*np.exp(-(x-x0)**2/(2*sigma**2)))

#Programma che carica un file di temperatura (raw) lo normalizza e me lo
#grafica in un plot per ottenere la mia funzione di Planck
from scipy.optimize import curve_fit
import numpy as np
import matplotlib.pyplot as plt
import sys

#standard integration range(i)  315-->1146
start = 315
end= 800

#Session where I load the file I need from the folder where the software is
x,y=np.loadtxt('../../test/Lucky/testData/T_62_1.txt',unpack=True) ##Raw file
xC,yC=np.loadtxt('../../test/Lucky/testData/Calib.txt',unpack=True) ##Calib file

#Session with all the normalization for the collected data
P=Planck(x,1,2436) ##Ideal Planck
#TPlanck ID24 =2436
#TPlanck I15 = 3200
Norm=y/yC*P #Normalization file
invX=1/x*10**9 #Inverse of wavelength for Wien function
W=Wien(Norm,x)#Define Wien function
#test=W
#for w in W:
#   test[numpy.isnan(w)]=[]
Two=TwoCol(Norm,x)
#array = np.array(a,dtype='float')
Two2=np.array(Two,dtype='float')
TwoInt=Two2[start:end]
bins=range(1000,3000,1)
hist=np.histogram(TwoInt,bins,density=False)
freq=np.array(hist[0])
control=len(hist[1])-1
value=np.array(np.delete(hist[1],control,0))
#del value[-1]
#xhist=numpy.histogram(x,density=False)

#Session where we save in the same folder .txt files with the new data
np.savetxt('TwoCol.txt',Two)
np.savetxt('Norm.txt',Norm)
np.savetxt('Wien.txt',W)
np.savetxt('frequence.txt',hist[0])

#Session dedicated to all the needed fits
#Planck fit
p0=[1,2000]
#Fit Planck in the range [start:end]
bestP,covarP = curve_fit(Planck, x[start:end], Norm[start:end], p0)
TP=round(bestP[1],2)#Save Planck Temperature
eP=bestP[0]#Save planck Emissivity
xp=x[start:end]
FP=Planck(xp,eP,TP)#Create the new Planck with the fit parameters
PRes=abs(Norm[start:end]-FP)#Planck Residual

#Wien fit
invX1=invX[start:end]
W1=W[start:end]
#Fit Wien and control that there are no inf or nan arguments in the fit
bestW,covarW = curve_fit(FWien,invX1[(np.isfinite(W1))],W1[(np.isfinite(W1))],p0=[1,TP])
Residual=W1-FWien(invX1[(np.isfinite(W1))],*bestW)
#Save Wien temperature
TW=round(bestW[1])

#Gaussian fit to the histogram two-colours
popt,pcov = curve_fit(gaus,value,freq,p0=[1000,TP,100])
Thist=round(popt[1],2)#Save Histogram temperature
errTot=round(popt[2])#Save hiistogram FWHM
#create a file with all the results needed
Results=(TP,errTot,TW,abs(((TP-TW)/TP*100)),Thist)
np.savetxt('Results.txt',Results,newline='\n',header="Name Planck STD Wien errPerc Hist")
print 'Planck:',TP,'K ','+-',errTot,'K ','Wien:',TW,'K','Hist:',Thist,'K'

#Defines the method for plotting all the graphs
def plots(x,y,yC,Norm,xp,FP,PFrom,PTo,invX,W,invX1,bestW,Two2,value,freq,popt,TwoInt,Residual,TP):
    fig=plt.figure(figsize=(8,11))#Defines dimension of the figure

    #Adding subplots to show
    ax1 = fig.add_subplot(3, 2, 1)
    ax2 = fig.add_subplot(3, 2, 2)
    ax3 = fig.add_subplot(3, 2, 3)
    ax4 = fig.add_subplot(3, 2, 4)
    ax5 = fig.add_subplot(3, 2, 5)
    plt.subplots_adjust(wspace=0.3,hspace=0.3)
    #Raw and calibration data subgraph
    ax1.plot(x, y, x, yC,'red')
    ax1.set_title('Raw vs Calib data')
    ax1.set_xlabel('wavelength (nm)')
    ax1.set_ylim(0,50000)
    ax1.grid(True, linestyle='-')
    ticklines = ax1.get_xticklines()
    ticklines.extend( ax1.get_yticklines() )
    gridlines = ax1.get_xgridlines()
    gridlines.extend( ax1.get_ygridlines() )
    ticklabels = ax1.get_xticklabels()
    ticklabels.extend( ax1.get_yticklabels() )

#     for line in ticklines:
#         line.set_linewidth(3)
#
#     for line in gridlines:
#         line.set_linestyle('-')
#
#     for label in ticklabels:
#         label.set_color('black')
#         label.set_fontsize('medium')

#     txt=plt.text(4500,33,TP)
#     txt1=plt.text(4200,33,'T=')
#     txt2=plt.text(2000,17,TW)
#     txt3=plt.text(1800,17,'T=')
#     txt.set_size(15)
#     txt1.set_size(15)
#     txt2.set_size(15)
#     txt3.set_size(15)
#     fig.canvas.draw()

    #Planck subgraph
    ax2.plot(x, Norm, xp, FP,'red')
    ax2.set_title('Planck')
    ax2.set_xlabel('wavelength (nm)')
    ax2.set_xlim(PFrom,PTo)
    ax2.set_yticks([])
    #ax2.grid(True)
    def on_button_press(event):
        #print dir(event)
        #print "BADGER"
        #print "Button:", event.button
        #print "Figure coordinates:", event.x, event.y
        print "Data coordinates:", event.xdata, event.ydata
        #start=event.xdata
        sys.stdout.flush()

    #Wien subgraph
    ax3.plot(invX,W,invX1,FWien(invX1,*bestW),'red',invX1,Residual)
    ax3.set_title('Wien')
    ax3.set_xlabel('1/wavelength (1/m)')
    ax3.set_ylabel("Wien function")
    ax3.set_xlim(10**9/PTo,10**9/PFrom)
    ax3.set_yticks([])
    #ax3.grid(True)

    #Two Colours subgraph
    ax4.plot(x,Two2,x[start:end],TwoInt,'red')
    ax4.set_title('Sliding Two-Colours')
    ax4.set_xlabel('wavelength (nm)')
    ax4.set_ylabel('T (K)')
    ax4.set_xlim(PFrom,PTo)
    ax4.grid(True)
    ticklines4 = ax4.get_xticklines()
    ticklines4.extend( ax4.get_yticklines() )
    gridlines4 = ax4.get_xgridlines()
    gridlines4.extend( ax4.get_ygridlines() )
    ticklabels4 = ax4.get_xticklabels()
    ticklabels4.extend( ax4.get_yticklabels() )

    for line in ticklines4:
        line.set_linewidth(3)

    for line in gridlines4:
        line.set_linestyle('-')

    for label in ticklabels4:
        label.set_color('black')
        label.set_fontsize('medium')

    #Histogram subgraph
    ax5.plot(value,freq,value,gaus(value,*popt),'red')
    ax5.set_title('Histogram')
    ax5.set_xlabel('T(K)')
    ax5.set_ylabel('# Counts')

    #pylab.show() #it plots everything
    fig.canvas.mpl_connect('button_press_event', on_button_press)
    plt.show()

#Plot range
PFrom=500
PTo=1000
plots(x,y,yC,Norm,xp,FP,PFrom,PTo,invX,W,invX1,bestW,Two2,value,freq,popt,TwoInt,Residual,TP)

max_stars_repo_name: ruggiamp/Lucky

max_issues_repo_name: ruggiamp/Lucky

max_forks_repo_name: ruggiamp/Lucky

reason: import numpy,from scipy


text:
import sys
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
import logging
logging.basicConfig(format = '[%(asctime)s]  %(message)s', level = logging.INFO)

from model import get_model
from utils import crps, real_to_cdf, preprocess, rotation_augmentation, shift_augmentation

seed = 1377713

def load_train_data():
    """
    Load training data from .npy files.
    """
    X = np.load('../input/X_train.npy')
    y = np.load('../input/y_train.npy')

    X = X.astype(np.float32)
    X /= 255

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(y)

    return X, y

def split_data(X, y, split_ratio=0.2):
    """
    Split data into training and testing.

    :param X: X
    :param y: y
    :param split_ratio: split ratio for train and test data
    """
    split = X.shape[0] * split_ratio
    X_test = X[:split, :, :, :]
    y_test = y[:split, :]
    X_train = X[split:, :, :, :]
    y_train = y[split:, :]

    return X_train, y_train, X_test, y_test

def train():
    """
    Training systole and diastole models.
    """
    logging.info('Loading and compiling models...')
    model_systole = get_model()
    model_diastole = get_model()

    logging.info('Loading training data...')
    X, y = load_train_data()

    logging.info('Pre-processing images...')
    X = preprocess(X)

    # split to training and test
    X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.15)

    nb_iter = 300
    epochs_per_iter = 1
    batch_size = 32
    calc_crps = 1  # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed)

    # remember min val. losses (best iterations), used as sigmas for submission
    min_val_loss_systole = sys.float_info.max
    min_val_loss_diastole = sys.float_info.max

    logging.info('-'*50)
    logging.info('Training...')
    logging.info('-'*50)

    for i in range(nb_iter):
        logging.info('-'*50)
        logging.info('Iteration {0}/{1}'.format(i + 1, nb_iter))
        logging.info('-'*50)

        logging.info('Augmenting images - rotations')
        X_train_aug = rotation_augmentation(X_train, 20)
        logging.info('Augmenting images - shifts')
        X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
        logging.info('Fitting systole model...')
        hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter,
                                         batch_size=batch_size, validation_data=(X_test, y_test[:, 0]))

        logging.info('Fitting diastole model...')
        hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter,
                                           batch_size=batch_size, validation_data=(X_test, y_test[:, 1]))

        # sigmas for predicted data, actually loss function values (RMSE)
        loss_systole = hist_systole.history['loss'][-1]
        loss_diastole = hist_diastole.history['loss'][-1]
        val_loss_systole = hist_systole.history['val_loss'][-1]
        val_loss_diastole = hist_diastole.history['val_loss'][-1]

        if calc_crps > 0 and i % calc_crps == 0:
            logging.info('Evaluating CRPS...')
            pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1)
            pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1)
            val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1)
            val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1)

            # CDF for train and test data (actually a step function)
            cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
            cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))

            # CDF for predicted data
            cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
            cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
            cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
            cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)

            # evaluate CRPS on training data
            crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
            logging.info('CRPS(train) = {0}'.format(crps_train))

            # evaluate CRPS on test data
            crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
            logging.info('CRPS(test) = {0}'.format(crps_test))

        logging.info('Saving weights...')
        # save weights so they can be loaded later
        model_systole.save_weights('../models/weights/weights_systole_1.hdf5', overwrite=True)
        model_diastole.save_weights('../models/weights/weights_diastole_1.hdf5', overwrite=True)

        # for best (lowest) val losses, save weights
        if val_loss_systole < min_val_loss_systole:
            min_val_loss_systole = val_loss_systole
            model_systole.save_weights('../models/weights/weights_systole_best_1.hdf5', overwrite=True)

        if val_loss_diastole < min_val_loss_diastole:
            min_val_loss_diastole = val_loss_diastole
            model_diastole.save_weights('../models/weights/weights_diastole_best_1.hdf5', overwrite=True)

        # save best (lowest) val losses in file (to be later used for generating submission)
        with open('./logs/val_loss.txt_1', mode='w+') as f:
            f.write(str(min_val_loss_systole))
            f.write('\n')
            f.write(str(min_val_loss_diastole))

train()

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle


text:
import numpy as np
import time

import neuron.ops as ops
from neuron.models.model import Model

__all__ = ['Tracker', 'OxUvA_Tracker']

class Tracker(Model):

    def __init__(self, name, is_deterministic=True,
                 input_type='image', color_fmt='RGB'):
        assert input_type in ['image', 'file']
        assert color_fmt in ['RGB', 'BGR', 'GRAY']
        super(Tracker, self).__init__()
        self.name = name
        self.is_deterministic = is_deterministic
        self.input_type = input_type
        self.color_fmt = color_fmt

    def init(self, img, init_bbox):
        self.init(img, init_bbox)

    def update(self, img):
        bboxes = self.update(img)
        return bboxes

    def forward_test(self, img_files, init_bbox, visualize=False):
        # state variables
        frame_num = len(img_files)
        bboxes = np.zeros((frame_num, 4))
        bboxes[0] = init_bbox
        times = np.zeros(frame_num)

        for f, img_file in enumerate(img_files):
            if self.input_type == 'image':
                img = ops.read_image(img_file, self.color_fmt)
            elif self.input_type == 'file':
                img = img_file

            begin = time.time()
            if f == 0:
                self.init(img, init_bbox)
            else:
                bboxes[f, :] = self.update(img)
            times[f] = time.time() - begin

            if visualize:
                ops.show_image(img, bboxes[f, :])

        return bboxes, times

class OxUvA_Tracker(Tracker):

    def update(self, img):
        r'''One needs to return (bbox, score, present) in
            function `update`.
        '''
        raise NotImplementedError

    def forward_test(self, img_files, init_bbox, visualize=False):
        frame_num = len(img_files)
        times = np.zeros(frame_num)
        preds = [{
            'present': True,
            'score': 1.0,
            'xmin': init_bbox[0],
            'xmax': init_bbox[2],
            'ymin': init_bbox[1],
            'ymax': init_bbox[3]}]

        for f, img_file in enumerate(img_files):
            if self.input_type == 'image':
                img = ops.read_image(img_file, self.color_fmt)
            elif self.input_type == 'file':
                img = img_file

            begin = time.time()
            if f == 0:
                self.init(img, init_bbox)
            else:
                bbox, score, present = self.update(img)
                preds.append({
                    'present': present,
                    'score': score,
                    'xmin': bbox[0],
                    'xmax': bbox[2],
                    'ymin': bbox[1],
                    'ymax': bbox[3]})
            times[f] = time.time() - begin

            if visualize:
                ops.show_image(img, bboxes[f, :])

        # update the preds as one-per-second
        frame_stride = 30
        preds = {f * frame_stride: pred for f, pred in enumerate(preds)}

        return preds, times

max_stars_repo_name: tsingqguo/ABA

max_issues_repo_name: tsingqguo/ABA

max_forks_repo_name: tsingqguo/ABA


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from collections import Counter
from sklearn import preprocessing

df = pd.read_csv('UIDAI.csv', dtype = object )

X = df[['Registrar', 'EnrolmentAgency', 'State', 'District', 'SubDistrict', 'PIN', 'Gender', 'Age', 'AadhaarGenerated', 'EnrolmentRejected', 'ResidentsProvidingEmail', 'ResidentsProvidingMobile']]
'''
X1 = X[['Registrar']].values
X1 = X1.ravel()
X1 = Counter(X1)
print(X1)

labels = ['CSC e-Gov. Services India Ltd.', 'NSDL e-Governance Infrastructure Ltd.' , 'DENA BANK' , 'Bank Of India' , 'MP State Electronics Dev. Corp. Ltd.  ' , 'Tamil Nadu eGovernance Agency' , 'Govt of Gujarat' , 'Dept of ITC Govt of Rajasthan' , 'Govt of Kerala' , 'Union Bank' , 'Others']
sizes = [209771,54214,33869,19791,17309,15468,13894,13565,11937,5536,45454]
explode = (0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2)

plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=0)
plt.legend(labels=labels)
plt.title('Registrars')
plt.show()

# Most Used Registrar: CSC e-Governance Services India Limited (2,09,771 Enrolled)
# Least Used Registrar: DIT Lakshadweep and UT of Puducherry (1 Person Enrolled for each)

X2 = X[['EnrolmentAgency']].values
X2 = X2.ravel()
X2 = Counter(X2)
print(X2)

labels = ['CSC SPV' , 'SRM Educ & Social Welfare Society' , 'SREI INFRASTRUCTURE FINANCES L' , 'Rajcomp Info Services Ltd', 'AKSH OPTIFIBRE LIMITED' , 'TN ARASU CABLE TV CORP. LTD' , 'Akshaya' , 'MPOnline Limited' , 'CMS Computers Ltd' , 'IAP COMPANY Pvt. Ltd', 'Others']
sizes = [100357,18101,16972,12910,12580,12114,11937,10808,9229,7537,228273]
explode = (0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('EnrolmentAgencies')
plt.show()

# Most Popular Enrollment Agency: CSC SPV (1,00,357 Entries)
# Least Popular Enrollment Agency: District IT Society Jind, District Family and Welfare Society  Jhajjar, SHRIKRISHNA KHANDASARI SUGAR M, Quick Data IT Services Pvt Ltd, District Health and Family Welfare Society Fatehabad, DIT Lakshadweep, Planning and Research Department, Integrated Systems & Services, VIKALP MULTIMEDIA, WEBEL, District Sukhmani Society Amritsar Punjab, District Family & Welfare Society Palwal (1 Entry for each)

X3 = X[['State']].values
X3 = X3.ravel()
X3 = Counter(X3)
print(X3)

labels = ['Bihar', 'Uttar Pradesh', 'West Bengal', 'Madhya Pradesh', 'Rajasthan', 'Gujarat', 'Tamil Nadu', 'Maharashtra', 'Karnataka', 'Kerala', 'Others']
sizes = [81776,69476,60485,37360,28659,24146,21196,19783,15755,12378,69805]
explode = (0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('States')
plt.show()

# Top State of Enrollment: Bihar (81776 Entries)
# Lowest State of Enrollment: Lakshadweep (5 Entries)

X4 = X[['Gender']].values
X4 = X4.ravel()
X4 = Counter(X4)
print(X4)

labels = ['Male', 'Female', 'UnDeclared']
sizes = [292798,148013,500]
explode = (0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('Gender')
plt.show()

# Male: 2,92798
# Female: 1,48,013
# UnDeclared: 7

X5 = X[['EnrolmentRejected']].values
X5 = X5.ravel()
X5 = Counter(X5)
print(X5)

labels = ['0', '1','>1']
sizes = [409275,27884,3669]
explode = (0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('Enrollments Rejected')
plt.show()

X6 = X[['ResidentsProvidingEmail']].values
X6 = X6.ravel()
X6 = Counter(X6)
print(X6)

labels = ['0', '1','>1']
sizes = [422997,16763,1058]
explode = (0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('Emails Provided')
plt.show()

X7 = X[['ResidentsProvidingMobile']].values
X7 = X7.ravel()
X7 = Counter(X7)
print(X7)

labels = ['0', '1', '2', '>2']
sizes = [112629, 267748,36068,24373]
explode = (0.1, 0.1, 0.1, 0.2)
plt.pie(sizes, autopct='%1.1f%%', explode=explode, startangle=180)
plt.legend(labels=labels)
plt.title('Mobile Numbers Provided')
plt.show()

X8 = X[['Age']].values
X8 = X8.ravel()
X8 = Counter(X8)
print(X8)

# Youngest <1 Year Old (16,797)
# Oldest 101 Years Old (1)
# Most Enrolled Age 4 (24,040)

X9 = X[['District']].values
X9 = X9.ravel()
X9 = Counter(X9)

print(X9)
'''

max_stars_repo_name: Malayanil/UIDAI-Dataset-Analysis

max_issues_repo_name: Malayanil/UIDAI-Dataset-Analysis

max_forks_repo_name: Malayanil/UIDAI-Dataset-Analysis


text:
import numpy as np

import matplotlib.pyplot as plt

def f(x, y): return (1 - x / 2 + x ** 5 + y ** 3) * np.exp(-x ** 2 - y ** 2)

n = 10
x = np.linspace(-3, 3, 4 * n)
y = np.linspace(-3, 3, 3 * n)
X, Y = np.meshgrid(x, y)
plt.imshow(f(X, Y))
plt.show()

max_stars_repo_name: alanland/ml-pratice

max_issues_repo_name: alanland/ml-pratice

max_forks_repo_name: alanland/ml-pratice


text:
# -*- coding: utf-8 -*-
import logging
import timeit

import click
import numpy as np
import torch
import torch.nn as nn
import torchvision.models as models
from efficientnet_pytorch import EfficientNet
from joblib import dump
from src.data.preprocess_data import DatasetManager
from src.features.dimension_reduction import extract_cnn_features
from src.models.aws_utils import load_torch_model, load_traditional_model
from torchvision import transforms

BATCH_SIZE = 8 # Lower batch size if GPU memory is lacking!
INPUT_SIZE = 224
N_FEATURES = 7
VAL_SIZE = 0.2
TEST_SIZE = 0.2

DATA_PATH = 'data/processed'
RESNET_NAME = 'resnet-50.pt'
MODEL_PATH = 'models/'

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def _create_cnn_embeddings(model):
    """
    Extract features, and save them locally.
    This needs to be done only once, and later we can load the features from disk.

    Parameters
    ----------
    model : torchvision.models
        Must inherit torch.nn.Module. In other words must have forward().
    """
    print(f"Creating CNN embeddings with device: {device}")
    start_t = timeit.default_timer()

    test_preprocess = transforms.Compose([
    transforms.ToTensor(),
    transforms.Resize(INPUT_SIZE),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])

    dataset_manager = DatasetManager(batch_size=BATCH_SIZE, test_size=TEST_SIZE,
                        validation_size=VAL_SIZE, transform=test_preprocess,
                        test_transform=test_preprocess)
    train_loader, test_loader, val_loader = dataset_manager.load_dataloaders()
    # Train
    embeddings, labels = extract_cnn_features(model, train_loader, BATCH_SIZE)
    np.save('data/processed/y_train_labels.npy', np.vstack(labels))
    np.save('data/processed/X_train_embeddings.npy', np.vstack(embeddings))
    # Test
    embeddings, labels = extract_cnn_features(model, test_loader, BATCH_SIZE)
    np.save('data/processed/y_test_labels.npy', np.vstack(labels))
    np.save('data/processed/X_test_embeddings.npy', np.vstack(embeddings))
    # Val
    embeddings, labels = extract_cnn_features(model, val_loader, BATCH_SIZE)
    np.save('data/processed/y_val_labels.npy', np.vstack(labels))
    np.save('data/processed/X_val_embeddings.npy', np.vstack(embeddings))

    print(f"CNN embedding saving done. Time taken: {timeit.default_timer() - start_t}s.")

def _create_model():
    """
    Create model that is used for generating embeddings.

    Returns
    -------
    resnet : EfficientNet model
    """
    model = EfficientNet.from_pretrained('efficientnet-b0')
    model.to(device)
    model.eval()

    return model

def _load_checkpoint_to_local():
    """
    Load model checkpoint from S3, and save it to local directory.
    """
    print("Loading model checkpoint from S3 to local repository..")
    start_t = timeit.default_timer()
    checkpoint = load_torch_model(RESNET_NAME)
    print(f"Model checkpoint loaded in {timeit.default_timer() - start_t}")

    checkpoint['model_state_dict']
    torch.save(checkpoint, MODEL_PATH + RESNET_NAME)
    print("Model checkpoint saved!")

def _load_traditional_models_to_local():
    """
    Loads traditional models from AWS S3 to local storage.
    Files are stored in ~/models/*_features/ depending on embedding types.
    Types are in ['cnn', 'pca', 'hog'].
    """
    print("Loading CNN models from S3 to local repository..")
    cnn_models = load_traditional_model('cnn')
    print("Loading PCA models from S3 to local repository..")
    pca_models = load_traditional_model('pca')
    print("Loading HOG models from S3 to local repository..")
    hog_models = load_traditional_model('hog')

    print("Saving CNN models to models/cnn_features..")
    [dump(model, f'models/cnn_features/{name}') for (model, name) in cnn_models]
    print("Saving PCA models to models/pca_features..")
    [dump(model, f'models/pca_features/{name}') for (model, name) in pca_models]
    print("Saving HOG models to models/hog_features..")
    [dump(model, f'models/hog_features/{name}') for (model, name) in hog_models]

    print("Traditional model saving done!")

def main():
    """
    Generates CNN embeddings and loads model checkpoint from S3 to local.
    """
    model = _create_model()
    _create_cnn_embeddings(model)
    _load_checkpoint_to_local()
    _load_traditional_models_to_local()

    print("Data processing and loading has been done.")
    print("You can now start using the notebooks!")

if __name__ == '__main__':
    main()

max_stars_repo_name: TeemuSo/Loop-Q-prize

max_issues_repo_name: TeemuSo/Loop-Q-prize

max_forks_repo_name: TeemuSo/Loop-Q-prize