text:
import math
import torch
import random
import numpy as np
import torch.nn as nn
from numpy import int64 as int64
import torchvision.transforms as transforms
from imgaug import BoundingBox, BoundingBoxesOnImage
import imgaug.augmenters as iaa
from PIL import Image, ImageOps, ImageFilter

class Normalize(object):
    """Normalize a tensor image with mean and standard deviation.
    Args:
        mean (tuple): means for each channel.
        std (tuple): standard deviations for each channel.
    """

    def __init__(self, mean=(0., 0., 0.), std=(1., 1., 1.)):
        self.mean = mean
        self.std = std

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        img = np.array(img).astype(np.float32)
        mask = np.array(mask).astype(np.float32)
        img /= 255.0
        img -= self.mean
        img /= self.std

        return {'image': img,
                'label': mask}

class ToTensor(object):
    """Convert ndarrays in sample to Tensors."""

    def __call__(self, sample):
        # swap color axis because
        # numpy image: H x W x C
        # torch image: C X H X W
        img = sample['image']
        mask = sample['label']
        img = np.array(img).astype(np.float32).transpose((2, 0, 1))
        mask = np.array(mask).astype(np.float32)

        img = torch.from_numpy(img).float()
        mask = torch.from_numpy(mask).float()

        return {'image': img,
                'label': mask}

class RandomHorizontalFlip(object):
    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        if random.random() < 0.5:
            img = img.transpose(Image.FLIP_LEFT_RIGHT)
            mask = mask.transpose(Image.FLIP_LEFT_RIGHT)

        return {'image': img,
                'label': mask}

class Translate_Y(object):
    def __init__(self, v):
        self.v = v # -0.3 - 0.3 ??

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.geometric.TranslateY(percent=self.v)
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

#TODO: Enhance this function so bounding boxes will account for change in actual object,
# i.e. if the translateY bbox moves the object up, the lower limit of the bbox should move up
class Translate_Y_BBoxes(object):
    def __init__(self, v):
        self.v = v

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels, foreground=iaa.geometric.TranslateY(percent=self.v))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Translate_X(object):
    def __init__(self, v):
        self.v = v

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.geometric.TranslateX(percent=self.v)
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Translate_X_BBoxes(object):
    def __init__(self, v):
        self.v = v

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels, foreground=iaa.geometric.TranslateX(percent=self.v))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class CutOut(object):
    def __init__(self, v):
        self.v = v # between 6 - 20

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.Cutout(nb_iterations=int(round(self.v)), size=0.05, fill_mode="gaussian")
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class CutOut_BBoxes(object):
    def __init__(self, v):
        self.v = v #self.v should be between 6 - 20

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.Cutout(nb_iterations=int(round(self.v)), size=0.05, fill_mode="gaussian"))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Rotate(object):
    def __init__(self, v):
        self.v = v # between -30 - 30

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.Rotate(rotate=self.v)
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

# TODO: Figure out how to make rotate just bboxes work correctly

# class Rotate_BBoxes(object):
#     def __init__(self):
#         pass
#
#     def __call__(self, sample):
#         img, annot = sample['img'], sample['annot']
#         unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()
#
#         bbs = BoundingBoxesOnImage(
#             [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
#             shape=img.shape)
#         aug = iaa.Rotate(30)
#         img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
#         rotate_bb_aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
#                                                     foreground=iaa.Rotate(30))
#         img_rotate_bb_aug, bbs_rotate_bb_aug = rotate_bb_aug(image=img, bounding_boxes=bbs_aug)
#         drawn_img = bbs_rotate_bb_aug.draw_on_image(img_rotate_bb_aug * 255, size=2, color=[0, 255., 0])
#         import skimage
#         skimage.io.imsave('draw10.png', drawn_img)

class ShearX(object):
    def __init__(self, v):
        self.v = v # between -30 - 30

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.ShearX(self.v)
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class ShearX_BBoxes(object):
    def __init__(self, v):
        self.v = v #self.v should be between -30 - 30

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.ShearX(self.v))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class ShearY(object):
    def __init__(self, v):
        self.v = v # between -30 - 30

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.ShearY(self.v)
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class ShearY_BBoxes(object):
    def __init__(self, v):
        self.v = v #self.v should be between -30 - 30

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.ShearY(self.v))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Equalize(object):
    def __init__(self, v):
        self.v = v # not applied

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.AllChannelsHistogramEqualization()
        img_aug, bbs_aug = aug(image=(img * 255.).astype('uint8'), bounding_boxes=bbs)
        img_aug = img_aug.astype('float32') / 255.
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Equalize_BBoxes(object):
    def __init__(self, v):
        self.v = v #not applied

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.AllChannelsHistogramEqualization())
        img_aug, bbs_aug = aug(image=(img * 255.).astype('uint8'), bounding_boxes=bbs)
        img_aug = img_aug.astype('float32') / 255.
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Solarize(object):
    def __init__(self, v):
        self.v = v # -1 - 1.

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.pillike.Solarize(threshold=self.v)
        img_aug, bbs_aug = aug(image=(img * 2. - 1.), bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        img_aug = (img_aug + 1.) / 2
        return {'img': img_aug, 'annot': annot_aug}

class Solarize_BBoxes(object):
    def __init__(self, v):
        self.v = v #-1 - 1

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.pillike.Solarize(threshold=0.))
        img_aug, bbs_aug = aug(image=(img * 2. - 1.), bounding_boxes=bbs)
        img_aug = (img_aug + 1.) / 2
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Color(object):
    def __init__(self, v):
        self.v = v # 0.0 - 3.0

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.pillike.EnhanceColor(self.v)
        img_aug, bbs_aug = aug(image=(img * 255.).astype('uint8'), bounding_boxes=bbs)
        img_aug = img_aug.astype('float32') / 255.
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class Color_BBoxes(object):
    def __init__(self, v):
        self.v = v #not applied?

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels,
                                          foreground=iaa.pillike.EnhanceColor(self.v))
        img_aug, bbs_aug = aug(image=(img * 255.).astype('uint8'), bounding_boxes=bbs)
        img_aug = img_aug.astype('float32') / 255.
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])
        # the shape has to be at least (0,5)
        if len(annot_aug) == 0:
            annot_aug = np.zeros((0,5))
        return {'img': img_aug, 'annot': annot_aug}

class RandomRotate(object):
    def __init__(self, degree):
        self.degree = degree

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']
        unique_labels = np.unique(annot[:, 4].astype('int').astype('str')).tolist()

        rotate_degree = random.uniform(-1 * self.degree, self.degree)
        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.BlendAlphaBoundingBoxes(labels=unique_labels, foreground=iaa.geometric.TranslateY(percent=0.1))
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        # drawn_img = bbs_aug.draw_on_image(img_aug * 255, size=2, color=[0, 255., 0])
        # import skimage
        # skimage.io.imsave('draw.png', drawn_img)
        # img = img.rotate(rotate_degree, Image.BILINEAR)
        # mask = mask.rotate(rotate_degree, Image.NEAREST)

        return {'image': img,
                'label': mask}

# FLIP LR BBOXES ONLY DOESNT SEEM to WORK WITH THIS LIBRARY SO FAR
class FlipLR(object):
    def __init__(self, v):
        self.v = v #ignore ??

    def __call__(self, sample):
        img, annot = sample['img'], sample['annot']

        bbs = BoundingBoxesOnImage(
            [BoundingBox(x1=ann[0], y1=ann[1], x2=ann[2], y2=ann[3], label=str(int(ann[4]))) for ann in annot],
            shape=img.shape)
        aug = iaa.Fliplr()
        img_aug, bbs_aug = aug(image=img, bounding_boxes=bbs)
        annot_aug = np.array([[bb.x1, bb.y1, bb.x2, bb.y2, np.float32(bb.label)] for bb in bbs_aug])

        return {'img': img_aug, 'annot': annot_aug}

# TODO: fix this later
class RandomGaussianBlur(object):
    def __init__(self, _):
        pass

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        if random.random() < 0.5:
            img = img.filter(ImageFilter.GaussianBlur(
                radius=random.random()))

        return {'image': img,
                'label': mask}

class RandomScaleCrop(object):
    def __init__(self, base_size, crop_size, fill=0):
        self.base_size = base_size
        self.crop_size = crop_size
        self.fill = fill

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        # random scale (short edge)
        short_size = random.randint(int(self.base_size * 0.5), int(self.base_size * 2.0))
        w, h = img.size
        if h > w:
            ow = short_size
            oh = int(1.0 * h * ow / w)
        else:
            oh = short_size
            ow = int(1.0 * w * oh / h)
        img = img.resize((ow, oh), Image.BILINEAR)
        mask = mask.resize((ow, oh), Image.NEAREST)
        # pad crop
        if short_size < self.crop_size:
            padh = self.crop_size - oh if oh < self.crop_size else 0
            padw = self.crop_size - ow if ow < self.crop_size else 0
            img = ImageOps.expand(img, border=(0, 0, padw, padh), fill=0)
            mask = ImageOps.expand(mask, border=(0, 0, padw, padh), fill=self.fill)
        # random crop crop_size
        w, h = img.size
        x1 = random.randint(0, w - self.crop_size)
        y1 = random.randint(0, h - self.crop_size)
        img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))
        mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))

        return {'image': img,
                'label': mask}

class FixScaleCrop(object):
    def __init__(self, crop_size):
        self.crop_size = crop_size

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        w, h = img.size
        if w > h:
            oh = self.crop_size
            ow = int(1.0 * w * oh / h)
        else:
            ow = self.crop_size
            oh = int(1.0 * h * ow / w)
        img = img.resize((ow, oh), Image.BILINEAR)
        mask = mask.resize((ow, oh), Image.NEAREST)
        # center crop
        w, h = img.size
        x1 = int(round((w - self.crop_size) / 2.))
        y1 = int(round((h - self.crop_size) / 2.))
        img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))
        mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))

        return {'image': img,
                'label': mask}

# resize to 512*1024
class FixedResize(object):
    """change the short edge length to size"""

    def __init__(self, resize=512):
        self.size1 = resize  # size= 512

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        assert img.size == mask.size

        w, h = img.size
        if w > h:
            oh = self.size1
            ow = int(1.0 * w * oh / h)
        else:
            ow = self.size1
            oh = int(1.0 * h * ow / w)
        img = img.resize((ow, oh), Image.BILINEAR)
        mask = mask.resize((ow, oh), Image.NEAREST)
        return {'image': img,
                'label': mask}

# random crop 321*321
class RandomCrop(object):
    def __init__(self, crop_size=320):
        self.crop_size = crop_size

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        w, h = img.size
        x1 = random.randint(0, w - self.crop_size)
        y1 = random.randint(0, h - self.crop_size)
        img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))
        mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size))
        return {'image': img,
                'label': mask}

class RandomScale(object):
    def __init__(self, scales=(1,)):
        self.scales = scales

    def __call__(self, sample):
        img = sample['image']
        mask = sample['label']
        w, h = img.size
        scale = random.choice(self.scales)
        w, h = int(w * scale), int(h * scale)
        return {'image': img,
                'label': mask}

class TransformTr(object):
    def __init__(self, resize, multi_scale=None):
        if multi_scale is None:
            self.composed_transforms = transforms.Compose([
                FixedResize(resize=resize),
                # RandomCrop(crop_size=args.crop_size),
                # tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255),
                # tr.RandomGaussianBlur(),
                # Normalize(mean, std),
                # ToTensor()
            ])
        else:
            self.composed_transforms = transforms.Compose([
                FixedResize(resize=args.resize),
                RandomScale(scales=args.multi_scale),
                RandomCrop(crop_size=args.crop_size),
                # tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255),
                # tr.RandomGaussianBlur(),
                Normalize(mean, std),
                ToTensor()])

    def __call__(self, sample):
        return self.composed_transforms(sample)

class TransformVal(object):
    def __init__(self, args, mean, std):
        self.composed_transforms = transforms.Compose([
            FixedResize(resize=args.resize),
            FixScaleCrop(crop_size=args.crop_size),  # TODO:CHECK THIS
            Normalize(mean, std),
            ToTensor()])

    def __call__(self, sample):
        return self.composed_transforms(sample)

reason: import numpy,from numpy


text:
"""Author: Brandon Trabucco.
An experiment where humans match captions with attributes.
Args:
    batches: Number of shuffled batches to annotate.
    examples: Number of examples per batch.
"""

import numpy as np
import random
import json
import argparse
from matplotlib.pyplot import title, figure, imshow, axis, text, close
from matplotlib.image import imread
from collections import namedtuple

random.seed(1234567)
np.random.seed(1234567)
Struct = namedtuple("Struct",
    ["data", "label"])

if __name__ == "__main__":
    """Program entry point, load the files.
    """

    parser = argparse.ArgumentParser("Matching experiment.")
    parser.add_argument("-b", "--batches",
        type=int, default=1, help="Number of shuffled batches to annotate.")
    parser.add_argument("-e", "--examples",
        type=int, default=3, help="Number of examples per batch.")
    args = parser.parse_args()

    with open("captions.json", "r") as f:
        data = json.load(f)

    answers = []

    # Loop and create batches.
    for b in range(args.batches):

        # Random indices into the dataset.
        indices = np.random.choice(
            len(data), args.examples, replace=False)

        # Separate the image and captions with ids.
        images, captions = [], []
        for i in indices:
            c = data[i]
            images.append(Struct(
                data=c["image_name"], label=i))
            captions.append(Struct(
                data=c["captions"][0][0], label=i))

        # Randomly shuffle each list.
        random.shuffle(images)
        random.shuffle(captions)

        # Construct a graphical quiz
        fig = figure()
        title("Image Batch {0} of {1}".format(b, args.batches))
        axis('off')
        for i, img in enumerate(images):

            # Display images.
            fig.add_subplot(2, args.examples, 1 + i)
            image = imread(img.data)
            imshow(image)
            axis('off')

            # Display text.
            fig.add_subplot(2, args.examples,
                1 + i + args.examples)
            text(0.5, 1.0, "Image {0:3d}".format(img.label),
                horizontalalignment="center")
            axis('off')

        # Build the captions selection.
        question = "Captions were:"
        for i, cap in enumerate(captions):
            question += "\n  ({0:3d}) {1}".format(i, cap.data)

        # Ask for user input.
        fig.show()
        print(question)

        # Collect the answers.
        for img in images:
            while True:

                # Read inputs separated by spaces
                iis = input("Image {0:3d} matches captions... ".format(
                    img.label)).strip().split(" ")

                # User has entered the empty string
                if len(iis) == 1 and iis[0] == "":
                    answers.append([img, []])
                    break

                try:
                    # Try to decode integers
                    jjs = list(set([int(i) for i in iis]))
                    if all([j >= 0 and j < args.examples for j in jjs]):
                        answers.append([img, [captions[j] for j in jjs]])
                        break
                    print("Inputs {0} not in range(0, {1})".format(jjs, args.examples))
                except:
                    print("Inputs {0} not numbers".format(iis))

        # Close the plot window.
        close()

    # Keep track of these.
    false_positives = 0
    false_negatives = 0
    true_positives = 0
    true_negatives = 0
    all_labels = set(range(args.examples))

    # Calculate how many answers are correct.
    for img, caps in answers:
        has_correct_label = False
        present_labels = set()

        for cap in caps:
            present_labels.add(cap.label)

            if cap.label == img.label:
                has_correct_label = True
                true_positives += 1

            if cap.label != img.label:
                false_positives += 1

        if not has_correct_label:
            false_negatives += 1

        for z in all_labels.difference(present_labels):
            if z != img.label:
                true_negatives += 1

    # Export the collected statistics.
    precision = true_positives / (true_positives + false_positives)
    recall = true_positives / (true_positives + false_negatives)
    print("Precision is {0:.3f} amnd recall is {1:.3f}".format(precision, recall))

    # Write a data file.
    with open("statistics.json", "w") as f:
        json.dump({
            "precision": precision, "recall": recall,
            "true_positives": true_positives, "true_negatives": true_negatives,
            "false_positives": false_positives, "false_negatives": false_negatives}, f)

ext: py

reason: import numpy


text:
import warnings
import numpy as np
from thimbles.sqlaimports import *

__all__ = """
from_spectre_chebyshev
from_spectre_legendre
""".split()

# ########################################################################### #

def from_spectre_chebyshev(pixels, coefficients):
    # THIS VERSION TAKEN FROM SPECTRE
    #c20    p = (point - c(6))/c(7)
    #c      xpt = (2.*p-(c(9)+c(8)))/(c(9)-c(8))
    # !! is this right?
    #transforming coefficients
    #wvs =  coeff[0] + xpts*coeff[1] + coeff[2]*(2.0*xpts**2.0-1.0) + coeff[3]*xpts*(4.0*xpts**2.0-3.0)+coeff[4]*(8.0*xpts**4.0-8.0*xpts**2.0+1.0)
    print("chebyshev with coefficients {}".format(coefficients))
    n = len(pixels)
    xpts = (2.0*pixels - float(n+1))/float(n-1)
    return np.polynomial.chebyshev.chebval(xpts, coefficients)

def from_spectre_legendre(pixels, coefficients):
    print("generating wavelengths from legendre polynomial coefficients {}".format(coefficients))
    pixels = np.asarray(pixels)
    n = len(pixels)
    xpts = (2.0*pixels - float(n+1))/float(n-1)
    return np.polyval(coefficients, xpts)

max_stars_repo_name: quidditymaster/thimbles

max_issues_repo_name: quidditymaster/thimbles

max_forks_repo_name: quidditymaster/thimbles


text:
from typing import List, Optional
import networkx as nx
import cirq
import numpy as np

def build_ising_model_hamiltonian(
    connectivity: List[int],
    coupling: int = 1,
    external: int = 1,
    transverse: Optional[int] = 1,
    random: bool = False,
    random_high: float = 1,
    random_low: float = -1,
):

    # Interaction either as complete graph (between all) or bipartite
    if len(connectivity) not in [1, 2]:
        raise NotImplementedError

    interaction_graph = None
    if len(connectivity) == 1:
        interaction_graph = nx.complete_graph(connectivity[0])
    else:
        interaction_graph = nx.complete_bipartite_graph(
            connectivity[0], connectivity[1]
        )

    # Build the hamiltonian
    hamiltonian = 0

    # Two qubit couplings
    for node1, node2 in interaction_graph.edges:
        c = (
            np.random.uniform(low=random_low, high=random_high)
            if random
            else coupling
        )
        hamiltonian += (
            c
            * cirq.Z(cirq.GridQubit(node1, 0))
            * cirq.Z(cirq.GridQubit(node2, 0))
        )

    # One qubit interaction
    for node in interaction_graph.nodes:
        e = (
            np.random.uniform(low=random_low, high=random_high)
            if random
            else external
        )
        hamiltonian += e * cirq.Z(cirq.GridQubit(node, 0))
        t = (
            np.random.uniform(low=random_low, high=random_high)
            if random
            else transverse
        )
        if transverse is not None:
            hamiltonian += t * cirq.X(cirq.GridQubit(node, 0))

    all_qubits = [cirq.GridQubit(node, 0) for node in interaction_graph.nodes]
    return hamiltonian, sorted(all_qubits)

def build_heisenberg_model_hamiltonian(
    connectivity: List[int],
    variant: str = "XYZ",
    coupling: List[int] = [1, 1, 1],
    external: int = 1,
    random: bool = False,
):

    # Interaction either as chain or 2d lattice
    if len(connectivity) not in [1, 2]:
        raise NotImplementedError

    interaction_graph = None
    if len(connectivity) == 1:
        interaction_graph = nx.path_graph(connectivity[0])
    else:
        interaction_graph = nx.grid_2d_graph(connectivity[0], connectivity[1])

    supported_variants = ["XYZ", "XXZ", "XXX"]
    if variant not in supported_variants:
        raise NotImplementedError

    # Initialize from given couplings or random
    J_x, J_y, J_z = coupling
    if random:
        J_x, J_y, J_z = np.random.uniform(size=[3], low=-1, high=1).tolist()

    # Depending on the variant restrict coupling
    if variant == "XXZ":
        J_y = J_x
    elif variant == "XXX":
        J_y = J_x
        J_z = J_x

    # Build the hamiltonian
    hamiltonian = 0

    # Two qubit couplings
    for node1, node2 in interaction_graph.edges:
        qubit1, qubit2 = cirq.GridQubit(*node1), cirq.GridQubit(*node2)
        hamiltonian += J_x * cirq.X(qubit1) * cirq.X(qubit2)
        hamiltonian += J_y * cirq.Y(qubit1) * cirq.Y(qubit2)
        hamiltonian += J_z * cirq.Z(qubit1) * cirq.Z(qubit2)

    # One qubit interaction
    if external != 0 or random == True:
        if random == True:
            external = np.random.uniform(low=-1, high=1)
        for node in interaction_graph.nodes:
            qubit = cirq.GridQubit(*node)
            hamiltonian += external * cirq.Z(qubit)

    all_qubits = [cirq.GridQubit(*node) for node in interaction_graph.nodes]
    return hamiltonian, sorted(all_qubits)

max_stars_repo_name: leevilehtonen/tfq-varqbm

max_issues_repo_name: leevilehtonen/tfq-varqbm

max_forks_repo_name: leevilehtonen/tfq-varqbm

reason: import numpy,import networkx


text:
import numpy as np
from scipy.spatial.distance import pdist, squareform
from vicsek_params import *

def init(L, N, v0, R, eta):
    ''' Initialise a single run '''
    state = np.zeros( (N, 5) )
    state[:,:2] = L*np.random.random( (N, 2) ) - 0.5*L # positions x,y
    state[:,2] = 2*np.pi * np.random.random(N) # angle
    state[:,3] = v0 * np.cos(state[:,2]) # x velocity
    state[:,4] = v0 * np.sin(state[:,2]) # y velocity

    return state

def step(state, L, N, v0, R, eta, t):
    ''' Perform one step for all particles '''

    # Update positions
    state[:,:2] = state[:,:2] + v0*dt*state[:,3:]

    # Periodic boundaries
    crossedX = np.where(abs(state[:,0]) > 0.5*L)
    crossedY = np.where(abs(state[:,1]) > 0.5*L)
    state[crossedX,0] = state[crossedX,0] - np.sign(state[crossedX,0])*L
    state[crossedY,1] = state[crossedY,1] - np.sign(state[crossedY,1])*L

    # Initialise heading with noise
    heading = eta*np.random.random(N) - 0.5*eta

    # Use adjacency matrix to determine neighbours
    A = squareform(pdist(state[:,:2]))
    for i in range(N):
        adj = np.where(A[i,:] < R)[0] # indices of adjacent particles
        theta = state[adj,2] # angles of all adjacent particles

        # Leaders must be treated separately on account of their weight
        ldr_adj = np.where(A[i,:N_ldr] < ldr_R)[0]
        ldr_theta = state[ldr_adj,2]

        # Sum sin and cos of angles
        sum_sin = np.sum(np.sin(theta)) + ldr_weight*np.sum(np.sin(ldr_theta))
        sum_cos = np.sum(np.cos(theta)) + ldr_weight*np.sum(np.cos(ldr_theta))

        # Compute heading for this particle
        heading[i] += np.arctan2(sum_sin, sum_cos)

    # Update state with new headings
    state[:,2] = heading # Can add an angle here for waveyness

    # Some leaders may have pre-defined trajectories
    state[:N_ldr_traj,2] = ldr_traj * t

    # Update velocities
    state[:,3] = v0 * np.cos(state[:,2])
    state[:,4] = v0 * np.sin(state[:,2])

    return

def order_parameter(state, V_coeff):
    ''' Calculate order parameter (V) '''

    Vx = np.sum(state[:,3])
    Vy = np.sum(state[:,4])
    return np.sqrt(Vx*Vx + Vy*Vy) * V_coeff

max_stars_repo_name: JustinTervala/PHoDMSs

max_issues_repo_name: JustinTervala/PHoDMSs

max_forks_repo_name: JustinTervala/PHoDMSs

reason: import numpy,from scipy


text:
from balutils.CatalogFeature import SimpleCatalog
import numpy as np

class Detection(SimpleCatalog):
    """
    Adds detection catalog functionality to the catalog.
    """
    def applyTo(self, catalog: Catalog) -> None:

        self.parent.applyTo(catalog)

        '''
        Balrog stack versions 1.4 and below have a small bug that
        seems to duplicate exactly 1 object, so check for these
        '''
        unq, unq_idx, unq_cnt = np.unique(
            catalog._cat['bal_id'],
            return_inverse=True,
            return_counts=True
        )

        Nunq = len(unq)

        if Nunq != catalog.Nobjs:
            Ndups = catalog.Nobjs - Nunq
            dup_ids = unq[np.where(unq_cnt > 1)]
            print('Warning: Detection catalog has {} duplicate(s)!'.format(Ndups))
            print('Removing the following duplicates from detection catalog:')
            print(dup_ids)

            Nbefore = catalog.Nobjs
            for did in dup_ids:
                indx = np.where(catalog._cat['bal_id']==did)[0]

                L = len(indx)
                for i in range(L-1): # keep last one
                    catalog._cat.remove_row(indx[i])

            catalog.Nobjs = len(catalog._cat)
            assert catalog.Nobjs == (Nbefore - Ndups)

            print('{} duplicates removed, catalog size now {}'.format(Ndups, catalog.Nobjs))

        return

max_stars_repo_name: mattkwiecien/balutils

max_issues_repo_name: mattkwiecien/balutils

max_forks_repo_name: mattkwiecien/balutils


text:
#!/bin/env python
# 2020/01/11
# K-means clustering
# Stanislaw Grams <sjg@fmdx.pl>
# 06-association_rules_and_clustering/02-k_means_clustering.py
import numpy    as np
import pandas   as pd
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

## data load
dataframe = pd.read_csv ("iris2D.csv", usecols=['PC1', 'PC2'])
x = dataframe.iloc[:, [0, 1]].values

## apply K-means clustering
kmeans = KMeans (n_clusters=3, init='k-means++', max_iter=300, n_init=10, random_state=0)
y_kmeans = kmeans.fit_predict (x);

## scatter chart
plt.scatter (x[y_kmeans == 0, 0], x[y_kmeans == 0, 1], s=100, c='red', label='Cluster 1')
plt.scatter (x[y_kmeans == 1, 0], x[y_kmeans == 1, 1], s=100, c='green', label='Cluster 2')
plt.scatter (x[y_kmeans == 2, 0], x[y_kmeans == 2, 1], s=100, c='blue', label='Cluster 3')
plt.scatter (kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:,1], s=100, c='magenta', label='Centroids')
plt.legend ()
plt.show ()

max_stars_repo_name: sgrams/ci

max_issues_repo_name: sgrams/ci

max_forks_repo_name: sgrams/ci


text:
from picamera.array import PiRGBArray
from picamera import PiCamera
import time
import cv2
import serial
from PIL import Image
from imutils import contours
import imutils
#from skimage import measure
import numpy as np
import time
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn import tree
import random

ser = serial.Serial('/dev/ttyACM0')
camera = PiCamera()
camera.resolution = (640, 480)
camera.shutter_speed = 2000
camera.framerate = 60
rawCapture = PiRGBArray(camera, size=(640, 480))

#rawCapture = PiRGBArray(camera)

start = time.time()
r1 =[]
r2=[]
val1 = []
val2 = []
Y = []
lower_red = np.array([0,50,50])
upper_red = np.array([10,255,255])

lower_red2 = np.array([165,50,50])
upper_red2 = np.array([180,255,255])
lower_blue = np.array([105,50,50])
upper_blue = np.array([127,255,255])

lower_green = np.array([44,50,50])
upper_green = np.array([70,255,255])

qwe = 'qwertyuiopasdfgh'
iii = 0
ser.write(qwe[0])
## Training Loop
time.sleep(2)
for characters in 'qwertyuiopasdfgh':
    ser.write(characters)
    time.sleep(0.5)
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
        xx = "{0:b}".format(iii).zfill(4)
        ser.write(qwe[iii])
        img = frame.array
        #img = cv2.imread('c4/%s.jpg' %xx, flags=cv2.IMREAD_COLOR)
        print(iii)
        Y.append(xx)
        firstpart, secondpart = xx[:int(len(xx)/2)], xx[int(len(xx)/2):]
        val1.append(firstpart)
        val2.append(secondpart)
        mask = []
        img = frame.array
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        maskR = cv2.inRange(hsv, lower_red, upper_red)
        maskR2 = cv2.inRange(hsv, lower_red2, upper_red2)
        #cv2.imshow("image4",img)
        #cv2.waitKey(0)
        mask.append(maskR+maskR2)
        maskG = cv2.inRange(hsv,lower_green, upper_green)
        mask.append(maskG)

        #maskB = cv2.inRange(hsv,lower_blue,upper_blue)
        #maskY = cv2.inRange(hsv,lower_yellow,upper_yellow)
        mask = np.array(mask)

        for colorx in range(2):
            img1 = mask[colorx,:,:]

            kernel = np.ones((5,5),np.uint8)
            blurred=cv2.GaussianBlur(img1,(11,11),0)
            threshold=cv2.threshold(blurred,200,255,cv2.THRESH_BINARY)[1]
            #threshold=cv2.erode(threshold,np.ones((1,1),np.uint8),iterations=1)
            threshold=cv2.dilate(threshold,np.ones((10,10),np.uint8),iterations=2)
            #cv2.imshow("image4",threshold)
            #cv2.waitKey(0)
            try:
                major = cv2.__version__.split('.')[0]
                if major == '3':
                    _, con, _ = cv2.findContours(threshold.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
                else:
                    con, _ = cv2.findContours(threshold.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
                cons=contours.sort_contours(con)[0]
                for(i,c) in enumerate(cons):
                        (x,y,w,h)=cv2.boundingRect(c)
                        #crop=masked_data[y:y+h,x:x+w]
                        ((cx,cy),radius)=cv2.minEnclosingCircle(c)
                        crop=cv2.circle(img,(int(cx),int(cy)),int(radius),(200,0,255),1)
                        #print(colorx,"  ",colorx,",",radius)
                        if(colorx==0):
                            r1.append("{0:.2f}".format(radius))
                        else:
                            r2.append("{0:.2f}".format(radius))
            except:
                    pass
        time.sleep(0.5)
        break
            #cv2.imshow("image1",img)
            #cv2.waitKey(0)
        #iii = iii+1

        #else:
            #break
        rawCapture.truncate(0)

print("time taken ", time.time()-start)
print("\n")
print(r1)
print(r2)
print("\n")
X1 = np.array(r1).reshape(-1,1)
Y1 = np.array(val1).reshape(-1,1)
X2 = np.array(r2).reshape(-1,1)
Y2 = np.array(val2).reshape(-1,1)
clf1 = DecisionTreeClassifier()
clf1 = clf1.fit(X1,Y1)
clf2 = DecisionTreeClassifier()
clf2 = clf2.fit(X2,Y2)

camera.close()
time.sleep(0.1)
camera = PiCamera()
camera.resolution = (640, 480)
camera.shutter_speed = 2000
camera.framerate = 60
rawCapture = PiRGBArray(camera, size=(640, 480))

iii = random.randint(0,15)
ser.write(qwe[iii])
time.sleep(0.5)
## Prediction Loop
for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
    time.sleep(0.5)
    r1 = []
    r2 = []
    Y1 = []
    iii = random.randint(0,15)
    ser.write(qwe[iii])
    time.sleep(0.1)
    img = frame.array
    mask = []
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    maskR = cv2.inRange(hsv, lower_red, upper_red)
    maskR2 = cv2.inRange(hsv, lower_red2, upper_red2)

    mask.append(maskR+maskR2)
    maskG = cv2.inRange(hsv,lower_green, upper_green)
    mask.append(maskG)
	#maskB = cv2.inRange(hsv,lower_blue,upper_blue)
    mask = np.array(mask)
    for colorx in range(2):
        img1 = mask[colorx,:,:]

        kernel = np.ones((5,5),np.uint8)
        blurred=cv2.GaussianBlur(img1,(11,11),0)
        threshold=cv2.threshold(blurred,200,255,cv2.THRESH_BINARY)[1]
        #threshold=cv2.erode(threshold,np.ones((1,1),np.uint8),iterations=1)
        threshold=cv2.dilate(threshold,np.ones((10,10),np.uint8),iterations=2)
        cv2.imshow("image4",threshold)
        cv2.waitKey(0)
        try:
            major = cv2.__version__.split('.')[0]
            if major == '3':
                _, con, _ = cv2.findContours(threshold.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
            else:
                con, _ = cv2.findContours(threshold.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
            cons=contours.sort_contours(con)[0]
            for(i,c) in enumerate(cons):
                    (x,y,w,h)=cv2.boundingRect(c)
                    #crop=masked_data[y:y+h,x:x+w]
                    ((cx,cy),radius)=cv2.minEnclosingCircle(c)
                    crop=cv2.circle(img,(int(cx),int(cy)),int(radius),(200,0,255),1)
                    print(colorx,"  ",colorx,",",radius)
                    if(colorx==0):
                        r1.append("{0:.2f}".format(radius))
                    else:
                        r2.append("{0:.2f}".format(radius))
        except:
                pass
    rawCapture.truncate(0)
    key = cv2.waitKey(1) & 0xFF
    if key == ord("q"):
	    break
    print("\n")
    print(r1)
    print(r2)
    print("\n")
    r1 = np.array(r1).reshape(-1,1)
    r2 = np.array(r2).reshape(-1,1)
    y_pred1 = clf1.predict(r1)
    y_pred2 = clf2.predict(r2)
    y_pred = [y_pred1[i]+y_pred2[i] for i in range(len(y_pred2))]
    x = "{0:b}".format(iii).zfill(4)
    print("For ",x," Output detected ",y_pred)
    #print("Accuracy:",accuracy_score(Y1, y_pred))

    rawCapture.truncate(0)
    if key == ord("q"):
	    break
camera.close()

max_stars_repo_name: nvanandsah/Spectroscopy

max_issues_repo_name: nvanandsah/Spectroscopy

max_forks_repo_name: nvanandsah/Spectroscopy


text:
# packages
import sys
sys.path.append("C:/BERTVision/code/torch")
import torch, h5py
import numpy as np

# prepare torch data set
class COLAH5Processor(torch.utils.data.Dataset):
    NAME = 'COLAH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args):
        self.type = type
        self.args = args

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'cola_bert_embeds.h5'
        self.train_label_path = self.file_path + 'cola_labels.h5'
        self.train_idx_path = self.file_path + 'cola_idx.h5'

        self.val_embed_path = self.file_path + 'cola_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'cola_dev_labels.h5'
        self.val_idx_path = self.file_path + 'cola_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

    def __len__(self):
        return self.dataset_len

    def __getitem__(self, idx):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

#

# prepare torch data set
class MNLIH5Processor(torch.utils.data.Dataset):
    NAME = 'MNLIH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args, shard=False, **kwargs):
        # init configurable string
        self.type = type
        # receive arg parser
        self.args = args
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = kwargs

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'mnli_bert_embeds.h5'
        self.train_label_path = self.file_path + 'mnli_labels.h5'
        self.train_idx_path = self.file_path + 'mnli_idx.h5'

        self.dev_matched_embed_path = self.file_path + 'mnli_dev_matched_bert_embeds.h5'
        self.dev_matched_label_path = self.file_path + 'mnli_dev_matched_labels.h5'
        self.dev_matched_idx_path = self.file_path + 'mnli_dev_matched_idx.h5'

        self.dev_mismatched_embed_path = self.file_path + 'mnli_dev_mismatched_bert_embeds.h5'
        self.dev_mismatched_label_path = self.file_path + 'mnli_dev_mismatched_labels.h5'
        self.dev_mismatched_idx_path = self.file_path + 'mnli_dev_mismatched_idx.h5'

        # if train, initialize the train data
        if self.type == 'train' and self.shard == True:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # np seed
            np.random.seed(self.seed['seed'])
            # 15% of data set
            self.random_indices_len = int(self.args.shard*self.dataset_len)
            # select randomly from indices, without replacement
            self.random_indices = np.random.choice(np.arange(self.dataset_len), size=self.random_indices_len, replace=False)
            # sort for now
            self.random_indices = np.sort(self.random_indices)

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        elif self.type == 'train' and self.shard == False:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        # if matched, initialize the dev data
        if self.type == 'dev_matched':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.dev_matched_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.dev_matched_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.dev_matched_idx_path, 'r')['idx']

            with h5py.File(self.dev_matched_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

        # if matched, initialize the dev data
        if self.type == 'dev_mismatched':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.dev_mismatched_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.dev_mismatched_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.dev_mismatched_idx_path, 'r')['idx']

            with h5py.File(self.dev_mismatched_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

    def __len__(self):
        if self.type == 'train' and self.shard == True:
            return len(self.random_indices)
        else:
            return self.dataset_len

    def __getitem__(self, random_indices):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(random_indices):
            random_indices = random_indices.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev_matched':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev_mismatched':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

#

# prepare torch data set
class MSRH5Processor(torch.utils.data.Dataset):
    NAME = 'MSRH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args, shard=False, **kwargs):
        # init configurable string
        self.type = type
        # receive arg parser
        self.args = args
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = kwargs

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'msr_bert_embeds.h5'
        self.train_label_path = self.file_path + 'msr_labels.h5'
        self.train_idx_path = self.file_path + 'msr_idx.h5'

        self.val_embed_path = self.file_path + 'msr_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'msr_dev_labels.h5'
        self.val_idx_path = self.file_path + 'msr_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train' and self.shard == True:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # np seed
            np.random.seed(self.seed['seed'])
            # 15% of data set
            self.random_indices_len = int(self.args.shard*self.dataset_len)
            # select randomly from indices, without replacement
            self.random_indices = np.random.choice(np.arange(self.dataset_len), size=self.random_indices_len, replace=False)
            # sort for now
            self.random_indices = np.sort(self.random_indices)

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        elif self.type == 'train' and self.shard == False:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

    def __len__(self):
        if self.type == 'train' and self.shard == True:
            return len(self.random_indices)
        else:
            return self.dataset_len

    def __getitem__(self, random_indices):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(random_indices):
            random_indices = random_indices.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

#

# prepare torch data set
class QNLIH5Processor(torch.utils.data.Dataset):
    NAME = 'QNLIH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args, shard=False, **kwargs):
        # init configurable string
        self.type = type
        # receive arg parser
        self.args = args
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = kwargs

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # h5 locs
        self.train_embed_path = self.file_path + 'qnli_bert_embeds.h5'
        self.train_label_path = self.file_path + 'qnli_labels.h5'
        self.train_idx_path = self.file_path + 'qnli_idx.h5'

        self.val_embed_path = self.file_path + 'qnli_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'qnli_dev_labels.h5'
        self.val_idx_path = self.file_path + 'qnli_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train' and self.shard == True:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # np seed
            np.random.seed(self.seed['seed'])
            # 15% of data set
            self.random_indices_len = int(self.args.shard*self.dataset_len)
            # select randomly from indices, without replacement
            self.random_indices = np.random.choice(np.arange(self.dataset_len), size=self.random_indices_len, replace=False)
            # sort for now
            self.random_indices = np.sort(self.random_indices)

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        elif self.type == 'train' and self.shard == False:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

    def __len__(self):
        if self.type == 'train' and self.shard == True:
            return len(self.random_indices)
        else:
            return self.dataset_len

    def __getitem__(self, random_indices):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(random_indices):
            random_indices = random_indices.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

#

# prepare torch data set
class QQPH5Processor(torch.utils.data.Dataset):
    NAME = 'QQPH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args, shard=False, **kwargs):
        # init configurable string
        self.type = type
        # receive arg parser
        self.args = args
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = kwargs

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # h5 locs
        self.train_embed_path = self.file_path + 'qqpairs_bert_embeds.h5'
        self.train_label_path = self.file_path + 'qqpairs_labels.h5'
        self.train_idx_path = self.file_path + 'qqpairs_idx.h5'

        self.val_embed_path = self.file_path + 'qqpairs_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'qqpairs_dev_labels.h5'
        self.val_idx_path = self.file_path + 'qqpairs_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train' and self.shard == True:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # np seed
            np.random.seed(self.seed['seed'])
            # 15% of data set
            self.random_indices_len = int(self.args.shard*self.dataset_len)
            # select randomly from indices, without replacement
            self.random_indices = np.random.choice(np.arange(self.dataset_len), size=self.random_indices_len, replace=False)
            # sort for now
            self.random_indices = np.sort(self.random_indices)

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        elif self.type == 'train' and self.shard == False:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

    def __len__(self):
        if self.type == 'train' and self.shard == True:
            return len(self.random_indices)
        else:
            return self.dataset_len

    def __getitem__(self, random_indices):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(random_indices):
            random_indices = random_indices.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

#

# prepare torch data set
class RTEH5Processor(torch.utils.data.Dataset):
    NAME = 'RTEH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args):
        self.type = type
        self.args = args

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'rte_bert_embeds.h5'
        self.train_label_path = self.file_path + 'rte_labels.h5'
        self.train_idx_path = self.file_path + 'rte_idx.h5'

        self.val_embed_path = self.file_path + 'rte_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'rte_dev_labels.h5'
        self.val_idx_path = self.file_path + 'rte_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

    def __len__(self):
        return self.dataset_len

    def __getitem__(self, idx):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

# prepare torch data set
class SQuADH5Processor(torch.utils.data.Dataset):
    NAME = 'SQuADH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''

    def __init__(self, type, args):
        self.type = type
        self.args = args

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        self.train_embed_path = self.file_path + 'squad_train_embeds.h5'
        self.train_start_path = self.file_path + 'squad_train_start_labels.h5'
        self.train_end_path = self.file_path + 'squad_train_end_labels.h5'
        self.train_idx_path = self.file_path + 'squad_train_indices.h5'

        self.val_embed_path = self.file_path + 'squad_dev_embeds.h5'
        self.val_start_path = self.file_path + 'squad_dev_start_labels.h5'
        self.val_end_path = self.file_path + 'squad_dev_end_labels.h5'
        self.val_idx_path = self.file_path + 'squad_dev_indices.h5'

        # if train, initialize the train data
        if self.type == 'train':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # start ids are shaped: [batch_sz, ]
            self.start = h5py.File(self.train_start_path, 'r')["start_ids"]
            # end ids are shaped: [batch_sz, ]
            self.end = h5py.File(self.train_end_path, 'r')["end_ids"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['indices']

            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # start ids are shaped: [batch_sz, ]
            self.start = h5py.File(self.val_start_path, 'r')["start_ids"]
            # end ids are shaped: [batch_sz, ]
            self.end = h5py.File(self.val_end_path, 'r')["end_ids"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['indices']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

    def __len__(self):
        return self.dataset_len

    def __getitem__(self, idx):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'start_ids': self.start[idx],
                    'end_ids': self.end[idx],
                    'idx': self.idx[idx]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'start_ids': self.start[idx],
                    'end_ids': self.end[idx],
                    'idx': self.idx[idx]}
            return sample

#

# prepare torch data set
class SSTH5Processor(torch.utils.data.Dataset):
    NAME = 'SSTH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args, shard=False, **kwargs):
        # init configurable string
        self.type = type
        # receive arg parser
        self.args = args
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = kwargs

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # h5 locs
        self.train_embed_path = self.file_path + 'sst_bert_embeds.h5'
        self.train_label_path = self.file_path + 'sst_labels.h5'
        self.train_idx_path = self.file_path + 'sst_idx.h5'

        self.val_embed_path = self.file_path + 'sst_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'sst_dev_labels.h5'
        self.val_idx_path = self.file_path + 'sst_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train' and self.shard == True:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # np seed
            np.random.seed(self.seed['seed'])
            # 15% of data set
            self.random_indices_len = int(self.args.shard*self.dataset_len)
            # select randomly from indices, without replacement
            self.random_indices = np.random.choice(np.arange(self.dataset_len), size=self.random_indices_len, replace=False)
            # sort for now
            self.random_indices = np.sort(self.random_indices)

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        elif self.type == 'train' and self.shard == False:
            # initialize ds length
            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

            self.random_indices = np.arange(self.dataset_len)

    def __len__(self):
        if self.type == 'train' and self.shard == True:
            return len(self.random_indices)
        else:
            return self.dataset_len

    def __getitem__(self, random_indices):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(random_indices):
            random_indices = random_indices.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[random_indices],
                    'labels': self.labels[random_indices],
                    'idx': self.idx[random_indices]}
            return sample

# prepare torch data set
class STSBH5Processor(torch.utils.data.Dataset):
    NAME = 'STSBH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args):
        self.type = type
        self.args = args

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'stsb_bert_embeds.h5'
        self.train_label_path = self.file_path + 'stsb_labels.h5'
        self.train_idx_path = self.file_path + 'stsb_idx.h5'

        self.val_embed_path = self.file_path + 'stsb_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'stsb_dev_labels.h5'
        self.val_idx_path = self.file_path + 'stsb_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

    def __len__(self):
        return self.dataset_len

    def __getitem__(self, idx):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

#
# prepare torch data set
class WNLIH5Processor(torch.utils.data.Dataset):
    NAME = 'WNLIH5'
    '''
    This class lazily emits batches from H5 files for deep learning.

    Parameters
    ----------
    type : string
        A string used to flag conditional statements in order to use/retrieve
        the right data set. [train or dev]

    Returns
    -------
    sample : tensor [layers, tokens, features]
        A single sample of data indexed by the torch data set.
    '''
    def __init__(self, type, args):
        self.type = type
        self.args = args

        if self.args.checkpoint == 'bert-base-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\'
        elif self.args.checkpoint == 'bert-large-uncased':
            self.file_path = 'C:\\w266\\data\\h5py_embeds\\bert_large\\'

        # todo: generalize locations
        self.train_embed_path = self.file_path + 'wnli_bert_embeds.h5'
        self.train_label_path = self.file_path + 'wnli_labels.h5'
        self.train_idx_path = self.file_path + 'wnli_idx.h5'

        self.val_embed_path = self.file_path + 'wnli_dev_bert_embeds.h5'
        self.val_label_path = self.file_path + 'wnli_dev_labels.h5'
        self.val_idx_path = self.file_path + 'wnli_dev_idx.h5'

        # if train, initialize the train data
        if self.type == 'train':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.train_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.train_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.train_idx_path, 'r')['idx']

            with h5py.File(self.train_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

        # if train, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [batch_sz, layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]
            # labels are shaped: [batch_sz, ]
            self.labels = h5py.File(self.val_label_path, 'r')["labels"]
            # idx ids are shaped: [batch_sz, ]
            self.idx = h5py.File(self.val_idx_path, 'r')['idx']

            with h5py.File(self.val_embed_path, 'r') as file:
                self.dataset_len = len(file["embeds"])

    def __len__(self):
        return self.dataset_len

    def __getitem__(self, idx):
        '''
        Torch's lazy emission system
        '''
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'train':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

        if self.type == 'dev':
            # emits [layers, tokens, features]
            sample = {'embeddings': self.embeddings[idx],
                    'labels': self.labels[idx],
                    'idx': self.idx[idx]}
            return sample

#

max_stars_repo_name: siduojiang/BERTVision

max_issues_repo_name: siduojiang/BERTVision


text:
#!/usr/bin/env python
# encoding:utf-8
"""
Author : Yuanqing Mei
Date : 2021/4/8
Time: 14:24
File: BenderMeta.py
HomePage : http://github.com/yuanqingmei
Email : dg1533019@smail.nju.edu.cn

Deriving Bender's method threshold by meta-analysis.
"""
import time

def bender_meta(t_dir="F:\\NJU\\MTmeta\\experiments\\supervised\\BenderThreshold\\",
                m_dir="F:\\NJU\\MTmeta\\experiments\\supervised\\BenderThreshold\\"):
    import os
    import csv
    from scipy.stats import norm  # norm.cdf() the cumulative normal distribution function in Python
    from scipy import stats  # 根据卡方分布计算p值: p_value=1.0-stats.chi2.cdf(chisquare,freedom_degree)
    import numpy as np
    import pandas as pd

    # 输入：两个匿名数组，effect_size中存放每个study的effect size，variance存放对应的方差
    # 输出：fixed effect model固定效应元分析后的结果，包括
    #      (1)fixedMean：固定效应元分析后得到的效应平均值；(2) fixedStdError：固定效应元分析的效应平均值对应的标准错
    def fixed_effect_meta_analysis(effect_size, variance):
        fixed_weight = []
        sum_Wi = 0
        sum_WiYi = 0
        d = {}  # return a dict
        study_number = len(variance)
        for i in range(study_number):
            if variance[i] == 0:
                continue
            fixed_weight.append(1 / variance[i])
            sum_Wi = sum_Wi + fixed_weight[i]
            sum_WiYi = sum_WiYi + effect_size[i] * fixed_weight[i]
        fixedMean = sum_WiYi / sum_Wi  # 固定模型元分析后得到的效应平均值
        fixedStdError = (1 / sum_Wi) ** 0.5  # 固定模型元分析的效应平均值对应的标准错
        d['fixedMean'] = fixedMean
        d['fixedStdError'] = fixedStdError
        return d

    # 输入：两个匿名数组，effect_size中存放每个study的effect size，variance存放对应的方差
    # 输出：random effect model随机效应元分析后的结果，包括：
    #      (1) randomMean：随机模型元分析后得到的效应平均值； (2) randomStdError：随机模型元分析的效应平均值对应的标准错
    def random_effect_meta_analysis(effect_size, variance):

        sum_Wi = 0
        sum_WiWi = 0
        sum_WiYi = 0  # Sum(Wi*Yi), where i ranges from 1 to k, and k is the number of studies
        sum_WiYiYi = 0  # Sum(Wi*Yi*Yi), where i ranges from 1 to k, and k is the number of studies

        sum_Wistar = 0
        sum_WistarYi = 0
        d = {}  # return a dict

        study_number = len(variance)
        fixed_weight = [0 for i in range(study_number)]  # 固定模型对应的权值
        random_weight = [0 for i in range(study_number)]  # 随机模型对应的权值

        for i in range(study_number):
            if variance[i] == 0:
                continue
            fixed_weight[i] = 1 / variance[i]
            sum_Wi = sum_Wi + fixed_weight[i]
            sum_WiWi = sum_WiWi + fixed_weight[i] * fixed_weight[i]
            sum_WiYi = sum_WiYi + effect_size[i] * fixed_weight[i]
            sum_WiYiYi = sum_WiYiYi + fixed_weight[i] * effect_size[i] * effect_size[i]

        Q = sum_WiYiYi - sum_WiYi * sum_WiYi / sum_Wi
        df = study_number - 1
        C = sum_Wi - sum_WiWi / sum_Wi

        # 当元分析过程中只有一个study研究时，没有研究间效应，故研究间的方差为零
        if study_number == 1:
            T2 = 0
        else:
            T2 = (Q - df) / C  # sample estimate of tau square

        if T2 < 0:
            T2 = (- 1) * T2  # 20190719，若T2小于0，取相反数

        for i in range(study_number):
            random_weight[i] = 1 / (variance[i] + T2)  # random_weight 随机模型对应的权值

        for i in range(study_number):
            sum_Wistar = sum_Wistar + random_weight[i]
            sum_WistarYi = sum_WistarYi + random_weight[i] * effect_size[i]

        randomMean = sum_WistarYi / sum_Wistar  # 随机模型元分析后得到的效应平均值
        randomStdError = (1 / sum_Wistar) ** 0.5  # 随机模型元分析的效应平均值对应的标准错
        # 当元分析过程中只有一个study研究时，没有研究间异质性，故异质性为零
        if study_number == 1:
            I2 = 0
        else:
            I2 = ((Q - df) / Q) * 100  # Higgins et al. (2003) proposed using a statistic, I2,
            # the proportion of the observed variance reflects real differences in effect size

        pValue_Q = 1.0 - stats.chi2.cdf(Q, df)  # pValue_Q = 1.0 - stats.chi2.cdf(chisquare, freedom_degree)

        d["C"] = C
        d["mean"] = randomMean
        d["stdError"] = randomStdError
        d["LL_CI"] = randomMean - 1.96 * randomStdError  # The 95% lower limits for the summary effect
        d["UL_CI"] = randomMean + 1.96 * randomStdError  # The 95% upper limits for the summary effect
        d["ZValue"] = randomMean / randomStdError  # a Z-value to test the null hypothesis that the mean effect is zero
        d["pValue_Z"] = 2 * (1 - norm.cdf(randomMean / randomStdError))  # norm.cdf() 返回标准正态累积分布函数值
        d["Q"] = Q
        d["df"] = df
        d["pValue_Q"] = pValue_Q
        d["I2"] = I2
        d["tau"] = T2 ** 0.5
        d["LL_ndPred"] = randomMean - 1.96 * (T2 ** 0.5)  # tau、randomMean 已知情况下的新出现的study的effctsize所落的区间
        d["UL_ndPred"] = randomMean + 1.96 * (T2 ** 0.5)  # tau、randomMean 已知情况下的新出现的study的effctsize所落的区间
        # tau、randomMean 未知情况（估计）下的新出现的study的effctsize所落的区间
        # stats.t.ppf(0.975,df)返回学生t分布单尾alpha=0.025区间点(双尾是alpha=0.05)的函数，它是stats.t.cdf()累积分布函数的逆函数
        d["LL_tdPred"] = randomMean - stats.t.ppf(0.975, df) * ((T2 + randomStdError * randomStdError) ** 0.5)
        # tau、randomMean 未知情况（估计）下的新出现的study的effctsize所落的区间
        d["UL_tdPred"] = randomMean + stats.t.ppf(0.975, df) * ((T2 + randomStdError * randomStdError) ** 0.5)

        fixedMean = sum_WiYi / sum_Wi  # 固定模型元分析后得到的效应平均值
        fixedStdError = (1 / sum_Wi) ** 0.5  # 固定模型元分析的效应平均值对应的标准错
        d['fixedMean'] = fixedMean
        d['fixedStdError'] = fixedStdError
        return d

    metric_dir = t_dir
    meta_dir = m_dir
    os.chdir(metric_dir)
    print(os.getcwd())

    # 显示所有行
    pd.set_option('display.max_rows', None)

    # 显示所有列
    pd.set_option('display.max_columns', None)

    # the item of row of dataframe
    pd.set_option('display.width', 5000)

    # read_csv(path, keep_default_na=False, na_values=[""])  只有一个空字段将被识别为NaN
    df = pd.read_csv(meta_dir + "Bender_thresholds.csv", keep_default_na=False, na_values=[""])

    metric_names = sorted(set(df.metric.values.tolist()))
    print("the metric_names are ", df.columns.values.tolist())
    print("the metric_names are ", metric_names)
    print("the len metric_names are ", len(metric_names))
    k = 0
    for metric in metric_names:

        print("the current metric is ", metric)

        threshold_effect_size = df[df["metric"] == metric].loc[:, "VARL_threshold"]
        print("the threshold_effect_size items are ", threshold_effect_size)
        print("the len of threshold_effect_size items is ", len(threshold_effect_size))

        threshold_variance = df[df["metric"] == metric].loc[:, "VARL_variance"]
        print("the threshold_variance items are ", threshold_variance)
        print("the len of threshold_variance items is ", len(threshold_variance))

        metaThreshold = pd.DataFrame()
        metaThreshold['EffectSize'] = threshold_effect_size
        metaThreshold['Variance'] = threshold_variance
        try:
            resultMetaAnalysis = random_effect_meta_analysis(
                np.array(metaThreshold[metaThreshold["EffectSize"] > 0].loc[:, "EffectSize"]),
                np.array(metaThreshold[metaThreshold["EffectSize"] > 0].loc[:, "Variance"]))

            with open(meta_dir + "Bender_meta_thresholds.csv", 'a+', encoding="utf-8", newline='') as f:
                writer_f = csv.writer(f)
                if os.path.getsize(meta_dir + "Bender_meta_thresholds.csv") == 0:
                    writer_f.writerow(
                        ["metric", "Bender_meta_threshold", "Bender_meta_threshold_stdError", "LL_CI", "UL_CI",
                         "ZValue", "pValue_Z", "Q", "df", "pValue_Q", "I2", "tau", "number_of_effect_size"])
                writer_f.writerow([metric, resultMetaAnalysis["mean"], resultMetaAnalysis["stdError"],
                                   resultMetaAnalysis["LL_CI"], resultMetaAnalysis["UL_CI"],
                                   resultMetaAnalysis["ZValue"], resultMetaAnalysis["pValue_Z"],
                                   resultMetaAnalysis["Q"], resultMetaAnalysis["df"], resultMetaAnalysis["pValue_Q"],
                                   resultMetaAnalysis["I2"], resultMetaAnalysis["tau"], len(threshold_effect_size)])

        except Exception as err1:
            print(err1)

        k += 1
        # break

if __name__ == '__main__':

    s_time = time.time()
    bender_meta()
    e_time = time.time()
    execution_time = e_time - s_time

    print("The __name__ is ", __name__, ". This is end of BenderMeta.py!\n",
          "The execution time of BenderMeta.py script is ", execution_time)

max_stars_repo_name: meiyuanqing/MetaThreshold

max_issues_repo_name: meiyuanqing/MetaThreshold

max_forks_repo_name: meiyuanqing/MetaThreshold


text:
import pandas as pd
import QuantLib2 as ql
import numpy as np
import math

def timestamp_to_qldate(ts):
    def _timestamp_to_qldate(t):
        t = pd.Timestamp(t)
        return ql.Date(t.day, t.month, t.year)
    try:
        return [_timestamp_to_qldate(t) for t in ts]
    except TypeError:
        return _timestamp_to_qldate(ts)

def annual_to_continuous(y):
    return math.ln(1+y)

def ql_array(iterable):
    A = ql.Array(len(iterable))
    for i,b in enumerate(iterable):
        A[i] = float(b)
    return A

# Nelson, Svensson, Siegel yield curve using Quantlib Kappa substitution
_maturities = np.arange(0.0, 121.0/12.0, 1.0/12.0)
_maturities[0] = .001
def nss_yield(params):
    [Beta0, Beta1, Beta2, Beta3, Kappa1, Kappa2] = [p for p in params]
    return  [Beta0 + Beta1 * (1 - np.exp(-n * Kappa1)) / (n * Kappa1) + \
             Beta2 * ((1 - np.exp(-n * Kappa1))/(n * Kappa1) - np.exp(-n * Kappa1)) + \
             Beta3 * ((1 - np.exp(-n * Kappa2))/(n * Kappa2) - np.exp(-n * Kappa2)) for n in _maturities]

def par_yield_semiannual(yc, mat_date, debug=False):
    dc = ql.SimpleDayCounter()  # We use a simple day counter for these theoretical rate constructions.
    calc_date = yc.referenceDate()
    if dc.yearFraction(calc_date, mat_date) < 1/30:
        return yc.zeroRate(mat_date, yc.dayCounter(), ql.Annual).rate()
    schedule = ql.Schedule(calc_date,
                           mat_date,
                           ql.Period(6, ql.Months),
                           ql.NullCalendar(),
                           ql.Unadjusted,
                           ql.Unadjusted,
                           ql.DateGeneration.Backward, False)
    discounts = np.array([yc.discount(d) for d in schedule])
    if schedule[0] != calc_date:  # schedule[0] is just today so we won't use it
        print("Schedule[0] wasn't today!")
    first_coupon_date = schedule[1]
    discounts = discounts[1:]
    accrfrac = (1 - dc.yearFraction(calc_date, first_coupon_date) * 2.0)
    par_yield = 2 * (1 - discounts[-1]) / (np.sum(discounts) - accrfrac)
    if debug:
        print(discounts)
        print(accrfrac)
    return par_yield

class QuantlibFitOptions:
    calendar = ql.UnitedStates(ql.UnitedStates.GovernmentBond)
    business_convention = ql.ModifiedFollowing  # Should this be ql.Unadjusted or ModifiedFollowing???
    day_counter = ql.ActualActual()
    end_of_month = True
    settlement_days = 0
    face_amount = 100
    coupon_frequency = ql.Semiannual
    compounding = ql.Compounded

max_stars_repo_name: lnsongxf/YieldCurveNoise

max_issues_repo_name: miabrahams/YieldCurveNoise

max_forks_repo_name: miabrahams/YieldCurveNoise