text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03_bbox_east_training.ipynb (unless otherwise specified).

__all__ = ['east_config', 'ObjectCategoryProcessor', 'ObjectCategoryList', 'ObjectItemList', 'bb_pad_collate',
           'gaussian_blur', 'UnetTypeConcat', 'EastModel', 'intersection', 'tlbr2hw', 'normalized_intersection',
           'tlbr2cthw', 'IoU', 'tlbr2hw', 'hw2area', 'match_anchors', 'create_grid', 'anchors', 'cthw2tlbr',
           'show_anchors_on_images', 'tlbr2cthw', 'bbox_to_target', 'bbox_to_target', 'tlbr2cc', 'prepare_bboxes',
           'prepare_labels', 'BBLossMetrics', 'target_to_bbox', 'nms', 'check_overlap', 'calc_precision_recall', 'F1',
           'iou_loss', 'dice_loss', 'encode_class', 'SigmaL1SmoothLoss', 'RetinaNetFocalLoss', 'threshold_images',
           'prepare_masked_bboxes', 'PreciseFocalLoss', 'MSEloss', 'MSE', 'mse']

# Cell
from fastai import *
from fastai.vision import *
import pandas as pd
import numpy as np
import cv2
from tqdm.notebook import tqdm
import xml.etree.ElementTree as ET
from torch.autograd import Variable
from fastai.callbacks import LossMetrics

# Cell
import matplotlib.pyplot as plt

# Cell
class east_config:
    HEIGHT = 512
    WIDTH = 384
    out_shape = (4,4) # 4 times smaller than input

east_config.input_shape = (east_config.HEIGHT, east_config.WIDTH)

# Cell
class ObjectCategoryProcessor(MultiCategoryProcessor):
    "`PreProcessor` for labelled bounding boxes."
    def __init__(self, ds:ItemList, pad_idx:int=0):
        super().__init__(ds)
        self.pad_idx = pad_idx
        self.state_attrs.append('pad_idx')

    def process(self, ds:ItemList):
        ds.pad_idx = self.pad_idx
        super().process(ds)

    def process_one(self,item): return [item[0], [self.c2i.get(o,None) for o in item[1]]]

    def generate_classes(self, items):
        "Generate classes from unique `items` and add `background`."
        classes = super().generate_classes([o[1] for o in items])
        classes = ['background'] + list(classes)
        return classes

def _get_size(xs,i):
    size = xs.sizes.get(i,None)
    if size is None:
        # Image hasn't been accessed yet, so we don't know its size
        _ = xs[i]
        size = xs.sizes[i]
    return size

# Cell
class ObjectCategoryList(MultiCategoryList):
    "`ItemList` for labelled bounding boxes."
    _processor = ObjectCategoryProcessor

    def get(self, i):
        return ImageBBox.create(*_get_size(self.x,i), *self.items[i], classes=self.classes, pad_idx=self.pad_idx)

    def analyze_pred(self, pred): return pred

    def reconstruct(self, t, x):
        (bboxes, labels, *_) = t # CHANGED
        if len((labels - self.pad_idx).nonzero()) == 0: return
        i = (labels - self.pad_idx).nonzero().min()
        bboxes,labels = bboxes[i:],labels[i:]
        return ImageBBox.create(*x.size, bboxes, labels=labels, classes=self.classes, scale=False)

# Cell
class ObjectItemList(ImageList):
    "`ItemList` suitable for object detection."
    _label_cls,_square_show_res = ObjectCategoryList,False

# Cell
def bb_pad_collate(samples:BatchSamples, pad_idx:int=0) -> Tuple[FloatTensor, Tuple[LongTensor, LongTensor]]:
    "Function that collect `samples` of labelled bboxes and adds padding with `pad_idx`."
    if isinstance(samples[0][1], int): return data_collate(samples)
    max_len = max([len(s[1].data[1]) for s in samples])
    bboxes = torch.zeros(len(samples), max_len, 4)
    labels = torch.zeros(len(samples), max_len).long() + pad_idx
    imgs = []
    for i, (image, bbox) in enumerate(samples):
        bbs, lbls = bbox.data
        imgs.append(image.data[None])
        assert len(bbs) == len(lbls)

        if bbs.nelement() != 0:
            bboxes[i,-len(bbs):] = bbs
            labels[i,-len(lbls):] = tensor(lbls)
    images = torch.cat(imgs,0)
    return images, (bboxes,labels,images)

# Cell
def _gaussian_blur(x, size:uniform_int):
    blurred = cv2.blur(image2np(x), (size,size)) # np.arr
#     blurred = cv2.GaussianBlur(image2np(x), (size,size), 0)
    return tensor(blurred).permute(2,0,1)

def gaussian_blur(size, p=1.0):
    return RandTransform(tfm=TfmPixel(_gaussian_blur), kwargs={'size':size}, p=p, resolved={}, do_run=True, is_random=True, use_on_y=False)

# Cell
class UnetTypeConcat(nn.Module):
    def __init__(self, unpool_layer, conv_layer):
        super().__init__()
        # conv_layer: has to ...
        # unpool_layer: has to increase [h,w] -> [hx2, wx2]
        self.unpool = unpool_layer
        self.conv = conv_layer

    def forward(self, e, d): # e/d - encoder/decoder output
        '''
        @param:  :encoder output
        @param:  :decoder output
        '''
        o = self.unpool(d)
        o = torch.cat((o, e), 1)
        return self.conv(o)

# Cell

# To work w/ ResNet18 change conv_layers in unet_concat:
# 3072 -> 768
# 640 -> 256
# 320 ->128
# (also pixelshuffle)

class EastModel(nn.Module):
    def __init__( self, pixelshuffle=False, conv_out=(listify(east_config.out_shape) == listify((8,8))) ):
        super().__init__()
        self.conv_out = conv_out

        self.means = tensor(imagenet_stats[0]).reshape(1,3,1,1) # [R,G,B]
        self.stds = tensor(imagenet_stats[1]).reshape(1,3,1,1) # [R,G,B]

        if pixelshuffle:
            unpool1 = nn.Sequential(nn.PixelShuffle(2), nn.Conv2d(512, 512*4, kernel_size=1))
            unpool2 = nn.Sequential(nn.PixelShuffle(2), nn.Conv2d(32,  32*4,  kernel_size=1))
            unpool3 = nn.Sequential(nn.PixelShuffle(2), nn.Conv2d(16,  16*4,  kernel_size=1))
        else:
            unpool1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)
            unpool2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)
            unpool3 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)

#         self.resnet = models.resnet50(pretrained=True)
#         self.unet_block1 = UnetTypeConcat(unpool1, conv_layer(ni=3072, nf=128, ks=1))
#         self.unet_block2 = UnetTypeConcat(unpool2, conv_layer(ni=640,  nf=64,  ks=1))
#         self.unet_block3 = UnetTypeConcat(unpool3, conv_layer(ni=320,  nf=64,  ks=1))

        self.resnet = models.resnet18(pretrained=False) # (if using pixelshuffle u have to change those params too)
        self.unet_block1 = UnetTypeConcat(unpool1, conv_layer(ni=768, nf=128, ks=1))
        self.unet_block2 = UnetTypeConcat(unpool2, conv_layer(ni=256,  nf=64,  ks=1))
        self.unet_block3 = UnetTypeConcat(unpool3, conv_layer(ni=128,  nf=64,  ks=1))

        self.conv_bonus1 = conv_layer(ni=128,  nf=128, ks=3)
        self.conv_bonus2 = conv_layer(ni=64,   nf=64,  ks=3)
        self.conv1 = conv_layer(ni=64,   nf=32,  ks=3)
        self.conv2 = conv_layer(ni=32,   nf=32,  ks=3)
        self.conv_score = conv_layer(ni=32,   nf=1,   ks=1, norm_type=None, use_activ=False)
        self.conv_geo   = conv_layer(ni=32,   nf=4,   ks=1, norm_type=None, use_activ=False)
#         self.conv10 = conv_layer(ni=32,  nf=1,   ks=1, norm_type=None, use_activ=False)

        if self.conv_out:
            self.conv_out = conv_layer(ni=32, nf=32, ks=3, stride=2, norm_type=None)

    def forward(self, x):
        x = (x - self.means.to(x.device)) / self.stds.to(x.device) # normalize

        f = []
        for layer in list(self.resnet.children())[:-1]:
            x = layer(x)
            if type(layer) == torch.nn.Sequential: f.append(x)

        # [b,128,h/16,w/16] = unet_block1([b,1024,h/16,w/16], [b,2048,h/32,w/32])
        o = self.unet_block1(f[2], f[3])
        o = self.conv_bonus1(o)
        # [b,64,h/8,w/8] = unet_block1([b,512,h/8,w/8], [b,128,h/16,w/16])
        o = self.unet_block2(f[1], o)
        o = self.conv_bonus2(o)
        # [b,64,h/4,w/4] = unet_block1([b,256,h/4,w/4], [b,64,h/8,w/8])
        o = self.unet_block3(f[0], o)

        o = self.conv1(o)  # [b,32,h/4,w/4]
        o = self.conv2(o)  # [b,32,h/4,w/4]

        if self.conv_out: o = self.conv_out(o) # [b,32,h/2,w/2]

        score = self.conv_score(o) # [b,1,h/4,w/4]
        geo_map = self.conv_geo(o) # [b,4,h/4,w/4]

        score = nn.Sigmoid()(score)
        geo_map = nn.Sigmoid()(geo_map)*2

#         angle_map = self.conv10(g)
#         angle_map = self.sigmoid(angle_map)
#         angle_map = (angle_map - 0.5) * math.pi / 2
#         geo = torch.cat((geo_map, angle_map), 1)  # bs 5 w/4 w/4
        geo = geo_map

        return score, geo

# Cell
# tlbr - top, left, bottom, right
# bb - bounding box
tlbr2hw = lambda bbs: bbs[:,2:] - bbs[:,:2] # [N,4] -> [N,2]

def intersection(bbs1, bbs2, normalize=False):
    ''' Calculates intersection area of bounding boxes
    @param: [N1,4]  :N1 tlbr coordinates
    @param: [N2,4]  :N2 tlbr coordinates
    @return [N1,N2] :area sizes for each coord combination
    '''
    N1, N2 = bbs1.size(0), bbs2.size(0)
    bbs1, bbs2 = bbs1.unsqueeze(1).expand(N1,N2,4), bbs2.unsqueeze(0).expand(N1,N2,4)
    intersection_tl_corner = torch.max(bbs1[...,:2], bbs2[...,:2]) # [N1,N2,2]
    intersection_br_corner = torch.min(bbs1[...,2:], bbs2[...,2:]) # [N1,N2,2]
    sizes = torch.clamp(intersection_br_corner - intersection_tl_corner, min=0) # [N1,N2,2]

    if normalize:
        hw = tlbr2hw(bbs1[:,0,:]) # [N1,2]
        sizes[...,:] /= hw.unsqueeze(1) # [N1,N2,2] * [N1,1,2]

    return sizes[...,0] * sizes[...,1] # [N1,N2]

# Cell
normalized_intersection = partial(intersection, normalize=True)

# Cell
def tlbr2cthw(bbs):
    ''' Convert top/left bottom/right format `boxes` to center/size corners
    @param: [N,4] :tlbr coordinates
    @return [N,4] :cthw coordinates
    '''
    center = (bbs[:,:2] + bbs[:,2:])/2 # [N,2]
    sizes = bbs[:,2:] - bbs[:,:2] # [N,2]
    return torch.cat([center, sizes], 1) #

# Cell
tlbr2hw = lambda bbs: bbs[:,2:] - bbs[:,:2] # [N,4] -> [N,2]
hw2area = lambda hw: hw[:,0] * hw[:,1] # [N,2] -> [N]

def IoU(bbs1, bbs2):
    ''' Calculates intersection area of bounding boxes
    @param: [N1,4]  :N1 tlbr coordinates
    @param: [N2,4]  :N2 tlbr coordinates
    @return [N1,N2] :area sizes for each coord combination
    '''
    inter = intersection(bbs1, bbs2) # [N1,N2]
    hw1, hw2 = tlbr2hw(bbs1), tlbr2hw(bbs2) # [N,2]
    bbs1_areas = hw2area(hw1).unsqueeze(1) # [N1,1]
    bbs2_areas = hw2area(hw2).unsqueeze(0) # [1,N2]
    union = bbs1_areas + bbs2_areas - inter
    return inter/(union+1e-8)

# Cell
# a bit problematic w/ int input and by_iou=False
def match_anchors(anchors, targets, match_thr=0.01, bg_thr=0.01, by_iou=False):
    ''' Match `anchors` to targets. -1 is match to background, -2 is ignore
    @param: [N1,4]  :N1 tlbr coordinates
    @param: [N2,4]  :N2 tlbr coordinates
    @param: int     :if this < IoU -> this match is a match (int index of match in targets)
    @param: int     :if this > IoU -> this match is a background (int -1)
    @param: bool    :to
    @return [N1]    :every match from anchors to targets index (if doesnt exist it is -2/-1)
    '''
    f = IoU if by_iou else normalized_intersection
    ious = f(anchors, targets) # [N1,N2]
#     print(anchors)
#     print(targets)
#     print('ious: \n', ious)
    matches = anchors.new(anchors.size(0)).zero_().long() - 2
#     matches = torch.zeros(anchors.size(0)).zero_().long() - 2

    if ious.shape[1] == 0:
        print('ious.shape[1] == 0')
        return matches

    iou, idxs = torch.max(ious, dim=1) # [N1]
    matches[iou < bg_thr] = -1
    matches[iou > match_thr] = idxs[iou > match_thr]

    return matches

# Cell
def create_grid(size):
    ''' Creates a x,y grid of size `size`, coords start from -1 to 1
    @param: tuple(H,W) :tuple of 2 ints H(height), W(width)
    @return [H,W,4]    :tlbr coords for each (h,w) cell
    '''
    H, W = size if is_tuple(size) else (size,size)
    grid = FloatTensor(H, W, 4)

    # not precise, but good enough
    linear_points_left = torch.linspace(0, 1-1/W, W) # [0, 1]
    linear_points_right = linear_points_left + 1/W
    linear_points_top = torch.linspace(0, 1-1/H, H) # [0, 1]
    linear_points_btm = linear_points_top + 1/H

    grid[:, :, 0] = linear_points_top.unsqueeze(1).expand(H,W)
    grid[:, :, 1] = linear_points_left.unsqueeze(0).expand(H,W)
    grid[:, :, 2] = linear_points_btm.unsqueeze(1).expand(H,W)
    grid[:, :, 3] = linear_points_right.unsqueeze(0).expand(H,W)
    return grid*2-1

# Cell
anchors = create_grid((east_config.HEIGHT//4, east_config.WIDTH//4)).view(-1,4)

# Cell
def _draw_outline(o:Patch, lw:int):
    "Outline bounding box onto image `Patch`."
    o.set_path_effects([patheffects.Stroke(
        linewidth=lw, foreground='black'), patheffects.Normal()])

def _draw_rect(ax:plt.Axes, b:Collection[int], color:str='white', text=None, text_size=14):
    "Draw bounding box on `ax`."
    patch = ax.add_patch(patches.Rectangle(b[:2], *b[-2:], fill=False, edgecolor=color, lw=2))
    _draw_outline(patch, 4)
    if text is not None:
        patch = ax.text(*b[:2], text, verticalalignment='top', color=color, fontsize=text_size, weight='bold')
        _draw_outline(patch,1)

# Cell
def cthw2tlbr(boxes):
    "Convert center/size format `boxes` to top/left bottom/right corners."
    top_left = boxes[:,:2] - boxes[:,2:]/2
    bot_right = boxes[:,:2] + boxes[:,2:]/2
    return torch.cat([top_left, bot_right], 1)

def show_anchors_on_images(data, anchors, figsize=(15,15)):
    all_boxes = []
    all_labels = []
    x, y = data.one_batch(DatasetType.Train, True, True)
    for image, bboxes, labels in zip(x, y[0], y[1]):
        image = Image(image.float().clamp(min=0, max=1))

        # 0=not found; 1=found; found 2=anchor
        processed_boxes = []
        processed_boxes_gt = []
        processed_boxes_bg = []
        processed_labels = []
        processed_labels_gt = []
        a = 0
        bgs = torch.zeros(len(anchors), dtype=bool) + 1
        for gt_box in bboxes[labels > 0]:
            matches = match_anchors(anchors, gt_box[None, :])
            non_bg = matches >= 0
            a += non_bg.sum()
#             print('1)', non_bg.sum(), (matches >= -1).sum(), (matches >= -2).sum())
            if non_bg.sum() != 0:
                processed_boxes_gt.append(to_np(gt_box))
                processed_labels_gt.append(2)
                for bb in anchors[non_bg]:
                    processed_boxes.append(to_np(bb))
                    processed_labels.append(3)
                bgs[non_bg] = 0
            else:
                processed_boxes.append(to_np(gt_box))
                processed_labels.append(0)
                val, idx = torch.max(IoU(anchors, gt_box[None, :]), 0)
                best_fitting_anchor = anchors[idx][0]
                processed_boxes.append(to_np(best_fitting_anchor))
                processed_labels.append(1)
            if a > 500:
                break
        all_boxes.extend(processed_boxes)
        all_labels.extend(processed_labels)

        processed_boxes = np.array(processed_boxes)
        processed_labels = np.array(processed_labels)

#         print('2)', np.unique(processed_labels, return_counts=True))

        _, ax = plt.subplots(nrows=1, ncols=3, figsize=figsize)
        ax[0].set_title("Matched Anchors")
        ax[1].set_title("Actual GT")
        ax[2].set_title("BG Anchors")

        # Anchors
        imageBB = ImageBBox.create(*image.size, tensor(processed_boxes[processed_labels > 1]),
#                                    labels=processed_labels[processed_labels > 1],
                                   classes=["", "", "", ""], scale=False)
#                                        classes=["", "", "Match", "Anchor"], scale=False)

        image.show(ax=ax[0], y=imageBB)

        # Actual
        imageBB = ImageBBox.create(*image.size, tensor(processed_boxes_gt),
#                                    labels=[processed_labels[0]],
                                   classes=["", "", "", ""], scale=False)
#                                        classes=["", "", "Match", "Anchor"], scale=False)
        image.show(ax=ax[1], y=imageBB)

        # BG Anchors
        imageBB = ImageBBox.create(*image.size, tensor(anchors[bgs == 1][1000:2000]),
                                   classes=["", "", "", ""], scale=False)
        image.show(ax=ax[2], y=imageBB)
        break

#     return np.array(all_boxes), np.array(all_labels)

# Cell
def tlbr2cthw(boxes): # [N,4] -> [N,4]
    center = (boxes[:,:2] + boxes[:,2:])/2
    sizes = boxes[:,2:] - boxes[:,:2]
    return torch.cat([center, sizes], 1)

def bbox_to_target(bboxes, anchors):
    ''' Return the target of the model on `anchors` for the `bboxes`
    @param: [N,4] :N tlbr coordinates (true bboxes), (bboxes must match anchors)
    @param: [N,4] :N tlbr coordinates (selected anchors)
    @return [N,4] :distance of how much bboxes are from anchors
    '''
    bboxes, anchors = tlbr2cthw(bboxes), tlbr2cthw(anchors)
    t_centers = (bboxes[...,:2] - anchors[...,:2]) / anchors[...,2:]
    t_sizes = torch.log(bboxes[...,2:] / anchors[...,2:] + 1e-8)
    return torch.cat([t_centers, t_sizes], -1)#.div_(bboxes.new_tensor([[0.1, 0.1, 0.2, 0.2]]))

# Cell
# tlbr to center coords
tlbr2cc = lambda boxes: (boxes[:,:2] + boxes[:,2:])/2 # [N,4] -> [N,2]

# Override
def bbox_to_target(bboxes, anchors):
    ''' Return the target of the model on `anchors` for the `bboxes`
    @param: [N,4] :N tlbr coordinates (true bboxes), (bboxes must match anchors)
    @param: [N,4] :N tlbr coordinates (selected anchors)
    @return [N,4] :distance of how much bboxes are from anchors
    '''
    a_centers = tlbr2cc(anchors)
    return torch.cat([a_centers - bboxes[...,:2], bboxes[...,2:] - a_centers], -1) # .div_(bboxes.new_tensor([[0.1, 0.1, 0.1, 0.1]]))

# Cell
def prepare_bboxes(anchors, bbox_true, bbox_pred, matches=None):
    ''' Return the target of the model on `anchors` for the `bboxes`
    @param: [N1,4] :tlbr coordinates (all anchors), N1 = H x W
    @param: [N2,4] :tlbr coordinates (true bboxes)
    @param: [N1,4] :tlbr coordinates (nn output bboxes)
    @param: [N3,4] :optional for caching
    @return ([N3,4], [N3,4], [N3,4]) or None :bbox_true, bbox_pred, matches (for caching)
    '''
    if matches is None: matches = match_anchors(anchors, bbox_true) # [N3, 4]
    non_bg = matches >= 0 # [N3, 4] (non background)

    if non_bg.sum() == 0:
#         print('non_bg.sum() == 0') # creates a bit of spam (0-2 per epoch)
        return None, None, None

    bbox_pred = bbox_pred[non_bg] # [N3,4]
    bbox_true = bbox_true[matches[non_bg]] # [N3,4] (matches return bbox_true indexes)
    anchors = anchors[non_bg] # [N3,4]
    return bbox_to_target(bbox_true, anchors), bbox_pred, matches

# Cell
def prepare_labels(y_true, y_pred, matches):
    ''' Prepare the probabilities of target and prediction of the model
    @param: [N1]           :
    @param: [H/4 x W/4, 1] :
    @param: [H/4 x W/4]    :
    @return [N2], [N2, 1]  :
    '''
    if matches is None: matches = torch.zeros_like(y_pred).squeeze()
    non_ignore = matches >= -1
    if non_ignore.sum() != matches.size(0):
        print('non_ignore.sum() != matches.size(0)', non_ignore.sum(), matches.size(0))

    target = y_true.new_zeros(len(matches)).float()
    non_bg = matches >= 0
    target[non_bg] = 1

    return target[non_ignore][:, None], y_pred[non_ignore]

# Cell
class BBLossMetrics(LossMetrics):
    def on_batch_end(self, last_target, train, **kwargs):
        "Update the metrics if not `train`"
        if train: return
        bs = last_target[0].size(0) # CHANGED
        for name in self.names:
            self.metrics[name] += bs * self.learn.loss_func.metrics[name].detach().cpu()
        self.nums += bs

# Cell
def target_to_bbox(output, anchors):
#     output.mul_(output.new_tensor([[0.1, 0.1, 0.1, 0.1]]))
    a_centers = tlbr2cc(anchors)
    return torch.cat([a_centers - output[...,:2], a_centers + output[...,2:]], -1)

# Cell
def nms(boxes, scores, threshold=0.2):
    ''' non maximum suppression
    @param: [N,4]
    @param: [N]
    @return [N1]
    '''
    sorted_idx = scores.argsort(descending=True) # [N]
    boxes, scores = boxes[sorted_idx], scores[sorted_idx]
    to_keep, indexes = [], torch.LongTensor(range(len(scores))) # [], [N]
    while len(scores) > 0:
        # put in highest scored index
        to_keep.append(sorted_idx[indexes[0]])
        # calculate IoU of highest_scored_box vs all_boxes
        ious = IoU(boxes, boxes[0].view(1,4)).squeeze() # [N]
        other_boxes = ious <= threshold # [N] (not the same boxes)
        if other_boxes.sum() == 0: break
        if other_boxes.sum() == len(boxes):
            if len(indexes) < 3: break
            boxes, scores, indexes = boxes[1:], scores[1:], indexes[1:]
            continue
        boxes, scores, indexes = boxes[other_boxes], scores[other_boxes], indexes[other_boxes]

    return LongTensor(to_keep)

# Cell
def _unpad(bbox_true, y_true, pad_idx=0):
    ''' [N_max,4], [N_max] -> [N1,4], [N1] '''
    i = torch.min(torch.nonzero(y_true - pad_idx)) if sum(y_true)>0 else 0
    return bbox_true[i:], y_true[i:] + pad_idx

def check_overlap(i_true, _list, is_overlaping):
    for i_pred in _list:
        if is_overlaping[i_pred, i_true]:
            return i_pred
    return None

def calc_precision_recall(bbox_pred, bbox_true, iou_thresh=0.5):
    ''' [P,4], [T,4] (Preds, Trues)'''
    num_preds, num_trues = len(bbox_pred), len(bbox_true)

    ious = IoU(bbox_pred, bbox_true)
    is_overlaping = ious > iou_thresh

    correct = 0
    pred_idxes =list(range(len(bbox_pred)))
    for i_true in range(len(bbox_true)):
        i_overlap = check_overlap(i_true, pred_idxes, is_overlaping)
        if i_overlap is not None:
            pred_idxes.remove(i_overlap)
            correct += 1

    precision = correct / num_preds
    recall = correct / num_trues
    # print(precision, recall, correct, num_preds)
    return precision, recall

def F1(last_output, bbox_true_b, y_true_b, images, detect_thresh=0.8, iou_thresh=0.5, **kwargs):
    y_pred_b, bbox_pred_b = last_output
    if len(y_true_b) == 0: return tensor(0.)

    f1 = 0
    for y_pred, bbox_pred, y_true, bbox_true in zip(y_pred_b, bbox_pred_b, y_true_b, bbox_true_b):

        # 1.get true bbox
        bbox_true, y_true = _unpad(bbox_true, y_true) # [N1,4]

        # 2. get pred bbox
        bbox_pred = bbox_pred.permute(1,2,0).view(-1,4)
        y_pred = y_pred.view(-1,1)

        bbox_pred = target_to_bbox(bbox_pred, anchors.to(y_pred.device))
    #     bbox_pred = bbox_pred.to('cpu')

        detect_mask = y_pred.squeeze() > detect_thresh # [H x W]
        if detect_mask.sum() == 0:
#             print('iou_metric: detect_mask.sum() == 0') # (too much spam when network start learning)
            return tensor(0)

        keep_idxes = nms(bbox_pred[detect_mask], y_pred[detect_mask].view(-1), threshold=0.2)
        bbox_pred = bbox_pred[detect_mask][keep_idxes] # [N2,4]

        # 3. calc result
        precision, recall = calc_precision_recall(bbox_pred, bbox_true, iou_thresh=iou_thresh)
        if precision + recall < 0.001: f1 += 0.
        else: f1 += 2 * (precision * recall) / (precision + recall)
    return tensor(f1 / len(y_true_b))

# Cell
def iou_loss(y_pred_geo, y_true_geo, eps=1.0):
    '''
    @param: [N,4] :
    @param: [N,4] :
    '''
    # d1 -> top, d2->left, d3->bottom, d4->right
    d1_true, d2_true, d3_true, d4_true = torch.split(y_true_geo, 1, 1) # [N,1] x 4
    d1_pred, d2_pred, d3_pred, d4_pred = torch.split(y_pred_geo, 1, 1) # [N,1] x 4
    area_true = (d1_true + d3_true) * (d2_true + d4_true)
    area_pred = (d1_pred + d3_pred) * (d2_pred + d4_pred)
    inter_width = torch.min(d2_true, d2_pred) + torch.min(d4_true, d4_pred)
    inter_height = torch.min(d1_true, d1_pred) + torch.min(d3_true, d3_pred)
    area_intersect = inter_width * inter_height
    area_union = area_true + area_pred - area_intersect
    L_AABB = -torch.log((area_intersect + eps) / (area_union + eps))
    return L_AABB

def dice_loss(y_pred, y_true):
    inter = torch.sum(y_true * y_pred)
    union = torch.sum(y_true) + torch.sum(y_pred) + 1e-5
    return 1. - (2 * inter / union)

# Cell
def encode_class(idxs, n_classes):
    target = idxs.new_zeros(len(idxs), n_classes).float()
    mask = idxs != 0
    i1s = LongTensor(list(range(len(idxs))))
    target[i1s[mask],idxs[mask]-1] = 1
    return target

class SigmaL1SmoothLoss(nn.Module):
    def forward(self, output, target):
        reg_diff = torch.abs(target - output)
        reg_loss = torch.where(torch.le(reg_diff, 1/9), 4.5 * torch.pow(reg_diff, 2), reg_diff - 1/18)
        return reg_loss.mean()

class RetinaNetFocalLoss(nn.Module):
    def __init__(self, anchors: Collection[float], gamma=1.0, pad_idx=0, class_loss_weight=1.0):
        super().__init__()
        self.class_weight = class_loss_weight
        self.gamma, self.pad_idx = gamma, pad_idx
        self.anchors, self.metric_names = anchors, ['bbox_loss', 'class_loss']

    def _unpad(self, bbox_true, y_true):
        ''' [N_max,4], [N_max] -> [N1,4], [N1] '''
        i = torch.min(torch.nonzero(y_true - self.pad_idx)) if sum(y_true)>0 else 0
        return bbox_true[i:], y_true[i:] + self.pad_idx

    def _cls_loss(self, y_pred, y_true, matches):
        ''' [N,1], [N,1] '''
#         return dice_loss(y_pred, y_true)
#         return F.binary_cross_entropy(y_pred, y_true)

        # CE = -log(pt), where pt is prob of correct class
        CE = F.binary_cross_entropy(y_pred, y_true, reduction='none')
        pt = torch.exp(-CE)
        return torch.mean( (1-pt)**self.gamma * CE )

    def _reg_loss(self, bbox_pred, bbox_true, y_pred):
        loss = iou_loss(bbox_pred*256, torch.clamp(bbox_true, min=0)*256)
        weights = -torch.log( torch.clamp( 1-y_pred, min=1e-3, max=0.7) )
#         weights = y_pred + 0.3
        return torch.mean(loss * weights)

    def _one_loss(self, y_pred, bbox_pred, y_true, bbox_true):
        ''' [N,C], [N,4], [N_max], [N_max,4] -> float, float '''
        bbox_true, y_true = self._unpad(bbox_true, y_true) # [N1,4], [N1]
        bbox_true, bbox_pred, matches = prepare_bboxes(self.anchors, bbox_true, bbox_pred) # [N2,4], [N2,4]
        if matches is None or matches is None: return tensor(0.)
        y_true, y_pred = prepare_labels(y_true, y_pred, matches) # [N2], [N2]
        bb_loss = self._reg_loss(bbox_pred, bbox_true, y_pred[matches[matches >= 0]])
        y_loss = self._cls_loss(y_pred, y_true, matches)
        self.metrics += tensor([bb_loss, y_loss])
        return (1-self.class_weight) * bb_loss + self.class_weight * y_loss

    def forward(self, nn_output, bbox_true, y_true, images):
        y_pred, bbox_pred = nn_output
        '''
        @param: [B, 1, H/4, W/4] :probability for each h,w cell
        @param: [B, 4, H/4, W/4] :bboxes for each h,w cell
        @param: [B, N_max, 4]    :true bboxes
        @param: [B, N_max]       :true label (1/0)
        @param: [B, H, W]
        '''
        if bbox_true.device != self.anchors.device:
            self.anchors = self.anchors.to(y_pred.device)

        loss = torch.tensor(0, dtype=torch.float32).to(y_pred.device)
        self.metrics = tensor([0., 0.])
        for yp, bp, yt, bt in zip(y_pred, bbox_pred, y_true, bbox_true):
            loss += self._one_loss(yp.view(-1,1), bp.permute(1,2,0).view(-1,4), yt, bt)

        bs = y_true.size(0)
        self.metrics = dict(zip(self.metric_names, [m/bs for m in self.metrics]))
        return loss/bs

# Cell
def threshold_images(imgs): # torch tensor [b,3,h,w] -> torch tensor [b,1,h,w] w/ 0 or 1
    # denormalize to 0-255
    imgs = (imgs - imgs.min())
    imgs = (imgs / imgs.max()) * 255.
    assert imgs.min() >= 0 and imgs.max() <= 255
    # threshold
    out_imgs = []
    for im in imgs:
        im = image2np(im).astype(np.uint8) # [h,w,3]
        im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        _,th = cv2.threshold(im,0,1,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
        out_imgs.append( tensor(th)[None] )
    return torch.cat(out_imgs, 0)

def prepare_masked_bboxes(anchors, bbox_true, bbox_pred, mask):
    matches = match_anchors(anchors, bbox_true) # [N3, 4]
    assert (tensor(matches.shape) == tensor(mask.shape)).all() == True
    matches[mask] = -1
    return prepare_bboxes(anchors, bbox_true, bbox_pred, matches)

class PreciseFocalLoss(RetinaNetFocalLoss):
    def forward(self, nn_output, bbox_true, y_true, images):
        y_pred, bbox_pred = nn_output

        # CHANGED
        b,c,h,w = y_pred.shape
        images = F.interpolate(images, size=(h,w), mode='bilinear', align_corners=True)
        text_mask = threshold_images(images).bool().to(device=y_pred.device)

        if bbox_true.device != self.anchors.device:
            self.anchors = self.anchors.to(y_pred.device)

        loss = torch.tensor(0, dtype=torch.float32).to(y_pred.device)
        self.metrics = tensor([0., 0.])
        for yp, bp, yt, bt, m in zip(y_pred, bbox_pred, y_true, bbox_true, text_mask): # CHANGED
            loss += self._one_loss(yp.view(-1,1), bp.permute(1,2,0).view(-1,4), yt, bt, m.view(-1)) # CHANGED

        bs = y_true.size(0)
        self.metrics = dict(zip(self.metric_names, [m/bs for m in self.metrics]))
        return loss/bs

    def _one_loss(self, y_pred, bbox_pred, y_true, bbox_true, mask):
        ''' [N,C], [N,4], [N_max], [N_max,4], [N_max] -> float, float '''
        bbox_true, y_true = self._unpad(bbox_true, y_true) # [N1,4], [N1]
        bbox_true, bbox_pred, matches = prepare_masked_bboxes(self.anchors, bbox_true, bbox_pred, mask) # [N2,4], [N2,4]
        if matches is None or matches is None: return tensor(0.)
        y_true, y_pred = prepare_labels(y_true, y_pred, matches) # CHANGED
        bb_loss = self._reg_loss(bbox_pred, bbox_true, y_pred[matches[matches >= 0]])
        y_loss = self._cls_loss(y_pred, y_true, matches)
        self.metrics += tensor([bb_loss, y_loss])
        return (1-self.class_weight) * bb_loss + self.class_weight * y_loss

# Cell
class MSEloss(RetinaNetFocalLoss):
    def _reg_loss(self, bbox_pred, bbox_true, y_pred):
        return nn.MSELoss()(bbox_pred, bbox_true)

    def _one_loss(self, y_pred, bbox_pred, y_true, bbox_true):
        ''' [N,C], [N,4], [N_max], [N_max,4] -> float, float '''
        bbox_true, y_true = self._unpad(bbox_true, y_true) # [N1,4], [N1]
        bbox_true, bbox_pred, matches = prepare_bboxes(self.anchors, bbox_true, bbox_pred) # [N2,4], [N2,4]
        if matches is None or matches is None: return tensor(0.)
        y_true, y_pred = prepare_labels(y_true, y_pred, matches) # [N2], [N2]
        bb_loss = self._reg_loss(bbox_pred, bbox_true, y_pred[matches[matches >= 0]])
        return bb_loss

# Cell
mse = MSEloss(anchors)
def MSE(last_output, bbox_true, y_true, **kwargs):
    return mse(last_output, bbox_true, y_true)


text:
import random

from tensorflow.keras.preprocessing.image import img_to_array
from PIL import Image
from skimage import transform
import numpy as np

def read_image(file):
    image = open_image(file)
    image = normalize_image(image)
    return image

def open_image(file):
    image = Image.open(file).convert('L')
    image = image.resize((100, 100))
    return np.array(image)

# Normalization Zero Mean and Unit Variance
def normalize_image(image):
    image = np.asarray(image, np.float32)

    image -= np.mean(image, keepdims=True)
    image /= np.std(image, keepdims=True)

    return img_to_array(image)

def data_augmentation(file, iterations):
    return [transform_image(file) for _ in range(iterations)]

def transform_image(files):
    random_degree = random.uniform(-10, 10)
    file = random.choice(files)

    image = open_image('../data/train/' + file)
    image = transform.rotate(image, random_degree)
    image = normalize_image(image)

    return image

ext: py

max_stars_repo_name: javiermzll/TF-ImageRecognition

max_issues_repo_name: javiermzll/TF-ImageRecognition

max_forks_repo_name: javiermzll/TF-ImageRecognition

reason: import numpy


text:
# Copyright (C) 2021 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Optimizer utility functions."""

import jax
import jax.numpy as jnp

def compute_global_norm(grads):
  return jnp.sqrt(
      sum([
          jnp.sum(jnp.square(x))
          for x in jax.tree_util.tree_leaves(grads)
      ])
  )

def clip_grads(grads, clip_by, clip_value):
  """Clips the gradient using the method clip_by."""
  if clip_by == 'global_norm':
    global_norm = compute_global_norm(grads)
    should_clip = global_norm > clip_value
    grads = jax.tree_map(
        lambda g: jnp.where(should_clip, g * clip_value / global_norm, g),
        grads
    )
  else:
    raise ValueError('Unexpected value for clip_by', clip_by)
  return grads

reason: import jax


text:
import mosek
import numpy as np
import scipy.io as sio
import scipy.sparse as ss
import pdb
import sys

def unpack(data):
    dims, vals = data[0, 0]
    dims = dims[0]
    return dims, vals

def vec_to_dense(dims, vals):
    sparse_mat = ss.coo_matrix((vals[:, 1], (vals[:, 0], np.zeros(vals.shape[0]))), shape=(dims[0], 1))
    mat = np.array(sparse_mat.todense())
    return mat[:, 0]

def load(filename=None):
    if filename is None:
        filename = 'A_b_obj.mat'
    data = sio.loadmat(filename)
    A_dims, A_vals = unpack(data["A"])
    b_dims, b_vals = unpack(data["b"])
    intflag_dims, intflag_vals = unpack(data["int_flag"])
    obj_dims, obj_vals = unpack(data["objective"])

    b = vec_to_dense(b_dims, b_vals)
    obj = vec_to_dense(obj_dims, obj_vals)
    try:
        intflag = intflag_vals[:,0]
    except:
        intflag = []

    A = ss.csr_matrix((A_vals[:, 2], (A_vals[:, 0], A_vals[:, 1])), shape=A_dims)

    return A_dims, A, b, obj, intflag

def solve_mosek(filename=None):
    A_dims, A, b, obj, intflag = load(filename)
    num_constraints, num_vars = A_dims
    print "Loaded", A_dims

    integer_solver = True

    task = make_task(A, b, obj, intflag, integer_solver)

    vprint('optimizing')
    task.optimize()
    task.solutionsummary(mosek.streamtype.msg)
    xx = np.zeros(num_vars, float)
    soltype = mosek.soltype.itg if integer_solver else mosek.soltype.itr
    task.getxx(soltype, xx)
    fval = task.getprimalobj(soltype)

    return xx, fval, obj#, task

def solve_mosek_nearby(task, nearby_partial_soln):
    num_vars = task.getnumvar()

    n_new_vars = 2*len(nearby_partial_soln)

    task.appendvars(n_new_vars)
    task.putboundlist(mosek.accmode.var, range(num_vars,num_vars+n_new_vars),
        [mosek.boundkey.lo]*n_new_vars, [0]*n_new_vars, [np.inf]*n_new_vars)

    num_cons = task.getnumcon()
    n_new_cons = len(nearby_partial_soln)
    task.appendcons(n_new_cons)
    (nearby_var_idx, nearby_val) = zip(*nearby_partial_soln)
    task.putboundlist(mosek.accmode.con, range(num_cons,num_cons+n_new_cons),
        [mosek.boundkey.fx]*n_new_cons, nearby_val, nearby_val)

    for i,row_idx in enumerate(range(num_cons,num_cons+n_new_cons)):
        task.putarow(row_idx, [nearby_var_idx[i], num_vars+2*i,num_vars+2*i+1], [1,-1,1])

    task.putclist(range(num_vars), np.zeros(n_new_vars))
    task.putclist(range(num_vars,num_vars+n_new_vars), np.ones(n_new_vars))

    vprint('optimizing')
    task.optimize()
    task.solutionsummary(mosek.streamtype.msg)
    xx = np.zeros(num_vars+n_new_vars, float)
    soltype = mosek.soltype.itr
    task.getxx(soltype, xx)
    fval = task.getprimalobj(soltype)

    task.removevars(range(num_vars,num_vars+n_new_vars))
    task.removecons(range(num_cons,num_cons+n_new_cons))

    return xx, fval, task

# Define a stream printer to grab output from MOSEK
def streamprinter(text):
    sys.stdout.write(text)
    sys.stdout.flush()

def vprint(text):
    verbose = True
    if verbose:
        print text

def make_task(A, b, obj, intflag, integer_solver):

    vprint('starting mosek environment...')
    env = mosek.Env()
    env.set_Stream(mosek.streamtype.log, streamprinter)

    vprint('creating mosek task...')
    task = env.Task(0,0)
    task.set_Stream(mosek.streamtype.log, streamprinter)

    num_constraints, num_vars = A.shape

    # set problem size
    task.appendcons(num_constraints)
    task.appendvars(num_vars)

    vprint('creating objective...')
    task.putclist(range(num_vars), obj)

    vprint('creating bounds...')
    for j in xrange(num_vars):
        # 0 <= x_j <= np.inf
        #task.putbound(mosek.accmode.var, j, mosek.boundkey.lo, 0, np.inf)
        task.putbound(mosek.accmode.var, j, mosek.boundkey.ra, 0, 1)

    vprint('creating equalities...')
    boundkeys = [mosek.boundkey.fx]*num_constraints
    #pdb.set_trace()
    task.putboundlist(mosek.accmode.con, range(num_constraints), boundkeys, b, b)
    [i_s, j_s, v_s] = ss.find(A)
    task.putaijlist(i_s, j_s, v_s)
    # we want to maximize
    task.putobjsense(mosek.objsense.maximize)

    if integer_solver:
        # define the integer variables
        task.putvartypelist(intflag, [mosek.variabletype.type_int]*len(intflag))

    # turn off basis identification
    task.putintparam(mosek.iparam.intpnt_basis, mosek.basindtype.never)
    return task

max_issues_repo_name: nayname/TerpreT

reason: import numpy,import scipy


text:
#!/usr/bin/python3
# Query GPU device with PyCUDA.
import pycuda.driver as drv
drv.init()

print("Detected {} CUDA capabable device(s)".format(drv.Device.count()))

for i in range(drv.Device.count()):
    gpu_device = drv.Device(i)
    print("Device {}: {}".format(i, gpu_device.name()))
    compute_capability = float('%d.%d' % gpu_device.compute_capability())
    print("\t Compute Capability: {}".format(compute_capability))
    print("\t Total Memory: {} megabytes".format(gpu_device.total_memory() // (1024**2)))

    device_attributes_tuples = gpu_device.get_attributes().items()
    device_attributes = {}

    for k, v in device_attributes_tuples:
        device_attributes[str(k)] = v

    num_mp = device_attributes['MULTIPROCESSOR_COUNT']
    cuda_cores_per_mp = {
        5.0: 128,
        5.1: 128,
        5.2: 128,
        6.0: 64,
        6.1: 128,
        6.2: 128
    }

    print("\t ({}) Multiprocessors, ({}) CUDA Cores / Multiprocessors: {} CUDA Cores.".format(
        num_mp, cuda_cores_per_mp[compute_capability], num_mp * cuda_cores_per_mp[compute_capability]))

#    device_attributes.pop("MULTIPROCESSOR_COUNT")
    for k in device_attributes.keys():
        print("\t {}: {}".format(k, device_attributes[k]))

reason: import pycuda


text:
import numpy as np

def count_contiguous_areas(u):
    shape = np.shape(u)

    counter = 0

    # for each block of four pixels
    for x in range(0, shape[0] - 1):
        for y in range(0, shape[1] - 1):

            # convert the block to binary
            block = u[x, y] + 2 * u[x+1, y] + 4 * u[x, y+1] + 8 * u[x+1, y+1]

            # just two cases -- top left set, and bottom right unset
            if block == 1:
                counter += 1
            elif block == 7:
                counter -= 1

    return counter

def test_universe(universe, expected_areas):

    print(f"{universe}")

    areas = count_contiguous_areas(universe)

    print(f"number of contiguous areas: {areas}")
    assert(areas == expected_areas)

def test_square_block():
    universe = np.array([
        [0, 0, 0, 0],
        [0, 1, 1, 0],
        [0, 1, 1, 0],
        [0, 0, 0, 0]])
    test_universe(universe, 1)

def test_two_square_blocks():
    universe = np.array([
        [0, 0, 0, 0, 0, 0, 0],
        [0, 1, 1, 0, 1, 1, 0],
        [0, 1, 1, 0, 1, 1, 0],
        [0, 0, 0, 0, 0, 0, 0]])
    test_universe(universe, 2)

def test_fancy_block():
    universe = np.array([
        [0, 0, 0, 0, 0, 0, 0],
        [0, 0, 1, 0, 1, 0, 0],
        [0, 1, 1, 1, 1, 1, 0],
        [0, 0, 1, 0, 1, 0, 0],
        [0, 0, 0, 0, 0, 0, 0]])
    test_universe(universe, 1)

def test_interleaved():
    universe = np.array([
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0],
        [0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0],
        [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
        [0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0],
        [0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],
        [0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0],
        [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        ])
    test_universe(universe, 4)

if __name__ == '__main__':
    test_square_block()
    test_two_square_blocks()
    test_fancy_block()
    test_interleaved()

max_stars_repo_name: MarcusRainbow/ContiguousAreas

max_issues_repo_name: MarcusRainbow/ContiguousAreas

max_forks_repo_name: MarcusRainbow/ContiguousAreas


text:
# %%
import tqdm
import numpy as np

from solver.word_ops import *

def main():
    words = wordVecDataframe(language="pt")
    codes= wordCodes(words.iloc[:,1:].to_numpy().astype(int))
    print(f"Codes vec: \n{codes}")
    print(f"Shape: {codes.shape}")

    n, let = words.iloc[:,1:].shape

    # Full matrix
    greens = np.zeros((n,n), dtype=np.uint8)
    yellows = np.zeros((n,n), dtype=np.uint8)

    # For each letter
    for i in tqdm.tqdm(range(26)):
        # Check matches between all words and all other words for that letter
        partial_greens = get_green_matches(codes[:,np.newaxis,i:i+1],codes[...,i:i+1])
        # Set bits in `greens` in positions that have matches
        greens +=  partial_greens[...,0]

        # Same for yellow, which depends on greens (green letters don't count for yellow matches)
        partial_yellows = get_yellow_matches(codes[:,np.newaxis,i:i+1],codes[...,i:i+1], partial_greens)
        yellows[...] += partial_yellows[...,0]

    file = "matches"
    np.savez(file, greens=greens, yellows=yellows, words=words.iloc[:,0].to_numpy())

if __name__=="__main__":
    main()

max_stars_repo_name: GuiBeal/termo-solver

max_issues_repo_name: GuiBeal/termo-solver

max_forks_repo_name: GuiBeal/termo-solver


text:
import numpy as np
import os
import os.path as osp

def file_path(name):
    this_dir = osp.dirname(__file__)
    path = osp.join(this_dir, name)
    return path

lights_status = {'go': 0, 'stop': 1, 'warning':2, 'ambiguous':3}
def read_data(file):
    '''
    @description: 将数据转出（N, 5）格式 xin, yin, xmax, ymax, status
    :param file:
    :return:
    '''
    f = open(file)
    lines = f.readlines()
    total_data = []
    data = []
    for line in lines:
        data_line = list(line.split())
        data.append(int(data_line[2]))
        data.append(int(data_line[3]))
        data.append(int(data_line[4]))
        data.append(int(data_line[5]))
        data.append(int(data_line[6]))
        status = list(data_line[-1].split('\''))
        if status[1] in {'go', 'stop', 'warning', 'ambiguous'}:
            data.append(lights_status[status[1]])
        total_data.append(data)
        data = []
    return total_data

file = './label.txt'
print(read_data(file))

def write_data(file, data):
    f = open(file, 'a')
    for i in range(len(data)):
        for j in range(5):                 # 一个object数据
            f.writelines(str(data[i][j]))
            if j < 4:
                f.writelines(',')
        if i < len(data)-1:
            f.writelines(' ')
    f.close()

def read_all_data(image_dir, label_dir):
    '''
    description:
    :param file_dir: 文件所在父文件夹
    :return: 读取所有的文件
    '''
    label_data = read_data(label_dir)
    image_name = []
    for image in os.listdir(image_dir):
        image_name.append(image_dir + image)   # 每张照片的地址
    print(image_name)
        #write_data(file_name, data)

    return 'write over'

image_dir = file_path('image')
label_dir = file_path('label.txt')
# read_all_data(image_dir, label_dir)


text:
import numpy as np
import copy
import pickle
import sys
import time
import math

from utils.tools import clip_gradients

class Model():
    def __init__(self, input_shape):
        shape = list(input_shape)
        shape.insert(0, None)
        self.input_shape = tuple(shape)
        self.layers = []
        self.inputs = None
        self.optimizer = None
        self.regularization = None

    def add(self, layer):
        self.layers.append(layer)

    def compile(self, optimizer, regularization=None):
        self.optimizer = optimizer
        self.regularization = regularization
        input_shape = self.input_shape
        for layer in self.layers:
            layer.update_output_shape(input_shape)
            input_shape = layer.output_shape

    def summary(self):
        print("====== MODEL SUMMARY ======")
        print("input shape:",self.input_shape)
        for layer in self.layers:
            print(layer.name,"output shape:",layer.output_shape)
        print("===========================")

    def fit(self, x=None, y=None, batch_size=None, epochs=1, verbose=1, callbacks=None, validation_split=0.0, validation_data=None, shuffle=True, class_weight=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None, validation_freq=1, max_queue_size=None, workers=1, use_multiprocessing=False):

        if callbacks!=None:
            print ("WARNING: Model.fit currently does not support callbacks.")
        if class_weight!=None:
            print ("WARNING: Model.fit currently does not support class_weight.")
        if sample_weight!=None:
            print ("WARNING: Model.fit currently does not support sample_weight.")
        if initial_epoch!=0:
            print ("WARNING: Model.fit currently does not support custom initial_epoch. Epoch always starts from 1.")
        if max_queue_size!=None:
            print ("WARNING: Model.fit currently does not support max_queue_size.")
        if use_multiprocessing!=False:
            print ("WARNING: Model.fit currently does not support multiprocessing.")
        if validation_split!=0:
            print ("WARNING: Model.fit currently does not support validation_split. You can provide validation data as (x_val, y_val) tuple in validation_data")

        full_size = x.shape[0]

        if steps_per_epoch != None:
            batch_size = int(full_size*1.0 / steps_per_epoch + 0.5)
        else:
            batch_size = batch_size if batch_size!=None else 32
            steps_per_epoch = math.ceil (full_size*1.0 / batch_size)

        train_results = []
        val_results = []
        for epoch in range(0, epochs, 1):

            if(verbose>0):
                print ("==>Epoch:",epoch+1,"/",epochs, " Steps:",steps_per_epoch)
            idxs = np.arange(full_size)
            if shuffle:
                np.random.shuffle(idxs)

            # train
            loss_total = 0
            count_correct_total = 0
            for step in range(0, steps_per_epoch, 1):
                iter = epoch*steps_per_epoch+step

                # prepare a batch
                idx_start = step*batch_size
                idx_end = idx_start+batch_size if step<steps_per_epoch-1 else full_size
                idx_batch = idxs[idx_start:idx_end]
                x_batch = x[idx_batch]
                y_batch = y[idx_batch]

                # train a batch
                loss, probs = self.forward(x_batch, y_batch)
                self.backward(y_batch)
                self.update(self.optimizer, iter)

                # update stats
                count_correct = np.sum(np.argmax(probs, axis=-1) == y_batch)
                if(verbose>1):
                    print ("Step:",step+1,"/",steps_per_epoch, " loss:",loss," acc:", count_correct*1.0/y_batch.shape[0])
                loss_total += loss
                count_correct_total += count_correct

            # save train stats
            train_loss = loss_total/steps_per_epoch
            train_acc = count_correct_total/full_size
            train_results.append([epoch, train_loss, train_acc])

            val_loss = 'NA'
            val_acc = 'NA'
            if (validation_data!=None) and ((epoch+1)%validation_freq == 0):
                # validate
                x_val = validation_data[0]
                y_val = validation_data[1]
                loss, probs = self.evaluate(x_val, y_val, None, 1, None, validation_steps, None, None, 1, False)

                # save validate stats
                val_loss = loss
                val_acc = np.sum(np.argmax(probs, axis=-1) == y_val)*1.0/y_val.shape[0]
                val_results.append([epoch, val_loss, val_acc])
            if(verbose>0):
                print('train_loss:',train_loss,' train_acc:', train_acc, ' val_loss:', val_loss, ' val_acc:', val_acc)

        return np.array(train_results), np.array(val_results)

    def evaluate(self, x=None, y=None, batch_size=None, verbose=1, sample_weight=None, steps=None, callbacks=None, max_queue_size=None, workers=1, use_multiprocessing=False):
        if callbacks!=None:
            print ("WARNING: Model.fit currently does not support callbacks.")
        if sample_weight!=None:
            print ("WARNING: Model.fit currently does not support sample_weight.")
        if max_queue_size!=None:
            print ("WARNING: Model.fit currently does not support max_queue_size.")
        if use_multiprocessing!=False:
            print ("WARNING: Model.fit currently does not support multiprocessing.")

        # set the mode into testing mode (because dropout layer need this flag)
        for layer in self.layers:
            layer.set_mode(training=False)

        full_size = x.shape[0]
        if steps != None:
            batch_size = int(full_size*1.0 / steps + 0.5)
        else:
            batch_size = batch_size if batch_size!=None else 32
            steps = math.ceil (full_size*1.0 / batch_size)

        loss_sum = 0
        probs_all = []
        for step in range(0, steps, 1):
            # prepare a batch
            idx_start = step*batch_size
            idx_end = idx_start+batch_size if step<steps-1 else full_size
            x_batch = x[idx_start:idx_end]
            y_batch = y[idx_start:idx_end]
            loss, probs = self.forward(x_batch, y_batch)
            loss_sum += loss
            probs_all.append(probs)

        # reset the mode into training for continous training
        for layer in self.layers:
            layer.set_mode(training=True)

        return loss_sum/steps, np.concatenate(probs_all, axis=0)

    def predict(self, x, batch_size=None, verbose=0, steps=None, callbacks=None, max_queue_size=None, workers=1, use_multiprocessing=False):
        if callbacks!=None:
            print ("WARNING: Model.predict currently does not support callbacks.")
        if max_queue_size!=None:
            print ("WARNING: Model.predict currently does not support max_queue_size.")
        if use_multiprocessing!=False:
            print ("WARNING: Model.predict currently does not support multiprocessing.")

        # set the mode into testing mode (because dropout layer need this flag)
        for layer in self.layers:
            layer.set_mode(training=False)

        full_size = x.shape[0]
        if steps != None:
            batch_size = int(full_size*1.0 / steps + 0.5)
        else:
            batch_size = batch_size if batch_size!=None else 32
            steps = math.ceil (full_size*1.0 / batch_size)

        loss_sum = 0
        probs_all = []
        for step in range(0, steps, 1):
            # prepare a batch
            idx_start = step*batch_size
            idx_end = idx_start+batch_size if step<steps-1 else full_size
            x_batch = x[idx_start:idx_end]
            _, probs = self.forward(x_batch)
            probs_all.append(probs)

        # reset the mode into training for continous training
        for layer in self.layers:
            layer.set_mode(training=True)

        return loss_sum/steps, np.concatenate(probs_all, axis=0)

    def forward(self, inputs, targets=None):
        self.inputs = []
        layer_inputs = inputs
        for l, layer in enumerate(self.layers):
            self.inputs.append(layer_inputs)
            if l == len(self.layers)-1:
                layer_inputs, probs = layer.forward(layer_inputs, targets)
            else:
                layer_inputs = layer.forward(layer_inputs)
        outputs = layer_inputs
        return outputs, probs

    def backward(self, targets):
        for l, layer in enumerate(self.layers[::-1]):
            if l == 0:
                grads = layer.backward(self.inputs[-1-l], targets)
            else:
                grads = layer.backward(grads, self.inputs[-1-l])

    def get_params(self):
        params = {}
        grads = {}
        for l, layer in enumerate(self.layers):
            if layer.trainable:
                layer_params, layer_grads = layer.get_params('layer-%dth' % l)
                params.update(layer_params)
                grads.update(layer_grads)

        if self.regularization:
            reg_grads = self.regularization.backward(params)
            for k, v in grads.items():
                grads[k] += reg_grads[k]
        return params, grads

    def update(self, optimizer, iteration):
        params, grads = self.get_params()

        # clip gradients
        # for k, v in grads.items():
        #     grads[k] = clip_gradients(v)
        #     print(k, np.mean(np.abs(v)))

        new_params = optimizer.update(params, grads, iteration)

        for l, layer in enumerate(self.layers):
            if layer.trainable:
                w_key = 'layer-%dth:' % l + layer.name + '/weights'
                b_key = 'layer-%dth:' % l + layer.name + '/bias'
                layer_params = {
                    w_key: new_params[w_key],
                    b_key: new_params[b_key]
                }
                layer.update(layer_params)

max_stars_repo_name: liyinnbw/ML

max_issues_repo_name: liyinnbw/ML

max_forks_repo_name: liyinnbw/ML


text:
import glob
import numpy as np
import automata

# read in metadata file entries to a list of lists
fname = "./outfiles/metadata.txt"
file = open(fname, "r")
metadata = []
for line in file:
    line = line.split()
    metadata.append(line)

# parse metadata variables
rows = int(metadata[0][1])
cols = int(metadata[0][2])
num_procs = int(metadata[1][1])
iprocs = int(metadata[2][1])
jprocs = int(metadata[2][2])
num_gen = int(metadata[3][1])
is_periodic = int(metadata[4][1])
freq_dump = int(metadata[5][1])

paths = glob.glob('./outfiles/dump*')
paths = sorted(paths, key=lambda x: int(x.split("p")[1]))
states = []
for i, path in enumerate([paths[0], paths[-1]]):
    data = np.fromfile(path, dtype=np.int8)
    n = int(data.size / (rows*cols))
    offset = i*rows*cols*(n-1)
    state = np.reshape(data[offset:offset+rows*cols], (rows, cols))
    states.append(state)

# test final state against benchmark game of life simulator (James Percival, ACSE-1)
initial_state = states[0]
final_state = states[1]
print("Running benchmark test on initial state...")
test_state = automata.life(initial_state, num_gen, is_periodic)

if np.all(final_state == test_state):
    print("Test Passed")
else:
    print("Test Failed")


text:
# nnls.py - Non-linear least squares solvers
# -----------------------------------------------
# This file is a part of DeerLab. License is MIT (see LICENSE.md).
# Copyright(c) 2019-2021: Luis Fabregas, Stefan Stoll and other contributors.

import numpy as np
import cvxopt as cvx
import math as m

def fnnls(AtA, Atb, tol=None, maxiter=None, verbose=False):
#=====================================================================================
    r"""
    FNNLS   Fast non-negative least-squares algorithm.
    x = fnnls(AtA,Atb) solves the problem min ||b - Ax|| if
        AtA = A'*A and Atb = A'*b.
    A default tolerance of TOL = MAX(SIZE(AtA)) * NORM(AtA,1) * EPS
    is used for deciding when elements of x are less than zero.
    This can be overridden with x = fnnls(AtA,Atb,TOL).

    [x,w] = fnnls(AtA,Atb) also returns dual vector w where
        w(i) < 0 where x(i) = 0 and w(i) = 0 where x(i) > 0.

    For the FNNLS algorithm, see
        R. Bro, S. De Jong
        A Fast Non-Negativity-Constrained Least Squares Algorithm
        Journal of Chemometrics 11 (1997) 393-401
    The algorithm FNNLS is based on is from
        Lawson and Hanson, "Solving Least Squares Problems", Prentice-Hall, 1974.
    """

    unsolvable = False
    count = 0

    # Use all-zero starting vector
    N = np.shape(AtA)[1]
    x = np.zeros(N)

    # Calculate tolerance and maxiter if not given.
    if tol is None:
        eps = np.finfo(float).eps
        tol = 10*eps*np.linalg.norm(AtA,1)*max(np.shape(AtA))
    if maxiter is None:
        maxiter = 5*N

    passive = x>0       # initial positive/passive set (points where constraint is not active)
    x[~passive] = 0
    w = Atb - AtA @ x     # negative gradient of error functional 0.5*||A*x-y||^2

    # Outer loop: Add variables to positive set if w indicates that fit can be improved.
    outIteration = 0
    maxIterations = 5*N
    while np.any(w>tol) and np.any(~passive):
        outIteration += 1

        # Add the most promising variable (with largest w) to positive set.
        t = np.argmax(w)
        passive[t] = True

        # Solve unconstrained problem for new augmented positive set.
        # This gives a candidate solution with potentially new negative variables.
        x_ =np.zeros(N)
        if np.sum(passive)==1:
            x_[passive] = Atb[passive]/AtA[passive,passive]
        else:
            x_[passive] = np.linalg.solve(AtA[np.ix_(passive,passive)],Atb[passive])

        # Inner loop: Iteratively eliminate negative variables from candidate solution.
        iIteration = 0
        while any((x_<=tol) & passive) and iIteration<maxIterations:
            iIteration += 1

            # Calculate maximum feasible step size and do step.
            negative = (x_<=tol) & passive
            alpha = min(x[negative]/(x[negative]-x_[negative]))
            x += alpha*(x_-x)

            # Remove all negative variables from positive set.
            passive[x<tol] = False

            # Solve unconstrained problem for reduced positive set.
            x_ = np.zeros(N)
            if np.sum(passive)==1:
                x_[passive] = Atb[passive]/AtA[passive,passive]
            else:
                x_[passive] = np.linalg.solve(AtA[np.ix_(passive,passive)],Atb[passive])

        # Accept non-negative candidate solution and calculate w.
        if all(x == x_):
            count += 1
        else:
            count = 0
        if count > 5:
            unsolvable = True
            break
        x = x_

        w = Atb - AtA@x
        w[passive] = -m.inf
        if verbose:
            print(f'{outIteration:10.0f}{iIteration:15.0f}{max(w):20.4e}\n')

    if verbose:
        if unsolvable:
            print('Optimization stopped because the solution cannot be further changed. \n')
        elif any(~passive):
            print('Optimization stopped because the active set has been completely emptied. \n')
        elif w>tol:
            print(f'Optimization stopped because the gradient (w) is inferior than the tolerance value TolFun = {tol:.6e}. \n')
        else:
            print('Solution found. \n')

    return x

#=====================================================================================

def cvxnnls(AtA, Atb, tol=None, maxiter=None):
#=====================================================================================
    """
    NNLS problem solved via CVXOPT

    References:
    -----------
    [1]
    Rein, Lewe, Andrade, Kacprzak and Weber. J. Magn. Reson., 295 (2018) 17–26.
    Global analysis of complex PELDOR time traces
    https://doi.org/10.1016/j.jmr.2018.07.015
    """

    N = np.shape(AtA)[1]
    if tol is None:
        eps = np.finfo(float).eps
        tol = 10*eps*np.linalg.norm(AtA,1)*max(np.shape(AtA))
    if maxiter is None:
        maxiter = 5*N

    x0 = np.zeros(N)

    cAtA = cvx.matrix(AtA)
    cAtb = -cvx.matrix(Atb)

    lb = cvx.matrix(np.zeros(N))
    I = -cvx.matrix(np.eye(N, N))

    # Set optimization stop criteria
    cvx.solvers.options['show_progress'] = False
    cvx.solvers.options['max_iters'] = maxiter
    cvx.solvers.options['abstol'] = tol
    cvx.solvers.options['reltol'] = tol

    P = cvx.solvers.qp(cAtA, cAtb, I, lb, initvals=cvx.matrix(x0))['x']
    P = np.squeeze(np.asarray(P))
    return P
#=====================================================================================

def nnlsbpp(AtA, AtB, x0=None):
#=====================================================================================
    """
    Non-Negative Least Squares using Block Principal Pivoting
    ==========================================================

    Usage:
    ------
        x = nnlsbpp(AtA,AtB,x0)

    Arguments:
    ----------
    AtA (NxN-element matrix)

    AtB (Nx1-element array or Nxk matrix)

    x0  (Nx1 vector or Nxk matrix)
        Initial guess(es) vector

    Returns:
    --------
    x (Nx1-element array or Nxk matrix)
        Solution vector(s)

    References:
    -----------
    [1]
    Portugal, Judice, Vicente, Mathematics of Computation, 1994, 63, 625-643
    A comparison of block pivoting and interior-point algorithms for linear
    least squares problems with nonnegative variables
    https://doi.org/10.1090/S0025-5718-1994-1250776-4

    [2]
    Kim, Park,SIAM J. Sci. Comput. 2011, 33(6), 3261-3281
    Fast Nonnegative Matrix Factorization: An Active-Set-Like Method and Comparisons
    https://doi.org/10.1137/110821172
    """

    # Size checks
    #-------------------------------------------------------------------------------
    n1,n2 = np.shape(AtA)

    if n1 is not n2:
        raise TypeError('AtA must be a square matrix. You gave a ',n1,'x',n2,' matrix.')
    n = np.size(AtB)
    k = 1
    if n is not n1:
        raise TypeError('AtB must have the same number of rows as AtA. You gave ',n,' instead of ',n1)

    if x0 is None:
        x0 = np.linalg.solve(AtA,AtB)

    # Loop over multiple right-hand sides
    #-------------------------------------------------------------------------------
    if k > 1:
        x = np.zeros((n1,k))
        for k_ in reversed(range(k)):
            x[:,k_] = nnlsbpp(AtA,AtB[:,k_],x0[:,k_])
        return  x

    # Calculate initial solution
    #-------------------------------------------------------------------------------
    x = np.zeros(n)
    if not np.all(x0):
        Fset = np.full((n,1),False)
        y = -AtB
    else:
        Fset = x0 > 0
        x = np.zeros(n)
        x[Fset] = np.linalg.solve(AtA[np.ix_(Fset,Fset)],AtB[Fset])
        y = AtA@x - AtB

    # Determine infeasible variables ( = variables with negative values)
    xFnegative = (x < 0) & Fset
    yGnegative = (y < 0) & ~Fset
    nInfeasible = sum(xFnegative | yGnegative)

    # Iterative algorithm
    #-------------------------------------------------------------------------------
    p = 3
    t = np.inf
    while nInfeasible > 0: # iterate until no infeasible variables left

        # Swap full blocks in/out of F set or swap a single element as backup plan.
        if nInfeasible < t:
            t = nInfeasible
            p = 3
            Fset[xFnegative] = False
            Fset[yGnegative] = True
        else:
            if p >= 1:
                p = p - 1
                Fset[xFnegative] = False
                Fset[yGnegative] = True
            else:
                idx_ = np.where(xFnegative | yGnegative)[-1]
                Fset[idx_] = ~Fset[idx_]

        # Solve linear system over F set
        x = np.zeros(n)
        x[Fset] = np.linalg.solve(AtA[np.ix_(Fset,Fset)],AtB[Fset])
        y = AtA@x - AtB

        # Determine infeasible variables
        xFnegative = (x < 0) & Fset
        yGnegative = (y < 0) & ~Fset
        nInfeasible = sum(xFnegative | yGnegative)

    return x
#=====================================================================================


text:
# Copyright 2021 Fedlearn authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
from collections import OrderedDict
import numpy as np
from dataclasses import dataclass, field
from typing import Optional, Union,Dict,List,Any

from datasets import load_dataset

from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_MASKED_LM_MAPPING,
    AutoConfig,
    AutoModelForMaskedLM,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
import math

from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    BertTokenizerFast,
    AlbertModel
    )
import os
import json
import argparse

#_TRAIN_DATA_FILE_='web_text_zh_testa.json'

@dataclass
class DataConfig():
    train_file:str
    val_file:str
    mlm_probability:float
    max_train_samples:int
    max_eval_samples:int
    pad_to_max_length:bool =True
    line_by_line:bool = True
    preprocessing_num_workers:int = 8
    max_seq_length:int = 256

TrainConfig = TrainingArguments

_ROOTDIR_ = 'demos/HFL/example/pytorch/hugging_face'
class BertMlM_Model():
    TEXT_FIELD = 'text'
    # MODEL_NAME_PAIR = {'bert-base-chinese':BertForMaskedLM, 'clue/albert_chinese_tiny':}
    MODEL_NAME = ['bert-base-chinese','uer/chinese_roberta_L-2_H-128']
    def __init__(self,
                 data_config:DataConfig,
                 train_config:TrainConfig,
                 pretrained_model_name='bert-base-chinese',
                 init_from_local_pretrained=False):

        self.data_config = data_config
        self.train_config =  train_config

        self.train_config.output_dir = os.path.join(
              self.train_config.output_dir,
              pretrained_model_name
        )

        # self.train_config.output_dir = \
            # '/'.join(self.train_config.output_dir.split('/')[:-1]) \
                # + '/'+ pretrained_model_name

        if pretrained_model_name not in BertMlM_Model.MODEL_NAME:
            raise(ValueError(f"{pretrained_model_name} is not a supported pretrained model"))

        self.tokenizer =  \
            BertTokenizerFast.from_pretrained(pretrained_model_name)

        model_name = pretrained_model_name \
            if not init_from_local_pretrained \
            else  f'{_ROOTDIR_}/{pretrained_model_name}'

        self.model = \
            BertForMaskedLM.from_pretrained(model_name)

        if self.train_config.do_train or self.train_config.do_eval:
            self.tokenized_datasets = self._load_dataset()

            pad_to_multiple_of_8 = self.data_config.line_by_line and  self.train_config.fp16 and not self.data_config.pad_to_max_length
            self.data_collator = DataCollatorForLanguageModeling(
                                    tokenizer=self.tokenizer,
                                    mlm_probability=self.data_config.mlm_probability,
                                    mlm = True,
                                    pad_to_multiple_of=8 if pad_to_multiple_of_8 else None,
                                )

        if self.train_config.do_train:
            if "train" not in self.tokenized_datasets:
                raise ValueError("--do_train requires a train dataset")
            self.train_dataset = self.tokenized_datasets["train"].shuffle()
            if data_config.max_train_samples is not None:
                self.train_dataset = self.train_dataset.select(range(self.data_config.max_train_samples))

        if self.train_config.do_eval:
            if "validation" not in self.tokenized_datasets:
                raise ValueError("--do_eval requires a validation dataset")
            self.eval_dataset = self.tokenized_datasets["validation"]
            if self.data_config.max_eval_samples is not None:
                self.eval_dataset = self.eval_dataset.select(range(self.data_config.max_eval_samples))

    def train(self):
        trainer = Trainer(
            model=self.model,
            args=self.train_config,
            train_dataset=self.train_dataset if self.train_config.do_train else None,
            eval_dataset=self.eval_dataset if self.train_config.do_eval else None,
            tokenizer=self.tokenizer,
            data_collator=self.data_collator,
        )

        num_train_sample = 1

        # Training
        #if self.train_config.do_train:

        checkpoint = self.train_config.resume_from_checkpoint

        train_result = trainer.train(resume_from_checkpoint=checkpoint)
        trainer.save_model()  # Saves the tokenizer too for easy upload

        metrics = train_result.metrics

        max_train_samples = self.data_config.max_train_samples \
                if self.data_config.max_train_samples is not None \
                else len(self.train_dataset)
        metrics["train_samples"] = min(max_train_samples, len(self.train_dataset))

        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

        num_train_sample = metrics["train_samples"]
        #if self.train_config.do_eval:
        # logger.info("*** Evaluate ***")

        # Evaluation
        metrics = trainer.evaluate()

        max_eval_samples = self.data_config.max_eval_samples if self.data_config.max_eval_samples is not None else len(self.eval_dataset)
        metrics["eval_samples"] = min(max_eval_samples, len(self.eval_dataset))
        try:
            perplexity = math.exp(metrics["eval_loss"])
        except OverflowError:
            perplexity = float("inf")

        metrics["perplexity"] = perplexity

        trainer.log_metrics("eval", metrics)
        trainer.save_metrics("eval", metrics)

        return_matrics ={
            'eval_loss':metrics["eval_loss"],
            'perplexity':metrics['perplexity'],
            'train_samples':num_train_sample
        }

        return self.get_model_parameters(), return_matrics

    def inference(self, inputs_sents:List[str])->Dict[str,List[Any]]:
        """ Retreive CLS vector (last layer's first vector), shape: [num_sents, vec_length:768 for regular Bert] """
        self.model.eval()

        outputs = self.model(**self.tokenizer(
                                            inputs_sents,
                                            return_tensors="pt",
                                            padding=True),
                            output_hidden_states=True)['hidden_states'][-1][:,0]
        np_list=[d.detach().cpu().numpy() for d in outputs]
        return {'CLS': np_list}

    def _load_dataset(self):
        extension = self.data_config.train_file.split(".")[-1]
        if extension == "txt":
            extension = "text"

        datasets = \
            load_dataset(
                extension,
                data_files={'train':self.data_config.train_file,
                            'validation':self.data_config.val_file},
                cache_dir='./cache'

                )

        padding = "max_length" if self.data_config.pad_to_max_length else False

        def tokenize_function(examples):
            field = BertMlM_Model.TEXT_FIELD
            examples[field] = [str(line) for line in examples[field] if len(line) > 0 and not line.isspace()]
            return self.tokenizer(
                examples[field],
                padding=padding,
                truncation=True,
                max_length=self.data_config.max_seq_length,
                # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
                # receives the `special_tokens_mask`.
                return_special_tokens_mask=True,
            )

        tokenized_datasets = datasets.map(
            tokenize_function,
            batched=True,
            #num_proc=self.data_config.preprocessing_num_workers,
            num_proc=1,
            load_from_cache_file=False
            #remove_columns=['qid', 'title', 'desc', 'topic', 'star', 'answer_id', 'answerer_tags']
            #remove_columns=['news_id', 'keywords', 'desc', 'title', 'source', 'time']
        )
        return tokenized_datasets

    def set_model_parameters(self,
                            params:Dict[str,np.ndarray],
                            mode='train'):
        if mode =='train':
            self.model.train()
        else:
            self.model.eval()

        state_dict = OrderedDict({k: torch.Tensor(v) for k, v in params.items()})
        self.model.load_state_dict(state_dict, strict=False)

    def get_model_parameters(self)->Dict[str,np.ndarray]:
        return {
            name: val.cpu().numpy()
            for name, val in self.model.state_dict().items()
            #if "bn" not in name
            }

    def eval(self):
        pass

def get_configs():

    config_file ='local_config.json'
    with open(os.path.join(_ROOTDIR_,config_file)) as f:
        configs = json.load(f)

    # configs['train_config']['output_dir'] = os.path.join(_ROOTDIR_, configs['train_config']['output_dir'])
    #
    input_configs = {'data_config': DataConfig(**configs['data_config']),
                'train_config':TrainConfig(**configs['train_config'])}
    for k,v in configs.items():
        if k not in ['data_config','train_config']:
            input_configs[k] = v

    return input_configs

def local_train():
    model = BertMlM_Model(**get_configs())
    model.train()

def local_predict():

    configs = get_configs()
    configs['train_config'].do_eval =False
    configs['train_config'].do_train=False

    model = BertMlM_Model(**configs,
                        pretrained_model_name='bert-base-chinese',
                        init_from_local_pretrained=True)

    text_test_file = f'{_ROOTDIR_}/test_sentences.txt'
    text_output_file = f'{_ROOTDIR_}/output_test_sentences.txt'
    texts = [line.strip() for line in  open(text_test_file,'r')]
    with torch.no_grad():
        cls_represnts:Dict[str,List] = model.inference(inputs_sents=texts)

    with open(text_output_file ,'w') as f:
        for v in cls_represnts['CLS']:
            f.write(str(v.tolist()) +'\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--predict_or_train', type=str,default='train',choices=['predict','train'])

    args = parser.parse_args()
    if args.predict_or_train == 'predict':
        local_predict()
    else:
        local_train()

max_stars_repo_name: yunqi-li/fedlearn-algo

max_issues_repo_name: yunqi-li/fedlearn-algo

max_forks_repo_name: yunqi-li/fedlearn-algo


text:
#!/usr/bin/env python
#=============================================================================#
#                                                                             #
# NAME:     util_tables.py                                                    #
#                                                                             #
# PURPOSE:  Helper functions and classes for manipulating numpy recarrays     #
#           and sqlite3 databases.                                            #
#                                                                             #
# MODIFIED: 23-Mar-2017 by C. Purcell                                         #
#                                                                             #
# CONTENTS:                                                                   #
#                                                                             #
#  register_sqlite3_numpy_dtypes ... setup sqlite3 to read numpy arrays       #
#  regexp                   ... regualr expression function for sqlite3       #
#  schema_to_dtypes         ... parse the schema file for numpy dtypes        #
#  columns_view             ... return a recarray view filtered by column     #
#  insert_arr_db            ... insert all recarray entries into the database #
#  update_arr_db            ... update DB entries using a recarray            #
#  select_into_arr          ... run a SQL query and return a numpy recarray   #
#  select_into_lod          ... run a SQL query and return a list-of-dicts    #
#  rowlst_to_rowdict        ... convert a LOD to a dictionary of dictionaries #
#                                                                             #
#=============================================================================#
import os
import sys
import time
import re
import numpy as np
import sqlite3

#-----------------------------------------------------------------------------#
def register_sqlite3_numpy_dtypes():
    """
    Map numpy data-types to the limited sqlite data-types. This must be called
    before using the sqlite3 database or INSERT statements will fail.
    """

    for t in (np.int8, np.int16, np.int32, np.int64,
          np.uint8, np.uint16, np.uint32, np.uint64):
        sqlite3.register_adapter(t, long)
    for t in (np.float16, np.float32, np.float64,
          np.float128, np.double):
        sqlite3.register_adapter(t, float)

#-----------------------------------------------------------------------------#
def regexp(expr, item):
    reg = re.compile(expr)
    return reg.search(item) is not None

#-----------------------------------------------------------------------------#
def schema_to_dtypes(schemaFile):
    """
    Parse the SQL file containing the CREATE TABLE definitions and convert to
    a dtype format that python can understand. This can be used to initialise
    numpy record arrays for storing the tables in memory. Assumes the
    create statement has the form:

    'CREATE TABLE myTableName (entry1 double [args+], entry2 int(2), ... );'

    The 'CREATE TABLE' statement must have the same case. WARNING: recognises
    only , 'DOUBLE', 'INTEGER' and 'VARCHAR(n)' types in the SQL statement.
    """

    # Return these
    tableNameLst = []
    tableDefDict = {}
    tableSQLdict = {}

    # Compile a few useful regular expressions
    spaces = re.compile('\s+')
    commaAndSpaces = re.compile(',\s+')
    comment = re.compile('#.*')
    quotes = re.compile('\'[^\']*\'')
    brackets = re.compile('[\[|\]\(|\)|\{|\}]')
    createRe =  re.compile('^(CREATE TABLE|create table) (\w+)\s*\((.+)\)\s*$')

    # Translation function for data types: SQL to recarray
    # RecArray dtypes:
    #   bytes                = b<n>, e.g. b1
    #   ints                 = i<n>, e.g. i2, i4, i8,
    #   unsigned ints        = u<n>, e.g. u1, u2, u4, u8
    #   floats               = f<n>, e.g. f2, f4, f8
    #   complex              = c<n>, e.g. c8, c16
    #   fixed length strings = a<n>, e.g. a10, a100
    # where <n> is the number of bytes / chars, so float32=f4, float64=f8
    def trtype(dtype):
        floatRe = re.compile('^FLOAT')
        doubleRe = re.compile('^DOUBLE')
        intRe = re.compile('^INT')
        charRe = re.compile('^VARCHAR\((\d+)\)')
        if floatRe.match(dtype):
            return 'f4'
        if doubleRe.match(dtype):
            return 'f8'
        if intRe.match(dtype):
            return 'i8'
        mch = charRe.match(dtype)
        if mch:
             return 'a' + mch.group(1)
        return 'f8'    # default to float64

    # Loop through the SQL statements
    sqlLst = open(schemaFile).read().split(';')
    for sql in sqlLst:

        # Simplify the SQL statement
        sql = sql.replace('\r', ' ')        # kill carriage-return
        sql = sql.replace('\n', ' ')        # kill newlines
        sql = sql.strip()                   # kill external whitespace
        sql = spaces.sub(' ', sql)          # shrink internal whitespaces
        sql = commaAndSpaces.sub(',', sql)  # kill ambiguous spaces

        mch = createRe.match(sql)
        if mch:
            tableName = mch.group(2)
            tableSQLdict[tableName] = mch.group(0)
            tableNameLst.append(tableName)
            colDefLst = mch.group(3).strip().split(',')
            colDefLst = [x.split(' ')[:2] for x in colDefLst]

            # Translate the data types into a python recarray dtype list
            for i in range(len(colDefLst)):
                colDefLst[i][1] = trtype(colDefLst[i][1])
                colDefLst[i] = tuple(colDefLst[i])

            # Add to the table definition dictionary
            tableDefDict[tableName] = colDefLst

    return tableDefDict, tableSQLdict

#-----------------------------------------------------------------------------#
def columns_view(arr, colNameLst=None):
    """
    Return a view of a numpy record array containing only the column names in
    the colNameLst argument. 'colNameLst' should be a list of column names.
    """

    # Default to all column
    if not colNameLst:
        colNameLst = arr.dtype.names

    dtype2 = np.dtype({name:arr.dtype.fields[name] for name in colNameLst})
    return np.ndarray(arr.shape, dtype2, arr, 0, arr.strides)

#-----------------------------------------------------------------------------#
def insert_arr_db(cursor, recArr, tabName, colNameLst=None,
                     insertSQL="INSERT OR REPLACE"):
    """
    Insert a numpy recarray into a database via a cursor object. It is assumed
    that the columns in the recarray and database have the same names and
    compatable datatypes. If the recarray contains columns NOT in the database
    the user must provide a list of existing columns to be to be inserted.
    """

    # Default to all cols
    if not colNameLst:
        colNameLst = recArr.dtype.names

    sql = '%s INTO %s (%s) ' % (insertSQL, tabName, ', '.join(colNameLst))
    sql += 'VALUES(%s) ' % (', '.join(['?']*len(colNameLst)))

    #print sql
    #print columns_view(recArr, colNameLst)
    #raw_input()
    cursor.executemany(sql, columns_view(recArr, colNameLst))

#-----------------------------------------------------------------------------#
def update_arr_db(cursor, recArr, tabName, keyName, colNameLst=None):
    """
    Do an UPDATE on existing rows. Unlike the INSERT OR REPLACE function
    above, this only updates the columns given in colNameLst.
    """

    # Default to all fields
    if not colNameLst:
        colNameLst = list(recArr.dtype.names)

    # Key must exist in list of field names
    if not keyName in colNameLst:
        print "ERR: Key '%s' not in column list" % keyName
        return

    # Remove the key from the list and format the SQL
    colNameLst.remove(keyName)
    sql = 'UPDATE %s SET ' % tabName
    sql += '=?, '.join(colNameLst) + '=? '
    sql += 'WHERE %s = ?' % keyName

    # Attach the key to the end of the field list and fetch a view
    # Use fancy indexing to force the key to the last column
    colNameLst.append(keyName)
    a = fields_view(recArr, colNameLst)[colNameLst]
    cursor.executemany(sql, a)

#-----------------------------------------------------------------------------#
def select_into_arr(cursor, sql, args=[]):
    """
    Run a SQL query and return a numpy recordarray.
    """

    if args == []:
        cursor.execute(sql)
    else:
        cursor.execute(sql, tuple(args))
    try:
        rows = cursor.fetchall()
        if len(rows) == 0:
            rows = np.array([], dtype='i4')
        else:
            columnNameLst = zip(*cursor.description)[0]
            rows = np.rec.fromrecords(rows, names=columnNameLst)
        return rows
    except Exception:
        print "WARNING: failed to convert SQL result to a recarray!"
        return None

#-----------------------------------------------------------------------------#
def select_into_lod(cursor, sql, args=[]):
    """
    Run a SQL query and return a List of Dictionaries.
    """

    rowLst = []
    if args == []:
        cursor.execute(sql)
    else:
        cursor.execute(sql, tuple(args))
    columnNameLst = zip(*cursor.description)[0]
    rows = cursor.fetchall()
    for row in rows:
        e = {}
        for i in range(len(columnNameLst)):
            e[columnNameLst[i]] = row[i]
        rowLst.append(e)
    return rowLst

#-----------------------------------------------------------------------------#
def rowlst_to_rowdict(rowLst, keyName):
    """
    Convert a list of dictionaries into a dictionary of dictionaries.
    """

    rowDict = {}
    for e in rowLst:
        key = e.pop(keyName)
        rowDict[key] = e

    return rowDict

max_stars_repo_name: crpurcell/CORNISH-S-PIPELINE

max_issues_repo_name: crpurcell/CORNISH-S-PIPELINE

max_forks_repo_name: crpurcell/CORNISH-S-PIPELINE


text:
import cv2
import numpy as np

url='http://192.168.1.33:8080/video'

def nothing(x):
 pass
cv2.namedWindow("Trackbars")
cv2.createTrackbar("L-H", "Trackbars", 43, 180, nothing)
cv2.createTrackbar("L-S", "Trackbars", 55, 255, nothing)
cv2.createTrackbar("L-V", "Trackbars", 19, 255, nothing)
cv2.createTrackbar("U-H", "Trackbars", 86, 180, nothing)
cv2.createTrackbar("U-S", "Trackbars", 255, 255, nothing)
cv2.createTrackbar("U-V", "Trackbars", 255, 255, nothing)

font = cv2.FONT_HERSHEY_COMPLEX

cap = cv2.VideoCapture(0) #use url in case

while True:

    _, frame = cap.read()
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
    l_h = cv2.getTrackbarPos("L-H", "Trackbars")
    l_s = cv2.getTrackbarPos("L-S", "Trackbars")
    l_v = cv2.getTrackbarPos("L-V", "Trackbars")
    u_h = cv2.getTrackbarPos("U-H", "Trackbars")
    u_s = cv2.getTrackbarPos("U-S", "Trackbars")
    u_v = cv2.getTrackbarPos("U-V", "Trackbars")

    lower_green = np.array([l_h, l_s, l_v])
    upper_green = np.array([u_h, u_s, u_v])
    mask = cv2.inRange(hsv, lower_green, upper_green)
    contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    for cnt in contours:
        area = cv2.contourArea(cnt)
        epsilon = 0.02 * cv2.arcLength(cnt, True)
        approx = cv2.approxPolyDP(cnt, epsilon, True)
        x = approx.ravel()[0]
        y = approx.ravel()[1]
        # print(area)

        if area > 400:
            cv2.drawContours(frame, [approx], 0, (0, 0, 0), 5)
            c = max(contours, key=cv2.contourArea)
            M = cv2.moments(c)
            cx = int(M["m10"] / area)
            print(cx)
            # print("forward")
            if area > 20000:
                # for i in range(10000):
                print("stop")

            elif cx > 600:
                print("right")

            elif cx < 300:
                print("left")
            elif 300 < cx < 600:
                print("forward")

    cv2.imshow("Frame", frame)
    cv2.imshow("Mask", mask)
    key = cv2.waitKey(1)
    if key == 27:
         break

cap.release()
cv2.destroyAllWindows()

max_stars_repo_path: using opencv/opencv.py

max_issues_repo_path: using opencv/opencv.py

max_forks_repo_path: using opencv/opencv.py