text:
import torch
import torch.nn as nn
import numpy as np
# from lib.nms.pth_nms import pth_nms

# def nms(dets, thresh):
#     "Dispatch to either CPU or GPU NMS implementations.\
#     Accept dets as tensor""
#     return pth_nms(dets, thresh)

def nms(dets, thresh):
    x1 = dets[:, 0].cpu().detach().numpy()
    y1 = dets[:, 1].cpu().detach().numpy()
    x2 = dets[:, 2].cpu().detach().numpy()
    y2 = dets[:, 3].cpu().detach().numpy()
    scores = dets[:, 4].cpu().detach().numpy()

    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        i = order[0]
        keep.append(i)
        intersect_x1 = np.maximum(x1[i], x1[order[1:]])
        intersect_y1 = np.maximum(y1[i], y1[order[1:]])
        intersect_x2 = np.minimum(x2[i], x2[order[1:]])
        intersect_y2 = np.minimum(y2[i], y2[order[1:]])

        intersect_w = np.maximum(0.0, intersect_x2 - intersect_x1)
        intersect_h = np.maximum(0.0, intersect_y2 - intersect_y1)
        inter = intersect_h * intersect_w
        IoU = inter / (areas[i] + areas[order[1:]] - inter)

        inds = np.where(IoU <= thresh)[0]
        order = order[inds + 1]

    return keep

def soft_nms(dets, thresh, type='gaussian'):
    x1 = dets[:, 0]
    y1 = dets[:, 1]
    x2 = dets[:, 2]
    y2 = dets[:, 3]
    scores = dets[:, 4]
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    order = scores.argsort()[::-1]
    scores = scores[order]

    keep = []
    while order.size > 0:
        i = order[0]
        dets[i, 4] = scores[0]
        keep.append(i)

        xx1 = np.maximum(x1[i], x1[order[1:]])
        yy1 = np.maximum(y1[i], y1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])
        yy2 = np.minimum(y2[i], y2[order[1:]])

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = w * h
        ovr = inter / (areas[i] + areas[order[1:]] - inter)

        order = order[1:]
        scores = scores[1:]

        if type == 'linear':
            inds = np.where(ovr > thresh)[0]
            scores[inds] *= (1 - ovr[inds])
        else:
            scores *= np.exp(- ovr ** 2 / thresh)

        inds = np.where(scores > 1e-3)[0]
        order = order[inds]
        scores = scores[inds]

        tmp = scores.argsort()[::-1]
        order = order[tmp]
        scores = scores[tmp]

    return keep

def soft_nms_with_var_voting(dets, confidence, thresh):
    confidence = np.exp(confidence / 2.)
    sigma_t = 0.01
    N = len(dets)
    x1 = dets[:, 0].cpu().detach().numpy().copy()
    y1 = dets[:, 1].cpu().detach().numpy().copy()
    x2 = dets[:, 2].cpu().detach().numpy().copy()
    y2 = dets[:, 3].cpu().detach().numpy().copy()
    scores = dets[:, 4].cpu().detach().numpy().copy()
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    ious = np.zeros((N, N))
    for i in range(N):
        xx1 = np.maximum(x1[i], x1)
        yy1 = np.maximum(y1[i], y1)
        xx2 = np.maximum(x2[i], x2)
        yy2 = np.minimum(y2[i], y2)
        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = w * h
        ovr = inter / (areas[i] + areas - inter)
        ious[i, :] = ovr.copy()
    i = 0
    while i < N:
        maxpos = dets[i:N, 4].argmax()
        maxpos += i
        dets[[maxpos, i]] = dets[[i, maxpos]]
        confidence[[maxpos, i]] = confidence[[i, maxpos]]
        ious[[maxpos, i]] = ious[[i, maxpos]]
        ious[:, [maxpos, i]] = ious[:, [i, maxpos]]

        ovr_bbox = np.where((ious[i, :N] > 0.01))[0] + i
        p = np.exp(-(1 - ious[i, ovr_bbox]) ** 2 / sigma_t)
        dets[i, :4] = p.dot(dets[ovr_bbox, :4] / confidence[ovr_bbox] ** 2) / \
            p.dot(1. / confidence[ovr_bbox] ** 2)
        pos = i + 1
        while pos < N:
            if ious[i, pos] > 0:
                ovr = ious[i, pos]
                dets[pos, 4] *= np.exp(-(ovr * ovr) / thresh)
                if dets[pos, 4] < 0.001:
                    dets[[pos, N - 1]] = dets[[N - 1, pos]]
                    confidence[[pos, N - 1]] = confidence[[N - 1, pos]]
                    ious[[pos, N - 1]] = ious[[N - 1, pos]]
                    ious[:, [pos, N - 1]] = ious[:, [N - 1, pos]]
                    N -= 1
                    pos -= 1
            pos += 1
        i += 1
    keep = [i for i in range(N)]
    return dets[keep]

def box_iou(box1, box2, order='xxyy'):
    if order == 'xywh':
        box1 = change_box_order(box1, 'xywh2xyxy')
        box2 = change_box_order(box2, 'xywh2xyxy')

    N = box1.size(0)
    M = box2.size(0)

    lt = torch.max(box1[:,None,:2], box2[:,:2])  # [N,M,2]
    rb = torch.min(box1[:,None,2:], box2[:,2:])  # [N,M,2]

    wh = (rb-lt+1).clamp(min=0)      # [N,M,2]
    inter = wh[:,:,0] * wh[:,:,1]  # [N,M]

    area1 = (box1[:,2]-box1[:,0]+1) * (box1[:,3]-box1[:,1]+1)  # [N,]
    area2 = (box2[:,2]-box2[:,0]+1) * (box2[:,3]-box2[:,1]+1)  # [M,]
    iou = inter / (area1[:,None] + area2 - inter)
    return iou

def box_c(box1, box2, order='xxyy'):
    if order == 'xxyy':
        b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2
        b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2
        b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2
        b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2
    else:
        # Get the coordinates of bounding boxes
        b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]
        b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]

    xmin = torch.min(b1_x1,b2_x1)
    ymin = torch.min(b1_y1,b2_y1)
    xmax = torch.max(b1_x2,b2_x2)
    ymax = torch.max(b1_y2,b2_y2)

    return (xmin,ymin,xmax,ymax)

def box_union(box1,box2,order='xxyy'):
    if order == 'xxyy':
        # Transform from center and width to exact coordinates
        b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2
        b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2
        b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2
        b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2
    else:
        # Get the coordinates of bounding boxes
        b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]
        b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]

    box1_area = (b1_y2 - b1_y1) * (b1_x2 - b1_x1)
    box2_area = (b2_y2 - b2_y1) * (b2_x2 - b2_x1)
    intersect = (torch.min(b2_x2,b1_x2) - torch.max(b1_x1,b2_x1)) * (torch.min(b2_y2,b1_y2) - torch.max(b1_y1,b2_y1))
    union_item = (box1_area + box2_area) - intersect + 1e-16

    return union_item

def giou(bbox1,bbox2,order='xxyy'):
    iou_item  = bbox_iou(bbox1, bbox2, order)
    boxc_item = box_c(bbox1, bbox2, order)
    boxc_area = (boxc_item[2] - boxc_item[0]) * (boxc_item[3] - boxc_item[1]) +  1e-7

    if boxc_area.eq(0).sum() > 0 :
        return iou_item
    u = box_union(bbox1, bbox2, order)
    giou_item = iou_item - (boxc_area - u) / boxc_area
    return giou_item

def change_box_order(boxes, order):
    assert order in ['xyxy2xywh','xywh2xyxy']
    a = boxes[:,:2]
    b = boxes[:,2:]
    if order == 'xyxy2xywh':
        return torch.cat([(a+b)/2,b-a+1], 1)
    return torch.cat([a-b/2,a+b/2], 1)

def meshgrid(x, y, row_major=True):
    a = torch.arange(0,x)
    b = torch.arange(0,y)
    xx = a.repeat(y).view(-1,1)
    yy = b.view(-1,1).repeat(1,x).view(-1,1)
    return torch.cat([xx,yy],1) if row_major else torch.cat([yy,xx],1)

def box_nms(bboxes, scores, threshold=0.9, mode='union'):
    x1 = bboxes[:, 0].cpu().detach().numpy()
    y1 = bboxes[:, 1].cpu().detach().numpy()
    x2 = bboxes[:, 2].cpu().detach().numpy()
    y2 = bboxes[:, 3].cpu().detach().numpy()
    scores = scores.cpu().detach().numpy()

    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        i = order[0]
        keep.append(i)
        intersect_x1 = np.maximum(x1[i], x1[order[1:]])
        intersect_y1 = np.maximum(y1[i], y1[order[1:]])
        intersect_x2 = np.minimum(x2[i], x2[order[1:]])
        intersect_y2 = np.minimum(y2[i], y2[order[1:]])

        intersect_w = np.maximum(0.0, intersect_x2 - intersect_x1)
        intersect_h = np.maximum(0.0, intersect_y2 - intersect_y1)
        inter = intersect_h * intersect_w
        IoU = inter / (areas[i] + areas[order[1:]] - inter)

        inds = np.where(IoU <= threshold)[0]
        order = order[inds + 1]

    return torch.LongTensor(keep)

class BBoxTransform(nn.Module):

    def __init__(self, mean=None, std=None):
        super(BBoxTransform, self).__init__()
        if mean is None:
            self.mean = torch.from_numpy(np.array([0, 0, 0, 0]).astype(np.float32)).cuda()
        else:
            self.mean = mean
        if std is None:
            self.std = torch.from_numpy(np.array([0.1, 0.1, 0.2, 0.2]).astype(np.float32)).cuda()
        else:
            self.std = std

    def forward(self, boxes, deltas):

        widths  = boxes[:, :, 2] - boxes[:, :, 0]
        heights = boxes[:, :, 3] - boxes[:, :, 1]
        ctr_x   = boxes[:, :, 0] + 0.5 * widths
        ctr_y   = boxes[:, :, 1] + 0.5 * heights

        dx = deltas[:, :, 0] * self.std[0] + self.mean[0]
        dy = deltas[:, :, 1] * self.std[1] + self.mean[1]
        dw = deltas[:, :, 2] * self.std[2] + self.mean[2]
        dh = deltas[:, :, 3] * self.std[3] + self.mean[3]

        pred_ctr_x = ctr_x + dx * widths
        pred_ctr_y = ctr_y + dy * heights
        pred_w     = torch.exp(dw) * widths
        pred_h     = torch.exp(dh) * heights

        pred_boxes_x1 = pred_ctr_x - 0.5 * pred_w
        pred_boxes_y1 = pred_ctr_y - 0.5 * pred_h
        pred_boxes_x2 = pred_ctr_x + 0.5 * pred_w
        pred_boxes_y2 = pred_ctr_y + 0.5 * pred_h

        pred_boxes = torch.stack([pred_boxes_x1, pred_boxes_y1, pred_boxes_x2, pred_boxes_y2], dim=2)

        return pred_boxes

class ClipBoxes(nn.Module):

    def __init__(self, width=None, height=None):
        super(ClipBoxes, self).__init__()

    def forward(self, boxes, img):

        batch_size, num_channels, height, width = img.shape

        boxes[:, :, 0] = torch.clamp(boxes[:, :, 0], min=0)
        boxes[:, :, 1] = torch.clamp(boxes[:, :, 1], min=0)

        boxes[:, :, 2] = torch.clamp(boxes[:, :, 2], max=width)
        boxes[:, :, 3] = torch.clamp(boxes[:, :, 3], max=height)

        return boxes

# class BasicBlock(nn.Module):
#     expansion = 1

#     def __init__(self, inplanes, planes, stride=1, downsample=None):
#         super(BasicBlock, self).__init__()
#         self.conv1 = conv3x3(inplanes, planes, stride)
#         self.bn1 = nn.BatchNorm2d(planes)
#         self.relu = nn.ReLU(inplace=True)
#         self.conv2 = conv3x3(planes, planes)
#         self.bn2 = nn.BatchNorm2d(planes)
#         self.downsample = downsample
#         self.stride = stride

#     def forward(self, x):
#         residual = x

#         out = self.conv1(x)
#         out = self.bn1(out)
#         out = self.relu(out)

#         out = self.conv2(out)
#         out = self.bn2(out)

#         if self.downsample is not None:
#             residual = self.downsample(x)

#         out += residual
#         out = self.relu(out)

#         return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out


text:
# %%
from agoge import InferenceWorker
import threading
import torch
import time
import numpy as np
from tqdm import tqdm
import jack
from matplotlib import pyplot as plt
from itertools import cycle
worker = InferenceWorker('/home/nintorac/agoge/artifacts/bluesy-chestnut-forest_0_2020-04-30_11-11-00x02v59be/checkpoint_220/model.box', with_data=True)

model = worker.model
data = worker.dataset
loader = data.loaders.test

n_samples = 32
n_latents = 8
loader.batch_sampler.batch_size = n_samples
features_all = []
features_half = []
for x in map(lambda x: x['x'], tqdm(loader)):
    q = model.features(x, torch.ones_like(x.float()).bool())
    features_all += [(q.mean.numpy(), q.stddev.numpy())]
    # features_half += [model.features(x, torch.rand_like(x.float())>torch.linspace(0, 1, 32).unsqueeze(-1)).mean.numpy()]

# for item in loader:
# %%

ext: py

reason: import numpy


text:
import sympy as sym

def shuffle_product(x, y):
    """
    Calculates shuffle product of two elements.
    """
    if '.' not in x:
        if '.' not in y:
            return x + '.' + y + '|' + y + '.' + x
        else:
            y_split = y.split('.')
            res_1 = x + '.' + y
            res_2 = '|'.join(map(lambda z: y_split[0] + '.' + z, shuffle_product('.'.join(y_split[1:]), x).split('|')))
            return res_1 + '|' + res_2
    elif '.' not in y:
        x_split = x.split('.')
        res_1 = '|'.join(map(lambda z: x_split[0] + '.' + z, shuffle_product('.'.join(x_split[1:]), y).split('|')))
        res_2 = y + '.' + x
        return res_1 + '|' + res_2
    else:
        x_split = x.split('.')
        y_split = y.split('.')
        res_1 = '|'.join(map(lambda z: x_split[0] + '.' + z, shuffle_product('.'.join(x_split[1:]), y).split('|')))
        res_2 = '|'.join(map(lambda z: y_split[0] + '.' + z, shuffle_product('.'.join(y_split[1:]), x).split('|')))
        return res_1 + '|' + res_2

def calc_shuffle_lin_comb(x='', y='', x_count=0, y_count=0):
    """
    Calculates shuffle product of linear combination of two moments. x_count, y_count - counts of times an element
    appears in product.
    """
    if not x and y_count == 1:
        return y
    elif not y and x_count == 1:
        return x
    elif x_count == 0:
        if y_count == 0:
            return ''
        elif y_count == 1:
            return y
    elif y_count == 0 and x_count == 1:
        return x
    x_times = x_count - 1
    y_times = y_count - 1
    if x and x_times >= 0:
        x_split = x.split('|')
        res = [x]
        times = 1
        while times <= x_times:
            new_res = []
            for element in res:
                temp = element.split('|')
                for moment_1_repr in temp:
                    if 'x' not in moment_1_repr:
                        if moment_1_repr.startswith('-'):
                            moment_1, coef_1 = moment_1_repr[1:], sym.Rational('-1')
                        else:
                            moment_1, coef_1 = moment_1_repr, sym.Rational('1')
                    else:
                        split_moment_1 = moment_1_repr.split('x')
                        moment_1, coef_1 = split_moment_1[1], sym.Rational(split_moment_1[0])
                    for moment_2_repr in x_split:
                        if 'x' not in moment_2_repr:
                            if moment_2_repr.startswith('-'):
                                moment_2, coef_2 = moment_2_repr[1:], sym.Rational('-1')
                            else:
                                moment_2, coef_2 = moment_2_repr, sym.Rational('1')
                        else:
                            split_moment_2 = moment_2_repr.split('x')
                            moment_2, coef_2 = split_moment_2[1], sym.Rational(split_moment_2[0])
                        coef = str(coef_1 * coef_2)
                        shuffle_prod = coef + 'x' + shuffle_product(moment_1, moment_2)
                        new_res.append(shuffle_prod.replace('|', '|' + coef + 'x'))
            times += 1
            res = new_res.copy()
        res = '|'.join(res)
        res = res.split('|')
        times = 0
    else:
        res = [y]
        times = 1
    y_split = y.split('|')
    while times <= y_times:
        new_res = []
        for element in res:
            temp = element.split('|')
            for moment_1_repr in temp:
                if 'x' not in moment_1_repr:
                    if moment_1_repr.startswith('-'):
                        moment_1, coef_1 = moment_1_repr[1:], sym.Rational('-1')
                    else:
                        moment_1, coef_1 = moment_1_repr, sym.Rational('1')
                else:
                    split_moment_1 = moment_1_repr.split('x')
                    moment_1, coef_1 = split_moment_1[1], sym.Rational(split_moment_1[0])
                for moment_2_repr in y_split:
                    if 'x' not in moment_2_repr:
                        if moment_2_repr.startswith('-'):
                            moment_2, coef_2 = moment_2_repr[1:], sym.Rational('-1')
                        else:
                            moment_2, coef_2 = moment_2_repr, sym.Rational('1')
                    else:
                        split_moment_2 = moment_2_repr.split('x')
                        moment_2, coef_2 = split_moment_2[1], sym.Rational(split_moment_2[0])
                    coef = str(coef_1 * coef_2)
                    shuffle_prod = coef + 'x' + shuffle_product(moment_1, moment_2)
                    new_res.append(shuffle_prod.replace('|', '|' + coef + 'x'))
        times += 1
        res = new_res.copy()
    return '|'.join(res)

def calculate_combinations(current_order, previous_elements):
    length = len(previous_elements)
    res = []
    for ind in range((current_order + 1) ** length):
        degrees = []
        divisor = ind
        sum_degr = 0
        for i in range(1, length + 1):
            denom = (current_order + 1) ** (length - i)
            value = divisor // denom
            degrees.append(value)
            divisor = divisor % denom
            sum_degr += previous_elements[i - 1]['order'] * value
        if sum_degr == current_order:
            res.append(degrees)
    return res

max_stars_repo_name: ViktorRusakov/napalm-control

max_issues_repo_name: ViktorRusakov/napalm-control

max_forks_repo_name: ViktorRusakov/napalm-control

reason: import sympy


text:
import os
import logging
import time
from contextlib import contextmanager
from pathlib import Path
from typing import IO, Iterator, Optional

import numpy as np

from sciencebeam_trainer_delft.sequence_labelling.tag_formatter import (
    TagOutputFormats,
    format_tag_result
)

LOGGER = logging.getLogger(__name__)

SCIENCEBEAM_DELFT_TAGGING_DEBUG_OUT = "SCIENCEBEAM_DELFT_TAGGING_DEBUG_OUT"

@contextmanager
def exclusive_prefixed_file(prefix: str, suffix: str = '') -> Iterator[IO]:
    for index in range(1, 10000):
        filename = '%s-%d%s' % (prefix, index, suffix)
        try:
            with open(filename, mode='x', encoding='utf-8') as fileobj:
                yield fileobj
                return
        except FileExistsError:
            continue
    raise FileExistsError('could not create any prefixed file: %s, suffix: %s' % (prefix, suffix))

class TagDebugReporter:
    def __init__(self, output_directory: str):
        self.output_directory = output_directory

    def get_base_output_name(self, model_name: str) -> str:
        return os.path.join(self.output_directory, 'sciencebeam-delft-%s-%s' % (
            round(time.time()),
            model_name
        ))

    def report_tag_results(
            self,
            texts: np.array,
            features: np.array,
            annotations,
            model_name: str):
        base_filename_prefix = self.get_base_output_name(model_name=model_name)
        with exclusive_prefixed_file(base_filename_prefix, '.json') as json_fp:
            output_file = json_fp.name
            filename_prefix = os.path.splitext(output_file)[0]
            LOGGER.info('tagger, output_file: %s', output_file)

            format_tag_result_kwargs = dict(
                tag_result=annotations,
                texts=texts,
                features=features,
                model_name=model_name
            )

            formatted_text = format_tag_result(
                output_format=TagOutputFormats.TEXT,
                **format_tag_result_kwargs
            )
            Path(filename_prefix + '.txt').write_text(formatted_text, encoding='utf-8')

            formatted_json = format_tag_result(
                output_format=TagOutputFormats.JSON,
                **format_tag_result_kwargs
            )
            json_fp.write(formatted_json)

            formatted_xml = format_tag_result(
                output_format=TagOutputFormats.XML,
                **format_tag_result_kwargs
            )
            Path(filename_prefix + '.xml').write_text(formatted_xml, encoding='utf-8')

            if features is not None:
                formatted_data = format_tag_result(
                    output_format=TagOutputFormats.DATA,
                    **format_tag_result_kwargs
                )
                Path(filename_prefix + '.data').write_text(formatted_data, encoding='utf-8')

def get_tag_debug_reporter_if_enabled() -> Optional[TagDebugReporter]:
    output_directory = os.environ.get(SCIENCEBEAM_DELFT_TAGGING_DEBUG_OUT)
    if not output_directory:
        return None
    return TagDebugReporter(output_directory)

max_stars_repo_name: elifesciences/sciencebeam-trainer-delft

max_issues_repo_name: elifesciences/sciencebeam-trainer-delft

max_forks_repo_name: elifesciences/sciencebeam-trainer-delft


text:
from __future__ import print_function

import os
import shutil
import ConfigParser
from operator import itemgetter
import cPickle as pickle
from sklearn.preprocessing import LabelEncoder
import matplotlib
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
import matplotlib.pyplot as plt
import numpy as np
import keras
from keras.utils import np_utils
from embeddings import EmbeddingsPretrainer
import utils
from model import build_model

font = {'size' : 10}
plt.rc('font', **font)

MODELS_DIR = '../models'
if not os.path.isdir(MODELS_DIR):
    os.mkdir(MODELS_DIR)

class Tagger:

    def __init__(self, config_path=None):
        if config_path:
            config = ConfigParser.ConfigParser()
            config.read(config_path)

            # parse the param
            param_dict = dict()
            for section in config.sections():
                for name, value in config.items(section):
                    if value.isdigit():
                        value = int(value)
                    elif value == "True":
                        value = True
                    elif value == "False":
                        value = False
                    param_dict[name] = value

            # set params:
            self.train_dir = param_dict['train_dir']
            self.dev_dir = param_dict['dev_dir']
            self.test_dir = param_dict['test_dir']
            self.model_name = param_dict['model_name']
            self.context_representation = param_dict['context_representation']
            self.pretrain_embeddings = param_dict['pretrain_embeddings']
            self.target_representation = param_dict['target_representation']
            self.include_target_one_hot = param_dict['include_target_one_hot']
            self.max_pooling = param_dict['max_pooling']
            self.std_token_len = param_dict['std_token_len']
            self.nb_left_tokens = param_dict['nb_left_tokens']
            self.left_char_len = param_dict['left_char_len']
            self.nb_right_tokens = param_dict['nb_right_tokens']
            self.right_char_len = param_dict['right_char_len']
            self.nb_epochs = param_dict['nb_epochs']
            self.batch_size = param_dict['batch_size']
            self.nb_filters = param_dict['nb_filters']
            self.filter_length = param_dict['filter_length']
            self.nb_dense_dims = param_dict['nb_dense_dims']
            self.embedding_dims = param_dict['embedding_dims']
            self.min_train_token_count = param_dict['min_train_token_count']

            self.model_path = os.sep.join((MODELS_DIR, self.model_name))
            if not os.path.isdir(self.model_path):
                os.mkdir(self.model_path)
            print(param_dict)

        # make sure that we can reproduce parametrization in case new model is trained:
        if not os.path.exists(os.sep.join((self.model_path, 'config.txt'))):
            shutil.copy(config_path, os.sep.join((self.model_path, 'config.txt')))

    def load_data(self, nb_instances, label_idxs):
        self.train_tokens, self.train_labels = \
                utils.load_data_dir(data_dir = self.train_dir,
                                    nb_instances = nb_instances,
                                    label_idxs=label_idxs)
        self.train_token_set = set(self.train_tokens) # for evaluation purposes (known vs unknown)
        if self.dev_dir:
            self.dev_tokens, self.dev_labels = \
                    utils.load_data_dir(data_dir = self.dev_dir,
                                        nb_instances = nb_instances,
                                        label_idxs=label_idxs)
        if self.test_dir:
            self.test_tokens, self.test_labels = \
                    utils.load_data_dir(data_dir = self.test_dir,
                                        nb_instances = nb_instances,
                                        label_idxs=label_idxs)

    def encode_labels(self, load_pickles=False):
        # ignore boundary markers
        self.train_labels = [l for l in self.train_labels if l not in ("@", "$")]
        if self.dev_dir:
            self.dev_labels = [l for l in self.dev_labels if l not in ("@", "$")]
        if self.test_dir:
            self.test_labels = [l for l in self.test_labels if l not in ("@", "$")]

        if not load_pickles:
            # fit a labelencoder on all labels:
            self.label_encoder = LabelEncoder()
            lbls = self.train_labels
            if self.dev_dir:
               lbls.extend(self.dev_labels)
            if self.test_dir:
                lbls.extend(self.test_labels)

            self.label_encoder.fit(lbls)

            # pickle the label encoder:
            with open(os.sep.join((self.model_path, 'label_encoder.p')), 'wb') as f:
                pickle.dump(self.label_encoder, f)
        else:
            self.label_encoder = pickle.load(open(os.sep.join((self.model_path, 'label_encoder.p')), 'rb'))

        # transform labels to int representation:
        self.train_int_labels = self.label_encoder.transform(self.train_labels)
        if self.dev_dir:
            self.dev_int_labels = self.label_encoder.transform(self.dev_labels)
        if self.test_dir:
            self.test_int_labels = self.label_encoder.transform(self.test_labels)

        print("> nb distinct train labels:", max(self.train_int_labels)+1)

        # convert labels to one-hot represenation for cross-entropy:
        self.train_y = np_utils.to_categorical(self.train_int_labels,
                                               len(self.label_encoder.classes_))
        if self.dev_dir:
            self.dev_y = np_utils.to_categorical(self.dev_int_labels,
                                               len(self.label_encoder.classes_))
        if self.test_dir:
            self.test_y = np_utils.to_categorical(self.test_int_labels,
                                               len(self.label_encoder.classes_))

    def vectorize_instances(self, tokens):
        left_X, tokens_X, right_X, target_one_hots = [], [], [], []
        filler = np.zeros(len(self.train_token_index))

        for token_idx, token in enumerate(tokens):
            if token in ("@", "$"):
                continue  # ignore boundary markers:

            if self.include_target_one_hot:
                try:
                    target_one_hots.append([self.train_token_index[token]])
                except KeyError:
                    target_one_hots.append([0])

            # vectorize target token:
            if self.target_representation in \
                    ('flat_characters', 'lstm_characters',\
                     'flat_convolution', 'lstm_convolution'):
                tokens_X.append(utils.vectorize_charseq(seq=token,
                                                        char_vector_dict=self.train_char_vector_dict,
                                                        std_seq_len=self.std_token_len))
            elif target_representation == 'embedding':
                try:
                    tokens_X.append([self.train_token_index[token]])
                except KeyError:
                    tokens_X.append([0])

            # vectorize context:
            if self.context_representation in \
                    ('flat_characters', 'lstm_characters',\
                     'flat_convolution', 'lstm_convolution'):

                # vectorize left context:
                left_context = [tokens[token_idx-(t+1)] for t in range(self.nb_left_tokens)
                                     if token_idx-(t+1) >= 0][::-1]
                left_str = " ".join(left_context)
                left_X.append(utiels.vectorize_charseq(seq=left_str,
                                                       char_vector_dict=self.train_char_vector_dict,
                                                       std_seq_len=self.left_char_len))

                # vectorize right context:
                right_str = " ".join([tokens[token_idx+t+1] for t in range(self.nb_right_tokens)
                                     if token_idx+t+1 < len(tokens)])
                right_X.append(utils.vectorize_charseq(seq=right_str,
                                                       char_vector_dict=self.train_char_vector_dict,
                                                       std_seq_len=self.right_char_len))

            elif self.context_representation in ('flat_embeddings', 'lstm_embeddings'):
                # vectorize left context:
                left_context_tokens = [tokens[token_idx-(t+1)]\
                                            for t in range(self.nb_left_tokens)\
                                                if token_idx-(t+1) >= 0][::-1]
                idxs = []
                if left_context_tokens:
                    idxs = [self.train_token_index[t]\
                                if t in self.train_token_index\
                                else 0 for t in left_context_tokens]
                while len(idxs) < self.nb_left_tokens:
                    idxs = [0] + idxs
                left_X.append(idxs)

                # vectorize right context:
                right_context_tokens = [tokens[token_idx+t+1]\
                                            for t in range(self.nb_right_tokens)\
                                                if token_idx+t+1 < len(tokens)]
                idxs = []
                if right_context_tokens:
                    idxs = [self.train_token_index[t]\
                                if t in self.train_token_index\
                                else 0 for t in right_context_tokens]
                while len(idxs) < self.nb_right_tokens:
                    idxs.append(0)
                right_X.append(idxs)

        tokens_X = np.asarray(tokens_X)
        left_X = np.asarray(left_X)
        right_X = np.asarray(right_X)
        target_one_hots = np.asarray(target_one_hots)

        return left_X, tokens_X, right_X, target_one_hots

    def vectorize_datasets(self, load_pickles=False):
        if not load_pickles:
            # fit dicts etc. on train data
            self.train_char_vector_dict = utils.get_char_vector_dict(self.train_tokens)
            self.train_token_index = utils.get_train_token_index(self.train_tokens,
                                                                 min_count=self.min_train_token_count)

            # dump fitted objects for reuse:
            # pickle the label encoder:
            with open(os.sep.join((self.model_path, 'train_char_vector_dict.p')), 'wb') as f:
                pickle.dump(self.train_char_vector_dict, f)
            with open(os.sep.join((self.model_path, 'train_token_index.p')), 'wb') as f:
                pickle.dump(self.train_token_index, f)

        else:
            self.train_char_vector_dict = pickle.load(open(os.sep.join((self.model_path, 'train_char_vector_dict.p')), 'rb'))
            self.train_token_index = pickle.load(open(os.sep.join((self.model_path, 'train_token_index.p')), 'rb'))

        # transform training, dev and test data:
        self.train_left_X, self.train_tokens_X,\
        self.train_right_X, self.train_target_one_hots = \
                        self.vectorize_instances(tokens=self.train_tokens)
        if self.dev_dir:
            self.dev_left_X, self.dev_tokens_X,\
            self.dev_right_X, self.dev_target_one_hots = \
                            self.vectorize_instances(tokens=self.dev_tokens)

        if self.test_dir:
            self.test_left_X, self.test_tokens_X,\
            self.test_right_X, self.test_target_one_hots = \
                        self.vectorize_instances(tokens=self.test_tokens)

    def set_model(self, load_pickles=False):
        # get embeddings if necessary:
        self.embeddings = None
        if self.pretrain_embeddings and self.context_representation != 'None':
            if not load_pickles:
                pretrainer = EmbeddingsPretrainer(tokens=self.train_tokens,
                                                  size=self.embedding_dims,
                                                  min_count=self.min_train_token_count)
                vocab = [k for k,v in sorted(self.train_token_index.items(),\
                                            key=itemgetter(1))]
                self.embeddings = pretrainer.get_weights(vocab)
                # dump embeddings:
                with open(os.sep.join((self.model_path, 'embeddings.p')), 'wb') as f:
                    pickle.dump(self.embeddings, f)
                # visualize etc:
                pretrainer.plot_mfi(outputfile=os.sep.join((self.model_path, 'embeddings.pdf')))
                pretrainer.most_similar(outputfile=os.sep.join((self.model_path, 'neighbors.txt')))
            else:
                self.embeddings = pickle.load(open(os.sep.join((self.model_path, 'embeddings.p')), 'rb'))

        self.model = build_model(std_token_len=self.std_token_len,
                                 left_char_len=self.left_char_len,
                                 right_char_len=self.right_char_len,
                                 filter_length=self.filter_length,
                                 nb_filters=self.nb_filters,
                                 nb_dense_dims=self.nb_dense_dims,
                                 nb_left_tokens=self.nb_left_tokens,
                                 nb_right_tokens=self.nb_right_tokens,
                                 nb_labels=len(self.label_encoder.classes_),
                                 char_vector_dict=self.train_char_vector_dict,
                                 train_token_index=self.train_token_index,
                                 target_representation=self.target_representation,
                                 context_representation=self.context_representation,
                                 embedding_dims=self.embedding_dims,
                                 include_target_one_hot=self.include_target_one_hot,
                                 max_pooling=self.max_pooling,
                                 pretrain_embeddings=self.pretrain_embeddings,
                                 embeddings=self.embeddings)

    def get_baseline(self):
        # Levenshtein baseline:
        if self.dev_dir:
            print("+++ BASELINE SCORE (dev)")
            all_acc, known_acc, unknown_acc, proportion_unknown = utils.baseline(train_tokens = self.train_tokens,
                                                                                train_labels = self.train_labels,
                                                                                test_tokens = self.dev_tokens,
                                                                                test_labels = self.dev_labels)
            print("\t - all acc:\t{:.2%}".format(all_acc))
            print("\t - known acc:\t{:.2%}".format(known_acc))
            print("\t - unknown acc:\t{:.2%}".format(unknown_acc))
            print("\t - proportion unknown:\t{:.2%}".format(proportion_unknown))

        print("+++ BASELINE SCORE (test)")
        all_acc, known_acc, unknown_acc, proportion_unknown = utils.baseline(train_tokens = self.train_tokens,
                                                                             train_labels = self.train_labels,
                                                                             test_tokens = self.test_tokens,
                                                                             test_labels = self.test_labels)
        print("\t - all acc:\t{:.2%}".format(all_acc))
        print("\t - known acc:\t{:.2%}".format(known_acc))
        print("\t - unknown acc:\t{:.2%}".format(unknown_acc))
        print("\t - proportion unknown:\t{:.2%}".format(proportion_unknown))

    def inspect_filters(self):
        with open(os.sep.join((self.model_path, 'filters_repr.txt')),\
            'wt') as f:
            for weights in self.model.get_weights():
                if len(weights.shape) == 4: # first conv layer
                    weight_tensor = weights
                    # get shape info:
                    nb_filters = weight_tensor.shape[0]
                    nb_chars = weight_tensor.shape[1]
                    filter_size = weight_tensor.shape[2]

                    chars = list('X'*len(self.train_char_vector_dict))
                    for char, vector in self.train_char_vector_dict.items():
                        chars[np.argmax(vector)] = char

                    for filter_idx, filter_ in enumerate(weight_tensor[:50]):
                        f.write('\t- filter '+str(filter_idx+1)+'\n')
                        filter_ = np.squeeze(filter_).transpose()
                        for position, position_weights in enumerate(filter_):
                            info = []
                            for score, char in sorted(zip(position_weights, chars), reverse=True)[:4]:
                                sc = "{:.3}".format(score)
                                info.append(char+' : '+sc)
                            f.write('\t\t+ pos '+str(position+1)+' > '+('\t|\t'.join(info))+'\n')

    def load_weights(self):
        self.model.load_weights(os.sep.join((self.model_path, 'weights.hdf5')))

    def fit(self):
        train_losses, dev_losses = [], []
        train_accs, dev_accs = [], []
        dev_known_accs, dev_unknown_accs = [], []

        for e in range(self.nb_epochs):
            if self.context_representation != 'None':
                print("-> epoch ", e+1, "...")
                d = {'left_input': self.train_left_X,
                            'target_input': self.train_tokens_X,
                            'right_input': self.train_right_X,
                            'target_one_hot_input': self.train_target_one_hots,
                            'label_output': self.train_y,
                    }
                self.model.fit(data=d,
                               nb_epoch = 1,
                               batch_size = self.batch_size)

                print("+++ TRAIN SCORE")
                train_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                print("\t - loss:\t{:.3}".format(train_loss))
                train_losses.append(train_loss)

                train_predictions = self.model.predict({'left_input': self.train_left_X,
                                          'target_input': self.train_tokens_X,
                                          'target_one_hot_input': self.train_target_one_hots,
                                          'right_input': self.train_right_X,
                                         },
                                         batch_size = self.batch_size)
                train_all_acc, _, _ = utils.accuracies(predictions = train_predictions,
                                                                   gold_labels = self.train_int_labels,
                                                                   test_tokens = self.train_tokens,
                                                                   train_token_set = self.train_token_set)
                print("\t - all acc:\t{:.2%}".format(train_all_acc))
                train_accs.append(train_all_acc)

                if self.dev_dir:
                    print("+++ DEV SCORE")
                    d = {'left_input': self.dev_left_X,
                         'target_input': self.dev_tokens_X,
                         'right_input': self.dev_right_X,
                         'target_one_hot_input': self.dev_target_one_hots,
                        }
                    dev_predictions = self.model.predict(data=d, batch_size = self.batch_size)
                    d['label_output'] = self.dev_y
                    dev_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                    print("\t - loss:\t{:.3}".format(dev_loss))
                    dev_losses.append(dev_loss)
                    dev_all_acc, dev_known_acc, dev_unknown_acc = utils.accuracies(predictions = dev_predictions,
                                                                       gold_labels = self.dev_int_labels,
                                                                       test_tokens = self.dev_tokens,
                                                                       train_token_set = self.train_token_set)
                    dev_accs.append(dev_all_acc)
                    dev_known_accs.append(dev_known_acc)
                    dev_unknown_accs.append(dev_unknown_acc)

                    print("\t - all acc:\t{:.2%}".format(dev_all_acc))
                    print("\t - known acc:\t{:.2%}".format(dev_known_acc))
                    print("\t - unknown acc:\t{:.2%}".format(dev_unknown_acc))

                if self.test_dir:
                    print("+++ TEST SCORE")
                    test_predictions = self.model.predict({'left_input': self.test_left_X,
                                              'target_input': self.test_tokens_X,
                                              'right_input': self.test_right_X,
                                              'target_one_hot_input': self.test_target_one_hots,
                                             },
                                             batch_size = self.batch_size)
                    test_all_acc, test_known_acc, test_unknown_acc = utils.accuracies(predictions = test_predictions,
                                                                       gold_labels = self.test_int_labels,
                                                                       test_tokens = self.test_tokens,
                                                                       train_token_set = self.train_token_set)
                    print("\t - all acc:\t{:.2%}".format(test_all_acc))
                    print("\t - known acc:\t{:.2%}".format(test_known_acc))
                    print("\t - unknown acc:\t{:.2%}".format(test_unknown_acc))

            elif self.context_representation == 'None':
                print("-> epoch ", e+1, "...")
                d = {'target_input': self.train_tokens_X,
                     'target_one_hot_input': self.train_target_one_hots,
                     'label_output': self.train_y,
                    }
                self.model.fit(data=d,
                            nb_epoch = 1,
                            batch_size = self.batch_size)

                print("+++ TRAIN SCORE")
                train_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                print("\t - loss:\t{:.3}".format(train_loss))
                train_losses.append(train_loss)

                train_predictions = self.model.predict({'target_input': self.train_tokens_X,
                                         'target_one_hot_input': self.train_target_one_hots},
                                         batch_size = self.batch_size)
                train_all_acc, _, _ = utils.accuracies(predictions = train_predictions,
                                                                   gold_labels = self.train_int_labels,
                                                                   test_tokens = self.train_tokens,
                                                                   train_token_set = self.train_token_set)
                train_accs.append(train_all_acc)

                print("\t - all acc:\t{:.2%}".format(train_all_acc))

                if self.dev_dir:
                    print("+++ DEV SCORE")
                    d = {'target_input': self.dev_tokens_X,
                         'target_one_hot_input': self.dev_target_one_hots,
                        }
                    dev_predictions = self.model.predict(data = d,
                                             batch_size = self.batch_size)
                    d['label_output'] = self.dev_y
                    dev_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                    dev_losses.append(dev_loss)
                    print("\t - loss:\t{:.3}".format(dev_loss))
                    dev_all_acc, dev_known_acc, dev_unknown_acc = utils.accuracies(predictions = dev_predictions,
                                                                       gold_labels = self.dev_int_labels,
                                                                       test_tokens = self.dev_tokens,
                                                                       train_token_set = self.train_token_set)

                    dev_accs.append(dev_all_acc)
                    dev_known_accs.append(dev_known_acc)
                    dev_unknown_accs.append(dev_unknown_acc)

                    print("\t - all acc:\t{:.2%}".format(dev_all_acc))
                    print("\t - known acc:\t{:.2%}".format(dev_known_acc))
                    print("\t - unknown acc:\t{:.2%}".format(dev_unknown_acc))

                if self.test_dir:
                    print("+++ TEST SCORE")
                    test_predictions = self.model.predict({'target_input': self.test_tokens_X,
                                             'target_one_hot_input': self.test_target_one_hots},
                                             batch_size = self.batch_size)
                    test_all_acc, test_known_acc, test_unknown_acc = utils.accuracies(predictions = test_predictions,
                                                                       gold_labels = self.test_int_labels,
                                                                       test_tokens = self.test_tokens,
                                                                       train_token_set = self.train_token_set)
                    print("\t - all acc:\t{:.2%}".format(test_all_acc))
                    print("\t - known acc:\t{:.2%}".format(test_known_acc))
                    print("\t - unknown acc:\t{:.2%}".format(test_unknown_acc))

            # keep track of results:
            plt.clf()
            f, ax = plt.subplots(1,1)
            plt.tick_params(axis='both', which='major', labelsize=6)
            plt.tick_params(axis='both', which='minor', labelsize=6)
            plt.xlabel('Epoch')
            plt.ylabel('Loss')
            ax.plot(train_losses, c='b')
            if self.dev_dir:
                ax.plot(dev_losses, c='g')
            ax2 = plt.gca().twinx()
            plt.tick_params(axis='both', which='major', labelsize=6)
            plt.tick_params(axis='both', which='minor', labelsize=6)
            ax2.plot(train_accs, label='Train Acc (all)', c='r')
            if self.dev_dir:
                ax2.plot(dev_accs, label='Devel Acc (all)', c='c')
                ax2.plot(dev_known_accs, label='Devel Acc (kno)', c='m')
                ax2.plot(dev_unknown_accs, label='Devel Acc (unk)',  c='y')

            # hack: add dummy legend items:
            ax2.plot(0, 0, label='Train Loss', c='b')
            if self.dev_dir:
                ax2.plot(0, 0, label='Devel Loss', c='g')
            plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
                        ncol=3, mode="expand", borderaxespad=0., fontsize = 'x-small')
            ax2.set_ylabel('Accuracy')
            plt.savefig(os.sep.join((self.model_path, 'losses.pdf')))

            with open(os.sep.join((self.model_path, 'losses.tsv')), 'w') as f:
                if self.dev_dir:
                    f.write('idx\ttrain\tdev\n')
                    for i in range(len(train_losses)):
                        f.write('\t'.join((str(i+1), str(train_losses[i]), str(dev_losses[i])))+'\n')
                else:
                    f.write('idx\ttrain\n')
                    for i in range(len(train_losses)):
                        f.write('\t'.join((str(i+1), str(train_losses[i])))+'\n')

            with open(os.sep.join((self.model_path, 'accs.tsv')), 'w') as f:
                if self.dev_dir:
                    f.write('idx\ttrain_all\tdev_all\tdev_known\tdev_unknown\n')
                    for i in range(len(train_losses)):
                        f.write('\t'.join((str(i+1), str(train_accs[i]),\
                                            str(dev_accs[i]), str(dev_known_accs[i]),\
                                            str(dev_unknown_accs[i])))+'\n')
                else:
                    f.write('idx\ttrain_all\n')
                    for i in range(len(train_losses)):
                        f.write('\t'.join((str(i+1), str(train_accs[i])))+'\n')

            # save
            self.model.save_weights(os.sep.join((self.model_path, 'weights.hdf5')),\
                overwrite=True)
            # visualize:
            self.inspect_filters()

    def test(self):
        if self.context_representation != 'None':
            d = {'left_input': self.train_left_X,
                    'target_input': self.train_tokens_X,
                    'right_input': self.train_right_X,
                    'target_one_hot_input': self.train_target_one_hots,
                    'label_output': self.train_y,
                }

            print("+++ TRAIN SCORE")
            train_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
            print("\t - loss:\t{:.3}".format(train_loss))

            train_predictions = self.model.predict({'left_input': self.train_left_X,
                                      'target_input': self.train_tokens_X,
                                      'target_one_hot_input': self.train_target_one_hots,
                                      'right_input': self.train_right_X,
                                     },
                                    batch_size = self.batch_size)
            train_all_acc, _, _ = utils.accuracies(predictions = train_predictions,
                                                   gold_labels = self.train_int_labels,
                                                   test_tokens = self.train_tokens,
                                                   train_token_set = self.train_token_set)
            print("\t - all acc:\t{:.2%}".format(train_all_acc))

            if self.dev_dir:
                print("+++ DEV SCORE")
                d = {'left_input': self.dev_left_X,
                     'target_input': self.dev_tokens_X,
                     'right_input': self.dev_right_X,
                     'target_one_hot_input': self.dev_target_one_hots,
                    }
                dev_predictions = self.model.predict(data=d, batch_size = self.batch_size)
                d['label_output'] = self.dev_y
                dev_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                print("\t - loss:\t{:.3}".format(dev_loss))
                dev_all_acc, dev_known_acc, dev_unknown_acc = utils.accuracies(predictions = dev_predictions,
                                                                   gold_labels = self.dev_int_labels,
                                                                   test_tokens = self.dev_tokens,
                                                                   train_token_set = self.train_token_set)
                print("\t - all acc:\t{:.2%}".format(dev_all_acc))
                print("\t - known acc:\t{:.2%}".format(dev_known_acc))
                print("\t - unknown acc:\t{:.2%}".format(dev_unknown_acc))

            if self.test_dir:
                print("+++ TEST SCORE")
                test_predictions = self.model.predict({'left_input': self.test_left_X,
                                          'target_input': self.test_tokens_X,
                                          'right_input': self.test_right_X,
                                          'target_one_hot_input': self.test_target_one_hots,
                                         },
                                         batch_size = self.batch_size)
                test_all_acc, test_known_acc, test_unknown_acc = utils.accuracies(predictions = test_predictions,
                                                                   gold_labels = self.test_int_labels,
                                                                   test_tokens = self.test_tokens,
                                                                   train_token_set = self.train_token_set)
                print("\t - all acc:\t{:.2%}".format(test_all_acc))
                print("\t - known acc:\t{:.2%}".format(test_known_acc))
                print("\t - unknown acc:\t{:.2%}".format(test_unknown_acc))

        elif self.context_representation == 'None':
            d = {'target_input': self.train_tokens_X,
                 'target_one_hot_input': self.train_target_one_hots,
                 'label_output': self.train_y,
                }

            print("+++ TRAIN SCORE")
            train_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
            print("\t - loss:\t{:.3}".format(train_loss))

            train_predictions = self.model.predict({'target_input': self.train_tokens_X,
                                         'target_one_hot_input': self.train_target_one_hots},
                                         batch_size = self.batch_size)
            train_all_acc, _, _ = utils.accuracies(predictions = train_predictions,
                                                                   gold_labels = self.train_int_labels,
                                                                   test_tokens = self.train_tokens,
                                                                   train_token_set = self.train_token_set)
            print("\t - all acc:\t{:.2%}".format(train_all_acc))

            if self.dev_dir:
                print("+++ DEV SCORE")
                d = {'target_input': self.dev_tokens_X,
                     'target_one_hot_input': self.dev_target_one_hots,
                    }
                dev_predictions = self.model.predict(data = d,
                                         batch_size = self.batch_size)
                d['label_output'] = self.dev_y
                dev_loss = self.model.evaluate(data=d, batch_size = self.batch_size)
                print("\t - loss:\t{:.3}".format(dev_loss))
                dev_all_acc, dev_known_acc, dev_unknown_acc = utils.accuracies(predictions = dev_predictions,
                                                                   gold_labels = self.dev_int_labels,
                                                                   test_tokens = self.dev_tokens,
                                                                   train_token_set = self.train_token_set)
                print("\t - all acc:\t{:.2%}".format(dev_all_acc))
                print("\t - known acc:\t{:.2%}".format(dev_known_acc))
                print("\t - unknown acc:\t{:.2%}".format(dev_unknown_acc))

            if self.test_dir:
                print("+++ TEST SCORE")
                test_predictions = self.model.predict({'target_input': self.test_tokens_X,
                                         'target_one_hot_input': self.test_target_one_hots},
                                         batch_size = self.batch_size)
                test_all_acc, test_known_acc, test_unknown_acc = utils.accuracies(predictions = test_predictions,
                                                                   gold_labels = self.test_int_labels,
                                                                   test_tokens = self.test_tokens,
                                                                   train_token_set = self.train_token_set)
                print("\t - all acc:\t{:.2%}".format(test_all_acc))
                print("\t - known acc:\t{:.2%}".format(test_known_acc))
                print("\t - unknown acc:\t{:.2%}".format(test_unknown_acc))

max_stars_repo_name: mikekestemont/tag

max_issues_repo_name: mikekestemont/tag

max_forks_repo_name: mikekestemont/tag


text:
#!/usr/bin/env python

# Samidha M. Verma

import random
import string
import numpy as np

def generate_population(n, l):
    population = []
    for i in range (0, n * l):
        population.append(random.choice(string.ascii_letters))
    population = np.reshape(population, (n, l))
    return population

def compute_fitness(x, target):
    length = len(target)
    fitness = 0
    for i in range(0, length):
        if (x[i] == target[i]):
            fitness = fitness + 1
    fitness = (int) (fitness / length * 100)
    return fitness

def mating_pool(pop, fitness):
    matingpool = []
    for i in range(0, len(pop)):
        if (fitness[i] == 0):
            break
        for j in range(fitness[i]):
            matingpool.append(pop[i])
    return matingpool

def crossover(parentA, parentB):
    child = parentA
    mid = random.randint(0, len(parentA))
    for i in range(0, len(parentA)):
        if(i < mid):
            child[i] = parentA[i]
        else:
            child[i] = parentB[i]
    return child

def mutation(x, rate):
    for i in range(0, len(x)):
        if (random.uniform(0, 1) < rate):
            x[i] = random.choice(string.ascii_letters)
    return x

max_stars_repo_name: samidhaVerma/optimization-algo

max_issues_repo_name: samidhaVerma/optimization-algo

max_forks_repo_name: samidhaVerma/optimization-algo


text:
"""
Visualize the features of the data to understand distributions and discover associations. Univariate
and bivariate analysis are important to fully understand the data. Trends may be discovered between
economic factors like GDP and default rate within a certain time period.
"""
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
import seaborn as sns

def barplot(data, x, y, hue=None):
    """
    Creates a bar plot to compare the values of two categories, stratified
    by another category (if applicable).

    :param data: dataset to visualize
    :param x: x-label of plot
    :param y: y-label of plot
    :param hue: extra condition to stratify in plot
    :return: barplot of inputted categories
    """
    ax = sns.barplot(x=x, y=y, hue=hue, data=data)
    plt.xlabel(x)
    plt.ylabel(y)
    plt.title(y + " based on " + x)
    plt.show()
    return ax

def histogram(column, data, normal=True, kde=False, bins=None):
    """
    Plots a histogram to visualize the distribution of a numerical
    column in the inputted data to find skewness.

    :param column: numerical column in data
    :param data: data to plot
    :param normal: normalize histogram
    :param kde: boolean of fitting kernel density estimate
    :param bins: bins of histogram
    :return: histogram of numerical column
    """
    ax = sns.distplot(data[column].dropna(), norm_hist=normal, kde=kde, bins=bins)
    plt.title("Distribution of " + column)
    return ax

def scatterplot(x, y, data, hue=None, regression=False):
    """
    This function returns a seaborn bar plot based on the data columns passed in.

    :param x: x-axis column as a string
    :param y: y-axis column as a string
    :param data: dataframe containing above columns
    :param hue: hue column as a string
    :param regression: boolean of whether to plot regression
    :returns: scatter plot of the columns
    """
    if not regression:
        return sns.relplot(x=x, y=y, hue=hue, data=data)
    else:
        assert hue is None, "Can't have Hue with Regression Plot"
        return sns.regplot(x=x, y=y, data=data)

def count_distribution(x, data, size=(14, 6)):
    """
    Plots and returns a count plot that displays the counts of each
    category within a given column.

    :param x: categorical column to display counts
    :param data: dataframe containing columns
    :param size: size of plot
    :return: count plot of x
    """
    plt.figure(figsize=size)

    g = sns.countplot(x=x, data=data)
    g.set_title(x+" Distribution", fontsize=15)
    g.set_xlabel(x, fontsize=13)
    g.set_ylabel('Count', fontsize=13)

    proportions = []
    for status in g.patches:
        height = status.get_height()
        proportions.append(height)
        g.text(status.get_x() + status.get_width() / 2.,
               height + 3,
               "{:1.2f}%".format(height / len(data) * 100),
               ha="center", fontsize=12)

    g.set_ylim(0, max(proportions) * 1.1)

max_stars_repo_name: samsonq/Macroeconomic-Default-Analysis

max_issues_repo_name: samsonq/Macroeconomic-Default-Analysis

max_forks_repo_name: samsonq/Macroeconomic-Default-Analysis


text:
from layerOfNeuron import *
import pickle
import random
import numpy as np

class NeuralNetwork:
    # с lambda не сохраняется
    #f  = lambda self, x: (1 / (1 + np.exp(-x)))
    #df = lambda self, x: (self.f(x) * (1 - self.f(x)))

    def  f(self, x): # передаточная функция
        try:
            return 1.0 / (1.0 + np.exp(-x))
        except OverflowError:
            print("NeuralNetwork.f: WARNING!!! OverflowError, returned 0.0!")
            return 0.0

    def df(self, x): # производная функции f
        try:
            return self.f(x) * (1.0 - self.f(x))
        except OverflowError:
            print("NeuralNetwork.df: WARNING!!! OverflowError, returned 0.0!")
            return 0.0

    __alpha = 0.0001 # постоянная момента

    def __init__(self, inp, h_layers, out):
        if type(inp)      != int:  raise TypeError("NeuralNetwork.__init__: number of inputs is not 'int'!")
        if type(h_layers) != list: raise TypeError("NeuralNetwork.__init__: list of hidden layers is not 'list'!")
        if type(out)      != int:  raise TypeError("NeuralNetwork.__init__: number of outputs is not 'int'!")

        if inp <= 0:           raise ValueError("NeuralNetwork.__init__: number of inputs <= 0!")
        if len(h_layers) == 0: raise ValueError("NeuralNetwork.__init__: length of list == 0!")
        if 0 in h_layers:      raise ValueError("NeuralNetwork.__init__: in list of hidden layers found 0!")
        if out <= 0:           raise ValueError("NeuralNetwork.__init__: number of outputs <= 0!")

        self.__inp                     = inp           # количество входов
        self.__num_neurons_in_h_layers = sum(h_layers) # количество нейронов в скрытых слоях
        self.__h_layers                = len(h_layers) # количество скрытых слоев
        self.__out                     = out           # количество выходов

        self.hidden_layers = [LayerOfNeuron(h_layers[i], inp if i == 0 else h_layers[i - 1], self.f) for i in range(self.__h_layers)] # создаем скрытые слои
        self.hidden_layers.append(LayerOfNeuron(out, h_layers[len(h_layers) - 1], self.f)) # и выходной считаем за последний скрытый

        self.__init_weights() # переинициализируем веса по некоторой формуле

    def __init_weights(self): # некоторые математические действия над весами
        p    = self.__num_neurons_in_h_layers
        beta = 0.7 * np.power(p, (1.0 / self.__inp))

        sm = 0
        for h in self.hidden_layers:
            for n in h.neurons:
                for s in n.synapses:
                    sm += s * s
        norm_syns = np.sqrt(sm)

        for h in self.hidden_layers:
            for n in h.neurons:
                for s in n.synapses:
                    s = (beta * s) / norm_syns
            n.s0 = random.uniform(-beta, beta)

    def __call__(self, x): # расчет реакции сети на входной вектор x
        if type(x) != list and type(x) != np.ndarray: raise TypeError("NeuralNetwork.__call__: input vector is not 'list' or 'numpy.ndarray'!")
        if len(x)  != self.__inp:                    raise ValueError("NeuralNetwork.__call__: incorrect input vector size!")

        for l in range(self.__h_layers + 1): # по всем скрытым + по выходному
            temp_x = np.fromiter((self.hidden_layers[l].neurons[n](x) for n in range(len(self.hidden_layers[l].neurons))), np.float)
            x = temp_x
        return x

    @classmethod
    def save(cls, network, name_file): # метод сохранения экземпляра сети
        if type(network)   != NeuralNetwork: raise TypeError("NeuralNetwork.save: SAVE ONLY NeuralNetwork OBJECTS!")
        if type(name_file) != str:           raise TypeError("NeuralNetwork.save: file name is invalid!")

        with open(name_file, 'wb') as file:
            pickle.dump(network, file)

    @classmethod
    def load(cls, name_file): # метод загрузки экземпляра сети
        with open(name_file, 'rb') as file:
            network = pickle.load(file)
            if type(network) != NeuralNetwork:
                raise TypeError("LOAD ONLY NeuralNetwork OBJECTS!")
            return network

    def training(self, inp, norm, a): # тренировка методом обратного распространения ошибки(a - скорость обучения)
        # полагаемся на адекватного учителя и не производим 2х лишних сравнений
        #if len(inp)  != self.__inp: raise ValueError("NeuralNetwork.training: incorrect input vector size!")
        #if len(norm) == self.__out: raise ValueError("NeuralNetwork.training: incorrect output vector size!")

        RES_input_layer = inp # выход входного слоя = входному вектору

        RES_hidden_layers = [] # запоминаем выходы всех скрытых слоев(далее выходной слой = последний скрытый)
        for l in range(self.__h_layers + 1):
            temp_inp = np.fromiter((self.hidden_layers[l].neurons[n](inp) for n in range(len(self.hidden_layers[l].neurons))), np.float)
            inp = temp_inp
            RES_hidden_layers.append(inp)

        delta = [[] for i in range(self.__h_layers + 1)] # массив локальных градиентов узлов сети

        E     = norm - RES_hidden_layers[len(RES_hidden_layers) - 1] # сигнал ошибки
        PHI   = np.fromiter((self.df(self.hidden_layers[self.__h_layers].neurons[n].v) for n in range(self.__out)), np.float)

        delta[self.__h_layers] = E * PHI # для выходного слоя

        for l in range(self.__h_layers - 1, -1, -1): # проходим по всем скрытым слоям КРОМЕ последнего
            num_neurons = len(self.hidden_layers[l].neurons)

            PHI = np.fromiter((self.df(self.hidden_layers[l].neurons[n].v) for n in range(num_neurons)), np.float)

            temp = np.empty(num_neurons)
            for n in range(num_neurons): # проходим по всем нейронам слоя
                W       = np.fromiter((self.hidden_layers[l + 1].neurons[ne].synapses[n] for ne in range(len(self.hidden_layers[l + 1].neurons))), np.float)
                SUM     = np.sum(delta[l + 1] * W)
                temp[n] = PHI[n] * SUM # расчитываем локальный градиент для нейрона n

            delta[l] = temp

        for l in range(self.__h_layers, 0, -1): # изменяем веса синаптических связей скрытых слоев КРОМЕ первого
            for n in range(len(self.hidden_layers[l].neurons)):
                self.hidden_layers[l].neurons[n].synapses       += a * self.hidden_layers[l].neurons[n].r * ((1 - self.__alpha) * delta[l][n] * RES_hidden_layers[l - 1] + self.__alpha * self.hidden_layers[l].neurons[n].memory_synapses)
                self.hidden_layers[l].neurons[n].memory_synapses = self.hidden_layers[l].neurons[n].synapses

        for n in range(len(self.hidden_layers[0].neurons)): # изменяем веса синаптических связей первого слоя
                self.hidden_layers[0].neurons[n].synapses       += a * self.hidden_layers[0].neurons[n].r * ((1 - self.__alpha) * delta[0][n] * RES_input_layer + self.__alpha * self.hidden_layers[0].neurons[n].memory_synapses)
                self.hidden_layers[0].neurons[n].memory_synapses = self.hidden_layers[0].neurons[n].synapses

        return (np.sum(E * E) / 2.0) # общая ошибка

max_stars_repo_name: allsilaevex/NeuralNetwork

max_issues_repo_name: allsilaevex/NeuralNetwork

max_forks_repo_name: allsilaevex/NeuralNetwork


text:
# coding=utf-8
###
###  Gabarit pour l'application de traitement des frequences de mots dans les oeuvres d'auteurs divers
###  Le traitement des arguments a ete inclus:
###     Tous les arguments requis sont presents et accessibles dans args
###     Le traitement du mode verbose vous donne un exemple de l'utilisation des arguments
###
###  Frederic Mailhot, 26 fevrier 2018
###    Revise 16 avril 2018
###    Revise 7 janvier 2020

###  Parametres utilises, leur fonction et code a generer
###
###  -d X  Deja traite dans le gabarit:  la variable rep_auth contiendra le chemin complet vers le repertoire d'auteurs
###       La liste d'auteurs est extraite de ce repertoire, et est comprise dans la variable authors
###
###  -P X Si utilise, indique au systeme d'utiliser la ponctuation.  Ce qui est considÃ©re comme un signe de ponctuation
###       est defini dans la liste PONC
###       Si -P EST utilise, cela indique qu'on dÃ©sire conserver la ponctuation (chaque signe est alors considere
###       comme un mot.  Par defaut, la ponctuation devrait etre retiree
###
###  -m X mode d'analyse:  -m 1 indique de faire les calculs avec des unigrammes, -m 2 avec des bigrammes.
###
###  -a X Auteur (unique a traiter).  Utile en combinaison avec -g, -G, pour la generation d'un texte aleatoire
###       avec les caracteristiques de l'auteur indique
###
###  -G   Indique qu'on veut generer un texte (voir -a ci-haut), le nombre de mots Ã  generer doit Ãªtre indique
###
###  -g   Indique qu'on veut generer un texte (voir -a ci-haut), le nom du fichier en sortie est indique
###
###  -F X Indique qu'on desire connaitre le rang d'un certain mot pour un certain auteur.  L'auteur doit etre
###       donnÃ© avec le parametre -a, et un mot doit suivre -F:   par exemple:   -a Verne -F Cyrus
###
###  -v   Deja traite dans le gabarit:  mode "verbose",  va imprimer les valeurs donnÃ©es en parametre
###
###
###  Le systeme doit toujours traiter l'ensemble des oeuvres de l'ensemble des auteurs.  Selon la presence et la valeur
###  des autres parametres, le systeme produira differentes sorties:
###
###  avec -a, -g, -G:  generation d'un texte aleatoire avec les caracteristiques de l'auteur identifie
###  avec -a, -F:  imprimer la frequence d'un mot d'un certain auteur.  Format de sortie:  "auteur:  mot  frequence"
###                la frequence doit Ãªtre un nombre reel entre 0 et 1, qui represente la probabilite de ce mot
###                pour cet auteur
###  avec -f:  indiquer l'auteur le plus probable du texte identifie par le nom de fichier qui suit -f
###            Format de sortie:  "nom du fichier: auteur"
###  avec ou sans -P:  indique que les calculs doivent etre faits avec ou sans ponctuation
###  avec -v:  mode verbose, imprimera l'ensemble des valeurs des paramÃ¨tres (fait deja partie du gabarit)
# -d D:\Simon\Desktop\Repo\S2_APP5\APP\TextesPourEtudiants -m 2 -g allo -G 10

#
# Ce code à été réalisé par Benjamin Gélinas(gelb2602) et Simon Leroux(lers0601)
# le 17 mars 2021
# Tutorat T3 et T2 respectivement
#

import math
import argparse
import glob
import sys
import os
import numpy
from pathlib import Path
from random import randint
from random import choice

### Ajouter ici les signes de ponctuation Ã  retirer
PONC = ["!", '"', "'", ")", "(", ",", ".", ";", ":", "?", "-", "_"]

###  Vous devriez inclure vos classes et mÃ©thodes ici, qui seront appellÃ©es Ã  partir du main
def dico_maker(texte, dico):
    dictionnairetemp = dico
    if not args.P:
        for i in range(len(PONC)):
            texte = texte.replace(PONC[i], " ")
    texte = texte.lower().split()
    gram = []
    if args.m == 1:
        for i in range(len(texte)):
            word = texte[i]
            if len(word) > 2:
                gram.append(word)
    else:
        i = 0
        while len(texte[i]) < 3:
            i += 1
        while i < len(texte):
            j = i
            if i + 1 != len(texte):
                while j + 1 != len(texte) and len(texte[j + 1]) < 3:
                    j += 1
                if j + 1 != len(texte):
                    gram.append((texte[i], texte[j + 1]))
            i = j
            i += 1
    for i in range(len(gram)):
        mot = gram[i]
        if mot not in dictionnairetemp:
            dictionnairetemp[mot] = 0
        dictionnairetemp[mot] += 1
    return dictionnairetemp

def sort(dictionnaires):
    return sorted(list(dictionnaires.items()), key=lambda x: x[1], reverse=True)

def calculDefrequence(rangeauteur, liste, i):
    freq = [[] for nombre in rangeauteur]
    for auteur in rangeauteur:
        for motEnCommun in range(len(liste[auteur])):
            freq[auteur].append(liste[auteur][motEnCommun][i])
        freq[auteur] = sum(freq[auteur])
    return freq

def ChaineDeMarkov(listeDeDicos):
    liste = [[] for auteur in range(len(listeDeDico))]
    for auteur in range(len(listeDeDicos)):
        chaineDeMarkov = []
        for bigramme in range(len(listeDeDicos[auteur])):
            motComparer, freq = listeDeDicos[auteur][bigramme]
            motComparer, suivant = motComparer
            if mot[auteur] == motComparer:
                chaineDeMarkov.append((suivant, freq))
        liste[auteur] += chaineDeMarkov
    return liste

### Main: lecture des paramÃ¨tres et appel des mÃ©thodes appropriÃ©es
###
###       argparse permet de lire les paramÃ¨tres sur la ligne de commande
###             Certains paramÃ¨tres sont obligatoires ("required=True")
###             Ces paramÃ¨tres doivent Ãªtres fournis Ã  python lorsque l'application est exÃ©cutÃ©e
if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog='markov_cip1_cip2.py')
    parser.add_argument('-d', required=True, help='Repertoire contenant les sous-repertoires des auteurs')
    parser.add_argument('-a', help='Auteur a traiter')
    parser.add_argument('-f', help='Fichier inconnu a comparer')
    parser.add_argument('-m', required=True, type=int, choices=range(1, 3),
                        help='Mode (1 ou 2) - unigrammes ou digrammes')
    parser.add_argument('-F', type=int, help='Indication du rang (en frequence) du mot (ou bigramme) a imprimer')
    parser.add_argument('-G', type=int, help='Taille du texte a generer')
    parser.add_argument('-g', help='Nom de base du fichier de texte a generer')
    parser.add_argument('-v', action='store_true', help='Mode verbose')
    parser.add_argument('-P', action='store_true', help='Retirer la ponctuation')
    parser.add_argument('-A', action='store_true', help='Tout les auteurs')
    args = parser.parse_args()

    ### Lecture du rÃ©pertoire des auteurs, obtenir la liste des auteurs
    ### Note:  args.d est obligatoire
    ### auteurs devrait comprendre la liste des rÃ©pertoires d'auteurs, peu importe le systÃ¨me d'exploitation
    cwd = os.getcwd()
    if os.path.isabs(args.d):
        rep_aut = args.d
    else:
        rep_aut = os.path.join(cwd, args.d)

    rep_aut = os.path.normpath(rep_aut)
    authors = os.listdir(rep_aut)

    ### Enlever les signes de ponctuation (ou non) - DÃ©finis dans la liste PONC
    if args.P:
        remove_ponc = True
    else:
        remove_ponc = False

    ### Si mode verbose, reflÃ©ter les valeurs des paramÃ¨tres passÃ©s sur la ligne de commande
    if args.v:
        print("Mode verbose:")
        print("Calcul avec les auteurs du repertoire: " + args.d)
        if args.f:
            print("Fichier inconnu a,"
                  " etudier: " + args.f)

        print("Calcul avec des " + str(args.m) + "-grammes")
        if args.F:
            print(str(args.F) + "e mot (ou digramme) le plus frequent sera calcule")

        if args.a:
            print("Auteur etudie: " + args.a)

        if args.P:
            print("Retirer les signes de ponctuation suivants: {0}".format(" ".join(str(i) for i in PONC)))

        if args.G:
            print("Generation d'un texte de " + str(args.G) + " mots")

        if args.g:
            print("Nom de base du fichier de texte genere: " + args.g)

        print("Repertoire des auteurs: " + rep_aut)
        print("Liste des auteurs: ")
        for a in authors:
            aut = a.split("/")
            print("    " + aut[-1])
    ### Ã€ partir d'ici, vous devriez inclure les appels Ã  votre code

    listeDeDico = []
    if args.a:
        lsdir = os.listdir(rep_aut + '/' + args.a)
        dictionnaire = {}
        for j in range(len(lsdir)):
            f = open(rep_aut + '/' + args.a + '/' + lsdir[j], "r", encoding="utf8")
            texte = f.read().lower()
            dictionnaire = dico_maker(texte, dictionnaire)
            f.close()
        listeDeDico.append(sort(dictionnaire))
        if args.F:
            print(listeDeDico[0][args.F - 1])
    else:
        for i in range(len(authors)):
            lsdir = os.listdir(rep_aut + '/' + authors[i])
            dictionnaire = {}
            for j in range(len(lsdir)):
                f = open(rep_aut + '/' + authors[i] + '/' + lsdir[j], "r", encoding="utf8")
                texte = f.read().lower()
                dictionnaire = dico_maker(texte, dictionnaire)
                f.close()
            listeDeDico.append(sort(dictionnaire))
        if args.F:
            for authors in range(len(listeDeDico)):
                print(listeDeDico[authors][args.F - 1])

    # Bloc de code pour la comparaison avec un texte inconnu
    if args.f:
        # Declaration des variables
        listeTexteInconnu = []
        dictionnaireInconnu = {}
        longeurduDico=range(len(listeDeDico))
        listeDeProbabilite = [[] for auteur in longeurduDico]
        FreqAuteur = [[] for nombre in longeurduDico]
        FreqAuteurInconnu = [[] for nombre in longeurduDico]
        sommeCarre = [[] for nombre in longeurduDico]
        listeDeMotCommun = [[] for auteur in longeurduDico]

        f = open(args.f, encoding="utf8")
        texte = f.read().lower()
        dictionnaireInconnu = dico_maker(texte, dictionnaireInconnu)
        f.close()

        # Le bout de code suivant passe a traver tout les textes de tout les auteurs dans la liste de mot ordonne et
        # si le mot est dans le dictionnaire du texte inconnu le rajoute dans une liste de mot commun. La liste de
        # mot commun contient en ordre:
        #                               0.Le gram en commun
        #                               1.La frequence dans le texte inconnu
        #                               2.La frequence dans le texte de l'auteur
        for auteur in longeurduDico:
            for motconnu in listeDeDico[auteur]:
                if args.m == 2:
                    if motconnu[0] in dictionnaireInconnu:
                        listeDeMotCommun[auteur].append((motconnu[0], dictionnaireInconnu[motconnu[0]], motconnu[1]))
                else:
                    if motconnu[0] in dictionnaireInconnu:
                        listeDeMotCommun[auteur].append((motconnu[0], dictionnaireInconnu[motconnu[0]], motconnu[1]))

        # Calcul de la frequence pour chaque auteur
        FreqAuteur = calculDefrequence(longeurduDico, listeDeMotCommun, 2)

        # Calcul de la frequence pour l'inconnu
        FreqAuteurInconnu = calculDefrequence(longeurduDico, listeDeMotCommun, 1)

        # Normalisation des frequences. Dans ce bout de code,
        # on crée un tableau avec comme informations:
        #                                               0. Fréquence normalisé du texte inconnu
        #                                               1. Fréquence normalisé du texte pour l'auteur
        for auteur in longeurduDico:
            for nombre in range(len(listeDeMotCommun[auteur])):
                listeDeProbabilite[auteur].append((listeDeMotCommun[auteur][nombre][1] / FreqAuteurInconnu[auteur],
                                                   listeDeMotCommun[auteur][nombre][2] / FreqAuteur[auteur]))

        # Calcul de la ressemblance avec l'auteur(La racine carré de la somme de toutes les différence entre la
        # fréquence normalisé de l'auteur et la fréquence normalisé de l'inconnu au carré)
        for auteur in longeurduDico:
            for nombre in range(len(listeDeProbabilite[auteur])):
                sommeCarre[auteur].append(pow(
                    (listeDeProbabilite[auteur][nombre][0] - listeDeProbabilite[auteur][nombre][1]), 2))
            sommeCarre[auteur] = sum(sommeCarre[auteur])
            sommeCarre[auteur] = math.sqrt(sommeCarre[auteur])

        # affichage
        if args.a:
            print(args.a + ": ", "%.4f" % sommeCarre[auteur])
        else:
            for auteur in range(len(listeDeDico)):
                print(authors[auteur] + ": ", "%.4f" % sommeCarre[auteur])

    # Bloc de code qui génere un texte
    if args.g and args.G:

        listeDeMarkov = [[] for auteur in range(len(listeDeDico))]
        mot = ["les" for auteur in range(len(listeDeDico))]
        listeDeTexte = [["les"] for auteur in range(len(listeDeDico))]
        nombreTotal = [1 for i in range(len(listeDeDico))]

        for nombreDeMots in range(args.G):  # Pour le faire le nombre de mot demandé
            # Crée une liste de chaines de markov pour tout les auteurs demmandées(donc la prochaine couche)
            listeDeSuivantelu = []
            listeDeMarkov = ChaineDeMarkov(listeDeDico)

            for auteur in range(len(listeDeMarkov)):
                tot = 0
                probabilite = []
                motsuivant = [listeDeMarkov[auteur][i][0] for i in range(len(listeDeMarkov[auteur]))]
                for i in range(len(listeDeMarkov[auteur])):
                    tot += listeDeMarkov[auteur][i][1]
                for i in range(len(listeDeMarkov[auteur])):
                    probabilite.append(listeDeMarkov[auteur][i][1] / tot)
                listeDeSuivantelu.append(numpy.random.choice(motsuivant, p=probabilite))
            for auteur in range(len(listeDeMarkov)):
                listeDeTexte[auteur].append(listeDeSuivantelu[auteur])
                mot[auteur] = listeDeSuivantelu[auteur]
            listeDeSuivantelu.clear()

        if args.a:
            with open(args.g, "w+") as doc:
                for auteur in range(len(listeDeTexte)):
                    print(args.a + ":", file=doc)
                    for mot in range(len(listeDeTexte[auteur])):
                        print(listeDeTexte[auteur][mot], file=doc, end=" ")
                    print("\n", file=doc, end=" ")
        else:
            with open(args.g, "w+") as doc:
                for auteur in range(len(listeDeTexte)):
                    print(authors[auteur] + ":", file=doc)
                    for mot in range(len(listeDeTexte[auteur])):
                        print(listeDeTexte[auteur][mot], file=doc, end=" ")
                    print("\n", file=doc, end=" ")

max_stars_repo_path: APP/Remise final/markov.py

max_issues_repo_path: APP/Remise final/markov.py

max_forks_repo_path: APP/Remise final/markov.py


text:
import sys, os
os.environ["CUDA_VISIBLE_DEVICES"] = '1,2,3'

import argparse
import torch
import numpy as np
import json

from ax.plot.contour import plot_contour
from ax.plot.trace import optimization_trace_single_method
from ax.service.managed_loop import optimize
from ax.utils.tutorials.cnn_utils import load_mnist, train, evaluate

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import train_interpolation as train
import dataloader

def hyperparameter_optimization(experiment_path,
                                batch_size,
                                total_trials,
                                model_name,
                                band=1,
                                epochs=5):

    def train_evaluate(parameterization):
        w = parameterization['w']
        s = parameterization['s']
        lr = 1e-4 #parameterization['lr']

        file_dir = os.path.dirname(os.path.abspath(__file__))
        data_path = '/nobackupp10/tvandal/nex-ai-opticalflow/geo/'\
                    '.tmp/training-data/Interp-ABI-L1b-RadM-15min-264x264-Large/'\
                     'Channel-%02i' % band

        model_path = os.path.join(experiment_path, 'LambdaW_%1.2f-LambdaS_%1.2f' % (w, s))
        mu, sd = dataloader.get_band_stats(band)
        dataset = dataloader.InterpLoader(data_path, patch_size=256, mean=mu, std=sd)

        print('Parameterization:', parameterization)
        loss = train.train_net(model_path=model_path,
                          dataset=dataset,
                          epochs=epochs,
                          batch_size=batch_size,
                          max_iterations=20000,
                          lr=lr,
                          lambda_w=w,
                          lambda_s=s,
                          model_name=model_name,
                          occlusion=True,
                          progress=20)
        if not np.isfinite(loss):
            loss = 1e6

        return dict(loss=(-min([loss, 1.]), 0.0))

    best_parameters, values, experiment, model = optimize(
            parameters=[
                        #{"name": "lr", "type": "range", "bounds": [1e-6, 1e-3], "log_scale": True},
                        {"name": "w", "type": "range", "bounds": [1e-2, 1.0]},
                        {"name": "s", "type": "range", "bounds": [1e-2, 1.0]},
                    ],
            evaluation_function=train_evaluate,
            objective_name='loss',
            total_trials=total_trials,
    )

    best_parameters['batch_size'] = batch_size

    print("---------Best Parameters----------")
    print(best_parameters)

    json_params = json.dumps(best_parameters)
    with open(os.path.join(experiment_path, 'best_parameters.json'), 'w') as f:
        f.write(json_params)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--gpus", default="0,1,2,3", type=str)
    parser.add_argument("--epochs", default=5, type=int)
    parser.add_argument("--total_trials", default=20, type=int)
    parser.add_argument("--batch_size", default=128, type=int)
    parser.add_argument("--band", default=1, type=int)
    parser.add_argument("--model_name", default='unet-medium', type=str)
    parser.set_defaults(multivariate=False)
    args = parser.parse_args()

    experiment_path = '/nobackupp10/tvandal/nex-ai-opticalflow/geo/'\
                    '.tmp/models/parameter-search-v3/%02i/' % args.band

    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus

    torch.manual_seed(1)
    hyperparameter_optimization(experiment_path,
                                args.batch_size,
                                args.total_trials,
                                args.model_name,
                                band=args.band,
                                epochs=args.epochs)

max_stars_repo_name: tjvandal/geostationary-superslomo

max_issues_repo_name: tjvandal/geostationary-superslomo

max_forks_repo_name: tjvandal/geostationary-superslomo


text:
# Init Solution
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set()

from IPython.display import display, Markdown
# Init Solution completed

display(Markdown("###### Sin(x) vs Cos(x)"))
x = np.linspace(-6,6, 1000)
fig, ax = plt.subplots(figsize=(10, 5))
ax.plot(x, np.sin(x), ":g", label="sin(x)")
ax.plot(x, np.cos(x), "--r", label="cos(x)")
ax.legend()
ax.set(title="sin(x) vs cos(x)", xlabel="x", ylabel="y")
plt.show()

display(Markdown("###### Normal Distribution (Histogram)"))
rng = np.random.RandomState(42)
x = rng.randn(1000000)
fig, ax = plt.subplots(figsize=(20,5))
ax.hist(x, bins=1000, ec="b")
ax.set(title="Normal Distribution", xlabel="x", ylabel="Height")
plt.show()

display(Markdown("###### Normal Distribution (Line)"))
rng = np.random.RandomState(42)
x = rng.randn(1000000)
size = 1000
bins, b = pd.cut(x, bins=size, retbins=True)
avg = [np.mean([b[i], b[i+1]]) for i in range(size)]
fig, ax = plt.subplots(figsize=(20,5))
ax.plot(avg, bins.value_counts())
ax.set(title="Normal Distribution", xlabel="x", ylabel="Height")
plt.show()

display(Markdown("###### 25% Pie Chart"))
data = np.full(4, .25)
fig, ax = plt.subplots(figsize=(5,5))
ax.pie(data, explode=[0,0,0,.25], autopct="%1.2f%%", startangle=45, shadow=True)
ax.set(title="25% Pie Chart")
plt.show()

display(Markdown("###### Random Triangles"))
rng = np.random.RandomState(42)
points = 40
x = rng.randn(points)
y = rng.randn(points)
fig, ax = plt.subplots(figsize=(10,10))
ax.scatter(x, y, s=(x**2)*200, c=y**2, cmap="rainbow", marker="^", alpha=.8)
plt.show()

max_stars_repo_name: KretschiGL/DataScienceLecture

max_issues_repo_name: KretschiGL/DataScienceLecture

max_forks_repo_name: KretschiGL/DataScienceLecture