text:
import os
from time import time
import json

import torch
import torch.nn as nn
import torchvision.transforms as transforms
from PIL import Image
import cv2
import numpy as np

from opt import opt
from SPPE.src.utils.img import load_image, cropBox, im_to_torch, torch_to_im
from SPPE.src.utils.eval import getPrediction
from pPose_nms import pose_nms
from yolo.darknet import Darknet
from yolo.util import dynamic_write_results
from dataloader import Mscoco
from SPPE.src.main_fast_inference import InferenNet, InferenNet_fast
from align import AlignPoints

args = opt
args.dataset = 'coco'

class ImageLoader:
    def __init__(self, format='yolo'):
        self.format = format
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
        ])

    def getitem_ssd(self, im_path):
        im = Image.open(im_path)
        if im.mode == 'L':
            im = im.convert('RGB')
        inp = load_image(im_path)
        ow = oh = 512
        im = im.resize((ow, oh))
        im = self.transform(im)

        return (im, inp, im_path)

    def getitem_yolo(self, im_path):
        inp_dim = int(opt.inp_dim)
        im, ori_im, im_dim_list = self.prep_image(im_path, inp_dim)

        ims = [im]
        ori_ims = [ori_im]
        im_dim_lists = [im_dim_list]
        im_names = [os.path.basename(im_path)]
        with torch.no_grad():
            # Human Detection
            ims = torch.cat(ims)
            im_dim_lists = torch.FloatTensor(im_dim_lists).repeat(1, 2)
        return (ims, ori_ims, im_names, im_dim_lists)

    def letterbox_image(self, img, inp_dim):
        '''resize image with unchanged asepect retio using padding'''
        img_w, img_h = img.shape[1], img.shape[0]
        w, h = inp_dim
        new_w = int(img_w * min(w / img_w, h / img_h))
        new_h = int(img_h * min(w / img_w, h / img_h))
        resized_image = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_CUBIC)
        canvas = np.full((inp_dim[1], inp_dim[0], 3), 128)
        canvas[(h - new_h) // 2:(h - new_h) // 2 + new_h,
        (w - new_w) // 2:(w - new_w) // 2 + new_w, :] = resized_image
        return canvas

    def prep_image(self, img, inp_dim):
        orig_im = cv2.imread(img)
        dim = orig_im.shape[1], orig_im.shape[0]
        img = (self.letterbox_image(orig_im, (inp_dim, inp_dim)))
        img_ = img[:, :, ::-1].transpose((2, 0, 1)).copy()
        img_ = torch.from_numpy(img_).float().div(255.0).unsqueeze(0)
        return img_, orig_im, dim

class DetectionLoader:
    def __init__(self, dataloder):
        # initialize the file video stream along with the boolean
        # used to indicate if the thread should be stopped or not
        self.det_model = Darknet("yolo/cfg/yolov3-spp.cfg")
        self.det_model.load_weights('models/yolo/yolov3-spp.weights')
        self.det_model.net_info['height'] = opt.inp_dim
        self.det_inp_dim = int(self.det_model.net_info['height'])
        assert self.det_inp_dim % 32 == 0
        assert self.det_inp_dim > 32
        self.det_model.cuda()
        self.det_model.eval()

        self.stopped = False
        self.dataloder = dataloder

    def detect_image(self, im_path):
        im, ori_im, im_name, im_dim_list = self.dataloder.getitem_yolo(im_path)

        with torch.no_grad():
            im = im.cuda()
        prediction = self.det_model(im, CUDA=True)
        # NMS process
        dets = dynamic_write_results(prediction, opt.confidence,
                                     opt.num_classes, nms=True,
                                     nms_conf=opt.nms_thesh)
        if isinstance(dets, int) or dets.shape[0] == 0:
            return (ori_im[0], im_name[0], None, None, None, None, None)

        dets = dets.cpu()
        im_dim_list = torch.index_select(im_dim_list, 0, dets[:, 0].long())
        scaling_factor = torch.min(self.det_inp_dim / im_dim_list, 1)[0] \
            .view(-1, 1)
        # coordinate transfer
        dets[:, [1, 3]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2
        dets[:, [2, 4]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2

        dets[:, 1: 5] /= scaling_factor
        for j in range(dets.shape[0]):
            dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0,
                                          im_dim_list[j, 0])
            dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0,
                                          im_dim_list[j, 1])
        boxes = dets[:, 1:5]
        scores = dets[:, 5:6]

        if boxes.shape[0] > 1:
            boxes = boxes[scores.argmax()].unsqueeze(0)
            scores = scores[scores.argmax()].unsqueeze(0)
            dets = dets[scores.argmax()].unsqueeze(0)
        # len(ori_im) === 1
        for k in range(len(ori_im)):

            boxes_k = boxes[dets[:, 0] == k]
            if isinstance(boxes_k, int) or boxes_k.shape[0] == 0:
                return (ori_im[k], im_name[k], None, None, None, None, None)
            inps = torch.zeros(boxes_k.size(0), 3,
                               opt.inputResH, opt.inputResW)
            pt1 = torch.zeros(boxes_k.size(0), 2)
            pt2 = torch.zeros(boxes_k.size(0), 2)
            return (ori_im[k], im_name[k], boxes_k, scores[dets[:, 0] == k],
                    inps, pt1, pt2)

class DetectionProcessor:

    def __init__(self, detectionLoader):
        self.detection_loader = detectionLoader

    def detect_image(self, im_path, outputdir):
        with torch.no_grad():
            ori_im, im_name, boxes, scores, inps, pt1, pt2 = \
                self.detection_loader.detect_image(im_path)

            if ori_im is None:
                return (None, None, None, None, None, None, None)
            if boxes is None or boxes.nelement() == 0:
                return (None, ori_im, im_name, boxes, scores, None, None)
            inp = im_to_torch(cv2.cvtColor(ori_im, cv2.COLOR_BGR2RGB))
            inps, pt1, pt2 = self.crop_from_dets(inp, boxes, inps, pt1, pt2)
            return (inps, ori_im, im_name, boxes, scores, pt1, pt2)

    def crop_from_dets(self, img, boxes, inps, pt1, pt2):

        imght = img.size(1)
        imgwidth = img.size(2)
        tmp_img = img

        tmp_img[0].add_(-0.406)
        tmp_img[1].add_(-0.457)
        tmp_img[2].add_(-0.480)
        for i, box in enumerate(boxes):
            upLeft = torch.Tensor((float(box[0]), float(box[1])))
            bottomRight = torch.Tensor((float(box[2]), float(box[3])))
            ht = bottomRight[1] - upLeft[1]
            width = bottomRight[0] - upLeft[0]
            if width > 100:
                scaleRate = 0.2
            else:
                scaleRate = 0.3
            upLeft[0] = max(0, upLeft[0] - width * scaleRate / 2)
            upLeft[1] = max(0, upLeft[1] - ht * scaleRate / 2)
            bottomRight[0] = max(
                min(imgwidth - 1, bottomRight[0] + width * scaleRate / 2),
                upLeft[0] + 5
            )
            bottomRight[1] = max(
                min(imght - 1, bottomRight[1] + ht * scaleRate / 2),
                upLeft[1] + 5
            )
            try:
                inps[i] = cropBox(tmp_img, upLeft, bottomRight,
                                  opt.inputResH, opt.inputResW)
            except IndexError:
                print(tmp_img.shape)
                print(upLeft)
                print(bottomRight)
                print('===')

            pt1[i] = upLeft
            pt2[i] = bottomRight
            return inps, pt1, pt2

def fetch_result(boxes, scores, hm_data, pt1, pt2, ori_im, im_name):
    ori_im = np.array(ori_im, dtype=np.uint8)
    if boxes is None:
        return None
    preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2,
                                                      opt.inputResH,
                                                      opt.inputResW,
                                                      opt.outputResH,
                                                      opt.outputResW)
    result = pose_nms(boxes, scores, preds_img, preds_scores)
    result = {
        'imgname': im_name,
        'result': result,
        'bbox': boxes,
    }
    return result

def generate_json(all_results, for_eval=False):
    form = opt.format
    json_results = []
    json_results_cmu = {}
    for im_res in all_results:
        im_name = im_res['imgname']
        # only support single person per image!!
        bbox = im_res['bbox'].numpy().tolist()
        result = {}
        result['bbox'] = bbox
        result['class'] = im_res['class'].numpy().tolist()

        for human in im_res['result']:
            keypoints = []
            if for_eval:
                result['image_id'] = \
                    int(im_name.split('/')[-1].split('.')[0].split('_')[-1])
            else:
                result['image_id'] = im_name.split('/')[-1]
            result['category_id'] = 1
            result['bbox'] = bbox
            kp_preds = human['keypoints']
            kp_scores = human['kp_score']
            pro_scores = human['proposal_score']
            for n in range(kp_scores.shape[0]):
                keypoints.append(float(kp_preds[n, 0]))
                keypoints.append(float(kp_preds[n, 1]))
                keypoints.append(float(kp_scores[n]))
            result['keypoints'] = keypoints
            result['score'] = float(pro_scores)
            if form == 'cmu':  # the form of CMU - Pose
                if result['image_id'] not in json_results_cmu.keys():
                    json_results_cmu[result['image_id']] = {}
                    json_results_cmu[result['image_id']]['version'] = "AlphaPose v0.2"
                    json_results_cmu[result['image_id']]['bodies'] = []
                tmp = {'joints': []}
                result['keypoints'].append(
                    (result['keypoints'][15] + result['keypoints'][18]) / 2)
                result['keypoints'].append(
                    (result['keypoints'][16] + result['keypoints'][19]) / 2)
                result['keypoints'].append(
                    (result['keypoints'][17] + result['keypoints'][20]) / 2)
                indexarr = [0, 51, 18, 24, 30, 15, 21, 27,
                            36, 42, 48, 33, 39, 45, 6, 3, 12, 9]
                for i in indexarr:
                    tmp['joints'].append(result['keypoints'][i])
                    tmp['joints'].append(result['keypoints'][i + 1])
                    tmp['joints'].append(result['keypoints'][i + 2])
                json_results_cmu[result['image_id']]['bodies'].append(tmp)
            elif form == 'open':  # the form of OpenPose
                if result['image_id'] not in json_results_cmu.keys():
                    json_results_cmu[result['image_id']] = {}
                    json_results_cmu[result['image_id']]['version'] = "AlphaPose v0.2"
                    json_results_cmu[result['image_id']]['people'] = []
                tmp = {'pose_keypoints 2d': []}
                result['keypoints'].append(
                    (result['keypoints'][15] + result['keypoints'][18]) / 2)
                result['keypoints'].append(
                    (result['keypoints'][16] + result['keypoints'][19]) / 2)
                result['keypoints'].append(
                    (result['keypoints'][17] + result['keypoints'][20]) / 2)
                indexarr = [0, 51, 18, 24, 30, 15, 21, 27,
                            36, 42, 48, 33, 39, 45, 6, 3, 12, 9]
                for i in indexarr:
                    tmp['pose_keypoints_2d'].append(result['keypoints'][i])
                    tmp['pose_keypoints 2d'].append(result['keypoints'][i + 1])
                    tmp['pose_keypoints 2d'].append(result['keypoints'][i + 2])
                json_results_cmu[result['image_id']]['people'].append(tmp)
            else:
                json_results.append(result)
    return json_results

def run_folder(inputdir, output_json_path, im_names, detection_processor, pose_model,
                pos_reg_model,aligner, missed="./missed"):
    idx = 0
    im_len = len(im_names)
    result_list = []
    for im_name in im_names:
        im_path = os.path.join(inputdir, im_name)
        temp_kps = []
        with torch.no_grad():
            (inps, ori_im, im_name, boxes, scores, pt1, pt2) = \
                detection_processor.detect_image(im_path, output_json_path)
            # Modified
            if boxes is None or boxes.nelement() == 0:
                with open(missed, "a") as f:
                    f.write(im_path + "\n")
                # pass
            try:
                inps_j = inps[0: inps.size(0)].cuda()
            except AttributeError:
                continue
            hm_j = pose_model(inps_j)
            hm = torch.cat([hm_j])
            hm = hm.cpu()
            result = fetch_result(boxes, scores, hm, pt1, pt2, ori_im,
                                  os.path.basename(im_name))

            pos = result['result'][0]['keypoints'].unsqueeze(0).numpy()
            pos = aligner.align_points(pos)[0]
            pos = (pos[..., :2] - 129) / 255
            pos = torch.FloatTensor(pos)
            kp = torch.cat((pos, result['result'][0]['kp_score']), 1)
            kp = kp.unsqueeze(0)
            if len(temp_kps) < 9:
                kp = kp.reshape([1, -1]).cuda()
                temp_kps.append(kp)
                kp = kp.repeat(9, 1).reshape(1,-1)
                outputs = pos_reg_model(kp)
                _, preds = torch.max(outputs, 1)
                result['class'] = preds.cpu()
                result_list.append(result)
            else:
                temp_kps.append(kp)
                temp_kps.pop(0)
                _temp_kps = torch.cat(temp_kps)
                _temp_kps.cuda()
                _temp_kps = _temp_kps.reshape([1, -1])
                outputs = pos_reg_model(kp)
                preds = torch.max(outputs, 1)
                result['class'] = preds.cpu()
                result_list.append(result)
        idx += 1
        print('{}- {} / {}'.format(inputdir, idx, im_len))
    datas = generate_json(result_list)
    with open(output_json_path, 'w') as f:
        f.write(json.dumps(datas))

if __name__ == '__main__':
    # root_dir = 'D:\pycharm_workspace\PoseRecognition\exps\\test1\\tt\\'
    root_dir = 'exps\\test_im'
    dest_path = 'exps\\'
    if not os.path.exists(dest_path):
        os.makedirs(dest_path)
    # Load pose model
    pose_dataset = Mscoco()
    if args.fast_inference:
        pose_model = InferenNet_fast(4 * 1 + 1, pose_dataset)
    else:
        pose_model = InferenNet(4 * 1 + 1, pose_dataset)
    pose_model.cuda()
    pose_model.eval()

    class NeuralNet(nn.Module):
        def __init__(self, input_size):
            super(NeuralNet, self).__init__()
            self.fc1 = nn.Linear(input_size, 512)
            self.fc2 = nn.Linear(512, 512)
            self.fc3 = nn.Linear(512, 4)
            self.drop = nn.Dropout(p=0.2)
            self.relu = nn.ReLU()

        def forward(self, x):
            out = self.fc1(x)
            out = self.relu(out)
            out = self.relu(self.fc2(out))
            out = self.fc3(out)
            return out

    pos_reg_model = NeuralNet(17 * 3 * 9).cuda()
    pos_reg_model.load_state_dict(torch.load('exps\\42_model.ckpt'))
    pos_reg_model.eval()

    aligner = AlignPoints()
    runtime_profile = {
        'dt': [],
        'pt': [],
        'pn': []
    }

    image_loader = ImageLoader()
    detection_loader = DetectionLoader(image_loader)
    detection_processor = DetectionProcessor(detection_loader)
    is_file = os.path.isfile(os.path.join(root_dir, os.listdir(root_dir)[0]))
    if is_file:
        dirname = root_dir.split('\\')[-1]
        # outputdir ='./output'
        ext_filters = ('.jpg', '.png')
        im_names = filter(lambda f: os.path.splitext(
            f)[-1].lower() in ext_filters, os.listdir(root_dir))
        im_names = sorted(list(im_names))
        run_folder(root_dir, os.path.join(dest_path, dirname + '_alphapose_old.json'), im_names,
                   detection_processor, pose_model, pos_reg_model, aligner)
    else:
        for dirname in os.listdir(root_dir):
            inputdir = os.path.join(root_dir, dirname)
        # outputdir = './output'
        ext_filters = ('.jpg', '.png')
        im_names = filter(lambda f: os.path.splitext(f)[-1].lower() in ext_filters,
                          os.listdir(inputdir))
        im_names = sorted(list(im_names))
        run_folder(inputdir, os.path.join(dest_path, dirname + '_alphapose_old.json'), im_names,
                   detection_processor, pose_model)


text:
# coding: utf-8

"""
General Utilities Module
"""

# register src directory path to PYTHONPATH
import sys
from os import path, pardir
current_dir = path.abspath(path.dirname(__file__))
parent_dir = path.abspath(path.join(current_dir, pardir))
parent_parent_dir = path.abspath(path.join(parent_dir, pardir))
sys.path.append(parent_dir)

import numpy as np
import pandas as pd
import tensorflow as tf
import pickle
import os

from modeldev import d_layer
from modeldev import d_model
from mylib import utils
from mylib import validation_utils

def remove_col_containing_specific_str(in_df, specific_str):
    """
    remove columns including specific string from input data frame

    :param  in_df        : pandas.DataFrame, input data frame
    :param  specific_str : str, deletion columns from in_df
    :return out_df       : pandas.DataFrame, data frame that spcific columns is removed
    """

    # see if input data frame has specific string on columns or not
    flag_containing = in_df.columns.str.contains(specific_str)

    # remove columns if input data frame has it
    num_containing = sum(flag_containing)
    if (num_containing >= 1):
        out_df = in_df.drop(columns=in_df.columns[flag_containing])
    else:
        out_df = in_df
    return out_df

def convert_parts_of_str(in_str, target_str, replacing_str):
    """
    convert target string into replacing string of input string
    e.g.
        s_before = 'ab-cd_e'
        s_after = convert_parts_of_str(s_before, 'c', 'K')
        print(s_after)  # ab-Kd_e

    :param  in_str        : str, input string
    :param  target_str    : str, string which is subject to be replaced
    :param  replacing_str : str, replacing string
    :return out_str       : str, output string
    """

    # convert str object into list object (because str is immutable)
    strlist = np.array(list(in_str))

    # detect target string
    ind = np.where(strlist == target_str)
    if (len(ind[0]) == 0):
        out_str = in_str
        return out_str

    # replace string
    strlist[ind[0]] = replacing_str
    out_str = ''.join(strlist)
    return out_str

def detect_first_timing_from_events(in_vec, interval):
    """
    detect first event timing from series of events
    regard events which are within iterval threshold as same event group

    e.g. interval = 5
        input  : [5, 8, 9, 11, 31, 32, 35, 39, 101, 103]
        output : [0, 4, 8]

    :param  in_vec  : ndarray, events timing vector
    :param  iterval : int, interval threshold
    :return out_vec : ndarray, timing vector that first timing is detected from events
    """

    # check differences from previous timestamp
    one_slide_vec = np.zeros(len(in_vec))
    one_slide_vec[1:] = in_vec[0:len(in_vec)-1]
    diff_vec = abs(in_vec - one_slide_vec)

    # first index must be first timing of events
    diff_vec[0] = interval + 1

    # detect groups
    timestamp_diff_df = pd.DataFrame(diff_vec)
    timestamp_diff_df = timestamp_diff_df[timestamp_diff_df[0] > interval]
    out_vec = np.array(timestamp_diff_df.index)

    return out_vec

def convert_ind_to_boolind(ind_vec, bool_list_length):
    """
    convert index numbers list to oolian index list
    e.g.
        input  : [0, 1, 5]
        output : [True, True, False, False, False, True]

    :param  ind_vec          : ndarray,
    :param  bool_list_length : int,
    :return bool_vec         : ndarray,
    """

    validation_utils.validate_vec_length_is_larger(bool_list_length, max(ind_vec), '>')

    if (len(ind_vec) == 0):
        return None
    else:
        out_vec = np.full(bool_list_length, False)
        out_vec[ind_vec] = True
        return out_vec

def load_dataset_X_y(dirname, opt):
    """
    load training data

    :param  : dirname : str, loading target directory
    :param  : opt     : str, option data format "pandas" or "numpy"
    :return : data_X  : numpy, training data
    :return : data_y  : numpy, true data
    """

    # X
    input_filename = dirname + '/data_X.csv'
    df_X = pd.read_csv(input_filename, encoding='shift-jis')

    # y
    input_filename = dirname + '/data_y.csv'
    df_y = pd.read_csv(input_filename, encoding='shift-jis')

    if (opt == 'pd'):
        return df_X, df_y
    elif (opt == 'np'):
        return df_X.values, df_y.values

def translate_contents_without_string(input):
    """
    translate string contents of list into contents without string
    e.g. function_list = ['function1', 'function2']
            -> translated_function_list = [function1, function2]

         input = 'hoge'
            -> translated_input = hoge

    :param  input  : input including string
    :return output : output excluding string
    """

    output_list = []

    if (type(input) is str):
        cmd = 'output_list.append(' + input + ')'
        exec(cmd)
        return output_list[0]
    elif (type(input) is list):
        if (len(input) == 0):
            return output_list
        else:
            for i in range(len(input)):
                cmd = 'output_list.append(' + input[i] + ')'
                exec(cmd)
            return output_list
    else:
        print('Please input str or "list" including "str".')

def save_result(sess, model, out_dirpath):
    """
    save training result (tf.Session and model)

    :param  sess       : tf.Session()
    :param  model      : d_model.Model()
    :param  out_dirpath: str, directory path to save results
    :return
    """
    if (not os.path.isdir(out_dirpath)):
        os.makedirs(out_dirpath)

    # save tf.Session()
    saver = tf.train.Saver()
    saver.save(sess, out_dirpath + 'model_sess.ckpt')

    # # save model
    # f = open(out_dirpath + 'model.textfile', 'w')
    # pickle.dump(model, f)
    # f.close
    pickle.dump(model, open(out_dirpath + 'model.pickle', 'wb'))

ext: py

max_stars_repo_name: hsmtknj/ml

max_issues_repo_name: hsmtknj/ml

max_forks_repo_name: hsmtknj/ml

reason: import numpy


text:
from typing import Tuple
import numpy as np

def get_dataset() -> Tuple[np.ndarray, np.ndarray]:
    """OR dataset."""
    x=np.array([[0,0],[1,0],[0,1],[1,1]])
    y= np.array([[0],[1],[1],[1]])
    return x,y

def accuracy_score(y_true:np.ndarray,y_pred: np.ndarray) -> float:
    n = y_true.shape[0]
    accuracy = np.sum(y_true == y_pred)/n
    return accuracy

def step_function(input_signal :np.ndarray) -> np.ndarray:
    output_signal = (input_signal > 0.0).astype(np.int_)
    return output_signal

class Perceptron:

    def __init__(self, learning_rate:float, input_dim:int) ->None:
        self.learning_rate = learning_rate
        self.input_dim = input_dim
        self.w = np.random.uniform(-1,1,size=(input_dim,1))

    def _update_weights(self, x:np.ndarray, y:np.ndarray, y_pred:np.ndarray)-> None:
        error = (y-y_pred)
        delta = error*x
        for delta_i in delta:
            self.w= self.w + self.learning_rate*delta_i.reshape(-1,1)

    def train(self, x:np.ndarray, y_pred:np.ndarray, epochs:int = 1) -> None:
        for epoch in range (1, epochs + 1):
            y_pred = self.predict(x)
            self._update_weights(x, y, y_pred)
            accuracy = accuracy_score(y,y_pred)
            print(f"Epoch: {epoch} Accuracy: {accuracy}")

    def predict(self, x:np.ndarray) -> np.ndarray:
        input_signal = np.dot(x, self.w)
        output_signal=step_function(input_signal)
        return output_signal

    def evaluate(self,x:np.ndarray,y:np.ndarray)-> None :
        y_pred = self.predict(x)
        return accuracy_score(y,y_pred)

if __name__ =="__main__":
    x,y = get_dataset()

    input_dim = x.shape[1]
    learning_rate = 0.5

    p= Perceptron(learning_rate,input_dim)
    p.train(x,y,epochs=10)


text:
import itertools
import re
import warnings

import numpy as np
import pandas as pd
import scipy
from sklearn.metrics.pairwise import cosine_similarity
from scattertext import ParsedCorpus
from scattertext.representations.Word2VecFromParsedCorpus import Word2VecDefault

class CategoryEmbeddingsResolver:
    def __init__(self, corpus, term_acceptance_re=re.compile('[a-z]{3,}')):
        self.corpus_ = corpus
        self.category_embeddings_ = {}
        self.category_word2vec_model_ = {}
        self.term_acceptance_re = term_acceptance_re

    def _verify_category(self, category):
        if category not in self.corpus_.get_categories():
            raise Exception("Category %s is not in corpus." % category)
        if category in self.category_embeddings_:
            raise Exception("You have already set embeddings by running set_embeddings or set_embeddings_model.")

    def embed_category(self, category, model=None):
        '''

        :param model: gensim word2vec.Word2Vec model
        :param term_acceptance_re : SRE_Pattern, Regular expression to identify
            valid terms, default re.compile('[a-z]{3,}')
        :return: EmbeddingsResolver
        '''
        self._verify_category(category)
        if self.term_acceptance_re is not None:
            acceptable_terms = set([t for t in self.corpus_.get_terms() if self.term_acceptance_re.match(t)])
        else:
            acceptable_terms = set(self.corpus_.get_terms())
        trained_model = CategorySpecificWord2VecFromParsedCorpus(self.corpus_, category, model).train()
        self.category_word2vec_model_[category] = trained_model
        word2dwe = {word: trained_model[word] for word in trained_model.wv.vocab.keys()}
        self.category_embeddings_[category] = word2dwe
        return self

    def embed_all_categories(self):
        for category in self.corpus_.get_categories():
            self.embed_category(category)
        return self

class CartegorySpecificCorpusAdapterForGensim(object):
    @staticmethod
    def get_sentences(corpus, category):
        '''
        Parameters
        ----------
        corpus, ParsedCorpus
        category, str

        Returns
        -------
        iter: [sentence1word1, ...], [sentence2word1, ...]
        '''
        #assert isinstance(corpus, ParsedCorpus)
        return itertools.chain(*[[[t.lower_ for t in sent if not t.is_punct]
                                  for sent in doc.sents]
                                 for doc_catgory, doc
                                 in zip(corpus.get_category_names_by_row(), corpus.get_parsed_docs())
                                 if category == doc_catgory])

class CategorySpecificWord2VecFromParsedCorpus(Word2VecDefault):
    def __init__(self, corpus, category, word2vec_model=None):
        '''
        Parameters
        ----------
        corpus: ParsedCorpus
          from which to build word2vec model
        category, str
        word2vec_model: word2vec.Word2Vec
            Gensim instance to be used to train word2vec model
        '''
        try:
            from gensim.models import word2vec
            assert word2vec_model is None or isinstance(word2vec_model, word2vec.Word2Vec)
        except:
            warnings.warn("You should really install gensim, but we're going to duck-type your model and hope it works")
        #print(type(corpus))
        #assert isinstance(corpus, ParsedCorpus)
        self.corpus = corpus
        self.category = category
        self.model = self._get_word2vec_model(word2vec_model)

    def train(self, epochs=2000, training_iterations=5):
        '''
        Parameters
        ----------
        epochs : int
          Number of epochs to train for.  Default is 2000.
        training_iterations : int
            Number of times to repeat training process. Default is training_iterations.

        Returns
        -------
        A trained word2vec model.
        '''

        self._scan_and_build_vocab()
        for _ in range(training_iterations):
            self.model.train(CartegorySpecificCorpusAdapterForGensim.get_sentences(self.corpus, self.category),
                             total_examples=self.model.corpus_count,
                             epochs=epochs)
        return self.model

    def _get_word2vec_model(self, word2vec_model):
        return (self._default_word2vec_model()
                if word2vec_model is None
                else word2vec_model)

    def _scan_and_build_vocab(self):
        try:
            self.model.scan_vocab(CartegorySpecificCorpusAdapterForGensim.get_sentences(self.corpus, self.category))
        except:
            pass
        self.model.build_vocab(CartegorySpecificCorpusAdapterForGensim.get_sentences(self.corpus, self.category))

class EmbeddingAligner(object):
    def __init__(self, category_embedding_resolver, category1, category2, prefix1=None, prefix2=None):
        '''

        :param category_embedding_resolver: CategoryEmbeddingsResolver
        :param category1: str
        :param category2: str
        :param prefix1: str
        :param prefix2: str
        '''
        #assert issubclass(type(category_embedding_resolver), CategoryEmbeddingsResolver)
        self.category_embedding_resolver = category_embedding_resolver
        valid_categories = category_embedding_resolver.corpus_.get_categories()
        assert category1 in valid_categories
        assert category2 in valid_categories
        self.category1 = category1
        self.category2 = category2
        cat1_dwe_dict = category_embedding_resolver.category_embeddings_[category1]
        cat2_dwe_dict = category_embedding_resolver.category_embeddings_[category2]
        self.terms = np.array(list((set(cat1_dwe_dict.keys()) & set(cat2_dwe_dict.keys()))))
        self.cat1_dwe_ar = np.stack([cat1_dwe_dict[word] for word in self.terms])
        self.cat2_dwe_ar = np.stack([cat2_dwe_dict[word] for word in self.terms])

        #self.cat1_dwe_ar_norm, self.cat2_dwe_ar_norm, self.disparity = \
        #    scipy.spatial.procrustes(self.cat1_dwe_ar, self.cat2_dwe_ar)
        self.pairwise_sim, sv = scipy.linalg.orthogonal_procrustes(self.cat1_dwe_ar, self.cat2_dwe_ar)

        import pdb; pdb.set_trace()

        #self.pairwise_sim = cosine_similarity(np.vstack([self.cat1_dwe_ar_norm,
        #                                                 self.cat2_dwe_ar_norm]))
        #

        self.pairwise_sim_sort = np.argsort(-self.pairwise_sim, axis=1)

        def distinct_prefix(x, y):
            for i, (xc, yc) in enumerate(zip(x, y)):
                if xc != yc:
                    return (x[:i + 1], y[:i + 1])
            return x, y

        myprefix1, myprefix2 = distinct_prefix(category1, category2)
        self.prefix1 = myprefix1 if prefix1 is None else prefix1
        self.prefix2 = myprefix2 if prefix2 is None else prefix2

        self.labeled_terms = np.array([self.prefix1 + '_' + w for w in self.terms]
                                      + [self.prefix2 + '_' + w for w in self.terms])

    def get_terms(self):
        return self.terms

    def project_separate(self, projector=None):
        if projector is None:
            from umap import UMAP
            projector = UMAP(n_components=2, metric='cosine')
        both_category_embeddings = np.vstack([self.cat1_dwe_ar_norm,
                                              self.cat2_dwe_ar_norm])
        projected_ar = projector.fit_transform(both_category_embeddings)
        df = pd.DataFrame(projected_ar, columns=['x', 'y'], index=self.labeled_terms)
        df['category'] = [self.category1] * len(self.terms) + [self.category2] * len(self.terms)
        return df

    def get_report_df(self, n_terms=5):
        conterpart_idx = np.hstack([np.arange(len(self.terms)) + len(self.terms),
                                    np.arange(len(self.terms))])
        idx = np.arange(len(self.terms))
        similarity_df = pd.DataFrame({
            'cosine_distance': self.pairwise_sim[[idx], conterpart_idx[idx]][0],
            'rank_' + self.prefix1: np.where(
                self.pairwise_sim_sort[idx] == conterpart_idx[idx][:, None]
            )[1],
            'rank_' + self.prefix2: np.where(
                self.pairwise_sim_sort[conterpart_idx[idx]] == idx[:, None]
            )[1],
            'context_' + self.prefix1: pd.DataFrame(
                self.labeled_terms[self.pairwise_sim_sort[idx, 1:1 + n_terms]]
            ).apply(', '.join, axis=1).values,
            'context_' + self.prefix2: pd.DataFrame(
                self.labeled_terms[self.pairwise_sim_sort[conterpart_idx[idx], 1:1 + n_terms]]
            ).apply(', '.join, axis=1).values,
        },
            index=self.terms
        )
        return pd.merge(
            similarity_df
                .assign(min_rank=lambda x: np.max(x[['rank_' + self.prefix1, 'rank_' + self.prefix1]], axis=1))
                .sort_values(by='min_rank', ascending=False),
            self.category_embedding_resolver.corpus_
                .get_term_freq_df(),
            left_index=True,
            right_index=True
        )

max_stars_repo_name: shettyprithvi/scattertext

max_issues_repo_name: shettyprithvi/scattertext

max_forks_repo_name: shettyprithvi/scattertext

reason: import numpy,import scipy


text:
import typing as tp

import numpy as np

from elegy import utils

ArrayLike = tp.Union[np.ndarray, np.ndarray]
T = tp.TypeVar("T")
Container = tp.Union[
    T,
    tp.Tuple["Container", ...],
    tp.Dict[str, "Container"],
]
ArrayHolder = Container[ArrayLike]

IndexLike = tp.Union[str, int, tp.Iterable[tp.Union[str, int]]]

Shape = tp.Sequence[int]
ShapeLike = tp.Union[int, Shape]
FloatLike = tp.Union[float, np.ndarray]
DType = tp.Any
ParamName = str

Params = tp.Mapping[str, tp.Mapping[ParamName, np.ndarray]]
State = tp.Mapping[str, tp.Mapping[str, np.ndarray]]
PadFn = tp.Callable[[int], tp.Tuple[int, int]]
PadFnOrFns = tp.Union[PadFn, tp.Sequence[PadFn]]
PRNGKey = np.ndarray
Parameters = tp.Dict[str, tp.Any]
States = tp.Dict[str, tp.Any]

class Initializer(utils.Protocol):
    def __call__(self, shape: Shape, dtype: DType) -> np.ndarray:
        ...

max_stars_repo_name: SamuelMarks/elegy

max_issues_repo_name: SamuelMarks/elegy

max_forks_repo_name: SamuelMarks/elegy


text:
"""GAN_model.py: Tensorflow model for the Generative Adversarial Networks"""
__author__      = "Khalid M. Kahloot"
__copyright__   = "Copyright 2019, Only for professionals"

import sys

sys.path.append('..')
import gc
import glob

from PIL import Image as PILImage
from IPython.display import display, Image
import os

from base.base_model import BaseModel
import tensorflow as tf
import numpy as np

from .GAN_graph import GANGraph
from .GANCNN_graph import GANCNNGraph

from utils.logger import Logger
from utils.early_stopping import EarlyStopping
from tqdm import tqdm
import sys
from collections import defaultdict

import utils.utils as utils
import utils.constants as const

from sklearn.decomposition import PCA
from utils.plots import plot_dataset, plot_samples, merge, resize_gif

class GANModel(BaseModel):
    def __init__(self, network_params, act_out=tf.nn.softplus,
                 transfer_fct=tf.nn.relu, learning_rate=0.002,
                 kinit=tf.contrib.layers.xavier_initializer(), batch_size=32,
                 latent_dim=10, dropout=0.2, batch_norm=True, epochs=200, checkpoint_dir='',
                 summary_dir='', result_dir='', restore=False, plot=False, colab=False, model_type=const.GAN):
        super().__init__(checkpoint_dir, summary_dir, result_dir)

        self.summary_dir = summary_dir
        self.result_dir = result_dir
        self.batch_size = batch_size
        self.latent_dim = latent_dim
        self.dropout = dropout

        self.epochs = epochs
        self.z_file = result_dir + '/z_file'
        self.restore = restore
        self.colab = colab
        self.plot = plot

        if self.plot:
            self.z_space_files = list()
            self.recons_files = list()

        # Creating computational graph for train and test
        self.graph = tf.Graph()
        with self.graph.as_default():
            if (model_type == const.GAN):
                self.model_graph = GANGraph(network_params, act_out,
                                            transfer_fct, learning_rate, kinit, batch_size,
                                            reuse=False)
            if (model_type == const.GANCNN):
                self.model_graph = GANCNNGraph(network_params, act_out,
                                               transfer_fct, learning_rate, kinit, batch_size,
                                               reuse=False)

            self.model_graph.build_graph()
            self.trainable_count = np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])

    def train_epoch(self, session, logger, data_train):
        loop = tqdm(range(data_train.num_batches(self.batch_size)))
        d_loss = []
        d_loss_real = []
        d_loss_fake = []
        dL2_loss = []

        g_loss = []
        gL2_loss = []

        for _ in loop:
            batch_x = next(data_train.next_batch(self.batch_size))
            self.batch_z = np.random.uniform(-1, 1, [self.batch_size, self.latent_dim]).astype(np.float32)
            g_loss_curr, gL2_loss_curr, d_loss_curr, d_loss_real_curr, d_loss_fake_curr, dL2_loss_curr = self.model_graph.partial_fit(
                session, batch_x, self.batch_z)

            g_loss.append(g_loss_curr)
            gL2_loss.append(gL2_loss_curr)

            d_loss.append(d_loss_curr)
            d_loss_real.append(d_loss_real_curr)
            d_loss_fake.append(d_loss_fake_curr)
            dL2_loss.append(dL2_loss_curr)

        g_loss_tr = np.mean(g_loss)
        gL2_loss_tr = np.mean(gL2_loss)

        d_loss_tr = np.mean(d_loss)
        d_loss_real_tr = np.mean(d_loss_real)
        d_loss_fake_tr = np.mean(d_loss_fake)
        dL2_loss_tr = np.mean(dL2_loss)

        cur_it = self.model_graph.global_step_tensor.eval(session)
        summaries_dict = {
            'g_loss_tr': g_loss_tr,
            'gL2_loss_tr': gL2_loss_tr,
            'd_loss_tr': d_loss_tr,
            'd_loss_real_tr': d_loss_real_tr,
            'd_loss_fake_tr': d_loss_fake_tr,
            'dL2_loss_tr': dL2_loss_tr
        }

        logger.summarize(cur_it, summaries_dict=summaries_dict)

        return g_loss_tr, gL2_loss_tr, d_loss_tr, d_loss_real_tr, d_loss_fake_tr, dL2_loss_tr

    def valid_epoch(self, session, logger, data_valid):
        loop = tqdm(range(data_valid.num_batches(self.batch_size)))
        d_loss = []
        d_loss_real = []
        d_loss_fake = []
        dL2_loss = []

        g_loss = []
        gL2_loss = []

        for _ in loop:
            batch_x = next(data_valid.next_batch(self.batch_size))
            g_loss_curr, gL2_loss_curr, d_loss_curr, d_loss_real_curr, d_loss_fake_curr, dL2_loss_curr = self.model_graph.evaluate(
                session, batch_x, self.batch_z)

            g_loss.append(g_loss_curr)
            gL2_loss.append(gL2_loss_curr)

            d_loss.append(d_loss_curr)
            d_loss_real.append(d_loss_real_curr)
            d_loss_fake.append(d_loss_fake_curr)
            dL2_loss.append(dL2_loss_curr)

        g_loss_val = np.mean(g_loss)
        gL2_loss_val = np.mean(gL2_loss)

        d_loss_val = np.mean(d_loss)
        d_loss_real_val = np.mean(d_loss_real)
        d_loss_fake_val = np.mean(d_loss_fake)
        dL2_loss_val = np.mean(dL2_loss)

        cur_it = self.model_graph.global_step_tensor.eval(session)
        summaries_dict = {
            'g_loss_val': g_loss_val,
            'gL2_loss_val': gL2_loss_val,
            'd_loss_val': d_loss_val,
            'd_loss_real_val': d_loss_real_val,
            'd_loss_fake_val': d_loss_fake_val,
            'dL2_loss_val': dL2_loss_val
        }

        logger.summarize(cur_it, summaries_dict=summaries_dict)

        return g_loss_val, gL2_loss_val, d_loss_val, d_loss_real_val, d_loss_fake_val, dL2_loss_val

    def train(self, data_train, data_valid, enable_es=1):

        with tf.Session(graph=self.graph) as session:
            tf.set_random_seed(1234)

            logger = Logger(session, self.summary_dir)
            # here you initialize the tensorflow saver that will be used in saving the checkpoints.
            # max_to_keep: defaults to keeping the 5 most recent checkpoints of your model
            saver = tf.train.Saver()
            early_stopping = EarlyStopping(name='total loss')

            if (self.restore and self.load(session, saver)):
                num_epochs_trained = self.model_graph.cur_epoch_tensor.eval(session)
                print('EPOCHS trained: ', num_epochs_trained)
            else:
                print('Initializing Variables ...')
                tf.global_variables_initializer().run()

            if (self.model_graph.cur_epoch_tensor.eval(session) == self.epochs):
                return

            for cur_epoch in range(self.model_graph.cur_epoch_tensor.eval(session), self.epochs + 1, 1):

                print('EPOCH: ', cur_epoch)
                self.current_epoch = cur_epoch

                g_loss_tr, gL2_loss_tr, d_loss_tr, d_loss_real_tr, d_loss_fake_tr, dL2_loss_tr = self.train_epoch(
                    session, logger, data_train)

                g_loss_val, gL2_loss_val, d_loss_val, d_loss_real_val, d_loss_fake_val, dL2_loss_val = self.valid_epoch(
                    session, logger, data_valid)

                print('TRAIN | GAN Generator: g_loss: ', g_loss_tr, ' | L2_loss: ', gL2_loss_tr)
                print('VALID | GAN Generator: g_loss: ', g_loss_val, ' | L2_loss: ', gL2_loss_val)
                print()
                print('TRAIN | GAN Discriminator: d_loss: ', d_loss_tr, ' | d_loss_real: ', d_loss_real_tr)
                print('      | d_loss_fake: ', d_loss_fake_tr, ' | L2_loss: ', dL2_loss_tr)
                print('VALID | GAN Discriminator: d_loss: ', d_loss_val, ' | d_loss_real: ', d_loss_real_val)
                print('      | d_loss_fake: ', d_loss_fake_val, ' | L2_loss: ', dL2_loss_val)

                if (cur_epoch == 1) or ((cur_epoch % const.SAVE_EPOCH == 0) and ((cur_epoch != 0))):
                    self.save(session, saver, self.model_graph.global_step_tensor.eval(session))
                    if self.plot:
                        self.generate_samples(data_train, session, cur_epoch)

                if cur_epoch % 50 == 0:
                    if self.colab:
                        self.push_colab()

                session.run(self.model_graph.increment_cur_epoch_tensor)

                # Early stopping
                # if(enable_es==1 and early_stopping.stop(g_loss_val)):
                # print('Early Stopping!')
                # break

            self.save(session, saver, self.model_graph.global_step_tensor.eval(session))
            if self.plot:
                self.generate_samples(data_train, session, cur_epoch)

            if self.colab:
                self.push_colab()

        return

    '''
    ------------------------------------------------------------------------------
                                         MODEL OPERATIONS
    ------------------------------------------------------------------------------
    '''

    def reconst(self, inputs):
        return self.decode(self.encode(inputs))

    def encode(self, inputs):
        self.model_graph.z_var = np.random.uniform(-1, 1, [self.batch_size, self.latent_dim]).astype(np.float32)
        for _ in range(10):
            self.model_graph.z_var = np.random.uniform(-1, 1, [self.batch_size, self.latent_dim]).astype(np.float32)
            ae_loss = self.batch_function(self.model_graph.encode, inputs)
            print('AE loss: {}'.format(np.mean(ae_loss)))
        z_var = np.repeat(self.model_graph.z_var, inputs.shape[0] // self.batch_size).reshape((-1, self.latent_dim))
        return z_var[:inputs.shape[0]]

    def decode(self, z):
        return self.batch_function(self.model_graph.decode, z)

    def interpolate(self, input1, input2):

        z1 = self.encode(input1)
        z2 = self.encode(input2)

        decodes = defaultdict(list)
        for idx, ratio in enumerate(np.linspace(0, 1, 10)):
            decode = dict()
            z = np.stack([self.slerp(ratio, r1, r2) for r1, r2 in zip(z1, z2)])
            z_decode = self.decode(z)

            for i in range(z_decode.shape[0]):
                try:
                    decode[i] = [z_decode[i].compute()]
                except:
                    decode[i] = [z_decode[i]]

            for i in range(z_decode.shape[0]):
                decodes[i] = decodes[i] + decode[i]

        imgs = []

        for idx in decodes:
            l = []

            l += [input1[idx:idx + 1][0]]
            l += decodes[idx]
            l += [input2[idx:idx + 1][0]]

            imgs.append(l)
        del decodes

        return imgs

    def reconst_loss(self, inputs):
        return self.batch_function(self.model_graph.reconst_loss, inputs)

    def slerp(self, val, low, high):
        """Code from https://github.com/soumith/dcgan.torch/issues/14"""
        omega = np.arccos(np.clip(np.dot(low / np.linalg.norm(low), high.transpose() / np.linalg.norm(high)), -1, 1))
        so = np.sin(omega)

        # l1 = lambda low, high, val: (1.0-val) * low + val * high
        # l2 = lambda low, high, val, so, omega: np.sin((1.0-val)*omega) / so * low + np.sin(val*omega) / so * high
        if so == 0:
            return (1.0 - val) * low + val * high  # L'Hopital's rule/LERP
        return np.sin((1.0 - val) * omega) / so * low + np.sin(val * omega) / so * high

    '''
    ------------------------------------------------------------------------------
                                         GENERATE SAMPLES
    ------------------------------------------------------------------------------
    '''

    def animate(self):
        if not hasattr(self, 'z_space_files') or len(self.recons_files) == 0:
            print('No images were generated during trainning!')

            path = self.summary_dir
            st = path + '/{} samples generation in epoch'.format(self.summary_dir.split('/')[-1:][0])
            self.recons_files = [f for f in glob.glob(path + "**/*.jpg", recursive=True) if f.startswith(st)]
            self.recons_files = list(map(lambda f: f.split('/')[-1], self.recons_files))

            self.recons_files.sort(key=utils.natural_keys)
            self.recons_files = list(map(lambda f: path + '/' + f, self.recons_files))

            st = path + '/{} Z space in epoch'.format(self.summary_dir.split('/')[-1:][0])
            self.z_space_files = [f for f in glob.glob(path + "**/*.jpg", recursive=True) if f.startswith(st)]
            self.z_space_files = list(map(lambda f: f.split('/')[-1], self.z_space_files))

            self.z_space_files.sort(key=utils.natural_keys)
            self.z_space_files = list(map(lambda f: path + '/' + f, self.z_space_files))

            if len(self.recons_files) == 0:
                print('No previous images found!')
                return None, None

        path = self.summary_dir
        st = path + '/{} samples generation in epoch'.format(self.summary_dir.split('/')[-1:][0])
        images = [PILImage.open(fn) for fn in self.recons_files]

        images[0].save(st + '_animate.gif', save_all=True, append_images=images[1:], duration=len(images) * 60,
                       loop=0xffff)
        images[0].save(st + '_res_animate.gif', save_all=True, append_images=images[1:], duration=len(images) * 60,
                       loop=0xffff, dpi=70)
        with open(st + '_res_animate.gif', 'rb') as f:
            img1 = Image(data=f.read(), format='gif')

        st = path + '/{} Z space in epoch'.format(self.summary_dir.split('/')[-1:][0])
        images = [PILImage.open(fn) for fn in self.z_space_files]
        images[0].save(st + '_animate.gif', save_all=True, append_images=images[1:], duration=len(images) * 60,
                       loop=0xffff, dpi=70)

        resize_gif(path=st + '_animate.gif', save_as=st + '_res_animate.gif', resize_to=(900, 450))
        with open(st + '_res_animate.gif', 'rb') as f:
            img2 = Image(data=f.read(), format='gif')

        return img1, img2

    def generate_samples(self, data, session, cur_epoch=''):
        # Generating Z Space
        print('Generating Z Space ...')
        z_recons_l = self.encode(data.x)
        np.savez(self.z_file, z_recons_l)

        pca = PCA(n_components=2)
        Z_pca = pca.fit_transform(z_recons_l)
        print('Z space dimensions: {}'.format(Z_pca.shape))
        print('Ploting Z space ...')
        z_space = self.summary_dir + '/{} Z space in epoch {}.jpg'.format(self.summary_dir.split('/')[-1:][0],
                                                                           cur_epoch)
        self.z_space_files.append(z_space)
        plot_dataset(Z_pca, y=data.labels, save=z_space)

        del Z_pca, z_recons_l
        gc.collect()

        # Generating Samples
        print('Generating Samples ...')

        x_recons_l = self.reconst(data.samples)
        recons_file = self.summary_dir + '/{} samples generation in epoch {}.jpg'.format(
            self.summary_dir.split('/')[-1:][0], cur_epoch)
        self.recons_files.append(recons_file)
        plot_samples(x_recons_l, scale=10, save=recons_file)

        del x_recons_l
        gc.collect()

    '''
     ------------------------------------------------------------------------------
                                          MODEL FUNCTIONS
     ------------------------------------------------------------------------------
     '''

    def batch_function(self, func, p1):
        with tf.Session(graph=self.graph) as session:
            saver = tf.train.Saver()
            if (self.load(session, saver)):
                num_epochs_trained = self.model_graph.cur_epoch_tensor.eval(session)
                print('EPOCHS trained: ', num_epochs_trained)
            else:
                return

            output_l = list()

            start = 0
            end = self.batch_size

            with tqdm(range(p1.shape[0] // self.batch_size)) as pbar:
                while end < p1.shape[0]:
                    output = func(session, p1[start:end])
                    output = np.array(output)
                    output = output.reshape([output.shape[0] * output.shape[1]] + list(output.shape[2:]))
                    output_l.append(output)

                    start = end
                    end += self.batch_size
                    pbar.update(1)
                else:

                    x1 = p1[start:]
                    xsize = len(x1)
                    p1t = np.zeros([self.batch_size - xsize] + list(x1.shape[1:]))

                    output = func(session, np.concatenate((x1, p1t), axis=0))
                    output = np.array(output)
                    output = output.reshape([output.shape[0] * output.shape[1]] + list(output.shape[2:]))[0:xsize]

                    output_l.append(output)

                    pbar.update(1)

        try:
            return np.vstack(output_l)
        except:
            output_l = list(map(lambda l: l.reshape(-1, 1), output_l))
        return np.vstack(output_l)


text:
import collections
import logging
import math
import random

import numpy as np
import pandas as pd
import plotly.plotly as py
import plotly.graph_objs as go
import tensorflow as tf
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans

from tools.metadata import get_hero_dict, get_last_patch

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

data_index = 0

def _build_vocabulary(words, vocabulary_size):
    """ Creates a dictionary representing a vocabulary and counts the appearances of each word
    In this context, each word represents a hero's index casted to string e.g. Anti-Mage -> "1"

    Args:
        words: list of strings representing the corpus
        vocabulary_size: number of words to be evaluated (the vocabulary will contain only this
            number of words, even if there are more unique words in the corpus)
    Returns:
        data: list of indices obtained by mapping the words' indices to the corpus
        count: list of [word, appearances] for the corpus
        dictionary: the vocabulary (the key is the word, and the value is the appearances)
        reverse_dictionary: the reversed vocabulary (the key are the appearances and the values
            are the words)
    """

    # create dictionary with the most common heroes
    count = [['UNK', -1]]
    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
    dictionary = dict()

    for word, _ in count:
        dictionary[word] = len(dictionary)

    data = list()
    unk_count = 0
    for word in words:
        if word in dictionary:
            index = dictionary[word]
        else:
            # the word is unknown
            index = 0
            unk_count = unk_count + 1
        data.append(index)

    count[0][1] = unk_count

    # save the dictionary's reversed version for later usage
    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return data, count, dictionary, reverse_dictionary

def _generate_batch(data, batch_size, num_skips, window_size):
    """ Generates a batch of data to be used in training using the skip-gram flavor of word2vec

    Args:
        data: list of indices obtained by mapping the words' indices to the corpus
        batch_size: number of samples to be used in a batch
        num_skips: number of skips hyperparameter of word2vec
        window_size: window size hyperparameter of word2vec
    Returns:
        batch: batch of data to be used in training
        labels: labels of each sample in the batch
    """

    global data_index
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    span = 2 * window_size + 1
    buffer = collections.deque(maxlen=span)

    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)

    for i in range(batch_size // num_skips):
        # target label at the center of the buffer
        target = window_size
        targets_to_avoid = [window_size]

        for j in range(num_skips):
            while target in targets_to_avoid:
                target = random.randint(0, span - 1)
            targets_to_avoid.append(target)
            batch[i * num_skips + j] = buffer[window_size]
            labels[i * num_skips + j, 0] = buffer[target]

        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)

    return batch, labels

def _train_word2vec(data,
                    batch_size,
                    vocabulary_size,
                    embedding_size,
                    neg_samples,
                    window_size,
                    num_steps,
                    reverse_dictionary,
                    heroes_dict):
    """ Given input data and hyperparameters, train the dataset of games using word2vec with
    skip-gram flavor

     Args:
        data: list of indices obtained by mapping the words' indices to the corpus
        batch_size: number of samples to be used in a batch
        vocabulary_size: number of words to be evaluated (the vocabulary will contain only this
            number of words, even if there are more unique words in the corpus)
        embedding_size: number of dimensions when creating word embeddings
        neg_samples: word2vec negative samples hyperparameter
        window_size: word2vec window size hyperparameter
        num_steps: number of steps to train for at (at least 10k should be fine)
        window_size: window size hyperparameter of
        reverse_dictionary: the reversed vocabulary (the key are the appearances and the values
            are the words)
        heroes_dict: dictionary that maps the hero's ID to its name
    Returns:
        final_embeddings: np.array of (samples, embedding_size) dimension corresponding to each
            hero's embeddings
    """

    valid_size = 15
    valid_examples = np.random.randint(0, vocabulary_size, valid_size)

    graph = tf.Graph()

    with graph.as_default(), tf.device('/cpu:0'):
        train_dataset = tf.placeholder(tf.int32, shape=[batch_size])
        train_labels = tf.placeholder(tf.float32, shape=[batch_size, 1])
        valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

        embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        softmax_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],
                                                          stddev=1.0 / math.sqrt(embedding_size)))
        softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))

        embed = tf.nn.embedding_lookup(embeddings, train_dataset)
        loss = tf.reduce_mean(
            tf.nn.sampled_softmax_loss(softmax_weights,
                                       softmax_biases,
                                       train_labels,
                                       embed,
                                       neg_samples,
                                       vocabulary_size))

        optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)

        norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / norm
        valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
        similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))

    with tf.Session(graph=graph) as session:
        session.run(tf.global_variables_initializer())
        logger.info('Initialized graph')
        average_loss = 0

        for step in range(num_steps):
            batch_data, batch_labels = _generate_batch(data,
                                                       batch_size,
                                                       2 * window_size,
                                                       window_size)

            feed_dict = {train_dataset: batch_data, train_labels: batch_labels}
            _, new_loss = session.run([optimizer, loss], feed_dict=feed_dict)

            average_loss += new_loss

            # print the loss every 2k steps
            if step % 2000 == 0:
                if step > 0:
                    average_loss = average_loss / 2000
                logger.info('Average loss at step %d: %f', step, average_loss)
                average_loss = 0

            # print a sample of similarities between heroes every 10k steps
            if step % 10000 == 0:
                sim = similarity.eval()

                for i in xrange(valid_size):
                    valid_word = reverse_dictionary[valid_examples[i]]

                    # ignore unknown and padding tokens
                    if valid_word != 'UNK' and valid_word != 'PAD':
                        valid_word = heroes_dict[int(reverse_dictionary[valid_examples[i]])]

                    top_k = 8  # number of nearest neighbors to print
                    nearest = (-sim[i, :]).argsort()[1:top_k + 1]
                    log = 'Nearest to %s:' % valid_word

                    for k in xrange(top_k):
                        index = reverse_dictionary[nearest[k]]

                        if index != 'UNK' and index != 'PAD':
                            close_word = heroes_dict[int(index)]
                        else:
                            close_word = index
                        log = '%s %s,' % (log, close_word)

                    logger.info(log)
        final_embeddings = normalized_embeddings.eval()

    return final_embeddings

def _plot_similarities(embeddings, heroes_dict, reverse_dictionary, perplexity=20):
    """ Plot the obtained hero embeddings using TSNE algorithm in 2D space.
    There are 4 assumed roles: Mid, Carry, Offlaner, Support, each category containing a
    representative hardcoded hero in order to correctly identify each cluster's role.

    Args:
        embeddings: hero embeddings obtained after training
        heroes_dict: dictionary that maps the hero's ID to its name
        reverse_dictionary: the reversed vocabulary (the key are the appearances and the values
            are the words)
        perplexity: hyperparameter of TSNE (15-30 seems to work best)
    """

    # Reduce the embeddings to 2D
    tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42)
    two_d_embeddings = tsne.fit_transform(embeddings)

    # Apply KMeans on the data in order to clusterize by role
    kmeans = KMeans(n_clusters=4, n_jobs=-1)
    kmeans.fit(tsne.embedding_)

    labels = kmeans.labels_
    labels = labels[2:]

    x_vals = two_d_embeddings[:, 0]
    y_vals = two_d_embeddings[:, 1]

    number_of_heroes = len(heroes_dict.keys())
    names = number_of_heroes * ['']
    for i in range(number_of_heroes):
        names[i] = heroes_dict[int(reverse_dictionary[i + 2])]

    x_vals = list(x_vals)
    y_vals = list(y_vals)

    # delete 'UNK' and 'PAD' when plotting
    del x_vals[1]
    del x_vals[0]
    del y_vals[1]
    del y_vals[0]

    traces = []
    for cluster in range(max(labels) + 1):
        indices = []
        for i in range(len(labels)):
            if labels[i] == cluster:
                indices.append(i)

        cluster_text = 'Mixed'
        heroes_in_cluster = [names[i] for i in indices]
        if 'Terrorblade' in heroes_in_cluster:
            cluster_text = 'Carry'
        elif 'Shadow Fiend' in heroes_in_cluster:
            cluster_text = 'Mid'
        elif 'Batrider' in heroes_in_cluster:
            cluster_text = 'Offlane'
        elif 'Dazzle' in heroes_in_cluster:
            cluster_text = 'Support'

        trace = go.Scatter(
            x=[x_vals[i] for i in indices],
            y=[y_vals[i] for i in indices],
            mode='markers+text',
            text=[names[i] for i in indices],
            name=cluster_text,
            textposition='top'
        )

        traces.append(trace)

    layout = go.Layout(
        title='Hero map test',
        xaxis=dict(
            autorange=True,
            showgrid=False,
            zeroline=False,
            showline=False,
            autotick=True,
            ticks='',
            showticklabels=False
        ),
        yaxis=dict(
            autorange=True,
            showgrid=False,
            zeroline=False,
            showline=False,
            autotick=True,
            ticks='',
            showticklabels=False
        )
    )

    data = traces
    figure = go.Figure(data=data, layout=layout)

    py.iplot(figure, filename='heromap')

def plot_hero_map(csv_path,
                  batch_size=128,
                  embedding_size=25,
                  window_size=2,
                  neg_samples=64,
                  num_steps=30001,
                  low_mmr=0,
                  high_mmr=9000):
    """ Creates a 2D plot of the heroes based on their similarity obtained with word2vec. The result
    is uploaded to plotly.

    Args:
        csv_path: path to the training dataset csv
        batch_size: size of the batch to be used in training
        embedding_size: number of dimensions when creating word embeddings
        window_size: word2vec window size hyperparameter
        neg_samples: word2vec negative samples hyperparameter
        num_steps: number of steps to train for at (at least 10k should be fine)
        low_mmr: lower bound of the MMRs filtered for plotting
        high_mmr: upper bound of the MMRs filtered for plotting
    """
    patch = get_last_patch()
    heroes_dict = get_hero_dict()
    dataset = pd.read_csv(csv_path)

    # filter the games by MMR and transform the dataset to a numpy array
    dataset = dataset[(dataset.avg_mmr > low_mmr) & (dataset.avg_mmr < high_mmr)].values

    vocabulary_size = patch['heroes_released'] + 1
    words = []

    # create corpus by separating each team and adding padding
    for match in dataset:
        radiant_list = match[2].split(',')
        dire_list = match[3].split(',')

        words.extend(radiant_list)
        words.append('PAD')
        words.append('PAD')
        words.extend(dire_list)
        words.append('PAD')
        words.append('PAD')

    # create vocabulary using the corpus
    data, count, dictionary, reverse_dictionary = _build_vocabulary(words, vocabulary_size)

    logger.info('Most common heroes (+UNK): %s', count[:5])
    logger.info('Sample data: %s', data[:10])

    # free unused memory
    del words

    final_embeddings = _train_word2vec(data,
                                       batch_size,
                                       vocabulary_size,
                                       embedding_size,
                                       neg_samples,
                                       window_size,
                                       num_steps,
                                       reverse_dictionary,
                                       heroes_dict)

    _plot_similarities(final_embeddings, heroes_dict, reverse_dictionary)


text:
"""Functions for generating and displaying maps of energy generation
and storage facilities, and predicted energy generation and storage
types"""

import os
import pandas as pd
import json
import numpy as np
import folium
import ela

# Set up color scheme.
type_colors = {'Solar': '#f6cc0a', 'Hydro': '#2B90F5', 'Nuclear': '#b9341b',
               'Biomass': '#55b64e', 'Oil': '#ff6700', 'Coal': '#fe7b89',
               'Gas': '#283b1a', 'Geothermal': '#d400ee', 'Wind': '#d9ffd8',
               'Other': '#000000',
               'Electro-mechanical': '#f6cc0a', 'Electro-chemical': '#2B90F5',
               'Pumped Hydro Storage': '#b9341b', 'Thermal Storage': '#55b64e'}

# Load state and county boundaries.
ela_dir, ela_filename = os.path.split(__file__)
data_dir = os.path.join(ela_dir, "data")
with open(os.path.join(data_dir, 'us-states.json')) as f:
    states = json.load(f)
with open(os.path.join(data_dir, 'us-counties.json')) as f:
    counties = json.load(f)

def geojson_to_df(geoj):
    """
    Read a geojson file into a pandas dataframe.

    Parameters
    ----------
    geoj : GeoJSON object (loaded from us-states.json or us-counties.json)

    Returns
    -------
    Pandas dataframe :
        containing columns for the feature ID, geometry type (Polygon or
        MultiPolygon) and geometry (coordinates)

    """
    geometry, geo_id, poly = [], [], []
    for feature in geoj['features']:
        geometry.append(feature['geometry']['coordinates'])
        geo_id.append(feature['id'])
        poly.append(feature['geometry']['type'])
    return pd.DataFrame(np.c_[geo_id, geometry, poly],
                        columns=['id', 'geometry', 'polygon'])

def geojson_centers(df):
    """
    Find the average coordinates of geojson features in a dataframe.

    Parameters
    ----------
    df : Pandas dataframe
        dataframe containing the output of geojson_to_df, with columns
        including 'polygon' and 'geometry'

    Returns
    -------
    None

    Side Effects
    ------------
    Adds a column 'centers' to the input dataframe, containing (latitude,
    longitude) tuples representing the average coordinates of all vertices
    in the GeoJSON geometry specification for each feature

    """
    centers = []
    for i in range(0, len(df)):
        row = df.iloc[i]
        poly = row.polygon

        if poly == 'MultiPolygon':
            geom = []
            for poly in row.geometry:
                geom = geom + poly[0]
        else:
            geom = row.geometry[0]

        lon, lat = list(zip(*geom))
        c_lon = (np.asarray(lon)).mean()
        c_lat = (np.asarray(lat)).mean()
        centers.append((c_lat, c_lon))
    df['centers'] = centers

def geojson_predict(df):
    """
    Predict generation and storage types for all features in a dataframe.

    Parameters
    ----------
    df : Pandas dataframe
        dataframe containing geographic features, with a column 'centers'
        which has (latitude, longitude) tuples
        This should be the output of geojson_to_df after modification by
        geojson_centers.

    Returns
    -------
    None

    Side Effects
    ------------
    Adds two columns 'pred_gen' and 'pred_stor' to the input dataframe,
    containing the predicted energy generation and storage types for each
    feature, based on ela.get_predicted_type() for the latitude and longitude
    in the 'centers' column.

    """

    gens = []
    stors = []
    for index, row in df.iterrows():
        latlon = row.centers
        gens.append(ela.get_predicted_type(latlon, 'gen'))
        stors.append(ela.get_predicted_type(latlon, 'stor'))
    df['pred_gen'] = gens
    df['pred_stor'] = stors

def pred_gen_to_colors(df):
    """
    Generate a dictionary mapping geographic feature ID to energy type color.

    Parameters
    ----------
    df : Pandas dataframe
        Dataframe containing geographic features in an 'id' column and
        predicted energy generation types in a 'pred_gen' column.

    Returns
    -------
    dictionary :
        keys : strings representing feature IDs from input dataframe
        values : strings representing colors (as hex codes) corresponding to
            the predicted energy generation type for each feature

    """
    return {df.iloc[i].id: type_colors[df.iloc[i].pred_gen] for i in df.index}

def pred_stor_to_colors(df):
    """
    Generate a dictionary mapping geographic feature ID to energy type color.

    Parameters
    ----------
    df : Pandas dataframe
        Dataframe containing geographic features in an 'id' column and
        predicted energy storage types in a 'pred_stor' column.

    Returns
    -------
    dictionary :
        keys : strings representing feature IDs from input dataframe
        values : strings representing colors (as hex codes) corresponding to
            the predicted energy storage type for each feature

    """
    return {df.iloc[i].id: type_colors[df.iloc[i].pred_stor] for i in df.index}

def prediction_map(geoj, gen_or_stor):
    """
    Create a Folium map layer with features colored by predicted energy type.

    Parameters
    ----------
    geoj : GeoJSON object
        Contains state or county boundaries

    gen_or_stor : string, either 'gen' or 'stor'
        Specify whether to map predicted generation or storage types.

    Returns
    -------
    Folium GeoJson layer
        Map layer showing the input geographic features colored according to
        their predicted energy types

    """

    df = geojson_to_df(geoj)
    geojson_centers(df)
    geojson_predict(df)

    if gen_or_stor == 'gen':
        map_colors = pred_gen_to_colors(df)
    elif gen_or_stor == 'stor':
        map_colors = pred_stor_to_colors(df)
    else:
        raise ValueError("Enter either 'gen' or 'stor'.")

    return folium.GeoJson(geoj,
                          style_function=lambda feature: {
                              'fillColor': map_colors[feature['id']],
                              'fillOpacity': 0.4, 'weight': 0})

def facility_map(state, gen_or_stor):
    """
    Plot generation facilities in one state on a map, colored by energy type.

    Parameters
    ----------
    state : str
        Two-letter abbreviation for the state to be visualized.
        If None, all facilities are visualized (this can be slow).
    gen_or_stor : string, either 'gen' or 'stor'
        Specify whether to map generation or storage facilities.

    Returns
    -------
    Folium map layer :
        FeatureGroup containing a marker for each facility.

    """

    if gen_or_stor == 'gen':
        state_df = ela.gen_data
    elif gen_or_stor == 'stor':
        state_df = ela.stor_data
    else:
        raise ValueError("Enter either 'gen' or 'stor'.")

    if state is not None:
        state_df = state_df[state_df.state == state]

    f = folium.map.FeatureGroup()
    if gen_or_stor == 'gen':
        for index, row in state_df.iterrows():
            tp = row.type
            f.add_child(folium.CircleMarker([row.lat, row.lon],
                                            radius=5, weight=1,
                                            fill_color=type_colors[tp],
                                            fill_opacity=1))
    elif gen_or_stor == 'stor':
        for index, row in state_df.iterrows():
            tp = row.type
            f.add_child(folium.RegularPolygonMarker([row.lat, row.lon],
                                                    number_of_sides=3,
                                                    radius=8,
                                                    rotation=30, weight=1,
                                                    fill_color=type_colors[tp],
                                                    fill_opacity=1))
    return f

def state_map(state):
    """
    Generate blank folium map centered at the input state.

    Parameters
    ----------
    state: string
        Two-letter abbreviation for the state to be visualized.

    Returns
    -------
    Folium map object

    """

    state_lat = ela.zip_data[ela.zip_data.state == state].lat.mean()
    state_lon = ela.zip_data[ela.zip_data.state == state].lon.mean()
    return folium.Map(location=[state_lat, state_lon], zoom_start=6)

max_stars_repo_name: DIRECT-Energy-Storage/ela

max_issues_repo_name: DIRECT-Energy-Storage/ela

max_forks_repo_name: DIRECT-Energy-Storage/ela


text:
import json
from mpmath import *
from probabilistic import *

with open(snakemake.input['likelihoods'],'r') as likelihoods, open(snakemake.input['prior'],'r') as priorfile:
    likelihoods = json.load(likelihoods)
    prior = mpf(priorfile.read())
    probabilities = {}

    for spaType in likelihoods:
        if likelihoods[spaType] == None: #or likelihoods[spaType] < snakemake.config['likelihoodCutoff']:
            probabilities[spaType] = None
        else:
            probabilities[spaType] = likelihoods[spaType]-prior

    sorted_probs = sorted(
        [(k,v) for k,v in probabilities.items()],
        key = lambda x : -float('inf') if x[1] is None else x[1],
        reverse = True
    )

    with open(snakemake.output['probabilities'],'w') as outfile:
        for p in sorted_probs:
            outfile.write(str(p[0])+'\t'+nstr(p[1],20)+'\n')

max_stars_repo_name: phspo/snakemake-ngs-spa-typing

max_issues_repo_name: phspo/snakemake-ngs-spa-typing

max_forks_repo_name: phspo/snakemake-ngs-spa-typing

reason: from mpmath


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""segnet package file tf_record

This module provides two main functions that are create_tfrecord and
compute_mean. Given a list of data generator object with the attributes
of 'give_length' and 'next'.

"""

import tensorflow as tf
import numpy as np

def _bytes_feature(value):
    """
    Serieliaze the data to bytes.
    """
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _int64_feature(value):
    """
    Serieliaze the data to int64.
    """
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def create_tfrecord(outname, dg_list):
    """
    Takes a list of data generator object  who has two methods: next and length
    and creates an associated TFRecord file.
    """
    writer = tf.python_io.TFRecordWriter(outname)
    for datagen in dg_list:
        n_iter_max = datagen.give_length()
        for _ in range(n_iter_max):
            img, annotation = datagen.next()
            annotation = annotation.astype(np.uint8)
            height = img.shape[0]
            width = img.shape[1]
            height_a = annotation.shape[0]
            width_a = annotation.shape[1]
            img_raw = img.tostring()
            annotation_raw = annotation.tostring()
            example = tf.train.Example(features=tf.train.Features(feature={
                'height_img': _int64_feature(height),
                'width_img': _int64_feature(width),
                'height_mask': _int64_feature(height_a),
                'width_mask': _int64_feature(width_a),
                'image_raw': _bytes_feature(img_raw),
                'mask_raw': _bytes_feature(annotation_raw)}))

            writer.write(example.SerializeToString())
    writer.close()

def compute_mean(outname, dg_list):
    """
    Takes a list of data generator object  who has two methods: next and length
    and creates an associated mean_file for RGB images.
    """
    mean_per_image = []
    for datagen in dg_list:
        n_iter_max = datagen.give_length()
        for _ in range(n_iter_max):
            img, _ = datagen.next()
            mean_per_image.append(np.mean(img, axis=(0, 1)))
    mean_array = np.mean(mean_per_image, axis=0)
    np.save(outname, mean_array)
    return mean_array


text:
import os.path
import torch
import torchvision.transforms as transforms

from PIL import Image
import numpy as np
import csv
#from  import getfeats_dlib_fromImg
from APDrawingGAN.data.face_landmark import getfeats_dlib_fromImg
from APDrawingGAN.data.base_dataset import BaseDataset, get_transform
from APDrawingGAN.data.image_folder import make_dataset
def getfeats(featpath):
    #print('getfeats0')
    trans_points = np.empty([5,2],dtype=np.int64)
    with open(featpath, 'r') as csvfile:
        reader = csv.reader(csvfile, delimiter=' ')
        for ind,row in enumerate(reader):
            trans_points[ind,:] = row
    return trans_points

def getSoft(size,xb,yb,boundwidth=5.0):
    xarray = np.tile(np.arange(0,size[1]),(size[0],1))
    yarray = np.tile(np.arange(0,size[0]),(size[1],1)).transpose()
    cxdists = []
    cydists = []
    for i in range(len(xb)):
        xba = np.tile(xb[i],(size[1],1)).transpose()
        yba = np.tile(yb[i],(size[0],1))
        cxdists.append(np.abs(xarray-xba))
        cydists.append(np.abs(yarray-yba))
    xdist = np.minimum.reduce(cxdists)
    ydist = np.minimum.reduce(cydists)
    manhdist = np.minimum.reduce([xdist,ydist])
    im = (manhdist+1) / (boundwidth+1) * 1.0
    im[im>=1.0] = 1.0
    return im

class SingleDataset(BaseDataset):
    @staticmethod
    def modify_commandline_options(parser, is_train):
        return parser

    def initialize(self, opt, img_background=None):
        self.opt = opt
        self.root = opt.dataroot
        self.dir_A = os.path.join(opt.dataroot)

        self.A_paths = make_dataset(self.dir_A)

        self.A_paths = sorted(self.A_paths)

        self.transform = get_transform(opt)
        self.A_path = opt.im_p
        self.img_background = opt.img_background
    def __getitem__(self, index):
        A_path = self.A_path#s[index]
        A_img = Image.open(A_path).convert('RGB')
        A = self.transform(A_img)
        #print(self.transform)
        #print('A0=',A.shape)
        if self.opt.which_direction == 'BtoA':
            input_nc = self.opt.output_nc
            output_nc = self.opt.input_nc
        else:
            input_nc = self.opt.input_nc
            output_nc = self.opt.output_nc

        if input_nc == 1:  # RGB to gray
            tmp = A[0, ...] * 0.299 + A[1, ...] * 0.587 + A[2, ...] * 0.114
            A = tmp.unsqueeze(0)

        item = {'A': A, 'A_paths': A_path}
        #print('A1=',A.shape)
        if self.opt.use_local:
            regions = ['eyel','eyer','nose','mouth']
            basen = os.path.basename(A_path)[:-4]+'.txt'
            featdir = self.opt.lm_dir
            featpath = os.path.join(featdir,basen)
            #feats = getfeats(featpath)
            feats = getfeats_dlib_fromImg(A_path)
            mouth_x = int((feats[3,0]+feats[4,0])/2.0)
            mouth_y = int((feats[3,1]+feats[4,1])/2.0)
            ratio = self.opt.fineSize / 256
            EYE_H = self.opt.EYE_H * ratio
            EYE_W = self.opt.EYE_W * ratio
            NOSE_H = self.opt.NOSE_H * ratio
            NOSE_W = self.opt.NOSE_W * ratio
            MOUTH_H = self.opt.MOUTH_H * ratio
            MOUTH_W = self.opt.MOUTH_W * ratio
            center = torch.tensor([[feats[0,0],feats[0,1]-4*ratio],[feats[1,0],feats[1,1]-4*ratio],[feats[2,0],feats[2,1]-NOSE_H/2+16*ratio],[mouth_x,mouth_y]])
            item['center'] = center
            rhs = [EYE_H,EYE_H,NOSE_H,MOUTH_H]
            rws = [EYE_W,EYE_W,NOSE_W,MOUTH_W]
            if self.opt.soft_border:
                soft_border_mask4 = []
                for i in range(4):
                    xb = [np.zeros(rhs[i]),np.ones(rhs[i])*(rws[i]-1)]
                    yb = [np.zeros(rws[i]),np.ones(rws[i])*(rhs[i]-1)]
                    soft_border_mask = getSoft([rhs[i],rws[i]],xb,yb)
                    soft_border_mask4.append(torch.Tensor(soft_border_mask).unsqueeze(0))
                    item['soft_'+regions[i]+'_mask'] = soft_border_mask4[i]
            for i in range(4):
                item[regions[i]+'_A'] = A[:,int(center[i,1]-rhs[i]/2):int(center[i,1]+rhs[i]/2),int(center[i,0]-rws[i]/2):int(center[i,0]+rws[i]/2)]
                if self.opt.soft_border:
                    item[regions[i]+'_A'] = item[regions[i]+'_A'] * soft_border_mask4[i].repeat(input_nc/output_nc,1,1)

            mask = torch.ones([output_nc,A.shape[1],A.shape[2]]) # mask out eyes, nose, mouth
            for i in range(4):
                mask[:,int(center[i,1]-rhs[i]/2):int(center[i,1]+rhs[i]/2),int(center[i,0]-rws[i]/2):int(center[i,0]+rws[i]/2)] = 0
            if self.opt.soft_border:
                imgsize = self.opt.fineSize
                maskn = mask[0].numpy()
                masks = [np.ones([imgsize,imgsize]),np.ones([imgsize,imgsize]),np.ones([imgsize,imgsize]),np.ones([imgsize,imgsize])]
                masks[0][1:] = maskn[:-1]
                masks[1][:-1] = maskn[1:]
                masks[2][:,1:] = maskn[:,:-1]
                masks[3][:,:-1] = maskn[:,1:]
                masks2 = [maskn-e for e in masks]
                bound = np.minimum.reduce(masks2)
                bound = -bound
                xb = []
                yb = []
                for i in range(4):
                    xbi = [center[i,0]-rws[i]/2, center[i,0]+rws[i]/2-1]
                    ybi = [center[i,1]-rhs[i]/2, center[i,1]+rhs[i]/2-1]
                    for j in range(2):
                        maskx = bound[:,xbi[j]]
                        masky = bound[ybi[j],:]
                        xb += [(1-maskx)*10000 + maskx*xbi[j]]
                        yb += [(1-masky)*10000 + masky*ybi[j]]
                soft = 1-getSoft([imgsize,imgsize],xb,yb)
                soft = torch.Tensor(soft).unsqueeze(0)
                mask = (torch.ones(mask.shape)-mask)*soft + mask

            bgdir = self.opt.bg_dir
            bgpath = os.path.join(bgdir,basen[:-4]+'.png')
            #im_bg = Image.open(bgpath)
            im_bg=self.img_background
            #tensor = torch.from_numpy(ndarray)
            mask2 = transforms.ToTensor()(im_bg) # mask out background

            mask2 = (mask2 >= 0.5).float()
            # hair_A = (A/2+0.5) * mask.repeat(input_nc/output_nc,1,1) * mask2.repeat(input_nc/output_nc,1,1) * 2 - 1
            # bg_A = (A/2+0.5) * (torch.ones(mask2.shape)-mask2).repeat(input_nc/output_nc,1,1) * 2 - 1
            hair_A = (A/2+0.5) * mask.repeat(3,1,1) * mask2.repeat(3,1,1) * 2 - 1
            bg_A = (A/2+0.5) * (torch.ones(mask2.shape)-mask2).repeat(3,1,1) * 2 - 1
            item['hair_A'] = hair_A
            item['bg_A'] = bg_A
            item['mask'] = mask
            item['mask2'] = mask2

        return item

    def __len__(self):
        return len(self.A_paths)

    def name(self):
        return 'SingleImageDataset'

max_stars_repo_name: shasph/dualFace

max_issues_repo_name: shasph/dualFace

max_forks_repo_name: shasph/dualFace


text:
from vispy import gloo, io
import rocket
import rocket.aux as aux
import icotree
import numpy as np
from uuid import uuid4

def main():
    global planet, camera, program, icosphere
    planet = aux.Mover()
    camera = aux.View(fov=45)
    program = aux.load_shaders('shaders/vertex.glsl', 'shaders/fragment.glsl')
    icosphere = icotree.Icosphere(0)

    rocket.prep(size=(512, 512))
    camera.move(z=(-6))
    planet.vel = aux.Velocity()
    planet.verts, planet.index, planet.color = icosphere.buffers()
    rocket.launch()

@rocket.attach
def update():
    """Update the game. Called for every frame, usually sixty per second."""
    planet.rotate(*tuple(planet.vel))
    planet.vel.damp()

@rocket.attach
def draw():
    program['a_position'] = planet.verts
    program['a_coloring'] = planet.color
    program['m_model'] = planet.transform
    program['m_view'] = camera.transform
    program['m_proj'] = camera.proj
    #program['u_color'] = (0.1, 0.2, 0.3)
    program.draw('triangles', gloo.IndexBuffer(planet.index))

@rocket.attach
def key_press(key):
    global planet
    if key == 'W':
        planet.vel.accel(y=1)
    elif key == 'A':
        planet.vel.accel(x=-1)
    elif key == 'S':
        planet.vel.accel(y=-1)
    elif key == 'D':
        planet.vel.accel(x=+1)
    elif key == 'P':
        path = '/home/john/media/pictures/'+str(uuid4())+'.png'
        print("Saving screenshot to '{}'...".format(path))
        rocket.screenshot(path)

@rocket.attach
def left_drag(start, end, delta):
    planet.vel.accel(x=delta[0]/5, y=delta[1]/5)

@rocket.attach
def right_drag(start, end, delta):
    planet.rotate(z=(0-delta[1])/2)

@rocket.attach
def middle_drag(start, end, delta):
    planet.rotate(x=delta[0], y=delta[1])
    planet.vel.decel(100, 100, 100)

@rocket.attach
def scroll(point, direction):
    camera.move(0, 0, direction/10)

if __name__ == '__main__':
    main()

max_stars_repo_name: Anavros/isosphere

max_issues_repo_name: Anavros/isosphere

max_forks_repo_name: Anavros/isosphere


text:
import pandas as pd
import networkx as nx
import collections
import numpy as np
from igraph import *

def get_most_detailed_categories(categories, dict_alias_title,
                                 dict_alias_depth):
    max_height = 0
    for category in categories:
        max_height = max(dict_alias_depth[category], max_height)
    new_categories = list()
    for category in categories:
        height = dict_alias_depth[category]
        if (height == max_height):
            new_categories.append(category)
    return new_categories

def cat_structs_igraph(catfilename):
    df_categories = pd.read_json(catfilename)

    dict_alias_title = dict()
    for index, row in df_categories.iterrows():
        dict_alias_title[row['alias']] = row['title']
        dict_alias_title[row['title']] = row['alias']

    category_tree = Graph(directed=False)
    category_tree.add_vertices(list(df_categories['alias']))
    category_tree.add_vertex('root')

    for index, row in df_categories.iterrows():
        if not row['parents']:
            category_tree.add_edge(row['alias'],
                                   'root')  # root node if no parents
        else:
            for parent_label in row['parents']:
                category_tree.add_edge(row['alias'], parent_label)

    dict_alias_depth = dict()
    for index, row in df_categories.iterrows():
        dict_alias_depth[row['alias']] = category_tree.shortest_paths(
            row['alias'], 'root')[0][0]

    df_categories = None

    return dict_alias_title, category_tree, dict_alias_depth

def cat_structs(catfilename):
    df_categories = pd.read_json(catfilename)  #"../data/categories.json"

    # dicionário alias title 2 way

    dict_alias_title = dict()
    for index, row in df_categories.iterrows():
        dict_alias_title[row['alias']] = row['title']
        dict_alias_title[row['title']] = row['alias']

    category_tree = nx.DiGraph()
    for index, row in df_categories.iterrows():
        if not row['parents']:
            category_tree.add_edge(row['alias'],
                                   'root')  # root node if no parents
        else:
            for parent_label in row['parents']:
                category_tree.add_edge(row['alias'], parent_label)

    undirected_category_tree = category_tree.to_undirected()
    # dict alias depth
    dict_alias_depth = dict()
    for index, row in df_categories.iterrows():
        dict_alias_depth[row['alias']] = nx.shortest_path_length(
            category_tree, row['alias'], 'root')

    df_categories = None

    return dict_alias_title, category_tree, dict_alias_depth

def get_users_cat_visits(training_matrix, poi_cats):
    users_cv = []
    for i in range(training_matrix.shape[0]):
        cats_visits = collections.defaultdict(int)
        lids = training_matrix[i].nonzero()[0]
        for lid in lids:
            for cat in poi_cats[lid]:
                cats_visits[cat] += training_matrix[i, lid]
        cats_visits = dict(cats_visits)
        users_cv.append(cats_visits)
    return users_cv

reason: import numpy,import networkx