text:
import os

import numpy as np
from dtaidistance import dtw
from cobras_ts.cobras_dtw import COBRAS_DTW
from cobras_ts.querier.labelquerier import LabelQuerier
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import DBA as dba
from matplotlib.patches import Ellipse
import datetime as dt
from queue import Queue
import random

ucr_path = '/Users/ary/Desktop/Thesis/UCR_TS_Archive_2015'
dataset = 'ECG200'
#dataset = 'Beef'
budget = 50
alpha = 0.5
window = 1
iteration = 1
cluster_idx = 0

# load the data
data = np.loadtxt(os.path.join(ucr_path,dataset,dataset + '_TEST'), delimiter=',')
data_train = np.loadtxt(os.path.join(ucr_path,dataset,dataset + '_TRAIN'), delimiter=',')
data = np.vstack((data,data_train))
series = data[:,1:][:20]
labels = data[:,0]

start = 0
end = series.shape[1]

print("series cnt = " + str(series.shape[0]))
print("series shape = " + str(series.shape[1]))

# construct the affinity matrix
dt1 = dt.datetime.now()
dists = dtw.distance_matrix(series, window=int(0.01 * window * series.shape[1]))
dt2 = dt.datetime.now()
print("distance matrix use time " + str((dt2 - dt1).seconds))
dists[dists == np.inf] = 0
dists = dists + dists.T - np.diag(np.diag(dists))
affinities = np.exp(-dists * alpha)

# initialise cobras_dtw with the precomputed affinities
clusterer = COBRAS_DTW(affinities, LabelQuerier(labels), budget)

clustering, intermediate_clusterings, runtimes, ml, cl = clusterer.cluster()
dt3 = dt.datetime.now()
print("cluster use time " + str((dt3 - dt2).seconds))

print("there are " + str(len(clustering.clusters)) + " clusters")

x = np.arange(0, series.shape[1])

def initFigure(fig):
    ax1 = fig.add_subplot(231)  # DBA
    ax2 = fig.add_subplot(232, sharex=ax1, sharey=ax1)  # adjusted dots
    ax3 = fig.add_subplot(233, sharex=ax1, sharey=ax1)  # eclipse
    ax4 = fig.add_subplot(234, sharex=ax1)  # dtw_vertical
    ax5 = fig.add_subplot(235, sharex=ax1, sharey=ax4)  # vertical
    ax6 = fig.add_subplot(236, sharex=ax1, sharey=ax4)  # dtw_horizontal

    ax_overall = [ax1, ax2, ax3]
    ax_statistic = [ax4, ax5, ax6]
    axs = ax_overall + ax_statistic

    # axis color
    for ax in axs:
        ax.spines['bottom'].set_color('blue')
        ax.spines['bottom'].set_linewidth(2)
        ax.spines['left'].set_linewidth(2)
        ax.set_xlabel("time")

    for ax in ax_overall:
        ax.spines['left'].set_color('pink')
    for ax in ax_statistic:
        ax.spines['left'].set_color('yellow')

    x = np.arange(0, series.shape[1])

    # y_label
    ax1.set_ylabel("DBA", color="purple")
    ax2.set_ylabel("Instances", color="blue")
    ax3.set_ylabel("DBA with Warping Deviations", color="blue")
    ax6.set_ylabel("Horizontal Warping Deviation", color="green")
    ax4.set_ylabel("Vertical Warping Deviation", color="red")
    ax5.set_ylabel("Standard Deviation", color="gray")
    return ax1, ax2, ax3, ax4, ax5, ax6

def findAllIndicesOfOneCluster(cluster_idx):
    superinstances = clustering.clusters[cluster_idx].super_instances
    indices_of_current_cluster = []
    for superinstance in superinstances:
        indices_of_current_cluster += superinstance.indices
    return indices_of_current_cluster

def findAllSeriesWithIndices(indices):
    target_series = list(map(lambda idx: series[idx], indices))
    return target_series

def getColorIter(size):
    color_arr = cm.rainbow(np.linspace(0, 1, size))
    colors = iter(color_arr)
    return colors

def plotDBA(plt, series_mean, range = None):
    if (range == None):
        plt.plot(x, series_mean, color='purple')
    elif (range[0] + 1 == range[1]):
        plt.scatter(x[range[0]], series_mean[range[0]], color = 'purple')
    else:
        plt.plot(x[range[0]: range[1]], series_mean[range[0]: range[1]], color = 'purple')

def plotEclipseAroundDBA(ax1, series_mean, series_dtw_horiz_var, series_dtw_vertic_var, span_tuple):
    color_type_num = 10
    color_arr = cm.rainbow(np.linspace(0, 1, color_type_num))
    ells = [Ellipse(xy=(i, series_mean[i]),
                    width=series_dtw_horiz_var[i], height=series_dtw_vertic_var[i], color=color_arr[i%color_type_num])
            for i in np.arange(span_tuple[0], span_tuple[1])
            ]
    for e in ells:
        ax1.add_artist(e)

def plotDeviations(plt, values, color, range = None) :
    if (range == None):
        plt.plot(x, values, color=color)
    elif (range[0] + 1 == range[1]):
        plt.scatter(x[range[0]], values[range[0]], color=color)
    else:
        plt.plot(x[range[0]: range[1]], values[range[0]: range[1]], color=color)

    # adjusted_series_weight_mat 往前映射的距离
    # series_mapping_mat         当前映射的index
    # 注意range是[ a, b )
def plotAdjustedSeries(plt, cluster_id ,series_mapping_mat, adjusted_series_weight_mat, range, series):
    plt.set_title(str(len(series)) + " instances of cluster " + str(cluster_id))
    colors = getColorIter(series_mapping_mat.shape[0])
    for series_index in np.arange(0, series_mapping_mat.shape[0]):
        s1, e1 = dba.getStartAndEndMapping(series_mapping_mat, adjusted_series_weight_mat, series_index, range[1] - 1)
        s2, e2 = dba.getStartAndEndMapping(series_mapping_mat, adjusted_series_weight_mat, series_index, range[0])

        x = np.arange(s2, e1)
        y = list(map(lambda  x: series[series_index][x], x))
        if len(x) == 1:
            plt.scatter(x, y, color=next(colors))
        else:
            plt.plot(x, y, color=next(colors))
        expected_range = np.arange(range[0], range[1])
        non_plotted_range = [item for item in expected_range if item not in x]
        for non_plotted_dot in non_plotted_range:
            plt.scatter(non_plotted_dot, series[series_index][non_plotted_dot], color = 'black')

def plotSelectedSpan(span_tuple, cluster_idx, ax1, ax2, ax3, ax4, ax5, ax6):
    cur_series = findAllSeriesWithIndices(findAllIndicesOfOneCluster(cluster_idx))
    dt4 = dt.datetime.now()

    series_mean, series_dtw_horiz_var, series_dtw_vertic_var, series_vertic_var, adjusted_series_mat, series_mapping_mat, adjusted_series_weight_mat, series_dtw_special_vertic_var = dba.performDBA(cur_series, iteration)
    dt5 = dt.datetime.now()
    print("getting DBA use time " + str((dt5 - dt4).seconds))

    plotDBA(ax1, series_mean, span_tuple)
    #plotRawAverage(ax1, cur_series)
    plotAdjustedSeries(ax2,  cluster_idx, series_mapping_mat, adjusted_series_weight_mat, span_tuple, cur_series)
    plotStatisticsCurves(ax4, ax5, ax6, series_dtw_horiz_var, series_dtw_vertic_var, series_vertic_var, series_dtw_special_vertic_var, span_tuple)
    plotDBA(ax3, series_mean, span_tuple)
    plotEclipseAroundDBA(ax3, series_mean, series_dtw_horiz_var, series_dtw_vertic_var, span_tuple)

def plotStatisticsCurves(ax4, ax5, ax6, series_dtw_horiz_var, series_dtw_vertic_var, series_vertic_var, series_dtw_special_vertic_var, span_tuple = None):
    plotDeviations(ax4, series_dtw_vertic_var, 'red', span_tuple)
    plotDeviations(ax5, series_vertic_var, 'grey', span_tuple)
    plotDeviations(ax6, series_dtw_horiz_var, 'green', span_tuple)
    # plotStatisticsCurve(ax6, series_dtw_special_vertic_var, 'purple', span_tuple)

def plott(i):
    fig = plt.figure(i)
    ax1, ax2, ax3, ax4, ax5, ax6 = initFigure(fig)
    plotSelectedSpan((start, end), i-1, ax1, ax2, ax3, ax4, ax5, ax6)

##for plotting the links
def listToDic(ml):
    dict = {}
    for cml in ml:
        if cml[0] in dict:
            dict[cml[0]].append(cml[1])
        else:
            dict[cml[0]] = [cml[1]]
        if cml[1] in dict:
            dict[cml[1]].append(cml[0])
        else:
            dict[cml[1]] = [cml[0]]
    return dict

ml_dict = listToDic(ml)
cl_dict = listToDic(cl)

def getClusterAndRepresentativeIds(clustering, firstId, secondId):
    found_first_sp = False
    found_second_sp = False
    representative_idx_first = 0
    representative_idx_second = 0
    cluster_idx_first = 0
    cluster_idx_second = 0

    for cluster_idx in range(0, len(clustering.clusters)):
        cluster = clustering.clusters[cluster_idx]
        for sp in cluster.super_instances:
            if found_first_sp == True and found_second_sp == True:
                break
            for idx in sp.indices:
                if found_first_sp == False and firstId == idx:
                    cluster_idx_first = cluster_idx
                    representative_idx_first = sp.representative_idx
                    found_first_sp = True
                if found_second_sp == False and secondId == idx:
                    cluster_idx_second = cluster_idx
                    representative_idx_second = sp.representative_idx
                    found_second_sp = True
    # assert (found_first_sp == True and found_second_sp == True)
    return cluster_idx_first, cluster_idx_second, representative_idx_first, representative_idx_second

# def getMustLinks(clustering, first_idx, second_idx):
#
#     # cluster_idx_left, cluster_idx_right = getClusteringIdxs(clustering, first_repre_idx, second_repre_idx)
#     cluster_idx_left, cluster_idx_right, repre_idx_left, repre_idx_right = getClusterAndRepresentativeIds(clustering, first_idx, second_idx)
#     assert (cluster_idx_left == cluster_idx_right)
#     must_links_between_first_and_second = findMustLinksBetweenSuperintances(ml_dict, repre_idx_left, repre_idx_right)
#     links = must_links_between_first_and_second
#     if len(links) >= 2:
#         if links[0] != first_idx:
#             links = [first_idx] + links
#         if links[len(links) - 1] != second_idx:
#             links = links + [second_idx]
#
#     return cluster_idx_left, links

def getMustLinks(first_idx, second_idx, repre_idx_left, repre_idx_right):
    must_links_between_first_and_second = findMustLinksBetweenSuperintances(ml_dict, repre_idx_left, repre_idx_right)
    links = must_links_between_first_and_second
    if len(links) >= 2:
        if links[0] != first_idx:
            links = [first_idx] + links
        if links[len(links) - 1] != second_idx:
            links = links + [second_idx]

    return links

def getCannotLinks(cluster_idx_left, cluster_idx_right, first_repre_idx, second_repre_idx, mock_1, mock_3):
    # cluster_idx_left, cluster_idx_right = getClusteringIdxs(clustering, first_repre_idx, second_repre_idx)
    # assert (cluster_idx_left != cluster_idx_right)
    cannot_link_between_cluster = getCannotLinkBetweenCluster(cluster_idx_left, cluster_idx_right)
    must_links_between_first_and_cluster = findMustLinksBetweenSuperintances(ml_dict, first_repre_idx, cannot_link_between_cluster[0])
    if len(must_links_between_first_and_cluster) > 0 and must_links_between_first_and_cluster[0] != mock_1:
        must_links_between_first_and_cluster = [mock_1] + must_links_between_first_and_cluster

    must_links_between_second_and_cluster = findMustLinksBetweenSuperintances(ml_dict, cannot_link_between_cluster[1], second_repre_idx)
    if len(must_links_between_second_and_cluster) > 0 and must_links_between_second_and_cluster[len(must_links_between_second_and_cluster) - 1] != mock_3:
        must_links_between_second_and_cluster = must_links_between_second_and_cluster + [mock_3]

    return must_links_between_first_and_cluster, must_links_between_second_and_cluster

def getCannotLinkBetweenCluster(cluster_idx_left, cluster_idx_right):
    right_cluster_sps_indices = []
    for sp in clustering.clusters[cluster_idx_right].super_instances:
        right_cluster_sps_indices.append(sp.representative_idx)

    for sp in clustering.clusters[cluster_idx_left].super_instances:
        sp_repre_idx = sp.representative_idx
        if sp_repre_idx in cl_dict:
            for second in cl_dict[sp_repre_idx]:
                if second in right_cluster_sps_indices:
                    return [sp_repre_idx, second]
    return []
def getClusteringIdxs(clustering, first_repre_idx, second_repre_idx):
    found_first_sp = False
    found_second_sp = False
    cluster_idx_first = 0
    cluster_idx_second = 0
    for cluster_idx in range(0, len(clustering.clusters)):
        cluster = clustering.clusters[cluster_idx]
        for sp in cluster.super_instances:
            if found_first_sp == False and sp.representative_idx == first_repre_idx:
                cluster_idx_first = cluster_idx
                found_first_sp = True
            if found_second_sp == False and sp.representative_idx == second_repre_idx:
                cluster_idx_second = cluster_idx
                found_second_sp = True
            if found_first_sp == True and found_second_sp == True:
                break

    # assert (found_first_sp == True and found_second_sp == True)
    return cluster_idx_first, cluster_idx_second

def checkLoop(path):
    dict = {}
    for e in path:
        if (e in dict):
            return False
        else:
            dict[e] = 1
    return True

def findMustLinksBetweenSuperintances(dict, first_repre_idx, second_repre_idx):
    visited_vertex = []
    visited_path = Queue()
    visited_path.put([first_repre_idx])
    visited_vertex.append(first_repre_idx)

    while (not visited_path.empty()):
        current_path = visited_path.get()
        last_of_path = current_path[len(current_path)-1]

        if last_of_path in dict:
            for next_elem in dict[last_of_path]:
                if (not next_elem in visited_vertex):
                    to_be_added_path = current_path + [next_elem]
                    if (next_elem == second_repre_idx):
                        return to_be_added_path
                    else:
                        visited_vertex.append(next_elem)
                        if checkLoop(to_be_added_path):
                            visited_path.put(to_be_added_path)

    return []

print("the number of clusters is "+ str(len(clustering.clusters)))

def getClusterSpsDict():
    cluster_sps_dict = {}
    for i in range(0, len(clustering.clusters)):
        size_of_sps_in_current_cluster = len(clustering.clusters[i].super_instances)
        cluster_sps_dict[i] = size_of_sps_in_current_cluster
        print(str(i) + " : " + str(size_of_sps_in_current_cluster))
    return cluster_sps_dict

cluster_sps_dict = getClusterSpsDict()

def getRandomMLPair():
    ml_pairs = []
    for cluster_idx in cluster_sps_dict:
        first = random.randint(0, cluster_sps_dict[cluster_idx] - 1)
        second = random.randint(0, cluster_sps_dict[cluster_idx] - 1)
        while (first == second):
            first = random.randint(0, cluster_sps_dict[cluster_idx] - 1)
            second = random.randint(0, cluster_sps_dict[cluster_idx] - 1)
        ml_pairs.append([[cluster_idx, first], [cluster_idx, second]])
    return ml_pairs

def getRandomCLPair():
    cl_pairs = []
    cnt = 5
    for i in range(0, len(cluster_sps_dict)):
        for j in range(i+1, len(cluster_sps_dict)):
            if cnt < 0:
                break
            else:
                cl_pairs.append([[i, 1], [j, 1]])

    return cl_pairs

def getYlabel(instanceid, clusterid):
    return "instance id: " + str(instanceid) + "                    \ncluster    id: " + str(clusterid) + "                     "

def plotMustLink(cluster_id, mock_1, mock_2,  repre_idx_left, repre_idx_right):
    must_link = getMustLinks(mock_1, mock_2, repre_idx_left, repre_idx_right)
    if (len(must_link) == 0):
        print("must 1: " + str(mock_1) + " must 2: " + str(mock_2))
    else:
        fig, axs = plt.subplots(len(must_link), sharex=True, sharey=True)

        fig.suptitle('must link')
        for plot_idx in range(0, len(must_link)):
            axs[plot_idx].plot(x, series[must_link[plot_idx]])
            axs[plot_idx].set_ylabel(getYlabel(must_link[plot_idx], cluster_id), color="blue", rotation=0)

        plt.show()

def plotCannotLink(cluster_id_1, cluster_id_2, repre_idx_left, repre_idx_right, mock_1, mock_3):
    cannot_link_1, cannot_link_2 = getCannotLinks(cluster_id_1, cluster_id_2, repre_idx_left, repre_idx_right, mock_1, mock_3)
    if (len(cannot_link_1) == 0 or len(cannot_link_2) == 0) :
        print("can not 1: " + str(mock_1) + "can not 2: " + str(mock_3))
    else:
        fig2, axs2 = plt.subplots(len(cannot_link_1) + len(cannot_link_2), sharex=True, sharey=True)
        fig2.suptitle('can not link')

        for plot_idx in range(0, len(cannot_link_1)):
            axs2[plot_idx].plot(x, series[cannot_link_1[plot_idx]], color = 'pink')
            axs2[plot_idx].set_ylabel(getYlabel(cannot_link_1[plot_idx], cluster_id_1), color = "pink", rotation=0)
            # axs2[plot_idx].tick_params(axis='y', labelrotation=90)
        for plot_idx in range(0, len(cannot_link_2)):
            axs2[len(cannot_link_1) +  plot_idx].plot(series[cannot_link_2[plot_idx]], color = 'purple')
            axs2[len(cannot_link_1) +  plot_idx].set_ylabel(getYlabel(cannot_link_2[plot_idx], cluster_id_2), color="purple", rotation=0)
        plt.show()

def getMock(cluster_idx, super_instance_idx):
    return clustering.clusters[cluster_idx].super_instances[super_instance_idx].representative_idx

def findAllSeriesWithIndices(indices):
    target_series = list(map(lambda idx: series[idx], indices))
    return target_series

def getColorIter(size):
    color_arr = cm.rainbow(np.linspace(0, 1, size))
    colors = iter(color_arr)
    return colors

def plotDBA(plt, series_mean, range = None):
    if (range == None):
        plt.plot(x, series_mean, color='purple')
    elif (range[0] + 1 == range[1]):
        plt.scatter(x[range[0]], series_mean[range[0]], color = 'purple')
    else:
        plt.plot(x[range[0]: range[1]], series_mean[range[0]: range[1]], color = 'purple')

def plotEclipseAroundDBA(ax1, series_mean, series_dtw_horiz_var, series_dtw_vertic_var, span_tuple):
    color_type_num = 10
    color_arr = cm.rainbow(np.linspace(0, 1, color_type_num))
    ells = [Ellipse(xy=(i, series_mean[i]),
                    width=series_dtw_horiz_var[i], height=series_dtw_vertic_var[i], color=color_arr[i%color_type_num])
            for i in np.arange(span_tuple[0], span_tuple[1])
            ]
    for e in ells:
        ax1.add_artist(e)

def plotDeviations(plt, values, color, range = None) :
    if (range == None):
        plt.plot(x, values, color=color)
    elif (range[0] + 1 == range[1]):
        plt.scatter(x[range[0]], values[range[0]], color=color)
    else:
        plt.plot(x[range[0]: range[1]], values[range[0]: range[1]], color=color)

def plotStatic():
    mock_1 = clustering.clusters[0].super_instances[1].representative_idx
    mock_2 = clustering.clusters[0].super_instances[2].representative_idx
    mock_3 = clustering.clusters[1].super_instances[1].representative_idx

    plotMustLink(mock_1, mock_2)
    plotCannotLink(mock_1, mock_3)

def plotRandom():
    mock_ml_pairs = getRandomMLPair()
    mock_cl_paris = getRandomCLPair()

    for mock_ml_pair in mock_ml_pairs:
        m1 = getMock(mock_ml_pair[0][0], mock_ml_pair[0][1])
        m2 = getMock(mock_ml_pair[1][0], mock_ml_pair[1][1])
        plotMustLink(m1, m2)

    for mock_cl_pari in mock_cl_paris:
        m1 = getMock(mock_cl_pari[0][0], mock_cl_pari[0][1])
        m2 = getMock(mock_cl_pari[1][0], mock_cl_pari[1][1])
        plotCannotLink(m1, m2)

def plotMuskAndCannotLinks():
    plotStatic()
    print("before getting random")
    plotRandom()

def plotAsRequired():
    while (True):
        idx1 = int(input("index1"))
        idx2 = int(input("index2"))
        cluster_idx_left, cluster_idx_right, repre_idx_left, repre_idx_right = getClusterAndRepresentativeIds(
            clustering, idx1, idx2)
        if cluster_idx_left == cluster_idx_right:
            plotMustLink(cluster_idx_left, idx1, idx2, repre_idx_left, repre_idx_right)
        else:
            plotCannotLink(cluster_idx_left, cluster_idx_right, repre_idx_left, repre_idx_right, idx1, idx2)

def main():
    for i in range(1, len(clustering.clusters) + 1):
        plott(i)
    plt.show()
    plotAsRequired()
    # plotMuskAndCannotLinks()

if __name__ == '__main__':
    main()

#experimental
def plotDeviationAroundDBA(ax1, series_mean, series_vertic_var, span_tuple = None):
    lower = series_mean - series_vertic_var
    upper = series_mean + series_vertic_var
    real_x = x
    if span_tuple != None:
        real_x = x[span_tuple[0]: span_tuple[1]]
        lower = lower[span_tuple[0]: span_tuple[1]]
        upper = upper[span_tuple[0]: span_tuple[1]]
    ax1.fill_between(real_x, lower, upper, alpha=0.1)

#experimental
def plotRawAverage(plt, cur_series):
    avg = np.divide(np.sum(cur_series, 0), len(cur_series))
    plt.plot(x, avg, color='purple')

#experimental
def plotMultiCurves(plt, indices, title, showLabel=False):
    plt.set_title(title + ' count=' + str(len(indices)))
    colors = getColorIter(len(indices))
    indices.sort()
    for index in indices:
        cur_color = next(colors)
        y = series[index]
        if showLabel == True:
            plt.plot(x, y, color=cur_color, label="index=" + str(index))
        else:
            plt.plot(x, y, color=cur_color)
    if showLabel == True & len(indices) < 15:
        plt.legend(loc = 9)

#deprecated
def plottt(i):
    # fig1 = plt.figure(1)
    exec('fig{} = plt.figure({})'.format(i, i))
    exec('ax{}, ax{},ax{}, ax{},ax{}, ax{} = initFigure(fig{})'.format(str(i)+'1', str(i)+'2', str(i)+'3', str(i)+'4', str(i)+'5', str(i)+'6', str(i)))
    exec('plotSelectedSpan((start, end), {}, ax{}, ax{},ax{}, ax{},ax{}, ax{})'.format(i-1, str(i)+'1', str(i)+'2', str(i)+'3', str(i)+'4', str(i)+'5', str(i)+'6'))

max_stars_repo_name: simiaolin/cobras

max_issues_repo_name: simiaolin/cobras

max_forks_repo_name: simiaolin/cobras


text:
#  ================ AlphaZero algorithm for Connect 4 game =================== #
# Name:             ResNet.py
# Description:      Includes both a dense and a resnet.
#                   Almost identical/taken from https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html
# Authors:          Jean-Philippe Bruneton & Adèle Douin & Vincent Reverdy
# Date:             2018
# License:          BSD 3-Clause License
# ============================================================================ #

# ================================= PREAMBLE ================================= #
# Packages
import torch
import torch.utils
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import torch.utils.data
import time
import math
from collections import OrderedDict
import torch.utils.data
import numpy as np
import config
import random

# ================================= CLASS : basic ResNet Block ================================= #

#no bias in conv
def conv3x3(in_planes, out_planes, stride=1):
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)

class BasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()

        m = OrderedDict()
        m['conv1'] = conv3x3(inplanes, planes, stride)
        m['bn1'] = nn.BatchNorm2d(planes)
        m['relu1'] = nn.ReLU(inplace=True)
        m['conv2'] = conv3x3(planes, planes)
        m['bn2'] = nn.BatchNorm2d(planes)
        self.group1 = nn.Sequential(m)

        self.relu = nn.Sequential(nn.ReLU(inplace=True))
        self.downsample = downsample

    def forward(self, x):
        if self.downsample is not None:
            residual = self.downsample(x)
        else:
            residual = x

        out = self.group1(x) + residual
        out = self.relu(out)

        return out

# ================================= CLASS : ResNet + two heads ================================= #

class ResNet(nn.Module):
    def __init__(self, block, layers):

        self.input_dim = config.L*config.H
        self.output_dim = config.L
        self.inplanes = config.convsize
        self.convsize=config.convsize
        super(ResNet, self).__init__()

        torch.set_num_threads(1)

        #as a start : the three features are mapped into a conv with 4*4 kernel
        self.ksize = (4, 4)
        self.padding = (1, 1)
        m = OrderedDict()
        m['conv1'] = nn.Conv2d(3, self.convsize, kernel_size=self.ksize, stride=1, padding=self.padding, bias=False)
        m['bn1'] = nn.BatchNorm2d(self.convsize)
        m['relu1'] = nn.ReLU(inplace=True)

        self.group1= nn.Sequential(m)

        #next : entering the resnet tower
        self.layer1 = self._make_layer(block, self.convsize, layers[0])

        #next : entering the policy head
        pol_filters = config.polfilters
        self.policy_entrance = nn.Conv2d(self.convsize, config.polfilters, kernel_size=1, stride=1, padding=0, bias=False)
        self.bnpolicy = nn.BatchNorm2d(config.polfilters)
        self.relu_pol = nn.ReLU(inplace=True)

        #if dense layer in policy head
        if config.usehiddenpol:
            self.hidden_dense_pol = nn.Linear(pol_filters * 30, config.hiddensize)
            self.relu_hidden_pol = nn.ReLU(inplace=True)
            self.fcpol1 = nn.Linear(config.hiddensize, 7)
        else:
            self.fcpol2= nn.Linear(pol_filters*30, 7)

        self.softmaxpol=nn.Softmax(dim=1)
        #end of policy head

        # in parallel: entering the value head
        val_filters = config.valfilters
        self.value_entrance = nn.Conv2d(self.convsize, config.valfilters, kernel_size=1, stride=1, padding=0, bias=False)
        self.bnvalue = nn.BatchNorm2d(config.valfilters)
        self.relu_val = nn.ReLU(inplace=True)

        #entering a dense hidden layer
        self.hidden_dense_value = nn.Linear(val_filters * 30, config.hiddensize)
        self.relu_hidden_val = nn.ReLU(inplace=True)
        self.fcval =  nn.Linear(config.hiddensize, 1)
        self.qval=nn.Tanh()
        #end value head

        # init weights
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / (5*n)))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        if type(x) == np.ndarray :
            x = x.reshape((3, config.H,config.L))
            x = torch.FloatTensor(x)
            x = torch.unsqueeze(x, 0)

        x = self.group1(x)
        x = self.layer1(x)

        x1 = self.policy_entrance(x)
        x1 = self.bnpolicy(x1)
        x1 = self.relu_pol(x1)
        x1 = x1.view(-1, config.polfilters*30)

        if config.usehiddenpol:
            x1 = self.hidden_dense_pol(x1)
            x1 = self.relu_hidden_pol(x1)
            x1 = self.fcpol1(x1)
        else:
            x1 = self.fcpol2(x1)

        x1 = self.softmaxpol(x1)

        x2 = self.value_entrance(x)
        x2 = self.bnvalue(x2)
        x2 = self.relu_val(x2)
        x2 = x2.view(-1, 30*config.valfilters)
        x2 = self.hidden_dense_value(x2)
        x2 = self.relu_hidden_val(x2)
        x2 = self.fcval(x2)
        x2 = self.qval(x2)

        return x2, x1

# -----------------------------------------------------------------#
# builds the model
def resnet18(pretrained=False, model_root=None, **kwargs):
    model = ResNet(BasicBlock, [config.res_tower, 2, 2, 2], **kwargs)
    return model

# ================================= CLASS : ResNet training ================================= #

class ResNet_Training:
    # -----------------------------------------------------------------#
    def __init__(self, net, batch_size, n_epoch, learning_rate, train_set, test_set, num_worker):
        self.net = net
        self.batch_size = batch_size
        self.n_epochs = n_epoch
        self.learning_rate = learning_rate
        self.num_worker = num_worker
        torch.set_num_threads(1)

        if config.use_cuda:
            self.net = self.net.cuda()

        self.train_set = train_set
        self.test_set = test_set

        self.train_loader = torch.utils.data.DataLoader(self.train_set, batch_size=self.batch_size, shuffle=True
                                                        , num_workers=self.num_worker, drop_last=True)
        self.test_loader = torch.utils.data.DataLoader(self.test_set, batch_size=64, shuffle=True
                                                       , num_workers=self.num_worker)
        self.valid_loader = torch.utils.data.DataLoader(self.train_set, batch_size=128, shuffle=True
                                                        , num_workers=self.num_worker)
        self.net.train()

    # -----------------------------------------------------------------#
    # Losses
    def Loss_value(self):
        loss = torch.nn.MSELoss()
        return loss

    def Loss_policy_bce(self):
        loss = torch.nn.BCELoss()
        return loss

    # -----------------------------------------------------------------#
    # Optimizers
    def Optimizer(self):

        if config.optim == 'sgd':
            optimizer = optim.SGD(self.net.parameters(), lr=self.learning_rate, momentum=config.momentum,
                                  weight_decay=config.wdecay)
        elif config.optim == 'adam':
            optimizer = optim.Adam(self.net.parameters(), lr=self.learning_rate, weight_decay=config.wdecay)

        elif config.optim == 'rms':
            optimizer = optim.RMSprop(self.net.parameters(), lr=self.learning_rate, momentum=config.momentum,
                                      weight_decay=config.wdecay)

        return optimizer

    # -----------------------------------------------------------------#
    # training function

    def trainNet(self):

        n_batches = len(self.train_loader)
        print(n_batches, 'batches')
        optimizer = self.Optimizer()

        # Loop for n_epochs
        for epoch in range(self.n_epochs):

            running_loss = 0.0
            print_every = n_batches // 2
            start_time = time.time()
            total_train_loss = 0

            for i, data in enumerate(self.train_loader, 0):

                sboard = config.L * config.H
                preinputs = data[:, 0:3*sboard]
                inputs = preinputs.view(self.batch_size, 3, config.H, config.L)
                probas = data[:, 3*sboard:3*sboard + self.net.output_dim]

                reward = data[:, -1]
                probas = probas.float()
                reward = reward.float()
                reward = reward.view(self.batch_size, 1)

                if config.use_cuda:
                    inputs, probas, reward = inputs.cuda(), probas.cuda(), reward.cuda()

                inputs = Variable(inputs.float())
                probas, reward = Variable(probas), Variable(reward)

                # Set the parameter gradients to zero
                optimizer.zero_grad()

                # Forward pass, backward pass, optimize
                vh, ph = self.net(inputs)
                loss = 0
                loss += self.Loss_value()(vh, reward)
                loss += self.Loss_policy_bce()(ph, probas)

                loss.backward()
                optimizer.step()

                # Print statistics
                running_loss += loss.data.item()
                total_train_loss += loss.data.item()

                if (i + 1) % (print_every + 1) == 0:
                    print("Epoch {}, {:d}% \t train_loss: {:.2f} took: {:.2f}s".format(
                        epoch+1, int(100 * (i+1) / n_batches), running_loss / print_every, time.time() - start_time))

        #    #Reset running loss and time
                    running_loss = 0.0
                    start_time = time.time()

        # send back the model to cpu for next self play games using forward in parallel cpu
        self.net.cpu()

# ================================= CLASS : DenseNet ================================= #

class DenseNet(nn.Module):
    # -----------------------------------------------------------------#
    def __init__(self):

        self.input_dim = 3*42
        torch.set_num_threads(1)
        self.hiddensize = 1024
        random.seed()

        super(DenseNet, self).__init__()

        self.first = nn.Linear(self.input_dim, self.hiddensize)
        self.second = nn.Linear(self.hiddensize, self.hiddensize)

        self.dense_policy = nn.Linear(self.hiddensize, 7)
        self.dense_value = nn.Linear(self.hiddensize, 1)

    # -----------------------------------------------------------------#
    def forward(self, x):
        if type(x) == np.ndarray:
            x = torch.FloatTensor(x)
            x = x.unsqueeze(0)
        else:
            x = x.squeeze(1)

        x = F.relu(self.first(x))
        x = F.relu(self.second(x))

        x1 = self.dense_policy(x)
        x1 = F.softmax(x1, dim=1)
        x2 = F.torch.tanh(self.dense_value(x))
        return x2, x1

# ================================= CLASS : DenseNet Training ================================= #

class DenseNet_Training:
    # -----------------------------------------------------------------#
    def __init__(self, net, batch_size, n_epoch, learning_rate, train_set, test_set, num_worker):
        self.net = net
        self.batch_size = batch_size
        self.n_epochs = n_epoch
        self.learning_rate = learning_rate
        self.num_worker = num_worker
        torch.set_num_threads(1)
        random.seed()
        self.optim = optim
        self.train_set = train_set
        self.test_set = test_set

        self.train_loader = torch.utils.data.DataLoader(self.train_set, batch_size=self.batch_size, shuffle=True
                                                        , num_workers=self.num_worker, drop_last=True)
        self.test_loader = torch.utils.data.DataLoader(self.test_set, batch_size=64, shuffle=True
                                                       , num_workers=self.num_worker)
        self.valid_loader = torch.utils.data.DataLoader(self.train_set, batch_size=128, shuffle=True
                                                        , num_workers=self.num_worker)
        self.net.train()

    # -----------------------------------------------------------------#
    # Losses
    def Loss_value(self):
        loss = torch.nn.MSELoss()
        return loss

    def Loss_policy_bce(self):
        loss = torch.nn.BCELoss()
        return loss

    def Optimizer_adam(self):

        optimizer = optim.Adam(self.net.parameters(), lr=self.learning_rate)
        return optimizer

    def Optimizer_sgd(self):
        optimizer = optim.SGD(self.net.parameters(), lr=self.learning_rate, momentum=config.momentum)

        return optimizer

    # -----------------------------------------------------------------#
    # Training
    def trainNet(self):

        n_batches = len(self.train_loader)
        print(n_batches, 'batches')

        for epoch in range(self.n_epochs):
            running_loss = 0.0
            print_every = n_batches // 2
            start_time = time.time()
            total_train_loss = 0

            for i, data in enumerate(self.train_loader, 0):

                sboard = config.L * config.H
                preinputs = data[:, 0:3*sboard]
                inputs = preinputs.view(self.batch_size, 3*config.H*config.L)
                probas = data[:, 3*sboard:3*sboard + 7]
                reward = data[:, -1]

                probas = probas.float()
                reward = reward.float()
                probas = probas.view(self.batch_size,7)
                reward = reward.view(self.batch_size, 1)
                inputs = Variable(inputs.float())

                # Set the parameter gradients to zero
                if config.optim == 'sgd':
                    self.Optimizer_sgd().zero_grad()
                elif config.optim == 'adam':
                    self.Optimizer_adam().zero_grad()

                # Forward pass, backward pass, optimize
                vh, ph = self.net(inputs)
                loss = 0
                loss += self.Loss_value()(vh, reward)
                loss += self.Loss_policy_bce()(ph, probas)
                loss.backward()

                if config.optim == 'sgd':
                    self.Optimizer_sgd().step()
                elif config.optim == 'adam':
                    self.Optimizer_adam().step()

                # Print statistics
                running_loss += loss.data.item()
                total_train_loss += loss.data.item()

                if (i + 1) % (print_every + 1) == 0:
                    print("Epoch {}, {:d}% \t train_loss: {:.2f} took: {:.2f}s".format(
                        epoch + 1, int(100 * (i + 1) / n_batches), running_loss / print_every,
                        time.time() - start_time))

                    #    #Reset running loss and time
                    running_loss = 0.0
                    start_time = time.time()

def densenet(pretrained=False, model_root=None, **kwargs):
    model = DenseNet(**kwargs)
    return model

ext: py

reason: import numpy


text:
import numpy as np
from PIL import Image
from AttnGAN.code.main_sampler import main_sampler

sentences = ['A man standing in a field',
             'A girl standing in the ocean']
print('\n\nGenerating Images now\n\n')
generated_images = main_sampler(sentences)

for i in range(len(generated_images)):
    im = Image.fromarray(generated_images[i])
    fullpath = './'+str(i)+'_.png'
    im.save(fullpath)

max_stars_repo_name: Amritds/AttnGAN

max_issues_repo_name: Amritds/AttnGAN

max_forks_repo_name: Amritds/AttnGAN


text:
"""Stop condition for DiviK.

stop.py

Copyright 2018 Spectre Team

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from functools import partial
from multiprocessing import Pool
from typing import Tuple

import numpy as np

import spdivik.distance as dst
import spdivik.types as ty
import spdivik.score as sc

def minimal_size(data: ty.Data, size: int = 2) -> bool:
    """Check if region is smaller than predefined size."""
    return data.shape[0] <= size

def _split_into_one(data: ty.Data) -> Tuple[ty.IntLabels, ty.Centroids]:
    labels = np.zeros(shape=(data.shape[0],), dtype=int)
    centroids = np.mean(data, axis=0, keepdims=True)
    return labels, centroids

class combine:
    """Combine stop conditions to be checked together."""

    def __init__(self, *args: ty.StopCondition):
        self._conditions = args

    def __call__(self, data: ty.Data) -> bool:
        """Check if there is any precaution for segmentation."""
        return any(precaution(data) for precaution in self._conditions)

class Gap:
    """GAP statistic-based stop condition."""

    def __init__(self, distance: dst.DistanceMetric,
                 split_into_two: ty.SegmentationMethod,
                 n_trials: int = 100, seed: int = 0, correction: bool=True,
                 pool: Pool = None):
        self._split_into_two = split_into_two
        self.correction = correction
        adjusted_gap = partial(sc.gap, distance=distance, seed=seed,
                               n_trials=n_trials, pool=pool)
        self._gap_of_two = partial(adjusted_gap, split=split_into_two)
        self._gap_of_one = partial(adjusted_gap, split=_split_into_one)

    def __call__(self, data: ty.Data) -> bool:
        """Check if segmentation is significantly justified."""
        labels, centroids = self._split_into_two(data)
        split_likelihood, split_deviation = self._gap_of_two(
            data, labels, centroids, return_deviation=True)
        labels, centroids = _split_into_one(data)
        dont_split_likelihood = self._gap_of_one(data, labels, centroids)
        if self.correction:
            return split_likelihood + split_deviation < dont_split_likelihood
        else:
            return split_likelihood < dont_split_likelihood

max_stars_repo_name: spectre-team/spectre-divik

max_issues_repo_name: spectre-team/spectre-divik

max_forks_repo_name: spectre-team/spectre-divik


text:
import torch
import torch.nn.functional as F
import sys
import numpy as np
import os, argparse
import cv2
from net import DFMNet
from data import test_dataset
import time

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
parser = argparse.ArgumentParser()
parser.add_argument('--testsize', type=int, default=256, help='testing size')
parser.add_argument('--gpu_id', type=str, default='0', help='select gpu id')
parser.add_argument('--test_path',type=str,default='./dataset/',help='test dataset path')
opt = parser.parse_args()

dataset_path = opt.test_path

#set device for test
if opt.gpu_id=='0':
    os.environ["CUDA_VISIBLE_DEVICES"] = "0"
    print('USE GPU 0')
elif opt.gpu_id=='1':
    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    print('USE GPU 1')
elif opt.gpu_id == '3':
    os.environ["CUDA_VISIBLE_DEVICES"] = "3"
    print('USE GPU 3')
elif opt.gpu_id=='all':
    os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
    print('USE GPU 0,1,2,3')

#load the model
model = DFMNet()
model.load_state_dict(torch.load('./pretrain/DFMNet_epoch_300.pth'))
model.to(device)
model.eval()

#test

def save(res,gt,notation=None,sigmoid=True):
    res = F.upsample(res, size=gt.shape, mode='bilinear', align_corners=False)
    res = res.sigmoid().data.cpu().numpy().squeeze() if sigmoid ==True else res.data.cpu().numpy().squeeze()
    res = (res - res.min()) / (res.max() - res.min() + 1e-8)
    print('save img to: ', os.path.join(save_path, name.replace('.png','_'+notation+'.png') if notation != None else name))
    cv2.imwrite(os.path.join(save_path, name.replace('.png','_'+notation+'.png') if notation != None else name), res * 255)

test_datasets = ['NJU2K','NLPR','STERE', 'RGBD135', 'LFSD','SIP']
for dataset in test_datasets:
    with torch.no_grad():
        save_path = './results/benchmark/' + dataset
        if not os.path.exists(save_path):
            os.makedirs(save_path)
        image_root = dataset_path + dataset + '/RGB/'
        gt_root = dataset_path + dataset + '/GT/'
        depth_root=dataset_path +dataset +'/depth/'
        test_loader = test_dataset(image_root, gt_root,depth_root, opt.testsize)

        for i in range(test_loader.size):
            image, gt,depth, name, image_for_post = test_loader.load_data()
            gt = np.asarray(gt, np.float32)
            gt /= (gt.max() + 1e-8)
            image = image.to(device)
            depth = depth.to(device)
            # torch.cuda().synchronize()
            time_s = time.time()
            out = model(image,depth)
            print(out)
            # torch.cdua().synchronize()
            time_e = time.time()
            t = time_e - time_s
            print("time: {:.2f} ms".format(t*1000))
            save(out[0],gt)
    print('Test Done!')


text:
"""Show a simple example of using MultiplePi."""
import matplotlib.pyplot as plt
import numpy as np

from matplot_fmt_pi.ticker import MultiplePi

fig = plt.figure(figsize=(4 * np.pi, 2.4))
axes = fig.add_subplot(111)
x = np.linspace(-2 * np.pi, 2 * np.pi, 512)
axes.plot(x, np.sin(x))

axes.grid(True)
axes.axhline(0, color='black', lw=2)
axes.axvline(0, color='black', lw=2)
axes.set_title("MultiplePi formatting")

pi_manager = MultiplePi(2)
axes.xaxis.set_major_locator(pi_manager.locator())
axes.xaxis.set_major_formatter(pi_manager.formatter())

plt.tight_layout()
plt.savefig("./pi_graph.png", dpi=120)

max_stars_repo_name: k-donn/format-pi

max_issues_repo_name: k-donn/format-pi

max_forks_repo_name: k-donn/format-pi


text:
import numpy as np
import tensorflow as tf
import pandas as pd

from utils.intersection_over_union import *

def ground_truth_generator(df):
    df['ratio'] = df.apply(lambda x: 'w' if x['x_max'] - x['x_min'] > x['y_max'] - x['y_min'] else 'h', axis=1)
    df['x_min'] = df.apply(lambda x: x['x_min'] - (x['x_max'] - x['x_min'])*.1 if x['ratio'] == 'h' else x['x_min'], axis=1)
    df['x_max'] = df.apply(lambda x: x['x_max'] + (x['x_max'] - x['x_min'])*.1 if x['ratio'] == 'h' else x['x_max'], axis=1)
    df['y_min'] = df.apply(lambda x: x['y_min'] - (x['y_max'] - x['y_min'])*.1 if x['ratio'] == 'w' else x['y_min'], axis=1)
    df['y_max'] = df.apply(lambda x: x['y_max'] + (x['y_max'] - x['y_min'])*.1 if x['ratio'] == 'w' else x['y_max'], axis=1)

    ground_truth = df.iloc[:,:1].copy()
    ground_truth['x_min'] = df['x_min'] - (df['x_max'] - df['x_min'])*.05
    ground_truth['x_max'] = df['x_max'] + (df['x_max'] - df['x_min'])*.05
    ground_truth['y_min'] = df['y_min'] - (df['y_max'] - df['y_min'])*.05
    ground_truth['y_max'] = df['y_max'] + (df['y_max'] - df['y_min'])*.05

    ground_truth['w'] = ground_truth['x_max'] - ground_truth['x_min']
    ground_truth['h'] = ground_truth['y_max'] - ground_truth['y_min']
    ground_truth['x'] = ground_truth['w']/2 + ground_truth['x_min']
    ground_truth['y'] = ground_truth['h']/2 + ground_truth['y_min']
    return ground_truth

def gt_generator(target):
    x_min = np.min(target[::2])
    x_max = np.max(target[::2])
    y_min = np.min(target[1::2])
    y_max = np.max(target[1::2])
    ratio = 'w' if x_max - x_min > y_max - y_min else 'h'

    x_min = x_min - (x_max - x_min)*.1 if ratio == 'h' else x_min
    x_max = x_max + (x_max - x_min)*.1 if ratio == 'h' else x_max
    y_min = y_min - (y_max - y_min)*.1 if ratio == 'w' else y_min
    y_max = y_max + (y_max - y_min)*.1 if ratio == 'w' else y_max

    ground_truth_x_min = x_min - (x_max - x_min)*.05
    ground_truth_x_max = x_max + (x_max - x_min)*.05
    ground_truth_y_min = y_min - (y_max - y_min)*.05
    ground_truth_y_max = y_max + (y_max - y_min)*.05

    ground_truth_w = ground_truth_x_max - ground_truth_x_min
    ground_truth_h = ground_truth_y_max - ground_truth_y_min
    ground_truth_x = ground_truth_w/2 + ground_truth_x_min
    ground_truth_y = ground_truth_h/2 + ground_truth_y_min
    # return [ground_truth_x, ground_truth_y, ground_truth_w, ground_truth_h], [ground_truth_x_min, ground_truth_y_min]
    return [ground_truth_x, ground_truth_y, ground_truth_w, ground_truth_h]

def label_generator(GT, anchor_boxes, out_boundaries_indxes):
    cls_label = - np.ones(shape=(anchor_boxes.shape[0]))
    pos_iou_threshold = 0.7
    neg_iou_threshold = 0.3
    n_sample = 32
    pos_ratio = 0.5
    n_pos = int(pos_ratio * n_sample)

    ious = np.apply_along_axis(IoU_np, 0, GT, anchor_boxes=anchor_boxes)
    cls_label[ious >= pos_iou_threshold] = 1
    cls_label[ious < neg_iou_threshold] = 0
    cls_label[np.argmax(ious)] = 1
    cls_label[out_boundaries_indxes] = -1

    pos_index = np.where(cls_label == 1)[0]
    if len(pos_index) > n_pos:
        disable_index = np.random.choice(
            pos_index,
            size = (len(pos_index) - n_pos),
            replace=False
        )
        cls_label[disable_index] = -1

    n_neg = n_sample - np.sum(cls_label == 1)
    neg_index = np.where(cls_label == 0)[0]
    if len(neg_index) > n_neg:
        disable_index = np.random.choice(
            neg_index,
            size = (len(neg_index) - n_neg),
            replace = False
        )
        cls_label[disable_index] = -1

    reg_label = anchor_boxes * np.broadcast_to(tf.cast(cls_label > 0, tf.int32), (4, len(cls_label))).T
    indices = np.where(reg_label != 0)[0][::4]
    x, y, w, h = GT[0], GT[1], GT[2], GT[3]

    tx = (x - reg_label[indices][:, 0]) / (reg_label[indices][:, 2])
    ty = (y - reg_label[indices][:, 1]) / (reg_label[indices][:, 3])
    tw = np.log(w / reg_label[indices][:, 2])
    th = np.log(h / reg_label[indices][:, 3])
    reg_label[indices] = np.stack([tx, ty, tw, th]).T

    return cls_label, reg_label

def classifier_label_generator(nms, gts, valid=False):
    num_roi = 128
    pos_ratio = .5
    num_pos = int(num_roi * pos_ratio)
    num_neg = int(num_roi - num_pos)
    ious = get_iou(nms, gts)
    pos_order = tf.argsort(ious * tf.cast(tf.math.greater_equal(ious, 0.5), tf.float64), direction='DESCENDING', axis=1)[:, :num_pos]
    neg_order = tf.argsort(ious * tf.cast(tf.math.less(ious, 0.5), tf.float64), direction='DESCENDING', axis=1)[:, :num_neg]

    if not valid:
        neg_cnt = tf.reduce_min(tf.math.count_nonzero(ious * tf.cast(tf.math.less(ious, 0.5), tf.float64), axis=1))
        neg_order = tf.argsort(ious * tf.cast(tf.math.less(ious, 0.5), tf.float64), direction='DESCENDING', axis=1)[:, :neg_cnt]
        indices = tf.range(start=0, limit=neg_cnt, dtype=tf.int32)
        shuffled_indices = tf.random.shuffle(indices)
        neg_order = tf.gather(neg_order, shuffled_indices, axis=1)[:, :num_neg]

    cls_labels = tf.concat([tf.ones_like(pos_order), tf.zeros_like(neg_order)], axis=1)
    label_order = tf.concat([pos_order, neg_order], axis=1)
    P_boxes = tf.gather(nms, label_order, batch_dims=1)

    tx = (gts[:, :1] - P_boxes[:, :, 0]) / P_boxes[:, :, 2]
    ty = (gts[:, 1:2] - P_boxes[:, :, 1]) / P_boxes[:, :, 3]
    tw = tf.math.log(gts[:, 2:3] / P_boxes[:, :, 2])
    th = tf.math.log(gts[:, 3:] / P_boxes[:, :, 3])
    tf.stack([tx, ty, tw, th], axis=-1)
    box_labels = tf.stack([tx, ty, tw, th], axis=-1)
    return box_labels, cls_labels, P_boxes

max_stars_repo_name: sooooner/faster-RCNN

max_issues_repo_name: sooooner/faster-RCNN

max_forks_repo_name: sooooner/faster-RCNN


text:
# coding: utf-8
# https://github.com/PaddlePaddle/Paddle-Inference-Demo/tree/master/python
import os, sys
import numpy as np
from paddle import fluid

from paddle.inference import Config
from paddle.inference import create_predictor

class PaddlePaddleOCR:
    def __init__(self, cfg):
        self.cfg = cfg
        self.predictor = self.init_predictor(cfg)
        print('model is inited')

    def init_predictor(self, cfg):
        model_dir = cfg['model_dir']
        params_file = cfg['params_file']
        use_gpu = cfg['use_gpu']

        config = Config(model_dir, params_file)

        # config.enable_memory_optim()

        if use_gpu:
            config.enable_use_gpu(1000, 0)
        else:
            # If not specific mkldnn, you can set the blas thread.
            # The thread num should not be greater than the number of cores in the CPU.
            config.set_cpu_math_library_num_threads(4)
            config.enable_mkldnn()
        config.disable_glog_info()

        predictor = create_predictor(config)
        return predictor

    def run(self, image_data):
        inp = np.expand_dims(image_data, 0)
        input_names = self.predictor.get_input_names()
        for i, name in enumerate(input_names):
            input_tensor = self.predictor.get_input_handle(name)
            input_tensor.reshape(inp.shape)
            input_tensor.copy_from_cpu(inp.copy())

        # do the inference
        self.predictor.run()

        results = []
        # get out data from output tensor
        output_names = self.predictor.get_output_names()
        for i, name in enumerate(output_names):
            output_tensor = self.predictor.get_output_handle(name)
            output_data = output_tensor.copy_to_cpu()
            results.append(output_data)

        return results

if __name__ == '__main__':
    print('begin...')

    USE_GPU = False
    # root_dir = './ch_ppocr_server_v2.0_det_infer'
    # root_dir = 'C:/workspace/repo/OCR/ai_groupOCR/ppocr_models/v2/ch_ppocr_mobile_v2.0_det_infer'
    # root_dir = 'C:/workspace/repo/OCR/ai_groupOCR/ppocr_models/v2/ch_ppocr_mobile_v2.0_cls_infer'
    root_dir = 'C:/workspace/repo/OCR/ai_groupOCR/ppocr_models/v2/ch_ppocr_server_v2.0_rec_infer'
    MODEL_DIR = 'inference.pdmodel'
    PARAMS_FILE = 'inference.pdiparams'

    os_cvd = '-1' if USE_GPU==False else '0'
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

    h = 32
    w = 320
    c = 3
    np.random.seed(666)
    # inp = np.random.rand(c, h, w).astype(np.float32)
    inp = np.random.randn(c, h, w).astype(np.float32)
    print('==> ',np.sum(inp), np.mean(inp), np.max(inp), np.min(inp))
    # print(inp, inp.shape, inp.dtype, '\n')

    import cv2

    image = cv2.imread('C:/workspace/repo/OCR/ai_groupOCR/ai_OCR_Recognizer/Snipaste.jpg')
    image = cv2.resize(image, (w, h))
    mean = 0.5
    std = 0.5
    scale = 1. / 255
    norm_img = (image * scale - mean) / std
    inp = norm_img
    inp = inp.transpose(2, 0, 1)
    print(np.sum(inp), np.mean(inp), np.max(inp), np.min(inp))

    #
    # import cv2
    # image = cv2.imread('6.jpg')
    # image = cv2.resize(image, (320, 448))
    # mean = np.array([0.485, 0.456, 0.406])
    # std = np.array([0.229, 0.224, 0.225])
    # scale = 1. / 255
    # norm_img = (image * scale - mean) / std
    # inp = norm_img
    # inp = inp.transpose(2,0,1)

    inp = inp.astype(np.float32)
    # print(inp, inp.shape, inp.dtype, '\n')

    model_file_path = os.path.abspath(os.path.join(root_dir, MODEL_DIR))
    params_file_path = os.path.abspath(os.path.join(root_dir, PARAMS_FILE))
    if not os.path.exists(model_file_path):
        print("not find model file path {}".format(model_file_path))
        sys.exit(0)
    if not os.path.exists(params_file_path):
        print("not find params file path {}".format(params_file_path))
        sys.exit(0)

    cfg = {}
    cfg['model_dir'] = model_file_path
    cfg['params_file'] = params_file_path
    cfg['use_gpu'] = USE_GPU

    paddleOCR = PaddlePaddleOCR(cfg)
    results = paddleOCR.run(inp)
    print(results[0].shape)
    print(np.sum(results), np.mean(results), np.max(results), np.min(results))
    print('done!')


text:
"""A parser for reading water level data from open Norwegian water level API

Example:
--------

    from datetime import datetime
    from analyx import parsers

    # XML file with water level data exists
    p = parsers.parse_file(parser_name='api_water_level_norway', file_path='api_water_level_norway')
    data = p.as_dict()

    # Water level data has to be downloaded from API
    p = parsers.parse_file(
                    parser_name='api_water_level_norway',
                    file_path='api_water_level_norway',
                    latitude=58.974339,
                    longitude=5.730121,
                    from_date=datetime(2021,11,21),
                    to_date=datetime(2021,11,22),
    )
    data = p.as_dict()

Description:
------------

See https://api.sehavniva.no/tideapi_no.html for an example
"""
# Standard library imports
from datetime import datetime
import pathlib
import pytz
from typing import Optional, Union
from xml.dom.minidom import parse

# Third party imports
import numpy as np
import pycurl

# Midgard imports
from midgard.data import dataset
from midgard.dev import log
from midgard.dev import plugins
from midgard.files import files
from midgard.math.unit import Unit
from midgard.parsers import Parser

@plugins.register
class ApiWaterLevelNorwayParser(Parser):
    """A parser for reading water level data from open Norwegian water level API

    See https://api.sehavniva.no/tideapi_no.html for an example

    Following **data** are available after water level data:

    | Parameter           | Description                                                                           |
    |---------------------|---------------------------------------------------------------------------------------|
    | flag                | Data flag (obs: observation, pre: prediction, weather: weather effect, forecast:      |
    |                     | forecast)                                                                             |
    | time                | Observation                                                                           |
    | water_level         | Water level in [cm]                                                                   |

    and **meta**-data:

    | Key                 | Description                                                                           |
    |---------------------|---------------------------------------------------------------------------------------|
    | __data_path__       | File path                                                                             |
    | __parser_name__     | Parser name                                                                           |
    | __url__             | URL of water level API                                                                |
    """

    URL = "http://api.sehavniva.no/tideapi.php"

    def __init__(
            self,
            file_path: Union[str, pathlib.Path],
            encoding: Optional[str] = None,
            url: Optional[str] = None,
            latitude: Optional[float] = None,
            longitude: Optional[float] = None,
            from_date: Optional[datetime] = None,
            to_date: Optional[datetime] = None,
    ) -> None:
        """Set up the basic information needed by the parser

        Args:
            file_path:    Path to file that will be read or downloaded.
            encoding:     Encoding of file that will be read.
            url:          Optional URL from where to download water level data.
            latitude:     Latitude of position in [deg]
            longitude:    Longitude of position in [deg]
            from_date:    Starting date of data period
            to_date:      Ending date of data period
        """
        super().__init__(file_path, encoding)
        if not self.file_path.exists() or self.file_path.stat().st_size == 0:
            self.download_xml(latitude, longitude, from_date, to_date, url)
            self.data_available = self.file_path.exists()

    def download_xml(
            self,
            latitude: float,
            longitude: float,
            from_date: datetime,
            to_date: datetime,
            url: Optional[str] = None,
    ) -> None:
        """Download XML file from url

        Args:
            latitude:     Latitude of position in [deg]
            longitude:    Longitude of position in [deg]
            from_date:    Starting date of data period
            to_date:      Ending date of data period
            url:          URL to download from, if None use self.URL instead.
        """

        # Get URL
        url = self.URL if url is None else url

        try:
            args = dict(
                lat=latitude,
                lon=longitude,
                fromtime=from_date.strftime("%Y-%m-%dT%H:%M"),
                totime=to_date.strftime("%Y-%m-%dT%H:%M"),
                datatype="all",
                refcode="cd",
                place="",
                file="",
                lang="nn",
                interval=10,
                dst=0,
                tzone="",
                tide_request="locationdata",
            )
        except AttributeError:
            log.fatal("Following arguments has to be set: latitude, longitude, from_date and to_date")

        url=f"{url}?{'&'.join([f'{k}={v}' for k, v in args.items()])}"
        print(f"Downloading {url} to {self.file_path}")

        # Read data from API to file path
        with files.open(self.file_path, mode="wb") as fid:
            c = pycurl.Curl()
            c.setopt(c.URL, url)
            c.setopt(c.WRITEDATA, fid)
            try:
                c.perform()
            finally:
                c.close()

        self.meta["__url__"] = url

    def read_data(self) -> None:
        """Read data from the XML file

        """
        self.data = {
            "value": list(),
            "time": list(),
            "flag": list(),
        }
        document = parse(str(self.file_path))
        elements = document.getElementsByTagName("waterlevel")

        # Read XML elements
        #
        # Example:
        #   <waterlevel value="104.5" time="2021-11-22T00:00:00+01:00" flag="pre"/>
        for element in elements:
            self.data["value"].append(float(element.attributes.get("value").value))

            self.data["time"].append(datetime.fromisoformat(element.attributes.get("time").value)) # Example to read: 2021-11-22T00:00:00+01:00

            flag = element.attributes.get("flag").value if "flag" in element.attributes.keys() else "weather"
            self.data["flag"].append(flag)

    #
    # GET DATASET
    #
    def as_dataset(self) -> "Dataset":
        """Return the parsed data as a Dataset

        Returns:
            dset (Dataset): The Dataset where water level observation are stored with following fields:

       |  Field               | Type              | Description                                                       |
       |----------------------|-------------------|-------------------------------------------------------------------|
       | flag                 | numpy.ndarray     | Data flag (obs: observation, pre: prediction, weather: weather    |
       |                      |                   | effect, forecast: forecast)                                       |
       | time                 | TimeTable         | Observation time given as TimeTable object                        |
       | water_level          | numpy.ndarray     | Water level in [m]                                                |

            and following Dataset `meta` data:

       |  Entry               | Type              | Description                                                       |
       |----------------------|-------------------|-------------------------------------------------------------------|
       | __data_path__        | str               | File path                                                         |
       | __url__              | str               | URL of water level API                                            |

        """
        dset = dataset.Dataset(num_obs=len(self.data["time"]))

        time = [d.astimezone(tz=pytz.utc).replace(tzinfo=None) for d in self.data["time"]] # Convert time to UTC
        dset.add_time("time", val=time, scale="utc", fmt="datetime")
        dset.add_float("water_level", val=np.array(self.data["value"]) * Unit.centimeter2meter)
        dset.add_text("flag", val=self.data["flag"])

        dset.meta.update(self.meta)
        return dset

max_stars_repo_name: kartverket/midgard

max_issues_repo_name: kartverket/midgard

max_forks_repo_name: kartverket/midgard


text:
import numpy as np

from scattering.utils.io import get_fn
from scattering.utils.features import find_local_maxima, find_local_minima

def test_local_maxima():
    data = np.loadtxt(get_fn('rdf.txt'))
    r = data[:, 0]
    g_r = data[:, 1]

    r_maxes = list()
    for i, r_guess in enumerate([0.3, 0.45, 0.65]):
        r_max, g_r_max = find_local_maxima(r, g_r, r_guess=r_guess)
        r_maxes.append(r_max)

    r_mins = list()
    for i, r_guess in enumerate([0.3, 0.5]):
        r_min, g_r_min = find_local_minima(r, g_r, r_guess=r_guess)
        r_mins.append(r_min)
    print(r_mins)

    assert np.allclose(r_maxes, [0.2775, 0.4375, 0.6675])
    assert np.allclose(r_mins, [0.3325, 0.5575])

max_stars_repo_name: XiaoboLinlin/scattering

max_issues_repo_name: XiaoboLinlin/scattering


text:
import numpy as np
from matplotlib import pyplot as plt

def latlt2polar(lat,lt,hemisphere):
    """
    Converts an array of latitude and lt points to polar for a top-down dialplot (latitude in degrees, LT in hours)
    i.e. makes latitude the radial quantity and MLT the azimuthal

    get the radial displacement (referenced to down from northern pole if we want to do a top down on the north,
    or up from south pole if visa-versa)
    """
    from numpy import pi
    if hemisphere=='N':
        r = 90.-lat
    elif hemisphere=='S':
        r = 90.-(-1*lat)
    else:
        raise ValueError('%s is not a valid hemisphere, N or S, please!' % (hemisphere))
    #convert lt to theta (azimuthal angle) in radians
    theta = lt/24. * 2*pi

    #the pi/2 rotates the coordinate system from
    #theta=0 at negative y-axis (local time) to
    #theta=0 at positive x axis (traditional polar coordinates)
    return r,theta

def polar2dial(ax,mlt_labels=True):
    """
    Turns a matplotlib axes polar plot into a dial plot
    """
    #Rotate the plot so that noon is at the top and midnight
    #is at the bottom, and fix the labels so radial direction
    #is latitude and azimuthal direction is local time in hours
    ax.set_theta_zero_location('S')
    theta_label_values = np.array([0.,3.,6.,9.,12.,15.,18.,21.])*180./12
    theta_labels = ['%d' % (int(th/180.*12)) for th in theta_label_values.flatten().tolist()]
    ax.set_thetagrids(theta_label_values,labels=theta_labels)

    r_label_values = 90.-np.array([80.,70.,60.,50.,40.])
    r_labels = [r'$%d^{o}$' % (int(90.-rv)) for rv in r_label_values.flatten().tolist()]
    ax.set_rgrids(r_label_values,labels=r_labels)
    ax.set_rlim([0.,40.])

max_stars_repo_name: lkilcommons/GDCmagnetic

max_issues_repo_name: lkilcommons/GDCmagnetic

max_forks_repo_name: lkilcommons/GDCmagnetic

reason: import numpy,from numpy


text:
"""
StyleMapGAN
Copyright (c) 2021-present NAVER Corp.

This work is licensed under the Creative Commons Attribution-NonCommercial
4.0 International License. To view a copy of this license, visit
http://creativecommons.org/licenses/by-nc/4.0/ or send a letter to
Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
"""

import argparse
import pickle
import os
import torch
from torch import nn
import numpy as np
from scipy import linalg
from tqdm import tqdm
from training.model import Generator
from metrics.calc_inception import load_patched_inception_v3
from torchvision import utils, transforms
from training.dataset import MultiResolutionDataset, DataSetFromDir
from torch.utils import data
from torch.nn import functional as F
from PIL import Image
from torch.utils.data import Dataset

class DPModel(nn.Module):
    def __init__(self, device, model_args):
        super(DPModel, self).__init__()
        self.g_ema = Generator(
            model_args.size,
            model_args.mapping_layer_num,
            model_args.latent_channel_size,
            model_args.latent_spatial_size,
            lr_mul=model_args.lr_mul,
            channel_multiplier=model_args.channel_multiplier,
            normalize_mode=model_args.normalize_mode,
            small_generator=model_args.small_generator,
        )

    def forward(self, real_img):
        z = real_img
        fake_img, _ = self.g_ema(z)

        return fake_img

def data_sampler(dataset, shuffle):
    if shuffle:
        return data.RandomSampler(dataset)

    else:
        return data.SequentialSampler(dataset)

def make_noise(batch, latent_channel_size, device):
    return torch.randn(batch, latent_channel_size, device=device)

@torch.no_grad()
def extract_feature_from_samples(generator, inception, batch_size, n_sample, device):
    n_batch = n_sample // batch_size
    resid = n_sample - (n_batch * batch_size)
    if resid > 0:
        batch_sizes = [batch_size] * n_batch + [resid]
    else:
        batch_sizes = [batch_size] * n_batch
    features = []

    for batch in tqdm(batch_sizes):
        latent = make_noise(batch, train_args.latent_channel_size, device)
        imgs = generator(latent)
        imgs = (imgs + 1) / 2  # -1 ~ 1 to 0~1
        imgs = torch.clamp(imgs, 0, 1, out=None)
        imgs = F.interpolate(imgs, size=(height, width), mode="bilinear")
        transformed = []

        for img in imgs:
            transformed.append(transforms.Normalize(mean=mean, std=std)(img))

        transformed = torch.stack(transformed, dim=0)

        assert transformed.shape == imgs.shape
        feat = inception(transformed)[0].view(imgs.shape[0], -1)
        features.append(feat.to("cpu"))

    features = torch.cat(features, 0)

    return features

def calc_fid(sample_mean, sample_cov, real_mean, real_cov, eps=1e-6):
    cov_sqrt, _ = linalg.sqrtm(sample_cov @ real_cov, disp=False)

    if not np.isfinite(cov_sqrt).all():
        print("product of cov matrices is singular")
        offset = np.eye(sample_cov.shape[0]) * eps
        cov_sqrt = linalg.sqrtm((sample_cov + offset) @ (real_cov + offset))

    if np.iscomplexobj(cov_sqrt):
        if not np.allclose(np.diagonal(cov_sqrt).imag, 0, atol=1e-3):
            m = np.max(np.abs(cov_sqrt.imag))

            raise ValueError(f"Imaginary component {m}")

        cov_sqrt = cov_sqrt.real

    mean_diff = sample_mean - real_mean
    mean_norm = mean_diff @ mean_diff

    trace = np.trace(sample_cov) + np.trace(real_cov) - 2 * np.trace(cov_sqrt)

    fid = mean_norm + trace

    return fid

def extract_feature_from_generated_samples(
    inception, batch_size, n_sample, device, transform
):

    features = []

    try:  # from LMDB
        dataset = MultiResolutionDataset(
            args.generated_image_path, transform, args.size
        )
    except:  # from raw images
        dataset = DataSetFromDir(args.generated_image_path, transform)

    loader = data.DataLoader(
        dataset,
        batch_size=batch_size,
        sampler=data_sampler(dataset, shuffle=True),
        num_workers=args.num_workers,
        pin_memory=True,
    )

    # generated images should match with n sample
    print(len(loader), n_sample, batch_size)

    if n_sample % batch_size == 0:
        assert len(loader) == n_sample // batch_size
    else:
        assert len(loader) == n_sample // batch_size + 1

    for i, real_img in enumerate(tqdm(loader)):
        real_img = real_img.to(device)

        if args.batch * (i + 1) > n_sample:
            real_img = real_img[: n_sample - args.batch * i]

        feat = inception(real_img)[0].view(real_img.shape[0], -1)
        features.append(feat.to("cpu"))

        if args.batch * (i + 1) > n_sample:
            break

    features = torch.cat(features, 0)
    print(len(features))
    assert len(features) == n_sample

    return features

if __name__ == "__main__":
    device = "cuda"

    parser = argparse.ArgumentParser()

    parser.add_argument("--batch", type=int, default=16)
    parser.add_argument("--size", type=int, default=256)
    parser.add_argument("--comparative_fid_pkl", type=str, default=None)
    parser.add_argument("--num_workers", type=int, default=10)
    parser.add_argument("--generated_image_path", type=str)
    parser.add_argument("--ckpt", metavar="CHECKPOINT")
    parser.add_argument("--dataset", type=str, required=True)

    args = parser.parse_args()
    assert ((args.generated_image_path is None) and (args.ckpt is not None)) or (
        (args.generated_image_path is not None) and (args.ckpt is None)
    )

    if args.dataset == "celeba_hq":
        n_sample = 29000
    elif args.dataset == "afhq":
        n_sample = 15130
    elif args.dataset in ["lsun/car", "lsun/church_outdoor"]:
        n_sample = 50000
    elif args.dataset == "ffhq":
        n_sample = 69000

    inception = nn.DataParallel(load_patched_inception_v3()).to(device)
    inception.eval()
    height, width = 299, 299
    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]

    if args.ckpt:
        assert args.comparative_fid_pkl is not None

        ckpt = torch.load(args.ckpt)
        train_args = ckpt["train_args"]
        assert args.size == train_args.size
        model = DPModel(device, train_args).to(device)
        model.g_ema.load_state_dict(ckpt["g_ema"])
        model = nn.DataParallel(model)
        model.eval()

        features = extract_feature_from_samples(
            model, inception, args.batch, n_sample, device
        ).numpy()

    else:
        transform = transforms.Compose(
            [
                transforms.Resize([args.size, args.size]),
                transforms.Resize([height, width]),
                transforms.ToTensor(),
                transforms.Normalize(mean=mean, std=std),
            ]
        )

        features = extract_feature_from_generated_samples(
            inception, args.batch, n_sample, device, transform
        ).numpy()

    print(f"extracted {features.shape[0]} features")

    sample_mean = np.mean(features, 0)
    sample_cov = np.cov(features, rowvar=False)

    if (args.generated_image_path is not None) and (args.comparative_fid_pkl is None):
        with open(
            f"metrics/fid_stats/{args.dataset}_stats_{args.size}_{n_sample}.pkl",
            "wb",
        ) as handle:
            pickle.dump({"mean": sample_mean, "cov": sample_cov}, handle)
    else:
        with open(args.comparative_fid_pkl, "rb") as f:
            embeds = pickle.load(f)
            real_mean = embeds["mean"]
            real_cov = embeds["cov"]

        fid = calc_fid(sample_mean, sample_cov, real_mean, real_cov)

        print("fid:", fid)

reason: import numpy,from scipy


text:
import unittest
import numpy
import pandas
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.decomposition import TruncatedSVD
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import FloatTensorType, Int64TensorType, StringTensorType
from test_utils import dump_data_and_model

class PipeConcatenateInput:
    def __init__(self, pipe):
        self.pipe = pipe

    def transform(self, inp):
        if isinstance(inp, (numpy.ndarray, pandas.DataFrame)):
            return self.pipe.transform(inp)
        elif isinstance(inp, dict):
            keys = list(sorted(inp.keys()))
            dim = inp[keys[0]].shape[0], len(keys)
            x2 = numpy.zeros(dim)
            for i in range(x2.shape[1]):
                x2[:, i] = inp[keys[i]]
            res = self.pipe.transform(x2)
            return res
        else:
            raise TypeError("Unable to predict with type {0}".format(type(inp)))

class TestSklearnPipeline(unittest.TestCase):

    def test_pipeline(self):
        from sklearn.preprocessing import StandardScaler
        from sklearn.pipeline import Pipeline

        data = numpy.array([[0, 0], [0, 0], [1, 1], [1, 1]], dtype=numpy.float32)
        scaler = StandardScaler()
        scaler.fit(data)
        model = Pipeline([('scaler1',scaler), ('scaler2', scaler)])

        model_onnx = convert_sklearn(model, 'pipeline', [('input', FloatTensorType([1, 2]))])
        self.assertTrue(model_onnx is not None)
        dump_data_and_model(data, model, model_onnx, basename="SklearnPipelineScaler")

    def test_combine_inputs(self):
        from sklearn.preprocessing import StandardScaler
        from sklearn.pipeline import Pipeline

        data = numpy.array([[0., 0.], [0., 0.], [1., 1.], [1., 1.]], dtype=numpy.float32)
        scaler = StandardScaler()
        scaler.fit(data)
        model = Pipeline([('scaler1', scaler), ('scaler2', scaler)])

        model_onnx = convert_sklearn(model, 'pipeline',
                                     [('input1', FloatTensorType([1, 1])),
                                      ('input2', FloatTensorType([1, 1]))])
        self.assertTrue(len(model_onnx.graph.node[-1].output) == 1)
        self.assertTrue(model_onnx is not None)
        data = {'input1': data[:, 0], 'input2': data[:, 1]}
        dump_data_and_model(data, PipeConcatenateInput(model), model_onnx,
                            basename="SklearnPipelineScaler11-OneOff")

    def test_combine_inputs_floats_ints(self):
        from sklearn.preprocessing import StandardScaler
        from sklearn.pipeline import Pipeline

        data = [[0, 0.],[0, 0.],[1, 1.],[1, 1.]]
        scaler = StandardScaler()
        scaler.fit(data)
        model = Pipeline([('scaler1', scaler), ('scaler2', scaler)])

        model_onnx = convert_sklearn(model, 'pipeline',
                                     [('input1', Int64TensorType([1, 1])),
                                      ('input2', FloatTensorType([1, 1]))])
        self.assertTrue(len(model_onnx.graph.node[-1].output) == 1)
        self.assertTrue(model_onnx is not None)
        data = numpy.array(data)
        data = {'input1': data[:, 0].astype(numpy.int64),
                'input2': data[:, 1].astype(numpy.float32)}
        dump_data_and_model(data, PipeConcatenateInput(model), model_onnx,
                            basename="SklearnPipelineScalerMixed-OneOff")

    def test_pipeline_column_transformer(self):

        iris = datasets.load_iris()
        X = iris.data[:, :3]
        y = iris.target
        X_train = pandas.DataFrame(X, columns=["vA", "vB", "vC"])
        X_train["vcat"] = X_train["vA"].apply(lambda x: "cat1" if x > 0.5 else "cat2")
        X_train["vcat2"] = X_train["vB"].apply(lambda x: "cat3" if x > 0.5 else "cat4")
        y_train = y % 2
        numeric_features = [0, 1, 2] # ["vA", "vB", "vC"]
        categorical_features = [3, 4] # ["vcat", "vcat2"]

        classifier = LogisticRegression(C=0.01, class_weight=dict(zip([False, True], [0.2, 0.8])),
                                        n_jobs=1, max_iter=10, solver='lbfgs', tol=1e-3)

        numeric_transformer = Pipeline(steps=[
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
        ])

        categorical_transformer = Pipeline(steps=[
            ('onehot', OneHotEncoder(sparse=True, handle_unknown='ignore')),
            ('tsvd', TruncatedSVD(n_components=1, algorithm='arpack', tol=1e-4))
        ])

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numeric_features),
                ('cat', categorical_transformer, categorical_features)
            ])

        model = Pipeline(steps=[
            ('precprocessor', preprocessor),
            ('classifier', classifier)
        ])

        model.fit(X_train, y_train)
        initial_type = [('numfeat', FloatTensorType([1, 3])),
                        ('strfeat', StringTensorType([1, 2]))]

        X_train = X_train[:11]
        model_onnx = convert_sklearn(model, initial_types=initial_type)

        dump_data_and_model(X_train, model, model_onnx,
                            basename="SklearnPipelineColumnTransformerPipeliner",
                            allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.2')")

        if __name__ == "__main__":
            from onnx.tools.net_drawer import GetPydotGraph, GetOpNodeProducer
            pydot_graph = GetPydotGraph(model_onnx.graph, name=model_onnx.graph.name, rankdir="TP",
                                        node_producer=GetOpNodeProducer("docstring"))
            pydot_graph.write_dot("graph.dot")

            import os
            os.system('dot -O -Tpng graph.dot')

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: wenbingl/sklearn-onnx

max_issues_repo_name: wenbingl/sklearn-onnx

max_forks_repo_name: wenbingl/sklearn-onnx


text:
from hpp.corbaserver.rbprm.hrp2 import Robot
## following info are already accessible once the urdf is loaded, but here we do not load the urdf ...
MASS = 55.88363633
## weight and gains used by TSID

fMin = 1.0                      # minimum normal force
fMax = 1000.                   # maximum normal force
w_com = 1.0           # weight of center of mass task
w_am = 1.
w_posture = 0.1                # weight of joint posture task
w_rootOrientation = 1.       # weight of the root's orientation task
w_forceRef = 1e-6             # weight of force regularization task
w_eff = 1.0                     # weight of the effector motion task
kp_contact = 300.0               # proportional gain of contact constraint
kp_com = 100.                # proportional gain of center of mass task
kp_am = 100.
kp_posture = 1.              # proportional gain of joint posture task
kp_rootOrientation = 500.     # proportional gain of the root's orientation task
kp_Eff = 1000.                   # proportional gain ofthe effectors motion task
level_eff = 0
level_com = 0
level_posture = 1
level_rootOrientation = 1
level_am = 1

IK_dt = 0.005

IK_eff_size = Robot.dict_size.copy()
IK_eff_size[Robot.rfoot] = [0.5,0.8] # diameter of the surface under the flexibilitie
IK_eff_size[Robot.lfoot] = [0.5,0.8]

## Settings for end effectors :
EFF_T_PREDEF = 0.3
EFF_T_DELAY = 0.05
p_max = 0.1

import numpy as np

gain_vector = np.matrix(
    [ 500. , 500.  , #chest
    100.,  100.,#head
    5.,   50.  , 10.,  10.,    10. ,  10. ,10.,  #larm
    5.,   50.  , 10., 10.,    10. ,  10. ,10., #rarm
    10. ,  5.  , 5. , 1. ,  1. ,  10., # lleg  #low gain on axis along y and knee
    10. ,  5.  , 5. , 1. ,  1. ,  10., #rleg
]).T   # gain vector for postural task :

#gain_vector = np.matrix(np.ones(30)).T

masks_posture = np.matrix(np.ones(30)).T
#masks_posture[-12:] = 0 # legID
#print "mask postural task",masks_posture


text:
import datetime as dt
import subprocess
import numpy as np
import os
import glob
import time
import sys

# process files for upload in archive, extracting only fields and subdomains used in paper
# to define domains, start is same as xml domain def, end is start+duration-1
# therefore T-grid N domain is -d x,112,208 -d y,644,773
# example:
# ncks -d x,112,208 -d y,644,773 -v e3t,PAR /results2/SalishSea/nowcast-green.201812/01jan15/SalishSea_1h_20150101_20150101_carp_T.nc ./SalishSea_1h_20150101_20150101_carp_T.nc

maxproc=4
saveloc='/data/eolson/MEOPAR/SS36runs/Archive/'
t0=dt.datetime(2015,1,1)
te=dt.datetime(2017,12,31) # included
fdurHC=1 # length of each results file in days
spath='/results2/SalishSea/nowcast-green.201812/'
fstencilC='{0}/SalishSea_1h_{1}_{1}_carp_T.nc'
ostencilC='SalishSea_1h_{1}_{1}_carp_T.nc'
fmax=maxproc*10
fstencilB='{0}/SalishSea_1h_{1}_{1}_ptrc_T.nc'
ostencilB='SalishSea_1h_{1}_{1}_ptrc_T.nc'
fstencilU='{0}/SalishSea_1h_{1}_{1}_grid_U.nc'
ostencilU='SalishSea_1h_{1}_{1}_grid_U.nc'
varnamesU=('vozocrtx',)
fstencilV='{0}/SalishSea_1h_{1}_{1}_grid_V.nc'
ostencilV='SalishSea_1h_{1}_{1}_grid_V.nc'
varnamesV=('vomecrty',)
fstencilW='{0}/SalishSea_1h_{1}_{1}_grid_W.nc'
ostencilW='SalishSea_1h_{1}_{1}_grid_W.nc'
varnamesW=('vovecrtz',)
fstencilT='{0}/SalishSea_1h_{1}_{1}_grid_T.nc'
ostencilT='SalishSea_1h_{1}_{1}_grid_T.nc'
varnamesT=('vosaline','votemper')
fstencilR='{0}/SalishSea_1h_{1}_{1}_prod_T.nc'
ostencilR='SalishSea_1h_{1}_{1}_prod_T.nc'
varnamesR=('PPDIATNO3','PPPHYNO3','PPMRUBNO3','PPDIAT','PPPHY','PPMRUB')

def listFiles(fdur,fstencil,ostencil):
    ifnames=list()
    ofnames=list()
    ffmt='%Y%m%d'
    dfmt='%d%b%y'
    iits=t0
    while iits<=te:
        iite=iits+dt.timedelta(days=(fdur-1))
        iitn=iits+dt.timedelta(days=fdur)
        try:
            iifstr=glob.glob(spath+fstencil.format(iits.strftime(dfmt).lower(),iits.strftime(ffmt),iite.strftime(ffmt)),recursive=True)[0]
            ifnames.append(iifstr)
            ofnames.append(saveloc+ostencil.format(iits.strftime(dfmt).lower(),iits.strftime(ffmt),iite.strftime(ffmt)))
        except:
            print('file does not exist:  '+spath+fstencil.format(iits.strftime(dfmt).lower(),iits.strftime(ffmt),iite.strftime(ffmt)))
            raise
        iits=iitn
    if not len(ifnames)==len(ofnames):
        print('lists are not equal lenght!')
    return ifnames, ofnames

def pidswait(pids):
    for ipid in pids.keys():
        while pids[ipid].poll() is None:
            time.sleep(10)
    plist=list()
    for ipid in pids.keys():
        for line in pids[ipid].stdout:
            print(line)
        for line in pids[ipid].stderr:
            print(line)
        if pids[ipid].returncode!=0:
            print(pids[ipid].returncode)
        pids[ipid].stdout.close()
        pids[ipid].stderr.close()
        plist.append(ipid)
    for el in plist:
        pids.pop(el)
    return pids

def process_carp_T_ndom(ifnames,ofnames):
    pids=dict()
    ii=0
    for ifname,ofname in zip(ifnames,ofnames):
        pids[ii]=subprocess.Popen('ncks -d x,112,208 -d y,644,773 -v e3t,PAR '+ifname+' '+ofname, shell=True,
                                           stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
        if ii%maxproc==(maxproc-1):
            pids[ii].wait() # wait for the last one in set
        if ii%fmax==(fmax-1):
            pids=pidswait(pids)
        ii+=1
    if len(pids)>0:
        pids=pidswait(pids)
    # check that no others are still running, wait for them
    return pids

def process_ptrc_T_ndom(ifnames,ofnames):
    pids=dict()
    ii=0
    for ifname,ofname in zip(ifnames,ofnames):
        pids[ii]=subprocess.Popen('ncks -d x,112,208 -d y,644,773 -v nitrate,diatoms,flagellates,ciliates '+ifname+' '+ofname, shell=True,
                                           stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
        if ii%maxproc==(maxproc-1):
            pids[ii].wait() # wait for the last one in set
        if ii%fmax==(fmax-1):
            pids=pidswait(pids)
        ii+=1
    if len(pids)>0:
        pids=pidswait(pids)
    # check that no others are still running, wait for them
    return pids

def process_ndom(ifnames,ofnames,varnames):
    pids=dict()
    ii=0
    for ifname,ofname in zip(ifnames,ofnames):
        pids[ii]=subprocess.Popen('ncks -d x,112,208 -d y,644,773 -v '+','.join(varnames)+' '+ifname+' '+ofname, shell=True,
                                           stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
        if ii%maxproc==(maxproc-1):
            pids[ii].wait() # wait for the last one in set
        if ii%fmax==(fmax-1):
            pids=pidswait(pids)
        ii+=1
    if len(pids)>0:
        pids=pidswait(pids)
    # check that no others are still running, wait for them
    return pids

if __name__ == "__main__":
    ## uncomment to process carp_T files:
    #ifnamesC,ofnamesC=listFiles(fdurHC,fstencilC,ostencilC);
    #print('done setup')
    #pids=process_carp_T_ndom(ifnamesC,ofnamesC)
    #print('done')
    ## uncomment to process ptrc_T files:
    #ifnamesB,ofnamesB=listFiles(fdurHC,fstencilB,ostencilB);
    #print('done setup')
    #pids=process_ptrc_T_ndom(ifnamesB,ofnamesB)
    #print('done')
    ## uncomment to process velocity files:
    #ifnamesU,ofnamesU=listFiles(fdurHC,fstencilU,ostencilU);
    #pids=process_ndom(ifnamesU,ofnamesU,varnamesU);
    #print('done U')
    #ifnamesV,ofnamesV=listFiles(fdurHC,fstencilV,ostencilV);
    #pids=process_ndom(ifnamesV,ofnamesV,varnamesV);
    #print('done V')
    #ifnamesW,ofnamesW=listFiles(fdurHC,fstencilW,ostencilW);
    #pids=process_ndom(ifnamesW,ofnamesW,varnamesW);
    #print('done W')
    ifnamesR,ofnamesR=listFiles(fdurHC,fstencilR,ostencilR);
    pids=process_ndom(ifnamesR,ofnamesR,varnamesR);
    print('done R')
    ifnamesT,ofnamesT=listFiles(fdurHC,fstencilT,ostencilT);
    pids=process_ndom(ifnamesT,ofnamesT,varnamesT);
    print('done T')