text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 20 13:46:42 2019

@author: dipu
"""

from PIL import Image
import numpy as np

data_dir = '/mnt/amber/scratch/Dipu/RICO/semantic_annotations/'

#%%
def dcg_at_k(r, k, method=1):
    r = np.asfarray(r)[:k]
    return np.sum(r / np.log2(np.arange(2, r.size + 2)))

def ndcg_at_k(r, k):
    dcg_max = dcg_at_k(sorted(r, reverse=True), k)
    if not dcg_max:
        return 0.
    return dcg_at_k(r, k) / dcg_max
#%% Pixel Accuracy
# Compute the Pixel Accurracy between the query image and retrieved images.
# Two version of the evla metrics:
# 1. Average Pix accuracy: for each  class(component/element) in query, compute the pixAccs and average them
# 2. Weighted Pix accuracy: for each class in query, compute the pixAccs. Computed the weighted mean where weights are proportional to areas covered by the components

def get_overall_PixAcc_ndcg(boundingBoxes,sort_inds,g_fnames,q_fnames, topk = [1,5,10,20,40]):
    n_query = len(q_fnames)
    n_topk = max(topk)

    allClasses = boundingBoxes.getClasses()
    classPixAcc = dict([(key, []) for key in allClasses])

#    aNdcg = np.empty((1,0),float)
#    wNdcg =np.empty((1,0),float)
#
#    aDCG = np.empty((1,0),float)
#    wDCG =np.empty((1,0),float)

    avgPixAccArray = np.zeros((n_query,n_topk))
    weightedPixAccArray = np.zeros((n_query,n_topk))

    for i in range((sort_inds.shape[0])):   #Iterate over all the query images
        qImageName = q_fnames[i]
        q_img  =  Image.open(data_dir+qImageName+'.png').convert('RGB')
        qBBoxes = boundingBoxes.getBoundingBoxesByImageName(qImageName)

        #accList = []
        #weightedAccList = []

        #for j in range(len(g_fnames)):     # Iterate over top-5 retrieved images
        for j in range(n_topk):
            #        print ('\nQuery: ', i, 'Retrieved Image: ', j )

            rImageName = g_fnames[sort_inds[i][j]]
#            r_img =   Image.open(data_dir+rImageName+'.png').convert('RGB')
            rBBoxes = boundingBoxes.getBoundingBoxesByImageName(rImageName)

            tempPixAcc = []
            weights = []
            qClasses = list(set([d.classId for d in qBBoxes]))

            for c in qClasses:
                mask1 = np.zeros(q_img.size, dtype=np.uint8)
                mask2 = np.zeros(q_img.size).astype(np.uint8)

                c_qboxes = [b for b in qBBoxes if b.classId == c]
                c_rboxes = [b for b in rBBoxes if b.classId == c]

                for cqbox in c_qboxes:
                    bb = cqbox.getBoundingBox()
                    mask1[bb[0] : bb[0]+bb[2], bb[1] : bb[1]+bb[3]] = 1

                for crbox in c_rboxes:
                    bb = crbox.getBoundingBox()
                    mask2[bb[0]: bb[0]+ bb[2], bb[1] : bb[1]+bb[3]] = 1

                sum_n_ii = np.sum(np.logical_and(mask1, mask2))
                sum_t_i  = np.sum(mask1)
                if (sum_t_i == 0):
                    pixAcc_c = 0
                else:
                    pixAcc_c = sum_n_ii / sum_t_i

                tempPixAcc.append(pixAcc_c)
                weights.append(sum_t_i)

                # Accumuate the pixel acc for all classes
                classPixAcc[c].append(pixAcc_c)
        #            print('Class:', c, ': ', pixAcc_c)

            current_acc = np.mean(tempPixAcc)

            weightTotal = np.sum(weights)
            weightvalues = np.divide(weights, weightTotal)
            current_weightedAcc = np.sum(tempPixAcc*weightvalues)

            #accList.append(current_acc)
            #weightedAccList.append(current_weightedAcc)

            avgPixAccArray[i][j] = current_acc
            weightedPixAccArray[i][j] = current_weightedAcc

        print('Computing NDCG Pix Acc: {}/{}'.format(i,50))

    aDCG = np.zeros((n_query,n_topk))   # stores avg-DCG for each query for each topk
    wDCG = np.zeros((n_query,n_topk))

    aNdcg = np.zeros((n_query,n_topk))
    wNdcg = np.zeros((n_query,n_topk))

    for k in range(len(topk)):
        for i in range(n_query):
            dcg_t =  dcg_at_k(avgPixAccArray[i][:], k)
            wdcg_t = dcg_at_k(weightedPixAccArray[i][:], k)
            aDCG[i,k] =  dcg_t
            wDCG[i,k] =  wdcg_t

            aGain = ndcg_at_k(avgPixAccArray[i][:], k)
            wGain = ndcg_at_k(weightedPixAccArray[i][:], k)

            aNdcg[i,k] = aGain
            wNdcg[i,k] = wGain

    avg_aDCG = list(np.mean(aDCG,  axis =1))
    avg_wDCG = list(np.mean(wDCG,  axis=1))
    avg_aNdcg = list(np.mean(aNdcg,axis=1))
    avg_wNdcg = list(np.mean(wNdcg,axis=1))

    print('Completed computing NCDG Pix Acc')
    return avg_aDCG, avg_wDCG, avg_aNdcg, avg_wNdcg

max_stars_repo_name: exnx/gcn-triplet-encoding

max_issues_repo_name: exnx/gcn-triplet-encoding

max_forks_repo_name: exnx/gcn-triplet-encoding


text:
#!/user/bin/python
# -*- coding: utf-8 -*-
#
# @ Author: Yao Shuang-Long
# @ Date: 2021/02/27 13:42:59
# @ Summary: the summary.
# @ Contact: xxxxxxxx@email.com
# @ Paper Link:
#
import torch
import numpy as np
import prettytable as pt
from torch._C import device

from tqdm import tqdm
from ..data import build_graph, src_T_dst
from .metrics import mr_score
from .metrics import mrr_score
from .metrics import hits_at_n_score

# calculate the mr, mrr and hits
def cal_mr_mrr_hits(ranks, des='results'):
    mr = mr_score(ranks)
    mrr = mrr_score(ranks)
    hits = [hits_at_n_score(ranks, i) for i in [1, 3, 5, 10, 100]]
    return [des, mr, mrr,] + hits

def _get_rank(current_index, pred_socres, filter_index, num_ent, constraint=None):

    def where(temp, index, break_sign=0):
        _rank, _frank = [], []
        num, j = 0, 0
        for i, x in enumerate(temp):
            if constraint is not None and index[i] not in constraint:
                j += 1
            if x == 1 or x == -1:
                if x == -1:
                    _frank.append(i-num-j+1)
                    _rank.append(i-j+1)
                num += 1
            if num == break_sign:
                break
        return np.array(_rank), np.array(_frank)

    all_labels_ternary = np.zeros(num_ent)
    filter_labels = list(filter_index)
    test_labels = list(current_index)
    all_labels_ternary[filter_labels] += 1
    all_labels_ternary[test_labels] = -1

    idx = np.argsort(pred_socres)[::-1]
    _break_sign = len(filter_labels)
    labels_ord = all_labels_ternary[idx]

    return where(labels_ord, idx, break_sign=_break_sign)

# calculate the ranks
def _calculate_ranks(pred_function, data, unique_entities, batch_size, device,
                     pair_filter, pred_tail=True, constraint=None):

    num_ent = len(unique_entities)
    pred_ents = np.array(unique_entities).reshape((-1, 1))

    graph = build_graph(data)

    num_batchs = (num_ent - 1) // batch_size + 1
    _ranks, _franks = [], []

    try:
        with tqdm(graph['pairs'], ncols=100, leave=False) as tbar:
            scores = np.zeros(num_ent)
            pred_ents = torch.from_numpy(pred_ents).long()

            for pair in tbar:
                scores -= scores
                _pair = torch.from_numpy(pair).long()
                per_pairs = _pair.view(1, -1).repeat(num_ent, 1)

                new_triples = torch.cat((per_pairs, pred_ents), 1)
                if not pred_tail:
                    src, rel, dst = new_triples.transpose(1, 0)
                    new_triples = torch.stack((dst, rel, src), 0).transpose(1, 0)

                for i in range(num_batchs):
                    j = i * batch_size
                    batch_data = new_triples[j: j+batch_size, :].to(device)
                    with torch.no_grad():
                        score = pred_function(batch_data).squeeze_()
                        if not isinstance(score, np.ndarray):
                            if score.is_cuda:
                                score = score.cpu().numpy()
                            else:
                                score = score.numpy()
                        scores[j: j + batch_size] += score

                _rank, _frank = _get_rank(graph['pair->rel_set'][(pair[0], pair[1])],
                                scores, pair_filter[(pair[0], pair[1])], num_ent=num_ent,
                                constraint=None if constraint is None else constraint[pair[1]])
                for i in _rank:
                    _ranks.append(i)
                for i in _frank:
                    _franks.append(i)
                # print(_ranks)
                # print(_franks)
    except KeyboardInterrupt:
        tbar.close()
        raise
    del graph
    return np.array(_ranks), np.array(_franks)

# ########################################
def calculate_ranks(function, dataset, for_test=True, batch_size=None, device='cpu', constraint=None):

    data = dataset.test if for_test else dataset.valid
    unique_entities = dataset.unique_entities
    batch_size = len(unique_entities) if batch_size is None else batch_size
    all_data = dataset.all_triples
    _constraint = None if constraint is None else dataset.load_constraint()
    pair_filter = build_graph(all_data)['pair->rel_set']
    rhs_ranks, rhs_franks = _calculate_ranks(function, data,
                    unique_entities, batch_size, device, pair_filter=pair_filter, pred_tail=True,
                    constraint=None if constraint is None else _constraint['rel_dst'])

    data = src_T_dst(data)
    all_data = src_T_dst(all_data)
    pair_filter = build_graph(all_data)['pair->rel_set']
    lhs_ranks, lhs_franks = _calculate_ranks(function, data,
                    unique_entities, batch_size, device, pair_filter=pair_filter, pred_tail=False,
                    constraint=None if constraint is None else _constraint['rel_src'])

    tb = pt.PrettyTable()
    tb.field_names = [' Evaluation ', 'MR', 'MRR (%)', 'Hits@1 (%)', 'Hits@3 (%)', 'Hits@5 (%)', 'Hits@10 (%)', 'Hits@100 (%)']
    tb.add_row(cal_mr_mrr_hits(lhs_ranks, des='Head'))
    tb.add_row(cal_mr_mrr_hits(rhs_ranks, des='Tail'))

    ranks = np.concatenate((rhs_ranks, lhs_ranks), 0)
    franks = np.concatenate((rhs_franks, lhs_franks), 0)
    tb.add_row(cal_mr_mrr_hits(ranks, des='Average'))
    tb.add_row(['', '', '', '', '', '', '', ''])
    tb.add_row(cal_mr_mrr_hits(lhs_franks, des='HeadFilter'))
    tb.add_row(cal_mr_mrr_hits(rhs_franks, des='TailFilter'))
    tb.add_row(cal_mr_mrr_hits(franks, des='AverageFilter'))
    tb.float_format = "2.4"

    return tb.get_string()

def calculate_n2n_ranks(function, dataset, batch_size=None, device='cpu', constraint=None):
    data = {'1 to 1': dataset.load_1_1(),
            '1 to N': dataset.load_1_n(),
            'N to 1': dataset.load_n_1(),
            'N to N': dataset.load_n_n()}
    unique_entities = dataset.unique_entities
    batch_size = len(unique_entities) if batch_size is None else batch_size
    all_data = dataset.all_triples
    pair_filter = build_graph(all_data)['pair->rel_set']
    _all_data = src_T_dst(all_data)
    _pair_filter = build_graph(_all_data)['pair->rel_set']
    _constraint = None if constraint is None else dataset.load_constraint()

    tb = pt.PrettyTable()
    tb.field_names = [f' {dataset.name} ', ' Evaluation ', 'MR', 'MRR (%)', 'Hits@1 (%)',
                      'Hits@3 (%)', 'Hits@5 (%)', 'Hits@10 (%)', 'Hits@100 (%)']

    for key, test_data in data.items():
        rhs_ranks, rhs_franks = _calculate_ranks(function, test_data, unique_entities,
                                            batch_size, device, pair_filter=pair_filter, pred_tail=True,
                                            constraint=None if constraint is None else _constraint['rel_dst'])
        _test_data = src_T_dst(test_data)
        lhs_ranks, lhs_franks = _calculate_ranks(function, _test_data, unique_entities,
                                            batch_size, device, pair_filter=_pair_filter, pred_tail=False,
                                            constraint=None if constraint is None else _constraint['rel_src'])
        ranks = np.concatenate((rhs_ranks, lhs_ranks), 0)
        franks = np.concatenate((rhs_franks, lhs_franks), 0)

        tb.add_row([f'{key}']+cal_mr_mrr_hits(lhs_ranks, des='Head'))
        tb.add_row([f'{key}']+cal_mr_mrr_hits(rhs_ranks, des='Tail'))
        tb.add_row([f'{key}']+cal_mr_mrr_hits(ranks, des='Average'))
        tb.add_row([f'{key}']+cal_mr_mrr_hits(lhs_franks, des='HeadFilter'))
        tb.add_row([f'{key}']+cal_mr_mrr_hits(rhs_franks, des='TailFilter'))
        tb.add_row([f'{key}']+cal_mr_mrr_hits(franks, des='AverageFilter'))
        tb.add_row(['', '', '', '', '', '', '', '', ''])
    tb.float_format = "2.4"

    return tb.get_string()

ext: py

max_stars_repo_name: vishalbelsare/kgraph

max_issues_repo_name: vishalbelsare/kgraph

max_forks_repo_name: vishalbelsare/kgraph

reason: import numpy


text:
# Import Camera base class
from ..Device import Device
from .PowerMeter import PowerMeter

# Import packages for data analysis
import numpy as np
import pandas as pd
import csv
from itertools import compress

# Import packages for plotting
import matplotlib.pyplot as plt
import seaborn as sns

class Ophir(PowerMeter):
    #   Helper functions for formatting and plotting
    #   data from Ophir power meters

    def __init__(self):
        # Initialize PowerMeter base class
        PowerMeter.__init__(self)

        # # Measured data as a pandas DataFrame
        # self.data = None

    # Load  measurement as a Pandas data frame
    def Load_Data(self, file_path = []):
        """Load power data from StarLab (Ophir) software. Returns the header separated into sections and a Pandas dataframe of power data. File paths must be entered in list format."""

        def Section_Header(header):
            ### Clean up header by sectioning into sublists
            # split header where empty lists occur
            # create new sublist when find empty element. Step when get to First Pulse Arrived string
            bool_list = [];
            sub_list = [];
            sectioned_header = [];
            units_list = [];
            power_meters_list = [];

            for row in header:
                if not row or '---' in row[0]: # if row is empty
                    if sub_list: # if sub_list is not empty append sub_list to sorted_list and reset sub_list
                        sectioned_header.append(sub_list)
                        sub_list = [];
                else:
                    sub_list.append(row[0]); # if row is not empty append to sub_list
                    if 'Units' in row[0]:
                        units_list.append(row[0].split(':')[1])
                    if 'Name' in row[0]:
                        power_meters_list.append(row[0].split(':')[1])

            sectioned_header.append(sub_list) # append last collected sub_list to sorted_list

            return sectioned_header, units_list, power_meters_list

        def Clean_Data(data):
            ### Prepare data to construct dataframe.
            # clean up labels in first row of data.
            stripped_labels = [item.strip() for item in data[0]]
            channel_labels = [item for item in stripped_labels if 'Channel' in item]
            time_labels = ['Timestamp ' + item.split()[1] for item in stripped_labels if 'Channel' in item]
            column_labels = time_labels + channel_labels

            # clean up numerical data in rest of data.
            numeric_data = data[1:]
            power_array = [];
            time_array = [];

            for i in range(len(channel_labels)):
                power_list = [];
                time_list = [];
                for j in range(len(numeric_data[:,i+1])):
                    # try to convert string to float value. If error, then the string is blank space and pass.
                    try:
                        power_list.append(float(numeric_data[j,i+1]))
                        time_list.append(float(numeric_data[j,0]))
                    except:
                        pass

                power_array.append(power_list);
                time_array.append(time_list);

            numerical_data = np.transpose(time_array + power_array)

            return column_labels, numerical_data

        # Override Device.Load_Data() method
        Device.Load_Data(self, file_path)

        ### Load and read csv file.
        file = [];

        # Unpack file path from list - incompatible with multiple file selection via tkinter
        unpacked_file_path = self.current_file_path[0]

        try:
            with open(unpacked_file_path, "r") as csvfile:
                # csvfile.seek(character_number) # reset cursor to specified character of csvfile
                # csvfile.read(character_number) # read to specified character number of csvfile
                myReader = csv.reader(csvfile, delimiter='\t')
                for row in myReader:
                    # file.append(row)
                    file += [row]
        except:
            print('FilePathError: invalid file path')
            self.log.append('FilePathError: invalid file path')

        ### Find beginning of data using 'Timestamp' string to split file into header and data.
        bool_list = [];

        for row in file:
            if not row: # if row is empty
                bool_list.append(False);
            else:
                bool_list.append('Timestamp' in row[0]);

        # compress finds row that contains 'Timestamp' string.
        data_start = list(compress(range(len(bool_list)), bool_list))[0] # Compress boolean array down to indices that contain True values.

        # split file into header and data
        header = file[:data_start];
        data = np.array(file[data_start:])[:,0:3];

        ### Section header
        sectioned_header, units_list, power_meters_list = Section_Header(header);

        ### Prepare data to construct dataframe
        column_labels, numerical_data = Clean_Data(data);

        ### Construct dataframe
        df = pd.DataFrame(columns = column_labels, data = numerical_data)

        self.data = [df];
        self.header = sectioned_header;
        self.power_meters = power_meters_list;
        self.units = units_list;

    def Plot_Power(self):
        """Plot centroid as a function of time"""

        # Check if power data has been properly loaded
        if not self._Is_Data_Loaded(self.data):
            self.Load_Data()

        # Unpack dataframe from data list - incompatible with multiple file selection via tkinter
        df = self.data[0]

        # Set figure format
        sns.set_context('poster',font_scale=1);

        # Get number of power meters
        num = len(self.power_meters);

        # Create new figure and axis object
        fig, ax = plt.subplots(nrows=num,ncols=1,figsize=(16,8), sharex=True, squeeze = False);

        # Plot data on axis object for each power meter
        for i in range(num):
            channel_label = chr(ord('A')+i);
            ax[i, 0].plot(df['Timestamp '+channel_label], df['Channel '+channel_label], label = self.power_meters[i]);
            ax[i, 0].set_ylabel('Power ['+self.units[i]+']'); # Set default y label
            ax[i, 0].grid(alpha = 0.5); # Turn grid on for axis
            ax[i, 0].legend(loc = 'best'); # Create legend for axis containing power meter name. Set its location to best.

            # Record index of last iteration
            last = i;

        # Set x label of last axis object to default label
        ax[last, 0].set_xlabel('Time [s]');

        # Call plot tight layout
        plt.tight_layout()

        return ax

max_stars_repo_name: UltrafastCornell/Devices

max_issues_repo_name: UltrafastCornell/Devices

max_forks_repo_name: UltrafastCornell/Devices


text:
#NAME:
#  powerspec2d
#PURPOSE:
#  Compute power spectrum of a 2D array by dividing into many overlapping sub-images
#  and averaging the power spectra. This improves the estimate at the expense of
#  spectral resolution. The normalization convention is the same as with
#  periodogram_fft.pro. This is a 2D analog of powerspec.pro, which uses Welch's
#  method with a Hanning window. Any sub-image containing NaN is ignored by
#  virtue of the /NAN keyword to mean() in periodogram_fft.pro.
#CALLING SEQUENCE:
#  result = powerspec2d(data [, Nfreq])
#INPUT PARAMETERS:
#  data = 2d array.
#  Nfreq = Size of square sub-image to analyze (optional). Default=16, min=4.
#OPTIONAL KEYWORD INPUTS:
#  dx = sample interval. Default=1. If included, then k will be converted into units
#     of cycles (or radians, if that keyword is set) per unit distance (or time),
#     corresponding to the units of dx.
#  dy = sample interval along the vertical direction (i.e., along the second
#     array index). Default: dy=dx.
#  radians = if set, then let k be in radians (rather than cycles).
#  retain_dc = if set, then retain the DC offset when calculating the power spectrum.
#     Ordinarily, the DC offset is subtracted before calculating the spectrum, because
#     the Hanning window tends to make the offset bleed into the nearest neighbors of
#     pixel [0,0] of the 2D power spectrum.
#OPTIONAL KEYWORD OUTPUTS:
#  kx = Nx-dimensional array of horizontal wavenumbers
#  ky = Ny-dimensional array of vertical wavenumbers
#  big_kx, big_ky = Nx x Ny arrays of horizontal and vertical wavenumbers, respectively.
#     This is handy for constructing digital filters as a function of the vector
#     wavenumber, to be applied to the FFT of an image.
#  k_magnitude = 2d array of wavenumber magnitudes.
#HISTORY:
#  2013-Aug-20 CCK
#  2013-Aug-21 CCK fixed dc offset issue, and added retain_dc keyword.
#  2015-Feb-21 CCK added /NaN in calls to mean(). I have verified that
#     the normalization is not affected by this change.
#  2019 Jun 25 JTE translated into Python 3

import numpy as np

def powerspec2d(data,
                Nfreq = 16,
                retain_dc = False,
                dx = dx,
                dy = dy,
                kx = kx,
                ky = ky,
                big_kx = big_kx,
                big_ky = big_ky,
                radians = radians):

    Nx, Ny = data.shape()

    Nfreq = round(Nfreq[0]) > 4 #Don't allow non-integer or smaller than 4.

    NsegX = np.ceil(np.divide(2.0 * Nx, Nfreq)) - 1.

    NsegY = np.ceil(np.divide(2.0 * Ny, Nfreq)) - 1.
       #How many segments  of length Nfreq fit in the interval, if we overlap them like
       #2 courses of bricks?

    if (NsegX < 2 | NsegY < 2):
        print('Data array too short to compute power spectrum using specified segment size.')

    xsep = np.divide(Nx - Nfreq, NsegX - 1) * (1 - 1e-6)
    ysep = np.divide(Ny - Nfreq, NsegY - 1) * (1 - 1e-6)
       #The last factor is to ensure we can fit Nseg segments within the data array.

    spectra = np.full([NsegX, NsegY], Nfreq)

    for i in range(NsegX):

       x0 = np.round(i * xsep)   #lower x index of the segment

       x1 = x0 + Nfreq - 1  #upper x index of the segment

       for j in range(NsegY):

          y0 = np.round(j * ysep)   #lower y index of the segment

          y1 = y0 + Nfreq - 1  #upper y index of the segment

          segment = data[x0:x1, y0:y1]

          spectra[i, j] = periodogram_fft(segment,
                                          retain_dc = retain_dc)

    power = np.nanmean(np.nanmean(spectra, axis = 1), axis = 1)

    #pick off leading dimension twice!

    k_magnitude = k_arr2d(Nfreq,
                          Nfreq,
                          dx =dx,
                          dy =dy,
                          radians =radians,
                          kx =kx,
                          ky =ky,
                          big_kx =big_kx,
                          big_ky =big_ky)

    return power #, k_magnitude

max_stars_repo_name: Jon-Eckberg/ffopy

max_issues_repo_name: Jon-Eckberg/ffopy

max_forks_repo_name: Jon-Eckberg/ffopy


text:
from .single_stage import SingleStageDetector
from ..registry import DETECTORS
from mmdet.core import bbox2result

import numpy as np
import cv2

@DETECTORS.register_module
class CSP(SingleStageDetector):

    def __init__(self,
                 backbone,
                 neck,
                 bbox_head,
                 train_cfg=None,
                 test_cfg=None,
                 pretrained=None,
                 return_feature_maps=False):
        super(CSP, self).__init__(backbone, neck, bbox_head, train_cfg,
                                   test_cfg, pretrained)
        self.return_feature_maps = return_feature_maps
    def show_input_debug(self, img, classification_maps, scale_maps, offset_maps):
        img_numpy = img.cpu().numpy().copy()[0]
        # img_numpy = np.transpose(img_numpy, [1, 2, 0]) * [58.395, 57.12, 57.375] + [123.675, 116.28, 103.53]
        img_numpy = np.transpose(img_numpy, [1, 2, 0]) + [102.9801, 115.9465, 122.7717]
        img_numpy = img_numpy[:, :, ::-1]
        img_numpy = img_numpy.astype(np.uint8)
        strides = [8, 16, 32, 64, 128]
        img_nows = []
        for i, stride in enumerate(strides):
            img_now = img_numpy.copy()
            # cls_numpy = classification_maps[0][i].cpu().numpy().copy()[0][2]
            cls_numpy = classification_maps[0][i].cpu().numpy().copy()[0][:80]
            scale_numpy = scale_maps[0][i].cpu().numpy().copy()[0][0] * stride
            offset_numpy = offset_maps[0][i].cpu().numpy().copy()[0][:2]
            cs, ys, xs = cls_numpy.nonzero()
            print(len(ys))
            for c, x, y in zip(cs, xs, ys):
                cv2.imshow(str(c), classification_maps[0][i].cpu().numpy().copy()[0][80+c])
                realx = x
                realy = y
                height = scale_numpy[y, x]
                realy = realy + 0.5 + offset_numpy[0][y, x]
                realx = realx + 0.5 + offset_numpy[1][y, x]
                realy = realy * stride
                realx = realx * stride
                top_y = int(realy - height/2)
                top_x = int(realx)
                down_y = int(realy + height/2)
                down_x = int(realx)
                top_left = (int(top_x - height * 0.1), int(top_y))
                down_right = (int(down_x + height * 0.1), down_y)
                cv2.rectangle(img_now, top_left, down_right, (255, 255, 5*int(c)), 2)
                img_nows.append(img_now)
            cv2.imshow(str(i) +'img', img_now)
        cv2.waitKey(0)

    def show_input_debug_caltech(self, img, classification_maps, scale_maps, offset_maps):
        for j in range(img.shape[0]):
            img_numpy = img.cpu().numpy().copy()[j]
            img_numpy = np.transpose(img_numpy, [1, 2, 0]) * [58.395, 57.12, 57.375] + [123.675, 116.28, 103.53]
            img_numpy = img_numpy[:, :, ::-1]
            img_numpy = img_numpy.astype(np.uint8)
            strides = [4]
            img_nows = []
            for i, stride in enumerate(strides):
                img_now = img_numpy.copy()
                cls_numpy = classification_maps[j][i].cpu().numpy().copy()[0][2]
                ignore_numpy = classification_maps[j][i].cpu().numpy().copy()[0][1]
                cv2.imshow('ignore', ignore_numpy)
                scale_numpy = scale_maps[j][i].cpu().numpy().copy()[0][0] * stride
                offset_numpy = offset_maps[j][i].cpu().numpy().copy()[0][:2]
                ys, xs = cls_numpy.nonzero()
                print(len(ys))
                for x, y in zip(xs, ys):
                    # cv2.imshow(str(c), classification_maps[j][i].cpu().numpy().copy()[0][c])
                    realx = x
                    realy = y
                    height = scale_numpy[y, x]
                    realy = realy + 0.5 + offset_numpy[0][y, x]
                    realx = realx + 0.5 + offset_numpy[1][y, x]
                    realy = realy * stride
                    realx = realx * stride
                    top_y = int(realy - height/2)
                    top_x = int(realx)
                    down_y = int(realy + height/2)
                    down_x = int(realx)
                    top_left = (int(top_x - height * 0.1), int(top_y))
                    down_right = (int(down_x + height * 0.1), down_y)
                    cv2.rectangle(img_now, top_left, down_right, (255, 255, 125), 2)
                    img_nows.append(img_now)
                cv2.imshow(str(i) +'img', img_now)
            cv2.waitKey(0)

    def show_input_debug_head(self, img, classification_maps, scale_maps, offset_maps):
        for j in range(img.shape[0]):
            img_numpy = img.cpu().numpy().copy()[j]
            img_numpy = np.transpose(img_numpy, [1, 2, 0]) * [58.395, 57.12, 57.375] + [123.675, 116.28, 103.53]
            img_numpy = img_numpy[:, :, ::-1]
            img_numpy = img_numpy.astype(np.uint8)
            strides = [4]
            img_nows = []
            for i, stride in enumerate(strides):
                img_now = img_numpy.copy()
                cls_numpy = classification_maps[j][i].cpu().numpy().copy()[0][2]
                ignore_numpy = classification_maps[j][i].cpu().numpy().copy()[0][1]
                cv2.imshow('ignore', ignore_numpy)
                scale_numpy = scale_maps[j][i].exp().cpu().numpy().copy()[0][0] * stride
                offset_numpy = offset_maps[j][i].cpu().numpy().copy()[0][:2]
                ys, xs = cls_numpy.nonzero()
                for x, y in zip(xs, ys):
                    # cv2.imshow(str(c), classification_maps[j][i].cpu().numpy().copy()[0][c])
                    realx = x
                    realy = y
                    height = scale_numpy[y, x]
                    realy = realy + 0.5 + offset_numpy[0][y, x]
                    realx = realx + 0.5 + offset_numpy[1][y, x]
                    realy = realy * stride
                    realx = realx * stride
                    top_y = int(realy)
                    top_x = int(realx)
                    down_y = int(realy + height)
                    down_x = int(realx)
                    top_left = (int(top_x - height * 0.41/2), int(top_y))
                    down_right = (int(down_x + height * 0.41/2), down_y)
                    cv2.rectangle(img_now, top_left, down_right, (255, 255, 125), 2)
                    img_nows.append(img_now)
                cv2.imshow(str(i) +'img', img_now)
            cv2.waitKey(0)

    def show_mot_input_debug(self, img, classification_maps, scale_maps, offset_maps):
        for j in range(img.shape[0]):
            img_numpy = img.cpu().numpy().copy()[j]
            img_numpy = np.transpose(img_numpy, [1, 2, 0]) * [58.395, 57.12, 57.375] + [123.675, 116.28, 103.53]
            # img_numpy = np.transpose(img_numpy, [1, 2, 0]) + [102.9801, 115.9465, 122.7717]
            img_numpy = img_numpy[:, :, ::-1]
            img_numpy = img_numpy.astype(np.uint8)
            strides = [4]
            img_nows = []
            for i, stride in enumerate(strides):
                img_now = img_numpy.copy()
                # cls_numpy = classification_maps[0][i].cpu().numpy().copy()[0][2]
                cls_numpy = classification_maps[j][i].cpu().numpy().copy()[0][2]
                instance_numpy = classification_maps[j][i].cpu().numpy().copy()[0][3]
                scale_numpy = scale_maps[j][i].cpu().numpy().copy()[0][0] * stride
                offset_numpy = offset_maps[j][i].cpu().numpy().copy()[0][:2]
                ys, xs = cls_numpy.nonzero()
                for x, y in zip(xs, ys):
                    c=0
                    cv2.imshow(str(c), classification_maps[j][i].cpu().numpy().copy()[0][2])
                    realx = x
                    realy = y
                    height = scale_numpy[y, x]
                    realy = realy + 0.5 + offset_numpy[0][y, x]
                    realx = realx + 0.5 + offset_numpy[1][y, x]
                    realy = realy * stride
                    realx = realx * stride
                    top_y = int(realy - height/2)
                    top_x = int(realx)
                    down_y = int(realy + height/2)
                    down_x = int(realx)
                    top_left = (int(top_x - height * 0.1), int(top_y))
                    down_right = (int(down_x + height * 0.1), down_y)
                    cv2.rectangle(img_now, top_left, down_right, (255, 255, 5*int(c)), 2)
                    instance = instance_numpy[y, x]
                    cv2.putText(img_now, str(instance), top_left, cv2.FONT_HERSHEY_COMPLEX, 1, 255)
                    img_nows.append(img_now)
                cv2.imshow(str(i) +'img', img_now)
            cv2.waitKey(0)

    def forward_train(self,
                      img,
                      img_metas,
                      gt_bboxes,
                      gt_labels,
                      gt_bboxes_ignore=None,
                      classification_maps=None,
                      scale_maps=None,
                      offset_maps=None):
        # for tracking data which batch is produced by dataset instead of data loader
        if type(img) == list:
            img=img[0]
            img_metas=img_metas[0]
            gt_bboxes=gt_bboxes[0]
            gt_labels=gt_labels[0]
            gt_bboxes_ignore = gt_bboxes_ignore[0]
            classification_maps = classification_maps[0]
            scale_maps = scale_maps[0]
            offset_maps = offset_maps[0]

        x = self.extract_feat(img)
        # self.show_input_debug(img, classification_maps, scale_maps, offset_maps)
        # self.show_input_debug_caltech(img, classification_maps, scale_maps, offset_maps)
        # self.show_mot_input_debug(img, classification_maps, scale_maps, offset_maps)
        # self.show_input_debug_head(img, classification_maps, scale_maps, offset_maps)
        outs = self.bbox_head(x)
        loss_inputs = outs + (gt_bboxes, gt_labels, classification_maps, scale_maps, offset_maps, img_metas, self.train_cfg)
        losses = self.bbox_head.loss(
            *loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore)
        return losses

    def simple_test(self, img, img_meta, rescale=False):
        x = self.extract_feat(img)
        outs = self.bbox_head(x)
        bbox_inputs = outs + (img_meta, self.test_cfg, rescale)
        if self.return_feature_maps:
            return self.bbox_head.get_bboxes_features(*bbox_inputs)
        bbox_list = self.bbox_head.get_bboxes(*bbox_inputs)
        bbox_results = [
            bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes)
            for det_bboxes, det_labels in bbox_list
        ]
        return bbox_results[0]

    def foward_features(self, features):
        bbox_list = self.bbox_head.get_bboxes(*features)
        bbox_results = [
            bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes)
            for det_bboxes, det_labels in bbox_list
        ]
        return bbox_results[0]

max_stars_repo_name: Ernstsen/Pedestron

max_issues_repo_name: Ernstsen/Pedestron

max_forks_repo_name: Ernstsen/Pedestron


text:
import pandas as pd
import os
import cv2
import numpy as np

# 1.解析数据
# 2.建立训练集测试集数据按7/3的比例

class Data_annotation:
    def __init__(self, root):
        self.root = root

    def get_faces_keypoints(self):

        FOLDER = ['I', 'II']
        tmp_lines = []
        DATA_info = {'path': [], 'face_rect': [], 'face_keypoints': []}

        for f in FOLDER:
            DATA_DIR = os.path.join(self.root, f)
            file_path = os.path.join(DATA_DIR, 'label.txt')
            with open(file_path) as file:
                lines = file.readlines()
            # map(lambda x, y: x + y, [1, 3, 5, 7, 9], [2, 4, 6, 8, 10])

            tmp_lines.extend(list(map(lambda x: os.path.join(DATA_DIR, x), lines)))

        for file in tmp_lines:

            file = file.strip().split()
            try:
                cv2.imdecode(np.fromfile(file[0], dtype=np.uint8), cv2.IMREAD_COLOR)
            except OSError:
                pass
            else:
                DATA_info['path'].append(file[0])
                DATA_info['face_rect'].append(list(map(float, file[1:5])))
                DATA_info['face_keypoints'].append(list(map(float, file[5:])))

        ANNOTATION = pd.DataFrame(DATA_info)
        ANNOTATION.to_csv('face_keypoints_annotation.csv')
        print('face_keypoints_annotation file is saved.')

    def get_train_val_data(self):
        FILE = 'face_keypoints_annotation.csv'
        DATA_info = pd.read_csv(FILE)
        DATA_info_anno = {'path': [], 'rect': [], 'points': []}

        expand_ratio = 0.2
        self.get_valid_data(DATA_info, DATA_info_anno, expand_ratio)
        data = pd.DataFrame(DATA_info_anno)
        # numpy choice, sample
        train_data = data.sample(frac=0.8, replace=False)

        val_data = data[data['path'].isin(list(set(data['path'])-set(train_data['path'])))]

        data.to_csv('train_data.csv')
        data.to_csv('val_data.csv')
        print("train_data:{:d}".format(train_data.shape[0]))
        print("val_data:{:d}".format(val_data.shape[0]))

    def get_valid_data(self, DATA_info, DATA_info_anno, expand_ratio=0.25):
        def expand_roi(rect , expand_ratio):
            left, top, right, bottom = rect
            rect_width = right-left
            rect_height = bottom-top
            left, top, right, bottom = left - expand_ratio * rect_width, \
                                       top - expand_ratio * rect_height,\
                                       right + expand_ratio * rect_width, \
                                       bottom + expand_ratio * rect_height
            return [left, top, right, bottom]

        for index, row in DATA_info.iterrows():
            is_invalid_sample = False
            img = cv2.imdecode(np.fromfile(row.loc['path'], dtype=np.uint8), cv2.IMREAD_COLOR)
            height, width, _ = img.shape
            # "[229.0, 38.0, 289.0, 99.0]"
            rect = list(map(lambda x: float(x), eval(row.loc['face_rect'])))
            rect = expand_roi(rect, expand_ratio)
            points = list(map(float, eval(row.loc['face_keypoints'])))
            x = points[0::2]
            y = points[1::2]
            points_zip = list(zip(x, y))
            # 处理Rect不超出图像边界
            # [left, top, right, bottom]
            rect_dstxy = [0 if i < 0 else i for i in rect]
            rect_dstx = [width if rect_dstxy[i] > width else rect_dstxy[i] for i in [0, 2]]
            rect_dsty = [height if rect_dstxy[i] > height else rect_dstxy[i] for i in [1, 3]]
            rect_dstx.extend(rect_dsty)
            # [left, right, top, bottom]
            rect = [rect_dstx[i] for i in [0, 2, 1, 3]]
            # 处理Points不超出Rect边界,如果超出则舍去该样本
            left, top, right, bottom = rect
            for point in points_zip:
                x, y = point  # (x,y)
                if x < left or x > right or y < top or y > bottom:
                    is_invalid_sample = True
                    print("{:s}:Points is out of rect boundary".format(row.loc['path']))
                    break
            if is_invalid_sample:
                continue
            DATA_info_anno['path'].append(row.loc['path'])
            DATA_info_anno['rect'].append(rect)
            DATA_info_anno['points'].append(points_zip)

if __name__ == '__main__':
    ROOTS = r'F:\DataSet\慧科\人脸关键点检测'
    data_anno = Data_annotation(ROOTS)
    data_anno.get_faces_keypoints()
    data_anno.get_train_val_data()

max_issues_repo_name: KyanChen/FaceKeypointsDetection

max_forks_repo_name: KyanChen/FaceKeypointsDetection


text:
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import torch

import torch.optim as optim

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(4, 32, 3, padding=1)
        self.conv2 = nn.Conv2d(32, 64, 3, padding=1)
        self.conv3 = nn.Conv2d(64, 128, 3, padding=1)

        self.p_fc1 = nn.Conv2d(128, 4, 1)
        self.p_fc2 = nn.Linear(4 * 8 * 8, 64)

        self.v_fc1 = nn.Conv2d(128, 2, 1)
        self.v_fc2 = nn.Linear(2 * 8 * 8, 1)

    def forward(self, x):
        a1 = F.relu(self.conv1(x))
        a2 = F.relu(self.conv2(a1))
        a3 = F.relu(self.conv3(a2))

        p1 = F.relu(self.p_fc1(a3))
        p_act = p1.view(-1, 4 * 8 * 8)

        p_out = F.softmax(self.p_fc2(p_act), dim=0)

        v1 = F.relu(self.v_fc1(a3))
        v_act = v1.view(-1, 2 * 8 * 8)

        v_out = torch.tanh(self.v_fc2(v_act))

        return p_out, v_out

class PolicyValueNet():
    def __init__(self, board_width=8, board_height=8, model_file=None):

        self.board_width = board_width
        self.board_height = board_height
        self.model = Net()
        self.v_loss = nn.MSELoss()
        # self.p_loss=nn.CrossEntropyLoss()
        self.optimizer = optim.Adam(self.model.parameters(), lr=0.01)
        if model_file:
            self.model.load_state_dict(torch.load(model_file))

    def policy_value_fn(self, board):
        legal_positions = board.available
        current_state = board.current_state()
        x = torch.FloatTensor(
            current_state.reshape(-1, 4, self.board_width, self.board_height))

        probs, value = self.model.forward(x)
        probs = probs.data.numpy()
        value = value.data.numpy()

        probs = zip(legal_positions, probs.flatten()[legal_positions])

        return probs, value

    def policy_value(self, x):
        x = torch.FloatTensor(np.reshape(
            x, (-1, 4, self.board_width, self.board_height)))
        probs, value = self.model.forward(x)
        probs = probs.data.numpy()
        value = value.data.numpy()
        return probs, value

    def train_step(self, state_batch, probs_batch, winner_batch, lr):

        state_batch = torch.FloatTensor(state_batch)
        probs_batch = torch.LongTensor(probs_batch)
        winner_batch = torch.FloatTensor(winner_batch)

        self.optimizer.zero_grad()
        p_out, v_out = self.model(state_batch)
        p_loss = -torch.mean(torch.sum(probs_batch * torch.log(p_out), 1))
        v_loss = self.v_loss(v_out, winner_batch)
        loss = p_loss + v_loss
        loss.backward()
        for param_group in self.optimizer.param_groups:
            param_group['lr'] = lr
        self.optimizer.step()

        return p_loss.item(), v_loss.item()

    def save_model(self, dir):
        torch.save(self.model.state_dict(), dir)

if __name__ == '__main__':
    net = PolicyValueNet()
    net.save_model("./testmodel")

max_stars_repo_name: LouisCaixuran/gomoku

max_issues_repo_name: LouisCaixuran/gomoku

max_forks_repo_name: LouisCaixuran/gomoku


text:
#!/usr/bin/python
# Copyright 2020 Makani Technologies LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""Generates a hover database for a specific apparent wind speed.

Example:

  ./generate_hover_database.py \
      --filename="output.dat" \
      --apparent_wind_speed=5.0 \
      --alphas_deg="-40.0, 100.0, 15" \
      --betas_deg="-60.0, 60.0, 7"
"""

import collections
import json
import logging
import multiprocessing
import sys
import gflags
from makani.analysis.aero import airfoil
from makani.analysis.aero.hover_model import hover_model
from makani.lib.python import dict_util
from makani.lib.python import flag_types
import numpy

FLAGS = gflags.FLAGS

gflags.DEFINE_float('apparent_wind_speed', '5.0', 'Apparent wind speed [m/s].')
gflags.DEFINE_boolean('use_wake_model', True,
                      'Boolean that determines whether the advected rotor wake '
                      'is included with the local apparent wind.')
gflags.DEFINE_string('filename', 'output.json', 'Output filename of database.')
gflags.DEFINE_integer('num_workers', 5,
                      'Number of processes to run in parallel.')
gflags.DEFINE_string('wing_serial', '01',
                     'Wing serial to generate a database for.')
gflags.DEFINE_string('alpha_intervals_deg', '-180.0,-30.0,-9.0,9.0,30.0,180.0',
                     'Comma separated list of angle-of-attack values [deg], '
                     'indicating the start and end points of multiple '
                     'linspaces.')
gflags.DEFINE_string('num_alphas', '8,5,10,5,8',
                     'Number of angle-of-attack points to place in between '
                     'each of the start / end pairs in alpha_intervals_deg.')
gflags.DEFINE_string('beta_intervals_deg', '-60.0,-15.0,15.0,60.0',
                     'Comma separated list of side-slip angles [deg], '
                     'indicating the start and end points of multiple '
                     'linspaces.')
gflags.DEFINE_string('num_betas', '4,4,4',
                     'Number of side-slip angle points to place in between '
                     'each of the start / end pairs in beta_intervals_deg.')
flag_types.DEFINE_linspace('delta1s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta1 angles [deg].')
flag_types.DEFINE_linspace('delta2s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta2 angles [deg].')
flag_types.DEFINE_linspace('delta3s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta3 angles [deg].')
flag_types.DEFINE_linspace('delta4s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta4 angles [deg].')
flag_types.DEFINE_linspace('delta5s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta5 angles [deg].')
flag_types.DEFINE_linspace('delta6s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta6 angles [deg].')
flag_types.DEFINE_linspace('delta7s_deg', '-110.0, 10.0, 24',
                           'Linspace range of delta7 angles [deg].')
flag_types.DEFINE_linspace('delta8s_deg', '-25.0, 25.0, 5',
                           'Linspace range of delta8 angles [deg].')

# Number of flaps to use in the hover database.
_NUM_DELTAS = 8

def _VariableLinspaceFromStrings(breaks_string, counts_string):
  """Construct a variable density increasing array from command line flags.

  For example:

    _VariableLinspaceFromString('0,2,8', '1,3')

  will return the same value as:

    np.array([0.0, 1.0, 2.0, 3.5, 5.0, 6.5, 8.0])

  Args:
    breaks_string: A string giving a comma separated list of n + 1 float values
        in increasing order.
    counts_string: A string giving a list of n non-negative integers prescribing
        how many values to place in between the break points.

  Raises:
    ValueError: if the values encoded by the string are invalid.

  Returns:
    A numpy array of values as described above.
  """
  try:
    counts = [int(c) for c in counts_string.split(',')]
    breaks = [float(r) for r in breaks_string.split(',')]
  except ValueError:
    raise ValueError('Bad format for range argument.')

  if len(counts) != len(breaks) - 1:
    raise ValueError('Inconsistent lengths for range arguments.')

  total_range = []
  for i in range(len(counts)):
    first_range = numpy.linspace(breaks[i], breaks[i + 1], 2 + counts[i])
    if i > 0:
      first_range = first_range[1:]
    total_range += first_range.tolist()

  return numpy.array(total_range)

def CalcZeroDeflectionCoeffs(alpha_list, beta_list,
                             nominal_local_apparent_winds_sph,
                             params):
  """Calculates the coefficients at zero flap deflection."""
  alphas, betas = numpy.meshgrid(alpha_list, beta_list, indexing='ij')
  zeros = numpy.zeros((_NUM_DELTAS,) + numpy.shape(alphas))
  return hover_model.CalcForceMomentCoeffs(
      alphas, betas, zeros, [0.0, 0.0, 0.0], FLAGS.apparent_wind_speed, params,
      local_apparent_winds_sph=nominal_local_apparent_winds_sph)

def CalcDeltaIncrementCoeffs(alpha_list, beta_list, delta_list, delta_index,
                             nominal_coeffs, nominal_local_apparent_winds_sph,
                             params):
  """Calculates coefficient increments for each flap."""
  alphas, betas, deltas_i = numpy.meshgrid(alpha_list, beta_list, delta_list,
                                           indexing='ij')
  deltas = numpy.zeros((_NUM_DELTAS,) + numpy.shape(alphas))
  deltas[delta_index] = deltas_i
  coeffs = hover_model.CalcForceMomentCoeffs(
      alphas, betas, deltas, [0.0, 0.0, 0.0], FLAGS.apparent_wind_speed, params,
      local_apparent_winds_sph=nominal_local_apparent_winds_sph)
  coeffs_0 = numpy.tile(numpy.expand_dims(nominal_coeffs, axis=2),
                        (1, 1, len(delta_list), 1))
  return coeffs - coeffs_0

def CalcAngularRateDerivativeCoeffs(alpha_list, beta_list, omega_hat_step,
                                    positive_rate_local_apparent_winds_sph,
                                    negative_rate_local_apparent_winds_sph,
                                    params):
  """Calculates angular rate derivatives in the nominal flap position."""
  omega_hat_step = numpy.array(omega_hat_step)
  alphas, betas = numpy.meshgrid(alpha_list, beta_list, indexing='ij')
  zeros = numpy.zeros((_NUM_DELTAS,) + numpy.shape(alphas))
  coeffs_p = hover_model.CalcForceMomentCoeffs(
      alphas, betas, zeros, omega_hat_step / 2.0, FLAGS.apparent_wind_speed,
      params, local_apparent_winds_sph=positive_rate_local_apparent_winds_sph)
  coeffs_n = hover_model.CalcForceMomentCoeffs(
      alphas, betas, zeros, -omega_hat_step / 2.0, FLAGS.apparent_wind_speed,
      params, local_apparent_winds_sph=negative_rate_local_apparent_winds_sph)
  return (coeffs_p - coeffs_n) / numpy.linalg.norm(omega_hat_step)

def CalcDeltaIncrementAngularRateDerivativeCoeffs(
    alpha_list, beta_list, delta_list, delta_index, omega_hat_step,
    nominal_derivative_coeffs,
    positive_rate_local_apparent_winds_sph,
    negative_rate_local_apparent_winds_sph, params):
  """Calculates angular rate derivative increments for each flap."""
  omega_hat_step = numpy.array(omega_hat_step)
  alphas, betas, deltas_i = numpy.meshgrid(alpha_list, beta_list, delta_list,
                                           indexing='ij')
  deltas = numpy.zeros((_NUM_DELTAS,) + numpy.shape(alphas))
  deltas[delta_index] = deltas_i

  coeffs_p = hover_model.CalcForceMomentCoeffs(
      alphas, betas, deltas, omega_hat_step / 2.0, FLAGS.apparent_wind_speed,
      params,
      local_apparent_winds_sph=positive_rate_local_apparent_winds_sph)
  coeffs_n = hover_model.CalcForceMomentCoeffs(
      alphas, betas, deltas, -omega_hat_step / 2.0, FLAGS.apparent_wind_speed,
      params,
      local_apparent_winds_sph=negative_rate_local_apparent_winds_sph)

  coeffs_0 = numpy.tile(numpy.expand_dims(nominal_derivative_coeffs, axis=2),
                        (1, 1, len(delta_list), 1))
  return (coeffs_p - coeffs_n) / numpy.linalg.norm(omega_hat_step) - coeffs_0

def GenerateHoverDatabase():
  """Generates a hover aerodynamics database in the DVL format.

  A hover aerodynamic database models all the aerodynamic surfaces as
  independent airfoils and accounts for the effect of the propwash on
  these surfaces.  The database is output in the DVL format which
  includes the following coefficients and derivatives:

    cx, cy, cz, cl, cm, cn,
    dcx1, dcy1, dcz1, dcl1, dcm1, dcn1,
    ...
    dcx8, dcy8, dcz8, dcl8, dcm8, dcn8,
    dcx/dp, dcy/dp, dcz/dp, dcl/dp, dcm/dp, dcn/dp
    dcx/dq, dcy/dq, dcz/dq, dcl/dq, dcm/dq, dcn/dq
    dcx/dr, dcy/dr, dcz/dr, dcl/dr, dcm/dr, dcn/dr
    dcx1/dp, dcy1/dp, dcz1/dp, dcl1/dp, dcm1/dp, dcn1/dp
    dcx1/dq, dcy1/dq, dcz1/dq, dcl1/dq, dcm1/dq, dcn1/dq
    dcx1/dr, dcy1/dr, dcz1/dr, dcl1/dr, dcm1/dr, dcn1/dr
    ...

  """
  alpha_list = _VariableLinspaceFromStrings(
      FLAGS.alpha_intervals_deg, FLAGS.num_alphas) * numpy.pi / 180.0
  beta_list = _VariableLinspaceFromStrings(
      FLAGS.beta_intervals_deg, FLAGS.num_betas) * numpy.pi / 180.0
  delta1s = numpy.pi / 180.0 * FLAGS.delta1s_deg
  delta2s = numpy.pi / 180.0 * FLAGS.delta2s_deg
  delta3s = numpy.pi / 180.0 * FLAGS.delta3s_deg
  delta4s = numpy.pi / 180.0 * FLAGS.delta4s_deg
  delta5s = numpy.pi / 180.0 * FLAGS.delta5s_deg
  delta6s = numpy.pi / 180.0 * FLAGS.delta6s_deg
  delta7s = numpy.pi / 180.0 * FLAGS.delta7s_deg
  delta8s = numpy.pi / 180.0 * FLAGS.delta8s_deg

  logging.info('Building wing parameters and airfoils for %s:%s.',
               FLAGS.wing_model, FLAGS.wing_serial)
  params = hover_model.GetParams(FLAGS.wing_model, FLAGS.wing_serial,
                                 use_wake_model=FLAGS.use_wake_model)

  def _ProcessTasks(tasks, group_name=''):
    """Iterates through a dictionary of tasks executing them in parallel."""
    def _CreateTaskTarget(func):
      def _WrapFun(args, pipe):
        pipe.put(func(*args))
        pipe.close()
      return _WrapFun

    queues = [multiprocessing.Queue() for _ in tasks]
    task_names = []
    processes = []
    for i, (name, (func, args)) in enumerate(tasks.iteritems()):
      task_names += [name]
      processes.append(multiprocessing.Process(
          target=_CreateTaskTarget(func), args=(args, queues[i])))

    pending = [True for _ in range(len(task_names))]
    running = [False for _ in range(len(task_names))]
    results = {
        task_name: None for task_name in task_names
    }
    while numpy.sum(running) > 0 or numpy.sum(pending) > 0:
      # Fill the running queue.
      while numpy.sum(running) < FLAGS.num_workers and numpy.sum(pending) > 0:
        start_idx = numpy.argwhere(pending)[0, 0]
        processes[start_idx].start()
        logging.info('Calculating %s%s...',
                     group_name + ': ' if group_name else '',
                     task_names[start_idx])
        pending[start_idx] = False
        running[start_idx] = True

      done = numpy.logical_and(running, [not queue.empty() for queue in queues])
      for done_idx in numpy.argwhere(done):
        results[task_names[done_idx[0]]] = queues[done_idx[0]].get(
            block=True, timeout=None)
        running[done_idx[0]] = False

    # Wait for work to finish.
    for process in processes:
      process.join()

    return results

  database = {
      'reynolds_number': (params['phys']['rho'] * FLAGS.apparent_wind_speed
                          * params['wing']['c']
                          / params['phys']['dynamic_viscosity']),
      'alphas': alpha_list,
      'betas': beta_list,
      'delta1s': delta1s,
      'delta2s': delta2s,
      'delta3s': delta3s,
      'delta4s': delta4s,
      'delta5s': delta5s,
      'delta6s': delta6s,
      'delta7s': delta7s,
      'delta8s': delta8s,
  }

  # Pre-compute the local apparent wind speeds for each sampling point
  # on each surface for all the different alphas, betas, and angular
  # rates.  This is the most expensive part of the database
  # calculation, so there are huge gains from pre-computing these.
  alphas, betas = numpy.meshgrid(alpha_list, beta_list, indexing='ij')
  omega_hat_step = 1e-2
  tasks = {
      'nominal': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [0.0, 0.0, 0.0], FLAGS.apparent_wind_speed, params)),
      'positive_p': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [omega_hat_step / 2.0, 0.0, 0.0],
           FLAGS.apparent_wind_speed, params)),
      'negative_p': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [-omega_hat_step / 2.0, 0.0, 0.0],
           FLAGS.apparent_wind_speed, params)),
      'positive_q': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [0.0, omega_hat_step / 2.0, 0.0],
           FLAGS.apparent_wind_speed, params)),
      'negative_q': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [0.0, -omega_hat_step / 2.0, 0.0],
           FLAGS.apparent_wind_speed, params)),
      'positive_r': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [0.0, 0.0, omega_hat_step / 2.0],
           FLAGS.apparent_wind_speed, params)),
      'negative_r': (
          hover_model.PrecomputeLocalApparentWindSph,
          (alphas, betas, [0.0, 0.0, -omega_hat_step / 2.0],
           FLAGS.apparent_wind_speed, params))
  }
  local_apparent_wind_sph = _ProcessTasks(tasks, 'local apparent wind')

  # The delta increment tasks require the base CFM so we run it first.
  tasks = {
      'cfm': (CalcZeroDeflectionCoeffs, (alpha_list, beta_list,
                                         local_apparent_wind_sph['nominal'],
                                         params))
  }

  rate_steps = {
      'p': [omega_hat_step, 0.0, 0.0],
      'q': [0.0, omega_hat_step, 0.0],
      'r': [0.0, 0.0, omega_hat_step]
  }
  for name, step in rate_steps.iteritems():
    derivative_name = 'dcfm_d%s' % name
    tasks.update({
        derivative_name: (CalcAngularRateDerivativeCoeffs,
                          (alpha_list, beta_list, step,
                           local_apparent_wind_sph['positive_%s' % name],
                           local_apparent_wind_sph['negative_%s' % name],
                           params))
    })
  database.update(_ProcessTasks(tasks))

  # The delta increment angular rate tasks require the results of the
  # previous computations so we run them in a separate sweep.
  tasks = {}
  deltas_list = [delta1s, delta2s, delta3s, delta4s, delta5s, delta6s,
                 delta7s, delta8s]
  for delta_index, delta_list in enumerate(deltas_list):
    derivative_name = 'dcfm%d' % (delta_index + 1)
    tasks.update({
        derivative_name: (CalcDeltaIncrementCoeffs,
                          (alpha_list, beta_list, delta_list, delta_index,
                           database['cfm'], local_apparent_wind_sph['nominal'],
                           params))
    })

  for rate_name, step in rate_steps.iteritems():
    for delta_index, delta_list in enumerate(deltas_list):
      nominal_derivative_coeffs = database['dcfm_d%s' % rate_name]
      derivative_name = 'dcfm%d_d%s' % (delta_index + 1, rate_name)
      tasks.update({
          derivative_name: (CalcDeltaIncrementAngularRateDerivativeCoeffs,
                            (alpha_list, beta_list, delta_list, delta_index,
                             step, nominal_derivative_coeffs,
                             local_apparent_wind_sph['positive_%s' % rate_name],
                             local_apparent_wind_sph['negative_%s' % rate_name],
                             params))
      })
  database.update(_ProcessTasks(tasks))

  logging.info('Writing database to file %s.', FLAGS.filename)
  WriteJsonFile(FLAGS.filename, params, database)

def WriteJsonFile(filename, params, database):
  """Write database out as a .dat file.

  Args:
    filename: Name of output file to write database to.
    params: Parameter structure used to generate the database.
    database: Dictionary of ndarrays of aerodynamic coefficients and
        derivatives.
  """
  def _PrepareCoefficientArray(array):
    return numpy.reshape(numpy.rollaxis(array, -1), (array.size,)).tolist()

  keys_and_values = [
      ('num_alphas', len(database['alphas'])),
      ('num_betas', len(database['betas'])),
      ('num_deltas', [len(database['delta1s']), len(database['delta2s']),
                      len(database['delta3s']), len(database['delta4s']),
                      len(database['delta5s']), len(database['delta6s']),
                      len(database['delta7s']), len(database['delta8s'])]),
      ('reynolds_number', database['reynolds_number']),
      ('alphas', database['alphas']),
      ('betas', database['betas']),
      ('delta1s', database['delta1s']),
      ('delta2s', database['delta2s']),
      ('delta3s', database['delta3s']),
      ('delta4s', database['delta4s']),
      ('delta5s', database['delta5s']),
      ('delta6s', database['delta6s']),
      ('delta7s', database['delta7s']),
      ('delta8s', database['delta8s']),
      ('cfm', _PrepareCoefficientArray(database['cfm'])),
      ('dcfm_dp', _PrepareCoefficientArray(database['dcfm_dp'])),
      ('dcfm_dq', _PrepareCoefficientArray(database['dcfm_dq'])),
      ('dcfm_dr', _PrepareCoefficientArray(database['dcfm_dr'])),
      ('dcfm1', _PrepareCoefficientArray(database['dcfm1'])),
      ('dcfm1_dp', _PrepareCoefficientArray(database['dcfm1_dp'])),
      ('dcfm1_dq', _PrepareCoefficientArray(database['dcfm1_dq'])),
      ('dcfm1_dr', _PrepareCoefficientArray(database['dcfm1_dr'])),
      ('dcfm2', _PrepareCoefficientArray(database['dcfm2'])),
      ('dcfm2_dp', _PrepareCoefficientArray(database['dcfm2_dp'])),
      ('dcfm2_dq', _PrepareCoefficientArray(database['dcfm2_dq'])),
      ('dcfm2_dr', _PrepareCoefficientArray(database['dcfm2_dr'])),
      ('dcfm3', _PrepareCoefficientArray(database['dcfm3'])),
      ('dcfm3_dp', _PrepareCoefficientArray(database['dcfm3_dp'])),
      ('dcfm3_dq', _PrepareCoefficientArray(database['dcfm3_dq'])),
      ('dcfm3_dr', _PrepareCoefficientArray(database['dcfm3_dr'])),
      ('dcfm4', _PrepareCoefficientArray(database['dcfm4'])),
      ('dcfm4_dp', _PrepareCoefficientArray(database['dcfm4_dp'])),
      ('dcfm4_dq', _PrepareCoefficientArray(database['dcfm4_dq'])),
      ('dcfm4_dr', _PrepareCoefficientArray(database['dcfm4_dr'])),
      ('dcfm5', _PrepareCoefficientArray(database['dcfm5'])),
      ('dcfm5_dp', _PrepareCoefficientArray(database['dcfm5_dp'])),
      ('dcfm5_dq', _PrepareCoefficientArray(database['dcfm5_dq'])),
      ('dcfm5_dr', _PrepareCoefficientArray(database['dcfm5_dr'])),
      ('dcfm6', _PrepareCoefficientArray(database['dcfm6'])),
      ('dcfm6_dp', _PrepareCoefficientArray(database['dcfm6_dp'])),
      ('dcfm6_dq', _PrepareCoefficientArray(database['dcfm6_dq'])),
      ('dcfm6_dr', _PrepareCoefficientArray(database['dcfm6_dr'])),
      ('dcfm7', _PrepareCoefficientArray(database['dcfm7'])),
      ('dcfm7_dp', _PrepareCoefficientArray(database['dcfm7_dp'])),
      ('dcfm7_dq', _PrepareCoefficientArray(database['dcfm7_dq'])),
      ('dcfm7_dr', _PrepareCoefficientArray(database['dcfm7_dr'])),
      ('dcfm8', _PrepareCoefficientArray(database['dcfm8'])),
      ('dcfm8_dp', _PrepareCoefficientArray(database['dcfm8_dp'])),
      ('dcfm8_dq', _PrepareCoefficientArray(database['dcfm8_dq'])),
      ('dcfm8_dr', _PrepareCoefficientArray(database['dcfm8_dr']))
  ]

  output_dict = collections.OrderedDict(
      keys_and_values + [('params', dict_util.OrderDict(params))])

  class _ParamsEncoder(json.JSONEncoder):
    """JSON encoder which handles the Airfoil objects and numpy arrays."""

    def default(self, o):
      if isinstance(o, airfoil.Airfoil):
        return str(o)
      elif isinstance(o, numpy.ndarray):
        return o.tolist()
      return json.JSONEncoder.default(self, o)

  with open(filename, 'w') as f:
    output_string = json.dumps(output_dict, separators=(', ', ':\n  '),
                               cls=_ParamsEncoder)
    output_string = (output_string
                     .replace(', \"', ',\n\"')
                     .replace('], [', '],\n   [')
                     .replace(' [[', '[[')
                     .replace('{', '{\n')
                     .replace('}', '\n}')) + '\n'
    f.write(output_string)

def main(argv):
  try:
    argv = FLAGS(argv)
  except gflags.FlagsError, e:
    print '%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS)
    sys.exit(1)
  logging.basicConfig(stream=sys.stdout,
                      format='%(asctime)s %(levelname)-8s %(message)s',
                      level=logging.INFO)
  logging.info('Generating hover database.')
  GenerateHoverDatabase()

if __name__ == '__main__':
  main(sys.argv)


text:
import numpy as np

a1 = np.array(([1,2,3],[4,5,6]))
a2 = np.add(a1,2)
print a2


text:
from setuptools import Extension, setup
from Cython.Build import cythonize
import numpy

setup(
    ext_modules = cythonize([Extension("grid_fast", ["grid_fast.pyx"])]),
    include_dirs=[numpy.get_include()]
)


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import math
import time

import numpy as np
import torch as th
import torch.nn.functional as F
import torch.optim as optim
from ogb.nodeproppred import DglNodePropPredDataset, Evaluator

from scipy import io
from sklearn import metrics
import itertools
import matplotlib.colors as colors
from matplotlib import pyplot as plt
from matplotlib.ticker import AutoMinorLocator, MultipleLocator

from models import AGNN

global device, in_feats, n_classes, epsilon

device = None
in_feats, n_classes = None, None
epsilon = 1 - math.log(2)

def gen_model(args):
    norm = "both" if args.use_norm else "none"

    if args.use_labels:
        model = AGNN(
            in_feats + n_classes,
            n_classes,
            n_hidden=args.n_hidden,
            n_layers=args.n_layers,
            n_heads=args.n_heads,
            activation=F.relu,
            dropout=args.dropout,
            attn_drop=args.attn_drop,
            norm=norm,
        )
    else:
        model = AGNN(
            in_feats,
            n_classes,
            n_hidden=args.n_hidden,
            n_layers=args.n_layers,
            n_heads=args.n_heads,
            activation=F.relu,
            dropout=args.dropout,
            attn_drop=args.attn_drop,
            norm=norm,
        )

    return model

def cross_entropy(x, labels):
    y = F.cross_entropy(x, labels[:, 0], reduction="none")
    y = th.log(epsilon + y) - math.log(epsilon)
    return th.mean(y)

def compute_acc(pred, labels, evaluator):
    return evaluator.eval({"y_pred": pred.argmax(dim=-1, keepdim=True), "y_true": labels})["acc"]

def add_labels(feat, labels, idx):
    onehot = th.zeros([feat.shape[0], n_classes]).to(device)
    onehot[idx, labels[idx, 0]] = 1
    return th.cat([feat, onehot], dim=-1)

def adjust_learning_rate(optimizer, lr, epoch):
    if epoch <= 50:
        for param_group in optimizer.param_groups:
            param_group["lr"] = lr * epoch / 50

def train(model, graph, labels, train_idx, optimizer, use_labels):
    model.train()

    feat = graph.ndata["feat"]

    if use_labels:
        mask_rate = 0.5
        mask = th.rand(train_idx.shape) < mask_rate

        train_labels_idx = train_idx[mask]
        train_pred_idx = train_idx[~mask]

        feat = add_labels(feat, labels, train_labels_idx)
    else:
        mask_rate = 0.5
        mask = th.rand(train_idx.shape) < mask_rate

        train_pred_idx = train_idx[mask]

    optimizer.zero_grad()
    pred = model(graph, feat)
    loss = cross_entropy(pred[train_pred_idx], labels[train_pred_idx])
    loss.backward()
    th.nn.utils.clip_grad_norm(model.parameters(),10)
    optimizer.step()

    return loss, pred

@th.no_grad()
def evaluate(model, graph, labels, train_idx, val_idx, test_idx, use_labels, evaluator):
    model.eval()

    feat = graph.ndata["feat"]

    if use_labels:
        feat = add_labels(feat, labels, train_idx)

    pred = model(graph, feat)
    train_loss = cross_entropy(pred[train_idx], labels[train_idx])
    val_loss = cross_entropy(pred[val_idx], labels[val_idx])
    test_loss = cross_entropy(pred[test_idx], labels[test_idx])

    return (
        compute_acc(pred[train_idx], labels[train_idx], evaluator),
        compute_acc(pred[val_idx], labels[val_idx], evaluator),
        compute_acc(pred[test_idx], labels[test_idx], evaluator),
        train_loss,
        val_loss,
        test_loss,
    )

def count_parameters(args):
    model = gen_model(args)
    print([np.prod(p.size()) for p in model.parameters() if p.requires_grad])
    return sum([np.prod(p.size()) for p in model.parameters() if p.requires_grad])

# %% Define class with the model arguments
class args:
    cpu = True #Run cpu only if true. This overrides the gpu value
    gpu = 0 #Change number if different GPU device ID
    n_runs = 1 #Number of model runs
    n_epochs = 1000 #2000 #Number of epochs
    use_labels = False #Use labels in the training set as input features
    use_norm = False #Use symmetrically normalized adjacency matrix
    lr = 0.002 #0.002 Learning rate
    n_layers = 2 #3 #Number of layers
    n_heads = 1 #3
    n_hidden = 256 #256
    dropout = 0.75 #0.75
    attn_drop = 0.05
    wd = 0
    log_every = 1 #print result every log_every-th epoch
    #plot_curves = True

# Define folder to save plots and model in
foldername = "test"

# set cpu or gpu
if args.cpu:
    device = th.device("cpu")
else:
    device = th.device("cuda:%d" % args.gpu)

# load data
data = DglNodePropPredDataset(name="ogbn-arxiv")
evaluator = Evaluator(name="ogbn-arxiv")

splitted_idx = data.get_idx_split()
train_idx, val_idx, test_idx = splitted_idx["train"], splitted_idx["valid"], splitted_idx["test"]
graph, labels = data[0]

# add reverse edges
srcs, dsts = graph.all_edges()
graph.add_edges(dsts, srcs)

# add self-loop
print(f"Total edges before adding self-loop {graph.number_of_edges()}")
graph = graph.remove_self_loop().add_self_loop()
print(f"Total edges after adding self-loop {graph.number_of_edges()}")

in_feats = graph.ndata["feat"].shape[1]
n_classes = (labels.max() + 1).item()
# graph.create_format_()

train_idx = train_idx.to(device)
val_idx = val_idx.to(device)
test_idx = test_idx.to(device)
labels = labels.to(device)
graph = graph.to(device)

# %% Run the model
val_accs = []
test_accs = []

# define model and optimizer
model = gen_model(args)
model = model.to(device)

optimizer = optim.RMSprop(model.parameters(), lr=args.lr, weight_decay=args.wd)

# training loop
total_time = 0
best_val_acc, best_test_acc, best_val_loss = 0, 0, float("inf")

#save accuracy and loss values
accs, train_accs, val_accs, test_accs = [], [], [], []
losses, train_losses, val_losses, test_losses = [], [], [], []

for epoch in range(1, args.n_epochs + 1):
    print("Starting Epoch ", epoch)

    tic = time.time()

    adjust_learning_rate(optimizer, args.lr, epoch)

    loss, pred = train(model, graph, labels, train_idx, optimizer, args.use_labels)
    acc = compute_acc(pred[train_idx], labels[train_idx], evaluator)

    train_acc, val_acc, test_acc, train_loss, val_loss, test_loss = evaluate(
        model, graph, labels, train_idx, val_idx, test_idx, args.use_labels, evaluator
    )

    toc = time.time()
    total_time += toc - tic

    print("Epoch run-time ", toc-tic)

    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_val_acc = val_acc
        best_test_acc = test_acc

    if epoch % args.log_every == 0:
        print(f"\nEpoch: {epoch}/{args.n_epochs}")
        print(
            f"Loss: {loss.item():.4f}, Acc: {acc:.4f}\n"
            f"Train/Val/Test loss: {train_loss:.4f}/{val_loss:.4f}/{test_loss:.4f}\n"
            f"Train/Val/Test/Best val/Best test acc: {train_acc:.4f}/{val_acc:.4f}/{test_acc:.4f}/{best_val_acc:.4f}/{best_test_acc:.4f}"
        )

    for l, e in zip(
        [accs, train_accs, val_accs, test_accs, losses, train_losses, val_losses, test_losses],
        [acc, train_acc, val_acc, test_acc, loss.item(), train_loss, val_loss, test_loss],
    ):
        l.append(e)

# %% Printouts

print("*" * 50)
print(f"Average epoch time: {total_time / args.n_epochs}")
print(f"Total Time: {total_time}")
print(f"Test acc: {best_test_acc}")
print()
print("Val Accs:", best_val_acc)
print("Test Accs:", best_test_acc)
print(f"Number of params: {count_parameters(args)}")

# %% Generate plots of accuracy and loss vs epochs

fig = plt.figure(figsize=(15, 12))
ax = fig.gca()
ax.tick_params(labelright=True)
for y, label in zip([train_accs, val_accs, test_accs], ["train acc", "val acc", "test acc"]):
    plt.plot(range(args.n_epochs), y, label=label)
ax.legend(prop={'size': 20})
ax.tick_params(axis='both', labelsize = 20)
plt.title("Accuracy vs Epochs", fontsize=30)
plt.ylabel('Accuracy', fontsize=20)
plt.xlabel('Epochs', fontsize=20)
plt.grid(which="major", color="silver", linestyle="dotted")
plt.grid(which="minor", color="silver", linestyle="dotted")
#plt.tight_layout()
plt.savefig(foldername + "/gat_accuracy.png", bbox_inches='tight')
plt.show()

fig = plt.figure(figsize=(15, 12))
ax = fig.gca()
ax.tick_params(labelright=True)
for y, label in zip([train_losses, val_losses, test_losses],
                    ["train loss", "val loss", "test loss"]):
    plt.plot(range(args.n_epochs), y, label=label)
ax.legend(prop={'size': 20})
ax.tick_params(axis='both', labelsize = 20)
plt.title("Loss vs Epochs", fontsize=30)
plt.ylabel('Loss', fontsize=20)
plt.xlabel('Epochs', fontsize=20)
plt.grid(which="major", color="silver", linestyle="dotted")
plt.grid(which="minor", color="silver", linestyle="dotted")
#plt.tight_layout()
plt.savefig(foldername + "/gat_loss.png", bbox_inches='tight')
plt.show()

# %% Generate histogram of predicted labels

category_names = ["cs.AI", "cs.AR", "cs.CC", "cs.CE", "cs.CG", "cs.CL", "cs.CR", "cs.CV", "cs.CY",
                  "cs.DB", "cs.DC", "cs.DL", "cs.DM", "cs.DS", "cs.ET", "cs.FL", "cs.GL", "cs.GR",
                  "cs.GT", "cs.HC", "cs.IR", "cs.IT", "cs.LG", "cs.LO", "cs.MA", "cs.MM", "cs.MS",
                  "cs.NA", "cs.NE", "cs.NI", "cs.OH", "cs.OS", "cs.PF", "cs.PL", "cs.RO", "cs.SC",
                  "cs.SD", "cs.SE", "cs.SI", "cs.SY"]

# Get predicted categories
feat = graph.ndata["feat"]
pred = model(graph, feat)
pred = pred.argmax(dim=-1, keepdim=True)

# Split predicted cateogories by train, validate and test sets
train_pred = th.flatten(pred[train_idx]).numpy()
val_pred = th.flatten(pred[val_idx]).numpy()
test_pred = th.flatten(pred[test_idx]).numpy()

# Get the ground truth labels for train set for sorting order later
train_labels = th.flatten(labels[train_idx]).numpy()

true_train_freq, train_freq, val_freq, test_freq = [], [], [], []

for i in range(n_classes):
    true_train_freq.append(np.count_nonzero(train_labels==i))
    train_freq.append(np.count_nonzero(train_pred==i))
    val_freq.append(np.count_nonzero(val_pred==i))
    test_freq.append(np.count_nonzero(test_pred==i))

train_freq, val_freq, test_freq = np.array(train_freq), np.array(val_freq), np.array(test_freq)

# Plot histogram in alphebetical order of paper categories
fig, ax = plt.subplots(figsize=(15, 8))
ax.bar(category_names, train_freq, color = 'tab:blue')
ax.bar(category_names, val_freq, bottom = train_freq, color = 'tab:purple')
ax.bar(category_names, test_freq, bottom = (val_freq + train_freq), color = 'tab:red')
ax.legend(labels=['Train', 'Validate', 'Test'], prop={'size': 15})
plt.setp(ax.get_xticklabels(), rotation = 90, horizontalalignment = 'center')
ax.tick_params(axis='both', labelsize = 13)
plt.title("Distribution of Predicted Paper Categories", fontdict={'fontsize':25})
plt.ylabel('Frequency', fontdict={'fontsize':15})
plt.savefig(foldername + "/pred_class_histogram.png",bbox_inches='tight')
plt.show()

# Plot histogram in frequency order of ground truth paper categories for training set

ordering = np.argsort(np.array(true_train_freq))
sorted_train_freq = train_freq[ordering]
sorted_val_freq = val_freq[ordering]
sorted_test_freq = test_freq[ordering]
sorted_names = []
for i in ordering:
    sorted_names.append(category_names[i])

fig, ax = plt.subplots(figsize=(15, 8))
ax.bar(sorted_names, sorted_train_freq, color = 'tab:blue')
ax.bar(sorted_names, sorted_val_freq, bottom = sorted_train_freq, color = 'tab:purple')
ax.bar(sorted_names, sorted_test_freq, bottom = (sorted_val_freq + sorted_train_freq), color = 'tab:red')
ax.legend(labels=['Train', 'Validate', 'Test'], prop={'size': 15})
plt.setp(ax.get_xticklabels(), rotation = 90, horizontalalignment = 'center')
ax.tick_params(axis='both', labelsize = 13)
plt.title("Distribution of Predicted Paper Categories", fontdict={'fontsize':25})
plt.ylabel('Frequency', fontdict={'fontsize':15})
plt.savefig(foldername + "/pred_class_histogram_sorted.png",bbox_inches='tight')
plt.show()

# %% Save the data

dict = {'predicted':pred, 'args':args, 'num_params': count_parameters(args),
        'accs':accs, 'train_accs':train_accs, 'val_accs':val_accs,'test_accs':test_accs,
        'losses':losses, 'train_losses':train_losses, 'val_losses':val_losses, 'test_losses':test_losses}
io.savemat(foldername+"/model_results.mat", dict)

#Info on saving/loading models: https://pytorch.org/tutorials/beginner/saving_loading_models.html

#Save model state only to make predictions
th.save(model.state_dict(), foldername + "/model_stateonly.pth")

#Save entire model and optimizer state so we can load and keep training
th.save({
            'epoch': epoch,
            'args':args,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'accs':accs, 'train_accs':train_accs, 'val_accs':val_accs,'test_accs':test_accs,
            'losses':losses, 'train_losses':train_losses, 'val_losses':val_losses, 'test_losses':test_losses
            }, foldername + "/checkpoint.pth")

# %% To load the model we would do:

# #Get args and also unpack everything else
# checkpoint = torch.load(foldername + "/checkpoint.pth")
# args = checkpoint['args']
# starting_epoch = checkpoint['epoch']
# accs = checkpoint['accs']
# train_accs, val_accs, test_accs = checkpoint['train_accs'], checkpoint['val_accs'], checkpoint['test_accs']
# losses = checkpoint['losses']
# train_losses, val_losses, test_losses = checkpoint['train_losses'], checkpoint['val_losses'], checkpoint['test_losses']

# #Re-initialize the model and the optimizer
# model = gen_model(args)
# model = model.to(device)
# optimizer = optim.RMSprop(model.parameters(), lr=args.lr, weight_decay=args.wd)

# #Load the states saved in the checkpoint
# model.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

# %% Plot the contigency matrices

# Function for plotting the confusion matrix. Borrowed from ECE 219 project 2
def plot_mat(mat, xticklabels = None, yticklabels = None, pic_fname = None, size=(-1,-1), if_show_values = True,
             num_decimals = 0, colorbar = True, grid = 'k', xlabel = None, ylabel = None, title = None,
             vmin=None, vmax=None, fontsize = {'title':15, 'axislabel': 15, 'small': 10}):
    if size == (-1, -1):
        size = (mat.shape[1] / 3, mat.shape[0] / 3)

    fig = plt.figure(figsize=size)
    ax = fig.add_subplot(1,1,1)

    # im = ax.imshow(mat, cmap=plt.cm.Blues)
    im = ax.pcolor(mat, cmap=plt.cm.Blues, linestyle='-', linewidth=0.5, edgecolor=grid, vmin=vmin, vmax=vmax)

    if colorbar:
        cbar = plt.colorbar(im, aspect = 30) #fraction=0.046, pad=0.07)
        cbar.ax.tick_params(labelsize=fontsize['axislabel'])
    # tick_marks = np.arange(len(classes))
    # Ticks
    lda_num_topics = mat.shape[0]
    nmf_num_topics = mat.shape[1]
    yticks = np.arange(lda_num_topics)
    xticks = np.arange(nmf_num_topics)
    ax.set_xticks(xticks + 0.5)
    ax.set_yticks(yticks + 0.5)
    if xticklabels:
        ax.tick_params(axis='x', labelrotation = 90)
    if xticklabels is None:
        xticklabels = [str(i) for i in xticks]
    if yticklabels is None:
        yticklabels = [str(i) for i in yticks]
    ax.set_xticklabels(xticklabels, fontsize = fontsize['small'])
    ax.set_yticklabels(yticklabels, fontsize = fontsize['small'])

    # Minor ticks
    # ax.set_xticks(xticks, minor=True);
    # ax.set_yticks(yticks, minor=True);
    # ax.set_xticklabels([], minor=True)
    # ax.set_yticklabels([], minor=True)

    # ax.grid(which='minor', color='k', linestyle='-', linewidth=0.5)

    # tick labels on left, right and bottom
    ax.tick_params(labelright = True, labeltop = False)

    if ylabel:
        plt.ylabel(ylabel, fontsize=fontsize['axislabel'])
    if xlabel:
        plt.xlabel(xlabel, fontsize=fontsize['axislabel'])
    if title:
        plt.title(title, fontsize=fontsize['title'])

    # im = ax.imshow(mat, interpolation='nearest', cmap=plt.cm.Blues)
    ax.invert_yaxis()

    # thresh = mat.max() / 2

    def show_values(pc, fmt="%." + str(num_decimals) + "f", **kw):
        pc.update_scalarmappable()
        ax = pc.axes
        for p, color, value in itertools.zip_longest(pc.get_paths(), pc.get_facecolors(), pc.get_array()):
            x, y = p.vertices[:-2, :].mean(0)
            if np.all(color[:3] > 0.5):
                color = (0.0, 0.0, 0.0)
            else:
                color = (1.0, 1.0, 1.0)
            ax.text(x, y, fmt % value, ha="center", va="center", color=color, **kw, fontsize=fontsize['small'])
    if if_show_values:
        show_values(im)
    # for i, j in itertools.product(range(mat.shape[0]), range(mat.shape[1])):
    #     ax.text(j, i, "{:.2f}".format(mat[i, j]), fontsize = 4,
    #              horizontalalignment="center",
    #              color="white" if mat[i, j] > thresh else "black")

    plt.tight_layout()
    if pic_fname:
        plt.savefig(pic_fname, dpi=200, facecolor='w', bbox_inches='tight')
    plt.show()

#sklearn documentation: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html
#We normalize against the true labels, so each matrix entry is divided by its row sum

# Get the ground truth labels
# From the histogram plot, the predicted labels are already in train_pred, val_pred, and test_pred
train_labels = th.flatten(labels[train_idx]).numpy()
val_labels = th.flatten(labels[val_idx]).numpy()
test_labels = th.flatten(labels[test_idx]).numpy()

# Generate the contingency matrix for training set
train_matrix = metrics.confusion_matrix(train_labels, train_pred, normalize='true')
pic_fname = foldername + "/train_matrix.png"
plot_mat(train_matrix,xlabel='Cluster Class', ylabel='Actual Class', title='Normalized Confusion Matrix', num_decimals = 2,
         xticklabels = category_names, yticklabels = category_names,
         size=(35,30), fontsize = {'title':35, 'axislabel':25, 'small':15}, pic_fname = pic_fname)

# Generate the contingency matrix for valation set
val_matrix = metrics.confusion_matrix(val_labels, val_pred, normalize='true')
pic_fname = foldername + "/val_matrix.png"
plot_mat(val_matrix, xlabel='Cluster Class', ylabel='Actual Class', title='Normalized Confusion Matrix', num_decimals = 2,
         xticklabels = category_names, yticklabels = category_names,
         size=(35,30), fontsize = {'title':35, 'axislabel':25, 'small':15}, pic_fname = pic_fname)

# Generate the contingency matrix for test set
test_matrix = metrics.confusion_matrix(test_labels, test_pred, normalize='true')
pic_fname = foldername + "/test_matrix.png"
plot_mat(train_matrix, xlabel='Cluster Class', ylabel='Actual Class', title='Normalized Confusion Matrix', num_decimals = 2,
         xticklabels = category_names, yticklabels = category_names,
         size=(35,30), fontsize = {'title':35, 'axislabel':25, 'small':15}, pic_fname = pic_fname)

reason: import numpy,from scipy


text:
import json
import warnings
import os
import sys
import numpy as np
import pandas as pd
from statsmodels.tools.sm_exceptions import ConvergenceWarning
from tqdm import tqdm
from statsmodels.tsa.arima.model import ARIMA
root_path = os.path.abspath(__file__)
root_path = '/'.join(root_path.split('/')[:-2])
sys.path.append(root_path)
from libcity.evaluator.utils import evaluate_model
from libcity.utils import preprocess_data
import warnings

warnings.filterwarnings('ignore')

config = {
    'model': 'ARIMA',
    'p_range': [0, 4],
    'd_range': [0, 3],
    'q_range': [0, 4],
    # 'dataset': 'METR_LA',
    'dataset': 'Ocean_sensor',
    'data_col': 'temp',
    'train_rate': 0.7,
    'eval_rate': 0.1,
    'input_window': 12,
    'output_window': 3,
    'metrics': ['MAE', 'MAPE', 'MSE', 'RMSE', 'masked_MAE',
                'masked_MAPE', 'masked_MSE', 'masked_RMSE', 'R2', 'EVAR']
}

def get_data(dataset):
    # path
    path = root_path + '/raw_data/' + dataset + '/'
    config_path = path + 'config.json'
    dyna_path = path + dataset + '.dyna'
    geo_path = path + dataset + '.geo'

    # read config
    with open(config_path, 'r') as f:
        json_obj = json.load(f)
        for key in json_obj:
            if key not in config:
                config[key] = json_obj[key]

    # read geo
    geo_file = pd.read_csv(geo_path)
    geo_ids = list(geo_file['geo_id'])

    # read dyna
    dyna_file = pd.read_csv(dyna_path)
    data_col = config.get('data_col', '')
    if data_col != '':  # 根据指定的列加载数据集
        if isinstance(data_col, list):
            data_col = data_col.copy()
        else:  # str
            data_col = [data_col].copy()
        data_col.insert(0, 'time')
        data_col.insert(1, 'entity_id')
        dyna_file = dyna_file[data_col]
    else:  # 不指定则加载所有列
        dyna_file = dyna_file[dyna_file.columns[2:]]  # 从time列开始所有列

    # 求时间序列
    time_slots = list(dyna_file['time'][:int(dyna_file.shape[0] / len(geo_ids))])

    # 转3-d数组
    feature_dim = len(dyna_file.columns) - 2
    df = dyna_file[dyna_file.columns[-feature_dim:]]
    len_time = len(time_slots)
    data = []
    for i in range(0, df.shape[0], len_time):
        data.append(df[i:i + len_time].values)
    data = np.array(data, dtype=float)  # (N, T, F)
    data = data.swapaxes(0, 1)  # (T, N, F)
    return data

# Try to find the best (p,d,q) parameters for ARIMA
def order_select_pred(data):
    # data: (T, F)
    res = ARIMA(data, order=(0, 1, 3)).fit()
    bic = res.bic
    return res
    p_range = config.get('p_range', [0, 4])
    d_range = config.get('d_range', [0, 3])
    q_range = config.get('q_range', [0, 4])
    with warnings.catch_warnings():
        warnings.simplefilter("error", category=ConvergenceWarning)
        warnings.simplefilter("error", category=RuntimeWarning)
        for p in range(p_range[0], p_range[1]):
            for d in range(d_range[0], d_range[1]):
                for q in range(q_range[0], q_range[1]):
                    try:
                        cur_res = ARIMA(data, order=(p, d, q)).fit()
                    except:
                        continue
                    if cur_res.bic < bic:
                        bic = cur_res.bic
                        res = cur_res
    return res

def arima(data):
    output_window = config.get('output_window', 3)
    y_pred = []  # (num_samples, N, out, F)
    data = data.swapaxes(1, 2)  # (num_samples, N, out, F)
    for time_slot in tqdm(data, 'ts'):  # (N, out, F)
        y_pred_ele = []  # (N, out, F)
        # Different nodes should be predict by different ARIMA models instance.
        for seq in time_slot:  # (out, F)
            pred = order_select_pred(seq).forecast(steps=output_window)
            pred = pred.reshape((-1, seq.shape[1]))  # (out, F)
            y_pred_ele.append(pred)
        y_pred.append(y_pred_ele)
    return np.array(y_pred).swapaxes(1, 2)  # (num_samples, out, N, F)

def main():
    print(config)
    data = get_data(config.get('dataset', ''))
    trainx, trainy, testx, testy = preprocess_data(data, config)
    y_pred = arima(testx)
    evaluate_model(y_pred=y_pred, y_true=testy, metrics=config['metrics'],
                   path=config['model']+'_'+config['dataset']+'_metrics.csv')

if __name__ == '__main__':
    main()

max_stars_repo_name: Neoyanghc/Bigscity-LibCity

max_issues_repo_name: Neoyanghc/Bigscity-LibCity

max_forks_repo_name: Neoyanghc/Bigscity-LibCity

reason: import numpy,from statsmodels


text:
from tqdm import tqdm
from keras import backend
import tensorflow as tf
import numpy as np
from keras import layers, models

def get_layer_names(model):
    res = []
    for i in model.layers:
        if isinstance(i, models.Model):
            for j in get_layer_names(i):
                res.append((i.name, j))
        else:
            res.append(i.name)
    return res

def contains_subnetwork(model):
    if [True for l in model.layers if isinstance(l, models.Model)]:
        return True
    else:
        return False

def set_layer_weights(layer0, layer1):
    if isinstance(layer0, layers.InputLayer):
        return
    elif isinstance(layer1, layers.TimeDistributed) and \
            isinstance(layer1.layer, layers.LocallyConnected1D) and \
            not isinstance(layer0, layers.TimeDistributed):
        s = layer1.input.get_shape().as_list()
        weight, bias = layer1.get_weights()
        weight2 = np.concatenate([weight for _ in range(s[1])], axis=0)
        bias2 = np.concatenate([np.expand_dims(bias, axis=0) for
                                _ in range(s[1])], axis=0)
        layer0.set_weights([weight2, bias2])
    else:
        layer0.set_weights(layer1.get_weights())

def set_model_weights(model0, model1):
    backend.get_session().run(tf.global_variables_initializer())
    if contains_subnetwork(model1) and \
            contains_subnetwork(model0):
        for l in model0.layers:
            set_layer_weights(
                model0.get_layer(l.name),
                model1.get_layer(l.name))
    elif (not contains_subnetwork(model1)) and \
            contains_subnetwork(model0):
        raise NotImplementedError()
    elif contains_subnetwork(model1) and \
            (not contains_subnetwork(model0)):
        layer_names = get_layer_names(model1)
        print(f'\nSetting weights of model {model0.name} to be the same as '
              f'model {model1.name}')
        for l in tqdm(model0.layers):
            fit = [i for i in layer_names if
                   isinstance(i, str) and l.name == i
                   or isinstance(i, tuple) and l.name == i[1]]
            assert len(fit) == 1, f'len(fit) == 1, but it is {len(fit)}'
            fit = fit[0]
            if isinstance(fit, tuple):
                set_layer_weights(
                    model0.get_layer(l.name),
                    model1.get_layer(fit[0]).get_layer(fit[1]))
            elif isinstance(fit, str):
                set_layer_weights(
                    model0.get_layer(l.name),
                    model1.get_layer(l.name))
            else:
                raise NotImplementedError
    elif (not contains_subnetwork(model1)) and \
            (not contains_subnetwork(model0)):
        for l in tqdm(model0.layers):
            set_layer_weights(
                model0.get_layer(l.name),
                model1.get_layer(l.name))
    else:
        raise ValueError
    return model0


text:
#!/usr/bin/env python

import sys, os, pickle
import pandas as pd
import numpy as np
from sklearn import linear_model, ensemble, metrics, impute, preprocessing

def load_listfile(path):
    names = []
    labels = []
    with open(path, 'r') as f:
        f.readline() # first line is header
        for line in f:
            line = line.strip()
            parts = line.split(',')
            names.append('_'.join(parts[0].split('_')[0:2]))
            labels.append(int(parts[-1]))
    return pd.DataFrame(index=names, columns=['label'], data=labels, dtype=int)

if len(sys.argv) < 3:
    print('usage: {} <listfile_dir> <feature_files...>'.format(sys.argv[0]))
    quit()
listfile_dir = sys.argv[1]
feature_files = sys.argv[2:]

print('loading features...')
features = []
for feature_file in feature_files:
    with open(feature_file, 'rb') as f:
        features.append(pickle.load(f))

print('processing listfiles...')
df_train = load_listfile(os.path.join(listfile_dir, 'train_listfile.csv'))
df_val = load_listfile(os.path.join(listfile_dir, 'val_listfile.csv'))
df_test = load_listfile(os.path.join(listfile_dir, 'test_listfile.csv'))
print(df_train.shape, df_val.shape, df_test.shape)
for feature in features:
    feature['df_train'] = feature['df'].loc[
        feature['df'].index.intersection(df_train.index)]
    feature['df_val'] = feature['df'].loc[
        feature['df'].index.intersection(df_val.index)]
    feature['df_test'] = feature['df'].loc[
        feature['df'].index.intersection(df_test.index)]
    del feature['df']
    print(feature['df_train'].shape, feature['df_val'].shape, feature['df_test'].shape)

print('imputing values...')
for feature in features:
    if feature['should_impute']:
        imputer = impute.SimpleImputer()
        imputer.fit(feature['df_train'])
        feature['df_train'][feature['df_train'].columns] = \
            imputer.transform(feature['df_train'][feature['df_train'].columns])
        feature['df_val'][feature['df_val'].columns] = \
            imputer.transform(feature['df_val'][feature['df_val'].columns])
        feature['df_test'][feature['df_test'].columns] = \
            imputer.transform(feature['df_test'][feature['df_test'].columns])

print('standardizing values...')
for feature in features:
    if feature['should_standardize']:
        scaler = preprocessing.StandardScaler()
        scaler.fit(feature['df_train'])
        std = 0.316
        #std = 0.1
        #std = 1
        feature['df_train'][feature['df_train'].columns] = \
            scaler.transform(feature['df_train'][feature['df_train'].columns])*std
        feature['df_val'][feature['df_val'].columns] = \
            scaler.transform(feature['df_val'][feature['df_val'].columns])*std
        feature['df_test'][feature['df_test'].columns] = \
            scaler.transform(feature['df_test'][feature['df_test'].columns])*std

print('concatenating features...')
df_train = pd.concat([df_train] + [feature['df_train'] for feature in features],
    axis=1, join='inner')
df_val = pd.concat([df_val] + [feature['df_val'] for feature in features],
    axis=1, join='inner')
df_test = pd.concat([df_test] + [feature['df_test'] for feature in features],
    axis=1, join='inner')
print(df_train.shape, df_val.shape, df_test.shape)

# fix for a weird bug where all-0 columns in BoC becomes NaN after concat
df_train.fillna(value=0, inplace=True)
df_val.fillna(value=0, inplace=True)
df_test.fillna(value=0, inplace=True)

train_X = df_train.drop('label', axis=1).values
train_y = df_train['label'].values
val_X = df_val.drop('label', axis=1).values
val_y = df_val['label'].values
test_X = df_test.drop('label', axis=1).values
test_y = df_test['label'].values

# uncomment the model to use
print('fitting model...')
model = linear_model.LogisticRegression(solver='lbfgs', random_state=42,
    penalty='l2', C=0.001, max_iter=10000, class_weight='balanced')
#model = ensemble.RandomForestClassifier(n_estimators=200,
#    class_weight='balanced', random_state=42, max_leaf_nodes=200)
model.fit(train_X, train_y)

train_pred = model.predict(train_X)
print('\n\n\ntrain:')
print(metrics.confusion_matrix(train_y, train_pred))
print(metrics.classification_report(train_y, train_pred))

val_pred = model.predict(val_X)
print('\n\n\nvalidation:')
print(metrics.confusion_matrix(val_y, val_pred))
print(metrics.classification_report(val_y, val_pred))

val_prob = model.predict_proba(val_X)[:, 1]
fpr, tpr, _ = metrics.roc_curve(val_y, val_prob)
roc_auc = metrics.auc(fpr, tpr)
print('ROC AUC:', roc_auc)

quit()

test_pred = model.predict(test_X)
test_prob = model.predict_proba(test_X)[:, 1]
fpr, tpr, _ = metrics.roc_curve(test_y, test_prob)
test_roc_auc = metrics.auc(fpr, tpr)
print('\\begin{tabular}{@{}c@{}} %.3f \\\\ \\textbf{%.3f} \end{tabular} &' %
    (metrics.accuracy_score(val_y, val_pred),
     metrics.accuracy_score(test_y, test_pred)))
print('\\begin{tabular}{@{}c@{}} %.3f \\\\ \\textbf{%.3f} \end{tabular} &' %
    (metrics.precision_score(val_y, val_pred),
     metrics.precision_score(test_y, test_pred)))
print('\\begin{tabular}{@{}c@{}} %.3f \\\\ \\textbf{%.3f} \end{tabular} &' %
    (metrics.recall_score(val_y, val_pred),
     metrics.recall_score(test_y, test_pred)))
print('\\begin{tabular}{@{}c@{}} %.3f \\\\ \\textbf{%.3f} \end{tabular} &' %
    (metrics.f1_score(val_y, val_pred),
     metrics.f1_score(test_y, test_pred)))
print('\\begin{tabular}{@{}c@{}} %.3f \\\\ \\textbf{%.3f} \end{tabular} \\\\ \\hline' %
    (roc_auc, test_roc_auc))

#variables = df_train.drop('label', axis=1).columns
#coefs = model.coef_
#significance = [(v, c) for (v, c) in zip(variables, coefs[0,:])]
#significance.sort(key=lambda x: x[1])
#print(significance[0:10])
#print(significance[-1:-11:-1])

max_stars_repo_name: jrrr/emr-semantic-representations

max_issues_repo_name: jrrr/emr-semantic-representations

max_forks_repo_name: jrrr/emr-semantic-representations