text:
import math
from collections import defaultdict

import numpy as np
import pandas as pd
from tqdm import tqdm

import sketch.compress_freq as cf
import sketch.sketch_frequent as f
import sketch.compress_dyadic as dyadic

class LinearBenchRunner:
    def __init__(
            self,
            size,
            segments,
            x_to_track
    ):
        self.size = size
        self.segments = segments
        self.x_to_track = x_to_track

    def run(self):
        print("Running Linear Bench with size: {} on {} segs".format(
            self.size,
            len(self.segments)
        ))

        compressors = [
            cf.TopValueCompressor(self.x_to_track),
            cf.RandomSampleCompressor(self.size),
            cf.TruncationCompressor(self.size),
            cf.HairCombCompressor(self.size),
            cf.IncrementalRangeCompressor(self.size),
        ]
        compressor_names = [
            "topvalue",
            "random_sample",
            "truncation",
            "pps",
            "incremental",
        ]

        dyadic_height = int(math.log2(len(self.segments)))
        dyadic_size = self.size/(dyadic_height+1)
        print("Dyadic Height: {}, Size:{}".format(dyadic_height, dyadic_size))
        dyadic_compressor = dyadic.DyadicFrequencyCompressor(
            size=dyadic_size,
            max_height=dyadic_height
        )

        sketches = [
            # lambda: f.SpaceSavingSketch(size=self.size, unbiased=False),
            lambda: f.CountMinSketchFast(size=self.size, unbiased=False, x_to_track=self.x_to_track)
            # lambda: f.CountMinSketchOld(size=self.size, unbiased=True),
        ]
        sketch_names = [
            # "spacesaving",
            "cms_min",
            # "cms_mean",
        ]
        results = []

        for cur_seg_idx, cur_seg in tqdm(enumerate(self.segments)):
            ec_current = f.ExactCounterSketch()
            ec_current.add(cur_seg)
            exact_dict = ec_current.get_dict()

            for compressor_idx, cur_compressor in enumerate(compressors):
                cur_compressor_name = compressor_names[compressor_idx]
                compressed_counts = cur_compressor.compress(item_dict=exact_dict)
                results.append({
                    "seg_idx": cur_seg_idx,
                    "method": cur_compressor_name,
                    "counts": compressed_counts,
                })

            dyadic_summaries = dyadic_compressor.compress(item_dict=exact_dict)
            for summ_height, cur_dyadic_summ in enumerate(dyadic_summaries):
                results.append({
                    "seg_idx": (summ_height, cur_seg_idx),
                    "method": "dyadic_truncation",
                    "counts": cur_dyadic_summ
                })

            for sketch_idx, cur_sketch_constructor in enumerate(sketches):
                cur_sketch_name = sketch_names[sketch_idx]
                cur_sketch = cur_sketch_constructor()
                cur_sketch.add(cur_seg)
                compressed_counts = cur_sketch.get_dict()
                results.append({
                    "seg_idx": cur_seg_idx,
                    "method": cur_sketch_name,
                    "counts": compressed_counts,
                })
        return results

def gen_data(num_segments=10, seg_size=1000, seed=0):
    r = np.random.RandomState(seed=seed)
    segments = []
    for i in range(num_segments):
        # seg_size = r.geometric(.001)
        # shift = r.randint(10)
        shift = 0
        xs = r.zipf(1.1, size=seg_size)
        segments.append(xs + shift)
    return segments

def run_single_bench():
    segments = gen_data(num_segments=256, seg_size=1024, seed=0)
    rr = LinearBenchRunner(size=64, segments=segments)
    results = rr.run()
    with open("output/linear_bench.out", "w") as f:
        f.write(repr(results))

def run_multi_grain(dataset="caida"):
    workload_granularities = [8, 32, 128, 512, 2048]
    total_space = 64*2048
    if dataset == "caida":
        df_in = pd.read_csv("notebooks/caida1M-dest-stream.csv")
        x_stream = df_in["Destination"].values
        data_name = "caida"
        x_df = pd.read_csv("notebooks/caida1M-xtrack.csv")
        x_to_track = x_df["x_track"].values
    elif dataset == "caida10M":
        df_in = pd.read_csv("/Users/edwardgan/Documents/Projects/datasets/caida-pcap/caida10M-ipdst.csv")
        x_stream = df_in["ip.dst"].values
        data_name = "caida10M"
        x_df = pd.read_csv("/Users/edwardgan/Documents/Projects/datasets/caida-pcap/caida10M-ipdst-xtrack.csv")
        x_to_track = x_df["x_track"].values
    elif dataset == "zipf":
        df_in = pd.read_csv("notebooks/zipf10M.csv", nrows=10_000_000)
        x_stream = df_in["x"].values
        # r = np.random.RandomState(seed=0)
        # total_size = 2_000_000
        # x_stream = r.zipf(1.1, size=total_size)
        data_name = "zipf"
        x_df = pd.read_csv("notebooks/zipf10M-xtrack.csv")
        x_to_track = x_df["x_track"].values
    x_to_track = np.sort(x_to_track)
    print("Dataset: {}".format(dataset))
    print("Tracking: {}".format(x_to_track[:20]))
    for cur_granularity in workload_granularities:
        segments = np.array_split(x_stream, cur_granularity)
        sketch_size = total_space // cur_granularity
        rr = LinearBenchRunner(size=sketch_size, segments=segments, x_to_track=x_to_track)
        print("Running Grain: {}".format(cur_granularity))
        results = rr.run()
        with open("output/grain_{}_{}.out".format(data_name,cur_granularity), "w") as f:
            f.write(repr(results))

def run_test_bench():
    segments = [
        [1,1,2,3],
        [1,1,2,3],
        [2,2,3,4],
        [3,3,4,5]
    ]
    rr = LinearBenchRunner(size=2, segments=segments, x_to_track=range(6))
    results = rr.run()
    return results

def main():
    # run_single_bench()
    # run_test_bench()
    # run_multi_grain(dataset="caida10M")
    run_multi_grain(dataset="zipf")

if __name__ == "__main__":
    main()

max_stars_repo_name: stanford-futuredata/sketchstore

max_issues_repo_name: stanford-futuredata/sketchstore

max_forks_repo_name: stanford-futuredata/sketchstore


text:
# ==============================================================
# Author: Rodoflo Ferro
# Email: ferro@cimat.mx
# Twitter: @FerroRodolfo
#
# ABOUT COPYING OR USING PARTIAL INFORMATION:
# This script has been originally created by Rodolfo Ferro,
# for his workshop at PyCon Colombia 2020. Any explicit usage
# of this script or its contents is granted according to
# the license provided and its conditions.
# ==============================================================

# -*- coding: utf-8 -*-

from flask import Flask
from flask import request
from flask import jsonify
import numpy as np

from utils import load_model
from utils import image_decoder

# Global variables
dl_model = '../model/model.json'
dl_weights = '../model/model.h5'
# model <- Call the load_model funtion
labels = [
    'Angry',
    'Disgust',
    'Fear',
    'Happy',
    'Sad',
    'Surprise',
    'Neutral'
]

# API from Flask instance
api = Flask(__name__)

def status():
    """
    GET method for API status verification.
    """

    message = {
        "status": 200,
        "message": [
            "This API is up and running!"
        ]
    }

    return

def predict_emotion():
    """
    POST method for emotion prediction endpoint.
    """

    # Get data as JSON from POST
    # data <- get_json() from request

    # Parse data from JSON
    # raw_img <- 'image' key from data

    # Build input vector for DL model
    # input_img <- Call image_decoder() on raw_img

    # Predict using DL model
    # prediction <- predict on input_img using the model
    # p_class <- grab the argmax of the prediciton vector

    # Serialize predictions for response
    prob_vector = None # TODO
    class_id = None    # TODO
    class_name = None  # TODO

    # Send response
    """
    Build message with:

    "message": [
            {
                "task": "Facial Emotion Recognition",
                "prob_vector": ???,
                "class_id": ???,
                "class_name": ???
            }
        ]
    """

    return response

@api.errorhandler(404)
def not_found(error=None):
    """
    GET method for not found routes.
    """

    message = {
        "status": 404,
        "message": [
            "[ERROR] URL not found."
        ]
    }
    response = jsonify(message)
    response.status_code = 404

    return response

if __name__ == '__main__':
    # Run API...
    pass

ext: py

reason: import numpy


text:
#!/bin/env python
from jinja2 import Template
import numpy as np

plumed_dist_template = Template("DISTANCE ATOMS={{atoms}} LABEL={{label}} ")
plumed_torsion_template = Template("TORSION ATOMS={{atoms}} LABEL={{label}} ")
plumed_angle_template = Template("ANGLE ATOMS={{atoms}} LABEL={{label}} ")
plumed_rmsd_template = Template("RMSD REFERENCE={{loc}} TYPE=OPTIMAL LABEL={{label}} ")
plumed_min_dist_template = Template("DISTANCES GROUPA={{group_a}} GROUPB={{group_b}} MIN={BETA={{beta}}} LABEL={{label}}")

plumed_matheval_template = Template("MATHEVAL ARG={{arg}} FUNC={{func}} LABEL={{label}} PERIODIC={{periodic}} ")

plumed_combine_template = Template("COMBINE LABEL={{label}} ARG={{arg}} COEFFICIENTS={{coefficients}} "+\
                                    "PERIODIC={{periodic}} ")

_SUPPORTED_FEATS=["Contact","Dihedral","AlphaAngle","Angle","Kappa"]

def create_torsion_label(inds, label):
    #t: TORSION ATOMS=inds
    return plumed_torsion_template.render(atoms=','.join(map(str, inds)), label=label) +"\n"

def create_angle_label(inds, label):
    #t: ANGLE ATOMS=inds
    return plumed_angle_template.render(atoms=','.join(map(str, inds)), label=label) +"\n"

def create_distance_label(inds, label):
    return plumed_dist_template.render(atoms=','.join(map(str, inds)), label=label) + "\n"

def create_min_dist_label(group_a,group_b,beta,label):
    return plumed_min_dist_template.render(group_a=','.join(map(str, group_a)),
                                           group_b=','.join(map(str, group_b)),
                                           beta=beta,
                                           label=label)+ "\n"

def create_rmsd_label(loc, label):
    return plumed_rmsd_template.render(loc=loc , label=label) + "\n"

def create_feature(argument, func, label, feature_mean=None, feature_scale=None, offset=None,**kwargs):
    arg = argument
    # if feature_scale is not None and feature_mean is not None:
    #     x = "((x-%s)/%s)"%(feature_mean, feature_scale)
    # else:
    #     x="x"
    x="x"
    if func is None:
        if feature_scale is not None and feature_mean is not None:
            f ="(%s-%s)/%s"%(x,feature_mean, feature_scale)
        else:
            f ="%s"%(x)

    elif func=="min":
        if feature_scale is not None and feature_mean is not None:
            f ="(%s-%s)/%s"%(x,feature_mean, feature_scale,)
        else:
            f ="%s"%(x)
    elif func=="exp":
        if feature_scale is not None and feature_mean is not None:
            f ="(%s(-(%s)^2/(2*%s^2)))-%s)/%s"%(func, x,kwargs.pop("sigma"),
                                                   feature_mean, feature_scale)
        else:
            f = "%s(-(%s)^2/(2*%s^2))"%(func, x, kwargs.pop("sigma"))

    elif func in ["sin","cos"]:
        if feature_scale is not None and feature_mean is not None:
            f = "(%s(%s)-%s)/%s"%(func,x,feature_mean, feature_scale)
        else:
            f = "%s(%s)"%(func,x)

    else:
        raise ValueError("Can't find function")

    if offset is not None:
        f += "-%s"%offset

    return plumed_matheval_template.render(arg=arg, func=f,\
                                           label=label,periodic="NO")

def get_feature_function(df, feature_index):
    possibles = globals().copy()
    possibles.update(locals())
    if df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])==1:
        func = possibles.get("create_distance_label")
    elif df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])>1:
        func = possibles.get("create_min_dist_label")
    elif df.featurizer[feature_index] == "LandMarkFeaturizer":
        func = possibles.get("create_rmsd_label")
    elif df.featurizer[feature_index] in ["Kappa","Angle"]:
        func = possibles.get("create_angle_label")
    else:
        func = possibles.get("create_torsion_label")
    return func

def get_feature_transform(df, feature_index):
    possibles = globals().copy()
    possibles.update(locals())
    if df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])==1:
        func = None
    elif df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])>1:
        func = "min"
    elif df.featurizer[feature_index] == "LandMarkFeaturizer":
        func = "exp"
        sigma =  df.otherinfo[feature_index]
    else:
        func = df.otherinfo[feature_index]
    return func

def render_atomic_feats(df,inds):
    output = []
    if not set(df.featurizer).issubset(set(_SUPPORTED_FEATS)):
        raise ValueError("Sorry only contact, landmark, and dihedral featuizers\
                         are supported for now")
    possibles = globals().copy()
    possibles.update(locals())

    already_done_list = []

    for j in df.iloc[inds].iterrows():
        feature_index = j[0]
        atominds = np.array(j[1]["atominds"])
        resids = j[1]["resids"]
        feat = j[1]["featuregroup"]
        func = get_feature_function(df, feature_index)
        feat_label = feat+"_%s"%'_'.join(map(str,resids))

        if feat_label not in already_done_list:
            #mdtraj is 0 indexed and plumed is 1 indexed
            if  df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])==1:
                output.append(func(inds=[np.array(atominds[0][0])+1,
                                         np.array(atominds[1][0])+1],
                                   label=feat_label))
            elif  df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])>1:
                output.append(func(group_a=np.array(atominds[0])+1,
                                   group_b=np.array(atominds[1])+1,
                                   beta=df.otherinfo[feature_index] ,
                                   label=feat_label))
            else:
                output.append(func(atominds + 1 , feat_label))
            output.append("\n")
            already_done_list.append(feat_label)

    return ''.join(output)

def render_df(df, nrm = None, inds=None, tica_mdl=None, output_label_prefix="f"):
    output = []

    if not set(df.featurizer).issubset(set(_SUPPORTED_FEATS)):
        raise ValueError("Sorry only contact, landmark, and dihedral featuizers\
                         are supported for now")

    if inds is None:
        inds = range(len(df))
    raw_feats = render_atomic_feats(df, inds)
    output.append(raw_feats)

    for j in df.iloc[inds].iterrows():
        feature_index = j[0]
        atominds = j[1]["atominds"]
        feat = j[1]["featuregroup"]
        resids = j[1]["resids"]
        feat = j[1]["featuregroup"]

        label = "%s0_%d"%(output_label_prefix, feature_index)

        func = get_feature_transform(df, feature_index,)
        if df.featurizer[feature_index] == "Contact" and len(df.atominds[feature_index][0])>1:
            feat_label = feat+"_%s"%'_'.join(map(str,resids))+".min"
        else:
            feat_label = feat+"_%s"%'_'.join(map(str,resids))
        sigma = None
        if nrm is not None:
            if hasattr(nrm, "center_"):
                nrm.mean_ = nrm.center_
        if tica_mdl and nrm:
            output.append(create_feature(argument=feat_label,\
                                             func=func,
                                             label=label,
                                             feature_mean = nrm.mean_[feature_index],
                                             feature_scale = nrm.scale_[feature_index],
                                             offset = tica_mdl.means_[feature_index]
                                         ) +"\n")
        elif nrm:
            output.append(create_feature(argument=feat_label,\
                                             func=func,
                                             label=label,
                                             feature_mean = nrm.mean_[feature_index],
                                             feature_scale = nrm.scale_[feature_index],
                                         ) +"\n")
        elif tica_mdl:
            output.append(create_feature(argument=feat_label,\
                                             func=func,
                                             label=label,
                                             offset = tica_mdl.means_[feature_index]
                                         ) +"\n")
        else:
            output.append(create_feature(argument=feat_label,\
                                             func=func,
                                             label=label,
                                         ) +"\n")

        output.append("\n")

    return ''.join(output)


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 21 15:58:27 2020
@author: aguemes
"""

import sys
import matplotlib
matplotlib.use('Agg')
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec

class TheArtist():

    def __init__(self, latex=True, font='serif', fontsize=8):

        self.inches_per_pt = 1.0 / 72.27
        plt.rc('text', usetex=latex)
        plt.rc('font', family=font, size=fontsize)

        return

    def draw(self, plot_dict):

        for plot in plot_dict.keys():

            if plot_dict[plot]['plot_type'] == 'pdf':

                self.plot_probability_density_function(plot_dict[plot]['data'], plot_dict[plot]['range'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])

            elif plot_dict[plot]['plot_type'] == 'cdf':

                self.plot_cumulative_density_function(plot_dict[plot]['data'], plot_dict[plot]['range'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])

            elif plot_dict[plot]['plot_type'] == 'jpdf':

                self.plot_joint_probability_density_function(plot_dict[plot]['data'], plot_dict[plot]['range'][0], plot_dict[plot]['range'][1], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])

            else:

                print('Plotting method not implemented yet. Exiting...')
                sys.exit(1)

            self.set_axis_lims(plot_dict[plot]['lims'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])
            self.set_ticks(plot_dict[plot]['ticks'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])
            self.set_labels(plot_dict[plot]['labels'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])
            self.set_title(plot_dict[plot]['title'], plot_dict[plot]['idx_row'], plot_dict[plot]['idx_col'])

        return

    def generate_figure_environment(self, cols=1, rows=1, fig_width_pt=384, ratio='golden', regular=True):

        if ratio == 'golden':

            ratio = 2 / (1 + 5 ** 0.5)

        elif ratio == 'square':

            ratio = 1

        else:

            print('This ratio has not been defined yet. Exiting code...')

            sys.exit(1)

        if regular:

            fig_width = fig_width_pt * self.inches_per_pt
            fig_height = fig_width * rows / cols * ratio

            self.fig, self.axs = plt.subplots(rows, cols, figsize=(fig_width, fig_height), squeeze=False)

        else:

            if type(cols) == list and len(cols) == rows:

                fig_width = fig_width_pt * self.inches_per_pt
                fig_height = fig_width * rows / max(cols) * ratio

                self.fig = plt.figure(figsize = (fig_width, fig_height), constrained_layout=False)

                gs = GridSpec(rows * 2, max(cols) * 2, figure=self.fig)

                self.axs = []

                for row, col in zip(range(rows), cols):

                    axs_row = []

                    if col == max(cols):

                        for idx_col in range(col):

                            axs_row.append(
                                    self.fig.add_subplot(gs[2*row:2*(row + 1), 2*idx_col:2*(idx_col + 1)])
                                )

                    else:

                        start = max(cols) - col

                        for idx_col in range(col):

                            axs_row.append(
                                self.fig.add_subplot(gs[2*row:2*(row + 1), (start + 2*idx_col):(start + 2*(idx_col + 1))])
                            )

                    self.axs.append(
                        axs_row
                    )

                self.axs = np.array(
                    [xi + [None] * (max(map(len, self.axs)) - len(xi)) for xi in self.axs]
                )

        self.im = []

        return

    def plot_probability_density_function(self, data, bins, idx_row, idx_col, color='darkblue', edgecolor='black', linewidth=1):

        sns.distplot(
            data,
            hist=True,
            kde=True,
            bins=bins,
            color=color,
            hist_kws={'edgecolor': edgecolor},
            kde_kws={'linewidth': linewidth, 'bw': (bins[1] - bins[0]) / 2},
            ax=self.axs[idx_row, idx_col]
        )

        return

    def plot_cumulative_density_function(self, data, bins, idx_row, idx_col, color='darkblue', edgecolor='black', linewidth=1):

        sns.distplot(
            data,
            hist=False,
            bins=bins,
            color=color,
            hist_kws={'edgecolor': edgecolor},
            kde_kws={'linewidth': linewidth, 'bw': (bins[1] - bins[0]) / 2, 'cumulative':True},
            ax=self.axs[idx_row, idx_col]
        )

        return

    def plot_joint_probability_density_function(self, data, xbins, ybins, idx_row, idx_col, levels=[0.1, 0.5, 0.9], color='darkblue', edgecolor='black', linewidth=1):

        jointProbs, xedges, yedges = np.histogram2d(x=data[0], y=data[1], bins=(xbins, ybins))
        jointProbs /= jointProbs.max()

        xedges = (xedges[1:] + xedges[:-1]) / 2
        yedges = (yedges[1:] + yedges[:-1]) / 2

        self.axs[idx_row, idx_col].contour(
            yedges,
            xedges,
            jointProbs,
            levels=levels,
            colors=color,
            linewidths=linewidth
        )

        return

    def plot_lines(self, x, y, idx_row, idx_col, color='darkblue', linewidth=1, linestyle='-', marker=None, markeredgecolor=None, markerfacecolor=None, markersize=3):

        self.axs[idx_row, idx_col].plot(
            x,
            y,
            color=color,
            linestyle=linestyle,
            linewidth=linewidth,
            marker=marker,
            markeredgecolor=markeredgecolor,
            markerfacecolor=markerfacecolor,
            markersize=markersize
        )

    def plot_panel_imshow(self, x, y, z, idx_row, idx_col, origin='lower', extent=None, cmap='Reds'):

        # self.axs[idx_row, idx_col].imshow(
        #     X=z,
        #     origin=origin,
        #     extent=None,
        #     cmap=cmap,
        #     vmin=0,
        #     vmax=z.max()
        # )

        self.im.append(
            self.axs[idx_row, idx_col].imshow(
            X=z,
            origin=origin,
            extent=None,
            cmap=cmap,
            vmin=0,
            vmax=z.max()
        )
        )

        return

    def plot_hist(self, x, bins, idx_row, idx_col, color='darkblue', histtype='bar', density=False, weights=None):

        self.axs[idx_row, idx_col].hist(
            x=x,
            bins=bins,
            color=color,
            histtype=histtype,
            density=density,
            weights=weights
        )

    def plot_lines_semi_y(self, x, y, idx_row, idx_col, color='darkblue', linewidth=1, linestyle='-', marker=None, markeredgecolor=None, markerfacecolor=None):

        self.axs[idx_row, idx_col].semilogy(
            x,
            y,
            color=color,
            linestyle=linestyle,
            linewidth=linewidth,
            marker=marker,
            markeredgecolor=markeredgecolor,
            markerfacecolor=markerfacecolor
        )

    def save_figure(self, fig_name, fig_format='png', dots_per_inch=600):

        filename = f'{fig_name}.{fig_format}'

        self.fig.tight_layout()

        self.fig.savefig(filename, dpi=dots_per_inch, bbox_inches='tight')

        return

    def set_axis_lims(self, values, idx_row, idx_col):

        self.axs[idx_row, idx_col].set_xlim(values[0])
        self.axs[idx_row, idx_col].set_ylim(values[1])

        return

    def set_colorbar(self, idx_row, idx_col, title=False):

        cbar = self.fig.colorbar(self.im[0], ax=self.axs[idx_row, idx_col], fraction=0.04)

        if title:

            cbar.ax.set_ylabel(title, rotation=-90, va="bottom")

        return

    def set_labels(self, labels, idx_row, idx_col):

        self.axs[idx_row, idx_col].set_xlabel(labels[0])
        self.axs[idx_row, idx_col].set_ylabel(labels[1])

        return

    def set_ticks(self, ticks, idx_row, idx_col):

        if list(ticks[0]) != None:

            self.axs[idx_row, idx_col].set_xticks(ticks[0])

        if list(ticks[1]) != None:

            self.axs[idx_row, idx_col].set_yticks(ticks[1])

        return

    def set_ticklabels(self, labels, idx_row, idx_col, position=[True, False, True, False], rotation=[0, 0], alignment_h=["center", "center"], alignment_v=["center", "center"], rotation_mode=[None, None]):

        if labels[0] != None:

            self.axs[idx_row, idx_col].set_xticklabels(labels[0], rotation=rotation[0], ha=alignment_h[0], va=alignment_v[0], rotation_mode=rotation_mode[0])

        if labels[1] != None:

            self.axs[idx_row, idx_col].set_yticklabels(labels[1], rotation=rotation[1], ha=alignment_h[1], va=alignment_v[1], rotation_mode=rotation_mode[1])

        self.axs[idx_row, idx_col].tick_params(top=position[0], bottom=position[1], labeltop=position[2], labelbottom=position[3])

        # plt.setp(cezanne.axs[0,0].get_xticklabels(), rotation=-30, ha="right", rotation_mode="anchor")

        return

    def set_title(self, title, idx_row, idx_col):

        self.axs[idx_row, idx_col].set_title(title)

        return

    def set_text(self, text, xpos, ypos, idx_row, idx_col):

        self.axs[idx_row, idx_col].text(xpos, ypos, text, transform=self.axs[idx_row,idx_col].transAxes)

        return

max_stars_repo_name: guemesturb/TheArtist

max_issues_repo_name: guemesturb/TheArtist

max_forks_repo_name: guemesturb/TheArtist


text:
#! python

"""
Library containing one or more functions to process incoming VOEvent XML strings. This library will
be imported by a long running process, so you can load large data files, etc, at import time, rather than
inside the processevent() function, to save time.

This module only handles MAXI and SWIFT VOEvents for Flare Stars.
"""

__version__ = "0.1.1"
__author__ = ["Paul Hancock", "Andrew Williams", "Gemma Anderson"]

import logging
import os
import astropy
from astropy.coordinates import Angle
from astropy.time import Time
import re
import sys
import voeventparse

from . import handlers
from . import triggerservice
from . import data_load

log = logging.getLogger('voevent.handlers.FlareStar_swift_maxi')   # Inherit the logging setup from handlers.py

# Settings
DEC_LIMIT = 32.

PROJECT_ID = 'G0056'
SECURE_KEY = handlers.get_secure_key(PROJECT_ID)

# Email these addresses when we trigger on an event
NOTIFY_LIST = ["Paul.Hancock@curtin.edu.au", "Gemma.Anderson@curtin.edu.au", "Andrew.Williams@curtin.edu.au", "jun.tian@postgrad.curtin.edu.au"]

# Email these addresses when we handle an event that is a GRB, but we don't trigger on it.
DEBUG_NOTIFY_LIST = ["Paul.Hancock@curtin.edu.au", "Gemma.Anderson@curtin.edu.au", "Andrew.Williams@curtin.edu.au", "jun.tian@postgrad.curtin.edu.au"]

EMAIL_TEMPLATE = """
The Flare Star MAXI+Swift handler triggered an MWA observation for
%(name)s at %(trigtime)s UTC.

Details are:
Trigger ID: %(triggerid)s
RA:         %(ra)s hours
Dec:        %(dec)s deg
"""

DEBUG_EMAIL_TEMPLATE = """
The Flare Star MAXI+Swift handler did NOT trigger an MWA observation for a
Flare Star. Log messages are:

%s

"""

EMAIL_SUBJECT_TEMPLATE = "Flare Star MAXI+Swift handler trigger for %s"

# state storage
xml_cache = {}
# list of star names
flare_stars = []

def make_flare_star_names():
    """
    Read a list of flare star names from a data file

    :return:
    None
    """
    global flare_stars
    data_file = data_load.FLARE_STAR_NAMES
    flare_stars = [a.strip().lower() for a in open(data_file, 'r').readlines() if not a.startswith("#")]
    # reformat ' ' into either '' or '_' in the list above
    flare_stars.extend([re.sub(' ', '_', f) for f in flare_stars if ' ' in f])
    flare_stars.extend([re.sub(' ', '', f) for f in flare_stars if ' ' in f])
    return

make_flare_star_names()

class FlareStar(handlers.TriggerEvent):
    """
    Subclass the TriggerEvent class to add a parameter 'short', relevant only for GRB type events.
    """
    def __init__(self, event=None):
        handlers.TriggerEvent.__init__(self, event=event)

    # Override or add source specific methods here if desired.

def processevent(event='', pretend=True):
    """
    Called externally by the voevent_handler script when a new VOEvent is received. Return True if
    the event was parsed by this handler, False if it was another type of event that should be
    examined by a different handler.

    :param event: A string containg the XML string in VOEvent format
    :param pretend: Boolean, True if we don't want to actually schedule the observations.
    :return: Boolean, True if this handler processed this event, False to pass it to another handler function.
    """

    v = voeventparse.loads(event.encode())

    # only respond to SWIFT and MAXI evetnts
    ivorn = v.attrib['ivorn']
    if not(('SWIFT' in ivorn) or ('MAXI' in ivorn)):
        return False

    log.info("Working on: %s" % ivorn)
    isflarestar = is_flarestar(v)
    log.debug("Flare Star ? {0}".format(isflarestar))
    if isflarestar:
        handle_flarestar(v, pretend=pretend)

    log.info("Finished.")
    return isflarestar     # True if we're handling this event, False if we're rejecting it

def is_flarestar(v):
    """
    Tests to see if this XML packet is a Flare Star from MAXI or SWIFT.

    :param v: string in VOEvent XML format
    :return: Boolean, True if this event is a Flare Star.
    """
    swift = False
    # SWIFT encodes a Why.Inference.Name
    if hasattr(v.Why.Inference, 'Name'):
        name = v.Why.Inference.Name
        log.debug("Found {0} in SWIFT format".format(name))
        swift = True
    else:
        # MAXI uses a Soruce_Name parameter
        src = v.find(".//Param[@name='Source_Name']")
        if src is None:
            return False
        # MAXI sometimes puts spaces at the start of the string!
        name = src.attrib['value'].strip()

        # move the name into some standard place so we can easily reference it later
        v.Why.Inference.Name = name
        log.debug("Found {0} in MAXI format".format(name))

    for f in flare_stars:
        # check if the name is within the "name" string since MAXI does stupid things sometimes
        if f in str(name).lower():
            # check if this is a sub_sub_threshold event and ignore if it is
            if swift and 'sub-sub-threshold' in str(v.What.Description):
                return False
            return True
    return False

def handle_flarestar(v, pretend=False):
    """
    Handles the actual VOEvent parsing, generating observations if appropriate.

    :param v: string in VOEvent XML format
    :param pretend: Boolean, True if we don't want to actually schedule the observations.
    :return: None
    """
    ivorn = v.attrib['ivorn']
    log.debug("processing Flare Star {0}".format(ivorn))

    name = v.Why.Inference.Name
    trig_id = v.find(".//Param[@name='TrigID']").attrib['value']
    c = voeventparse.get_event_position(v)
    if c.dec > DEC_LIMIT:
        msg = "Flare Star {0} above declination cutoff of +10 degrees".format(name)
        log.debug(msg)
        log.debug("Not triggering")
        handlers.send_email(from_address='mwa@telemetry.mwa128t.org',
                            to_addresses=DEBUG_NOTIFY_LIST,
                            subject='GRB_fermi_swift debug notification for trigger: %s' % trig_id,
                            msg_text=DEBUG_EMAIL_TEMPLATE % msg,
                            attachments=[('voevent.xml', voeventparse.dumps(v))])
        return

    if trig_id not in xml_cache:
        fs = FlareStar(event=v)
        fs.trigger_id = trig_id
        xml_cache[trig_id] = fs
    else:
        fs = xml_cache[trig_id]
        fs.add_event(v)

    ra = c.ra
    dec = c.dec
    fs.add_pos((ra, dec, 0.))
    fs.debug("Flare Star {0} is detected at RA={1}, Dec={2}".format(name, ra, dec))

    req_time_min = 30

    # look at the schedule
    obslist = triggerservice.obslist(obstime=1800)
    if obslist is not None and len(obslist) > 0:
        fs.debug("Currently observing:")
        fs.debug(str(obslist))
        # are we currently observing *this* GRB?
        obs = str(obslist[0][1])  # in case the obslist is returning unicode strings
        fs.debug("obs {0}, trig {1}".format(obs, trig_id))

        # Same GRB trigger from same telescope
        if obs == trig_id:
            fs.info("already observing this star")
            fs.info("not triggering again")
            handlers.send_email(from_address='mwa@telemetry.mwa128t.org',
                                to_addresses=DEBUG_NOTIFY_LIST,
                                subject='GRB_fermi_swift debug notification for trigger: %s' % trig_id,
                                msg_text=DEBUG_EMAIL_TEMPLATE % '\n'.join([str(x) for x in fs.loglist]),
                                attachments=[('voevent.xml', voeventparse.dumps(v))])
            return
    else:
        fs.debug("Current schedule empty")

    fs.debug("Triggering")
    # label as SWIFT or MAXI for the trigger type
    ttype = v.attrib['ivorn'].split('/')[-1].split('#')[0]

    emaildict = {'triggerid': fs.trigger_id,
                 'trigtime': Time.now().iso,
                 'ra': Angle(fs.ra[-1], unit=astropy.units.deg).to_string(unit=astropy.units.hour, sep=':'),
                 'dec': Angle(fs.dec[-1], unit=astropy.units.deg).to_string(unit=astropy.units.deg, sep=':'),
                 'name': name}
    email_text = EMAIL_TEMPLATE % emaildict
    email_subject = EMAIL_SUBJECT_TEMPLATE % fs.trigger_id
    # Do the trigger
    result = fs.trigger_observation(ttype=ttype,
                                    obsname=trig_id,
                                    time_min=req_time_min,
                                    pretend=pretend,
                                    project_id=PROJECT_ID,
                                    secure_key=SECURE_KEY,
                                    email_tolist=NOTIFY_LIST,
                                    email_text=email_text,
                                    email_subject=email_subject,
                                    creator='VOEvent_Auto_Trigger: FlareStar_swift_maxi=%s' % __version__,
                                    voevent=voeventparse.dumps(v))
    if result is None:
        handlers.send_email(from_address='mwa@telemetry.mwa128t.org',
                            to_addresses=DEBUG_NOTIFY_LIST,
                            subject='GRB_fermi_swift debug notification for trigger: %s' % trig_id,
                            msg_text=DEBUG_EMAIL_TEMPLATE % '\n'.join([str(x) for x in fs.loglist]),
                            attachments=[('voevent.xml', voeventparse.dumps(v))])

if __name__ == "__main__":
    print("Flare stars are:{0}".format(flare_stars))

reason: import astropy,from astropy


text:
# -*- coding: utf-8 -*-

import unittest
import numpy as np
import itertools

from .. import interpolate

class test_interpolate(unittest.TestCase):
    def _generate(self, shape):
        axes = [range(n) for n in shape]
        data = np.random.normal(size=shape).astype(np.float32) * 100
        return data, axes

    def test_nd(self):
        def identity(x):
            return x

        def meshgrid(axes):
            return np.meshgrid(*axes, indexing="ij")

        parameters = (
            (0, 1, 2),
            (True,),
            (
                (interpolate.interpolate_regular, identity),
                (interpolate.interpolate_irregular, meshgrid),
            ),
        )

        for parameters in itertools.product(*parameters):
            degree, asgrid, (interp, meshgrid) = parameters

            rtol = 1e-3

            # 1D
            data1, axes = self._generate((6,))
            axnew = (0,)
            data2 = interp(data1, axes, axnew, degree=degree, asgrid=asgrid)
            data3 = data1[axnew]
            np.testing.assert_allclose(data2, data3, rtol=rtol)
            axnew = ([0, 5],)
            data2 = interp(data1, axes, axnew, degree=degree, asgrid=asgrid)
            data3 = data1[axnew]
            np.testing.assert_allclose(data2, data3, rtol=rtol)

            # 2D
            data1, axes = self._generate((6, 8))
            if asgrid:
                axnew = ([0, 3, 5], [1, 3, 6])
            else:
                axnew = ([0, 3, 5], [1, 3])
            data2 = interp(data1, meshgrid(axes), axnew, degree=degree, asgrid=asgrid)
            if asgrid:
                axnew = tuple(np.meshgrid(*axnew, indexing="ij"))
            data3 = data1[axnew]
            np.testing.assert_allclose(data2, data3, rtol=rtol)

            # 3D
            data1, axes = self._generate((6, 8, 9))
            if asgrid:
                axnew = ([0, 3, 5], [1, 3, 6], [2, 5, 7])
            else:
                axnew = ([0, 3, 5], [1, 3], [2, 7])
            data2 = interp(data1, meshgrid(axes), axnew, degree=degree, asgrid=asgrid)
            if asgrid:
                axnew = tuple(np.meshgrid(*axnew, indexing="ij"))
            data3 = data1[axnew]
            np.testing.assert_allclose(data2, data3, rtol=rtol)

def test_suite():
    """Test suite including all test suites"""
    testSuite = unittest.TestSuite()
    testSuite.addTest(test_interpolate("test_nd"))
    return testSuite

if __name__ == "__main__":
    import sys

    mysuite = test_suite()
    runner = unittest.TextTestRunner()
    if not runner.run(mysuite).wasSuccessful():
        sys.exit(1)

max_stars_repo_name: woutdenolf/spectrocrunch

max_issues_repo_name: woutdenolf/spectrocrunch

max_forks_repo_name: woutdenolf/spectrocrunch


text:
import numpy as np
import matplotlib.pyplot as plt
class LPMModel():
    """
    input:
        x -----(feature_num, batch)
        y -------(1,batch)
    """
    def __init__(self, x, y,epochs,lr,eplison):
        self.x = x
        self.y = y
        print("input:x",np.shape(x),"input:y",np.shape(y))
        self.w = np.random.rand(1, len(self.x))
        # self.w = [[0.70,-1.0]]
        self.b = 0.0
        self.epochs = epochs
        self.lr = lr
        self.eplison = eplison
        self.loss_list = []

    def predict_model(self):
        """
          self.predict ----[1.batch]
          self.predictClas ---[1,batch]
        :return:
        """
        self.predict = np.matmul(self.w, self.x) + self.b
        self.predictClas = self.predict.copy()
        self.predictClas[self.predict>=0] = 1.0
        self.predictClas[self.predict<0] = -1.0

    def cal_loss_and_gradient(self):
        equal = np.equal(np.array(self.predictClas),np.array(self.y))
        self.missClasY = np.expand_dims(self.y[~equal],0)
        self.missClasX = []
        for i in range(len(self.x)):
            k = self.x[i]
            self.missClasX.append(k[(~equal)[0]])
        # print(np.shape(self.missClasY),np.shape(self.missClasX))
        self.loss = -np.sum(np.matmul(self.missClasY,np.transpose(self.predict[self.predictClas != self.y])))
        self.w_gradient = -np.matmul(self.missClasY, np.transpose(self.missClasX))
        self.b_gradient = -np.sum(self.missClasY)

    def update_gradient(self,lr):

        self.w = self.w -lr* self.w_gradient
        self.b = self.b -lr*self.b_gradient
        print(self.w,self.b)

    def training(self):
        epoch = 0
        last_loss = 0
        while (epoch <self.epochs):
            self.predict_model()
            self.cal_loss_and_gradient()
            print("epoch:",epoch,"loss", self.loss,"miss clas:",len(self.missClasY[0]))
            self.loss_list.append(self.loss)
            if (self.loss - last_loss< self.eplison or len(self.missClasY[0]) == 0):
                print("training finish")
                break
            else:
                self.update_gradient(self.lr)
            epoch = epoch + 1

    def plot_loss(self):
        x = np.arange(0,70)
        # print(np.shape(self.loss))
        plt.xlabel("Epochs")
        plt.ylabel("Training loss")
        plt.legend()
        plt.plot(x, self.loss_list, linestyle='--')
        plt.show()

    # def plot_curve(self):
    #     plt.scatter(self.x[self.y == 0], self.x[self.y == 0], marker='x', label='品种1')
    #     plt.scatter(self.x[self.y == 1], self.x[self.y == 1], marker='o', label='品种2')
    #     x = np.arange(4, 8)
    #     plt.plot(x, (x * 0.7 - 0.48), linestyle='--')
    #     plt.show()

max_stars_repo_path: LP Model/LPMModel.py

max_stars_repo_name: FindTheTruth/Machine-learning

max_issues_repo_path: LP Model/LPMModel.py

max_issues_repo_name: FindTheTruth/Machine-learning

max_forks_repo_path: LP Model/LPMModel.py

max_forks_repo_name: FindTheTruth/Machine-learning


text:
import numpy
import solution
import pylab

def draw_decision(X, y, classifier, at1, at2, grid=50):

    points = numpy.take(X, [at1, at2], axis=1)
    maxx, maxy = numpy.max(points, axis=0)
    minx, miny = numpy.min(points, axis=0)
    difx = maxx - minx
    dify = maxy - miny
    maxx += 0.02*difx
    minx -= 0.02*difx
    maxy += 0.02*dify
    miny -= 0.02*dify

    for c,(x,y) in zip(y,points):
        pylab.text(x,y,str(c), ha="center", va="center")
        pylab.scatter([x],[y],c=["b","r"][c!=0], s=200)

    num = grid
    prob = numpy.zeros([num, num])
    for xi,x in enumerate(numpy.linspace(minx, maxx, num=num)):
        for yi,y in enumerate(numpy.linspace(miny, maxy, num=num)):
            #probability of the closest example
            diff = points - numpy.array([x,y])
            dists = (diff[:,0]**2 + diff[:,1]**2)**0.5 #euclidean
            ind = numpy.argsort(dists)
            prob[yi,xi] = classifier(X[ind[0]])[1]

    pylab.imshow(prob, extent=(minx,maxx,maxy,miny))

    pylab.xlim(minx, maxx)
    pylab.ylim(miny, maxy)
    pylab.xlabel(at1)
    pylab.ylabel(at2)

    pylab.show()

X,y = solution.load('reg.data')

learner = solution.LogRegLearner(lambda_=0.)
classifier = learner(X,y)

draw_decision(X, y, classifier, 0, 1)

learner = solution.LogRegLearner(lambda_=0.01)
classifier = learner(X,y)

draw_decision(X, y, classifier, 0, 1)

learner = solution.LogRegLearner(lambda_=0.3)
classifier = learner(X,y)

draw_decision(X, y, classifier, 0, 1)

for lam in [0.0, 0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.5]:
    learner = solution.LogRegLearner(lambda_=lam)
    res = solution.test_cv(learner, X, y)
    res1 = solution.test_learning(learner, X, y)
    print("Tocnost cv:", round(solution.CA(y, res), 3))  # argumenta sta pravi razredi, napovedani
    print("Tocnost learning:", round(solution.CA(y, res1), 3))  # argumenta sta pravi razredi, napovedani
    print("\n")

max_stars_repo_path: Logisticna regresija/rezultati.py

max_stars_repo_name: mbencina/data-mining

max_issues_repo_path: Logisticna regresija/rezultati.py

max_issues_repo_name: mbencina/data-mining

max_forks_repo_path: Logisticna regresija/rezultati.py

max_forks_repo_name: mbencina/data-mining


text:
#!/usr/bin/python
'''
Module: read_and_map_mod_aerosol.py
==========================================================================================
Disclaimer: The code is for demonstration purposes only. Users are responsible to check for accuracy and revise to fit their objective.

Author: Justin Roberts-Pierel, 2015
Organization: NASA ARSET
Purpose: To extract AOD data from a MODIS HDF4 file (or series of files) and create a map of the resulting data

See the README associated with this module for more information.
==========================================================================================
'''

#import necessary modules
from pyhdf import SD
import numpy as np
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import sys

#This uses the file "fileList.txt", containing the list of files, in order to read the files
try:
	fileList=open('fileList.txt','r')
except:
	print('Did not find a text file containing file names (perhaps name does not match)')
	sys.exit()

#loops through all files listed in the text file
for FILE_NAME in fileList:
	FILE_NAME=FILE_NAME.strip()
	user_input=input('\nWould you like to process\n' + FILE_NAME + '\n\n(Y/N)')
	if(user_input == 'N' or user_input == 'n'):
		continue
	else:
		if '3K' in FILE_NAME:#then this is a 3km MODIS file
			print('This is a 3km MODIS file. Here is some information: ')
			SDS_NAME='Optical_Depth_Land_And_Ocean' # The name of the sds to read
		elif 'L2' in FILE_NAME: #Same as above but for 10km MODIS file
			print('This is a 10km MODIS file. Here is some information: ')
			SDS_NAME='AOD_550_Dark_Target_Deep_Blue_Combined'
		else:#if it is neither 3km nor 10km, then this will skip the rest of this loop iteration
			print('The file :',FILE_NAME, ' is not a valid MODIS file (Or is named incorrectly). \n')
			continue
		try:
			# open the hdf file for reading
			hdf=SD.SD(FILE_NAME)
		except:
			print('Unable to open file: \n' + FILE_NAME + '\n Skipping...')
			continue

		# Get lat and lon info
		lat = hdf.select('Latitude')
		latitude = lat[:]
		min_lat=latitude.min()
		max_lat=latitude.max()
		lon = hdf.select('Longitude')
		longitude = lon[:]
		min_lon=longitude.min()
		max_lon=longitude.max()

		#get AOD SDS, or exit if it doesn't find the SDS in the file
		try:
			sds=hdf.select(SDS_NAME)
		except:
			print('Sorry, your MODIS hdf file does not contain the SDS:',SDS_NAME,'. Please try again with the correct file type.')
			sys.exit()
		#get scale factor for AOD SDS
		attributes=sds.attributes()
		scale_factor=attributes['scale_factor']
		#get valid range for AOD SDS
		range=sds.getrange()
		min_range=min(range)
		max_range=max(range)

		#get SDS data
		data=sds.get()
		#get data within valid range
		valid_data=data.ravel()
		valid_data=[x for x in valid_data if x>=min_range]
		valid_data=[x for x in valid_data if x<=max_range]
		valid_data=np.asarray(valid_data)
		#scale the valid data
		valid_data=valid_data*scale_factor
		#find the average
		average=sum(valid_data)/len(valid_data)
		#find the standard deviation
		stdev=np.std(valid_data)
		#print information
		print('\nThe valid range of values is: ',round(min_range*scale_factor,3), ' to ',round(max_range*scale_factor,3),'\nThe average is: ',round(average,3),'\nThe standard deviation is: ',round(stdev,3))
		print('The range of latitude in this file is: ',min_lat,' to ',max_lat, 'degrees \nThe range of longitude in this file is: ',min_lon, ' to ',max_lon,' degrees')

		#Asks user if they would like to see a map
		is_map=str(input('\nWould you like to create a map of this data? Please enter Y or N \n'))
		#if user would like a map, view it
		if is_map == 'Y' or is_map == 'y':
			attrs = sds.attributes(full=1)
			fillvalue=attrs['_FillValue']
			# fillvalue[0] is the attribute value (-9999)
			fv = fillvalue[0]
			#turn fillvalues to NaN
			data=data.astype(float)
			data[data == fv] = np.nan
			#create the map
			data = np.ma.masked_array(data, np.isnan(data))
			m = Basemap(projection='cyl', resolution='l', llcrnrlat=min_lat, urcrnrlat = max_lat, llcrnrlon=min_lon, urcrnrlon = max_lon)
			m.drawcoastlines(linewidth=0.5)
			m.drawparallels(np.arange(-90., 120., 5.), labels=[1, 0, 0, 0])
			m.drawmeridians(np.arange(-180., 181., 5.), labels=[0, 0, 0, 1])
			x, y = m(longitude, latitude)
			m.pcolormesh(x, y, data*scale_factor)
			plt.autoscale()
			#create colorbar
			cb = m.colorbar()
			#label colorboar
			cb.set_label('AOD')

			#title the plot
			plotTitle=FILE_NAME[:-4]
			plt.title('{0}\n {1}'.format(plotTitle, SDS_NAME))
			fig = plt.gcf()
			# Show the plot window.
			plt.show()
			#once you close the map it asks if you'd like to save it
			is_save=str(input('\nWould you like to save this map? Please enter Y or N \n'))
			if is_save == 'Y' or is_save == 'y':
				#saves as a png if the user would like
				pngfile = '{0}.png'.format(plotTitle)
				fig.savefig(pngfile)
print('\nAll valid files have been processed.')


text:
"""
  Synthetic multi-objective function for 3D optimisation.
  -- kirthevasank
"""

import numpy as np

num_objectives = 2

def compute_objectives(x):
  """ Computes the objectives. """
  vol1 = x[0] # Na2SO4
  vol2 = x[1] # NaNO3
  vol3 = x[2] # LiNO3
  vol4 = x[3] # Li2SO4
  vol5 = x[4] # NaClO4
  vol6 = 10 - (vol1 + vol2 + vol3 + vol4 + vol5) # Water
  # Synthetic functions
  conductivity = 0.9 * (vol1 + vol5) + 0.1 * (vol2 + vol3) ** 2 + \
                 0.8 * vol4 * vol3 + 2.1 * vol6 * (vol1 ** 1.5)
  voltage_window = 0.5 * (vol3 + vol2) ** 1.7 + 0.9 * vol1 * vol4 + \
                   1.2 * (vol4 ** 0.5) * (vol6 ** 1.5)
  # Add Gaussian noise to simulate experimental noise
  conductivity += np.random.normal() * 0.01
  voltage_window += np.random.normal() * 0.01
  return [conductivity, voltage_window]


text:
# !/usr/bin/python
# -*- coding:utf-8 -*-
# Author: Shengjia Yan
# Date: 2017/3/3
# Email: sjyan@seu.edu.cn
# 实验的第四步：利用多知识库间sameAs关系提升链接质量

from urllib import unquote
import networkx as nx
from table import *
import json
import random

class SameAs(object):
    # table_name: 表格文件的名称
    # table_path: 表格文件的路径
    # baidubaike_edg_path: baidubaike 的实体消岐图路径
    # hudongbaike_edg_path: hudongbaike 的实体消岐图路径
    # zhwiki_edg_path: zhwiki 的实体消岐图路径
    # result_path: 最终实体链接结果的输出路径
    # baidubaike_hudongbaike_sameas_path: baidubaike 和 hudongbaike 的 sameas 数据文件路径
    # hudongbaike_zhwiki_sameas_path: hudongbaike 和 zhwiki 的 sameas 数据文件路径
    # zhwiki_baidubaike_sameas_path: zhwiki 和 baidubaike 的 sameas 数据文件路径
    def __init__(self, table_name, table_path, baidubaike_edg_path, hudongbaike_edg_path, zhwiki_edg_path, baidubaike_hudongbaike_sameas_path, hudongbaike_zhwiki_sameas_path, zhwiki_baidubaike_sameas_path, result_path):
        table_manager = TableManager(table_path)
        self.tables = table_manager.get_tables()  # 表格文件中所有表格，tables 是 Table 类型数组
        self.table_name = table_name
        self.table_path = table_path
        self.table_quantity = table_manager.table_quantity
        self.baidubaike_edg_path = baidubaike_edg_path
        self.hudongbaike_edg_path = hudongbaike_edg_path
        self.zhwiki_edg_path = zhwiki_edg_path
        self.result_path = result_path
        self.baidubaike_edg = nx.Graph()
        self.hudongbaike_edg = nx.Graph()
        self.zhwiki_edg = nx.Graph()
        self.baidubaike_hudongbaike_sameas_path = baidubaike_hudongbaike_sameas_path
        self.hudongbaike_zhwiki_sameas_path = hudongbaike_zhwiki_sameas_path
        self.zhwiki_baidubaike_sameas_path = zhwiki_baidubaike_sameas_path
        self.baidubaike_hudongbaike_sameas = []
        self.hudongbaike_zhwiki_sameas = []
        self.zhwiki_baidubaike_sameas = []
        self.kb_quantity = 3

    # 从 sameAs 文件中抽取三个知识库两两之间的 sameAs 关系
    def extract_sameAs(self):
        # input path
        baidubaike_hudongbaike_sameas_path = '../../../data/raw/sameAs/2.9_baidubaike_hudongbaike_links_zh.nt'
        hudongbaike_zhwiki_sameas_path = '../../../data/raw/sameAs/2.9_hudongbaike_zhwiki_links_zh.nt'
        zhwiki_baidubaike_sameas_path = '../../../data/raw/sameAs/2.9_zhwiki_baidubaike_links_zh.nt'

        # output path
        baidubaike_hudongbaike_sameas_output_path = '../../../data/sameas/baidubaike_hudongbaike_sameas.txt'
        hudongbaike_zhwiki_sameas_output_path = '../../../data/sameas/hudongbaike_zhwiki_sameas.txt'
        zhwiki_baidubaike_sameas_output_path = '../../../data/sameas/zhwiki_baidubaike_sameas.txt'

        # baidubaike - hudongbaike
        try:
            baidubaike_hudongbaike_sameas_ifile = open(baidubaike_hudongbaike_sameas_path, 'r')     # quantity: 1742106
            baidubaike_hudongbaike_sameas_ofile = open(baidubaike_hudongbaike_sameas_output_path, 'a')

            for line in baidubaike_hudongbaike_sameas_ifile.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                baidubaike_entity = split[0]
                hudongbaike_entity = split[2]

                # clean
                baidubaike_entity = baidubaike_entity[1:]
                baidubaike_entity = baidubaike_entity.replace('http://zhishi.me/baidubaike/resource/', '')
                hudongbaike_entity = hudongbaike_entity[:-3]
                hudongbaike_entity = hudongbaike_entity.replace('http://zhishi.me/hudongbaike/resource/', '')

                # convert
                baidubaike_entity = unquote(baidubaike_entity)
                hudongbaike_entity = unquote(hudongbaike_entity)

                baidubaike_hudongbaike_link = '<' + baidubaike_entity + '> <' + hudongbaike_entity + '>\n'

                baidubaike_hudongbaike_sameas_ofile.write(baidubaike_hudongbaike_link)

        finally:
            if baidubaike_hudongbaike_sameas_ifile:
                baidubaike_hudongbaike_sameas_ifile.close()

            if baidubaike_hudongbaike_sameas_ofile:
                baidubaike_hudongbaike_sameas_ofile.close()

        # hudongbaike - zhwiki
        try:
            hudongbaikee_zhwiki_sameas_ifile = open(hudongbaike_zhwiki_sameas_path, 'r')        # quantity: 189024
            hudongbaike_zhwiki_sameas_ofile = open(hudongbaike_zhwiki_sameas_output_path, 'a')

            for line in hudongbaikee_zhwiki_sameas_ifile.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                hudongbaike_entity = split[0]
                zhwiki_entity = split[2]

                # clean
                hudongbaike_entity = hudongbaike_entity[1:]
                hudongbaike_entity = hudongbaike_entity.replace('http://zhishi.me/hudongbaike/resource/', '')
                zhwiki_entity = zhwiki_entity[:-3]
                zhwiki_entity = zhwiki_entity.replace('http://zhishi.me/zhwiki/resource/', '')

                # convert
                hudongbaike_entity = unquote(hudongbaike_entity)
                zhwiki_entity = unquote(zhwiki_entity)

                hudongbaike_zhwiki_link = '<' + hudongbaike_entity + '> <' + zhwiki_entity + '>\n'

                hudongbaike_zhwiki_sameas_ofile.write(hudongbaike_zhwiki_link)

        finally:
            if hudongbaikee_zhwiki_sameas_ifile:
                hudongbaikee_zhwiki_sameas_ifile.close()

            if hudongbaike_zhwiki_sameas_ofile:
                hudongbaike_zhwiki_sameas_ofile.close()

        # zhwiki - baidubaike
        try:
            zhwiki_baidubaike_sameas_ifile = open(zhwiki_baidubaike_sameas_path, 'r')  # quantity: 206814
            zhwiki_baidubaike_sameas_ofile = open(zhwiki_baidubaike_sameas_output_path, 'a')

            for line in zhwiki_baidubaike_sameas_ifile.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                zhwiki_entity = split[0]
                baidubaike_entity = split[2]

                # clean
                zhwiki_entity = zhwiki_entity[1:]
                zhwiki_entity = zhwiki_entity.replace('http://zhishi.me/zhwiki/resource/', '')
                baidubaike_entity = baidubaike_entity[:-3]
                baidubaike_entity = baidubaike_entity.replace('http://zhishi.me/baidubaike/resource/', '')

                # convert
                zhwiki_entity = unquote(zhwiki_entity)
                baidubaike_entity = unquote(baidubaike_entity)

                zhwiki_baidubaike_link = '<' + zhwiki_entity + '> <' + baidubaike_entity + '>\n'

                zhwiki_baidubaike_sameas_ofile.write(zhwiki_baidubaike_link)

        finally:
            if zhwiki_baidubaike_sameas_ifile:
                zhwiki_baidubaike_sameas_ifile.close()

            if zhwiki_baidubaike_sameas_ofile:
                zhwiki_baidubaike_sameas_ofile.close()

    # 加载实体消岐图
    # table_number: 当前表格的编号
    def load_entity_disambiguation_graph(self, table_number):
        # baidubaike
        baidubaike_edg_path = self.baidubaike_edg_path + 'edg' + str(table_number) + '.txt'
        self.baidubaike_edg = nx.read_gpickle(baidubaike_edg_path)

        # hudongbaike
        hudongbaike_edg_path = self.hudongbaike_edg_path + 'edg' + str(table_number) + '.txt'
        self.hudongbaike_edg = nx.read_gpickle(hudongbaike_edg_path)

        # zhwiki
        zhwiki_edg_path = self.zhwiki_edg_path + 'edg' + str(table_number) + '.txt'
        self.zhwiki_edg = nx.read_gpickle(zhwiki_edg_path)

    # 判断分别来自 kb1 和 kb2 的 e1 与 e2 是否存在 sameAs 关系
    def isSameAs(self, e1, kb1, e2, kb2):
        # e1 (baidubaike) - e2 (hudongbaike)
        if kb1 == 'baidubaike' and kb2 == 'hudongbaike':
            for dict in self.baidubaike_hudongbaike_sameas:
                if e1 == dict['baidubaike_entity'] and e2 == dict['hudongbaike_entity']:
                    return True

        # e1 (hudongbaike) - e2 (baidubaike)
        if kb1 == 'hudongbaike' and kb2 == 'baidubaike':
            for dict in self.baidubaike_hudongbaike_sameas:
                if e1 == dict['hudongbaike_entity'] and e2 == dict['baidubaike_entity']:
                    return True

        # e1 (hudongbaike) - e2 (zhwiki)
        if kb1 == 'hudongbaike' and kb2 == 'zhwiki':
            for dict in self.hudongbaike_zhwiki_sameas:
                if e1 == dict['hudongbaike_entity'] and e2 == dict['zhwiki_entity']:
                    return True

        # e1 (zhwiki) - e2 (hudongbaike)
        if kb1 == 'zhwiki' and kb2 == 'hudongbaike':
            for dict in self.baidubaike_hudongbaike_sameas:
                if e1 == dict['zhwiki_entity'] and e2 == dict['hudongbaike_entity']:
                    return True

        # e1 (baidubaike) - e2 (zhwiki)
        if kb1 == 'baidubaike' and kb2 == 'zhwiki':
            for dict in self.zhwiki_baidubaike_sameas:
                if e1 == dict['baidubaike_entity'] and e2 == dict['zhwiki_entity']:
                    return True

        # e1 (zhwiki) - e2 (baidubaike)
        if kb1 == 'zhwiki' and kb2 == 'baidubaike':
            for dict in self.baidubaike_hudongbaike_sameas:
                if e1 == dict['zhwiki_entity'] and e2 == dict['baidubaike_entity']:
                    return True

        return False

    def conbine_single_kb_el_result(self):
        baidubaike_result_path = '../../../data/disambiguation/baidubaike/result/'
        hudongbaike_result_path = '../../../data/disambiguation/hudongbaike/result/'
        zhwiki_result_path = '../../../data/disambiguation/zhwiki/result/'

        single_kb_el_result_path = self.result_path + 'single_kb_el_result.txt'

        try:
            single_kb_el_result_file = open(single_kb_el_result_path, 'w')
            whole = []

            for i in range(self.table_quantity):
                table = self.tables[i]
                row_num = table.row_num
                col_num = table.col_num
                t = []

                # baidubaike
                baidubaike_result_file_path = baidubaike_result_path + str(i) + '.txt'
                baidubaike_result_file = open(baidubaike_result_file_path, 'r').read()
                baidubaike_result_json = json.loads(baidubaike_result_file)

                # hudongbaike
                hudongbaike_result_file_path = hudongbaike_result_path + str(i) + '.txt'
                hudongbaike_result_file = open(hudongbaike_result_file_path, 'r').read()
                hudongbaike_result_json = json.loads(hudongbaike_result_file)

                # zhwiki
                zhwiki_result_file_path = zhwiki_result_path + str(i) + '.txt'
                zhwiki_result_file = open(zhwiki_result_file_path, 'r').read()
                zhwiki_result_json = json.loads(zhwiki_result_file)

                for r in range(row_num):
                    row = []
                    for c in range(col_num):
                        dict = {}       # {'header': h} or {'mention': m, 'entity': [(entity1, kb1), (entity2, kb2), (entity3, kb3)]}

                        if r == 0:
                            dict['header'] = baidubaike_result_json[r][c]['header']
                            row.append(dict)
                        else:
                            dict['mention'] = baidubaike_result_json[r][c]['mention']

                            entity_set = []     # [(entity1, kb1), (entity2, kb2), (entity3, kb3)]

                            # baidubaike
                            kb = 'baidubaike'
                            baidubaike_entity = baidubaike_result_json[r][c]['entity']
                            tuple = (baidubaike_entity, kb)
                            entity_set.append(tuple)

                            # hudongbaike
                            kb = 'hudongbaike'
                            hudongbaike_entity = hudongbaike_result_json[r][c]['entity']
                            tuple = (hudongbaike_entity, kb)
                            entity_set.append(tuple)

                            # zhwiki
                            kb = 'zhwiki'
                            zhwiki_entity = zhwiki_result_json[r][c]['entity']
                            tuple = (zhwiki_entity, kb)
                            entity_set.append(tuple)

                            dict['entity'] = entity_set
                            row.append(dict)
                    t.append(row)
                whole.append(t)

        finally:
            single_kb_el_result_json = json.dumps(whole, ensure_ascii=False)
            single_kb_el_result_file.write(single_kb_el_result_json)

            if single_kb_el_result_file:
                single_kb_el_result_file.close()

    # 挑选每个 mention 的候选实体概率排名列表中的前3个，来做多知识库消岐
    # 我们认为来自同一个知识库的候选实体是不存在 SameAs 关系的，因为同一个实体文件中不同实体就不存在 SameAs 关系
    def rerank(self):
        tables = self.tables
        half_number_of_kb = float(self.kb_quantity) / 2

        try:
            baidubaike_hudongbaike_sameas_file = open(self.baidubaike_hudongbaike_sameas_path, 'r')
            hudongbaike_zhwiki_sameas_file = open(self.hudongbaike_zhwiki_sameas_path, 'r')
            zhwiki_baidubaike_sameas_file = open(self.zhwiki_baidubaike_sameas_path, 'r')
            single_kb_el_result_path = self.result_path + 'single_kb_el_result.txt'
            single_kb_el_result_file = open(single_kb_el_result_path, 'r').read()
            single_kb_el_result_json = json.loads(single_kb_el_result_file)
            multiple_kb_el_result_path = self.result_path + 'multiple_kb_el_result.txt'
            multiple_kb_el_result_file = open(multiple_kb_el_result_path, 'w')

            # process baidubaike - hudongbaiek sameas data
            for line in baidubaike_hudongbaike_sameas_file.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                baidubaike_entity = split[0]
                hudongbaike_entity = split[1]

                # clean
                baidubaike_entity = baidubaike_entity[1:]
                hudongbaike_entity = hudongbaike_entity[:-1]

                dict = {}
                dict['baidubaike_entity'] = baidubaike_entity
                dict['hudongbaike_entity'] = hudongbaike_entity
                self.baidubaike_hudongbaike_sameas.append(dict)

            # process hudongbaike - zhwiki sameas data
            for line in hudongbaike_zhwiki_sameas_file.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                hudongbaike_entity = split[0]
                zhwiki_entity = split[1]

                # clean
                hudongbaike_entity = hudongbaike_entity[1:]
                zhwiki_entity = zhwiki_entity[:-1]

                dict = {}
                dict['hudongbaike_entity'] = hudongbaike_entity
                dict['zhwiki_entity'] = zhwiki_entity
                self.hudongbaike_zhwiki_sameas.append(dict)

            # process zhwiki - baidubaike sameas data
            for line in zhwiki_baidubaike_sameas_file.readlines():
                line = line.strip('\n')

                # split
                split = line.split('> <')
                zhwiki_entity = split[0]
                baidubaike_entity = split[1]

                # clean
                zhwiki_entity = zhwiki_entity[1:]
                baidubaike_entity = baidubaike_entity[:-1]

                dict = {}
                dict['zhwiki_entity'] = zhwiki_entity
                dict['baidubaike_entity'] = baidubaike_entity
                self.zhwiki_baidubaike_sameas.append(dict)

            whole = []

            # i: 第i张表格，从0开始
            for i in range(self.table_quantity):
                t = []
                table = tables[i]
                table_number = i
                row_num = table.row_num
                col_num = table.col_num
                mention_counter = 0

                self.load_entity_disambiguation_graph(table_number)

                baidubaike_edg = self.baidubaike_edg
                hudongbaike_edg = self.hudongbaike_edg
                zhwiki_edg = self.zhwiki_edg

                # 逐行逐列遍历表格 i 中的每个单元格
                # i: table number
                # r: row number
                # c: column number
                for r in range(row_num):
                    row = []

                    for c in range(col_num):
                        cell = {}   # {'header': h} or {'mention': m, 'entity': [(entity1, kb1), (entity2, kb2), (entity3, kb3)]}
                        mention = table.get_cell(r, c)  # unicode

                        if r == 0:  # 表头不做实体链接
                            cell['header'] = mention
                            row.append(cell)
                        else:
                            cell['mention'] = mention

                            baidubaike_ranking_list = []    # [(entity1, kb1, rank), (entity2, kb1, rank), (entity3, kb1, rank)]
                            hudongbaike_ranking_list = []
                            zhwiki_ranking_list = []

                            # get baidubaike ranking list
                            kb = 'baidubaike'
                            ranking = baidubaike_edg.node[mention_counter]['ranking']   # [(entity1_number, probability1), (entity2_number, probability2)]
                            rank = 1

                            if baidubaike_edg.node[mention_counter]['NIL'] == False:
                                if len(ranking) < 3:
                                    for tuple in ranking:
                                        entity_number = tuple[0]
                                        entity = baidubaike_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        baidubaike_ranking_list.append(entity_kb_tuple)
                                        rank += 1
                                else:
                                    for p in range(3):
                                        entity_number = ranking[p][0]
                                        entity = baidubaike_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        baidubaike_ranking_list.append(entity_kb_tuple)
                                        rank += 1

                            # get hudongbaike ranking list
                            kb = 'hudongbaike'
                            ranking = hudongbaike_edg.node[mention_counter]['ranking']  # [(entity1_number, probability1), (entity2_number, probability2)]
                            rank = 1

                            if hudongbaike_edg.node[mention_counter]['NIL'] == False:
                                if len(ranking) < 3:
                                    for tuple in ranking:
                                        entity_number = tuple[0]
                                        entity = hudongbaike_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        hudongbaike_ranking_list.append(entity_kb_tuple)
                                        rank += 1
                                else:
                                    for p in range(3):
                                        entity_number = ranking[p][0]
                                        entity = hudongbaike_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        hudongbaike_ranking_list.append(entity_kb_tuple)
                                        rank += 1

                            # get zhwiki ranking list
                            kb = 'zhwiki'
                            ranking = zhwiki_edg.node[mention_counter]['ranking']  # [(entity1_number, probability1), (entity2_number, probability2)]
                            rank = 1

                            if zhwiki_edg.node[mention_counter]['NIL'] == False:
                                if len(ranking) < 3:
                                    for tuple in ranking:
                                        entity_number = tuple[0]
                                        entity = zhwiki_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        zhwiki_ranking_list.append(entity_kb_tuple)
                                        rank += 1
                                else:
                                    for p in range(3):
                                        entity_number = ranking[p][0]
                                        entity = zhwiki_edg.node[entity_number]['candidate']
                                        entity_kb_tuple = (entity, kb, rank)
                                        zhwiki_ranking_list.append(entity_kb_tuple)
                                        rank += 1

                            mention_counter += 1

                            list = []   # [{'set': [(entity1, kb1, rank), (entity2, kb2, rank)], 'average_ranking': 1.25, 'average_ranking_rank': 1, 'highest_ranking': 1, 'highest_ranking_rank': 1, 'set_size': size of the set, 'flag_less': True}]

                            # baidubaike - hudongbaike - zhwiki
                            for baidubaike_candidate_index in range(len(baidubaike_ranking_list)):
                                set = []    # [(entity1, kb1, rank), (entity2, kb2, rank)]
                                baidubaike_entity = baidubaike_ranking_list[baidubaike_candidate_index][0]
                                set.append(baidubaike_ranking_list[baidubaike_candidate_index])
                                flag_hudongbaike_pop = False

                                for hudongbaike_candidate_index in range(len(hudongbaike_ranking_list)):
                                    hudongbaike_entity = hudongbaike_ranking_list[hudongbaike_candidate_index][0]

                                    if self.isSameAs(baidubaike_entity, 'baidubaike', hudongbaike_entity, 'hudongbaike'):
                                        set.append(hudongbaike_ranking_list[hudongbaike_candidate_index])
                                        hudongbaike_pop_index = hudongbaike_candidate_index
                                        flag_hudongbaike_pop = True
                                        flag_zhwiki_pop = False

                                        for zhwiki_candidate_index in range(len(zhwiki_ranking_list)):
                                            zhwiki_entity = zhwiki_ranking_list[zhwiki_candidate_index][0]

                                            if self.isSameAs(hudongbaike_entity, 'hudongbaike', zhwiki_entity, 'zhwiki'):
                                                set.append(zhwiki_ranking_list[zhwiki_candidate_index])
                                                zhwiki_pop_index = zhwiki_candidate_index
                                                flag_zhwiki_pop = True
                                                break

                                        if flag_zhwiki_pop:
                                            zhwiki_ranking_list.pop(zhwiki_pop_index)

                                        break

                                if flag_hudongbaike_pop:
                                    hudongbaike_ranking_list.pop(hudongbaike_pop_index)

                                flag_zhwiki_pop = False
                                for zhwiki_candidate_index in range(len(zhwiki_ranking_list)):
                                    zhwiki_entity = zhwiki_ranking_list[zhwiki_candidate_index][0]

                                    if self.isSameAs(baidubaike_entity, 'baidubaike', zhwiki_entity, 'zhwiki'):
                                        set.append(zhwiki_ranking_list[zhwiki_candidate_index])
                                        zhwiki_pop_index = zhwiki_candidate_index
                                        flag_zhwiki_pop = True
                                        break

                                if flag_zhwiki_pop:
                                    zhwiki_ranking_list.pop(zhwiki_pop_index)

                                # build dict
                                dict = {}  # {'set': [(entity1, kb1, rank), (entity2, kb2, rank)], 'average_ranking': 1.25, 'average_ranking_rank': 1, 'highest_ranking': 1, 'highest_ranking_rank': 1, 'set_size': size of the set, 'flag_less': True}
                                set_size = len(set)
                                dict['set_size'] = set_size
                                dict['set'] = set

                                if set_size < half_number_of_kb:
                                    dict['flag_less'] = True
                                else:
                                    dict['flag_less'] = False

                                average_ranking = 0
                                highest_ranking = 3

                                for candidate in set:
                                    rank = candidate[2]

                                    if rank < highest_ranking:
                                        highest_ranking = rank

                                    average_ranking += rank

                                average_ranking /= float(set_size)
                                dict['average_ranking'] = average_ranking
                                dict['highest_ranking'] = highest_ranking
                                dict['average_ranking_rank'] = 1
                                dict['highest_ranking_rank'] = 1
                                list.append(dict)

                            # hudongbaike - zhwiki
                            for hudongbaike_candidate_index in range(len(hudongbaike_ranking_list)):
                                set = []        # [(entity1, kb1, rank), (entity2, kb2, rank)]
                                hudongbaike_entity = hudongbaike_ranking_list[hudongbaike_candidate_index][0]
                                set.append(hudongbaike_ranking_list[hudongbaike_candidate_index])
                                flag_zhwiki_pop = False

                                for zhwiki_candidate_index in range(len(zhwiki_ranking_list)):
                                    zhwiki_entity = zhwiki_ranking_list[zhwiki_candidate_index][0]

                                    if self.isSameAs(hudongbaike_entity, 'hudongbaike', zhwiki_entity, 'zhwiki'):
                                        set.append(zhwiki_ranking_list[zhwiki_candidate_index])
                                        zhwiki_pop_index = zhwiki_candidate_index
                                        flag_zhwiki_pop = True
                                        break

                                if flag_zhwiki_pop:
                                    zhwiki_ranking_list.pop(zhwiki_pop_index)

                                # build dict
                                dict = {}
                                set_size = len(set)
                                dict['set_size'] = set_size
                                dict['set'] = set

                                if set_size < half_number_of_kb:
                                    dict['flag_less'] = True
                                else:
                                    dict['flag_less'] = False

                                average_ranking = 0
                                highest_ranking = 3

                                for candidate in set:
                                    rank = candidate[2]

                                    if rank < highest_ranking:
                                        highest_ranking = rank

                                    average_ranking += rank

                                average_ranking /= float(set_size)
                                dict['average_ranking'] = average_ranking
                                dict['highest_ranking'] = highest_ranking
                                dict['average_ranking_rank'] = 1
                                dict['highest_ranking_rank'] = 1
                                list.append(dict)

                            # zhwiki
                            for zhwiki_candidate_index in range(len(zhwiki_ranking_list)):
                                set = []        # [(entity1, kb1, rank), (entity2, kb2, rank)]
                                set.append(zhwiki_ranking_list[zhwiki_candidate_index])

                                # build dict
                                dict = {}
                                set_size = len(set)
                                dict['set_size'] = set_size
                                dict['set'] = set

                                if set_size < half_number_of_kb:
                                    dict['flag_less'] = True
                                else:
                                    dict['flag_less'] = False

                                average_ranking = 0
                                highest_ranking = 3

                                for candidate in set:
                                    rank = candidate[2]

                                    if rank < highest_ranking:
                                        highest_ranking = rank

                                    average_ranking += rank

                                average_ranking /= float(set_size)
                                dict['average_ranking'] = average_ranking
                                dict['highest_ranking'] = highest_ranking
                                dict['average_ranking_rank'] = 1
                                dict['highest_ranking_rank'] = 1
                                list.append(dict)

                            # rank the average_ranking
                            list.sort(key=lambda x: x['average_ranking'])  # 按照 average_ranking 顺序排序，下标越大 average_ranking 越大
                            current_rank = 1

                            for index in range(len(list)):
                                if index == 0:
                                    list[index]['average_ranking_rank'] = current_rank
                                else:
                                    if list[index]['average_ranking'] > list[index-1]['average_ranking']:
                                        current_rank += 1
                                        list[index]['average_ranking_rank'] = current_rank
                                    else:
                                        list[index]['average_ranking_rank'] = current_rank

                            # rank the highest_ranking
                            list.sort(key=lambda x: x['highest_ranking'])  # 按照 highest_ranking 顺序排序，下标越大 highest_ranking 越大
                            current_rank = 1

                            for index in range(len(list)):
                                if index == 0:
                                    list[index]['highest_ranking_rank'] = current_rank
                                else:
                                    if list[index]['highest_ranking'] > list[index - 1]['highest_ranking']:
                                        current_rank += 1
                                        list[index]['highest_ranking_rank'] = current_rank
                                    else:
                                        list[index]['highest_ranking_rank'] = current_rank

                            # Rule 3
                            flag_less = True

                            for d in list:
                                if d['flag_less'] == False:
                                    flag_less = False
                                    break

                            if flag_less:
                                cell['entity'] = single_kb_el_result_json[i][r][c]['entity']
                                row.append(cell)
                                continue

                            # Rule 1 & Rule 2
                            final_candidate_set_list = []   # [[(entity1, kb1), (entity2, kb2)], [(entity3, kb3)]]

                            for d in list:
                                if d['average_ranking_rank'] == 1 and d['highest_ranking_rank'] == 1 and d['flag_less'] == False:
                                    set = d['set']
                                    entity_kb_set = []      # [(entity1, kb1), (entity2, kb2)]

                                    for triple in set:
                                        entity = triple[0]
                                        kb = triple[1]
                                        tuple = (entity, kb)
                                        entity_kb_set.append(tuple)

                                    final_candidate_set_list.append(entity_kb_set)

                            if len(final_candidate_set_list) == 0:
                                cell['entity'] = single_kb_el_result_json[i][r][c]['entity']
                                row.append(cell)
                            elif len(final_candidate_set_list) == 1:
                                cell['entity'] = final_candidate_set_list[0]
                                row.append(cell)
                            else:
                                cell['entity'] = final_candidate_set_list[random.randint(0, len(final_candidate_set_list) - 1)]
                                row.append(cell)
                    t.append(row)
                whole.append(t)

        finally:
            multiple_kb_el_result_json = json.dumps(whole, ensure_ascii=False)
            multiple_kb_el_result_file.write(multiple_kb_el_result_json)

            if baidubaike_hudongbaike_sameas_file:
                baidubaike_hudongbaike_sameas_file.close()

            if hudongbaike_zhwiki_sameas_file:
                hudongbaike_zhwiki_sameas_file.close()

            if zhwiki_baidubaike_sameas_file:
                zhwiki_baidubaike_sameas_file.close()

            if multiple_kb_el_result_file:
                multiple_kb_el_result_file.close()

max_stars_repo_name: yanshengjia/seu-graduation-project

max_issues_repo_name: yanshengjia/seu-graduation-project

max_forks_repo_name: yanshengjia/seu-graduation-project

reason: import networkx


text:
from threading import Thread

import cv2
import numpy as np
import math

from .camera import MaproomCamera
from .fps import FPS
from . import constants as c
from . import util as u

MIN_CONTOUR_AREA = 500

class IRFinder:
  def __init__(self, irCam, clientIPs=[], clientPort=5201):
    self.irCam = irCam
    self.clients = u.clientsFromIPs(clientIPs, clientPort)
    self.running = False

    self.avgPos = np.array([0,0], np.float32)
    self.normAvgPos = np.array([0,0], np.float32)

  def start(self):
    self.running = True
    self.irCam.setPreundistort(True)

    t = Thread(target=self.update, args=(), daemon=True)
    t.start()
    return self

  def findCircle(self, ir, draw=False):
    irROI = ir[c.mappedImageUL[0]:c.mappedImageBR[0], c.mappedImageUL[1]:c.mappedImageBR[1]]
    irROI = cv2.blur(irROI, (25,25))

    minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(irROI)

    pos = np.array(maxLoc, np.float32)
    normPos = pos / np.array(irROI.shape, np.float32)
    ret = None
    if maxVal > 25:
      self.avgPos += (pos - self.avgPos) * 0.3
      self.normAvgPos = self.avgPos / np.array(irROI.shape, np.float32)
      dist = np.linalg.norm(self.normAvgPos - normPos)

      ret = self.normAvgPos

    if draw:
      cv2.circle(ir, u.tup(pos + c.mappedImageUL), 50, 255, 2)
      cv2.circle(ir, u.tup(self.avgPos + c.mappedImageUL), 50, 255, 8)

    if ret is None:
      return ret
    else:
      return ret

  def update(self):
    fps = FPS()
    while True:
      if not self.running:
        return

      ir = self.irCam.grayUndistorted
      if ir is None:
        continue

      self.circleCenter = self.findCircle(ir)
      u.sendToClients(self.clients, "/ir", { 'circleCenter': self.circleCenter.tolist() })

      p, f, frameidx = fps.update()
      if p:
        print('IR FPS', f)

  def stop(self):
    self.running = False
    self.irCam.setPreundistort(False)

  def isRunning(self):
    return self.running

if __name__ == "__main__":
  cam1 = MaproomCamera(0)
  cam1.load('./calibrations')
  cam1.setPreundistort(True)
  cam1.start()

  irFinder = IRFinder(cam1, clientIPs=['192.168.7.20', '192.168.7.22', '192.168.7.23'])

  draw = True
  fps = FPS()
  lastcam1frameidx = 0
  lastcam2frameidx = 0
  while True:
    p, f, frameidx = fps.update()

    cam1.update()
    ir = cam1.grayUndistorted
    if ir is None:
      continue

    circleCenter = irFinder.findCircle(ir, draw=draw)
    u.sendToClients(irFinder.clients, "/ir", { 'circleCenter': circleCenter })

    if draw:
      frame = ir

      cv2.imshow('frame', frame)
      if cv2.waitKey(1) & 0xFF == ord('q'):
        break

    if p:
      print('FPS', f)

max_stars_repo_name: O-C-R/maproom-robots


text:
import numpy as np
import pytest
import unyt as u
from unyt.testing import assert_allclose_units

from gmso import Topology
from gmso.formats.mol2 import from_mol2
from gmso.tests.base_test import BaseTest
from gmso.utils.io import get_fn

class TestMol2(BaseTest):
    def test_read_mol2(self):
        top = Topology.load(get_fn("parmed.mol2"))
        assert top.name == "parmed"
        assert top.n_sites == 8
        assert_allclose_units(
            top.box.lengths,
            ([8.2693, 7.9100, 6.6460] * u.Å).to("nm"),
            rtol=1e-5,
            atol=1e-8,
        )
        assert list(top.sites)[0].element.name == "carbon"
        assert_allclose_units(
            list(top.sites)[0].element.mass,
            np.array(1.9944733e-26) * u.kg,
            rtol=1e-5,
            atol=1e-8,
        )

        top = Topology.load(get_fn("tip3p.mol2"))
        assert top.name == "tip3p"
        assert top.n_sites == 3
        assert_allclose_units(
            top.box.lengths, 3.0130 * np.ones(3) * u.Å, rtol=1e-5, atol=1e-8
        )
        positions_check = [
            [0.061, 0.1, 0.1],
            [0.017, 0.09, 0.177],
            [0.011, 0.154, 0.04],
        ]
        for check, site in zip(positions_check, top.sites):
            assert_allclose_units(
                site.position,
                check * u.nm,
                rtol=1e-5,
                atol=1e-8,
            )

        top = Topology.load(get_fn("vmd.mol2"))
        assert top.name == "vmd"
        assert top.n_sites == 6
        assert len(top.bonds) == 5
        assert top.bonds[0].connection_members[0] == top.sites[0]
        assert top.box == None

        with pytest.warns(
            UserWarning,
            match=r"No charges were detected for site C with index 1",
        ):
            top = Topology.load(get_fn("ethane.mol2"))
        assert list(top.sites)[0].charge is None

        with pytest.warns(
            UserWarning,
            match=r"No element detected for site C with index1\, consider manually adding the element to the topology",
        ):
            Topology.load(get_fn("benzene.mol2"))

    def test_residue(self):
        top = Topology.load(get_fn("ethanol_aa.mol2"))
        assert np.all([site.residue_name == "ETO" for site in top.sites])
        assert np.all([site.residue_number == 1 for site in top.sites])

        top = Topology.load(get_fn("benzene_ua.mol2"), site_type="lj")
        assert np.all(
            [
                site.residue_name == "BEN1"
                for site in top.iter_sites("residue_name", "BEN1")
            ]
        )
        assert np.all(
            [
                site.residue_number == 1
                for site in top.iter_sites("residue_name", "BEN1")
            ]
        )
        assert np.all(
            [
                site.residue_name == "BEN2"
                for site in top.iter_sites("residue_name", "BEN2")
            ]
        )
        assert np.all(
            [
                site.residue_number == 2
                for site in top.iter_sites("residue_name", "BEN2")
            ]
        )

    def test_lj_system(self):
        top = Topology.load(get_fn("methane.mol2"), site_type="lj")
        assert np.all([site.element == None for site in top.sites])

    def test_wrong_path(self):
        with pytest.raises(
            OSError, match=r"Provided path to file that does not exist"
        ):
            Topology.load("not_a_file.mol2")
        top = Topology.load(get_fn("ethanegro.mol2"))
        assert len(top.sites) == 0
        assert len(top.bonds) == 0

    def test_broken_files(self):
        with pytest.warns(
            UserWarning,
            match=r"The record type indicator @<TRIPOS>MOLECULE_extra_text\n is not supported. Skipping current section and moving to the next RTI header.",
        ):
            Topology.load(get_fn("broken.mol2"))
        with pytest.warns(
            UserWarning,
            match=r"This mol2 file has two boxes to be read in, only reading in one with dimensions Box\(a=0.72",
        ):
            Topology.load(get_fn("broken.mol2"))

max_stars_repo_name: chrisiacovella/gmso

max_issues_repo_name: chrisiacovella/gmso

max_forks_repo_name: chrisiacovella/gmso


text:
import digitaldna as ddna
import csv
from digitaldna import Verbosity
from digitaldna import SequencePlots
import pandas as pd
import numpy as np
from digitaldna.lcs import LongestCommonSubsequence
from os import listdir
from matplotlib import pyplot as plt
import time
from digitaldna import SequencePlots

def main():

    #from os.path import isfile, join
    savepath1 = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/plot/"
    savepath = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/log_plot/"
    intra_seq_path = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/intra_seq_plot/"
    inter_seq_path = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/inter_seq_plot/"
    mypath="D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/timeline_per_id/"

    listafile=listdir(mypath)
    #print(listafile)

    for f in listafile:
        filepath=mypath+f
        print (filepath)
        #nome = f[3:]
        #nome = nome[:-4]
        nome = "non_scaricati_polarizzate" #assegnare nome in caso di file unico
        path_matrici = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/matrici/"+nome+"_glcr_cache"
        df = pd.read_csv(filepath)
        est = LongestCommonSubsequence(
            out_path= path_matrici,
            overwrite=True, window=5, threshold= "auto", verbosity= Verbosity.FILE_EXTENDED)
        est.fit(df["dna"])
        y = est.fit_predict(df["dna"])
        time.sleep(5)

        lcslog = est.plot_LCS_log()

        fname = savepath + nome +'log.png'
        lcslog.savefig(fname, dpi=None)
        lcslog.close()

        lcs_plot = est.plot_LCS()

        fname1 = savepath1 + nome + '.png'
        lcs_plot.savefig(fname1, dpi=None)
        lcs_plot.close()

        fname2 = intra_seq_path + nome +'.png'
        plotter = SequencePlots(alphabet='b3_type')
        intra_seq = plotter.plot_intrasequence_entropy(df["dna"])
        intra_seq.savefig(fname2, dpi=None)
        intra_seq.close()

        fname3 = inter_seq_path + nome + '.png'
        inter_seq = plotter.plot_intersequence_entropy(df["dna"])
        inter_seq.savefig(fname3, dpi=None)
        inter_seq.close()

        bot_path = "D:/Google Drive/Europee2019 CNR/SISP 2019/timeline utenti/non_scaricati_polarizzate/bot_python/"
        pd.DataFrame(y).to_csv(
            bot_path + nome +"_bot.csv",
            sep=';', encoding='utf-8')
        #exit()

    #salvare plot
    #salvare y (classificazione bot)
    #tagliare "df_" e "".csv" dai file png dei plot

if __name__ == '__main__':
    main()


text:
#!/usr/bin/env python
""" This is for E4 offline processing
The input data should be in the standard format of Empatica E4.

__Author__='Guangtao Nie'
__Institution__='RASL Lab, Vanderbilt Univ'
__version__='0.1'
"""

import pandas as pd
import numpy as np
import xgboost as xgb
import joblib
import argparse
from utils import Reader, Extract_Features
import os
import warnings

warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)

freqs = {'acc': 32, 'bvp': 64, 'eda': 4, 'temp': 4}
labels_annotation = {0: 'low', 1: 'high'}

def main(args):
    clf = joblib.load('GN_E4_stress.joblib')

    print(args.start,args.end)
    step_ratio=args.step/args.window
    reader = Reader(freqs, step_ratio, args.window)
    if len(args.start) > 1:
        start=[int(x) for x in args.start if x.isnumeric()]
        start.append(args.start[-1])
        start = reader.date_converter(*start)
    else:
        start = float(args.start[0])
    if len(args.end) > 1:
        end=[int(x) for x in args.end if x.isnumeric()]
        end.append(args.end[-1])
        end = reader.date_converter(*end)
    else:
        end = float(args.end[0])
    print('session start time, end time:', start, end)
    acc, bvp, eda, temp = reader.get_data(args.direc, start, end)

    extractor = Extract_Features(freqs)
    acc_features = extractor.acc_features(acc)
    bvp_features = extractor.bvp_features(bvp)
    eda_features = extractor.eda_features(eda)
    temp_features = extractor.temp_features(temp)
    extractor.eda_bvp_nan(eda_features)
    extractor.eda_bvp_nan(bvp_features)
    print('shapes:', acc.shape, bvp.shape,
          eda.shape, temp.shape)
    print('feature shapes:', acc_features.shape,
          bvp_features.shape, eda_features.shape, temp_features.shape)

    features = np.hstack([acc_features, bvp_features,
                          eda_features, temp_features])
    D_test = xgb.DMatrix(features)
    preds = clf.predict(D_test)
    best_preds = np.asarray([np.argmax(line) for line in preds])
    labels = np.asarray([labels_annotation[i]
                         for i in best_preds]).reshape((-1, 1))
    best_preds = best_preds.reshape((-1, 1))
    print('preds shape:', best_preds.shape, labels.shape)

    timestamp=np.arange(int(start)+args.window,int(end),args.step).reshape((-1,1))
    #print(timestamp.shape)
    df = pd.DataFrame(np.hstack([timestamp,best_preds, labels]), columns=[
                      'timestamp','output', 'annotation'], index=None)
    df.to_csv(os.path.join(args.direc, 'output_stress.csv'), index=None)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--direc", help="directory of single session", type=str, default='1602171700_A019C2')
    parser.add_argument(
        "--window", help="input signal length in seconds", type=int, default=60)
    parser.add_argument(
        "--start", help="session start time (can either be datetime or unix timestamp in UTC)", nargs='+', default=[2020, 10, 8, 10, 43, 30, 300, 's'])
    parser.add_argument(
        "--end", help="session end time (can either be datetime or unix timestamp in UTC)", nargs='+', default=[2020, 10, 8, 10, 55, 30, 300, 's'])
    parser.add_argument(
        "--step", help="step length relative to window length", type=int, default=1)
    args = parser.parse_args()
    argv = args.__dict__
    main(args)