text:
"""
Test model.py module.
"""
import numpy as np
import pandas as pd
import pytest

from src import model

@pytest.fixture
def dummy_df():
    """Example data to test modeling utility functions"""
    dummy_df = pd.DataFrame(data={"col1": list(range(100)), "target": list(range(100))})
    return dummy_df

def test_parse_ratio():
    """Parses ratio as expected."""
    ratio = "6:2:2"
    sizes = model._parse_ratio(ratio)
    assert sizes == [0.6, 0.2, 0.2]

def test_parse_ratio_missing_component():
    """Ratio only provides two components."""
    ratio = "6:4"
    sizes = model._parse_ratio(ratio)
    assert sizes == [0.6, 0., 0.4]

def test_parse_ratio_invalid_ratio():
    """Ratio uses incorrect separator."""
    ratio = "6;2;2"
    with pytest.raises(ValueError):
        model._parse_ratio(ratio)

def test_split_predictors_response(dummy_df):
    """Separate predictor variables from response."""
    nrows, ncols = dummy_df.shape
    features, target = model.split_predictors_response(dummy_df, target_col="target")

    assert features.shape == (nrows, ncols - 1)
    assert target.shape == (nrows, )

def test_split_predictors_response_invalid_target_col(dummy_df):
    """Specified target column does not exist in DataFrame."""
    with pytest.raises(KeyError):
        model.split_predictors_response(dummy_df, "class_dne")

def test_split_predictors_response_only_one_column():
    """Splitting a 1-column DataFrame yields a 0-col DF and Series."""
    df = pd.DataFrame(data={"col1": list(range(100))})
    features, target = model.split_predictors_response(df, target_col="col1")

    pd.testing.assert_frame_equal(features, pd.DataFrame(index=list(range(100))))
    pd.testing.assert_series_equal(target, pd.Series(data=list(range(100)), name="col1"))

def test_split_train_val_test(dummy_df):
    """Splits into train/val/test with dimensions as expected."""
    target = dummy_df["target"]
    features = dummy_df.drop("target", axis=1)
    splits = model.split_train_val_test(features, target, "6:2:2")
    assert len(splits) == 6  # Features and response for each of train/val/test

    X_train, X_val, X_test, y_train, y_val, y_test = splits

    assert X_train.shape == (60, 1)
    assert X_val.shape == (20, 1)
    assert X_test.shape == (20, 1)
    assert y_train.shape == (60, )
    assert y_val.shape == (20, )
    assert y_test.shape == (20, )

def test_split_train_val_test_no_validation(dummy_df):
    """Splits into train/test (no val) with dimensions as expected."""
    target = dummy_df["target"]
    features = dummy_df.drop("target", axis=1)

    # Ratio specifies all three numbers
    X_train, X_test, y_train, y_test = model.split_train_val_test(features, target, "3:0:1")

    assert X_train.shape == (75, 1)
    assert X_test.shape == (25, 1)
    assert y_train.shape == (75,)
    assert y_test.shape == (25,)

    # Ratio provides only two numbers
    X_train, X_test, y_train, y_test = model.split_train_val_test(features, target, "3:1")

    assert X_train.shape == (75, 1)
    assert X_test.shape == (25, 1)
    assert y_train.shape == (75, )
    assert y_test.shape == (25, )

def test_split_train_val_test_bad_ratio(dummy_df):
    """Ratio results in an empty train, validation, or test set."""
    target = dummy_df["target"]
    features = dummy_df.drop("target", axis=1)

    with pytest.raises(ValueError):
        model.split_train_val_test(features, target, "10:0:0")

    with pytest.raises(ValueError):
        model.split_train_val_test(features, target, "0:10:0")

    with pytest.raises(ValueError):
        model.split_train_val_test(features, target, "0:0:10")

def test_parse_dict_to_dataframe():
    """Convert a dictionary to `pandas.DataFrame` format."""
    colnames = ["col1", "col2", "col3"]
    sample_data = [1.5, 3.0, 4.5]

    data_dict = dict(zip(colnames, sample_data))
    actual_df = model.parse_dict_to_dataframe(data_dict)
    expected_df = pd.DataFrame(data=[sample_data], columns=colnames)

    pd.testing.assert_frame_equal(actual_df, expected_df)

def test_parse_dict_to_dataframe_empty_dict():
    """An empty dict yields an empty DataFrame."""
    data_dict = dict()
    actual_df = model.parse_dict_to_dataframe(data_dict)

    # An empty `pandas.DataFrame` still has one index entry
    expected_df = pd.DataFrame(index=[0])

    pd.testing.assert_frame_equal(actual_df, expected_df)

def test_validate_dataframe():
    """Create and reorder columns for model pipeline."""
    expected_columns = model.PREDICTION_COLUMNS

    sample_data = list(range(len(expected_columns)))
    shuffled_columns = np.random.shuffle(expected_columns)
    sample_df = pd.DataFrame(sample_data, columns=shuffled_columns)

    actual = model.validate_dataframe(sample_df)

    assert len(actual.columns) == len(expected_columns)
    assert sorted(actual.columns) == sorted(expected_columns)

def test_validate_dataframe_empty_input():
    """An empty input DataFrame yields an empty output DataFrame."""
    actual = model.validate_dataframe(pd.DataFrame())

    expected_columns = model.PREDICTION_COLUMNS
    # `NaN` values make the dtype `float64`, not `object`, even though it's empty
    expected = pd.DataFrame(columns=expected_columns, dtype=np.float64)

    pd.testing.assert_frame_equal(actual, expected)


text:
from datetime import datetime
from numpy import (
    array,
    unravel_index,
    datetime64,
    asarray,
    atleast_1d,
    nanmax,
    nanmin,
    nan,
    isfinite,
)
from scipy.interpolate import interp1d
from argparse import ArgumentParser

#
from pymap3d.haversine import anglesep
from pymap3d import aer2ecef, ecef2aer

EPOCH = datetime(1970, 1, 1, 0, 0, 0)

def projectisrhist(isrlla, beamazel, optlla, optazel, heightkm):
    """
    intended to project ISR beam at a single height into optical data.

    output:
    az,el,slantrange in degrees,meters
    """
    isrlla = asarray(isrlla)
    optlla = asarray(optlla)
    assert isrlla.size == optlla.size == 3
    x, y, z = aer2ecef(beamazel[0], beamazel[1], heightkm * 1e3, isrlla[0], isrlla[1], isrlla[2])
    try:
        az, el, srng = ecef2aer(x, y, z, optlla[0], optlla[1], optlla[2])
    except IndexError:
        az, el, srng = ecef2aer(x, y, z, optlla["lat"], optlla["lon"], optlla["alt_km"])

    return {"az": az, "el": el, "srng": srng}

def timesync(tisr, topt, tlim=None):
    """
    TODO: for now, assume optical is always faster
    inputs
    tisr: vector of datetime for ISR
    topt: vector of datetime for camera
    tlim: start,stop UT1 Unix time request

    output
    iisr: indices (integers) of isr to playback at same time as camera
    iopt: indices (integers) of optical to playback at same time as isr
    """
    if isinstance(tisr[0], datetime):
        tisr = array([t.timestamp() for t in tisr])  # must be ndarray
    elif isinstance(tisr[0], datetime64):
        tisr = tisr.astype(float) / 1e9

    assert ((tisr > 1e9) & (tisr < 2e9)).all(), "date sanity check"

    if tlim is not None and isinstance(tlim[0], datetime):
        tlim = array([t.timestamp() for t in tlim])

    assert isinstance(
        tisr[0], float
    ), "datetime64 is not wanted here, lets use ut1_unix float for minimum conversion effort"
    # separate comparison
    if topt is None:
        topt = (nan, nan)
    # %% interpolate isr indices to opt (assume opt is faster, a lot of duplicates iisr)
    if tlim is not None:
        tstart = nanmax([tlim[0], tisr[0], topt[0]])
        tend = nanmin([tlim[1], tisr[-1], topt[-1]])
    else:
        tstart = nanmax([tisr[0], topt[0]])
        tend = nanmin([tisr[-1], topt[-1]])

    if topt is not None and isfinite(topt[0]):
        f = interp1d(tisr, range(tisr.size), "nearest", assume_sorted=True)

        # optical:  typically treq = topt
        ioptreq = ((tstart <= topt) & (topt <= tend)).nonzero()[0]

        toptreq = topt[ioptreq]
        iisrreq = f(toptreq).astype(int)

        # tisrreq = tisr[(tstart<=tisr) & (tisr<=tend)]
    else:
        ioptreq = (None,) * tisr.size
        iisrreq = ((tstart <= tisr) & (tisr <= tend)).nonzero()[0]

    return iisrreq, ioptreq

def findindex2Dsphere(azimg, elimg, az, el):
    """
    finds nearest row,column index of projection on 2-D sphere.
    E.g. an astronomical or auroral camera image

    inputs:
    azimg: azimuth of image pixels (deg)
    elimg: elevation of image pixels (deg)
    az: azimuth of point you seek to find the index for (deg)
    el: elevation of point you seek to find the index for (deg)

    output:
    row, column of 2-D image closest to az,el

    brute force method of finding the row,col with the closest az,el using haversine method
    useful for:
    azimuth, elevation
    right ascension, declination
    latitude, longitude
    """
    az = atleast_1d(az)
    el = atleast_1d(el)
    assert azimg.ndim == 2 and elimg.ndim == 2
    assert isinstance(az[0], float) and isinstance(el[0], float)

    adist = anglesep(azimg, elimg, az, el)
    return unravel_index(adist.argmin(), azimg.shape)

def boilerplateapi(descr="loading, processing, plotting raw ISR data"):
    p = ArgumentParser(description=descr)
    p.add_argument("isrfn", help="HDF5 file (or path) to read")
    p.add_argument(
        "-r", "--rtype", help="0: alt code. 1: plasma line. 3: long pulse", type=int, default=3
    )
    p.add_argument("-c", "--optfn", help="optical data HDF5 to read")  # ,nargs='+',default=('',)
    p.add_argument("-a", "--azelfn", help="plate scale file hdf5")  # ,nargs='+',default=('',)
    p.add_argument("--t0", help="time to extract 1-D vertical plot")
    p.add_argument("--acf", help="show autocorrelation function (ACF)", action="store_true")
    p.add_argument(
        "--beamid", help="beam id 64157 is magnetic zenith beam", type=int, default=64157
    )
    p.add_argument(
        "--vlim", help="min,max for SNR plot [dB]", type=float, nargs=2, default=(None, None)
    )
    p.add_argument(
        "--zlim", help="min,max for altitude [km]", type=float, nargs=2, default=(90.0, None)
    )
    p.add_argument(
        "--tlim", help="min,max time range yyyy-mm-ddTHH:MM:SSz", nargs=2, default=[None, None]
    )
    p.add_argument(
        "--flim", help="frequency limits to plots", type=float, nargs=2, default=(None, None)
    )
    p.add_argument("-o", "--odir", help="directory to write files to", default=".")
    p = p.parse_args()

    return p

ext: py

max_stars_repo_name: scivision/isrutils

max_issues_repo_name: scivision/isrutils

max_forks_repo_name: scivision/isrutils

reason: from numpy,from scipy


text:
import codecs
import json
import os
import pickle

import imgaug.augmenters as iaa
from imblearn.over_sampling import SMOTE  # NOTE must be install from fork https://github.com/Devin-Taylor/imbalanced-learn
import multiaug.augmenters as aug
import numpy as np
import pandas as pd
from scipy.io import loadmat
import torch
from torch.utils.data import dataset

from utils.ppmi_data import load_ppmi

ROOT = "./"
DATA = os.path.join(ROOT, "data/")

np.random.seed(1477)

class PPMIDataset(dataset.Dataset):
    def __init__(self, train, classification=False, balance=False, augmentation=False, fraction=0.2, max_angle=5, fold=None, folds=5):
        self.seed = 1447
        self.classification = classification

        if fold is None: # don't use kfolds
            with open(os.path.join(DATA, "ppmi_all_train_test_indices_unbalanced.json")) as fd:
                data = json.load(fd)
        else:
            with open(os.path.join(DATA, "ppmi_all_train_test_indices_folds.json")) as fd:
                data = json.load(fd)

        cont_meth, cont_spect, _, cont_meta = load_ppmi(dataset="CONTROL", normalise=True)
        pd_meth, pd_spect, _, pd_meta = load_ppmi(dataset="PD", normalise=True)

        if train:
            if fold is None:
                cont_train_ids = data['control']['train_unbalanced']
                pd_train_ids = data['pd']['train_unbalanced']
            else:
                train_folds = [x for x in range(folds) if x != fold]
                cont_train_ids = []
                pd_train_ids = []
                for f in train_folds:
                    cont_train_ids += data['control']['folds'][str(f)]
                    pd_train_ids += data['pd']['folds'][str(f)]

            cont_meth = cont_meth.iloc[cont_train_ids, :]
            pd_meth = pd_meth.iloc[pd_train_ids, :]
            cont_meta = cont_meta.iloc[cont_train_ids, :]
            pd_meta = pd_meta.iloc[pd_train_ids, :]
            cont_spect = cont_spect[cont_train_ids, :, :, :]
            pd_spect = pd_spect[pd_train_ids, :, :, :]
        else:
            if fold is None:
                cont_test_ids = data['control']['test_unbalanced']
                pd_test_ids = data['pd']['test_unbalanced']
            else:
                cont_test_ids = data['control']['folds'][str(fold)]
                pd_test_ids = data['pd']['folds'][str(fold)]

            cont_meth = cont_meth.iloc[cont_test_ids, :]
            pd_meth = pd_meth.iloc[pd_test_ids, :]
            cont_meta = cont_meta.iloc[cont_test_ids, :]
            pd_meta = pd_meta.iloc[pd_test_ids, :]
            cont_spect = cont_spect[cont_test_ids, :]
            pd_spect = pd_spect[pd_test_ids, :]

        meth = pd.concat((cont_meth, pd_meth), axis=0).reset_index(drop=True).values
        metadata = pd.concat((cont_meta, pd_meta), axis=0).reset_index(drop=True)
        spect = np.concatenate((cont_spect, pd_spect), axis=0)
        age = metadata.Age # TODO need to account for this when augmenting

        metadata['diag_label'] = np.where(metadata['DIAGNOSIS'] == 'CONTROL', 0, 1)

        if balance and train:
            smt = SMOTE(k_neighbors=5)
            meth, class_ind = smt.fit_sample(meth, metadata.diag_label)
            augmenter = aug.OneOf(smt._row_ids.tolist(), image3d_transforms=[aug.image3d_augmenters.Rotate3d(angle=max_angle)])
            spect, _ = augmenter.apply_image3d(spect, metadata.diag_label)
        else:
            class_ind = metadata['diag_label']

        if augmentation and train and fraction > 0:
            augmenter = aug.OneOf(augment=fraction, image3d_transforms=[aug.image3d_augmenters.Rotate3d(angle=max_angle)],
                                  tabular_transforms=[aug.tabular_augmenters.GaussianPerturbation(fraction=0.05)])
            spect, class_ind = augmenter.apply_image3d(spect, class_ind)
            meth, _ = augmenter.apply_tabular(meth, class_ind)

        self.meth = torch.from_numpy(meth)
        self.spect = torch.from_numpy(spect)
        self.age = torch.from_numpy(np.array(age))
        self.class_ind = torch.from_numpy(np.array(class_ind))

    def __getitem__(self, idx):
        if self.classification:
            return self.meth[idx, :], self.spect[idx, :], self.class_ind[idx]
        else:
            return self.meth[idx, :], self.spect[idx, :], self.age[idx]

    def __len__(self):
        return len(self.meth)

class NoisyMNISTDataset(dataset.Dataset):
    def __init__(self, set="train"):
        self.set = set
        awgn = loadmat("data/mnist/mnist-with-awgn.mat")
        blur = loadmat("data/mnist/mnist-with-motion-blur.mat")
        contrast = loadmat("data/mnist/mnist-with-reduced-contrast-and-awgn.mat")

        self.awgn_x = awgn['{}_x'.format(set)]
        self.awgn_x = torch.from_numpy(np.array(np.expand_dims(self.awgn_x.reshape((self.awgn_x.shape[0], 28, 28)), axis=1)/255, dtype=np.float32))
        self.awgn_y = torch.from_numpy(awgn['{}_y'.format(set)].argmax(axis=1))
        self.blur_x = blur['{}_x'.format(set)]
        self.blur_x = torch.from_numpy(np.array(np.expand_dims(self.blur_x.reshape((self.blur_x.shape[0], 28, 28)), axis=1)/255, dtype=np.float32))
        self.blur_y = torch.from_numpy(blur['{}_y'.format(set)].argmax(axis=1))
        self.contrast_x = contrast['{}_x'.format(set)]
        self.contrast_x = torch.from_numpy(np.array(np.expand_dims(self.contrast_x.reshape((self.contrast_x.shape[0], 28, 28)), axis=1)/255, dtype=np.float32))
        self.contrast_y = torch.from_numpy(contrast['{}_y'.format(set)].argmax(axis=1))

    def __getitem__(self, idx):
        return self.awgn_x[idx, :], self.blur_x[idx, :], self.contrast_x[idx, :], self.awgn_y[idx]

    def __len__(self):
        return len(self.awgn_y)

class CIFAR10Dataset(dataset.Dataset):
    def __init__(self, train: bool, fmt: str = 'standard'):

        if train:
            setname = 'train'
        else:
            setname = 'test'
        with open("data/cifar10/{}_{}.pkl".format(setname, fmt), 'rb') as fd:
            data = pickle.load(fd)

        self.images = torch.from_numpy(np.array(data['X'].reshape((-1, 3, 32, 32))/255, dtype=np.float32))
        self.labels = torch.from_numpy(np.array(data['y']))

    def __getitem__(self, idx):
        return self.images[idx, :], self.labels[idx]

    def __len__(self):
        return len(self.images)

class MyMNISTDataset(dataset.Dataset):
    def __init__(self, train=True, awgn_frac=0.):
        self.set = set

        assert awgn_frac <= 1., "Noise cannot exceed 100%"

        if train:
            image_file = "data/mnist/train-images.idx3-ubyte"
            label_file = "data/mnist/train-labels.idx1-ubyte"
        else:
            image_file = "data/mnist/t10k-images.idx3-ubyte"
            label_file = "data/mnist/t10k-labels.idx1-ubyte"

        images = read_image_file(image_file)
        labels = read_label_file(label_file)

        seq = iaa.Sequential([iaa.AdditiveGaussianNoise(scale=awgn_frac*255)])
        aug_images = seq.augment_images(images.numpy())

        self.mnist = torch.from_numpy(np.array(np.expand_dims(images, axis=1)/255, dtype=np.float32))
        self.aug_mnist = torch.from_numpy(np.array(np.expand_dims(aug_images, axis=1)/255, dtype=np.float32))
        self.labels = labels

    def __getitem__(self, idx):
        return self.mnist[idx, :], self.aug_mnist[idx, :], torch.zeros((2, 2)), self.labels[idx]

    def __len__(self):
        return len(self.labels)

def get_int(b):
    return int(codecs.encode(b, 'hex'), 16)

def read_label_file(path):
    with open(path, 'rb') as f:
        data = f.read()
        assert get_int(data[:4]) == 2049
        length = get_int(data[4:8])
        parsed = np.frombuffer(data, dtype=np.uint8, offset=8)
        return torch.from_numpy(parsed).view(length).long()

def read_image_file(path):
    with open(path, 'rb') as f:
        data = f.read()
        assert get_int(data[:4]) == 2051
        length = get_int(data[4:8])
        num_rows = get_int(data[8:12])
        num_cols = get_int(data[12:16])
        parsed = np.frombuffer(data, dtype=np.uint8, offset=16)
        return torch.from_numpy(parsed).view(length, num_rows, num_cols)

max_stars_repo_name: Devin-Taylor/multi-head-co-attention

max_issues_repo_name: Devin-Taylor/multi-head-co-attention

max_forks_repo_name: Devin-Taylor/multi-head-co-attention

reason: import numpy,from scipy


text:
#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt
plt.ion()
from figurefirst import FigureLayout

layout = FigureLayout('example_hello_world_layout.svg')
layout.make_mplfigures()

d = np.array([[144, 57], [138, 57], [138, 59], [141, 61], [141, 82], [138, 84], [138, 85], [142, 85], [147, 85], [147, 84], [144, 82], [144, 57], [144, 57], [155, 57], [149, 57], [149, 59], [152, 61], [152, 82], [149, 84], [149, 85], [153, 85], [158, 85], [158, 84], [155, 82], [155, 57], [155, 57], [273, 57], [267, 57], [267, 59], [270, 61], [270, 82], [267, 84], [267, 85], [271, 85], [276, 85], [276, 84], [273, 82], [273, 57], [273, 57], [295, 57], [289, 57], [289, 59], [292, 61], [292, 70], [287, 67], [278, 76], [287, 85], [292, 83], [292, 85], [298, 85], [298, 84], [295, 81], [295, 57], [295, 57], [90, 57], [90, 59], [91, 59], [94, 61], [94, 82], [91, 84], [90, 84], [90, 85], [96, 85], [102, 85], [102, 84], [101, 84], [98, 82], [98, 71], [110, 71], [110, 82], [107, 84], [106, 84], [106, 85], [112, 85], [118, 85], [118, 84], [117, 84], [113, 82], [113, 61], [117, 59], [118, 59], [118, 57], [112, 58], [106, 57], [106, 59], [107, 59], [110, 61], [110, 70], [98, 70], [98, 61], [101, 59], [102, 59], [102, 57], [96, 58], [90, 57], [90, 57], [193, 57], [193, 59], [197, 60], [205, 85], [205, 86], [206, 85], [213, 65], [219, 85], [220, 86], [221, 85], [229, 61], [233, 59], [233, 57], [229, 58], [224, 57], [224, 59], [228, 61], [227, 62], [221, 80], [215, 60], [215, 60], [218, 59], [218, 57], [213, 58], [208, 57], [208, 59], [211, 60], [212, 63], [207, 80], [200, 60], [200, 60], [203, 59], [203, 57], [198, 58], [193, 57], [193, 57], [128, 67], [120, 76], [129, 85], [135, 80], [135, 80], [134, 80], [129, 84], [125, 82], [123, 76], [134, 76], [135, 75], [128, 67], [128, 67], [169, 67], [160, 76], [169, 85], [178, 76], [169, 67], [169, 67], [240, 67], [231, 76], [240, 85], [249, 76], [240, 67], [240, 67], [257, 67], [251, 68], [251, 69], [254, 71], [254, 82], [251, 84], [251, 85], [256, 85], [261, 85], [261, 84], [260, 84], [257, 82], [257, 75], [262, 68], [262, 68], [261, 70], [263, 71], [265, 70], [262, 67], [257, 71], [257, 67], [257, 67], [128, 68], [133, 75], [123, 75], [128, 68], [128, 68], [169, 68], [173, 70], [174, 76], [173, 81], [169, 84], [164, 82], [163, 76], [164, 70], [169, 68], [169, 68], [240, 68], [244, 70], [246, 76], [245, 81], [240, 84], [235, 82], [234, 76], [235, 70], [240, 68], [240, 68], [287, 68], [292, 70], [292, 72], [292, 80], [292, 82], [287, 84], [283, 82], [281, 76], [283, 71], [287, 68], [287, 68]])

ax = layout.axes['ax_name']['axis']

ax.plot(d[:,0], -d[:,1], lw=4)

layout.insert_figures('target_layer_name')

layout.write_svg('example_hello_world_output.svg')

max_stars_repo_name: clbarnes/figurefirst

max_issues_repo_name: clbarnes/figurefirst

max_forks_repo_name: clbarnes/figurefirst

reason: import numpy


text:
import argparse
import logging
import os
import random
import shutil
import sys
import time

import numpy as np
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from tensorboardX import SummaryWriter
from torch.nn import BCEWithLogitsLoss
from torch.nn.modules.loss import CrossEntropyLoss
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.utils import make_grid
from tqdm import tqdm

from dataloaders import utils
from dataloaders.dataset import BaseDataSets, RandomGenerator
from networks.net_factory import net_factory
from utils import losses, metrics, ramps
from val_2D import test_single_volume, test_single_volume_ds

parser = argparse.ArgumentParser()
parser.add_argument('--root_path', type=str,
                    default='../data/ACDC', help='Name of Experiment')
parser.add_argument('--exp', type=str,
                    default='ACDC_pCE', help='experiment_name')
parser.add_argument('--fold', type=str,
                    default='fold1', help='cross validation')
parser.add_argument('--sup_type', type=str,
                    default='scribble', help='supervision type')
parser.add_argument('--model', type=str,
                    default='unet', help='model_name')
parser.add_argument('--num_classes', type=int,  default=4,
                    help='output channel of network')
parser.add_argument('--max_iterations', type=int,
                    default=30000, help='maximum epoch number to train')
parser.add_argument('--batch_size', type=int, default=24,
                    help='batch_size per gpu')
parser.add_argument('--deterministic', type=int,  default=1,
                    help='whether use deterministic training')
parser.add_argument('--base_lr', type=float,  default=0.01,
                    help='segmentation network learning rate')
parser.add_argument('--patch_size', type=list,  default=[256, 256],
                    help='patch size of network input')
parser.add_argument('--seed', type=int,  default=1337, help='random seed')
parser.add_argument('--labeled_num', type=int, default=50,
                    help='labeled data')
args = parser.parse_args()

def train(args, snapshot_path):
    base_lr = args.base_lr
    num_classes = args.num_classes
    batch_size = args.batch_size
    max_iterations = args.max_iterations

    model = net_factory(net_type=args.model, in_chns=1, class_num=num_classes)
    db_train = BaseDataSets(base_dir=args.root_path, split="train", transform=transforms.Compose([
        RandomGenerator(args.patch_size)
    ]), fold=args.fold, sup_type=args.sup_type)
    db_val = BaseDataSets(base_dir=args.root_path, split="val")

    def worker_init_fn(worker_id):
        random.seed(args.seed + worker_id)

    trainloader = DataLoader(db_train, batch_size=batch_size, shuffle=True,
                             num_workers=8, pin_memory=True, worker_init_fn=worker_init_fn)
    valloader = DataLoader(db_val, batch_size=1, shuffle=False,
                           num_workers=1)

    model.train()

    optimizer = optim.SGD(model.parameters(), lr=base_lr,
                          momentum=0.9, weight_decay=0.0001)
    ce_loss = CrossEntropyLoss(ignore_index=4)
    dice_loss = losses.DiceLoss(num_classes)

    writer = SummaryWriter(snapshot_path + '/log')
    logging.info("{} iterations per epoch".format(len(trainloader)))

    iter_num = 0
    max_epoch = max_iterations // len(trainloader) + 1
    best_performance = 0.0
    iterator = tqdm(range(max_epoch), ncols=70)
    for epoch_num in iterator:
        for i_batch, sampled_batch in enumerate(trainloader):

            volume_batch, label_batch = sampled_batch['image'], sampled_batch['label']
            volume_batch, label_batch = volume_batch.cuda(), label_batch.cuda()

            outputs = model(volume_batch)
            outputs_soft = torch.softmax(outputs, dim=1)

            loss_ce = ce_loss(outputs, label_batch[:].long())
            loss = loss_ce
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            lr_ = base_lr * (1.0 - iter_num / max_iterations) ** 0.9
            for param_group in optimizer.param_groups:
                param_group['lr'] = lr_

            iter_num = iter_num + 1
            writer.add_scalar('info/lr', lr_, iter_num)
            writer.add_scalar('info/total_loss', loss, iter_num)
            writer.add_scalar('info/loss_ce', loss_ce, iter_num)

            logging.info(
                'iteration %d : loss : %f, loss_ce: %f' %
                (iter_num, loss.item(), loss_ce.item()))

            if iter_num % 20 == 0:
                image = volume_batch[1, 0:1, :, :]
                image = (image - image.min()) / (image.max() - image.min())
                writer.add_image('train/Image', image, iter_num)
                outputs = torch.argmax(torch.softmax(
                    outputs, dim=1), dim=1, keepdim=True)
                writer.add_image('train/Prediction',
                                 outputs[1, ...] * 50, iter_num)
                labs = label_batch[1, ...].unsqueeze(0) * 50
                writer.add_image('train/GroundTruth', labs, iter_num)

            if iter_num > 0 and iter_num % 200 == 0:
                model.eval()
                metric_list = 0.0
                for i_batch, sampled_batch in enumerate(valloader):
                    metric_i = test_single_volume(
                        sampled_batch["image"], sampled_batch["label"], model, classes=num_classes)
                    metric_list += np.array(metric_i)
                metric_list = metric_list / len(db_val)
                for class_i in range(num_classes-1):
                    writer.add_scalar('info/val_{}_dice'.format(class_i+1),
                                      metric_list[class_i, 0], iter_num)
                    writer.add_scalar('info/val_{}_hd95'.format(class_i+1),
                                      metric_list[class_i, 1], iter_num)

                performance = np.mean(metric_list, axis=0)[0]

                mean_hd95 = np.mean(metric_list, axis=0)[1]
                writer.add_scalar('info/val_mean_dice', performance, iter_num)
                writer.add_scalar('info/val_mean_hd95', mean_hd95, iter_num)

                if performance > best_performance:
                    best_performance = performance
                    save_mode_path = os.path.join(snapshot_path,
                                                  'iter_{}_dice_{}.pth'.format(
                                                      iter_num, round(best_performance, 4)))
                    save_best = os.path.join(snapshot_path,
                                             '{}_best_model.pth'.format(args.model))
                    torch.save(model.state_dict(), save_mode_path)
                    torch.save(model.state_dict(), save_best)

                logging.info(
                    'iteration %d : mean_dice : %f mean_hd95 : %f' % (iter_num, performance, mean_hd95))
                model.train()

            if iter_num % 3000 == 0:
                save_mode_path = os.path.join(
                    snapshot_path, 'iter_' + str(iter_num) + '.pth')
                torch.save(model.state_dict(), save_mode_path)
                logging.info("save model to {}".format(save_mode_path))

            if iter_num >= max_iterations:
                break
        if iter_num >= max_iterations:
            iterator.close()
            break
    writer.close()
    return "Training Finished!"

if __name__ == "__main__":
    if not args.deterministic:
        cudnn.benchmark = True
        cudnn.deterministic = False
    else:
        cudnn.benchmark = False
        cudnn.deterministic = True

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    snapshot_path = "../model/{}_{}/{}".format(
        args.exp, args.fold, args.sup_type)
    if not os.path.exists(snapshot_path):
        os.makedirs(snapshot_path)
    if os.path.exists(snapshot_path + '/code'):
        shutil.rmtree(snapshot_path + '/code')
    shutil.copytree('.', snapshot_path + '/code',
                    shutil.ignore_patterns(['.git', '__pycache__']))

    logging.basicConfig(filename=snapshot_path+"/log.txt", level=logging.INFO,
                        format='[%(asctime)s.%(msecs)03d] %(message)s', datefmt='%H:%M:%S')
    logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
    logging.info(str(args))
    train(args, snapshot_path)


text:
from typing import Dict, Union, List
import logging

from overrides import overrides
import numpy

from allennlp.data.fields.field import DataArray, Field
from allennlp.data.fields.label_field import LabelField
from allennlp.data.fields.list_field import ListField
from allennlp.data.vocabulary import Vocabulary
from allennlp.common.checks import ConfigurationError
from allennlp.data.fields.sequence_field import SequenceField
from allennlp.common.util import pad_sequence_to_length

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

import IPython as ipy

class MapLabelField(ListField):
    """
    A ``MapLabelField`` extends the ``LabelField`` class (for categorical labels which are either
    strings of text or 0-indexed integers) by allowing a single field to be indexed in multiple
    namespaces. This allows the representation of a mapping between different sets of labels (for
    example, to identify a set of possible options for a prediction to choose among).

    Like a ListField, this field will get converted into a tensor that has one more mode than the
    items in the list (from target_namespace).
    If this is a list of ``TextFields`` that have shape (num_words, num_characters), this
    ``ListField`` will output a tensor of shape (num_sentences, num_words, num_characters).

    Parameters
    ----------
    index_label : ``Union[str, int]``
    target_label : ``Union[str, int]``
    index_label_namespace : ``str``, optional (default="labels")
        The namespace to use for converting label strings into integers.  We map label strings to
        integers for you (e.g., "entailment" and "contradiction" get converted to 0, 1, ...),
        and this namespace tells the ``Vocabulary`` object which mapping from strings to integers
        to use (so "entailment" as a label doesn't get the same integer id as "entailment" as a
        word).  If you have multiple different label fields in your data, you should make sure you
        use different namespaces for each one, always using the suffix "labels" (e.g.,
        "passage_labels" and "question_labels").
    target_label_namespace : ``str``, optional (default="labels")
        The namespace to use for converting label strings into integers.  We map label strings to
        integers for you (e.g., "entailment" and "contradiction" get converted to 0, 1, ...),
        and this namespace tells the ``Vocabulary`` object which mapping from strings to integers
        to use (so "entailment" as a label doesn't get the same integer id as "entailment" as a
        word).  If you have multiple different label fields in your data, you should make sure you
        use different namespaces for each one, always using the suffix "labels" (e.g.,
        "passage_labels" and "question_labels").
    skip_indexing : ``bool``, optional (default=False)
        If your labels are 0-indexed integers, you can pass in this flag, and we'll skip the indexing
        step.  If this is ``False`` and your labels are not strings, this throws a ``ConfigurationError``.
        For a MapLabelField this option is not recommended.
    """
    def __init__(self,
                 index_label: Union[str, int],
                 set_label: Union[str, int],
                 target_label: Union[str, int],
                 index_label_namespace: str = 'primary_labels',
                 set_label_namespace: str = 'secondary_labels',
                 target_label_namespace: str = 'tertiary_labels',
                 set_index_map: str = None,
                 set_target_map: str = None,
                 skip_indexing: bool = False) -> None:
        self.index_label = index_label # predicate token lemma
        self.set_label = set_label # predicate ID
        self.target_label = target_label # predicate sense

        self.target_label_set = set()

        self._index_label_namespace = index_label_namespace
        self._set_label_namespace = set_label_namespace
        self._target_label_namespace = target_label_namespace

        if set_index_map is None:
            self._set_index_map = index_label_namespace + '_map'
        if set_target_map is None:
            self._set_target_map = set_label_namespace + '_map'

        self._index_label_id = None
        self._set_label_id = None
        self._target_label_id = None

        namespace_err = None
        if not (self._set_label_namespace.endswith("labels") or self._set_label_namespace.endswith("tags")):
            namespace_err = 'primary'
            namespace = self._set_label_namespace
        if not (self._target_label_namespace.endswith("labels") or self._target_label_namespace.endswith("tags")):
            namespace_err = 'secondary'
            namespace = self._target_label_namespace

        """
        if namespace_err is not None:
            logger.warning("Your {} label namespace was '{}'. ".format(namespace_err, namespace) +
                           "We recommend you use a namespace ending with 'labels' or 'tags', " +
                           "so we don't add UNK and PAD tokens by default to your vocabulary.  " +
                           "See documentation for `non_padded_namespaces` parameter in Vocabulary.",
                           self._set_label_namespace)
        """

        if skip_indexing:
            if not isinstance(index_label, int) and isinstance(target_label, int):
                raise ConfigurationError("In order to skip indexing, your labels must be integers. "
                                         "Found index_label = {}, target_label = {}".format(index_label, target_label))
            else:
                self._index_label_id = index_label
                self.target_label_set = set([target_label])
        else:
            if not isinstance(index_label, str):
                raise ConfigurationError("LabelFields must be passed a string label if skip_indexing=False. "
                                         "Found label: {} with type: {}.".format(index_label, type(index_label)))

    @overrides
    def count_vocab_items(self, indexer: Dict[str, Dict[str, Union[int, List[int]]]]):
        # target_label is counted elsewhere
        if self._index_label_id is None:
            # count set_label normally here
            indexer[self._index_label_namespace][self.index_label] += 1  # type: ignore
        if self._set_label_id is None:
            # count set_label normally here
            indexer[self._set_label_namespace][self.set_label] += 1  # type: ignore
        if len(self.target_label_set) == 0:
            # ''count'' target and index labels here by adding set_label to the maps
            try:
                if self.set_label not in indexer[self._set_index_map]:
                    #override the 'int' default value of the defaultdict
                    indexer[self._set_index_map][self.set_label] = []
                if self.index_label not in indexer[self._set_index_map][self.set_label]:
                    indexer[self._set_index_map][self.set_label].append(self.index_label) # type: ignore
                if self.set_label not in indexer[self._set_target_map]:
                    #override the 'int' default value of the defaultdict
                    indexer[self._set_target_map][self.set_label] = []
                if self.target_label not in indexer[self._set_target_map][self.set_label]:
                    indexer[self._set_target_map][self.set_label].append(self.target_label) # type: ignore
            except:
                print("Problem with constructing maps in MapLabelField")
                ipy.embed()

    @overrides
    def index(self, vocab: Vocabulary):
        if self._index_label_id is None:
            self._index_label_id = vocab.get_token_index(self.index_label, self._index_label_namespace)  # type: ignore
        if self._set_label_id is None:
            self._set_label_id = vocab.get_token_index(self.set_label, self._set_label_namespace)  # type: ignore
        if len(self.target_label_set) == 0:
            set_set = vocab.get_token_from_index(self.index_label, self._set_index_map)
            flat_set = []
            for tgt_set in set_set:
                flat_set += vocab.get_token_index(tgt_set, self._set_target_map)  # type: ignore
            self.target_label_set = set(flat_set)
            target_ind_set = [vocab.get_token_index(lb, self._target_label_namespace) for lb in self.target_label_set]
            self.target_ind_set = sorted(list(target_ind_set))

    @overrides
    def get_padding_lengths(self) -> Dict[str, int]:
        # sequences are currently not supported; see ListField for a template to
        # implement sequence padding
        padding_lengths = {'num_fields': len(self.target_label_set)}
        return padding_lengths

    @overrides
    def sequence_length(self) -> int:
        return len(self.target_label_set)

    @overrides
    def as_array(self, padding_lengths: Dict[str, int]) -> DataArray:
        padded_label_list = pad_sequence_to_length(self.target_ind_set,
                                                   padding_lengths['num_fields'],
                                                   lambda: -1)
        return numpy.expand_dims(numpy.asarray(padded_label_list),1)

    @overrides
    def empty_field(self):  # pylint: disable=no-self-use
        raise RuntimeError("Nested MapLabelFields are not implemented, and if you want this "
                           "you should /definitely/ try to simplify your data type, anyway")

max_stars_repo_name: pmulcaire/allennlp

max_issues_repo_name: pmulcaire/allennlp

max_forks_repo_name: pmulcaire/allennlp


text:
import numpy as np

from element_search import find_element

from particle_scripts import compute_Uf
#==============================================================================
def vorticity(flow, n, theta_plus, dr, id_geometry=0):
    elements = flow.elements
    geometry_nodes = flow.geometries[id_geometry]

    allNodes = np.array(list(zip(flow.nodes_X[n], flow.nodes_Y[n])))
    allUf = np.array(list(zip(flow.Us[n], flow.Vs[n])))

    nodes_coord = allNodes[geometry_nodes - 1]
    barycenter = np.mean(nodes_coord, axis=0)

    n_nodes = len(geometry_nodes)

    # Looking for the arc which crosses (O, theta_plus)
    for i in range(n_nodes - 1):
        p2, p3 = nodes_coord[[i, i+1]]

        Op2 = p2 - barycenter
        Op3 = p3 - barycenter

        theta_plus2 = np.pi - np.arctan2(Op2[1], Op2[0])
        theta_plus3 = np.pi - np.arctan2(Op3[1], Op3[0])

        if (theta_plus2 - theta_plus) * (theta_plus3 - theta_plus) <= 0:
            break

    # Intersection point M
    p2p3 = p3 - p2
    d23 = np.linalg.norm(p2p3)

    t_p2p3= p2p3/d23
    n_p2p3 = np.array([t_p2p3[1], -t_p2p3[0]])

    theta_plus2 = np.pi - np.arctan2(Op2[1], Op2[0])
    theta_plus3 = np.pi - np.arctan2(Op3[1], Op3[0])

    norm_Op2 = np.linalg.norm(Op2)
    norm_OM = norm_Op2*np.cos(theta_plus2 - theta_plus)

    # Quick verification
    norm_Op3 = np.linalg.norm(Op3)
    norm_OM_via_p3 = norm_Op3*np.cos(theta_plus - theta_plus3)
    if abs(norm_OM_via_p3 - norm_OM) > 1e-3:
        print('norm via p2 = %s' % norm_OM)
        print('norm via p3 = %s' % norm_OM_via_p3)
        raise Exception('The two norms should be equal. Something wrong in the' \
                        + ' algebra.')

    OM = barycenter + norm_OM*np.array([-np.cos(theta_plus), np.sin(theta_plus)])

    nearby_point = OM + dr*n_p2p3
    nearby_element = find_element(nearby_point, elements, allNodes)

    Unearby = compute_Uf(nearby_point, allUf, allNodes, nearby_element)

    norm_Unearby_theta = np.dot(Unearby,
                                np.array([np.sin(theta_plus),
                                          np.cos(theta_plus)]))

    return norm_Unearby_theta/dr


text:
# PROJECT TO SHOW AN IMAGE WITH MULTIPLE COLORS BASED ON SUB-SQUARES
# Santiago Garcia Arango, August 2020

import numpy as np
import cv2 as cv
import os

# This way we can access upper package to save images in output folder
os.sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from SaveImg import save_img

# Get the path for the folder that contains the input and output images
upper_dir = os.path.dirname(__file__)  # Upper dir
img_folder_path = os.path.abspath(os.path.join(upper_dir, "imgs"))

# Get specific path to the images we will use inside the "imgs" folder
path_delfin = os.path.join(img_folder_path, "delfin.jpg")

# Loading our image with its path and parameter to change format
imgColor = cv.imread(path_delfin, 1)

h, w = imgColor.shape[:2]  # Get the shape of the image (height, width, c)

# Let the user insert "h" and "w" percentage where the colors change
h_user = int(input("Insert height percentage to cut image"))
w_user = int(input("Insert width percentage to cut image"))

# Convert the user's input percentages into valid "h" and "w" values
h_user = (h_user * h)/100
w_user = (w_user * w)/100

# Now we play to generate 3 new images with Blue, Green, Red backgrounds
new_img = np.zeros((h, w, 3), np.uint8)

for i in range(0, h):
    for j in range(0, w):
        # Check upper left portion (keep original format)
        if (i < h_user and j < w_user):
            new_img[i, j] = imgColor[i, j]  # Same original pixels

        # Check upper right portion (change to blue pixels only)
        if (i < h_user and j > w_user):
            new_img[i, j] = [imgColor[i, j][0], 0, 0]  # Blue pixels only

        # Check lower left portion (change to green pixels only)
        if (i > h_user and j < w_user):
            new_img[i, j] = [0, imgColor[i, j][1], 0]  # Green pixels only

        # Check lower right portion
        if (i > h_user and j > w_user):
            new_img[i, j] = [0, 0, imgColor[i, j][2]]  # Red pixels only

cv.imshow("Original", imgColor)
cv.imshow("New Image", new_img)

# Save obtained images
try:
    # save_img.SaveResult(new_img, "Multi_colors_65_25.jpg", __file__)
    pass
except:
    pass

cv.waitKey(0)  # Show image until a key is pressed
cv.destroyAllWindows()  # (always at the end)


text:
import unittest

import pandas as pd
import numpy.testing as npt

from .. import warp

class TestSplitByColumn(unittest.TestCase):
    def test_creates_a_new_dataframe_for_each_passed_value(self):
        df = pd.DataFrame.from_dict({'hemi': ['N', 'S', 'S', 'E']}, orient='columns')

        dfs = warp.split_by_column(df, 'hemi', ['N'])
        self.assertEqual(len(dfs), 1)
        self.assertEqual(set(dfs[0].hemi.values), {'N'})

        dfs = warp.split_by_column(df, 'hemi', ['N', 'S'])
        self.assertEqual(len(dfs), 2)
        self.assertEqual(set(dfs[0].hemi.values), {'N'})
        self.assertEqual(set(dfs[1].hemi.values), {'S'})

class TestOrderByRank(unittest.TestCase):
    def test_sorts_by_rank(self):
        df = pd.DataFrame(
            data={
                'extent': [100, 20, 30, 40],
                'year': [2000, 1920, 1930, 1940],
            },
            index=pd.DatetimeIndex(['2001', '2002', '2003', '2004'])
        )

        expected_ranked_extent = pd.Series([20, 30, 40, 100])
        expected_ranked_years = pd.Series([1920, 1930, 1940, 2000])

        actual = warp.order_by_rank(df)

        npt.assert_array_equal(expected_ranked_years, actual['ranked-year'].values)
        npt.assert_array_equal(expected_ranked_extent, actual['ranked-extent'].values)

class Test_restrict_filename(unittest.TestCase):

    def setUp(self):
        filenames1 = ['nt_20150731_f17_v1.1_s.bin',
                      'nt_20150801_f17_v1.1_s.bin',
                      'nt_20150802_f17_v1.1_s.bin']
        filenames7 = ['nt_20150807_f17_v1.1_s.bin']
        filenames8 = []

        df = pd.DataFrame([[pd.Timestamp('2015-08-01'), filenames1],
                           [pd.Timestamp('2015-08-07'), filenames7],
                           [pd.Timestamp('2015-08-08'), filenames8]],
                          columns=['date', 'filename'])
        df = df.set_index('date')

        self.df = df

    def test_picks_filename_matching_the_date(self):
        actual = warp.restrict_filename(self.df)

        actual_filename = actual.loc['2015-08-01', 'filename']

        expected = 'nt_20150801_f17_v1.1_s.bin'

        self.assertEqual(actual_filename, expected)

    def test_uses_the_only_available_filename(self):
        actual = warp.restrict_filename(self.df)

        actual_filename = actual.loc['2015-08-07', 'filename']

        expected = 'nt_20150807_f17_v1.1_s.bin'

        self.assertEqual(actual_filename, expected)

    def test_with_empty_filename_list(self):
        actual = warp.restrict_filename(self.df)

        actual_filename = actual.loc['2015-08-08', 'filename']

        expected = ''

        self.assertEqual(actual_filename, expected)

max_stars_repo_name: andypbarrett/nsidc-seaice

max_issues_repo_name: andypbarrett/nsidc-seaice

max_forks_repo_name: andypbarrett/nsidc-seaice


text:
# -*- coding: utf-8 -*-
"""
Tencent is pleased to support the open source community by making GameAISDK available.

This source code file is licensed under the GNU General Public License Version 3.
For full details, please refer to the file "LICENSE.txt" which is provided as part of this source code package.

Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
"""

import json
import logging
import shutil
import tarfile
import os
from collections import OrderedDict
import numpy as np
import cv2
from qtpy import QtWidgets
from qtpy import QtGui
from qtpy import QtCore

from PyQt5.QtWidgets import QComboBox
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QImage, QPixmap, QIcon
from PyQt5.QtWidgets import QTreeWidgetItem, QTreeWidgetItemIterator, QPushButton

from ..common.define import UI_HIDDEN_KEYS, SCENE_HIDDEN_KEYS, Mode, Number_Float_Key, Number_Int_Key, \
    BOOL_Keys, VALID_SYMBOL, ITEM_TYPE_IMAGE, ITEM_TYPE_IMAGE_FLODER, BOOL_FLAGS, \
    LEARNING_CONFIG_EXCITATIONFUNCTION_Key, LEARNING_CONFIG_TASKID_LIST
from ..ui.dialog.label_dialog import LabelDialog
from ..ui.main_window.tool_window import ui
from ..ui.tree.project_data_manager import g_project_data_mgr, change_image_path
from ..config_manager.task.task_manager import TaskManager
from .dialog.tip_dialog import show_warning_tips
from ..common.define import AI_NAME

logger = logging.getLogger("sdktool")

def create_tree_item(key=None, value=None, node_type=None, edit=True):
    child = QTreeWidgetItem()
    child.setText(0, str(key))
    if value is not None:
        child.setText(1, str(value))

    if node_type is not None:
        child.setText(2, node_type)

    if edit is True:
        child.setFlags(child.flags() | Qt.ItemIsEditable)
    return child

def show_critical_item(item):
    if item is None:
        return

    if item.text(0) in UI_HIDDEN_KEYS:
        if item.childCount() > 0:
            item.setExpanded(False)
        else:
            item.setHidden(True)
        return

    if item.text(0) in SCENE_HIDDEN_KEYS:
        if item.childCount() > 0:
            item.setExpanded(False)
        else:
            item.setHidden(True)
        return

    count = item.childCount()
    if count > 0:
        for index in range(count):
            sub_item = item.child(index)
            show_critical_item(sub_item)
    return

def show_detail_item(item):
    iterator = QTreeWidgetItemIterator(item)
    while iterator.value() is not None:
        iterator.value().setHidden(False)
        iterator.__iadd__(1)

def show_work_tree(tree, mode):
    if tree is None:
        return

    if mode not in [Mode.UI, Mode.SCENE, Mode.AI, Mode.RUN, Mode.UI_AUTO_EXPLORE]:
        return

    project_node = tree.topLevelItem(0)
    nodes = get_sub_nodes(project_node)
    for node in nodes:
        key = node.text(0)
        if (mode == Mode.UI and key == 'UI') or \
                (mode == Mode.SCENE and key == 'Scene') or \
                (mode == Mode.AI and key == 'AI') or \
                (mode == Mode.RUN and key == 'Run') or \
                mode == Mode.UI_AUTO_EXPLORE:
            node.setExpanded(True)
        else:
            node.setExpanded(False)

def get_value(value, key: str, default):
    if not isinstance(value, dict):
        return default
    if key not in value.keys():
        return default
    return value[key]

def is_video(url):
    suffix = url.split('.')[-1]
    return suffix in ['mp4', 'avi']

def is_image(url):
    suffix = url.split('.')[-1]
    return suffix in ['jpg', 'png', 'bmp']

def is_float_number(number: str):
    try:
        float(number)
        return True

    except ValueError:
        return False

def is_number(number: str):
    if is_int_number(number) or is_float_number(number):
        return True
    return False

def is_int_number(number: str):
    try:
        int(number)
        return True

    except ValueError:
        return False

def is_json_file(file_name: str):
    try:
        with open(file_name) as f:
            json.load(f)
        return True
    except IOError as err:
        logger.error("err: %s", str(err))
        return False

def get_sub_nodes(parent):
    nodes = list()
    if parent is None:
        return nodes

    for index in range(parent.childCount()):
        node = parent.child(index)
        nodes.append(node)

    return nodes

def get_ai_node_by_type(left_tree, node_type):
    project_node = left_tree.topLevelItem(0)
    if len(project_node.text(0)) == 0:
        show_warning_tips("please new project first")
        return False

    nodes = get_sub_nodes(project_node)

    ai_node = None
    for node in nodes:
        if node.text(0) == AI_NAME:
            ai_node = node
            break

    if ai_node is None:
        ai_node = create_tree_item(key='AI', edit=False)
        project_node.addChild(ai_node)
        project_node.setExpanded(True)

    sub_nodes = get_sub_nodes(ai_node)

    for node in sub_nodes:
        if node.text(0) == node_type:
            return node, ai_node

    return None, ai_node

def _recursive_get_item_value_by_type(item_type, parent, result):
    if not parent:
        return

    if parent.text(2) == item_type:
        result.append(parent.text(1))
        return

    sub_nodes = get_sub_nodes(parent)
    for sub_node in sub_nodes:
        _recursive_get_item_value_by_type(item_type, sub_node, result)

def get_item_value_by_type(item_type, parent):
    """ 查找节点parent下的指定类型的项的值

    :param item_type:
    :param parent:
    :return:
    """
    result = []
    _recursive_get_item_value_by_type(item_type, parent, result)
    cnt = len(result)
    if cnt == 0:
        return None
    if cnt == 1:
        return result[0]
    raise ValueError('found multi-result')

def _recursive_get_item_by_type(item_type, parent, result):
    if not parent:
        return

    parent_type = parent.text(2)
    if parent_type == item_type:
        result.append(parent)
        return

    sub_nodes = get_sub_nodes(parent)
    for sub_node in sub_nodes:
        _recursive_get_item_by_type(item_type, sub_node, result)

def get_item_by_type(item_type, parent):
    """ 查找节点parent下的指定类型的项

    :param item_type:
    :param parent:
    :return:
    """
    result = []
    _recursive_get_item_by_type(item_type, parent, result)
    return result

def get_tree_top_nodes(tree):
    nodes = list()
    for index in range(tree.topLevelItemCount()):
        node = tree.topLevelItem(index)
        nodes.append(node)
    return nodes

def is_roi_invalid(roi_node):
    if roi_node is None:
        return False

    w = int(roi_node.child(2).text(1))
    h = int(roi_node.child(3).text(1))

    if w == 0 and h == 0:
        return False
    return True

def valid_number_value(config=None):
    if config is None:
        return

    for key, value in config.items():
        if isinstance(value, (str, int, float)):
            if key in Number_Int_Key:
                config[key] = int(float(value))
            elif key in Number_Float_Key:
                config[key] = float(value)
            elif key in BOOL_Keys:
                config[key] = (value == 'True')
        elif isinstance(value, (dict, OrderedDict)):
            valid_number_value(value)
        elif isinstance(value, (list)):
            for sub_value in value:
                if isinstance(sub_value, (dict, OrderedDict)):
                    valid_number_value(sub_value)

def clear_child(node):
    if node is None:
        return
    for _ in range(node.childCount()):
        node.takeChild(0)

def cvimg_to_qtimg(cvimg):
    cvimg = cvimg.astype('uint8')
    height, width, depth = cvimg.shape
    cvimg = cv2.cvtColor(cvimg, cv2.COLOR_BGR2RGB)
    Qimg = QImage(cvimg.data, width, height, width * depth, QImage.Format_RGB888)
    pixmap = QPixmap.fromImage(Qimg)

    return pixmap

def qtpixmap_to_cvimg(qtpixmap):
    qimg = qtpixmap.toImage()
    temp_shape = (qimg.height(), qimg.bytesPerLine() * 8 // qimg.depth())
    temp_shape += (4,)
    ptr = qimg.bits()
    ptr.setsize(qimg.byteCount())
    result = np.array(ptr, dtype=np.uint8).reshape(temp_shape)
    result = result[..., :3]
    result = result.astype('uint8')
    return result

def set_log_text(text: str):
    ui.set_text(text)

def save_action(node):
    if node is None:
        return

    params = OrderedDict()
    for roi_index in range(node.childCount()):
        action_item = node.child(roi_index)
        name = action_item.text(0)
        value = action_item.text(1)

        if action_item.childCount() > 0:
            params[name] = save_action(action_item)
        else:
            params[name] = value
    return params

def sdk_to_tool_path(config):
    project_data_path = g_project_data_mgr.get_project_data_path()
    sdk_data_path = g_project_data_mgr.get_sdk_data_path()
    change_image_path(config, sdk_data_path, project_data_path)

def tool_to_sdk_path(config):
    project_data_path = g_project_data_mgr.get_project_data_path()
    sdk_data_path = g_project_data_mgr.get_sdk_data_path()
    change_image_path(config, project_data_path, sdk_data_path)

class ExecResult(object):
    def __init__(self):
        self.flag = True

def filter_log(text, valid_log_levels):
    show_log = False
    cur_log_level = ''
    for level in valid_log_levels:
        if level in text:
            show_log = True
            cur_log_level = level
            break

    if not show_log:
        return ''

    valid_log = text.split(cur_log_level)[1]
    if len(valid_log.split(':')) > 1:
        str_valid_log = valid_log.split(':')[1]
    else:
        str_valid_log = valid_log
    return "[{}]: {}".format(cur_log_level, str_valid_log)

def filter_error_log(text):
    return filter_log(text, ['ERROR'])

def filter_warn_log(text):
    return filter_log(text, ['ERROR', 'WARN'])

def filter_info_log(text):
    return filter_log(text, ['ERROR', 'WARN', 'INFO'])

def filter_debug_log(text):
    return filter_log(text, ['ERROR', 'WARN', 'INFO', 'DEBUG'])

def right_tree_value_changed(canvas, node, column, pre_number_value):
    if node is None:
        logger.error("item value changed failed, current item is none")
        return
    logger.debug("right tree value changed key %s, value %s", node.text(0), node.text(1))
    if canvas.mouse_move_flag:
        logger.info("mouse is moving")
        return

    # key can not been changed
    if column == 0:
        return

    changed = True
    node_text0 = node.text(0)
    if node_text0 in Number_Int_Key:
        node_text1 = node.text(1)
        if node_text1 and not is_int_number(node_text1):
            changed = False
            dlg = LabelDialog(text="type of {} should be int".format(node.text(0)))
            dlg.pop_up()

    if node_text0 in Number_Float_Key:
        node_text1 = node.text(1)
        if node_text1 and not is_float_number(node_text1):
            changed = False
            dlg = LabelDialog(text="type of {} should be float".format(node.text(0)))
            dlg.pop_up()

    if not changed:
        node.setText(1, str(pre_number_value))

def is_str_valid(name: str):
    for word in name:
        if word not in VALID_SYMBOL:
            return False
    return True

def clear_files(dir_name, formats=None):
    """ 删除指定目录下的指定格式的文件

    :param dir_name:
    :param formats:
    :return:
    """
    if not formats:
        formats = [".png", ".bmp", ".jpg"]

    if not isinstance(formats, list):
        return

    if not dir_name or not os.path.exists(dir_name):
        return

    file_list = []
    for root, _, file_names in os.walk(dir_name):
        for file_name in file_names:
            if os.path.splitext(file_name)[1] in formats:
                file_list.append(os.path.join(root, file_name))

    for f in file_list:
        os.remove(f)

def get_files_count(dir_name, formats=None):
    if formats is None:
        formats = [".png", ".bmp", ".jpg"]

    if not dir_name or not os.path.exists(dir_name):
        return 0

    file_count = 0
    for _, _, file_names in os.walk(dir_name):
        for file in file_names:
            if os.path.splitext(file)[1] in formats:
                file_count = file_count + 1

    return file_count

def get_file_list(file_path, tree_item, level):
    """ 将file_path目录下的文件添加到树节点tree_item下

    :param file_path:
    :param tree_item:
    :param level:
    :return:
    """
    if tree_item is None:
        print("GetFileList failed, tree_item is None")
        return

    if not file_path or not os.path.exists(file_path):
        print("GetFileList failed, file_path is {}".format(file_path))
        return

    # 获取文件名
    child = QTreeWidgetItem()
    _, file_name = os.path.split(file_path)
    if level > 1:
        # 过滤某些特定文件
        if file_name in ["project.json", "task.json~", "project.json~"]:
            return

        # 创建树的节点，根据后缀名类型的不同来设置不同的icon
        extension = os.path.splitext(file_name)[1]
        if extension in [".jpg", ".png", ".bmp"]:
            child.setText(0, file_name)
            child.setText(2, ITEM_TYPE_IMAGE)
            child.setText(3, file_path)

            icon = QIcon()
            icon.addPixmap(QPixmap(":/menu/image-gallery.png"), QIcon.Normal, QIcon.Off)
            child.setIcon(0, icon)
            tree_item.addChild(child)

        elif extension in ["", ".0", ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9"]:
            child.setText(0, file_name)
            child.setText(2, ITEM_TYPE_IMAGE_FLODER)
            child.setText(3, file_path)

            icon = QIcon()
            icon.addPixmap(QPixmap(":/menu/file.png"), QIcon.Normal, QIcon.Off)
            child.setIcon(0, icon)
            tree_item.addChild(child)

        elif extension in [".json"]:
            return

    # 若dir为目录，则递归
    if os.path.isdir(file_path):
        child.setText(0, file_name)
        child.setText(2, ITEM_TYPE_IMAGE_FLODER)
        child.setText(3, file_path)

        items = []
        for item_name in os.listdir(file_path):
            items.append(os.path.join(file_path, item_name))
        items.sort()
        for file_item in items:
            if level > 1:
                get_file_list(file_item, child, level + 1)
            else:
                get_file_list(file_item, tree_item, level + 1)

def make_tar_gz(output_file, source_file):
    if None in [output_file, source_file]:
        raise ValueError("failed: input file or output file is None")
    if not os.path.exists(source_file):
        raise FileNotFoundError("failed: source file {} is not exists".format(source_file))

    with tarfile.open(output_file, "w:gz") as tar:
        tar.add(source_file, arcname=os.path.basename(source_file))
        return True

def create_bool_tree_item(value):
    """ 创建combobox item

    :param value:
    :return:
    """
    combobox = QComboBox()
    combobox.addItems(BOOL_FLAGS)
    combobox.setCurrentText(str(value))
    return combobox

def new_icon(icon):
    return QIcon(':/' + icon)

def new_button(text, icon=None, slot=None):
    b = QPushButton(text)
    if icon is not None:
        b.setIcon(new_icon(icon))
    if slot is not None:
        b.clicked.connect(slot)
    return b

def del_files(obj_dir):
    for root, dirs, files in os.walk(obj_dir):
        for name in files:
            file_path = os.path.join(root, name)
            os.remove(file_path)
        for dir_name in dirs:
            dir_path = os.path.join(root, dir_name)
            shutil.rmtree(dir_path)

def get_image_list(folder_dir):
    result = []
    for path, _, file_list in os.walk(folder_dir):
        for file_name in file_list:
            if is_image(file_name):
                result.append(os.path.join(path, file_name))

    return result

def exchange_value(key, dictvalue):
    if key != LEARNING_CONFIG_EXCITATIONFUNCTION_Key:
        return dictvalue

    result = OrderedDict()
    for key_, value in dictvalue.items():
        if key_ in LEARNING_CONFIG_TASKID_LIST:
            result[key_] = __get_task_value(value)
        else:
            result[key_] = value
    return result

def __get_task_value(value):
    task_manager = TaskManager()
    task = task_manager.get_task()

    for sub_task in task.get_all():
        if sub_task[1] == value:
            return sub_task[0]
    return 0

def str_to_bool(value):
    return True if value.lower() == 'true' else False

def create_common_button_box(dialog, accept, reject):
    bb = QtWidgets.QDialogButtonBox(
        QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel,
        QtCore.Qt.Horizontal,
        dialog)

    bb.button(bb.Ok).setIcon(QtGui.QIcon(":/menu/done.png"))
    bb.button(bb.Cancel).setIcon(QtGui.QIcon(":/menu/undo.png"))
    bb.accepted.connect(accept)
    bb.rejected.connect(reject)
    return bb

max_stars_repo_name: Passer-D/GameAISDK

max_issues_repo_name: guokaiSama/GameAISDK

max_forks_repo_name: guokaiSama/GameAISDK


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
from six import reraise
from tblib import Traceback
from multiprocessing import Manager, Process
import posix_ipc, mmap

import numpy as np

def to_lodtensor(data, place):
    """convert tensor to lodtensor
    """
    seq_lens = [len(seq) for seq in data]
    cur_len = 0
    lod = [cur_len]
    for l in seq_lens:
        cur_len += l
        lod.append(cur_len)
    flattened_data = numpy.concatenate(data, axis=0).astype("int64")
    flattened_data = flattened_data.reshape([len(flattened_data), 1])
    res = fluid.LoDTensor()
    res.set(flattened_data, place)
    res.set_lod([lod])
    return res

def lodtensor_to_ndarray(lod_tensor):
    """conver lodtensor to ndarray
    """
    dims = lod_tensor.get_dims()
    ret = np.zeros(shape=dims).astype('float32')
    for i in xrange(np.product(dims)):
        ret.ravel()[i] = lod_tensor.get_float_element(i)
    return ret, lod_tensor.lod()

def batch_to_ndarray(batch_samples, lod):
    frame_dim = batch_samples[0][0].shape[1]
    batch_feature = np.zeros((lod[-1], frame_dim), dtype="float32")
    batch_label = np.zeros((lod[-1], 1), dtype="int64")
    start = 0
    for sample in batch_samples:
        frame_num = sample[0].shape[0]
        batch_feature[start:start + frame_num, :] = sample[0]
        batch_label[start:start + frame_num, :] = sample[1]
        start += frame_num
    return (batch_feature, batch_label)

def split_infer_result(infer_seq, lod):
    infer_batch = []
    for i in xrange(0, len(lod[0]) - 1):
        infer_batch.append(infer_seq[lod[0][i]:lod[0][i + 1]])
    return infer_batch

class DaemonProcessGroup(object):
    def __init__(self, proc_num, target, args):
        self._proc_num = proc_num
        self._workers = [
            Process(
                target=target, args=args) for _ in xrange(self._proc_num)
        ]

    def start_all(self):
        for w in self._workers:
            w.daemon = True
            w.start()

    @property
    def proc_num(self):
        return self._proc_num

class EpochEndSignal(object):
    pass

class CriticalException(Exception):
    pass

class SharedNDArray(object):
    """SharedNDArray utilizes shared memory to avoid data serialization when
    data object shared among different processes. We can reconstruct the
    `ndarray` when memory address, shape and dtype provided.

    Args:
        name (str): Address name of shared memory.
        whether_verify (bool): Whether to validate the writing operation.
    """

    def __init__(self, name, whether_verify=False):
        self._name = name
        self._shm = None
        self._buf = None
        self._array = np.zeros(1, dtype=np.float32)
        self._inited = False
        self._whether_verify = whether_verify

    def zeros_like(self, shape, dtype):
        size = int(np.prod(shape)) * np.dtype(dtype).itemsize
        if self._inited:
            self._shm = posix_ipc.SharedMemory(self._name)
        else:
            self._shm = posix_ipc.SharedMemory(
                self._name, posix_ipc.O_CREAT, size=size)
        self._buf = mmap.mmap(self._shm.fd, size)
        self._array = np.ndarray(shape, dtype, self._buf, order='C')

    def copy(self, ndarray):
        size = int(np.prod(ndarray.shape)) * np.dtype(ndarray.dtype).itemsize
        self.zeros_like(ndarray.shape, ndarray.dtype)
        self._array[:] = ndarray
        self._buf.flush()
        self._inited = True

        if self._whether_verify:
            shm = posix_ipc.SharedMemory(self._name)
            buf = mmap.mmap(shm.fd, size)
            array = np.ndarray(ndarray.shape, ndarray.dtype, buf, order='C')
            np.testing.assert_array_equal(array, ndarray)

    @property
    def ndarray(self):
        return self._array

    def recycle(self, pool):
        self._buf.close()
        self._shm.close_fd()
        self._inited = False
        pool[self._name] = self

    def __getstate__(self):
        return (self._name, self._array.shape, self._array.dtype, self._inited,
                self._whether_verify)

    def __setstate__(self, state):
        self._name = state[0]
        self._inited = state[3]
        self.zeros_like(state[1], state[2])
        self._whether_verify = state[4]

class SharedMemoryPoolManager(object):
    """SharedMemoryPoolManager maintains a multiprocessing.Manager.dict object.
    All available addresses are allocated once and will be reused. Though this
    class is not process-safe, the pool can be shared between processes. All
    shared memory should be unlinked before the main process exited.

    Args:
        pool_size (int): Size of shared memory pool.
        manager (dict): A multiprocessing.Manager object, the pool is
                        maintained by the proxy process.
        name_prefix (str): Address prefix of shared memory.
    """

    def __init__(self, pool_size, manager, name_prefix='/deep_asr'):
        self._names = []
        self._dict = manager.dict()

        for i in xrange(pool_size):
            name = name_prefix + '_' + str(i)
            self._dict[name] = SharedNDArray(name)
            self._names.append(name)

    @property
    def pool(self):
        return self._dict

    def __del__(self):
        for name in self._names:
            # have to unlink the shared memory
            posix_ipc.unlink_shared_memory(name)

def suppress_signal(signo, stack_frame):
    pass

def suppress_complaints(verbose, notify=None):
    def decorator_maker(func):
        def suppress_warpper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            except:
                et, ev, tb = sys.exc_info()

                if notify is not None:
                    notify(except_type=et, except_value=ev, traceback=tb)

                if verbose == 1 or isinstance(ev, CriticalException):
                    reraise(et, ev, Traceback(tb).as_traceback())

        return suppress_warpper

    return decorator_maker

class ForceExitWrapper(object):
    def __init__(self, exit_flag):
        self._exit_flag = exit_flag

    @suppress_complaints(verbose=0)
    def __call__(self, *args, **kwargs):
        self._exit_flag.value = True

    def __eq__(self, flag):
        return self._exit_flag.value == flag

max_stars_repo_name: buptwds/models

max_issues_repo_name: buptwds/models

max_forks_repo_name: buptwds/models


text:
import numpy as np
from sympy import simplify, Matrix, var

xi = var('xi')
eta = var('eta')

print('Chebyshev Polynomials of the Second Kind')
u = [1, 2*xi]
for i in range(2, 15):
    ui = simplify(2*xi*u[i-1] - u[i-2])
    u.append(ui)
for i, ui in enumerate(u):
    print('u({0}) = {1}'.format(i, ui))

with open('cheb.txt', 'w') as f:
    f.write('Number of terms: {0}\n\n'.format(len(u)))
    f.write(','.join(map(str, u)).replace('**','^') + '\n\n')
    f.write(','.join(map(str, u)).replace('xi', 'eta').replace('**','^'))

print np.outer(u, [ui.subs({xi:eta}) for ui in u])

if False:
    m = Matrix([[2*xi, 1, 0, 0],
                [1, 2*xi, 1, 0],
                [0, 1, 2*xi, 1],
                [0, 0, 1, 2*xi]])

    print('m.det() {0}'.format(simplify(m.det())))

max_stars_repo_name: mrosemeier/compmech

max_issues_repo_name: mrosemeier/compmech

max_forks_repo_name: mrosemeier/compmech

reason: import numpy,from sympy


text:
#Importing Libraries
import matplotlib.pyplot as plt
plt.style.use('ggplot')

import numpy as np
import pandas as pd
import seaborn as sns
plt.rcParams['figure.figsize'] = (12, 8)

#Loading the data
data = pd.read_csv('bike_sharing_data.txt')
data.head()
data.info()

#Visualising the Data
ax = sns.scatterplot(x = " Total Population", y = " Total Profit", data = data)
ax.set_title = ("Total Profit in $10,000 vs. City population in 10,000's")

#Cost Function
def Cost(X, y, theta):
    m = len(y)
    y_pred = X.dot(theta)
    result = 1 / (2 * m) * np.sum((y_pred - y) ** 2)

    return result

#Calculating the Cost
m = data.population.values.size
X = np.append(np.ones((m, 1)), data.population.values.reshape(m, 1), axis = 1)
y = data.Profit.values.reshape(m, 1)
theta = zeros((2, 1))

Cost(X, y, theta)

#Gradient descent
def Grad(X, y, theta, alpha, iterations):
    m = len(y)
    costs = []
    for i in range(iterations):
        y_pred = X.dot(theta)
        result = 1 / (2 * m) * np.sum((y_pred - y) ** 2)
        theta = theta - (alpha / m) * (np.dot((X.transpose()), (y_pred - y)))

max_stars_repo_path: Linear Regression/Linear Regression.py

max_issues_repo_path: Linear Regression/Linear Regression.py

max_forks_repo_path: Linear Regression/Linear Regression.py


text:
from __future__ import print_function

from pprint import pprint
from functools import reduce
import pickle
import numpy as np
import pandas as pd
import logging

from annette.estimation.layers.base import BaseLayer

class AdditionLayer(BaseLayer):
    """Addition estimation"""

    def __init__(self, name, layer_type = "Addition", est_type = "roofline", op_s = 1*1e9, bandwidth = 1*1e9, architecture = None):
        self.name = name
        self.layer_type = layer_type
        self.estimation = est_type

        # Model parameters
        self.op_s = op_s
        self.bandwidth = bandwidth
        if not architecture:
            self.architecture = {"bit_act": 8, "bit_weights": 8}
        else:
            self.architecture = architecture

        # Layer parameters
        self.layer = {}
        self.layer['num_inputs'] = None
        self.layer['num_outputs'] = None
        self.layer['num_ops'] = None
        self.layer['parents'] = None

        # Layer description dictionary, add information for rebuilding here
        self.desc = self.gen_dict()

    def compute_parameters(self, layer = None):
        """Compute Parameters for Base Layer prediction"""
        self.layer = self.compute_nums(self.layer)
        self.layer['data_bytes'] = self.layer['num_outputs'] * self.architecture['bit_act']*3 / 8 # Element wise addition layer moves data *3
        return self.layer

    def estimate_roofline(self):
        """returns roofline estimated AdditionLayer execution time (ms)"""
        self.layer['data_roof'] = self.layer['data_bytes'] / self.bandwidth
        self.layer['time_ms'] = np.max([self.layer['data_roof']])*1000 # to milliseconds
        return self.layer['time_ms']

max_stars_repo_name: embedded-machine-learning/annette

max_issues_repo_name: embedded-machine-learning/annette

max_forks_repo_name: embedded-machine-learning/annette


text:
import matplotlib.pyplot as plt
import numpy as np
import math
import matplotlib
from matplotlib import pyplot as plt
from matplotlib.collections import LineCollection
from scipy.cluster import hierarchy as hier
from scipy.spatial import distance as ssd
from sklearn.manifold import MDS
from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt
import pylab
import rf_dist
import T_dist
import pylab
import Q_dist
import KC_dist

def main(a,auto):
    bins=8
    if (auto.upper()=="Y"): bins="auto"
    plt.hist(a, bins=bins)
    plt.show()

dist=[]
array=[]
with open("Random_200.txt") as f:
    for line in f:
         array.append(line.replace('\n', ''))

n_samples = len(array)

Histro=[]

for x in range (0,len(array)):

    for y in range (0,len(array)):
        dist=T_dist.main(array[x], array[y])
        Histro.append(dist)

main(Histro,"y")


text:
from django.shortcuts import render, redirect
from django.views.generic.edit import FormView
from .forms import ExelForm
# from .exchange import DataFrame
import matplotlib.pyplot as plt
from scipy import stats
import os

class FileFieldView(FormView):

    form_class = ExelForm  # Upload from from forms.py for template
    template_name = os.path.join("exelchange", "index.html")  # Basic template

    def get(self, request, **kwargs):
        form = self.form_class()
        check(request)
        path = os.path.join('exelchange', 'static', 'media')
        path = os.path.join(path, request.COOKIES.get('sessionid', ), 'graph.png')

        cook = os.path.join(request.COOKIES['sessionid'], 'graph.png') if os.path.exists(path) else None

        return render(request, os.path.join("exelchange", "index.html"), {'form': form,
                                                                          'img': cook,
                                                                          'result': request.session.get('result', '')},)

    def post(self, request, *args, **kwargs):
        form_class = self.get_form_class()
        form = self.get_form(form_class)  # Getting form for template
        if form.is_valid():
            x = list(map(float, request.POST['x_coord'].split()))
            y = list(map(float, request.POST['y_coord'].split()))
            appr = stats.linregress(x, y)
            a = round(appr.slope, 4)
            b = round(appr.intercept, 4)
            y1 = [a * i + b for i in x]
            label = f'y = {a}*x {b}' if b < 0 else f'{a}*x + {b}'

            result = [((float(i) - b) / a, float(i)) for i in request.POST['absorb'].split()]
            request.session['result'] = result
            fig = plt.figure(figsize=(15, 10))
            ax = fig.add_subplot(111)
            ax.set_title('Approximation curve',
                         fontsize=25,
                         color='black',
                         pad=10)
            ax.plot(x, y, label='initial', color='black')
            ax.plot(x, y1, label=label, color='orange')
            for i, j in result:
                ax.plot(i, j, 'o', color='green', markersize=20)

            plt.legend(fontsize='xx-large')
            plt.grid()
            check(request)
            fig.savefig(os.path.join('exelchange', 'static', 'media', request.COOKIES['sessionid'], 'graph.png'))
            # print(request.COOKIES['result'])
        return redirect(request.path)

def check(request):
    if not request.COOKIES.get('sessionid', '') or not request.COOKIES['sessionid']:  # set session cookie
        request.session.create()
        request.COOKIES['sessionid'] = request.session.session_key

    path = os.path.join('exelchange', 'static')
    if not os.path.exists(path):
        os.mkdir(path)
    path = os.path.join(path, 'media')
    if not os.path.exists(path):
        os.mkdir(path)
    path = os.path.join(path, request.COOKIES.get('sessionid', ))
    if not os.path.exists(path):
        os.mkdir(path)

reason: from scipy


text:
# @author Kyle
# Copyright Kyle 2019
# Generates a plot of ups and downs in a given story structure
# Inspired by https://github.com/IndicoDataSolutions/plotlines
# (see above for use of a trained sentiment api and other nifty things, including an awesome description of everything)
#bokeh code based on https://bokeh.pydata.org/en/1.0.0/docs/user_guide/examples/tools_hover_tooltip_formatting.html
import numpy as np

from bokeh.io import output_file, show
from bokeh.models import ColumnDataSource, HoverTool
from bokeh.plotting import figure

from textblob import TextBlob
import heapq

fileName = "t1.txt"

file = open(fileName)
text = TextBlob(file.read())
word_counts = text.word_counts

vals = []
nums = []
sents = []
i = 0

for thing in text.sentences:
    sents.append(thing)
    vals.append(thing.sentiment.polarity)
    nums.append(i)
    i = i + 1

# Thanks to ideas from https://medium.com/incedge/text-summarization-96079bf23e83
sentence_scores = {}
for sent in sents:
    count = 0
    words = sent.words
    for word in words:
        if word in word_counts:
            count += word_counts[word]
    sentence_scores[sent] = count

summary_sent = heapq.nlargest(15, sentence_scores, key=sentence_scores.get)

summ = []
for sent in summary_sent:
    summ.append(sents.index(sent))
    summ.sort()

vals = []
nums = []

for thing in summ:
    vals.append(sents[thing].sentiment.polarity)
    nums.append(thing)
    print sents[thing]

sent_list = []

for thing in nums:
    sent_list.append(str(sents[thing]))

output_file("sentiment_quantifier.html")

source = ColumnDataSource(data={
    'numsent'      : nums,
    'sent val' : vals,
    'sent list' : sent_list,
    })

p = figure(plot_height=250, x_axis_type="linear", tools="", toolbar_location=None,
           title="Sentiment Quantifier", sizing_mode="scale_width")
p.background_fill_color="#f5f5f5"
p.grid.grid_line_color="white"
p.xaxis.axis_label = 'Sentence Number'
p.yaxis.axis_label = 'Sentiment Polarity'
p.axis.axis_line_color = None

p.line(x='numsent', y='sent val', line_width=2, color='#ebbd5b', source=source)

p.add_tools(HoverTool(
    tooltips=[
        ( 'Sentence Number   :',   '@numsent{%1.4f}'            ),
        ( 'Sentiment Polarity:',  '@{sent val}{%1.4f}' ),
        ( 'Sentence          :', '@{sent list}{%s}' ),
    ],

    formatters={
        'numsent'      : 'printf',
        'sent val' : 'printf',
        'sent list' : 'printf',
    },
    mode='vline'
))

show(p)

max_stars_repo_name: Mytholody/sentimentPlotter

max_issues_repo_name: Mytholody/sentimentPlotter

max_forks_repo_name: Mytholody/sentimentPlotter


text:
# -*- coding: utf-8 -*-
# 比特币走势预测，使用时间序列ARMA
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from statsmodels.tsa.arima_model import ARMA
import warnings
from itertools import product
from datetime import datetime
warnings.filterwarnings('ignore')
# 数据加载
df = pd.read_csv('./bitcoin_2012-01-01_to_2018-10-31.csv')
# 将时间作为df的索引
df.Timestamp = pd.to_datetime(df.Timestamp)
df.index = df.Timestamp
# 数据探索
print(df.head())
# 按照月，季度，年来统计
df_month = df.resample('M').mean()
df_Q = df.resample('Q-DEC').mean()
df_year = df.resample('A-DEC').mean()
# 按照天，月，季度，年来显示比特币的走势
fig = plt.figure(figsize=[15, 7])
plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签
plt.suptitle('比特币金额（美金）', fontsize=20)
plt.subplot(221)
plt.plot(df.Weighted_Price, '-', label='按天')
plt.legend()
plt.subplot(222)
plt.plot(df_month.Weighted_Price, '-', label='按月')
plt.legend()
plt.subplot(223)
plt.plot(df_Q.Weighted_Price, '-', label='按季度')
plt.legend()
plt.subplot(224)
plt.plot(df_year.Weighted_Price, '-', label='按年')
plt.legend()
plt.show()
# 设置参数范围
ps = range(0, 3)
qs = range(0, 3)
parameters = product(ps, qs)
parameters_list = list(parameters)
# 寻找最优ARMA模型参数，即best_aic最小
results = []
best_aic = float("inf") # 正无穷
for param in parameters_list:
    try:
        model = ARMA(df_month.Weighted_Price,order=(param[0], param[1])).fit()
    except ValueError:
        print('参数错误:', param)
        continue
    aic = model.aic
    if aic < best_aic:
        best_model = model
        best_aic = aic
        best_param = param
    results.append([param, model.aic])
# 输出最优模型
result_table = pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
print('最优模型: ', best_model.summary())
# 比特币预测
df_month2 = df_month[['Weighted_Price']]
date_list = [datetime(2018, 11, 30), datetime(2018, 12, 31), datetime(2019, 1, 31), datetime(2019, 2, 28), datetime(2019, 3, 31),
             datetime(2019, 4, 30), datetime(2019, 5, 31), datetime(2019, 6, 30)]
future = pd.DataFrame(index=date_list, columns= df_month.columns)
df_month2 = pd.concat([df_month2, future])
df_month2['forecast'] = best_model.predict(start=0, end=91)
# 比特币预测结果显示
plt.figure(figsize=(20,7))
df_month2.Weighted_Price.plot(label='实际金额')
df_month2.forecast.plot(color='r', ls='--', label='预测金额')
plt.legend()
plt.title('比特币金额（月）')
plt.xlabel('时间')
plt.ylabel('美金')
plt.show()

max_stars_repo_name: dailishan/bitcoin

max_issues_repo_name: dailishan/bitcoin

max_forks_repo_name: dailishan/bitcoin

reason: import numpy,from statsmodels


text:
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Functions for running experiments (colabs, xm) and storing/saving data."""
import dataclasses
import os
from typing import (Any, Callable, List, MutableMapping, Optional, Text, Tuple,
                    Union)

import graph_nets
import more_itertools
import numpy as np
import sonnet as snt
import tensorflow as tf
from graph_attribution import datasets
from graph_attribution import graphnet_models as models
from graph_attribution import graphnet_techniques as techniques
from graph_attribution import graphs as graph_utils
from graph_attribution import tasks, templates

# Typing alias.
GraphsTuple = graph_nets.graphs.GraphsTuple
TransparentModel = templates.TransparentModel
AttributionTechnique = templates.AttributionTechnique
AttributionTask = templates.AttributionTask
NodeEdgeTensors = templates.NodeEdgeTensors
MethodDict = MutableMapping[Text, AttributionTechnique]
OrderedDict = MutableMapping

def set_seed(random_seed: int):
    """Sets initial seed for random numbers."""
    tf.random.set_seed(random_seed)
    np.random.seed(random_seed)

def get_graph_block(block_type: models.BlockType, node_size: int,
                    edge_size: int, global_size: int, index: int) -> snt.Module:
    """Gets a GNN block based on enum and sizes."""
    name = f'{block_type.name}_{index+1}'
    if block_type == models.BlockType.gcn:
        return models.GCNLayer(models.get_mlp_fn([node_size] * 2), name=name)
    elif block_type == models.BlockType.gat:
        return models.SelfAttention(
            node_size, models.get_mlp_fn([node_size] * 2))
    elif block_type == models.BlockType.mpnn:
        return models.NodeEdgeLayer(
            models.get_mlp_fn([node_size] * 2),
            models.get_mlp_fn([edge_size] * 2),
            name=name)
    elif block_type == models.BlockType.graphnet:
        use_globals = index != 0
        return graph_nets.modules.GraphNetwork(
            node_model_fn=models.get_mlp_fn([node_size] * 2),
            edge_model_fn=models.get_mlp_fn([edge_size] * 2),
            global_model_fn=models.get_mlp_fn([global_size] * 2),
            edge_block_opt={'use_globals': use_globals},
            node_block_opt={'use_globals': use_globals},
            global_block_opt={'use_globals': use_globals},
            name=name)
    else:
        raise ValueError(f'block_type={block_type} not implemented')

class GNN(snt.Module, templates.TransparentModel):
    """A general graph neural network for graph property prediction."""

    def __init__(self,
                 node_size: int,
                 edge_size: int,
                 global_size: int,
                 y_output_size: int,
                 block_type: models.BlockType,
                 activation: models.Activation,
                 target_type: templates.TargetType,
                 n_layers: int = 3):
        super(GNN, self).__init__(name=block_type.name)

        # Graph encoding step, basic linear mapping.
        self.encode = graph_nets.modules.GraphIndependent(
            node_model_fn=lambda: snt.Linear(node_size),
            edge_model_fn=lambda: snt.Linear(edge_size))
        # Message passing steps or GNN blocks.
        gnn_layers = [
            get_graph_block(
                block_type,
                node_size,
                edge_size,
                global_size,
                index)
            for index in range(0, n_layers)
        ]
        self.gnn = models.SequentialWithActivations(gnn_layers)
        if target_type == templates.TargetType.globals:
            readout = models.ReadoutGAP(global_size, tf.nn.softmax)
        else:
            readout = graph_nets.modules.GraphIndependent()

        self.readout = readout
        self.linear = snt.Linear(y_output_size, with_bias=False)
        self.activation = models.cast_activation(activation)
        self.pred_layer = snt.Sequential([self.linear, self.activation])
        self.block_type = block_type
        self.target_type = target_type

    def cast_task_batch_index(
            self,
            task_index: Optional[int] = None,
            batch_index: Optional[int] = None) -> Tuple[int, Union[int, slice]]:
        """Provide defaults for task and batch indices when not present."""
        task_index = 0 if task_index is None else task_index
        batch_index = slice(None) if batch_index is None else batch_index
        return task_index, batch_index

    @tf.function(experimental_relax_shapes=True)
    def get_graph_embedding(self, x: GraphsTuple) -> tf.Tensor:
        """Build a graph embedding."""
        out_graph = self.readout(self.gnn(self.encode(x)))
        return models.get_graph_attribute(out_graph, self.target_type)

    def __call__(self, x: GraphsTuple) -> tf.Tensor:
        """Typical forward pass for the model."""
        graph_emb = self.get_graph_embedding(x)
        y = self.pred_layer(graph_emb)
        return y

    def predict(self,
                x: GraphsTuple,
                task_index: Optional[int] = None,
                batch_index: Optional[int] = None) -> tf.Tensor:
        """Forward pass with output set on the task of interest (y[batch_index, task_index])."""
        task_index, batch_index = self.cast_task_batch_index(
            task_index, batch_index)
        return self(x)[batch_index, task_index]

    @tf.function(experimental_relax_shapes=True)
    def get_gradient(self,
                     x: GraphsTuple,
                     task_index: Optional[int] = None,
                     batch_index: Optional[int] = None) -> NodeEdgeTensors:
        """Gets gradient of inputs wrt to the target."""
        with tf.GradientTape(watch_accessed_variables=False) as gtape:
            gtape.watch([x.nodes, x.edges])
            y = self.predict(x, task_index, batch_index)
        nodes_grad, edges_grad = gtape.gradient(y, [x.nodes, x.edges])
        return nodes_grad, edges_grad

    @tf.function(experimental_relax_shapes=True)
    def get_gap_activations(self, x: GraphsTuple) -> NodeEdgeTensors:
        """Gets node-wise and edge-wise contributions to graph embedding."""
        return self.readout.get_activations(self.gnn(self.encode(x)))

    @tf.function(experimental_relax_shapes=True)
    def get_prediction_weights(self,
                               task_index: Optional[int] = None) -> tf.Tensor:
        """Gets last layer prediction weights."""
        task_index, _ = self.cast_task_batch_index(task_index, None)
        w = self.linear.w[:, task_index]
        return w

    @tf.function(experimental_relax_shapes=True)
    def get_intermediate_activations_gradients(
        self,
        x: GraphsTuple,
        task_index: Optional[int] = None,
        batch_index: Optional[int] = None
    ) -> Tuple[List[NodeEdgeTensors], List[NodeEdgeTensors], tf.Tensor]:
        """Gets intermediate layer activations and gradients."""
        task_index, batch_index = self.cast_task_batch_index(
            task_index, batch_index)
        acts = []
        grads = []
        with tf.GradientTape(
                persistent=True, watch_accessed_variables=False) as gtape:
            gtape.watch([x.nodes, x.edges])
            x = self.encode(x)
            outputs, acts = self.gnn.call_with_activations(x)
            outputs = self.readout(outputs)
            embs = models.get_graph_attribute(outputs, self.target_type)
            y = self.pred_layer(embs)[batch_index, task_index]
        acts = [(act.nodes, act.edges) for act in acts]
        grads = gtape.gradient(y, acts)
        return acts, grads, y

    @tf.function(experimental_relax_shapes=True)
    def get_attention_weights(self, inputs: GraphsTuple) -> List[tf.Tensor]:
        if self.block_type != models.BlockType.gat:
            raise ValueError(
                f'block_type={self.block_type.name}, attention only works with "gat" blocks'
            )

        outs = self.encode(inputs)
        weights = []
        for block in self.gnn._layers:  # pylint: disable=protected-access
            outs, w = block.apply_attention(outs)
            weights.append(w)

        return weights

    @classmethod
    def from_hparams(cls, hp, task:AttributionTask) -> 'GNN':
      return cls(node_size = hp.node_size,
               edge_size = hp.edge_size,
               global_size = hp.global_size,
               y_output_size = task.n_outputs,
               block_type = models.BlockType(hp.block_type),
               activation = task.get_nn_activation_fn(),
               target_type = task.target_type,
               n_layers = hp.n_layers)

def get_batched_attributions(method: AttributionTechnique,
                             model: TransparentModel,
                             inputs: GraphsTuple,
                             batch_size: int = 2500) -> List[GraphsTuple]:
    """Batched attribution since memory (e.g. IG) can be an issue."""
    n = graph_utils.get_num_graphs(inputs)
    att_pred = []
    actual_batch_size = int(np.ceil(batch_size / method.sample_size))
    for chunk in more_itertools.chunked(range(n), actual_batch_size):
        x_chunk = graph_utils.get_graphs_tf(inputs, np.array(chunk))
        att = method.attribute(x_chunk, model)
        att_pred.extend(att)
    return att_pred

def generate_result(model: templates.TransparentModel,
                    method: templates.AttributionTechnique,
                    task: templates.AttributionTask,
                    inputs: GraphsTuple,
                    y_true: np.ndarray,
                    true_atts: List[GraphsTuple],
                    pred_atts: Optional[List[GraphsTuple]] = None,
                    reducer_fn: Optional[Callable[[np.ndarray],
                                                  Any]] = np.nanmean,
                    batch_size: int = 1000) -> MutableMapping[Text, Any]:
    """For a given model, method and task, generate metrics."""
    if pred_atts is None:
        pred_atts = get_batched_attributions(method, model, inputs, batch_size)
    result = task.evaluate_attributions(
        true_atts, pred_atts, reducer_fn=reducer_fn)
    # Need to reshape since predict returns a 1D array.
    y_pred = model.predict(inputs).numpy().reshape(-1, 1)
    result.update(task.evaluate_predictions(y_true, y_pred))
    result['Task'] = task.name
    result['Technique'] = method.name
    result['Model'] = model.name
    return result

@dataclasses.dataclass(frozen=True)
class ExperimentData:
    """Helper class to hold all data relating to an experiment."""
    x_train: GraphsTuple
    x_test: GraphsTuple
    y_train: np.ndarray
    y_test: np.ndarray
    att_test: List[GraphsTuple]
    x_aug: Optional[GraphsTuple] = None
    y_aug: Optional[np.ndarray] = None

    @classmethod
    def from_data_and_splits(cls, x, y, att, train_index,
                             test_index, x_aug=None, y_aug=None):
        """Build class from data and split indices."""
        if np.intersect1d(train_index, test_index).shape[0]:
            raise ValueError('train/test indices have overlap!.')
        return cls(
            x_train=graph_utils.get_graphs_tf(x, np.array(train_index)),
            x_test=graph_utils.get_graphs_tf(x, np.array(test_index)),
            y_train=y[train_index],
            y_test=y[test_index],
            att_test=[att[i] for i in test_index],
            x_aug=x_aug, y_aug=y_aug)

def get_experiment_setup(
    task_type: Union[tasks.Task, Text], block_type: Union[models.BlockType,
                                                          Text]
) -> Tuple[ExperimentData, AttributionTask, MethodDict]:
    """Get experiment data based on task_name."""
    task_type = tasks.Task(task_type)
    task = tasks.get_task(task_type)
    fnames = datasets.get_default_experiment_filenames(task_type)
    has_null_features = os.path.exists(fnames['null'])
    expects_aug = isinstance(task.task_type, tasks.BinaryClassificationTaskType
                             ) and has_null_features

    # Load inputs, labels and attributions.
    x = datasets.load_graphstuples(fnames['x'])
    if len(x) > 1:
        raise ValueError(f'Expected a single graph for x={fnames["x"]}')
    x = x[0]

    y = datasets.load_npz(fnames['y'])['y']
    att = datasets.load_graphstuples(fnames['att'])
    train_index, test_index = datasets.load_train_test_indices(
        fnames['splits'])

    x_aug = datasets.load_graphstuples(
        fnames['x_aug']) if expects_aug else None
    y_aug = datasets.load_npz(fnames['y_aug'])[
        'y_aug'] if expects_aug else None

    # Setup experiment data
    exp = ExperimentData.from_data_and_splits(x, y, att, train_index, test_index,
                                              x_aug, y_aug)
    # Null vectors for IG are possible when a graph has node/edge features.
    if has_null_features:
        null_data = datasets.load_npz(fnames['null'])
        node_null, edge_null = null_data['node'], null_data['edge']
    else:
        node_null, edge_null = None, None

    # Get available methods for the problem.
    use_attention = models.BlockType(block_type) == models.BlockType.gat
    use_gap_readout = task.target_type == templates.TargetType.globals
    methods = techniques.get_techniques_dict(node_null, edge_null,
                                             use_gap_readout, use_attention)

    return exp, task, methods

max_stars_repo_name: edwardelson/graph-attribution

max_issues_repo_name: edwardelson/graph-attribution

max_forks_repo_name: edwardelson/graph-attribution


text:
import pandas as pd
import pandas_profiling
import numpy as np

df = pd.read_csv("examples/Meteorite_Landings.csv",
                 parse_dates=['year'], encoding='UTF-8')

# Note: Pandas does not support dates before 1880, so we ignore these for this analysis
df['year'] = pd.to_datetime(df['year'], errors='coerce')

# Example: Constant variable
df['source'] = "NASA"

# Example: Boolean variable
df['boolean'] = np.random.choice([True, False], df.shape[0])

# Example: Mixed with base types
df['mixed'] = np.random.choice([1, "A"], df.shape[0])

# Example: Highly correlated variables
df['reclat_city'] = df['reclat'] + np.random.normal(scale=5, size=(len(df)))

# Example: Duplicate observations
duplicates_to_add = pd.DataFrame(df.iloc[0:10])
duplicates_to_add[u'name'] = duplicates_to_add[u'name'] + " copy"

df = df.append(duplicates_to_add, ignore_index=True)

max_stars_repo_name: MWTA/Natural-Language-Processing-Python

max_issues_repo_name: MWTA/Natural-Language-Processing-Python

max_forks_repo_name: MWTA/Natural-Language-Processing-Python