text:
'''
    General support functions
'''
import pandas as pd
import numpy as np
import torch
import random
import sys
import os
import matplotlib.pyplot as plt
import logging

from tqdm import tqdm
from copy import deepcopy
from typing import List

def seed_torch(seed=42):
    '''Make the code reproducible'''
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True


text:
import numpy as np
from PIL import ImageDraw

from eda.image.transforms.transform import EdaTransform
from eda.image.transforms.utils import categorize_value

class Cutout(EdaTransform):
    def __init__(self, name=None, prob=1.0, level=0, max_pixel=20, color=None):
        self.max_pixel = max_pixel
        self.value_range = (0, self.max_pixel)
        self.color = color
        super().__init__(name, prob, level)

    def transform(self, pil_img, label, **kwargs):
        pil_img = pil_img.copy()
        degree = categorize_value(self.level, self.value_range, "int")
        width, height = pil_img.size

        x0 = np.random.uniform(width)
        y0 = np.random.uniform(height)

        x0 = int(max(0, x0 - degree / 2.0))
        y0 = int(max(0, y0 - degree / 2.0))
        x1 = min(width, x0 + degree)
        y1 = min(height, y0 + degree)

        xy = (x0, y0, x1, y1)

        if self.color is not None:
            color = self.color
        elif pil_img.mode == "RGB":
            color = (125, 123, 114)
        elif pil_img.mode == "L":
            color = 121
        else:
            raise ValueError(f"Unspported image mode {pil_img.mode}")

        ImageDraw.Draw(pil_img).rectangle(xy, color)

        return pil_img, label

    def __repr__(self):
        return (
            f"<Transform ({self.name}), prob={self.prob}, level={self.level}, "
            f"max_pixel={self.max_pixel}>"
        )

ext: py

max_stars_repo_name: simran-arora/emmental-tutorials

max_issues_repo_name: simran-arora/emmental-tutorials

max_forks_repo_name: simran-arora/emmental-tutorials

reason: import numpy


text:
# -*- coding:utf-8 -*-

import re
import os
import csv
import argparse
import pickle
from tqdm import tqdm
from itertools import chain
import numpy as np
import glob
from nltk.tokenize import RegexpTokenizer
import logging
import pandas as pd

import torch
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
from torchtext import data
from torchtext import vocab
from torchtext.vocab import GloVe

train_data_txt = "../data/EMTC_data/emtc_train.txt"
train_data_csv = "../data/EMTC_data/emtc_train.csv"
test_data_txt = "../data/EMTC_data/emtc_test.txt"
test_data_csv = "../data/EMTC_data/emtc_test.csv"

def txt_to_csv():
    with open(train_data_txt, 'r') as fin, open(train_data_csv, 'w') as fout:
        o = csv.writer(fout)
        for line in fin:
            o.writerow(''.join(re.sub(':', '"', line, 1)).rsplit('#',1))

def cofirm_csv():
    # たしかめ
    with open(train_data_csv, 'r') as f:
        lines = f.readlines()
        print(lines)

def pandas_txt2csv():
    train_data = pd.read_csv(train_data_txt, header=None, sep=':|#', error_bad_lines=False, engine='python')
    train_data.to_csv(train_data_csv, index=False, header=None)
    print(train_data[:5])

    test_data = pd.read_csv(test_data_txt, header=None, sep=':|#', error_bad_lines=False, engine='python')
    test_data.to_csv(test_data_csv, index=False, header=None)
    print(test_data[:5])

def data_to_torch():
    tokenizer = lambda x: x.split()
    TEXT = data.Field(sequential=True, tokenize=tokenizer, lower=True, include_lengths=True, batch_first=True, fix_length=50)
    LABEL = data.LabelField()

    # train_dataset = data.TabularDataset(path=train_data_csv, format='csv', fields=[('text', TEXT), ('label', LABEL)])
    # test_dataset = data.TabularDataset(path=test_data_csv, format='csv', fields=[('text', TEXT), ('label', LABEL)])

    train_dataset, test_dataset = data.TabularDataset.splits(path='../data/EMTC_data/', train='emtc_train.csv', test='emtc_test.csv', format='csv', fields=[('text', TEXT), ('label', LABEL)])
    train_dataset, val_dataset = train_dataset.split()

    print('len(train): {}'.format(len(train_dataset)))
    print('len(validation): {}'.format(len(val_dataset)))
    print('len(test): {}'.format(len(test_dataset)))

    # 単語に番号を振る
    TEXT.build_vocab(train_dataset, vectors=GloVe(name='840B', dim=300), min_freq=2)
    LABEL.build_vocab(train_dataset)

    # 単語カウント結果
    print('単語カウント: {}'.format(TEXT.vocab.freqs.most_common(10)))
    # print(LABEL.vocab.freqs)
    print('上位10個: {}'.format(TEXT.vocab.itos[:10])) #単語10個
    print('サイズ:{}'.format(TEXT.vocab.vectors.size()))

    # バッチ化
    train_iter, val_iter, test_iter = data.BucketIterator.splits((train_dataset, val_dataset, test_dataset), batch_size=32, sort_key=lambda x: len(x.text), repeat=False,shuffle=True)

    vocab_size = len(TEXT.vocab)
    print(vocab_size)

    word_embeddings = TEXT.vocab.vectors
    print(TEXT.vocab.vectors.size())

    # 入力前のTensorの確認
    for batch in enumerate(train_iter):
        import pdb;pdb.set_trace()
        print(batch.text[0].size())
        print(batch.text[1].size())
        print(label.size())
        print("1データ目の単語列を表示")
        print(batch.text[0][0])
        print(batch.text[1][0])
        print(batch.label[0])
        print([TEXT.vocab.itos[data] for data in batch.text[0][0]].tolist())
        print('ラベル')
        print(batch.label[0].item())
        break

if __name__ == '__main__':
    # txt_to_csv()
    # cofirm_csv()
    # pandas_txt2csv()
    data_to_torch()


text:
#%%
import numpy as np
import pandas as pd
import tqdm

# Load the complete gene-linked dataset
data = pd.read_csv('../../data/go_biological_processes.csv')

# %%
# Make a dictionary of all possible combinations
processes = data['go_biological_process'].unique()

connection_dict = {f'{p1}:::{p2}':0 for p1 in processes for p2 in processes if p1 != p2}

# %%
# Iterate through each gene and populate the connections
for g, d in tqdm.tqdm(data.groupby('gene')):
    # Find the processes.
    member_of = d['go_biological_process'].values
    for p1 in member_of:
        for p2 in member_of:
            if p1 != p2:
                connection_dict[f'{p1}:::{p2}'] += 1


text:
# Fix paths for imports to work in unit tests ----------------

if __name__ == "__main__":

    from _fix_paths import fix_paths
    fix_paths()

# ------------------------------------------------------------

# Load libraries ---------------------------------------------

from typing import Dict

import numpy as np
import pandas as pd

from ssa_sim_v2.simulator.modules.multi_state_simulator_module import MultiStateSimulatorModule
from ssa_sim_v2.simulator.modules.average_position.average_position_base_module import AveragePositionModule

# ------------------------------------------------------------

class AveragePositionDateHoWModule(MultiStateSimulatorModule):
    """
    Module for the average position in AdWords results for segmented data based on the max click probability
    and the actual probability with a separate prior for every date and hour of week in a specified range.

    A normal noise can be added to this value.

    :ivar pd.DataFrame priors: Prior in the form of a DataFrame with three columns: date, hour_of_week,
            prior. The last column defines priors (in the form of dictionaries)
            for given dates and hours of week.
    :ivar pd.DataFrame base_classes: DataFrame with three columns: date, hour_of_week,
            base_class. The last column defines class for for given dates and hours of week.
    :ivar Dict[str, AveragePositionModule] models: Dict with average price models for every valid pair date and hour of week.
    """

    def __init__(self, priors=None, base_classes=None, seed=123):
        """
        :param pd.DataFrame priors: DataFrame with three columns: date, hour_of_week,
            prior. The last column defines priors (in the form of dictionaries)
            for given dates and hours of week.
        :param pd.DataFrame base_classes: DataFrame with three columns: date, hour_of_week,
            base_class. The last column defines class for given dates and hours of week.
        :param int seed: Seed for the random number generator.
        """

        self.models = {}  # type: Dict[str, AveragePositionModule]

        super().__init__(priors, base_classes, seed)

    def get_average_position(self, click_probability, date, how, attr=(0,)):
        """
        Returns average position for the given click probability using an underlying click probability -> average
        position model for the given date and hour of week and attributes.

        :param float click_probability:
        :param str date: Date string in the format yyyy-mm-dd.
        :param int how: Integer value for the hour of week in the range 0-167.
        :param tuple attr:

        :return: Average position from a click probability.
        :rtype: float
        """

        return self.models["{}.{}".format(date, how)].get_average_position(click_probability, attr)

# ==============================================================================
# Unit tests
# ==============================================================================

if __name__ == "__main__":

    import unittest
    from ssa_sim_v2.simulator.modules.average_position.average_position_base_module import AveragePositionHyperbolicModule

    class TestAveragePositionHyperbolicDateHoWModule(unittest.TestCase):
        def test_sanity(self):
            print("AveragePositionDateHoWModule class sample run -------------")

            import math

            date_range = pd.DataFrame(pd.date_range("2017-01-05", "2017-01-08"), columns=["date"])
            date_range["key"] = 1
            hours = pd.DataFrame(np.array(range(24)), columns=["hour_of_day"])
            hours["key"] = 1
            priors = pd.merge(date_range, hours, how="left", on="key")
            priors["hour_of_week"] = priors["date"].dt.dayofweek * 24 + priors["hour_of_day"]
            priors.loc[:, "date"] = priors["date"].dt.strftime("%Y-%m-%d")
            priors = priors[["date", "hour_of_week"]]

            base_classes = priors.copy()

            priors.loc[:, "prior"] = None
            # print(priors)
            base_classes.loc[:, "base_class"] = None

            i = 0

            for index, row in priors.iterrows():
                if math.floor(float(i) / 8) == 0:
                    Params = AveragePositionHyperbolicModule.Params
                    priors.loc[index, "prior"] = [{
                        (0, 0): Params(max_cp=1.0, noise_level=0.1, noise_type="multiplicative"),
                        (0, 1): Params(max_cp=0.5, noise_level=0.1, noise_type="multiplicative"),
                        (1, 0): Params(max_cp=0.3, noise_level=0.1, noise_type="multiplicative"),
                        (1, 1): Params(max_cp=0.1, noise_level=0.1, noise_type="multiplicative"),
                    }]

                    base_classes.loc[index, "base_class"] = AveragePositionHyperbolicModule

                elif math.floor(float(i) / 8) == 1:
                    Params = AveragePositionHyperbolicModule.Params
                    priors.loc[index, "prior"] = [{
                        (0, 0): Params(max_cp=1.0, noise_level=0.1, noise_type="additive"),
                        (0, 1): Params(max_cp=0.5, noise_level=0.1, noise_type="additive"),
                        (1, 0): Params(max_cp=0.3, noise_level=0.1, noise_type="additive"),
                        (1, 1): Params(max_cp=0.1, noise_level=0.1, noise_type="additive"),
                    }]

                    base_classes.loc[index, "base_class"] = AveragePositionHyperbolicModule

                elif math.floor(float(i) / 8) == 2:
                    Params = AveragePositionHyperbolicModule.Params
                    priors.loc[index, "prior"] = [{
                        (0, 0): Params(max_cp=1.0, noise_level=0.5, noise_type="multiplicative"),
                        (0, 1): Params(max_cp=0.5, noise_level=0.5, noise_type="multiplicative"),
                        (1, 0): Params(max_cp=0.3, noise_level=0.5, noise_type="multiplicative"),
                        (1, 1): Params(max_cp=0.1, noise_level=0.5, noise_type="multiplicative"),
                    }]

                    base_classes.loc[index, "base_class"] = AveragePositionHyperbolicModule

                i = (i + 1) % 24

            model = AveragePositionDateHoWModule(priors, base_classes, seed=9)

            attributes = [(0, 0), (0, 1), (1, 0), (1, 1)]
            cps = np.linspace(0.1, 1.0, 8)
            t = 0

            for attr in attributes:
                while t < len(priors):
                    cp = cps[t % 8]
                    avg_pos = model.get_average_position(cp, priors["date"][t], priors["hour_of_week"][t], attr)
                    print("date={} how={} hod={} attr={} cp={} avg_pos={}".format(
                        priors["date"][t], priors["hour_of_week"][t],
                        priors["hour_of_week"][t] % 24, attr, round(cp, 2), avg_pos))
                    t += 1
                t = 0

            self.assertTrue(True)

            print("")

    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(TestAveragePositionHyperbolicDateHoWModule))
    unittest.TextTestRunner().run(suite)


text:
import numpy as np
import cv2

from fast_rcnn.config import cfg
from utils.blob import im_list_to_blob

def get_image_blob(im):
    """Converts an image into a network input.

    Arguments:
        im (ndarray): a color image in BGR order

    Returns:
        blob (ndarray): a data blob holding an image pyramid
        im_scale_factors (list): list of image scales (relative to im) used
            in the image pyramid
    """
    im_orig = im.astype(np.float32, copy=True)
    im_orig -= cfg.PIXEL_MEANS

    im_shape = im_orig.shape
    im_size_min = np.min(im_shape[0:2])
    im_size_max = np.max(im_shape[0:2])

    processed_ims = []
    im_scale_factors = []

    for target_size in cfg.TEST.SCALES:
        im_scale = float(target_size) / float(im_size_min)
        # Prevent the biggest axis from being more than MAX_SIZE
        if np.round(im_scale * im_size_max) > cfg.TEST.MAX_SIZE:
            im_scale = float(cfg.TEST.MAX_SIZE) / float(im_size_max)
        im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
                        interpolation=cv2.INTER_LINEAR)
        im_scale_factors.append(im_scale)
        processed_ims.append(im)

    # Create a blob to hold the input images
    blob = im_list_to_blob(processed_ims)

    return blob, np.array(im_scale_factors)

def get_rois_blob(im_rois, im_scale_factors):
    """Converts RoIs into network inputs.

    Arguments:
        im_rois (ndarray): R x 4 matrix of RoIs in original image coordinates
        im_scale_factors (list): scale factors as returned by _get_image_blob

    Returns:
        blob (ndarray): R x 5 matrix of RoIs in the image pyramid
    """
    rois, levels = _project_im_rois(im_rois, im_scale_factors)
    rois_blob = np.hstack((levels, rois))
    return rois_blob.astype(np.float32, copy=False)

def get_gt_boxes_blob(boxes, clss, pids, im_scale_factors):
    assert boxes.shape[0] == pids.shape[0]
    gt_boxes = boxes * im_scale_factors[0]
    gt_boxes = np.hstack([gt_boxes, clss[:, np.newaxis], pids[:, np.newaxis]])
    return gt_boxes.astype(np.float32, copy=False)

def _project_im_rois(im_rois, scales):
    """Project image RoIs into the image pyramid built by _get_image_blob.

    Arguments:
        im_rois (ndarray): R x 4 matrix of RoIs in original image coordinates
        scales (list): scale factors as returned by _get_image_blob

    Returns:
        rois (ndarray): R x 4 matrix of projected RoI coordinates
        levels (list): image pyramid levels used by each projected RoI
    """
    im_rois = im_rois.astype(np.float, copy=False)

    if len(scales) > 1:
        widths = im_rois[:, 2] - im_rois[:, 0] + 1
        heights = im_rois[:, 3] - im_rois[:, 1] + 1

        areas = widths * heights
        scaled_areas = areas[:, np.newaxis] * (scales[np.newaxis, :] ** 2)
        diff_areas = np.abs(scaled_areas - 224 * 224)
        levels = diff_areas.argmin(axis=1)[:, np.newaxis]
    else:
        levels = np.zeros((im_rois.shape[0], 1), dtype=np.int)

    rois = im_rois * scales[levels]

    return rois, levels


text:
import math
import pandas as pd
import numpy as np
import scipy.stats

# Geodistance between ci and cf (WGS84)
def geodistance(ci , cf , m):
    r = []

    lambda1 = ci[1] * math.pi / 180 #longitude
    phi1 = ci[0] * math.pi / 180 #latitude
    lambda2 = cf[1] * math.pi / 180
    phi2 = cf[0] * math.pi / 180

    L = lambda2 - lambda1

    if (lambda1 == lambda2) & (phi1 == phi2):
        return 0.0

    a = 6378137.0
    f = 1/298.257223563
    b = a * (1 - f)
    axa = a**2
    bxb = b**2

    U1 = math.atan((1 - f) * math.tan(phi1))
    U2 = math.atan((1 - f) * math.tan(phi2))

    lambda_cur = L
    lambda_old = 1 - 1j # There is no way a complex number is going to coincide with a real number!

    ntrials = 0 # Just in case...

    while (abs(lambda_cur - lambda_old) > 1e-9):
        ntrials = ntrials + 1

        lambda_old = lambda_cur
        sin_sigma = math.sqrt((math.cos(U2) * math.sin(lambda_cur))**2 +
                              (math.cos(U1) * math.sin(U2) -
                               math.sin(U1) * math.cos(U2) * math.cos(lambda_cur))**2)
        cos_sigma = math.sin(U1) * math.sin(U2) + math.cos(U1) * math.cos(U2) * math.cos(lambda_cur)
        sigma = math.atan2(sin_sigma, cos_sigma)
        sin_alpha = math.cos(U1) * math.cos(U2) * math.sin(lambda_cur) / sin_sigma
        cos2_alpha = 1 - sin_alpha**2
        cos_2sigmam = cos_sigma - 2 * math.sin(U1) * math.sin(U2) / cos2_alpha

        C = (f/16) * cos2_alpha * (4 + f * (4 - 3 * cos2_alpha))

        lambda_cur = L + (1 - C) * f * sin_alpha * (sigma + C * sin_sigma *
                         (cos_2sigmam + C * cos_sigma * (-1 + 2 * (cos_2sigmam)**2)))

        #Stop the function if convergence is not achieved:
        if ntrials > 1000:
            print('Convergence failure...')
            return

        # Convergence achieved? get the distance:
        uxu = cos2_alpha * (axa - bxb) / bxb;
        A = 1 + (uxu/16384) * (4096 + uxu * (-768 + uxu * (320 - 175 * uxu)))
        B = (uxu/1024) * ( 256 + uxu * (-128 + uxu * (74 - 47 * uxu)))
        delta_sigma = B * sin_sigma * (cos_2sigmam + (B/4) * (cos_sigma * (-1 + 2 * cos_2sigmam**2) - (B/6)*cos_2sigmam * (-3 + 4 * sin_sigma**2) * ( 3 + 4 * cos_2sigmam**2)))
        r = b * A * (sigma - delta_sigma)
    return r

# minimum distance for every point to other points
def min_distance(coors):
    distance_dir = {}
    for i, coor_i in enumerate(coors):
        for j, coor_j in enumerate(coors):
            if i >= j:
                continue
            dis = geodistance(coor_i , coor_j , 6)/1000
            distance_dir.setdefault(i,[]).append(dis)
            distance_dir.setdefault(j,[]).append(dis)
    min_dis_list = [min(distance_dir[i]) for i in distance_dir]
    return min_dis_list

# Average nearest neighbor
def ANN(coors, n, area):
    min_dis_list = min_distance(coors)

    de = 0.5/(np.sqrt(n/area))
    do = np.sum(min_dis_list)/n
    se = 0.26136/(np.sqrt(n*n/area))
    ann = do/de
    z = (do-de)/se
    p = scipy.stats.norm.cdf(z)
    return ann, z, p

def main():
    rows = []
    df = pd.read_csv(input_file, sep=',')
    dates = df['Date'].unique()
    for date in dates:
        df_day = df.loc[df['Date']==date]
        df_crime = df_day.loc[df_day['Crime']==crime_type]
        coors = [[lat,lon] for lat, lon in zip(df_crime['X'],df_crime['Y'])]
        ann, z, p = ANN(coors, len(df_crime), area)
        rows.append([date[:-8], round(ann,3), round(z,3), round(p,3)])

    # output the results of ann
    headers = ['Date','ANN','Z_value','P_value']
    data_dir = {h:v for h,v in zip(headers,zip(*rows))}
    outputdata = pd.DataFrame(data_dir)
    outputdata.to_csv(output_file,sep=',',index=False,header=True)

if __name__ == '__main__':
    input_file = './data/dc-washington.txt'
    output_file = './result/ann-dc-washington.csv'
    area = 177
    crime_type = 'Burglary'

    main()

max_stars_repo_path: 6. ANN/ANN.py

max_stars_repo_name: stccenter/crime-data-analysis

max_issues_repo_path: 6. ANN/ANN.py

max_issues_repo_name: stccenter/crime-data-analysis

max_forks_repo_path: 6. ANN/ANN.py

max_forks_repo_name: stccenter/crime-data-analysis

reason: import numpy,import scipy


text:
from theano import gof
from theano import gradient as G
from function_module import orig_function

class OpFromGraph(gof.Op):
    """
    This create an L{Op} from a list of input variables and a list of output
    variables.

    The signature is the same as the signature of L{FunctionFactory}
    and/or function and the resulting L{Op}'s perform will do the same
    operation as::
      function(inputs, outputs, **kwargs)

    Take note that the following options, if provided, must take the
    value(s) listed below:
      unpack_single = False
      borrow_outputs = False

    OpFromGraph takes an additional input, grad_depth. If grad_depth
    is n, OpFromGraph will make special Ops for gradients up to the
    nth level, allowing the user to differentiate this op up to n
    times. The parameter defaults to 1. If grad_depth == 0, the op
    will not be differentiable.

    Example:
      x, y, z = tensor.scalars('xyz')
      e = x + y * z
      op = OpFromGraph([x, y, z], [e], linker='c')
      # op behaves like a normal theano op
      e2 = op(x, y, z) + op(z, y, x)
      fn = function([x, y, z], [e2])
    """

    def __init__(self, inputs, outputs, grad_depth = 1, **kwargs):
        if not isinstance(outputs, list):
            raise TypeError('outputs must be list', outputs)
        for i in inputs + outputs:
            if not isinstance(i, gof.Variable):
                raise TypeError('inputs and outputs must be Variable instances', i)
        if 'updates' in kwargs:
            raise TypeError('updates are not allowed in kwargs')
        # TODO: the graph may have implicit inputs like Value and SharedVariable instances.
        #       what impact to they have on the validity of this Op?
        self.fn = orig_function(inputs, outputs, **kwargs)
        self.inputs = inputs
        self.outputs = outputs
        self.input_types = [input.type for input in inputs]
        self.output_types = [output.type for output in outputs]

        if grad_depth > 0:
            output_grads = [t() for t in self.output_types]
            gd = G.grad_sources_inputs(zip(self.outputs, output_grads), self.inputs)
            gs = map(gd.get, self.inputs)
            self.grad_ops = []
            for g in gs:
                if g is None:
                    self.grad_ops.append(lambda *args: None)
                else:
                    self.grad_ops.append(OpFromGraph(inputs + output_grads,
                                                     [g],
                                                     grad_depth = grad_depth - 1))
    def __eq__(self, other):
        #TODO: recognize a copy
        return self is other

    def __hash__(self):
        #TODO: use internal variables in hash
        return hash(type(self))

    def make_node(self, *inputs):
        for input, type in zip(inputs, self.input_types):
            if not type == input.type:
                raise TypeError("Wrong type, expected %s but got %s" % (type, input.type))
        return gof.Apply(self,
                         inputs,
                         [type() for type in self.output_types])

    def perform(self, node, inputs, outputs):
        variables = self.fn(*inputs)
        assert len(variables) == len(outputs)
        for output, variable in zip(outputs, variables):
            ##TODO: when function's output-borrowing semantics are correct, we wont need this
            # copy anymore
            output[0] = variable.copy()

    def grad(self, inputs, output_grads):
        if hasattr(self, 'grad_ops'):
            return [go(*(inputs + output_grads)) for go in self.grad_ops]
        else:
            raise NotImplementedError

max_stars_repo_name: arnaudsj/Theano

max_issues_repo_name: arnaudsj/Theano

max_forks_repo_name: arnaudsj/Theano

reason: from theano


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 12 18:44:09 2019

@author: z
"""
import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

birddata = pd.read_csv("bird_tracking.csv")
bird_names = pd.unique(birddata.bird_name)

#路线分析
plt.figure(figsize=(7,7))
for bird_name in bird_names:
    ix = birddata.bird_name == bird_name
    x , y = birddata.longitude[ix],birddata.latitude[ix]
    plt.plot(x,y,".",label = bird_name)
plt.xlabel("Longitude")
plt.ylabel("Latitude")
plt.legend(loc="lower right")

#速度分析
plt.figure(figsize=(7,7))
speed = birddata.speed_2d[birddata.bird_name == "Eric"]
ind = np.isnan(speed)
plt.hist(speed[~ind],bins=np.linspace(0,30,20),density=True)
plt.xlabel("2D speed (m/s)")
plt.ylabel("Frequency")
#birddata.speed_2d.plot(kind="hist",range=(0,30))

#时间戳
timestamps = []
for time in birddata.date_time:
    timestamps.append(datetime.datetime.strptime(time[0:-3],"%Y-%m-%d %H:%M:%S"))
birddata["timestamp"] = pd.Series(timestamps,index=birddata.index)
times = birddata.timestamp[birddata.bird_name == "Eric"]
elapsed_time = [time - times[0] for time in times]
elapsed_days = np.array(elapsed_time)/datetime.timedelta(days=1)

#速度图
data = birddata[birddata.bird_name == "Eric"]
next_day = 1
inds = []
daily_mean_speed = []
for (i,t) in enumerate(elapsed_days):
    if t < next_day:
        inds.append(i)
    else:
        #计算平均速度
        daily_mean_speed.append(np.mean(data.speed_2d[inds]))
        next_day +=1
        inds = []
plt.figure(figsize=(8,6))
plt.plot(daily_mean_speed)
plt.xlabel("Day")
plt.ylabel("Mean Speed (m/s)");

#Cartopy地图模块使用
import cartopy.crs as ccrs
import cartopy.feature as cfeature

proj = ccrs.Mercator()

plt.figure(figsize=(10,10))
ax = plt.axes(projection=proj)
ax.set_extent((-25.0,20.0,52.0,10.0))
ax.add_feature(cfeature.LAND)
ax.add_feature(cfeature.OCEAN)
ax.add_feature(cfeature.COASTLINE)
ax.add_feature(cfeature.BORDERS, linestyle=':')

for name in bird_names:
	ix = birddata['bird_name'] == name
	x , y = birddata.longitude[ix],birddata.latitude[ix]
	ax.plot(x,y,'.',transform=ccrs.Geodetic(),label=name)
plt.legend(loc="upper left")
plt.show();

max_stars_repo_name: linghtiin/test

max_issues_repo_name: linghtiin/test

max_forks_repo_name: linghtiin/test


text:
import numpy as np
from typing import List, Callable

from lightRaven.sampling import SamplingBase
from lightRaven.policy import FAPolicy, TabularPolicy, PolicyBase

class CEMSeldonian:
    """
    The Seldonian version of the Cross Entropy Method learning algorithm.
    """
    def __init__(self, epochs: int, pop_size: int, elite_ratio: float,
                 ci_ub: Callable, ref_size: int, g_funcs: List[Callable], c=1, delta=0.05,
                 gamma=0.9, extra_std=2.0, extra_decay_time=10, n_proc=8):
        """
        Parameters
        ----------
        epochs : int
            The total number of iterations that the algorithm will try to train and improve on its current policies.
        pop_size : int
            The total number of candidate policies (population) that will be generated at each epoch.
        elite_ratio : float
            The percentage of candidates that we will keep (elites) to improve on the next policy generation.
            The algorithm will keep the top `elite_ratio` percent of the population to update its mean and variance
                parameter.
        ci_ub : Callable
            A function calculates the concentration upper bounds for safety test.
        ref_size : int
            The size of the dataset in the safety test.
            Used in the concentration bound calculation to prevent producing a bound that is overly conservative
        g_funcs : List[Callable]
            A :obj:`list` of user-defined constraint functions for safety test.
        c : int
            A hyperparameter used to scale the concentration interval generated by `ci_ub`.
            For student t, use `2` in candidate selection; Otherwise use `1`.
        delta : float
            The significance level for the safety test in order to get a high confidence performance lower bound
            of a candidate policy
        gamma : float
            The discount factor. Mostly predefined by the MDP.
        extra_std : float
            A hyperparameter which adds extra variance to the Covariance matrix
        extra_decay_time : float
            A hyperparameter which controls the scale of the extra variance adds to the Covariance matrix
        n_proc : int
            The total number of processes that can be spawned for parallellization
       """
        self.epochs = epochs
        self.pop_size = pop_size
        self.elite_ratio = elite_ratio
        assert 0 < elite_ratio < 1
        self.elite_size = int(self.pop_size * self.elite_ratio)
        self.extra_std = extra_std
        self.extra_decay_time = extra_decay_time
        self.n_proc = n_proc
        self.gamma = gamma

        self.p_type = None
        self.sampler = None
        self.obs_shape = None
        self.act_shape = None

        self.theta_dim = None
        self.means = None
        self.stds = None
        self.elites = None

        self.sampler = None
        self.best_thetas = np.array([])
        self.ci_ub = ci_ub
        self.delta = delta
        self.ref_size = ref_size
        self.c = c
        self.g_funcs = g_funcs
        self.penalty = 1e5

    @staticmethod
    def get_best_idx(rewards: np.ndarray) -> np.ndarray:
        """ Compute any indices of the candidate policies with non-zero performance after safety test

        Parameters
        ----------
        rewards : np.ndarray
            A vector of estimated performance for each candidate policy
        """
        return np.where(rewards > 0)

    def load_setup(self, obs_shape: int, act_shape: int, p_type: str) -> None:
        """ Load a supported Gym environment into the algorithm

        Parameters
        ----------
        obs_shape : int
            Observation space shape
        act_shape : int
            Action space shape
        p_type : str
            Indicating policy type from 'fa' (functional approximation), 'tb' (tabular method)
        """
        self.obs_shape, self.act_shape = obs_shape, act_shape
        if p_type not in ['fa', 'tb']:
            ValueError(f'policy type {p_type} is not supported!')
        self.p_type = p_type

    def load_sampler(self, sampler: SamplingBase) -> None:
        """ Load the performance estimator for candidate evaluation

        Parameters
        ----------
        sampler : SamplingBase
            An estimator used for candidate evaluation
        """
        self.sampler = sampler

    def init_params(self, theta=None) -> None:
        """ Define the policy shape. Initialize the mean and std vector for policy generation of CEM.
        """
        assert all(isinstance(val, int) for val in [self.obs_shape, self.act_shape])
        self.theta_dim = self.obs_shape * self.act_shape
        if theta is not None:
            self.means = theta
        else:
            self.means = np.random.uniform(size=self.theta_dim)
        self.stds = np.ones(self.theta_dim)

    def update_params(self, elites: np.ndarray) -> None:
        """ Use a predefined policy matrix to initialize the mean and std vector of CEM.

        Parameters
        ----------
        elites : np.ndarray
            A predefined policy that is previously trained by CEM
        """
        self.means = np.mean(elites, axis=0)
        self.stds = np.std(elites, axis=0)

    def candidate_eval(self, theta: np.ndarray) -> float:
        """ Run the given candidate policy over the environment and returns its average performance.

        Parameters
        ----------
        theta : np.ndarray
            A candidate policy generated by CEM
        """
        assert isinstance(self.sampler, SamplingBase)

        policy = None
        if self.p_type == 'fa':
            policy = FAPolicy(self.obs_shape, self.act_shape, theta)
        elif self.p_type == 'tb':
            policy = TabularPolicy(self.obs_shape, self.act_shape, theta)

        assert isinstance(policy, PolicyBase)
        self.sampler.load_eval_policy(policy)

        rewards = self.sampler.get_est()
        perf_est = rewards.mean()

        for g in self.g_funcs:
            if self.ci_ub(g(rewards), ref_size=self.ref_size, c=self.c, delta=self.delta) > 0:
                perf_est -= self.penalty
                break
        return perf_est

    def get_elite_idx(self, rewards: np.ndarray) -> np.ndarray:
        """ Compute the indices of the candidate policies by their estimated performance (descending order)

        Parameters
        ----------
        rewards : np.ndarray
            A vector of estimated performance for each candidate policy
        """
        return np.argsort(rewards)[::-1][:self.elite_size]

    def train(self):
        """ Iterate over all candidate policies and update parameters using elite policies.
        """
        # Check if parameters are set
        assert all(isinstance(val, np.ndarray) for val in [self.means, self.stds])
        for epoch in range(self.epochs):

            extra_cov = max(1.0 - epoch / self.extra_decay_time, 0) * self.extra_std**2

            candidates = np.random.multivariate_normal(
                mean=self.means,
                cov=np.diag(np.array(self.stds**2) + extra_cov),
                size=self.pop_size
            )

            g_candidates = list(map(self.candidate_eval, candidates))
            g_candidates = np.array(g_candidates).reshape(-1)

            elite_mask = self.get_elite_idx(g_candidates)
            best_mask = self.get_best_idx(g_candidates)
            self.elites = candidates[elite_mask]
            self.best_thetas = candidates[best_mask]

            self.update_params(self.elites)

    def get_best_candidates(self):
        """ Return the last generated elite policies by Seldonian CEM
        """
        if self.best_thetas.shape[0] == 0:
            return self.means
        else:
            return self.best_thetas

    def get_best_rewards(self, n: int):
        """ Return the performance estimation of top-n elite policies.

        Parameters
        ----------
        n : int
            The top-n policies that will be evaluated and monitored.
        """
        if self.best_thetas.shape[0] == 0:
            return self.means.reshape(1, -1)
        if len(self.best_thetas.shape) == 1:
            return self.best_thetas.reshape(1, -1)
        else:
            return [self.candidate_eval(theta) for theta in self.best_thetas[:n]]

# Add any constraint functions here
def g0(d: np.ndarray) -> np.ndarray:
    return 20 - d

def g1(d: np.ndarray) -> np.ndarray:
    return 25 - d

def g2(d: np.ndarray) -> np.ndarray:
    return 30 - d


text:
# Needed to convert a folder of images into a hd5 file

import os
import cv2
import h5py
import glob
import parmap
import argparse
import numpy as np
from tqdm import tqdm as tqdm
import matplotlib.pylab as plt

def format_image(img_path, size):
    """
    Load img with opencv and reshape
    """

    img_color = cv2.imread(img_path)
    img_color = img_color[:, :, ::-1]

    # Following line: Slice image to center around face (only needed when just the center square is wanted)
    # img_color = img_color[30:-30, 20:-20, :]

    img_color = cv2.resize(img_color, (size, size), interpolation=cv2.INTER_AREA)

    img_color = img_color.reshape((1, size, size, 3)).transpose(0, 3, 1, 2)

    return img_color

def build_HDF5(jpeg_dir):
    """
    Gather the data in a single HDF5 file.
    """

    # Put train data in HDF5
    hdf5_file = os.path.join(output_dir, NAME)
    with h5py.File(hdf5_file, "w") as hfw:

            list_img = glob.glob(os.path.join(jpeg_dir, "*.jpg"))
            list_img = np.array(list_img)

            data_color = hfw.create_dataset("data",
                                            (0, 3, SIZE, SIZE),
                                            maxshape=(None, 3, SIZE, SIZE),
                                            dtype=np.uint8)

            num_files = len(list_img)
            chunk_size = TO BE SPECIFIED #set to num_files if dataset is small
            num_chunks = num_files / chunk_size
            arr_chunks = np.array_split(np.arange(num_files), num_chunks)

            for chunk_idx in tqdm(arr_chunks):

                list_img_path = list_img[chunk_idx].tolist()
                output = parmap.map(format_image, list_img_path, SIZE, parallel=True)

                arr_img_color = np.concatenate(output, axis=0)

                # Resize HDF5 dataset
                data_color.resize(data_color.shape[0] + arr_img_color.shape[0], axis=0)

                data_color[-arr_img_color.shape[0]:] = arr_img_color.astype(np.uint8)

def check_HDF5():
    """
    Plot images with landmarks to check the processing
    """

    # Get hdf5 file
    hdf5_file = os.path.join(output_dir, NAME)

    with h5py.File(hdf5_file, "r") as hf:
        data_color = hf["data"]
        for i in range(data_color.shape[0]):
            plt.figure()
            img = data_color[i, :, :, :].transpose(1,2,0)
            plt.imshow(img)
            plt.show()
            plt.clf()
            plt.close()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('jpeg_dir', type=str) #path to images
    parser.add_argument('--do_plot', default=True, type=bool)
    args = parser.parse_args()

    SIZE = 32
    NAME = "NAME.h5"
    output_dir = "WHERE TO PUT THE h5-FILE"

    build_HDF5(args.jpeg_dir)

    if args.do_plot:
        check_HDF5()

max_stars_repo_name: developerator/urban-tribble

max_issues_repo_name: developerator/urban-tribble

max_forks_repo_name: developerator/urban-tribble


text:
from __future__ import print_function, absolute_import
import inspect
import re
from collections import Mapping, defaultdict
import textwrap
from contextlib import closing
from numba.io_support import StringIO
from numba import ir

class SourceLines(Mapping):
    def __init__(self, func):

        try:
            lines, startno = inspect.getsourcelines(func)
        except IOError:
            self.lines = ()
            self.startno = 0
        else:
            self.lines = textwrap.dedent(''.join(lines)).splitlines()
            self.startno = startno

    def __getitem__(self, lineno):
        try:
            return self.lines[lineno - self.startno].rstrip()
        except IndexError:
            return ''

    def __iter__(self):
        return iter((self.startno + i) for i in range(len(self.lines)))

    def __len__(self):
        return len(self.lines)

    @property
    def avail(self):
        return bool(self.lines)

class TypeAnnotation(object):
    def __init__(self, interp, typemap, calltypes, lifted):
        self.filename = interp.bytecode.filename
        self.func = interp.bytecode.func
        self.blocks = interp.blocks
        self.typemap = typemap
        self.calltypes = calltypes
        self.lifted = lifted

    def annotate(self):
        source = SourceLines(self.func)
        # if not source.avail:
        #     return "Source code unavailable"

        # Prepare annotations
        groupedinst = defaultdict(list)
        for blkid, blk in self.blocks.items():
            groupedinst[blk.loc.line].append("label %s" % blkid)
            for inst in blk.body:
                lineno = inst.loc.line

                if isinstance(inst, ir.Assign):
                    if (isinstance(inst.value, ir.Expr) and
                            inst.value.op ==  'call'):
                        atype = self.calltypes[inst.value]
                    else:
                        atype = self.typemap[inst.target.name]

                    aline = "%s = %s  :: %s" % (inst.target, inst.value, atype)
                elif isinstance(inst, ir.SetItem):
                    atype = self.calltypes[inst]
                    aline = "%s  :: %s" % (inst, atype)
                else:
                    aline = "%s" % inst
                groupedinst[lineno].append("  %s" % aline)

        # Format annotations
        io = StringIO()
        with closing(io):
            if source.avail:
                print("# File: %s" % self.filename, file=io)
                for num in source:
                    srcline = source[num]
                    ind = _getindent(srcline)
                    print("%s# --- LINE %d --- " % (ind, num), file=io)
                    for inst in groupedinst[num]:
                        print('%s# %s' % (ind, inst), file=io)
                    print(file=io)
                    print(srcline, file=io)
                    print(file=io)
                if self.lifted:
                    print("# The function contains lifted loops", file=io)
                    for loop in self.lifted:
                        print("# Loop at line %d" % loop.bytecode.firstlineno,
                              file=io)
                        print("# Has %d overloads" % len(loop.overloads),
                              file=io)
                        for cres in loop._compileinfos.values():
                            print(cres.type_annotation, file=io)
            else:
                print("# Source code unavailable", file=io)
                for num in groupedinst:
                    for inst in groupedinst[num]:
                        print('%s' % (inst,), file=io)
                    print(file=io)

            return io.getvalue()

    def __str__(self):
        return self.annotate()

re_longest_white_prefix = re.compile('^\s*')

def _getindent(text):
    m = re_longest_white_prefix.match(text)
    if not m:
        return ''
    else:
        return ' ' * len(m.group(0))

max_stars_repo_name: meawoppl/numba

max_issues_repo_name: meawoppl/numba

max_forks_repo_name: meawoppl/numba

reason: from numba


text:
#!/usr/bin/env python
# coding: utf-8

# # Customer Churn Modeling
# </imag>
#
# ##### Can you predict if bank customers will turnover next cycle?
#
# Churn prevention allows companies to develop loyalty programs and retention campaigns to keep as many customers as possible.

# # Machine Learning Process Step by Step
# </imag>
#
#

# ### Importing Libraries
# - numpy
# - matplotlib
# - seaborn
# - scikit-learn
# - imblearn
#

# In[ ]:

# Importing Librarys
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV # to split the data
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, fbeta_score, roc_curve #To evaluate our model
from sklearn.metrics import plot_confusion_matrix
from sklearn.externals import joblib
from scipy import stats
import seaborn as sns
import matplotlib.pyplot as plt

from pylab import rcParams

import warnings
warnings.filterwarnings("ignore")

# ### Loading Dataset
# https://www.kaggle.com/barelydedicated/bank-customer-churn-modeling
#
# __Given below are columns information__
# 1. RowNumber
# 2. CustomerId
# 3. Surname
# 4. CreditScore
# 5. Geography
# 6. Gender
# 7. Age
# 8. Tenure
# 9. Balance
# 10. NumOfProductsHow many accounts, bank account affiliated products the person has
# 11. HasCrCard
# 12. IsActiveMemberSubjective, but for the concept
# 13. EstimatedSalary
# 14. Exited Did they leave the bank after all?

# In[ ]:

# Using pandas
# Data downloaded from above link and stored in local folder
dataframe = pd.read_csv("../../../input/barelydedicated_bank-customer-churn-modeling/Churn_Modelling.csv")

# ## Exploratory Data Analysis
# - __Summarizing, Describing and Data Distributions__
# - __Univariate and bivariate Analysis__
# - __Outliers and their influence__
# - __Metadata errors__
# - __Missing Data__
# - __Correlation analysis between variables__
#

# In[ ]:

dataframe.head(5)

# In[ ]:

# Find out the total number of rows and columns
dataframe.shape

# In[ ]:

#columns and their corresponding data types,along with finding whether they contain null values or not
dataframe.info()

# In[ ]:

# The describe() function in pandas is very handy in getting various summary statistics.
# This function returns the count, mean, standard deviation, minimum and maximum values and the quantiles of the data
dataframe.describe()

# In[ ]:

# Checking data sampling
dataframe.Exited.unique()
dataframe.Exited.value_counts()

# In[ ]:

# Data Visualization
import seaborn as sns
sns.countplot(dataframe['Exited'],label="Count")
print()

# In[ ]:

# Data to plot
male_dataframe = dataframe[dataframe.Gender=="Male"]
sizes = male_dataframe['Exited'].value_counts(sort = True)
colors = ["Green","Red"]
rcParams['figure.figsize'] = 5,5
# Plot
plt.pie(sizes,  colors=colors, autopct='%1.1f%%', shadow=True, startangle=270,)
plt.title('Percentage of Churn in Dataset for Male')
print()

# In[ ]:

# Data to plot
female_dataframe = dataframe[dataframe.Gender=="Female"]
sizes = female_dataframe['Exited'].value_counts(sort = True)
colors = ["Green","Red"]
rcParams['figure.figsize'] = 5,5
# Plot
plt.pie(sizes,  colors=colors, autopct='%1.1f%%', shadow=True, startangle=270,)
plt.title('Percentage of Churn in Dataset for Female')
print()

# ### Correlation Analysis

# In[ ]:

correlations = dataframe.corr()
# plot correlation matrix
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(correlations, vmin=-1, vmax=1)
print()
ticks = np.arange(0,9,1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_xticklabels(dataframe.columns)
ax.set_yticklabels(dataframe.columns)
rcParams['figure.figsize'] = 40,15
print()

# ### Finding Outliers

# In[ ]:

rcParams['figure.figsize'] = 10,10
boxplot = dataframe.boxplot(column=['EstimatedSalary', 'Balance'])

# In[ ]:

rcParams['figure.figsize'] = 10,8
sns.boxplot(x="Geography",y="Exited",data=dataframe,palette='rainbow')

# ## Data Preprocessing
# - __Imputing Missing Data__
# - __Handling Unbalanced Data(Under Sampling and OverSampling)__
# - __Handling Outliers__
# - __Transforming, Encoding, Scaling, and Shuffling__

# ### Imputing Missing Data

# In[ ]:

# removing null values to avoid errors
dataframe.dropna(inplace = True)

# ## Feature Engineering
# - __Adding or dropping features__
# - __Combining multiple features into one feature__
# - __Binning__
# - __One Hot Encoding__

# In[ ]:

#remove the fields from the data set that we don't
# want to include in our model
del dataframe['RowNumber']
del dataframe['CustomerId']
del dataframe['Surname']

# In[ ]:

from sklearn.preprocessing import LabelEncoder

# In[ ]:

label1 = LabelEncoder()
dataframe['Geography'] = label1.fit_transform(dataframe['Geography'])

# In[ ]:

label2 = LabelEncoder()
dataframe['Gender'] = label2.fit_transform(dataframe['Gender'])

# In[ ]:

features_dataframe = pd.get_dummies(dataframe, columns=['Geography'])

# In[ ]:

#Remove the Exited from the feature data
del features_dataframe['Exited']

# In[ ]:

X = features_dataframe.values
y = dataframe['Exited']
features_dataframe.columns

# ### Handling Imbalanced Data

# In[ ]:

# apply near miss
import imblearn
from imblearn.under_sampling import NearMiss
nr = NearMiss()
X, y = nr.fit_sample(X, y)
X[100]

# In[ ]:

y[100]

# In[ ]:

#Split the data set in a traning set (80%) and a test set(20%)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0, stratify = y)

# ## Modelling

# ### Training and Model Selection

# In[ ]:

# Algorithmns models to be compared
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.ensemble import GradientBoostingClassifier
from xgboost import XGBClassifier

# In[ ]:

# ### Choosed GradientBoostingClassifier Algorithm and List Metrics

# In[ ]:

# From above models we can see,
# We are getting highest accuracy and better values of precision and recall for XGBClassifier
model = GradientBoostingClassifier()
model.fit(X_train, y_train)
print('Accuracy = ',accuracy_score(y_test, model.predict(X_test)))
print('classification_report = ',classification_report(y_test, model.predict(X_test)))

# ### Hyper Parameter Tuning

# In[ ]:

# ### Train model based on best hyper parameter

# In[ ]:

# ## Model Evaluation

# #### Accuracy and Classification_report

# In[ ]:

print('Accuracy = ',accuracy_score(y_test, model.predict(X_test)))
print('classification_report = ',classification_report(y_test, model.predict(X_test)))

# ### Plot Confusion Matrix

# In[ ]:

# Plot non-normalized confusion matrix
titles_options = [("Confusion matrix, without normalization", None), ("Normalized confusion matrix", 'true')]
for title, normalize in titles_options:
    disp = plot_confusion_matrix(model, X_test, y_test, display_labels=['Exited','Not Exited'], cmap=plt.cm.Blues, normalize=normalize)
    disp.ax_.set_title(title)

    print(title)
    print(disp.confusion_matrix)

print()

# ### ROC Curve

# In[ ]:

y_pred_prob = model.predict_proba(X_test)[:,1]
fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic')
print()

# In[ ]:

# ### Save Model for Production Deployment

# In[ ]:

# Save the trained model to a file so we can use it in other programs
joblib.dump(model,"customer_churn_mlmodel.pkl")

# ### List Important Features

# In[ ]:

# These are the features labels from out data set
feature_labels = np.array(['CreditScore', 'Gender', 'Age', 'Tenure', 'Balance', 'NumOfProducts', 'HasCrCard', 'IsActiveMember', 'EstimatedSalary', 'Geography_0','Geography_1', 'Geography_2'])

# In[ ]:

# Create a numpy array based on the model's feature importances
importance = model.feature_importances_

# In[ ]:

# Sort the feature labels based on the feature importance rankings from the model
feature_indexes_by_importance = importance.argsort()

# In[ ]:

# Print each feature label, from most important to least important (reverse order)
for index in feature_indexes_by_importance:
    print("{} - {:.2f}%".format(feature_labels[index], (importance[index] * 100.0)))

# In[ ]:

max_stars_repo_name: Chenguang-Zhu/relancer

max_issues_repo_name: Chenguang-Zhu/relancer

max_forks_repo_name: Chenguang-Zhu/relancer

reason: import numpy,from scipy


text:
"""
Script to launch non deep methods with UMAP reduction dimension algorithm
See parse_arguments() function for details on arguments

Author:
Baptiste Lafabregue 2019.25.04
"""
import os
import json
import numpy as np
import argparse
from sklearn import metrics
from sklearn.cluster import KMeans

import tensorflow as tf
import umap

import utils

def parse_arguments():
    parser = argparse.ArgumentParser(
        description='Classification tests for UCR repository datasets'
    )
    parser.add_argument('--dataset', type=str, metavar='d', required=True,
                        help='dataset name')
    parser.add_argument('--archives', type=str, metavar='DIR', required=True,
                        help='archive name')
    parser.add_argument('--itr', type=str, metavar='X', default='0',
                        help='iteration index')
    parser.add_argument('--seeds_itr', type=int, metavar='X', default=None,
                        help='seeds index, do not specify or -1 if none')
    parser.add_argument('--root_dir', type=str, metavar='PATH', default='.',
                        help='path of the root dir where archives and results are stored')
    parser.add_argument('--not_use_previous', default=False, action="store_true",
                        help='Flag to not use previous results and only computes one with errors, '
                             'but computes everything again, only for All option')

    return parser.parse_args()

if __name__ == '__main__':
    args = parse_arguments()

    tf.keras.backend.set_floatx('float64')

    root_dir = args.root_dir
    dataset_name = args.dataset
    archive_name = args.archives
    use_previous = not args.not_use_previous
    itr = args.itr
    seeds_itr = args.seeds_itr
    seeds = None
    if seeds_itr is not None and seeds_itr >= 0:
        seeds = utils.read_seeds(root_dir, archive_name, dataset_name, seeds_itr)

    train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, True)
    x_train = train_dict[dataset_name][0]
    y_train = train_dict[dataset_name][1]
    input_shape = x_train.shape[1:]
    nb_classes = np.shape(np.unique(y_train, return_counts=True)[1])[0]

    train_dict = utils.read_dataset(root_dir, archive_name, dataset_name, False)
    x_test = train_dict[dataset_name][0]
    y_test = train_dict[dataset_name][1]

    x_train = np.reshape(x_train, (x_train.shape[0], -1))
    x_test = np.reshape(x_test, (x_test.shape[0], -1))

    UMAP_MIN_DIST = 0
    UMAP_NEIGHBORS = 10

    md = float(UMAP_MIN_DIST)
    x_train = umap.UMAP(
        random_state=0,
        metric='euclidean',
        n_components=nb_classes,
        n_neighbors=UMAP_NEIGHBORS,
        min_dist=md).fit_transform(x_train)
    x_test = umap.UMAP(
        random_state=0,
        metric='euclidean',
        n_components=nb_classes,
        n_neighbors=UMAP_NEIGHBORS,
        min_dist=md).fit_transform(x_test)

    stats_dir = root_dir + '/stats/' + str(itr) + '/' + str(seeds_itr) + '/'

    print('Launch euclidien kmeans on UMAP features : ' + dataset_name)
    y_pred = KMeans(n_clusters=nb_classes, random_state=0).fit_predict(x_train)
    y_pred_test = KMeans(n_clusters=nb_classes, random_state=0).fit_predict(x_test)

    acc = np.round(utils.cluster_acc(y_train, y_pred), 5)
    nmi = np.round(metrics.normalized_mutual_info_score(y_train, y_pred), 5)
    ari = np.round(metrics.adjusted_rand_score(y_train, y_pred), 5)
    summarized = str(acc)+","+str(0)+","+str(nmi)+","+str(0)+","+str(ari)+","+str(0)
    print('acc : ' + str(acc))
    print('nmi : ' + str(nmi))
    print('ari : ' + str(ari))

    acc_test = np.round(utils.cluster_acc(y_test, y_pred_test), 5)
    nmi_test = np.round(metrics.normalized_mutual_info_score(y_test, y_pred_test), 5)
    ari_test = np.round(metrics.adjusted_rand_score(y_test, y_pred_test), 5)
    summarized_test = str(acc_test)+","+str(0)+","+str(nmi_test)+","+str(0)+","+str(ari_test)+","+str(0)

    utils.create_directory(stats_dir)
    with open(stats_dir + "kmeans_UMAP_None_0.0_" + dataset_name, "w") as f:
        f.write(summarized + '\n')
        f.write(summarized_test + '\n')
        f.write('0\n')

max_stars_repo_name: blafabregue/TimeSeriesDeepClustering

max_issues_repo_name: blafabregue/TimeSeriesDeepClustering

max_forks_repo_name: blafabregue/TimeSeriesDeepClustering


text:
#!/usr/bin/env python
# coding: utf-8
# Author: Vladimir M. Zaytsev <zaytsev@usc.edu>

try:
    import numpy as np
except:
    import numpypy as np

XDTYPE = np.float64
YDTYPE = np.int16


text:
import os
import re
from libsbml import *
from peitho.errors_and_parsers.abc_sysbio.abcsysbio.relations import *
from peitho.errors_and_parsers.abc_sysbio.abcsysbio_parser.Writer import Writer

class ODEPythonWriter(Writer):
    def __init__(self, sbmlFileName, modelName="", inputPath="", outputPath=""):
        Writer.__init__(self, sbmlFileName, modelName, inputPath, outputPath)
        self.out_file = open(os.path.join(outputPath,self.parsedModel.name+".py"),"w")

    def write(self):
        p=re.compile('\s')
        #Import the necessaries
        self.out_file.write("from math import *\nfrom numpy import *\nfrom abcsysbio.relations import *\n\n")
        #The user-defined functions used in the model must be written in the file

        for i in range(0, len(self.parsedModel.listOfFunctions)):
            self.out_file.write("def ")
            self.out_file.write(self.parsedModel.listOfFunctions[i].getId())
            self.out_file.write("(")
            for j in range(0, self.parsedModel.listOfFunctions[i].getNumArguments()):
                self.out_file.write(self.parsedModel.functionArgument[i][j])
                self.out_file.write(",")
            self.out_file.write("):\n\n\toutput=")
            self.out_file.write(self.parsedModel.functionBody[i])
            self.out_file.write("\n\n\treturn output\n\n")

        #Write the modelfunction

        self.out_file.write("def modelfunction((")

        for i in range(0, len(self.parsedModel.species)):
            ##if (self.parsedModel.species[i].getConstant() == False):
            self.out_file.write(self.parsedModel.speciesId[i])
            self.out_file.write(",")
        for i in range(0, len(self.parsedModel.listOfParameter)):
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        self.out_file.write(self.parsedModel.parameterId[i])
                        self.out_file.write(",")

        self.out_file.write("),time,parameter=(")
        for i in range(0,len(self.parsedModel.parameterId)):
            dontPrint = False
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        dontPrint=True
            if dontPrint == False:
                self.out_file.write(repr(self.parsedModel.parameter[i]))
                self.out_file.write(",")

        ##for i in range(0, self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == True):
            ##self.out_file.write(repr(self.parsedModel.initValues[i]))
            ##self.out_file.write(",")
        self.out_file.write(")):\n\n")

        counter=0
        for i in range(0,len(self.parsedModel.parameterId)):
            dontPrint = False
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]): dontPrint=True
            if dontPrint == False:
                self.out_file.write("\t"+self.parsedModel.parameterId[i]+"=parameter["+repr(counter)+"]\n")
                counter=counter+1

        ##for i in range(0, self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == True):
            ##self.out_file.write("\t"+self.parsedModel.speciesId[i]+"=parameter["+repr(counter)+"]\n")
            ##counter = counter+1

        self.out_file.write("\n")

        #write the derivatives

        for i in range(0,self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == False):
            self.out_file.write("\td_"+self.parsedModel.speciesId[i]+"=")
            if (self.parsedModel.species[i].isSetCompartment() == True):
                self.out_file.write("(")
            for k in range(0,self.parsedModel.numReactions):
                if(not self.parsedModel.stoichiometricMatrix[i][k]==0.0):
                    self.out_file.write("(")
                    self.out_file.write(repr(self.parsedModel.stoichiometricMatrix[i][k]))
                    self.out_file.write(")*(")
                    string=p.sub('',self.parsedModel.kineticLaw[k])
                    self.out_file.write(string)
                    self.out_file.write(")+")
            self.out_file.write("0")
            if (self.parsedModel.species[i].isSetCompartment() == True):
                self.out_file.write(")/")
                mySpeciesCompartment = self.parsedModel.species[i].getCompartment()
                for j in range(0, len(self.parsedModel.listOfParameter)):
                    if (self.parsedModel.listOfParameter[j].getId() == mySpeciesCompartment):
                        self.out_file.write(self.parsedModel.parameterId[j])
                        break
            self.out_file.write("\n")

        for i in range(0,len(self.parsedModel.listOfRules)):
            if self.parsedModel.listOfRules[i].isRate() == True:
                self.out_file.write("\td_")
                self.out_file.write(self.parsedModel.ruleVariable[i])
                self.out_file.write("=")
                self.out_file.write(self.parsedModel.ruleFormula[i])
                self.out_file.write("\n")

        self.out_file.write("\n\treturn(")

        for i in range(0, len(self.parsedModel.species)):
            ##if (self.parsedModel.species[i].getConstant() == False):
            self.out_file.write("d_"+self.parsedModel.speciesId[i])
            self.out_file.write(",")
        for i in range(0, len(self.parsedModel.listOfParameter)):
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        self.out_file.write("d_"+self.parsedModel.parameterId[i])
                        self.out_file.write(",")

        self.out_file.write(")\n")

        #Write the rules
        self.out_file.write("\ndef rules((")

        for i in range(0, len(self.parsedModel.species)):
            ##if (self.parsedModel.species[i].getConstant() == False):
            self.out_file.write(self.parsedModel.speciesId[i])
            self.out_file.write(",")
        for i in range(0, len(self.parsedModel.listOfParameter)):
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        self.out_file.write(self.parsedModel.parameterId[i])
                        self.out_file.write(",")
        self.out_file.write("),(")
        for i in range(0,len(self.parsedModel.parameterId)):
            dontPrint = False
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        dontPrint=True
            if dontPrint == False:
                self.out_file.write(self.parsedModel.parameterId[i])
                self.out_file.write(",")

        ##for i in range(0, self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == True):
            ##self.out_file.write(self.parsedModel.speciesId[i])
            ##self.out_file.write(",")

        self.out_file.write("),time):\n\n")

        #Write the events

        for i in range(0, len(self.parsedModel.listOfEvents)):
            self.out_file.write("\tif ")
            self.out_file.write(mathMLConditionParser(self.parsedModel.eventCondition[i]))
            self.out_file.write(":\n")
            listOfAssignmentRules = self.parsedModel.listOfEvents[i].getListOfEventAssignments()
            for j in range(0, len(listOfAssignmentRules)):
                self.out_file.write("\t\t")
                self.out_file.write(self.parsedModel.eventVariable[i][j])
                self.out_file.write("=")
                self.out_file.write(self.parsedModel.eventFormula[i][j])
                self.out_file.write("\n")
            self.out_file.write("\n")

        self.out_file.write("\n")

        #write the rules

        for i in range(0, len(self.parsedModel.listOfRules)):
            if self.parsedModel.listOfRules[i].isAssignment():
                self.out_file.write("\t")
                self.out_file.write(self.parsedModel.ruleVariable[i])
                self.out_file.write("=")
                self.out_file.write(mathMLConditionParser(self.parsedModel.ruleFormula[i]))
                self.out_file.write("\n")

        self.out_file.write("\n\treturn((")
        for i in range(0, self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == False):
            self.out_file.write(self.parsedModel.speciesId[i])
            self.out_file.write(",")
        for i in range(0, len(self.parsedModel.listOfParameter)):
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        self.out_file.write(self.parsedModel.parameterId[i])
                        self.out_file.write(",")
        self.out_file.write("),(")

        for i in range(0,len(self.parsedModel.parameterId)):
            dontPrint = False
            if (self.parsedModel.listOfParameter[i].getConstant() == False):
                for k in range(0, len(self.parsedModel.listOfRules)):
                    if (self.parsedModel.listOfRules[k].isRate() and self.parsedModel.ruleVariable[k] == self.parsedModel.parameterId[i]):
                        dontPrint=True
            if dontPrint == False:
                self.out_file.write(self.parsedModel.parameterId[i])
                self.out_file.write(",")

        ##for i in range(0, self.parsedModel.numSpecies):
            ##if (self.parsedModel.species[i].getConstant() == True):
            ##self.out_file.write(self.parsedModel.speciesId[i])
            ##self.out_file.write(",")
        self.out_file.write("))\n\n")
        self.out_file.close()

max_stars_repo_name: MichaelPHStumpf/Peitho

max_issues_repo_name: MichaelPHStumpf/Peitho

max_forks_repo_name: MichaelPHStumpf/Peitho

reason: from numpy


text:
import numpy as np
from collections import namedtuple
from .hmc_utils.base_hmc import BaseTHMC
from .hmc_utils.stats import TNStepStats
from .sample_trace import TNTrace
from .nuts import Tree, NUTS

__all__ = ['TNUTS', 'TTree']

# A proposal for the next position
TProposal = namedtuple("Proposal", "q, u, weight, energy, logp, p_accept")

class TTree(Tree):

    def _get_proposal(self, point, p_accept):
        return TProposal(point.q, point.u, point.weight, point.energy,
                         point.logp, p_accept)

    def stats(self):
        return {
            'u': self.proposal.u,
            'weight': self.proposal.weight,
            'logp': self.proposal.logp,
            'energy': self.proposal.energy,
            'tree_depth': self.depth,
            'tree_size': self.n_proposals,
            'mean_tree_accept': self.accept_sum / self.n_proposals,
            'energy_change': self.proposal.energy - self.start.energy,
            'max_energy_change': self.max_energy_change,
        }

class TNUTS(BaseTHMC, NUTS):

    _expected_trace = TNTrace

    _expected_stats = TNStepStats

    _expected_tree = TTree

max_stars_repo_name: HerculesJack/bayesfast

max_issues_repo_name: HerculesJack/bayesfast

max_forks_repo_name: HerculesJack/bayesfast


text:
import numpy as np
import os
import cv2
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans, MiniBatchKMeans
from sklearn.metrics import pairwise_distances_argmin
from sklearn.datasets import load_sample_image
from sklearn.utils import shuffle
from time import time
from PIL import Image

n_colors = 32
dir_path = "./scaled"
out_path = "vectorized"

try:
  os.mkdir(out_path)
except:
  pass

dir = os.listdir(dir_path)
for i, filename in enumerate(dir):
  # Load the photo
  img = Image.open(f'{dir_path}/{filename}')
  img.load()
  img = np.asarray(img, dtype="uint8")

  # Convert to floats (need to be in the range [0-1])
  img = np.array(img, dtype=np.float64) / 255

  # Apply Gaussian Blur to get rid of some problematic noise
  # cv2.fastNlMeansDenoisingMulti(img, 2)
  # img = cv2.GaussianBlur(img, (3, 3), 2)

  # Load image and transform to a 2D numpy array.
  w, h, d = original_shape = tuple(img.shape)
  assert d == 3
  image_array = np.reshape(img, (w * h, d))

  print("Fitting model on a small sub-sample of the data")
  image_array_sample = shuffle(image_array, random_state=0)[:1000]
  kmeans = KMeans(n_clusters=n_colors, random_state=0).fit(image_array_sample)
  # kmeans = MiniBatchKMeans(n_clusters=n_colors).fit(image_array_sample)

  # Get labels for all points
  print("Predicting color indices on the full image (k-means)")
  labels = kmeans.predict(image_array)

  codebook_random = shuffle(image_array, random_state=0)[:n_colors]
  print("Predicting color indices on the full image (random)")
  labels_random = pairwise_distances_argmin(codebook_random,
                                            image_array,
                                            axis=0)

  def recreate_image(codebook, labels, w, h):
      # Recreate the image from the code book & labels
      d = codebook.shape[1]
      image = np.zeros((w, h, d))
      label_idx = 0
      for i in range(w):
          for j in range(h):
              image[i][j] = codebook[labels[label_idx]]
              label_idx += 1
      return image

  # Make images
  image_kmeans = recreate_image(kmeans.cluster_centers_, labels, w, h)
  image_random = recreate_image(codebook_random, labels_random, w, h)

  # Save the images
  cv2.imwrite(f"{out_path}/{filename[:-4]}_{n_colors}_kmeans.png", cv2.cvtColor((image_kmeans*255).astype(np.uint8), cv2.COLOR_RGB2BGR))
  cv2.imwrite(f"{out_path}/{filename[:-4]}_{n_colors}_random.png", cv2.cvtColor((image_random*255).astype(np.uint8), cv2.COLOR_RGB2BGR))
  print(f'{i+1}/{len(dir)}: {filename[:-4]} vectorized')

max_stars_repo_name: adacienciala/Salvador

max_issues_repo_name: adacienciala/Salvador

max_forks_repo_name: adacienciala/Salvador


text:
import operator as op

import numpy as np

from jax.config import config

import pytest

# enable 64-bit mode for output dtype checks
config.update("jax_enable_x64", True)
from typing import Optional

import jax

import scico.numpy as snp
from scico import linop
from scico.numpy import BlockArray
from scico.random import randn
from scico.typing import JaxArray, PRNGKey

def adjoint_test(
    A: linop.LinearOperator,
    key: Optional[PRNGKey] = None,
    rtol: float = 1e-4,
    x: Optional[JaxArray] = None,
    y: Optional[JaxArray] = None,
):
    """Check the validity of A.conj().T as the adjoint for a LinearOperator A.

    Args:
        A: LinearOperator to test.
        key: PRNGKey for generating `x`.
        rtol: Relative tolerance.
    """

    assert linop.valid_adjoint(A, A.H, key=key, eps=rtol, x=x, y=y)

def test_valid_adjoint():

    diagonal, key = randn((10,), dtype=np.float32)
    D = linop.Diagonal(diagonal=diagonal)
    assert linop.valid_adjoint(D, D.T, key=key, eps=None) < 1e-4
    x, key = randn((5,), dtype=np.float32)
    y, key = randn((5,), dtype=np.float32)
    with pytest.raises(ValueError):
        linop.valid_adjoint(D, D.T, key=key, x=x)
    with pytest.raises(ValueError):
        linop.valid_adjoint(D, D.T, key=key, y=y)

class AbsMatOp(linop.LinearOperator):
    """Simple LinearOperator subclass for testing purposes.

    Similar to linop.MatrixOperator, but does not use the specialized
    MatrixOperator methods (.T, adj, etc). Used to verify the
    LinearOperator interface.
    """

    def __init__(self, A, adj_fn=None):
        self.A = A
        super().__init__(
            input_shape=A.shape[1], output_shape=A.shape[0], input_dtype=A.dtype, adj_fn=adj_fn
        )

    def _eval(self, x):
        return self.A @ x

class LinearOperatorTestObj:
    def __init__(self, dtype):
        M, N = (8, 16)
        key = jax.random.PRNGKey(12345)
        self.dtype = dtype

        self.A, key = randn((M, N), dtype=dtype, key=key)
        self.B, key = randn((M, N), dtype=dtype, key=key)
        self.C, key = randn((N, M), dtype=dtype, key=key)
        self.D, key = randn((M, N - 1), dtype=dtype, key=key)

        self.x, key = randn((N,), dtype=dtype, key=key)
        self.y, key = randn((M,), dtype=dtype, key=key)
        scalar, key = randn((1,), dtype=dtype, key=key)
        self.scalar = scalar.item()
        self.Ao = AbsMatOp(self.A)
        self.Bo = AbsMatOp(self.B)
        self.Co = AbsMatOp(self.C)
        self.Do = AbsMatOp(self.D)

@pytest.fixture(scope="module", params=[np.float32, np.float64, np.complex64, np.complex128])
def testobj(request):
    yield LinearOperatorTestObj(request.param)

@pytest.mark.parametrize("operator", [op.add, op.sub])
def test_binary_op(testobj, operator):
    # Our AbsMatOp class does not override the __add__, etc
    # so AbsMatOp + AbsMatOp -> LinearOperator
    # So to verify results, we evaluate the new LinearOperator on a random input

    comp_mat = operator(testobj.A, testobj.B)  # composite matrix
    comp_op = operator(testobj.Ao, testobj.Bo)  # composite linop

    assert isinstance(comp_op, linop.LinearOperator)  # Ensure we don't get a Map
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.x, comp_op @ testobj.x, rtol=5e-5)

    # linops of different sizes
    with pytest.raises(ValueError):
        operator(testobj.Ao, testobj.Co)
    with pytest.raises(ValueError):
        operator(testobj.Ao, testobj.Do)

@pytest.mark.parametrize("operator", [op.mul, op.truediv])
def test_scalar_left(testobj, operator):
    comp_mat = operator(testobj.A, testobj.scalar)
    comp_op = operator(testobj.Ao, testobj.scalar)
    assert isinstance(comp_op, linop.LinearOperator)  # Ensure we don't get a Map
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.x, comp_op @ testobj.x, rtol=5e-5)

    np.testing.assert_allclose(comp_mat.conj().T @ testobj.y, comp_op.adj(testobj.y), rtol=5e-5)

@pytest.mark.parametrize("operator", [op.mul, op.truediv])
def test_scalar_right(testobj, operator):
    if operator == op.truediv:
        pytest.xfail("scalar / LinearOperator is not supported")
    comp_mat = operator(testobj.scalar, testobj.A)
    comp_op = operator(testobj.scalar, testobj.Ao)
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.x, comp_op @ testobj.x, rtol=5e-5)

def test_negation(testobj):
    comp_mat = -testobj.A
    comp_op = -testobj.Ao
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.x, comp_op @ testobj.x, rtol=5e-5)

@pytest.mark.parametrize("operator", [op.add, op.sub])
def test_invalid_add_sub_array(testobj, operator):
    # Try to add or subtract an ndarray with AbsMatOp
    with pytest.raises(TypeError):
        operator(testobj.A, testobj.Ao)

@pytest.mark.parametrize("operator", [op.add, op.sub])
def test_invalid_add_sub_scalar(testobj, operator):
    # Try to add or subtract a scalar with AbsMatOp
    with pytest.raises(TypeError):
        operator(1.0, testobj.Ao)

def test_matmul_left(testobj):
    comp_mat = testobj.A @ testobj.C
    comp_op = testobj.Ao @ testobj.Co
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.y, comp_op @ testobj.y, rtol=5e-5)

def test_matmul_right(testobj):
    comp_mat = testobj.C @ testobj.A
    comp_op = testobj.Co @ testobj.Ao
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ testobj.x, comp_op @ testobj.x, rtol=5e-5)

def test_matvec_left(testobj):
    comp_mat = testobj.A @ testobj.x
    comp_op = testobj.Ao @ testobj.x
    assert comp_op.dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat, comp_op, rtol=5e-5)

def test_matvec_right(testobj):
    comp_mat = testobj.C @ testobj.y
    comp_op = testobj.Co @ testobj.y
    assert comp_op.dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat, comp_op, rtol=5e-5)

def test_gram(testobj):
    Ao = testobj.Ao
    a = Ao.gram(testobj.x)
    b = Ao.conj().T @ Ao @ testobj.x
    c = Ao.gram_op @ testobj.x

    comp_mat = testobj.A.conj().T @ testobj.A @ testobj.x

    np.testing.assert_allclose(a, comp_mat, rtol=5e-5)
    np.testing.assert_allclose(b, comp_mat, rtol=5e-5)
    np.testing.assert_allclose(c, comp_mat, rtol=5e-5)

def test_matvec_call(testobj):
    # A @ x and A(x) should return same
    np.testing.assert_allclose(testobj.Ao @ testobj.x, testobj.Ao(testobj.x), rtol=5e-5)

def test_adj_composition(testobj):
    Ao = testobj.Ao
    Bo = testobj.Bo
    A = testobj.A
    B = testobj.B
    x = testobj.x

    comp_mat = A.conj().T @ B
    a = Ao.conj().T @ Bo
    b = Ao.adj(Bo)
    assert a.input_dtype == testobj.A.dtype
    assert b.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ x, a @ x, rtol=5e-5)
    np.testing.assert_allclose(comp_mat @ x, b @ x, rtol=5e-5)

def test_transpose_matvec(testobj):
    Ao = testobj.Ao
    y = testobj.y

    a = Ao.T @ y
    b = y.T @ Ao

    comp_mat = testobj.A.T @ y

    assert a.dtype == testobj.A.dtype
    assert b.dtype == testobj.A.dtype
    np.testing.assert_allclose(a, comp_mat, rtol=5e-5)
    np.testing.assert_allclose(a, b, rtol=5e-5)

def test_transpose_matmul(testobj):
    Ao = testobj.Ao
    Bo = testobj.Bo
    x = testobj.x
    comp_op = Ao.T @ Bo
    comp_mat = testobj.A.T @ testobj.B
    assert comp_op.input_dtype == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ x, comp_op @ x, rtol=5e-5)

def test_conj_transpose_matmul(testobj):
    Ao = testobj.Ao
    Bo = testobj.Bo
    x = testobj.x
    comp_op = Ao.conj().T @ Bo
    comp_mat = testobj.A.conj().T @ testobj.B
    assert comp_mat == testobj.A.dtype
    np.testing.assert_allclose(comp_mat @ x, comp_op @ x, rtol=5e-5)

def test_conj_matvec(testobj):
    Ao = testobj.Ao
    x = testobj.x
    a = Ao.conj() @ x
    comp_mat = testobj.A.conj() @ x
    assert a.dtype == testobj.A.dtype
    np.testing.assert_allclose(a, comp_mat, rtol=5e-5)

def test_adjoint_matvec(testobj):
    Ao = testobj.Ao
    y = testobj.y

    a = Ao.adj(y)
    b = Ao.conj().T @ y
    c = (y.conj().T @ Ao).conj()

    comp_mat = testobj.A.conj().T @ y

    assert a.dtype == testobj.A.dtype
    assert b.dtype == testobj.A.dtype
    assert c.dtype == testobj.A.dtype
    np.testing.assert_allclose(a, comp_mat, rtol=5e-5)
    np.testing.assert_allclose(a, b, rtol=5e-5)
    np.testing.assert_allclose(a, c, rtol=5e-5)

def test_adjoint_matmul(testobj):
    # shape mismatch
    Ao = testobj.Ao
    Co = testobj.Co

    with pytest.raises(ValueError):
        Ao.adj(Co)

def test_hermitian(testobj):
    Ao = testobj.Ao
    y = testobj.y

    np.testing.assert_allclose(Ao.conj().T @ y, Ao.H @ y)

def test_shape(testobj):
    Ao = testobj.Ao
    x = testobj.x
    y = testobj.y

    with pytest.raises(ValueError):
        _ = Ao @ y

    with pytest.raises(ValueError):
        _ = Ao(y)

    with pytest.raises(ValueError):
        _ = Ao.T @ x

    with pytest.raises(ValueError):
        _ = Ao.adj(x)

class TestDiagonal:
    def setup_method(self, method):
        self.key = jax.random.PRNGKey(12345)

    input_shapes = [(8,), (8, 12), ((3,), (4, 5))]

    @pytest.mark.parametrize("diagonal_dtype", [np.float32, np.complex64])
    @pytest.mark.parametrize("input_shape", input_shapes)
    def test_eval(self, input_shape, diagonal_dtype):
        diagonal, key = randn(input_shape, dtype=diagonal_dtype, key=self.key)
        x, key = randn(input_shape, dtype=diagonal_dtype, key=key)

        D = linop.Diagonal(diagonal=diagonal)
        assert (D @ x).shape == D.output_shape
        snp.testing.assert_allclose((diagonal * x), (D @ x), rtol=1e-5)

    @pytest.mark.parametrize("diagonal_dtype", [np.float32, np.complex64])
    def test_eval_broadcasting(self, diagonal_dtype):
        # array broadcast
        diagonal, key = randn((3, 1, 4), dtype=diagonal_dtype, key=self.key)
        x, key = randn((5, 1), dtype=diagonal_dtype, key=key)
        D = linop.Diagonal(diagonal, x.shape)
        assert (D @ x).shape == (3, 5, 4)
        np.testing.assert_allclose((diagonal * x).ravel(), (D @ x).ravel(), rtol=1e-5)

        # blockarray broadcast
        diagonal, key = randn(((3, 1, 4), (5, 5)), dtype=diagonal_dtype, key=self.key)
        x, key = randn(((5, 1), (1,)), dtype=diagonal_dtype, key=key)
        D = linop.Diagonal(diagonal, x.shape)
        assert (D @ x).shape == ((3, 5, 4), (5, 5))
        snp.testing.assert_allclose((diagonal * x), (D @ x), rtol=1e-5)

        # blockarray x array -> error
        diagonal, key = randn(((3, 1, 4), (5, 5)), dtype=diagonal_dtype, key=self.key)
        x, key = randn((5, 1), dtype=diagonal_dtype, key=key)
        with pytest.raises(ValueError):
            D = linop.Diagonal(diagonal, x.shape)

        # array x blockarray -> error
        diagonal, key = randn((3, 1, 4), dtype=diagonal_dtype, key=self.key)
        x, key = randn(((5, 1), (1,)), dtype=diagonal_dtype, key=key)
        with pytest.raises(ValueError):
            D = linop.Diagonal(diagonal, x.shape)

    @pytest.mark.parametrize("diagonal_dtype", [np.float32, np.complex64])
    @pytest.mark.parametrize("input_shape", input_shapes)
    def test_adjoint(self, input_shape, diagonal_dtype):
        diagonal, key = randn(input_shape, dtype=diagonal_dtype, key=self.key)
        D = linop.Diagonal(diagonal=diagonal)

        adjoint_test(D)

    @pytest.mark.parametrize("operator", [op.add, op.sub])
    @pytest.mark.parametrize("diagonal_dtype", [np.float32, np.complex64])
    @pytest.mark.parametrize("input_shape1", input_shapes)
    @pytest.mark.parametrize("input_shape2", input_shapes)
    def test_binary_op(self, input_shape1, input_shape2, diagonal_dtype, operator):

        diagonal1, key = randn(input_shape1, dtype=diagonal_dtype, key=self.key)
        diagonal2, key = randn(input_shape2, dtype=diagonal_dtype, key=key)
        x, key = randn(input_shape1, dtype=diagonal_dtype, key=key)

        D1 = linop.Diagonal(diagonal=diagonal1)
        D2 = linop.Diagonal(diagonal=diagonal2)

        if input_shape1 != input_shape2:
            with pytest.raises(ValueError):
                a = operator(D1, D2) @ x
        else:
            a = operator(D1, D2) @ x
            Dnew = linop.Diagonal(operator(diagonal1, diagonal2))
            b = Dnew @ x
            snp.testing.assert_allclose(a, b, rtol=1e-5)

    @pytest.mark.parametrize("operator", [op.add, op.sub])
    def test_binary_op_mismatch(self, operator):
        diagonal_dtype = np.float32
        input_shape1 = (8,)
        input_shape2 = (12,)
        diagonal1, key = randn(input_shape1, dtype=diagonal_dtype, key=self.key)
        diagonal2, key = randn(input_shape2, dtype=diagonal_dtype, key=key)

        D1 = linop.Diagonal(diagonal=diagonal1)
        D2 = linop.Diagonal(diagonal=diagonal2)
        with pytest.raises(ValueError):
            operator(D1, D2)

    @pytest.mark.parametrize("operator", [op.mul, op.truediv])
    def test_scalar_right(self, operator):
        if operator == op.truediv:
            pytest.xfail("scalar / LinearOperator is not supported")

        diagonal_dtype = np.float32
        input_shape = (8,)

        diagonal1, key = randn(input_shape, dtype=diagonal_dtype, key=self.key)
        scalar = np.random.randn()
        x, key = randn(input_shape, dtype=diagonal_dtype, key=key)

        D = linop.Diagonal(diagonal=diagonal1)
        scaled_D = operator(scalar, D)

        np.testing.assert_allclose(scaled_D @ x, operator(scalar, D @ x), rtol=5e-5)

    @pytest.mark.parametrize("operator", [op.mul, op.truediv])
    def test_scalar_left(self, operator):
        diagonal_dtype = np.float32
        input_shape = (8,)

        diagonal1, key = randn(input_shape, dtype=diagonal_dtype, key=self.key)
        scalar = np.random.randn()
        x, key = randn(input_shape, dtype=diagonal_dtype, key=key)

        D = linop.Diagonal(diagonal=diagonal1)
        scaled_D = operator(D, scalar)

        np.testing.assert_allclose(scaled_D @ x, operator(D @ x, scalar), rtol=5e-5)

def test_adj_lazy():
    dtype = np.float32
    M, N = (8, 16)
    A, key = randn((M, N), dtype=np.float32, key=None)
    y, key = randn((M,), dtype=np.float32, key=key)
    Ao = AbsMatOp(A, adj_fn=None)  # defer setting the linop

    assert Ao._adj is None
    a = Ao.adj(y)  # Adjoint is set when .adj() is called
    b = A.T @ y
    np.testing.assert_allclose(a, b, rtol=1e-5)

def test_jit_adj_lazy():
    dtype = np.float32
    M, N = (8, 16)
    A, key = randn((M, N), dtype=np.float32, key=None)
    y, key = randn((M,), dtype=np.float32, key=key)
    Ao = AbsMatOp(A, adj_fn=None)  # defer setting the linop
    assert Ao._adj is None
    Ao.jit()  # Adjoint set here
    assert Ao._adj is not None
    a = Ao.adj(y)
    b = A.T @ y
    np.testing.assert_allclose(a, b, rtol=1e-5)

class PowerIterTestObj:
    def __init__(self, dtype):
        M, N = (4, 4)
        key = jax.random.PRNGKey(12345)
        self.dtype = dtype

        A, key = randn((M, N), dtype=dtype, key=key)
        self.A = A.conj().T @ A  # ensure symmetric

        self.Ao = linop.MatrixOperator(self.A)
        self.Bo = AbsMatOp(self.A)

        self.key = key
        self.ev = snp.linalg.norm(
            self.A, 2
        )  # The largest eigenvalue of A is the spectral norm of A

@pytest.fixture(scope="module", params=[np.float32, np.complex64])
def pitestobj(request):
    yield PowerIterTestObj(request.param)

def test_power_iteration(pitestobj):
    """Verify that power iteration calculates largest eigenvalue for real and complex
    symmetric matrices.
    """
    # Test using the LinearOperator MatrixOperator
    mu, v = linop.power_iteration(A=pitestobj.Ao, maxiter=100, key=pitestobj.key)
    assert np.abs(mu - pitestobj.ev) < 1e-4

    # Test using the AbsMatOp for test_linop.py
    mu, v = linop.power_iteration(A=pitestobj.Bo, maxiter=100, key=pitestobj.key)
    assert np.abs(mu - pitestobj.ev) < 1e-4

def test_operator_norm():

    I = linop.Identity(8)
    Inorm = linop.operator_norm(I)
    assert np.abs(Inorm - 1.0) < 1e-5
    key = jax.random.PRNGKey(12345)
    for dtype in [np.float32, np.complex64]:
        d, key = randn((16,), dtype=dtype, key=key)
        D = linop.Diagonal(d)
        Dnorm = linop.operator_norm(D)
        assert np.abs(Dnorm - snp.abs(d).max()) < 1e-5

class SliceTestObj:
    def __init__(self, dtype):
        self.x = snp.zeros((4, 5, 6, 7), dtype=dtype)

@pytest.fixture(scope="module", params=[np.float32, np.complex64])
def slicetestobj(request):
    yield SliceTestObj(request.param)

slice_examples = [
    np.s_[1:],
    np.s_[:, 2:],
    np.s_[..., 3:],
    np.s_[1:, :-3],
    np.s_[1:, :, :3],
    np.s_[1:, ..., 2:],
]

@pytest.mark.parametrize("idx", slice_examples)
def test_slice_eval(slicetestobj, idx):
    x = slicetestobj.x
    A = linop.Slice(idx=idx, input_shape=x.shape, input_dtype=x.dtype)
    assert (A @ x).shape == x[idx].shape

@pytest.mark.parametrize("idx", slice_examples)
def test_slice_adj(slicetestobj, idx):
    x = slicetestobj.x
    A = linop.Slice(idx=idx, input_shape=x.shape, input_dtype=x.dtype)
    adjoint_test(A)

block_slice_examples = [
    1,
    np.s_[0:1],
    np.s_[:1],
]

@pytest.mark.parametrize("idx", block_slice_examples)
def test_slice_blockarray(idx):
    x = BlockArray((snp.zeros((3, 4)), snp.ones((3, 4, 5, 6))))
    A = linop.Slice(idx=idx, input_shape=x.shape, input_dtype=x.dtype)
    assert (A @ x).shape == x[idx].shape

def test_transpose():
    shape = (1, 2, 3, 4)
    perm = (1, 0, 3, 2)
    x, _ = randn(shape)
    H = linop.Transpose(shape, perm)
    np.testing.assert_array_equal(H @ x, x.transpose(perm))

    # transpose transpose is transpose inverse
    np.testing.assert_array_equal(H.T @ H @ x, x)

def test_pad():
    shape = (2, 3, 4)
    pad = 1
    x, _ = randn(shape)
    H = linop.Pad(shape, pad)

    pad_shape = tuple(n + 2 * pad for n in shape)
    y = snp.zeros(pad_shape)
    y = y.at[pad:-pad, pad:-pad, pad:-pad].set(x)
    np.testing.assert_array_equal(H @ x, y)

    # pad transpose is crop
    y, _ = randn(pad_shape)
    np.testing.assert_array_equal(H.T @ y, y[pad:-pad, pad:-pad, pad:-pad])

max_stars_repo_name: lanl/scico

max_issues_repo_name: lanl/scico

max_forks_repo_name: lanl/scico

reason: import numpy,import jax,from jax