text:
#!/usr/bin/python
# _*_ coding: utf-8 _*_

"""

@author: Qing Liu, sunnymarkliu@163.com
@github: https://github.com/sunnymarkLiu
@time  : 2019/9/5 15:20
"""
# coding = utf-8
import numpy as np

class RougeL(object):
    def __init__(self, gamma=1.2):
        self.gamma = gamma  # gamma 为常量
        self.inst_scores = []
        self.r_scores = []
        self.p_scores = []

    def _lcs(self, x, y):
        """
        Computes the length of the longest common subsequence (lcs) between two
        strings. The implementation below uses a DP programming algorithm and runs
        in O(nm) time where n = len(x) and m = len(y).
        Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence
        Args:
          x: collection of words
          y: collection of words
        Returns:
          Table of dictionary of coord and len lcs
        """
        n, m = len(x), len(y)
        table = dict()
        for i in range(n + 1):
            for j in range(m + 1):
                if i == 0 or j == 0:
                    table[i, j] = 0
                elif x[i - 1] == y[j - 1]:
                    table[i, j] = table[i - 1, j - 1] + 1
                else:
                    table[i, j] = max(table[i - 1, j], table[i, j - 1])

        return table

    def lcs(self, x, y):
        """
        Returns the length of the Longest Common Subsequence between sequences x
        and y.
        Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence
        Args:
          x: sequence of words
          y: sequence of words
        Returns
          integer: Length of LCS between x and y
        """
        table = self._lcs(x, y)
        n, m = len(x), len(y)
        return table[n, m]

    def add_inst(self, cand: str, ref: str):
        """根据参考答案分析出预测答案的分数
        Arguments:
            cand {str} -- 预测答案
            ref {str} -- 参考答案
        """

        basic_lcs = self.lcs(cand, ref)
        p_denom = len(cand)
        r_denom = len(ref)
        prec = basic_lcs / p_denom if p_denom > 0. else 0.
        rec = basic_lcs / r_denom if r_denom > 0. else 0.
        if prec != 0 and rec != 0:
            score = ((1 + self.gamma ** 2) * prec * rec) / \
                float(rec + self.gamma**2 * prec)
        else:
            score = 0
        self.inst_scores.append(score)
        self.r_scores.append(rec)
        self.p_scores.append(prec)
        return self

    def get_score(self) -> float:
        """计算cand预测数据的RougeL分数
        Returns:
            float -- RougeL分数
        """
        return 1. * sum(self.inst_scores) / len(self.inst_scores)


text:
# -*- coding: utf-8 -*-

from __future__ import division, absolute_import

import sympy
from sympy import exp, Derivative
import numpy as np

from symvarsub import NumTransformer
from symvarsub.utilities import RealFunction

def test_NumTransformer(tempdir=None, logger=None):
    x, y = sympy.symbols('x y')
    t = NumTransformer([(x+1)**2, (x+1)**3-y], [x, y], tempdir=tempdir,
                       save_temp=True, logger=logger)
    x_, y_ = np.linspace(0, 10, 10), np.linspace(10, 20, 10)
    out = t(x_, y_)
    assert np.allclose(out, np.vstack(((x_+1)**2, (x_+1)**3-y_)).transpose())

def test_NumTransformer__complex_argument_names(tempdir=None, logger=None):
    # y'(t) = t**4; y(0)=1.0  ===> y = t**5/5 + 1.0
    # z(t) = log(y(t)) ===> y=exp(z(t)), dzdt = t**4*exp(-z(t))

    num_t = np.linspace(0.0, 3.0, 4)
    exact_y = num_t**5/5.0+1.0
    exact_dydt = num_t**4

    num_z = np.log(exact_y)
    num_dzdt = num_t**4*np.exp(-num_z)

    z = sympy.Function('z')
    t = sympy.Symbol('t')

    y_in_z = sympy.exp(z(t))
    dydt_in_z = y_in_z.diff(t)

    z_data = {t: num_t, z(t): num_z, z(t).diff(t): num_dzdt}

    exprs = [y_in_z, dydt_in_z]
    inp = [t, z(t), z(t).diff(t)]
    tfmr = NumTransformer(exprs, inp, tempdir=tempdir,
                          save_temp=True, logger=logger)
    result = tfmr(*[z_data[k] for k in inp])

    num_y = result[:, 0]
    assert np.allclose(num_y, exact_y)

    num_dydt = result[:, 1]
    assert np.allclose(num_dydt, exact_dydt)

def test_NumTransformer__complex_argument_names2(tempdir=None, logger=None):
    # y,t -> f,g
    # f = exp(y(t))
    # g = exp(y(t))*Derivative(y(t), t)

    n = 5
    num_y = np.linspace(5.0, 7.0, n)
    num_dydt = np.linspace(0.0, 3.0, n)

    exact_f = np.exp(num_y)
    exact_g = np.exp(num_y)*num_dydt

    out = ['f', 'g']
    exact = {'f': exact_f, 'g': exact_g}

    y = sympy.Function('y')
    t = sympy.Symbol('t')
    dydt = y(t).diff(t)

    exprs = [sympy.exp(y(t)), sympy.exp(y(t))*y(t).diff(t)]
    inp = [y(t), dydt]
    num_data = {y(t): num_y, dydt: num_dydt}
    tfmr = NumTransformer(exprs, inp, tempdir=tempdir,
                          save_temp=True, logger=logger)
    result = tfmr(*[num_data[k] for k in inp])

    for i, s in enumerate(out):
        assert np.allclose(result[:, i], exact[s])

def test_NumTransformer__write_code(tempdir=None, logger=None):
    t = sympy.Symbol('t', real=True)
    lny0, lny1, lny2, lny3, lny4 = [RealFunction(s)(t) for s
                                    in 'lny0 lny1 lny2 lny3 lny4'.split()]

    exprs = [
        exp(lny0), exp(lny0)*Derivative(lny0, t),
        exp(lny1), exp(lny1)*Derivative(lny1, t),
    ]
    args = [Derivative(lny1, t), lny1, Derivative(lny0, t), lny0]
    tfmr = NumTransformer(exprs, args, tempdir=tempdir, save_temp=True)
    # Compilation might fail: (that's what we are testing)
    tfmr(*([np.ones(3)]*len(args)))

if __name__ == '__main__':
    # When this test file is run from the command line
    # we print some extra info using a logger and save the generated
    # code in the output directories tmp${N}

    import logging
    logging.basicConfig(level=logging.DEBUG)
    logger = logging.getLogger(__name__)
    test_NumTransformer__complex_argument_names('./tmp1/', logger=logger)
    test_NumTransformer__complex_argument_names2('./tmp2/', logger=logger)
    test_NumTransformer('./tmp3/', logger=logger)
    test_NumTransformer__write_code(tempdir='./tmp4/', logger=logger)

ext: py

max_stars_repo_name: bjodah/symvarsub

max_issues_repo_name: bjodah/symvarsub

max_forks_repo_name: bjodah/symvarsub

reason: import numpy,import sympy,from sympy


text:
#!/usr/bin/python
# -*- coding:utf-8 -*-

"""
dp - value iteration 示例
走迷宫
"""

import numpy as np
from Reinforcement.Environments.grid_world import GridworldEnv

def value_iteration():
    """
    进行训练
    """
    V = np.zeros(env.nS)
    optimal_v = optimal_value_function(V)
    policy = optimal_policy_extraction(optimal_v)
    return policy, optimal_v

def optimal_value_function(V):
    """
    计算最优方案
    """
    while True:
        delta = 0
        for s in range(env.nS):
            Q_sa = np.zeros(env.nA)
            for a in range(env.nA):
                for prob_s, next_state, reward, _ in env.P[s][a]:
                    Q_sa[a] += prob_s * (reward + gamma * V[next_state])
            max_value_function_s = np.max(Q_sa)
            delta = max(delta, np.abs(max_value_function_s - V[s]))
            V[s] = max_value_function_s
        if delta < 0.00001:
            break
    return V

def optimal_policy_extraction(V):
    """
    获取规则
    """
    policy = np.zeros([env.nS, env.nA])
    for s in range(env.nS):
        Q_sa = np.zeros(env.nA)
        for a in range(env.nA):
            for prob_s, next_state, reward, _ in env.P[s][a]:
                Q_sa[a] += prob_s * (reward + gamma * V[next_state])
        best_action = np.argmax(Q_sa)
        policy[s] = np.eye(env.nA)[best_action]
    return policy

if __name__ == "__main__":
    gamma = 1.0
    env = GridworldEnv()

    final_policy,final_v = value_iteration()

    print("Final Policy ")
    print(final_policy)

    print("Final Policy grid : (0=up, 1=right, 2=down, 3=left)")
    print(np.reshape(np.argmax(final_policy, axis=1), env.shape))

    print("Final Value Function grid")
    print(final_v.reshape(env.shape))

max_stars_repo_name: neohope/MLStudy

max_issues_repo_name: neohope/MLStudy

max_forks_repo_name: neohope/MLStudy

reason: import numpy


text:
import collections.abc as cabc
from typing import Iterable, Collection, Union

import numpy as np
import pandas as pd
from anndata import AnnData

from ... import logging

def wishbone(
    adata: AnnData,
    start_cell: str,
    branch: bool = True,
    k: int = 15,
    components: Iterable[int] = (1, 2, 3),
    num_waypoints: Union[int, Collection] = 250,
):
    """\
    Wishbone identifies bifurcating developmental trajectories from single-cell data
    [Setty16]_.

    Wishbone is an algorithm for positioning single cells along bifurcating
    developmental trajectories with high resolution. Wishbone uses multi-dimensional
    single-cell data, such as mass cytometry or RNA-Seq data, as input and orders cells
    according to their developmental progression, and it pinpoints bifurcation points
    by labeling each cell as pre-bifurcation or as one of two post-bifurcation cell
    fates.

    .. note::
       More information and bug reports `here
       <https://github.com/dpeerlab/wishbone>`__.

    Parameters
    ----------
    adata
        Annotated data matrix.
    start_cell
        Desired start cell from `obs_names`.
    branch
        Use True for Wishbone and False for Wanderlust.
    k
        Number of nearest neighbors for graph construction.
    components
        Components to use for running Wishbone.
    num_waypoints
        Number of waypoints to sample.

    Returns
    -------
    Updates `adata` with the following fields:

    `trajectory_wishbone` : (`adata.obs`, dtype `float64`)
        Computed trajectory positions.
    `branch_wishbone` : (`adata.obs`, dtype `int64`)
        Assigned branches.

    Example
    -------

    >>> import scanpy.external as sce
    >>> import scanpy as sc

    **Loading Data and Pre-processing**

    >>> adata = sc.datasets.pbmc3k()
    >>> sc.pp.normalize_per_cell(adata)
    >>> sc.pp.pca(adata)
    >>> sc.tl.tsne(adata=adata, n_pcs=5, perplexity=30)
    >>> sc.pp.neighbors(adata, n_pcs=15, n_neighbors=10)
    >>> sc.tl.diffmap(adata, n_comps=10)

    **Running Wishbone Core Function**

    Usually, the start cell for a dataset should be chosen based on high expression of
    the gene of interest:

    >>> sce.tl.wishbone(
    ...     adata=adata, start_cell='ACAAGAGACTTATC-1',
    ...     components=[2, 3], num_waypoints=150,
    ... )

    **Visualizing Wishbone results**

    >>> sc.pl.tsne(adata, color=['trajectory_wishbone', 'branch_wishbone'])
    >>> markers = ['C1QA', 'PSAP', 'CD79A', 'CD79B', 'CST3', 'LYZ', 'MALAT1']
    >>> sce.pl.wishbone_marker_trajectory(adata, markers, show=True)

    For further demonstration of Wishbone methods and visualization please follow the
    notebooks in the package `Wishbone_for_single_cell_RNAseq.ipynb
    <https://github.com/dpeerlab/wishbone/tree/master/notebooks>`_.\
    """
    try:
        from wishbone.core import wishbone as c_wishbone
    except ImportError:
        raise ImportError(
            "\nplease install wishbone:\n\n\thttps://github.com/dpeerlab/wishbone"
        )

    # Start cell index
    s = np.where(adata.obs_names == start_cell)[0]
    if len(s) == 0:
        raise RuntimeError(
            f"Start cell {start_cell} not found in data. "
            "Please rerun with correct start cell."
        )
    if isinstance(num_waypoints, cabc.Collection):
        diff = np.setdiff1d(num_waypoints, adata.obs.index)
        if diff.size > 0:
            logging.warning(
                "Some of the specified waypoints are not in the data. "
                "These will be removed"
            )
            num_waypoints = diff.tolist()
    elif num_waypoints > adata.shape[0]:
        raise RuntimeError(
            "num_waypoints parameter is higher than the number of cells in the "
            "dataset. Please select a smaller number"
        )
    s = s[0]

    # Run the algorithm
    components = list(components)
    res = c_wishbone(
        adata.obsm['X_diffmap'][:, components],
        s=s,
        k=k,
        l=k,
        num_waypoints=num_waypoints,
        branch=branch,
    )

    # Assign results
    trajectory = res["Trajectory"]
    trajectory = (trajectory - np.min(trajectory)) / (
        np.max(trajectory) - np.min(trajectory)
    )
    adata.obs['trajectory_wishbone'] = np.asarray(trajectory)

    # branch_ = None
    if branch:
        branches = res["Branches"].astype(int)
        adata.obs['branch_wishbone'] = np.asarray(branches)

def _anndata_to_wishbone(adata: AnnData):
    from wishbone.wb import SCData, Wishbone

    scdata = SCData(adata.to_df())
    scdata.diffusion_eigenvectors = pd.DataFrame(
        adata.obsm['X_diffmap'], index=adata.obs_names
    )
    wb = Wishbone(scdata)
    wb.trajectory = adata.obs["trajectory_wishbone"]
    wb.branch = adata.obs["branch_wishbone"]
    return wb


text:
import onnxruntime
import numpy as np
import cv2
import torch

import torchvision
import torchvision.transforms as transforms
from models import *
from collections import OrderedDict

def get_classify_result(image,model,device):
    transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])
    img_tensor = transform_test(image).unsqueeze(0)
    print(img_tensor)
#     print("img_tensor")
#     print(img_tensor)
#     print(img_tensor)
    img_tensor = img_tensor.permute(0, 2, 3, 1).to(device)
    print(img_tensor.size())
    print(img_tensor.dtype)
#     print(img_tensor.shape)
    with torch.no_grad():
        outputs = model(img_tensor)
#     print("outputs")
#     print(outputs)
    return outputs

image = cv2.imread("/home/liweiliang/project/pytorch-cifar/data/onnx_test/off/101.jpg")
# imgsz = 480
img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# print(img[:5, :5, :])
# img = cv2.resize(img, (imgsz, imgsz))
img = cv2.resize(img, (32, 96))
# print(img[:5, :5, :] / 255.0)
# img = torch.from_numpy(img).unsqueeze(0).permute(0, 3, 1, 2)
# img = img.to(device, non_blocking=True)
# img = img.half() if half else img.float()  # uint8 to fp16/32
# img_detect = (img / 255.0).astype(np.float32).reshape(1,192,256,3)
# print(img_detect[:5, :5, :])
# input_array= np.ones([1,480,480,3]).astype(np.float32)
transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

img_tensor = transform_test(img)
img_array = img_tensor.unsqueeze(0).permute(0, 2, 3, 1).numpy()
print(img_array)
print(img_tensor)
# print(img_array)
print(img_array.shape)
sess = onnxruntime.InferenceSession('/home/liweiliang/project/pytorch-cifar/checkpoint/resnet.onnx')
input_nodes = sess.get_inputs()[0].name
for i in range(len(sess.get_outputs())):
    print(sess.get_outputs()[i].name)
class_data = sess.get_outputs()[0].name
print(class_data)
# print(input_array[0,0,0,0])
output = sess.run([   class_data], {input_nodes: img_array})
result  = output[0]
print(result.shape)
print("onnx result")
print(result)

device = 'cuda' if torch.cuda.is_available() else 'cpu'
net = ResNet18()
net = net.to(device)
checkpoint = torch.load('/home/liweiliang/project/pytorch-cifar/checkpoint/ckpt_190.pth')
new_state_dict = OrderedDict()
for k, v in checkpoint['net'].items():
#     print(k)
    name = k[7:]  # remove module.
    new_state_dict[name] = v
#     print(v)
net.load_state_dict(new_state_dict)
net.eval()
# image=cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

result = get_classify_result(img,net,device)
print("pytorch result")
print(result)

max_stars_repo_name: neolixcn/pytorch-cifar

max_issues_repo_name: neolixcn/pytorch-cifar

max_forks_repo_name: neolixcn/pytorch-cifar


text:
import argparse
import math
import os
import sys
import typing

import cv2
import numpy

def load_binary(filepath: str, input_threshold: float = 127, input_maxvalue: float = 255) -> numpy.array:
    """
    しきい値適用済み2値画像読み込み
    :param filepath: 入力ファイルパス
    :param input_threshold: 入力閾値
    :param input_maxvalue: 閾値最大値
    :return: 2値画像データ
    """
    filepath = str(filepath)
    img = cv2.imread(filepath, cv2.IMREAD_GRAYSCALE | cv2.IMREAD_IGNORE_ORIENTATION)
    _, thr = cv2.threshold(img, input_threshold, 255, cv2.ADAPTIVE_THRESH_MEAN_C)
    return thr

def detect_lines(img: numpy.array, min_length: float = 20, line_gap: float = 3) -> list[numpy.array]:
    """
    直線検出
    :param img: 入力画像（2値画像）
    :return: 検出直線リスト
    """
    lines = cv2.HoughLinesP(img, rho=1, theta=math.pi/180, threshold=200, minLineLength=min_length, maxLineGap=line_gap)
    return lines

def detect_area(img: numpy.array, area_threshold: float = 0.0001) -> list[numpy.array]:
    """
    閾値を超える面積を持つ輪郭の検出
    :param img: 入力画像
    :param area_threshold: 面積閾値（画像全体の何%を`(0, 1]`で指定）
    :return: 閾値を超えた面積の領域リスト
    """
    height, width = img.shape
    img_area = width * height
    contours, _ = cv2.findContours(img, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
    contours = [cnt for cnt in contours if (cv2.contourArea(cnt) / img_area) > area_threshold]
    return contours

T = typing.TypeVar("T")

def clamp(v: T, min_v: T, max_v: T) -> T:
    """
    clamp関数
    入力値を`[min_v, max_v]`の範囲に収める
    :param T v: 入力値
    :param T min_v: 最小値
    :param T max_v: 最大値
    :return: `[min_v, max_v]`内に収めた値
    """
    return min(max_v, max(v, min_v))

BoundingRect = tuple[tuple[float, float], tuple[float, float]]

def fill_area(img: numpy.array, contours: list[numpy.array], buffer_ratio: float = 1.1, color: typing.Optional[float] = None) -> numpy.array:
    """
    領域の外接矩形で塗りつぶす
    :param img: 入力画像
    :param contours: 領域リスト
    :param buffer_ratio: バッファ拡張率
    :param color: 塗りつぶしの色（未指定の場合は入力画像の中央値で塗りつぶす）
    :return: 塗りつぶし後の画像
    """
    height, width = img.shape
    x_buffer = int(width * buffer_ratio)
    y_buffer = int(height * buffer_ratio)
    # detect fill color
    if color is None:
        color = numpy.median(img)
    # fill bounding rect
    for cnt in contours:
        hull = cv2.convexHull(cnt)
        M = cv2.moments(hull)
        cx = M["m10"] / M["m00"]
        cy = M["m01"] / M["m00"]
        temp = []
        for c in hull:
            x, y = c[0]
            vx = x - cx
            vy = y - cy
            x = clamp(vx * buffer_ratio + cx, 0, width)
            y = clamp(vy * buffer_ratio + cy, 0, height)
            temp.append([int(x), int(y)])
        img = cv2.fillPoly(img, pts=[numpy.asarray(temp)], color=(color,))
    return img

def line_length(line: numpy.array) -> float:
    """
    直線の長さの算出
    `cv2.HoughLinesP()`の返り値は`[ [ [start_x, start_y, end_x, end_y] ], ... ]`形式になっている
    :param line: `[ [start_x, start_y, end_x, end_y] ]`であること
    :return: 直線の長さ
    """
    assert len(line) == 1
    sx, sy, ex, ey = line[0]
    dx = ex - sx
    dy = ey - sy
    return math.sqrt(dx * dx + dy * dy)

def detect_meteor(filepath: str, input_threshold: float = 127, input_maxvalue: float = 255, area_threshold: float = 0.0001, buffer_ratio: float = 1.1, line_threshold: float = 100) -> typing.Optional[tuple[list[numpy.array], list[BoundingRect], tuple[int, int]]]:
    """
    流星の検出
    :param str filepath: 入力画像ファイルパス
    :param float input_threshold: 画像のしきい値処理
    :param float input_maxvalue: 画像のしきい値処理最大値
    :param float area_threshold: 面積のある領域検知用閾値（`detect_area()`関数`threshold`参照）
    :param float buffer_ratio: 面積拡張率
    :param float line_threshold: 検出した直線を流星と判定する最小の長さ
    :return: (検出した直線 or None, 塗りつぶした領域 or None)
    """
    img = load_binary(filepath, input_threshold, input_maxvalue)
    area_contours = detect_area(img, area_threshold)
    if area_contours:
        img = fill_area(img, area_contours, buffer_ratio=buffer_ratio, color=0)
    lines = detect_lines(img)
    if lines is not None:
        line_lengthes = [line_length(x) for x in lines]
        length = max(line_lengthes)
        if length > line_threshold:
            filtered_lines = [x for x, y in zip(lines, line_lengthes) if y > line_threshold]
            return filtered_lines, area_contours, img.shape
    return None, area_contours, img.shape

def main(argv: list[str]) -> int:
    from tqdm import tqdm
    parser = argparse.ArgumentParser()
    parser.add_argument("directory")
    parser.add_argument("--input-threshold", type=float, default=127)
    parser.add_argument("--input-maxvalue", type=float, default=255)
    parser.add_argument("--area-threshold", type=float, default=0.0001)
    parser.add_argument("--buffer-ratio", type=float, default=1.1)
    parser.add_argument("--line-threshold", type=float, default=100)

    args = parser.parse_args(argv[1:])

    # 拡張子が`.jpg`の画像リストを作成
    image_list = []
    for dirname, _, filenames in os.walk(args.directory):
        image_list.extend([os.path.join(dirname, x) for x in filenames if x.lower().endswith(".jpg") or x.lower().endswith(".jpeg")])
    image_list.sort()

    # 流星の写っていると思われる画像を抽出
    result = []
    for filepath in tqdm(image_list):
        filepath = str(filepath)
        lines, _, _ = detect_meteor(
            filepath,
            args.input_threshold,
            args.input_maxvalue,
            args.area_threshold,
            args.buffer_ratio,
            args.line_threshold
        )
        if lines is not None:
            result.append((filepath, lines))
    print("detected: {}/{}".format(len(result), len(image_list)))
    if result:
        print("files:")
        for filepath, lines in result:
            print(filepath)

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))

max_stars_repo_name: tail-feather/MeteorDetector

max_issues_repo_name: tail-feather/MeteorDetector

max_forks_repo_name: tail-feather/MeteorDetector


text:
#!/usr/bin/env python

"""
Most of the code was provided by the wonderful Nick Higginbottom, who can be found on GitHub here,
	https://github.com/Higginbottom/
"""

import sys
import numpy as np
import pluto_hdf_utils
from astropy import constants as c

def write_z_v_density(fname, z, density, Ric):
	with open(fname, "w") as f:
		f.write("# R = {:2.4f} Ric\n".format(Ric))
		f.write("# z          rho\n")
		for i in range(len(z)):
			f.write("{:^+7.5e} {:^+7.5e}\n".format(z[i], density[i]))
	return

try:
	fname=sys.argv[1]
except:
	print("No filename provided")
	sys.exit(1)

# Read in data using pyhdf
data=pluto_hdf_utils.get_hdf_data(fname)

if len(sys.argv) > 3:
	itheta_disk=int(sys.argv[3])
else:
	itheta_disk = -1
if len(sys.argv) > 4:
	iradmax = int(sys.argv[4])
else:
	iradmax = -1

try:
	print("HDF file produced at t=",data["Time"])
except:
	print("No time information")

# Add the data into their own arrays
density=data["Data"]["DENSITY"]["data"]
nd=density/(1.43*c.m_p.value)
energy=data["Data"]["TOTAL ENERGY"]["data"]
v_r=data["Data"]["1-VELOCITY"]["data"]
v_t=data["Data"]["2-VELOCITY"]["data"]
v_p=data["Data"]["3-VELOCITY"]["data"]
line_c=data["Data"]["LINE_C"]["data"]
xray_h=data["Data"]["XRAY_H"]["data"]
comp_c=data["Data"]["COMP_C"]["data"]
comp_h=data["Data"]["COMP_H"]["data"]
brem_c=data["Data"]["BREM_C"]["data"]
adiab_hc=data["Data"]["ADIABATIC"]["data"]
line_c_pre=data["Data"]["LINE_PRE"]["data"]
xray_h_pre=data["Data"]["XRAY_PRE"]["data"]
comp_c_pre=data["Data"]["COMP_C_PRE"]["data"]
comp_h_pre=data["Data"]["COMP_H_PRE"]["data"]
brem_c_pre=data["Data"]["BREM_PRE"]["data"]
theta=(data["Data"]["2-VELOCITY"]["x1"])
r=(data["Data"]["2-VELOCITY"]["x2"])
zeus_density=data["Data"]["DENSITY"]["data"]

# Various constants and numbers from the data
t_x=5.6e7
R_ic=c.G.cgs.value*7.0*c.M_sun.cgs.value*0.6*c.m_p.cgs.value/(c.k_B.cgs.value*t_x/4.0)
zeus_nh=zeus_density/(1.43*c.m_p.cgs).value
zeus_ne=zeus_nh*1.21
temp=(2.0/3.0)*energy
temperature=temp/((density/(c.m_p.cgs.value*0.6))*c.k_B.cgs.value)
pressure=energy*(5./3.-1.)
xi=data["Data"]["XI"]["data"]
nt=len(theta)
nr=len(r)

try:
	rindex = int(sys.argv[2])
except:
	print("No r index given")
	sys.exit(1)

print("------------------")
print("Ric = {:e}".format(R_ic))
print("index = {}".format(rindex))
print("r[index] = {:e}".format(r[rindex]))
print("r/Ric = {}".format(r[rindex] / R_ic))
print("n theta = {}".format(nt))
print("n r = {}".format(nr))
print("density.shape = {}".format(density.shape))
print("temperature.shape = {}".format(temperature.shape))
print("------------------")

zidx = 0
nlim = 37
z = np.zeros(nlim)
rho = np.zeros(nlim)
print("\nR = {:2.4} Ric".format(r[rindex] / R_ic))
print("# z          rho\n")
for i in range(nt-nlim, nt):
	ztemp = z[zidx] = r[rindex] * np.cos(theta[i])
	dtemp = rho[zidx] = density[i][rindex]
	T = temperature[i][rindex]
	zidx += 1
	print("{:^+7.5e} {:^+7.5e} {:^+7.5e}".format(ztemp, dtemp, T))
write_z_v_density("grid.out", z,  rho, r[rindex] / R_ic)

max_stars_repo_name: saultyevil/Python-scripts

max_issues_repo_name: saultyevil/Python-scripts

max_forks_repo_name: saultyevil/Python-scripts

reason: import numpy,from astropy


text:
#!/usr/bin/env python

##############################################################################
##
# This file is part of Sardana
##
# http://www.sardana-controls.org/
##
# Copyright 2011 CELLS / ALBA Synchrotron, Bellaterra, Spain
##
# Sardana is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
##
# Sardana is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
##
# You should have received a copy of the GNU Lesser General Public License
# along with Sardana.  If not, see <http://www.gnu.org/licenses/>.
##
##############################################################################

"""This is the macro server scan data output recorder module"""

__all__ = ["SPSRecorder", "ShmRecorder"]

__docformat__ = 'restructuredtext'

import time
import numpy
import operator

from sardana.macroserver.scan.recorder import BaseSharedMemoryRecorder
from sardana.macroserver.scan.recorder.datarecorder import DataRecorder
import numbers

class SPSRecorder(BaseSharedMemoryRecorder):

    maxenv = 50
    envlen = 1024

    def __init__(self, program=None, array=None, shape=None, **kwpars):
        """ @param[in] program SPS program name
            @param[in] array SPS array name
            @param[in] shape tuple (cols, rows)
            @param[in] pars keyword extra parameters
        """
        BaseSharedMemoryRecorder.__init__(self, **kwpars)
        try:
            import sps  # check if sps format is supported by this system
            self.sps = sps
        except ImportError:
            raise Exception("SPS is not available")
        self.shape = shape
        self.owner = False
        self.owner_ENV = False
        self.setID(program, array)

    def setID(self, program, array):
        self.program = program.replace('/', '')
        self.array = array.replace('_', '')
        if not array is None:
            self.array_ENV = "%s_ENV" % self.array
        else:
            self.array_ENV = None
        if program and array:
            self.init()

    def init(self):
        pass

    def setSize(self, rows, cols):
        self.shape = (cols, rows)
        self.rows = rows
        self.cols = cols

    def isInitialized(self):
        ret = not (
            self.program is None or self.array_ENV is None or self.array is None)
        return ret and not self.shape is None

    def putEnv(self, name, value):
        if not self.isInitialized():
            return
        self.sps.putenv(self.program, self.array_ENV, name, str(value))

    def putAllEnv(self, d):
        if not self.isInitialized():
            return
        p, a = self.program, self.array_ENV
        for k, v in d.items():
            self.sps.putenv(p, a, k, str(v))

    def _startRecordList(self, recordlist):
        if not self.isInitialized():
            return

        arraylist = self.sps.getarraylist(self.program)

        if self.array in arraylist:
            shm = self.sps.attach(self.program, self.array)
        else:
            cols, rows = self.shape
            self.sps.create(self.program, self.array,
                            rows, cols, self.sps.DOUBLE)
            self.owner = True

        if self.array_ENV in arraylist:
            shm_env = self.sps.attach(self.program, self.array_ENV)
        else:
            self.sps.create(self.program, self.array_ENV, self.maxenv, self.envlen,
                            self.sps.STRING)
            self.owner_ENV = True

        self.nopts = 0

        env = recordlist.getEnviron()

        self.labels = [col.label for col in env['datadesc']]

        env = {'title': env['title'],
               'started': env['starttime'].ctime(),
               'ended': '',
               'axistitles': ' '.join(self.labels),
               'ylabel': 'Counts',
               'nopts': self.nopts,
               'xbeg': 0,
               'xend': 200,
               'aborted': 0,
               'command': 'done',
               'fitresult': '0'}

        self.putAllEnv(env)

    def _writeRecord(self, record):
        if not self.isInitialized():
            return

        vals = []

        for colname in self.labels:
            val = record.data.get(colname)
            if ((val is not None)
                    and (isinstance(val, numbers.Number)
                         and (type(val) in [int, float]))):
                vals.append(val)
            elif (val is not None) and (isinstance(val, numbers.Number)):
                valsmca = []
                for i in range(0, len(val)):
                    valsmca.append(val[i])
                sufix = "1D"
                if self.array.endswith(sufix):
                    valsmca = numpy.array(valsmca)
                    self.sps.putdatarow(
                        self.program, self.array, record.recordno, valsmca)

        sufix = "0D"
        if self.array.endswith(sufix):
            vals = numpy.array(vals)
            self.sps.putdatarow(self.program, self.array,
                                record.recordno, vals)

        self.nopts += 1

        env = {'nopts': self.nopts,
               'peak': 111,
               'peakpos': 34,
               'fwhm': 12.3,
               'fwhmpos': 45,
               'com': 23}

        self.putAllEnv(env)

    def _endRecordList(self, recordlist):
        if not self.isInitialized():
            return
        env = recordlist.getEnviron()
        self.putEnv('ended', env.get('endtime').ctime())

class ShmRecorder(DataRecorder):

    """ Sets data in shared memory to be used by sps """

    maxenv = 50
    envlen = 1024

    def setShmID(self, shmid):
        self.shm_id = shmid
        self.shm_id_ENV = shmid + "_ENV"

    def setShmMntGrp(self, mnt_grp):
        self.mnt_grp = mnt_grp

    def setProgram(self, progname):
        self.progname = progname

    def isInitialized(self):
        try:
            getattr(self, "shm_id")
            getattr(self, "shm_id_env")
            getattr(self, "progname")
            return True
        except:
            return False

    def setSize(self, rows, cols):
        self.rows = rows
        self.cols = cols

    def putenv(self, name, value):
        self.sps.putenv(self.progname, self.shm_id_env, name, str(value))

    def setChanDimList(self, chandimlist):
        self.chandimlist = chandimlist

    def _startRecordList(self, recordlist):

        if not self.isInitialized():
            return

        arraylist = self.sps.getarraylist(self.progname)

        if self.shm_id in arraylist:
            shm = self.sps.attach(self.progname, self.shm_id)
        else:
            self.sps.create(self.progname, self.shm_id, self.rows, self.cols,
                            self.sps.DOUBLE)

        if self.shm_id_env in arraylist:
            shm_env = self.sps.attach(self.progname, self.shm_id_env)
        else:
            self.sps.create(self.progname, self.shm_id_env, self.maxenv, self.envlen,
                            self.sps.STRING)

        print("Starting new SHM recording")

        self.putenv('title', recordlist.getEnvironValue('title'))

        for env, val in list(recordlist.getEnviron().items()):
            if env != 'title' and env != 'labels':
                self.putenv(env, val)

        self.nopts = 0

        self.putenv('started', time.ctime(
            recordlist.getEnvironValue('starttime')))
        self.putenv('ended', '')
        self.putenv('axistitles', ' '.join(
            recordlist.getEnvironValue('labels')))
        self.putenv('ylabel', 'Counts')
        self.putenv('nopts', self.nopts)
        self.putenv('xbeg', 100)
        self.putenv('xend', 200)
        self.putenv('aborted', 0)
        self.putenv('command', 'done')
        self.putenv('fitresult', '0')

        self.labels = recordlist.getEnvironValue('labels')

    def _writeRecord(self, record):
        # uhmm. only numeric values can be written

        if not self.isInitialized():
            return

        vals = []

        dim_list = []
        for colname in self.labels:
            dim_list.append(0)
            val = record.data.get(colname)
            if (val is not None) and (type(val) in [int, float]):
                vals.append(val)

        myj = 0

        for val2 in record.data:
            tmp = val2 + '_value'
            # esto me da el nombre del canal
            for dim in self.chandimlist:
                if tmp == dim:
                    dim_list[myj] = self.chandimlist[dim]
                    myj = myj + 1

        myj = 0

        for val2 in list(record.data.values()):
            valsmca = []
            if type(val2) in [list]:
                if dim_list[myj] == 1:
                    for i in range(0, len(val2)):
                        valsmca.append(val2[i])
                    tmp_name = self.mnt_grp + "_1D"
                    if self.shm_id == tmp_name:
                        valsmca = numpy.array(valsmca)
                        self.sps.putdatarow(
                            self.progname, self.shm_id, record.recordno, valsmca)
            myj = myj + 1

        vals = numpy.array(vals)
        tmp_name = self.mnt_grp + "_0D"
        if self.shm_id == tmp_name:
            self.sps.putdatarow(self.progname, self.shm_id,
                                record.recordno, vals)

        self.nopts += 1
        self.putenv('nopts', self.nopts)
        self.putenv('peak', 111)
        self.putenv('peakpos', 34)
        self.putenv('fwhm', 12.3)
        self.putenv('fwhmpos', 45)
        self.putenv('com', 23)

    def _endRecordList(self, recordlist):
        if not self.isInitialized():
            return
        self.putenv('ended', time.ctime(recordlist.getEnvironValue('endtime')))


text:
#!/usr/bin/python3

# Run the optimization step to place all the features and camera poses
# in a way that minimizes the mean reprojection error for the
# collective data set.

import argparse
import pickle
import cv2
import math
import numpy as np
import os

from lib import Groups
from lib import Optimizer
from lib import ProjectMgr
from lib import transformations

def optmizer(project_dir, optmize_options):

    group_id = optmize_options[0]
    refine = optmize_options[1]
    cam_calibration = optmize_options[2]

    d2r = math.pi / 180.0
    r2d = 180.0 / math.pi

    # return a 3d affine tranformation between current camera locations
    # and original camera locations.
    def get_recenter_affine(src_list, dst_list):
        print('get_recenter_affine():')
        src = [[], [], [], []]      # current camera locations
        dst = [[], [], [], []]      # original camera locations
        for i in range(len(src_list)):
            src_ned = src_list[i]
            src[0].append(src_ned[0])
            src[1].append(src_ned[1])
            src[2].append(src_ned[2])
            src[3].append(1.0)
            dst_ned = dst_list[i]
            dst[0].append(dst_ned[0])
            dst[1].append(dst_ned[1])
            dst[2].append(dst_ned[2])
            dst[3].append(1.0)
            # print("{} <-- {}".format(dst_ned, src_ned))
        A = transformations.superimposition_matrix(src, dst, scale=True)
        print("A:\n", A)
        return A

    # transform a point list given an affine transform matrix
    def transform_points( A, pts_list ):
        src = [[], [], [], []]
        for p in pts_list:
            src[0].append(p[0])
            src[1].append(p[1])
            src[2].append(p[2])
            src[3].append(1.0)
        dst = A.dot( np.array(src) )
        result = []
        for i in range(len(pts_list)):
            result.append( [ float(dst[0][i]),
                            float(dst[1][i]),
                            float(dst[2][i]) ] )
        return result

    proj = ProjectMgr.ProjectMgr(project_dir)
    proj.load_images_info()

    source_file = os.path.join(proj.analysis_dir, 'matches_grouped' )
    print('Match file:', source_file)
    matches = pickle.load( open(source_file, "rb") )
    print('Match features:', len(matches))

    # load the group connections within the image set
    groups = Groups.load(proj.analysis_dir)
    # sort from smallest to largest: groups.sort(key=len)

    opt = Optimizer.Optimizer(project_dir)
    opt.setup( proj, groups, group_id, matches, optimized=refine,
            cam_calib=cam_calibration)
    cameras, features, cam_index_map, feat_index_map, fx_opt, fy_opt, cu_opt, cv_opt, distCoeffs_opt = opt.run()

    # mark all the optimized poses as invalid
    for image in proj.image_list:
        opt_cam_node = image.node.getChild('camera_pose_opt', True)
        opt_cam_node.setBool('valid', False)

    for i, cam in enumerate(cameras):
        image_index = cam_index_map[i]
        image = proj.image_list[image_index]
        ned_orig, ypr_orig, quat_orig = image.get_camera_pose()
        print('optimized cam:', cam)
        rvec = cam[0:3]
        tvec = cam[3:6]
        Rned2cam, jac = cv2.Rodrigues(rvec)
        cam2body = image.get_cam2body()
        Rned2body = cam2body.dot(Rned2cam)
        Rbody2ned = np.matrix(Rned2body).T
        (yaw, pitch, roll) = transformations.euler_from_matrix(Rbody2ned, 'rzyx')
        #print "orig ypr =", image.camera_pose['ypr']
        #print "new ypr =", [yaw/d2r, pitch/d2r, roll/d2r]
        pos = -np.matrix(Rned2cam).T * np.matrix(tvec).T
        newned = pos.T[0].tolist()[0]
        print(image.name, ned_orig, '->', newned, 'dist:', np.linalg.norm(np.array(ned_orig) - np.array(newned)))
        image.set_camera_pose( newned, yaw*r2d, pitch*r2d, roll*r2d, opt=True )
        image.placed = True
    proj.save_images_info()
    print('Updated the optimized camera poses.')

    # update and save the optimized camera calibration
    proj.cam.set_K(fx_opt, fy_opt, cu_opt, cv_opt, optimized=True)
    proj.cam.set_dist_coeffs(distCoeffs_opt.tolist(), optimized=True)
    proj.save()

    # compare original camera locations with optimized camera locations and
    # derive a transform matrix to 'best fit' the new camera locations
    # over the original ... trusting the original group gps solution as
    # our best absolute truth for positioning the system in world
    # coordinates.
    #
    # each optimized group needs a separate/unique fit

    matches_opt = list(matches) # shallow copy
    refit_group_orientations = True
    if refit_group_orientations:
        group = groups[group_id]
        print('refitting group size:', len(group))
        src_list = []
        dst_list = []
        # only consider images that are in the current   group
        for name in group:
            image = proj.findImageByName(name)
            ned, ypr, quat = image.get_camera_pose(opt=True)
            src_list.append(ned)
            ned, ypr, quat = image.get_camera_pose()
            dst_list.append(ned)
        A = get_recenter_affine(src_list, dst_list)

        # extract the rotation matrix (R) from the affine transform
        scale, shear, angles, trans, persp = transformations.decompose_matrix(A)
        print('  scale:', scale)
        print('  shear:', shear)
        print('  angles:', angles)
        print('  translate:', trans)
        print('  perspective:', persp)
        R = transformations.euler_matrix(*angles)
        print("R:\n{}".format(R))

        # fixme (just group):

        # update the optimized camera locations based on best fit
        camera_list = []
        # load optimized poses
        for image in proj.image_list:
            if image.name in group:
                ned, ypr, quat = image.get_camera_pose(opt=True)
            else:
                # this is just fodder to match size/index of the lists
                ned, ypr, quat = image.get_camera_pose()
            camera_list.append( ned )

        # refit
        new_cams = transform_points(A, camera_list)

        # update position
        for i, image in enumerate(proj.image_list):
            if not image.name in group:
                continue
            ned, [y, p, r], quat = image.get_camera_pose(opt=True)
            image.set_camera_pose(new_cams[i], y, p, r, opt=True)
        proj.save_images_info()

        if True:
            # update optimized pose orientation.
            dist_report = []
            for i, image in enumerate(proj.image_list):
                if not image.name in group:
                    continue
                ned_orig, ypr_orig, quat_orig = image.get_camera_pose()
                ned, ypr, quat = image.get_camera_pose(opt=True)
                Rbody2ned = image.get_body2ned(opt=True)
                # update the orientation with the same transform to keep
                # everything in proper consistent alignment

                newRbody2ned = R[:3,:3].dot(Rbody2ned)
                (yaw, pitch, roll) = transformations.euler_from_matrix(newRbody2ned, 'rzyx')
                image.set_camera_pose(new_cams[i], yaw*r2d, pitch*r2d, roll*r2d,
                                    opt=True)
                dist = np.linalg.norm( np.array(ned_orig) - np.array(new_cams[i]))
                print('image: {}'.format(image.name))
                print('  orig pos: {}'.format(ned_orig))
                print('  fit pos: {}'.format(new_cams[i]))
                print('  dist moved: {}'.format(dist))
                dist_report.append( (dist, image.name) )
            proj.save_images_info()

            dist_report = sorted(dist_report,
                                key=lambda fields: fields[0],
                                reverse=False)
            print('Image movement sorted lowest to highest:')
            for report in dist_report:
                print('{} dist: {}'.format(report[1], report[0]))

        # tranform the optimized point locations using the same best
        # fit transform for the camera locations.
        new_feats = transform_points(A, features)

        # update any of the transformed feature locations that have
        # membership in the currently processing group back to the
        # master match structure.  Note we process groups in order of
        # little to big so if a match is in more than one group it
        # follows the larger group.
        for i, feat in enumerate(new_feats):
            match_index = feat_index_map[i]
            match = matches_opt[match_index]
            in_group = False
            for m in match[2:]:
                if proj.image_list[m[0]].name in group:
                    in_group = True
                    break
            if in_group:
                #print(' before:', match)
                match[0] = feat
                #print(' after:', match)
    else:
        # not refitting group orientations, just copy over optimized
        # coordinates
        for i, feat in enumerate(features):
            match_index = feat_index_map[i]
            match = matches_opt[match_index]
            match[0] = feat

    # write out the updated match_dict
    print('Updating matches file:', len(matches_opt), 'features')
    pickle.dump(matches_opt, open(source_file, 'wb'))

    #proj.cam.set_K(fx_opt/scale[0], fy_opt/scale[0], cu_opt/scale[0], cv_opt/scale[0], optimized=True)
    #proj.save()

    # temp write out just the points so we can plot them with gnuplot
    f = open(os.path.join(proj.analysis_dir, 'opt-plot.txt'), 'w')
    for m in matches_opt:
        try:
            f.write('%.2f %.2f %.2f\n' % (m[0][0], m[0][1], m[0][2]))
        except:
            pass
    f.close()

    # temp write out direct and optimized camera positions
    f1 = open(os.path.join(proj.analysis_dir, 'cams-direct.txt'), 'w')
    f2 = open(os.path.join(proj.analysis_dir, 'cams-opt.txt'), 'w')
    for name in groups[group_id]:
        image = proj.findImageByName(name)
        ned1, ypr1, quat1 = image.get_camera_pose()
        ned2, ypr2, quat2 = image.get_camera_pose(opt=True)
        f1.write('%.2f %.2f %.2f\n' % (ned1[1], ned1[0], -ned1[2]))
        f2.write('%.2f %.2f %.2f\n' % (ned2[1], ned2[0], -ned2[2]))
    f1.close()
    f2.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Keypoint projection.')
    parser.add_argument('--project', required=True, help='project directory')
    parser.add_argument('--group', type=int, default=0, help='group number')
    parser.add_argument('--refine', action='store_true', help='refine a previous optimization.')
    parser.add_argument('--cam-calibration', action='store_true', help='include camera calibration in the optimization.')

    args = parser.parse_args()

    optmize_options = [args.group, args.refine, args.cam_calibration]
    optmizer(args.project, optmize_options)

max_stars_repo_name: skywalkerisnull/ImageAnalysis

max_issues_repo_name: skywalkerisnull/ImageAnalysis

max_forks_repo_name: skywalkerisnull/ImageAnalysis


text:
#!/usr/bin/python

import os
import fnmatch
import sys
import re
import subprocess
import json
import numpy as np
#import natsort

def sortkey_natural(s):
    return tuple(int(part) if re.match(r'[0-9]+$', part) else part for part in re.split(r'([0-9]+)', s))

def get_sweep_xmls(data_path):

    file_list = []

    # Walk through directory
    for dname, sdname, files in os.walk(data_path):
        for filename in files:
            if fnmatch.fnmatch(filename, "room.xml"): # Match search string
                file_list.append(os.path.join(dname, filename))
    file_list = sorted(file_list, key=sortkey_natural)

    return file_list

def get_objects(s):

    dir_list = []

    sweep_path = os.path.abspath(os.path.join(os.path.abspath(s), os.path.pardir))
    objects_path = os.path.join(sweep_path, "consolidated_objects")
    if not os.path.exists(objects_path):
        return []
    for dname, sdname, files in os.walk(objects_path):
        for dirname in sdname:
            if fnmatch.fnmatch(dirname, "object*"):
                dir_list.append(os.path.join(dname, dirname))
    dir_list = sorted(dir_list, key=sortkey_natural)

    return dir_list

def inspect_objects(data_path):

    sweeps = get_sweep_xmls(data_path)

    backward_detected = 0
    forward_detected = 0
    backward_propagated = 0
    forward_propagated = 0

    for i, s in enumerate(sweeps):
        print s
        objects = get_objects(s)
        for o in objects:
            print i, o
            object_file = os.path.join(o, "segmented_object.json")
            if not os.path.exists(object_file):
                print object_file, "does not exist, skipping..."
                continue
            with open(object_file) as data_file:
                object_dict = json.load(data_file)['object']
            if object_dict['going_backward']:
                if object_dict['object_type'] == "detected":
                    backward_detected += 1
                elif object_dict['object_type'] == "propagated":
                    backward_propagated += 1
            else:
                if object_dict['object_type'] == "detected":
                    forward_detected += 1
                elif object_dict['object_type'] == "propagated":
                    forward_propagated += 1

    print "Nbr backward detected: ", backward_detected
    print "Nbr forward detected: ", forward_detected
    print "Nbr backward propagated: ", backward_propagated
    print "Nbr forward propagated: ", forward_propagated

if __name__ == '__main__':

    if len(sys.argv) < 2:
        print "Usage: ", sys.argv[0], " path/to/data"
    else:
        inspect_objects(sys.argv[1])


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sb
import os
abspath = os.path.abspath(__file__)
dname = os.path.dirname(abspath)
os.chdir(dname)

fuel_econ = pd.read_csv('./data/fuel_econ.csv')
# print(fuel_econ.head())

most_makes = fuel_econ['make'].value_counts().index[:18]
fuel_econ_sub = fuel_econ.loc[fuel_econ['make'].isin(most_makes)]

make_means = fuel_econ_sub.groupby('make').mean()
comb_order = make_means.sort_values('comb', ascending=False).index
g = sb.FacetGrid(data=fuel_econ_sub, col='make', col_wrap=6, size=2,
                 col_order=comb_order)
# try sb.distplot instead of plt.hist to see the plot in terms of density!
g.map(plt.hist, 'comb', bins=np.arange(12, fuel_econ_sub['comb'].max()+2, 2))
g.set_titles('{col_name}')
# plt.show()
plt.clf()

base_color = sb.color_palette()[0]
sb.barplot(data=fuel_econ_sub, x='comb', y='make',
           color=base_color, order=comb_order, ci='sd')
plt.show()

max_stars_repo_name: jpmassena/AIPND

max_issues_repo_name: jpmassena/AIPND

max_forks_repo_name: jpmassena/AIPND


text:
import pytest
import numpy as np
import pandas as pd

from tensortrade.rewards import RiskAdjustedReturns

@pytest.fixture
def net_worths():
    return pd.Series([100, 400, 350, 450, 200, 400, 330, 560], name="net_worth")

class TestRiskAdjustedReturns:

    def test_sharpe_ratio(self, net_worths):
        scheme = RiskAdjustedReturns(return_algorithm='sharpe', risk_free_rate=0)

        returns = net_worths.diff()

        sharpe_ratio = scheme._sharpe_ratio(returns)

        expected_ratio = returns.mean() / (returns.std() + 1E-9)

        assert sharpe_ratio == expected_ratio

    @pytest.mark.skip("Needs to be reevaluated.")
    def test_sortino_ratio(self, net_worths):
        scheme = RiskAdjustedReturns(
            return_algorithm='sortino',
            risk_free_rate=0,
            target_returns=0
        )

        returns = net_worths.diff()

        sortino_ratio = scheme._sortino_ratio(returns)

        expected_return = returns.mean()
        downside_std = returns[returns < 0].std()

        expected_ratio = expected_return / downside_std

        assert sortino_ratio == expected_ratio

max_stars_repo_name: bwcknr/tensortrade

max_issues_repo_name: bwcknr/tensortrade

max_forks_repo_name: bwcknr/tensortrade


text:
#!/usr/bin/env python3

################################################################################

from __future__ import print_function
from math import log10
import numpy as np
import matplotlib.pyplot as plt
from scipy import integrate
from math import log10, sqrt

from astropy import units as u
from astropy.coordinates import SkyCoord
from astroML.plotting import setup_text_plots

setup_text_plots(fontsize=20, usetex=True)

### CONSTANTS ###
omega_m   = 0.3
omega_lam = 0.7
H0        = 70.         # Hubble parameter at z=0, km/s/Mpc
c_kms     = 299792.458  # speed of light, km/s
dH        = c_kms / H0  # Hubble distance, Mpc

################################################################################

def inv_efunc(z):
    """ Used to calculate the comoving distance to object at redshift
        xs    z. Eqn 14 from Hogg, astro-ph/9905116."""
    return 1. / sqrt(omega_m * (1. + z)**3 + omega_lam)

################################################################################

def sizehist( data_fname='coma_vicinity.dat',
              udg_size_fname='re.dat',
              plot_fname='sizehist_final.pdf' ):

    #############################   SDSS GALAXIES   ############################

    # Load galaxy positions, magnitudes and redshifts and r-i colours.
    r = np.genfromtxt(data_fname, dtype=None, skip_header=2,
                      names='ra,dec,cz,g,dm1,re',
                      usecols=(0, 1, 2, 3, 5, 6))
    # Apply Redshift & Size Cuts
    condition1 = (r['cz']>0.015) & (r['cz']<0.0333) & (r['re']>0)
    r = r[condition1]
    #Calculate the comoving distance corresponding to each object's redshift
    dist = np.array([dH * integrate.quad(inv_efunc, 0, cz)[0] for cz in r['cz']])

    #################################   UDGS   #################################

    t = np.genfromtxt(udg_size_fname, dtype=None, skip_header=0,
                      names='re',usecols=(0))
    dm   = 5*log10(98.0*1020000.0) - 5
    rlen = len(r['g'])

    for i in range(0,rlen):
        r['dm1'][i] = (5*log10(dist[i]*1020000.0) - 5)

    r['g']  = r['g'] - r['dm1']
    r['re'] = (r['re']*1000*dist/206265.0)

    condition3 = (r['g']<-17) & (r['g']>-20) & (r['re']>2.4) & (r['re']<10)
    r = r[condition3]

    ############################   SIZE HISTOGRAM   ############################

    # Create Figure
    fig = plt.figure(figsize=(7.5, 7.5))
    ax = fig.add_subplot(111)
    fig.subplots_adjust(bottom=0.2, top=0.9,
                        left=0.2, right=0.9, wspace=None, hspace = None)
    # SDSS Galaxies
    n, bins, patches = plt.hist(r['re'], 25, density=1, histtype='stepfilled',
                                facecolor='darkseagreen', alpha=0.45)
    # UDGs
    n, bins, patches = plt.hist(t['re'],10, density=1, facecolor='r', alpha=0.3)
                               # facecolor='darkslategrey', alpha=0.45)
    # Plot Formatting
    plt.tick_params(which='both', direction='in', pad =10, labelsize=20)
    plt.yscale('log')
    plt.ylabel(r'log(Normalized Number)')
    plt.xlabel(r'$r_e$ (kpc)')
    plt.axis([2.4,10,0.001,3])

    # Save Figure
    plt.tight_layout()
    plt.savefig(plot_fname, bbox_inches='tight')
    plt.show()

################################################################################

if __name__=="__main__":

    sizehist( data_fname='../data/coma_vicinity.dat',
              udg_size_fname='../data/re.dat',
              plot_fname='../plots/sizehist_final.pdf' )

################################################################################

reason: import numpy,from scipy,from astropy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 20 10:44:15 2019

@author: dipu
"""
import numpy as np
from PIL import Image
import time

#%% IoU Classwise

def get_overall_Classwise_IOU(boundingBoxes,sort_inds,g_fnames,q_fnames, topk = [1,5,10,20,40]):
    #data_dir = '/mnt/amber/scratch/Dipu/RICO/semantic_annotations/'
    n_topk = max(topk)
    n_query = len(q_fnames)

    avgClassIouArray = np.zeros((n_query,n_topk))
    weightedClassIouArray = np.zeros((n_query,n_topk))
    allClasses = boundingBoxes.getClasses()
    classwiseClassIou = dict([(key, []) for key in allClasses])

    for i in  range((sort_inds.shape[0])):   #Iterate over all the query images
        ts = time.time()
        qImageName = q_fnames[i]
        #q_img  =  Image.open(data_dir+qImageName+'.png').convert('RGB')
        #q_img_size = q_img.size
        q_img_size = (1440, 2560)
        qBBoxes = boundingBoxes.getBoundingBoxesByImageName(qImageName)
        qClasses = list(set([d.classId for d in qBBoxes]))

        for j in range(n_topk):     # Iterate over top-5 retrieved images
            rImageName = g_fnames[sort_inds[i][j]]
            rBBoxes = boundingBoxes.getBoundingBoxesByImageName(rImageName)

            iouTemp = []
            weights = []

            #Iterate over each element(boudingbox)
            for c in qClasses:                               # qbbs query bounding boxes
                mask1 = np.zeros(q_img_size, dtype=np.uint8)
                mask2 = np.zeros(q_img_size).astype(np.uint8)

                c_qboxes = [b for b in qBBoxes if b.classId == c]
                c_rboxes = [b for b in rBBoxes if b.classId == c]

                for cqbox in c_qboxes:
                    bb = cqbox.getBoundingBox()
                    mask1[bb[0] : bb[0]+bb[2], bb[1] : bb[1]+bb[3]] = 1
    #            ax[0,1].imshow(Image.fromarray(np.transpose(mask1)))

                for crbox in c_rboxes:
                    bb = crbox.getBoundingBox()
                    mask2[bb[0]: bb[0]+ bb[2], bb[1] : bb[1]+bb[3]] = 1

                intersec = np.sum(np.logical_and(mask1, mask2))
                union = np.sum(np.logical_or(mask1, mask2))
                iou_c = intersec/union

                iouTemp.append(iou_c)
                weights.append(np.sum(mask1))

                # Accumuate the pixel acc for all classes
                classwiseClassIou[c].append(iou_c)

            avgClassIouArray[i][j] = np.mean(iouTemp)

            weightTotal = np.sum(weights)
            weightvalues = np.divide(weights, weightTotal)
            weightedClassIouArray[i][j] = np.sum(iouTemp*weightvalues)

        #print('Time for query{} = {}'.format(i, time.time()-ts))
        #print('Computing Classwise IoU: {}/{} in time {}'.format(i,n_query, time.time()-ts))
        #ts = time.time()

    overallMeanClassIou_list = []
    overallMeanWeightedClassIou_list = []

    for k in topk:
        meanAvgPixAcc = np.mean(avgClassIouArray[:,:k], axis=1)
        overallMeanClassIou = np.mean(meanAvgPixAcc)
        overallMeanClassIou_list.append(overallMeanClassIou)

        meanWeightedPixAcc = np.mean(weightedClassIouArray[:,:k], axis=1)
        overallMeanWeightedClassIou = np.mean(meanWeightedPixAcc)
        overallMeanWeightedClassIou_list.append(overallMeanWeightedClassIou)
    print('Completed computing Classwise IoU: {}/{}'.format(i+1,n_query))

    return overallMeanClassIou_list, overallMeanWeightedClassIou_list, avgClassIouArray

max_stars_repo_name: exnx/gcn-triplet-encoding

max_issues_repo_name: exnx/gcn-triplet-encoding

max_forks_repo_name: exnx/gcn-triplet-encoding


text:
import PanoramAI as PA
import numpy as np
import numpy.testing as npt
import pytest

def test_VAEorama_smoketest():
    M, N = 8, 8
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data)

def test_attributes():
    M, N = 16, 128
    LD = 100
    BS = 25
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data, latent_dim = LD, BATCH_SIZE = BS)
    npt.assert_equal(V.dimensions, [M, N])
    npt.assert_equal(V.latent_dim, LD)
    npt.assert_equal(V.BATCH_SIZE, BS)

def test_training():
    M, N = 16, 128
    LD = 100
    BS = 25
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data, latent_dim = LD, BATCH_SIZE = BS)
    V.train(1)

def test_sample():
    M, N = 16, 128
    LD = 100
    BS = 25
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data, latent_dim = LD, BATCH_SIZE = BS)

    examples = V.generate_samples(1)[0]
    assert examples.shape == (M, N, 3)

def test_encode_decode():
    #Test shapes of the encode->decode process
    M, N = 16, 128
    LD = 100
    BS = 25
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data, latent_dim = LD, BATCH_SIZE = BS)

    for x in V.train_dataset:
        mean, logvar = V.model.encode(x)
        npt.assert_equal(
            mean.get_shape().as_list(), (V.BATCH_SIZE, V.latent_dim,))
        npt.assert_equal(
            logvar.get_shape().as_list(), (V.BATCH_SIZE, V.latent_dim,))
        z = V.model.reparameterize(mean, logvar)
        npt.assert_equal(z.shape, mean.shape)

        xprime = V.model.decode(z)
        npt.assert_equal(x.shape, xprime.shape)
        break

def test_save_and_load():
    import os
    M, N = 16, 128
    LD = 100
    BS = 25
    data = np.random.randn(128, M, N, 3)
    V = PA.VAEorama(data, latent_dim = LD, BATCH_SIZE = BS)
    V.save_model_weights("temp/")
    V.load_model_weights("temp/")
    res = os.system("rm -rf temp")
    npt.assert_equal(res, 0)

if __name__ == "__main__":
    #test_VAEorama_smoketest()
    #test_attributes()
    test_training()

max_stars_repo_name: tmcclintock/GANorama

max_issues_repo_name: tmcclintock/GANorama

max_forks_repo_name: tmcclintock/GANorama


text:
import os
import numpy as np
from PIL import Image

class Augmentation(object):
	def __init__(self, data_dir, flip=True):
		super(Augmentation, self).__init__()
		self.data_dir = data_dir
		self.flip = flip
		self.array_data = []
		self.flip_data = []
		self.augment_data = []
		self.shifting_rate = 1e-1

	def get_data(self):
		try:
			return os.listdir(self.data_dir)
		except FileNotFoundError as e:
			print("Error! Directory Not Found ")

	def data2array(self):
		data_list = self.get_data()
		for data in data_list:
			img = Image.open(os.path.join(self.data_dir, data))
			img = np.array(img)
			img = img[:,:,:3]
			self.array_data.append(img)
			self.augment_data.append(img)

	def flipped(self):
		for data in self.array_data:
			flipped_img = np.fliplr(data)
			self.flip_data.append(flipped_img)
			self.augment_data.append(flipped_img)

	def Shift_Top_left(self, data):
		self.augment_data.append(data[:-int(data.shape[0] * self.shifting_rate),
									  :-int(data.shape[0] * self.shifting_rate)])

	def Shift_Bottom_Right(self, data):
		self.augment_data.append(data[:int(data.shape[0] * self.shifting_rate),
									   int(data.shape[0] * self.shifting_rate):])

	def Shift_Center(self, data):
		self.augment_data.append(
						data[int(data.shape[0] * self.shifting_rate / 2):-int(data.shape[0] * self.shifting_rate / 2),
							 int(data.shape[0] * self.shifting_rate / 2):-int(data.shape[0] * self.shifting_rate / 2)])

	def main(self):
		self.get_data()
		self.data2array()

		if self.flip == True:
			self.flipped()
			for flip_img in self.flip_data:
				self.Shift_Top_left(flip_img)
				self.Shift_Bottom_Right(flip_img)
				self.Shift_Center(flip_img)

		for iter_data in self.array_data:
			self.Shift_Top_left(iter_data)
			self.Shift_Bottom_Right(iter_data)
			self.Shift_Center(iter_data)

if __name__ == '__main__':
	data_path = './data'
	arg = Augmentation(data_path)
	arg.main()

max_stars_repo_name: tasbirul/Deep-Learning-With-Keras

max_issues_repo_name: tasbirul/Deep-Learning-With-Keras

max_forks_repo_name: tasbirul/Deep-Learning-With-Keras


text:
from torcherist.dataloader.text import *
from torcherist.dataset.text import read_ptb
import pytest
import numpy as np
import random

def test_sequential_text_sampler():
    my_seq = np.arange(35)
    for X, y in sequential_text_sampler(my_seq, batch_size=2, m=5):
        X, y = X.numpy(), y.numpy()
        assert np.all(X + 1 == y)
    prev_seq_end = None
    for X, y in sequential_text_sampler(my_seq, by="sequential", batch_size=2, m = 5):
        X, y = X.numpy(), y.numpy()
        if prev_seq_end is not None:
            assert np.all(prev_seq_end + 1 == X[:,0])
        prev_seq_end = X[:,-1]
        assert np.all(X + 1 == y)

def test_time_machine():
    data_iter, vocab = load_time_machine(batch_size=32,
                                         m=35,
                                         by="random")
    # 26 en char & empty space & <unk>
    assert len(vocab) == 28
    for X, y in data_iter:
        # for each batch
        for XX, yy in zip(X, y):
            XX, yy = XX.numpy().flatten(), yy.numpy().flatten()
            assert len(XX) == len(yy)
            assert np.all(XX[1:] == yy[:-1])

    data_iter, vocab = load_time_machine(batch_size=32,
                                         m=35,
                                         by="sequential")
    # 26 en char & empty space & <unk>
    assert len(vocab) == 28
    for X, y in data_iter:
        # for each batch
        for XX, yy in zip(X, y):
            XX, yy = XX.numpy().flatten(), yy.numpy().flatten()
            assert len(XX) == len(yy)
            assert np.all(XX[1:] == yy[:-1])

    # test if data_iter can be iter multiple times
    for X, y in data_iter:
        # for each batch
        for XX, yy in zip(X, y):
            XX, yy = XX.numpy().flatten(), yy.numpy().flatten()
            assert len(XX) == len(yy)
            assert np.all(XX[1:] == yy[:-1])


text:
import numpy as np
import platform
import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib.ticker as mticker
import matplotlib.gridspec as gridspec
from tqdm import tqdm
from scipy import constants
import h5py
import os
import pnptransport.utils as utils
# import shutil
import itertools
import pandas as pd

base_path = r'G:\My Drive\Research\PVRD1\Manuscripts\PNP_Draft\simulations\SiO2_figure'
output_file = r'sio2_vfb_different_temperatures.h5'
pnp_file = r'G:\My Drive\Research\PVRD1\Manuscripts\PNP_Draft\simulations\SiO2_figure\MU3_SiOx_CV2_VS_1C_D69D70D71D72_4Na_D69D70D71D72_smoothed_cont_minus_clean_5_test1.h5'

er = 3.9
thickness = 100E-7  # cm
experimental_files = [
    {
        'filename': r'50C_MU2_SiOx_CV2_VS_1C_D29D30D31D32_4Na_D29D30D31D32 - Copy_smoothed_cont_minus_clean_5_erf_fit.h5',
        'temperature': 50.,
        'sqrt_t_sat': 2.4
    },
    {
        'filename': r'55C_MU3_SiOx_CV2_VS_1C_D93D94D95D96_4Na_D93D94D95D96 - Copy_smoothed_cont_minus_clean_8_erf_fit.h5',
        'temperature': 55.,
        'sqrt_t_sat': 2.0
    },
    {
        'filename': r'60C_MU3_SiOx_CV2_VS_1C_D69D70D71D72_4Na_D69D70D71D72_smoothed_cont_minus_clean_5_erf_fit.h5',
        'temperature': 60,
        'sqrt_t_sat': 2.84
    },
    {
        'filename': r'70C_fast_MU3_SiOx_CV2_VS_1C_D57D58D59D60_4Na_D57D58D59D60_smoothed_cont_minus_clean_8_erf_fit.h5',
        'temperature': 70,
        'sqrt_t_sat': 1.7
    },
    # {
    #     'filename': r'70C_slow_MU3_SiOx_CV2_VS_1C_D57D58D59D60_4Na_D57D58D59D60_smoothed_cont_minus_clean_7_erf_fit.h5',
    #     'temperature': 70
    # }
]

defaultPlotStyle = {
    'font.size': 11,
    'font.family': 'Arial',
    'font.weight': 'regular',
    'legend.fontsize': 11,
    'mathtext.fontset': 'stix',
    'xtick.direction': 'in',
    'ytick.direction': 'in',
    'xtick.major.size': 4.5,
    'xtick.major.width': 1.75,
    'ytick.major.size': 4.5,
    'ytick.major.width': 1.75,
    'xtick.minor.size': 2.75,
    'xtick.minor.width': 1.0,
    'ytick.minor.size': 2.75,
    'ytick.minor.width': 1.0,
    'xtick.top': False,
    'ytick.right': False,
    'lines.linewidth': 2.5,
    'lines.markersize': 10,
    'lines.markeredgewidth': 0.85,
    'axes.labelpad': 5.0,
    'axes.labelsize': 12,
    'axes.labelweight': 'regular',
    'legend.handletextpad': 0.2,
    'legend.borderaxespad': 0.2,
    'axes.linewidth': 1.25,
    'axes.titlesize': 12,
    'axes.titleweight': 'bold',
    'axes.titlepad': 6,
    'figure.titleweight': 'bold',
    'figure.dpi': 100
}

vfb_dtype_2 = np.dtype([
    ('time_s', 'd'),
    ('vsh', 'd'),
    ('vsh_std', 'd'),
    ('Qs', 'd'),
    ('Q0', 'd')
])

def normalize_vfb(v_fb: np.ndarray) -> np.ndarray:
    return -(v_fb - v_fb[0]) / (v_fb.max() - v_fb[0])

def estimate_vfb_norm_error_factor(
        v_fb: np.ndarray, v_fb_std: np.ndarray, v_fb_sat: float, v_fb_sat_std: float
) -> np.ndarray:
    x = v_fb
    x_sat = v_fb_sat
    dx = v_fb_std
    dx_sat = v_fb_sat_std
    factor = np.empty_like(v_fb)
    for i in range(len(x)):
        xx = x[i] if i > 0 else x[1]
        var = np.array([dx[i] / xx, dx_sat / x_sat])
        factor[i] = np.sqrt(var.T.dot(var))
    return factor

marker = itertools.cycle(('o', 's', '^', 'v', '>', '<', 'd', 'p', 's', ',', '+', '.', '*'))

if __name__ == '__main__':
    if platform.system() == 'Windows':
        base_path = r'\\?\\' + base_path

    # Load the style
    mpl.rcParams.update(defaultPlotStyle)

    n_files = len(experimental_files)
    cmap = mpl.cm.get_cmap('cool')
    normalize = mpl.colors.Normalize(vmin=0, vmax=n_files)
    plot_colors = [cmap(normalize(i)) for i in range(n_files)]

    fig_s = plt.figure(1)
    fig_s.set_size_inches(7.2, 3.0, forward=True)
    fig_s.subplots_adjust(hspace=0.1, wspace=0.1)
    gs_s_0 = gridspec.GridSpec(ncols=1, nrows=1, figure=fig_s)
    gs_s_00 = gridspec.GridSpecFromSubplotSpec(
        nrows=1, ncols=2, subplot_spec=gs_s_0[0], hspace=0.1, wspace=0.6
    )

    ax_c_0 = fig_s.add_subplot(gs_s_00[0, 0])
    # Set the axis labels
    ax_c_0.set_xlabel(r'Depth ($\mathregular{\mu}$m)')
    # ax_c_0.set_ylabel(r'${C}$ ($\mathregular{cm^{-3}}$)')
    ax_c_0.set_ylabel(r'${C/C_0}$')
    ax_c_0.set_title('(a)')

    # Make the y axis log
    ax_c_0.set_yscale('log')
    # Set the ticks for the y axis
    ax_c_0.yaxis.set_major_locator(mpl.ticker.LogLocator(base=10.0, numticks=6))
    ax_c_0.yaxis.set_minor_locator(mpl.ticker.LogLocator(base=10.0, numticks=60, subs=np.arange(2, 10) * .1))
    # ax_c_0.set_ylim(1E14, 1E20)
    ax_c_0.set_ylim(1E-4, 50.0)
    # Set the ticks for the x axis
    # Configure the ticks for the x axis
    ax_c_0.xaxis.set_major_locator(mticker.MaxNLocator(6, prune=None))
    ax_c_0.xaxis.set_minor_locator(mticker.AutoMinorLocator(2))
    color_map = 'viridis_r'
    # Get the color map
    cm = mpl.cm.get_cmap(color_map)

    with h5py.File(pnp_file, 'r') as hf:
        # Get the time dataset
        time_s = np.array(hf['time'])
        # Get the sinx group
        grp_sinx = hf['L1']
        # Get the position vector in SiNx in nm
        x_sin = np.array(grp_sinx['x'])
        thickness_sin = np.max(x_sin)
        n_profiles = len(time_s)
        normalize = mpl.colors.Normalize(vmin=1E-3, vmax=(np.amax(time_s / 3600)))
        # Get a 20 time points geometrically spaced
        requested_time = utils.geometric_series_spaced(max_val=np.amax(time_s), min_delta=240, steps=15)
        # requested_time = np.linspace(0.0, np.amax(time_s), 50)
        requested_indices = utils.get_indices_at_values(x=time_s, requested_values=requested_time)
        time_profile = np.empty(len(requested_indices))
        C0 = np.array(grp_sinx['concentration']['ct_0'])
        c0 = C0[0]

        print(requested_indices)

        model_colors = [cm(normalize(t)) for t in time_s / 3600.]
        scalar_maps = mpl.cm.ScalarMappable(cmap=cm, norm=normalize)
        with tqdm(requested_indices, leave=True, position=0) as pbar:
            for j, idx in enumerate(requested_indices):
                time_j = time_s[idx] / 3600.
                time_profile[j] = time_j
                # Get the specific profile
                ct_ds = 'ct_{0:d}'.format(idx)
                c_sin = np.array(grp_sinx['concentration'][ct_ds])
                color_j = cm(normalize(time_j))
                ax_c_0.plot(x_sin, c_sin/c0, color=color_j, zorder=0)
                pbar.set_description('Extracting profile {0} at time {1:.1f} h...'.format(ct_ds, time_j))
                pbar.update()
                pbar.refresh()

    # Set the limits for the x axis of the concentration plot
    ax_c_0.set_xlim(left=np.amin(x_sin), right=np.amax(x_sin))
    # Add the color bar
    from mpl_toolkits.axes_grid1 import make_axes_locatable
    divider = make_axes_locatable(ax_c_0)
    cax = divider.append_axes("right", size="7.5%", pad=0.03)
    cbar = fig_s.colorbar(scalar_maps, cax=cax)
    cbar.set_label(r'$t$ (h)', rotation=90, fontsize=14)
    cbar.ax.tick_params(labelsize=11)
    cbar.ax.yaxis.set_major_locator(mticker.MaxNLocator(6, prune=None))

    ax_s_0 = fig_s.add_subplot(gs_s_00[0, 1])
    # Set the axis labels
    # ax_s_0.set_xlabel(r'$\sqrt{t}$ (h$^{1/2}$)')
    # ax_s_0.set_ylabel(r"$Q'_s/q$ ($10^{11}/\mathregular{cm}^2$)")
    ax_s_0.set_xlabel(r'$t$ (h)')
    ax_s_0.set_ylabel(r"$Q_s/Q_0$")
    ax_s_0.set_title('(b)')

    # Configure the ticks for the x axis
    ax_s_0.xaxis.set_major_locator(mticker.MaxNLocator(6, prune=None))
    ax_s_0.xaxis.set_minor_locator(mticker.AutoMinorLocator(2))

    # Create the output storage file
    output_h5 = os.path.join(base_path, output_file)
    if os.path.exists(output_h5):
        os.remove(output_h5)

    # A dataset with all the vfb data for different temperatures (assuming vfb was
    # taken at the same times)
    vfb_df = pd.DataFrame()
    average_vfb_norm_cols = []
    average_vfb_std_norm_cols = []

    for i, f in enumerate(experimental_files):
        fn = f['filename']
        temperature = f['temperature']
        full_path = os.path.join(base_path, fn)
        print("Trying to read file \n'{0}'.".format(full_path))
        with h5py.File(full_path, 'r') as hf_exp:
            exp_ds = np.array(hf_exp['/vfb_data'])
            bias = hf_exp['/vfb_data'].attrs['stress_bias']
            temp_c = hf_exp['/vfb_data'].attrs['temp_c']
            thickness = hf_exp['/vfb_data'].attrs['thickness']
            time_s = exp_ds['time_s']
            if i == 0:
                vfb_df['time (s)'] = time_s
            # print("Length of vfb dataset: {0:d}".format(len(time_s)))
            # print("Time point: ")
            # print(time_s)

            sqrt_t = np.sqrt(time_s / 3600.)
            Qs = exp_ds['vsh'] * constants.epsilon_0 * er / thickness / 100.
            idx_sat = np.abs(sqrt_t - f['sqrt_t_sat']).argmin()
            Q0 = np.mean(Qs[sqrt_t >= f['sqrt_t_sat']])
            vsh = exp_ds['vsh']
            vsh_std = exp_ds['vsh_std']
            vfb_sat = np.mean(vsh[sqrt_t >= f['sqrt_t_sat']])
            vfb_sat_std = np.linalg.norm(vsh_std[sqrt_t >= f['sqrt_t_sat']])
            print('vsh_sat = {0:.4f} ± {1:.4f}'.format(vfb_sat, vfb_sat_std))
            vfb_col = 'vsh @ {0}C'.format(temperature)
            vfb_std_col = 'vsh_std @ {0}C'.format(temperature)
            vfb_norm_col = 'vsh_norm @ {0}C'.format(temperature)
            vfb_std_norm_col = 'vsh_norm_std @ {0}C'.format(temperature)
            average_vfb_norm_cols.append(vfb_norm_col)
            average_vfb_std_norm_cols.append(vfb_std_norm_col)
            vfb_df[vfb_col] = exp_ds['vsh']
            vfb_df[vfb_std_col] = exp_ds['vsh_std']
            vfb_norm_std = np.sign(vfb_sat) * exp_ds['vsh'] / vfb_sat
            vfb_df[vfb_norm_col] = vfb_norm_std
            vfb_df[vfb_std_norm_col] = estimate_vfb_norm_error_factor(
                v_fb=exp_ds['vsh'], v_fb_std=exp_ds['vsh_std'], v_fb_sat=vfb_sat,
                v_fb_sat_std=vfb_sat_std
            ) * vfb_norm_std
            vfb_attrs = {}
            for k in list(hf_exp['vfb_data'].attrs):
                vfb_attrs[k] = hf_exp['vfb_data'].attrs[k]

        if os.path.exists(output_h5):
            h5_mode = 'a'
        else:
            h5_mode = 'w'
        with h5py.File(output_h5, h5_mode) as hf_out:
            output_ds = np.empty(len(time_s), dtype=vfb_dtype_2)
            for j in range(len(time_s)):
                output_ds[j] = (exp_ds['time_s'][j], exp_ds['vsh'][j], exp_ds['vsh_std'][j], Qs[j], Q0)
            qs_ds = hf_out.create_dataset(
                name='{0:.0f}C_{1:d}'.format(temperature, i), data=output_ds, compression='gzip'
            )
            # for a in list(vfb_attrs):
            #     print('Saving attribute \'{0}\' = {1}'.format(a, vfb_attrs[a]))
            #     qs_ds.attrs[a] = vfb_attrs[a]

    vfb_df['vfb_norm_temp_mean'] = vfb_df[average_vfb_norm_cols].mean(axis=1)
    vfb_df['vfb_norm_temp_mean_std'] = vfb_df[average_vfb_norm_cols].std(axis=1)
    print(vfb_df)
    # ax_s_0.errorbar(
    #     time_s / 3600., Qs / abs(Q0), yerr=(2 ** 0.5) * exp_ds['vsh_std'], color=plot_colors[i],
    #     marker=next(marker), fillstyle='none', ls='none', capsize=4, elinewidth=1.5,
    #     label='{0:.0f} °C'.format(temperature), zorder=i
    # )
    ax_s_0.errorbar(
        time_s / 3600., np.abs(vfb_df['vfb_norm_temp_mean']), yerr=vfb_df['vfb_norm_temp_mean_std'],
        color='C0', marker=next(marker), fillstyle='none', ls='none', capsize=4,
        elinewidth=1.5, label='Experiment', zorder=0
    )

    # Plot the simulated vfb
    h5_pnp = os.path.join(base_path, pnp_file)
    with h5py.File(h5_pnp, 'r') as hf:
        # Get the time dataset
        time_sim = np.array(hf['time'])
        vfb = -np.array(hf['vfb'])  # / L * 1E4
        ax_s_0.plot(
            time_sim / 3600., (vfb - vfb[0]) / (vfb.max() - vfb[0]), color='tab:red', label='Model',
            zorder=n_files
        )

    leg = ax_s_0.legend(loc='lower right', frameon=True)
    fig_s.tight_layout()
    fig_s.savefig(os.path.join(base_path, 'qs_sio2_50-70C.png'), dpi=600)
    fig_s.savefig(os.path.join(base_path, 'qs_sio2_50-70C.svg'), dpi=600)
    fig_s.savefig(os.path.join(base_path, 'qs_sio2_50-70C.eps'), dpi=600)
    plt.show()

max_stars_repo_name: erickmartinez/pnptransport

max_issues_repo_name: erickmartinez/pnptransport

max_forks_repo_name: erickmartinez/pnptransport

reason: import numpy,from scipy


text:
import matplotlib.pyplot as plt
import numpy as np
import math
import sys
import os
import itertools

cur_dir = os.getcwd()

files = [_ for _ in os.listdir(cur_dir) if '.dat' in _]
# print(files)

data = {}
for file in files:
    x, y = [], []
    # print(cur_dir + '/file')
    with open(cur_dir + '/' + file, 'r') as rf:
        for _, line in enumerate(rf):
            line = line.strip().split('\t')
            print(line)
            r, gr = float(line[0]), float(line[1])
            x.append(r)
            y.append(gr)
        data[file[:len(file) - 6]] = (x, y)

# print(data)
f, ax = plt.subplots(figsize=(9.5, 7.5))
for ele in sorted(data.keys()):
    x, y = data[ele][0], data[ele][1]
    # print(x, y)
    ax.plot(x, y, label=str(ele), linestyle='-')
plt.xlabel('r/meters', fontsize=30)
plt.ylabel('g(r)', fontsize=30)
plt.axhline(y=1, linestyle='-', color='k')
plt.xticks(fontsize=25, rotation=0)
plt.yticks(fontsize=20, rotation=0)
plt.title('MC simulation', fontsize=30)
plt.legend(fontsize=15, ncol=1)
plt.xlim([-0.2, 6.0])
plt.ylim([-0.2, 4.0])
plt.show()


text:
"""
classify.py
"""

import re
import pickle
import configparser
from collections import Counter, defaultdict
from itertools import chain, combinations
import glob
import numpy as np
import os
from scipy.sparse import csr_matrix
from sklearn.cross_validation import KFold
from sklearn.linear_model import LogisticRegression

def get_unique_tweets(filename):
    print("getting unique tweets from pickle file")
    readtweets = open(filename, 'rb')
    tweets = pickle.load(readtweets)
    readtweets.close()
    utlist = set()
    for t in tweets:
        utlist.add(t['text'].encode('utf8').decode('unicode_escape').encode('ascii','ignore').decode("utf-8"))
    print("found %d unique tweets from file" % len(utlist))

    return list(utlist)

def get_afinn_sentiment(affin_filename):
    print("forming pos and neg word list from affin sentiment file")
    pos = []
    neg = []
    with open(affin_filename) as f:
        for line in f:
            tl = re.split(r'\t+', line.rstrip('\t'))
            if int(tl[1]) > 0:
                pos.append(tl[0].encode('ascii','ignore').decode("utf-8"))
            elif int(tl[1]) < 0:
                neg.append(tl[0].encode('ascii','ignore').decode("utf-8"))

    return pos,neg

def dsum(*dicts):
    ret = defaultdict(int)
    for d in dicts:
        for k, v in d.items():
            ret[k] += v
    return dict(ret)

def read_data(path):
    fnames = sorted([f for f in glob.glob(os.path.join(path, 'pos', '*.txt'))])
    data = [(1, open(f).readlines()[0]) for f in sorted(fnames[:1000])]
    fnames = sorted([f for f in glob.glob(os.path.join(path, 'neg', '*.txt'))])
    data += [(0, open(f).readlines()[0]) for f in sorted(fnames[:1000])]
    data = sorted(data, key=lambda x: x[1])
    return np.array([d[1] for d in data]), np.array([d[0] for d in data])

def tokenize(doc):
    tnp = []
    for x in doc.lower().split():
        tnp.append(re.sub('^\W+', '',re.sub('\W+$', '',x)))
    #return tnp
    return np.array(tnp)

def token_features(tokens, feats,pos,neg):
    feats.update(dsum(dict(Counter(Counter(["token=" + s for s in tokens]))),feats))

def token_pair_features(tokens,feats,pos,neg):
    k=3
    for i in range(len(tokens)-k+1):
        for e in list(combinations(list(tokens[i:k+i]), 2)):
            feats['token_pair='+e[0]+'__'+e[1]] += 1

def lexicon_features(tokens,feats,pos_words,neg_words):
    feats.update(dsum(dict(Counter({'pos_words': len([x for x in list(s.lower() for s in tokens) if x in pos_words]) , 'neg_words' : len([x for x in list(s.lower() for s in tokens) if x in neg_words]) })),feats))

def featurize(tokens,feature_fns,pos,neg):
    feats = defaultdict(lambda : 0)
    for fn in feature_fns:
        fn(tokens,feats,pos,neg)
    return sorted(list(feats.items()), key=lambda x: (x[0]))

def vectorize(tokens_list,pos,neg,vocab=None):
    feature_fns = [token_pair_features,lexicon_features]
    #feature_fns = [token_pair_features,lexicon_features,token_features]
    min_freq = 2

    vf = []
    vocabSec = {}

    for t in tokens_list:
        vf.append(list(featurize(t,feature_fns,pos,neg)))

    if vocab is None:
        vocabSec = {i:x for x,i in enumerate(sorted(list([k for k,v in dict(Counter(list([e[0] for e in list(chain(*vf)) if e[1]>0]))).items() if v >=min_freq])))}
    else:
        vocabSec = vocab

    column=[]
    data=[]
    rows=[]

    row=0
    for f in vf:
        for e in f:
            if e[0] in vocabSec:
                rows.append(row)
                column.append(vocabSec[e[0]])
                data.append(e[1])
        row+=1

    data=np.array(data,dtype='int64')
    rows=np.array(rows,dtype='int64')
    column=np.array(column,dtype='int64')
    X=csr_matrix((data, (rows,column)), shape=(len(tokens_list), len(vocabSec)))

    #print (X.toarray())
    #print (sorted(vocabSec.items(), key=lambda x: x[1]))

    # for x in vocabSec:
    #     x1 = re.sub('token=', '', x)
    #     line = re.sub('token_pair=', '', x1)
    #     print(line.encode('ascii','ignore').decode("utf-8"))

    return X,vocabSec

def fit_train_classifier(docs, labels,pos,neg):
    tokens_list = [ tokenize(d) for d in docs ]
    X,vocab = vectorize(tokens_list,pos,neg)
    model = LogisticRegression()
    model.fit(X,labels)

    return model,vocab

def parse_test_data(tweets,vocab,pos,neg):
    tokenslist = [ tokenize(d) for d in tweets ]
    X_test,vocb=vectorize(tokenslist,pos,neg,vocab)
    return X_test

def print_classification(tweets,X_test,clf):
    predicted = clf.predict(X_test)
    saveclassifydata = {}
    print("Number of pos and neg tweets: "+str(Counter(predicted)))
    for idx,t in  enumerate(tweets[:10]):
        label = ''
        if predicted[idx] == 1:
            label = "Positive"
            saveclassifydata['positivetweet'] = t
        elif predicted[idx] == 0:
            label = "Negative"
            saveclassifydata['negativetweet']  = t
        print("Classified as : %s Tweet Text: %s " % (label,t))

    saveclassifydata['pos'] = dict(Counter(predicted))[1]
    saveclassifydata['neg'] = dict(Counter(predicted))[0]

    outputfile = open('dataclassify.pkl', 'wb')
    pickle.dump(saveclassifydata, outputfile)
    outputfile.close()

def main():
    config =  configparser.ConfigParser()
    config.read('twitter.cfg')
    internalData =  str(config.get('twitter', 'useDataFile'))
    affin_filename = 'data/affin/AFINN-111.txt'
    filename = ''
    if internalData == "True":
        print("As internalData is set to True we will use tweets in file mytweets.pkl")
        filename = 'data/mydownloadeddata/mytweets.pkl'
    elif internalData == "False":
        print("As internalData is set to False we will use new tweets file newmytweets.pkl")
        filename = 'newmytweets.pkl'
    tweets = get_unique_tweets(filename)
    pos,neg = get_afinn_sentiment(affin_filename)
    print("Total pos words are %d" % int(len(pos)))
    print("Total neg words are %d" % int(len(neg)))
    print("Reading and fitting train data")
    docs, labels = read_data(os.path.join('data', 'train'))
    clf, vocab = fit_train_classifier(docs,labels,pos,neg)
    print ("Length of vocab is %d" % int(len(vocab)))
    X_test = parse_test_data(tweets,vocab,pos,neg)
    print_classification(tweets,X_test,clf)

if __name__ == '__main__':
    main()

max_stars_repo_name: kungfumas/analisa-twitter

max_issues_repo_name: viyatgandhi/Online-Social-Networking

max_forks_repo_name: viyatgandhi/Online-Social-Networking


text:
import numpy as np
import pandas as pd

# import our modules
from modules import loader, graph
from modules.graph import Timeseries
import time
import os
import yaml
from typing import List
import math
import sys

import matplotlib.pyplot as plt
import matplotlib.cm as cm

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import PolynomialFeatures

import operator

root_data_folder = "./data/2"
# read the data from the csv file

filenames = ["results_write", "results_write_batch",
             "results_read", "results_read_batch"]
titles = ["Write average latency", "Batch write average latency",
          "Read average latency", "Batch read average latency"]
scales = [None, None, None, None]

add_mean = False

def fit_model(x, y, degree):
    # create the polynomal features to be used for training the regression model
    polynomial_features = PolynomialFeatures(degree=degree)
    x_poly = polynomial_features.fit_transform(x)
    # create a linear regression model using scikit-learn
    model = LinearRegression()
    # fit the model to the data points (x,y)
    model.fit(x_poly, y)
    # use the model to predict the data points
    y_poly_pred = model.predict(x_poly)

    print(model.coef_)
    # get the prediction error as RMSE
    rmse = np.sqrt(mean_squared_error(y, y_poly_pred))
    # get the regression score
    r2 = r2_score(y, y_poly_pred)
    return x_poly, y_poly_pred, rmse, r2

def plot_line(x, y_poly_pred):
    # sort the values of x before line plot
    sort_axis = operator.itemgetter(0)
    sorted_zip = sorted(zip(x, y_poly_pred), key=sort_axis)
    x, y_poly_pred = zip(*sorted_zip)
    plt.plot(x, y_poly_pred)

def create_timeseries(data, header, datax=None):
    tss: List[Timeseries] = []
    colors = ['blue', 'red', 'green', 'orange']
    # colors = ['Red', 'Orange', 'Yellow', 'Green', 'Blue', 'Indigo']

    ck = 0

    sdata = np.shape(data)
    rows = sdata[0]
    cols = sdata[1]

    print(sdata)

    # colors = cm.rainbow(np.linspace(0, 1, cols))
    # colors = cm.viridis(np.linspace(0, 1, cols))

    for j in range(cols):
        ts: Timeseries = Timeseries()
        try:
            ts.label = header[j]
        except:
            ts.label = "unknown"
        ts.color = colors[ck]
        ck += 1
        if ck >= len(colors):
            ck = 0

        for i in range(rows):
            if datax is not None:
                ts.x.append(datax[i][j])
            else:
                ts.x.append(i)
            ts.y.append(data[i][j])
        tss.append(ts)
        ts = None

    return tss

# create separate models for each data file
for i, filename in enumerate(filenames):
    data_file = root_data_folder + "/" + filename + ".csv"
    result_file = root_data_folder + "/img/" + filename
    x, y, xheader, yheader = loader.load_dataset(data_file)

    print(xheader)
    print(yheader)

    start_index = 0
    end_index = None

    # end_index = 9600
    # end_index = 4800 * 5

    x = x[start_index:, :]
    y = y[start_index:, :]

    # print(x)

    # quit()

    if end_index is not None:
        x = x[:end_index, :]
        y = y[:end_index, :]

    sx = np.shape(x)
    sy = np.shape(y)

    print(np.shape(x))
    print(np.shape(y))

    # plot_data(x, y, xheader, yheader)

    x_poly, y_poly_pred, rmse, r2 = fit_model(x, y, 0)

    tss_xy = create_timeseries(y, ["data"], x)
    plot_tss = [tss_xy[0]]
    if add_mean:
        tss_xy_poly = create_timeseries(
            y_poly_pred, ["mean: " + str(int(y_poly_pred[0]))], x)
        tss_xy_poly[0].color = "orange"
        plot_tss.append(tss_xy_poly[0])

    fig, _ = graph.plot_timeseries_multi(
        i, plot_tss, titles[i], xheader[0], yheader[0], True, scales[i], (10, 8))
    graph.save_figure(fig, result_file)


text:
import pandas as pd
import numpy as np

import scipy

# interpolation function using scipy's linear interpolation method:
# parameters: name of objective (for instance 'FUELFLOW'), data frame, point of interest (for interpolation)
def scipy_interpolation_linear(objective_name, data, interpolation_point):
    points, values = closest_points(objective_name, data, interpolation_point, 2500)

    interpolated_val = scipy.interpolate.griddata(points, values, interpolation_point, method='linear', rescale='TRUE')

    return interpolated_val

# interpolation function using scipy's linear interpolation method:
# parameters: name of objective (for instance 'FUELFLOW'), data frame, point of interest (for interpolation)
def scipy_interpolation_nearest(objective_name, data, interpolation_point):
    points, values = closest_points(objective_name, data, interpolation_point, 16)

    interpolated_val = scipy.interpolate.griddata(points, values, interpolation_point, method=interpolation_point, rescale='TRUE')

    return interpolated_val

# function for calculating closest neighbours for a given point using euclidean distance
# parameters: name of objective (for instance 'FUELFLOW'), data frame with neighbours, point of interest,
#             number of neighbours returned
def closest_points(objective_name, df, interpolation_point, n):
    temp_df = df.copy()
    temp_df['distance'] = temp_df.sub(point).pow(2).sum(1).pow(0.5) # euclidean distance

    # sort values from closest to farthest, including nbr_neighbours amount of points
    df_sort_by_dist = temp_df.sort_values('distance').iloc[0:nbr_neighbours]
    df_sort_by_dist = df_sort_by_dist.drop(['distance'],axis = 1)
    df_sort_by_dist = df_sort_by_dist.reset_index()
    df_sort_by_dist = df_sort_by_dist.drop(['index'], axis=1)

    val = df_sort_by_dist[objective_name]
    poi = df_sort_by_dist.drop([objective_name], axis=1)

    return val, poi

max_stars_repo_name: Navidhaddad/Interpolation-jeppesen

max_issues_repo_name: Navidhaddad/Interpolation-jeppesen

max_forks_repo_name: Navidhaddad/Interpolation-jeppesen

reason: import numpy,import scipy


text:
import torch
import torch.nn as nn
from math import floor
import os
import random
import numpy as np
import pdb
import time
from datasets.dataset_h5 import Dataset_All_Bags, Whole_Slide_Bag
from torch.utils.data import DataLoader
from models.resnet_custom import resnet50_baseline
import argparse
from utils.utils import print_network, collate_features
from PIL import Image
import h5py

device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

def save_hdf5(output_dir, asset_dict, mode='a'):
	file = h5py.File(output_dir, mode)

	for key, val in asset_dict.items():
		data_shape = val.shape
		if key not in file:
			data_type = val.dtype
			chunk_shape = (1, ) + data_shape[1:]
			maxshape = (None, ) + data_shape[1:]
			dset = file.create_dataset(key, shape=data_shape, maxshape=maxshape, chunks=chunk_shape, dtype=data_type)
			dset[:] = val
		else:
			dset = file[key]
			dset.resize(len(dset) + data_shape[0], axis=0)
			dset[-data_shape[0]:] = val

	file.close()
	return output_dir

def compute_w_loader(file_path, output_path, model, batch_size = 8, verbose = 0,
	  				 print_every=20, pretrained=True, target_patch_size=-1):
	"""
	args:
		file_path: directory of bag (.h5 file)
		output_path: directory to save computed features (.h5 file)
		model: pytorch model
		batch_size: batch_size for computing features in batches
		verbose: level of feedback
		pretrained: use weights pretrained on imagenet
	"""
	dataset = Whole_Slide_Bag(file_path=file_path, pretrained=pretrained,
							  target_patch_size=target_patch_size)
	x, y = dataset[0]
	kwargs = {'num_workers': 4, 'pin_memory': True} if device.type == "cuda" else {}
	loader = DataLoader(dataset=dataset, batch_size=batch_size, **kwargs, collate_fn=collate_features)

	if verbose > 0:
		print('processing {}: total of {} batches'.format(file_path,len(loader)))

	mode = 'w'
	for count, (batch, coords) in enumerate(loader):
		with torch.no_grad():
			if count % print_every == 0:
				print('batch {}/{}, {} files processed'.format(count, len(loader), count * batch_size))
			batch = batch.to(device, non_blocking=True)
			mini_bs = coords.shape[0]

			features = model(batch)

			features = features.cpu().numpy()

			asset_dict = {'features': features, 'coords': coords}
			save_hdf5(output_path, asset_dict, mode=mode)
			mode = 'a'

	return output_path

parser = argparse.ArgumentParser(description='Feature Extraction')
parser.add_argument('--data_dir', type=str)
parser.add_argument('--csv_path', type=str)
parser.add_argument('--feat_dir', type=str)
parser.add_argument('--batch_size', type=int, default=256)
parser.add_argument('--slide_ext', type=str, default= '.svs')
parser.add_argument('--no_auto_skip', default=False, action='store_true')
parser.add_argument('--target_patch_size', type=int, default=-1,
					help='the desired size of patches for optional scaling before feature embedding')
args = parser.parse_args()

if __name__ == '__main__':

	print('initializing dataset')
	csv_path = args.csv_path
	bags_dataset = Dataset_All_Bags(csv_path)

	os.makedirs(args.feat_dir, exist_ok=True)
	dest_files = os.listdir(args.feat_dir)

	print('loading model checkpoint')
	model = resnet50_baseline(pretrained=True)
	model = model.to(device)

	# print_network(model)
	if torch.cuda.device_count() > 1:
		model = nn.DataParallel(model)

	model.eval()
	total = len(bags_dataset)

	for bag_candidate_idx in range(total):
		slide_id = bags_dataset[bag_candidate_idx].split(args.slide_ext)[0]
		bag_name = slide_id + '.h5'
		bag_candidate = os.path.join(args.data_dir, 'patches', bag_name)

		print('\nprogress: {}/{}'.format(bag_candidate_idx, total))
		print(bag_name)
		if not args.no_auto_skip and slide_id+'.pt' in dest_files:
			print('skipped {}'.format(slide_id))
			continue

		output_path = os.path.join(args.feat_dir, 'h5_files', bag_name)
		file_path = bag_candidate
		time_start = time.time()
		output_file_path = compute_w_loader(file_path, output_path,
											model = model, batch_size = args.batch_size,
											verbose = 1, print_every = 20,
											target_patch_size=args.target_patch_size)
		time_elapsed = time.time() - time_start
		print('\ncomputing features for {} took {} s'.format(output_file_path, time_elapsed))
		file = h5py.File(output_file_path, "r")

		features = file['features'][:]
		print('features size: ', features.shape)
		print('coordinates size: ', file['coords'].shape)
		features = torch.from_numpy(features)
		bag_base, _ = os.path.splitext(bag_name)
		torch.save(features, os.path.join(args.feat_dir, 'pt_files', bag_base+'.pt'))