text:
import matplotlib.pyplot as plt
import numpy as np
from math import log

#function applies specifically to binary-like mixture initial states
#i.e. states where only qudits belonging to Va undergo dephasing
#function determines the updated fidelity after one round of protocol P1
def one_parameter_family(Fin,dim,subset_node_num):
    Fout=(Fin**2)/(Fin**2 + ((1-Fin)**2)/(dim**subset_node_num -1))
    return Fout

def one_param_fam_psucc(Fin,dim,subset_node_num):
    psucc=Fin**2 + ((1-Fin)**2)/(dim**subset_node_num - 1)
    return psucc

def one_param_fam_Von_Neuman_Ent(Fid,dim,subset_node_num):
    return (Fid*(log((1-Fid)/(Fid*(dim**subset_node_num -1)))/log(dim)) - log((1-Fid)/(dim**subset_node_num -1))/log(dim))

def plot_one_param_fam_fout_vs_fin(subset_node_num,dimList):

    plt.figure(figsize=(12,8))
    cmap = plt.cm.get_cmap('plasma')
    ind=np.linspace(0,0.75,6)
    colors = [cmap(ind[0]),cmap(ind[1]),cmap(ind[2]),cmap(ind[3]),cmap(ind[4]),cmap(ind[5])]
    track=0
    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        fidList=np.arange(1/(dim**subset_node_num),1,0.001)
        fidOutList=[]
        for x in range(0,np.size(fidList)):
            fidOutList.append(one_parameter_family(fidList[x],dim,subset_node_num))
        plt.plot(fidList,fidOutList,c=colors[track], label="dim = %s " % (dim))
        track+=1
    plt.legend(loc='lower right',fontsize=14)
    plt.xlim([-0.04,1.04])
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    plt.xlabel('Input Fidelity',fontsize=20)
    plt.ylabel('Output Fidelity',fontsize=20)
    plt.show()

#print(one_parameter_family(0.501,3,1))

def plot_one_param_fam_psucc_vs_fin(subset_node_num,dimList,fidList):

    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        psuccList=[]
        for x in range(0,np.size(fidList)):
            psuccList.append(one_param_fam_psucc(fidList[x],dim,subset_node_num))
        plt.plot(fidList,psuccList,label="dim = %s " % (dim))
    plt.legend()
    plt.xlabel('Input Fidelity')
    plt.ylabel('Protocol Probability of Success')
    plt.show()

def plot_one_param_fam_entin_vs_fid(subset_node_num,dimList):
    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        # entDiff=[]
        ents=[]
        min_fid=1/(dim**subset_node_num)
        fidList=np.arange(min_fid+0.001,1,0.001)
        for x in range(0,np.size(fidList)):
            fid=fidList[x]
            ents.append(one_param_fam_Von_Neuman_Ent(fid,dim,subset_node_num))
            # fidOut=one_parameter_family(fid,dim,subset_node_num)
            # entOut=one_param_fam_Von_Neuman_Ent(fidOut,dim,subset_node_num)
            # entDiff.append(entIn-entOut)
        plt.plot(ents,fidList,label="d = %s " % (dim))
    plt.legend(loc='upper right',fontsize=14)
    plt.ylabel('Initial Fidelity',fontsize=20)
    plt.xlabel('Initial Von Neumann Entropy',fontsize=20)
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    plt.ylim([0,1])
    plt.show()

def plot_one_param_fam_entdiff_vs_entin(subset_node_num,dimList):
    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        entDiff=[]
        entsIn=[]
        min_fid=1/(dim**subset_node_num)
        fidList=np.arange(min_fid+0.001,1,0.01)
        for x in range(0,np.size(fidList)):
            fid=fidList[x]
            entIn=one_param_fam_Von_Neuman_Ent(fid,dim,subset_node_num)
            entsIn.append(entIn)
            fidOut=one_parameter_family(fid,dim,subset_node_num)
            entOut=one_param_fam_Von_Neuman_Ent(fidOut,dim,subset_node_num)
            entDiff.append(entIn-entOut)
        plt.plot(entsIn,entDiff,label="d = %s " % (dim))
    plt.legend()
    plt.xlabel('Initial Von Neuman Entropy')
    plt.ylabel('Difference between Initial and Final Von Neuman Entropy')
    plt.show()

def plot_one_param_fam_entratio_vs_entin(subset_node_num,dimList):
    plt.figure(figsize=(12,8))
    cmap = plt.cm.get_cmap('plasma')
    ind=np.linspace(0,0.75,6)
    colors = [cmap(ind[0]),cmap(ind[1]),cmap(ind[2]),cmap(ind[3]),cmap(ind[4]),cmap(ind[5])]
    track=0
    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        entRat=[]
        entsIn=[]
        min_fid=1/(dim**subset_node_num)
        fidList=np.arange(min_fid+0.001,1,0.002)
        for x in range(0,np.size(fidList)):
            fid=fidList[x]
            entIn=one_param_fam_Von_Neuman_Ent(fid,dim,subset_node_num)
            entsIn.append(entIn)
            fidOut=one_parameter_family(fid,dim,subset_node_num)
            entOut=one_param_fam_Von_Neuman_Ent(fidOut,dim,subset_node_num)
            entRat.append(entIn/entOut)
        plt.plot(entsIn,entRat,'-',c=colors[track],label="d = %s " % (dim))
        track+=1
    plt.legend(loc='upper right',fontsize=14)
    plt.yscale('log')
    plt.xlabel('Initial Entropy',fontsize=20)
    plt.ylabel('Ratio of Initial to Final Entropy',fontsize=20)
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    plt.ylim((1,10e6))
    plt.show()

def plot_one_param_fam_entratio_vs_fidin(subset_node_num,dimList):
    plt.figure()
    for y in range(0,np.size(dimList)):
        dim=dimList[y]
        entRat=[]
        min_fid=1/(dim**subset_node_num)
        fidList=np.arange(min_fid+0.001,1,0.002)
        for x in range(0,np.size(fidList)):
            fid=fidList[x]
            entIn=one_param_fam_Von_Neuman_Ent(fid,dim,subset_node_num)
            fidOut=one_parameter_family(fid,dim,subset_node_num)
            entOut=one_param_fam_Von_Neuman_Ent(fidOut,dim,subset_node_num)
            entRat.append(entIn/entOut)
        plt.plot(fidList,entRat,'-',label="d = %s " % (dim))
    plt.legend(loc='upper right',fontsize=14)
    plt.yscale('log')
    plt.xlabel('Initial Fidelity',fontsize=20)
    plt.ylabel('Ratio of Initial to Final Entropy',fontsize=20)
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    # plt.ylim((1,10e6))
    plt.show()

# plot_one_param_fam_entin_vs_fid(3,[2,3,5,7,9,11,13])
plot_one_param_fam_entratio_vs_entin(3,[2,3,5,7,11,13])
# plot_one_param_fam_entratio_vs_fidin(3,[2,3,5,7,9,11,13])
# plot_one_param_fam_fout_vs_fin(1,[2,3,5,7,11,13])

#plot_one_param_fam_psucc_vs_fin(5,[2,3,5,7,11,13],np.arange(0.501,1,0.01))


text:
import numpy as np
import math

def spatial_accuracy(ps1, ps2, thresh):
    '''
        Args) ps1, ps2 : normalized point sets
        Retern) acc: spatial accuracy
    '''
    assert len(ps1) == len(ps2), \
        f"length of given point sets are differenct: len(ps1)={len(ps1)}, len(ps2)={len(ps2)}"

    dists = (ps1 - ps2) ** 2
    dists = np.sum(dists, axis=-1)
    dists = np.sqrt(dists)

    acc = np.mean(dists <= thresh)
    return acc

def temporal_accuracy(ps1, ps2, prev_ps1, prev_ps2, thresh):
    '''
        Args) ps1, ps2 : normalized point sets
        Retern) acc: temporal accuracy
    '''
    assert len(ps1) == len(ps2), \
            f"length of given point sets are differenct: len(ps1)={len(ps1)}, len(ps2)={len(ps2)}"
    assert len(prev_ps1) == len(prev_ps2), \
            f"length of given point sets are differenct: len(prev_ps1)={len(prev_ps1)}, len(prev_ps2)={len(prev_ps2)}"

    dists_prev = ps1 - ps2
    dists_next = prev_ps1 - prev_ps2

    diffs = (dists_prev - dists_next) ** 2
    diffs = np.sum(diffs, axis=-1)
    diffs = np.sqrt(diffs)

    acc = np.mean(diffs <= thresh, axis=-1)
    acc = np.mean(acc)
    return acc

ext: py

max_stars_repo_name: ghnam-ken/PoST

max_issues_repo_name: ghnam-ken/PoST

max_forks_repo_name: ghnam-ken/PoST

reason: import numpy


text:
import pandas as pd
import pytest
import numpy as np

import dask.dataframe as dd
from dask.dataframe.utils import assert_eq, PANDAS_VERSION

N = 40
df = pd.DataFrame(
    {
        "a": np.random.randn(N).cumsum(),
        "b": np.random.randint(100, size=(N,)),
        "c": np.random.randint(100, size=(N,)),
        "d": np.random.randint(100, size=(N,)),
        "e": np.random.randint(100, size=(N,)),
    }
)
ddf = dd.from_pandas(df, 3)

idx = (
    pd.date_range("2016-01-01", freq="3s", periods=100)
    | pd.date_range("2016-01-01", freq="5s", periods=100)
)[:N]

ts = pd.DataFrame(
    {
        "a": np.random.randn(N).cumsum(),
        "b": np.random.randint(100, size=(N,)),
        "c": np.random.randint(100, size=(N,)),
        "d": np.random.randint(100, size=(N,)),
        "e": np.random.randint(100, size=(N,)),
    },
    index=idx,
)
dts = dd.from_pandas(ts, 3)

def shifted_sum(df, before, after, c=0):
    a = df.shift(before)
    b = df.shift(-after)
    return df + a + b + c

def ts_shifted_sum(df, before, after, c=0):
    a = df.shift(before.seconds)
    b = df.shift(-after.seconds)
    return df + a + b + c

@pytest.mark.parametrize("npartitions", [1, 4])
def test_map_overlap(npartitions):
    ddf = dd.from_pandas(df, npartitions)
    for before, after in [(0, 3), (3, 0), (3, 3), (0, 0)]:
        # DataFrame
        res = ddf.map_overlap(shifted_sum, before, after, before, after, c=2)
        sol = shifted_sum(df, before, after, c=2)
        assert_eq(res, sol)

        # Series
        res = ddf.b.map_overlap(shifted_sum, before, after, before, after, c=2)
        sol = shifted_sum(df.b, before, after, c=2)
        assert_eq(res, sol)

def test_map_overlap_names():
    npartitions = 3
    ddf = dd.from_pandas(df, npartitions)

    res = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=2)
    res2 = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=2)
    assert set(res.dask) == set(res2.dask)

    res3 = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=3)
    assert res3._name != res._name
    # Difference is just the final map
    diff = set(res3.dask).difference(res.dask)
    assert len(diff) == npartitions

    res4 = ddf.map_overlap(shifted_sum, 3, 0, 0, 3, c=2)
    assert res4._name != res._name

def test_map_overlap_errors():
    # Non-integer
    with pytest.raises(ValueError):
        ddf.map_overlap(shifted_sum, 0.5, 3, 0, 2, c=2)

    # Negative
    with pytest.raises(ValueError):
        ddf.map_overlap(shifted_sum, 0, -5, 0, 2, c=2)

    # Partition size < window size
    with pytest.raises(NotImplementedError):
        ddf.map_overlap(shifted_sum, 0, 100, 0, 100, c=2).compute()

    # Offset with non-datetime
    with pytest.raises(TypeError):
        ddf.map_overlap(shifted_sum, pd.Timedelta("1s"), pd.Timedelta("1s"), 0, 2, c=2)

def test_map_overlap_provide_meta():
    df = pd.DataFrame(
        {"x": [1, 2, 4, 7, 11], "y": [1.0, 2.0, 3.0, 4.0, 5.0]}
    ).rename_axis("myindex")
    ddf = dd.from_pandas(df, npartitions=2)

    # Provide meta spec, but not full metadata
    res = ddf.map_overlap(
        lambda df: df.rolling(2).sum(), 2, 0, meta={"x": "i8", "y": "i8"}
    )
    sol = df.rolling(2).sum()
    assert_eq(res, sol)

def mad(x):
    return np.fabs(x - x.mean()).mean()

rolling_method_args_check_less_precise = [
    ("count", (), False),
    ("sum", (), False),
    ("mean", (), False),
    ("median", (), False),
    ("min", (), False),
    ("max", (), False),
    ("std", (), True),
    ("var", (), True),
    ("skew", (), True),  # here and elsewhere, results for kurt and skew are
    ("kurt", (), True),  # checked with check_less_precise=True so that we are
    # only looking at 3ish decimal places for the equality check
    # rather than 5ish. I have encountered a case where a test
    # seems to have failed due to numerical problems with kurt.
    # So far, I am only weakening the check for kurt and skew,
    # as they involve third degree powers and higher
    ("quantile", (0.38,), False),
    ("apply", (mad,), False),
]

@pytest.mark.parametrize(
    "method,args,check_less_precise", rolling_method_args_check_less_precise
)
@pytest.mark.parametrize("window", [1, 2, 4, 5])
@pytest.mark.parametrize("center", [True, False])
def test_rolling_methods(method, args, window, center, check_less_precise):
    # DataFrame
    prolling = df.rolling(window, center=center)
    drolling = ddf.rolling(window, center=center)
    if method == "apply":
        kwargs = {"raw": False}
    else:
        kwargs = {}

    assert_eq(
        getattr(prolling, method)(*args, **kwargs),
        getattr(drolling, method)(*args, **kwargs),
        check_less_precise=check_less_precise,
    )

    # Series
    prolling = df.a.rolling(window, center=center)
    drolling = ddf.a.rolling(window, center=center)
    assert_eq(
        getattr(prolling, method)(*args, **kwargs),
        getattr(drolling, method)(*args, **kwargs),
        check_less_precise=check_less_precise,
    )

if PANDAS_VERSION <= "0.25.0":
    filter_panel_warning = pytest.mark.filterwarnings(
        "ignore::DeprecationWarning:pandas[.*]"
    )
else:
    filter_panel_warning = lambda f: f

@filter_panel_warning
@pytest.mark.parametrize("window", [1, 2, 4, 5])
@pytest.mark.parametrize("center", [True, False])
def test_rolling_cov(window, center):
    # DataFrame
    prolling = df.drop("a", 1).rolling(window, center=center)
    drolling = ddf.drop("a", 1).rolling(window, center=center)
    assert_eq(prolling.cov(), drolling.cov())

    # Series
    prolling = df.b.rolling(window, center=center)
    drolling = ddf.b.rolling(window, center=center)
    assert_eq(prolling.cov(), drolling.cov())

def test_rolling_raises():
    df = pd.DataFrame(
        {"a": np.random.randn(25).cumsum(), "b": np.random.randint(100, size=(25,))}
    )
    ddf = dd.from_pandas(df, 3)
    pytest.raises(ValueError, lambda: ddf.rolling(1.5))
    pytest.raises(ValueError, lambda: ddf.rolling(-1))
    pytest.raises(ValueError, lambda: ddf.rolling(3, min_periods=1.2))
    pytest.raises(ValueError, lambda: ddf.rolling(3, min_periods=-2))
    pytest.raises(ValueError, lambda: ddf.rolling(3, axis=10))
    pytest.raises(ValueError, lambda: ddf.rolling(3, axis="coulombs"))
    pytest.raises(NotImplementedError, lambda: ddf.rolling(100).mean().compute())

def test_rolling_names():
    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
    a = dd.from_pandas(df, npartitions=2)
    assert sorted(a.rolling(2).sum().dask) == sorted(a.rolling(2).sum().dask)

def test_rolling_axis():
    df = pd.DataFrame(np.random.randn(20, 16))
    ddf = dd.from_pandas(df, npartitions=3)

    assert_eq(df.rolling(3, axis=0).mean(), ddf.rolling(3, axis=0).mean())
    assert_eq(df.rolling(3, axis=1).mean(), ddf.rolling(3, axis=1).mean())
    assert_eq(
        df.rolling(3, min_periods=1, axis=1).mean(),
        ddf.rolling(3, min_periods=1, axis=1).mean(),
    )
    assert_eq(
        df.rolling(3, axis="columns").mean(), ddf.rolling(3, axis="columns").mean()
    )
    assert_eq(df.rolling(3, axis="rows").mean(), ddf.rolling(3, axis="rows").mean())

    s = df[3]
    ds = ddf[3]
    assert_eq(s.rolling(5, axis=0).std(), ds.rolling(5, axis=0).std())

def test_rolling_partition_size():
    df = pd.DataFrame(np.random.randn(50, 2))
    ddf = dd.from_pandas(df, npartitions=5)

    for obj, dobj in [(df, ddf), (df[0], ddf[0])]:
        assert_eq(obj.rolling(10).mean(), dobj.rolling(10).mean())
        assert_eq(obj.rolling(11).mean(), dobj.rolling(11).mean())
        with pytest.raises(NotImplementedError):
            dobj.rolling(12).mean().compute()

def test_rolling_repr():
    ddf = dd.from_pandas(pd.DataFrame([10] * 30), npartitions=3)
    res = repr(ddf.rolling(4))
    assert res == "Rolling [window=4,center=False,axis=0]"

def test_time_rolling_repr():
    res = repr(dts.rolling("4s"))
    assert res == "Rolling [window=4000000000,center=False,win_type=freq,axis=0]"

def test_time_rolling_constructor():
    result = dts.rolling("4s")
    assert result.window == "4s"
    assert result.min_periods is None
    assert result.win_type is None

    assert result._win_type == "freq"
    assert result._window == 4000000000  # ns
    assert result._min_periods == 1

@pytest.mark.parametrize(
    "method,args,check_less_precise", rolling_method_args_check_less_precise
)
@pytest.mark.parametrize("window", ["1S", "2S", "3S", pd.offsets.Second(5)])
def test_time_rolling_methods(method, args, window, check_less_precise):
    # DataFrame
    if method == "apply":
        kwargs = {"raw": False}
    else:
        kwargs = {}
    prolling = ts.rolling(window)
    drolling = dts.rolling(window)
    assert_eq(
        getattr(prolling, method)(*args, **kwargs),
        getattr(drolling, method)(*args, **kwargs),
        check_less_precise=check_less_precise,
    )

    # Series
    prolling = ts.a.rolling(window)
    drolling = dts.a.rolling(window)
    assert_eq(
        getattr(prolling, method)(*args, **kwargs),
        getattr(drolling, method)(*args, **kwargs),
        check_less_precise=check_less_precise,
    )

@filter_panel_warning
@pytest.mark.parametrize("window", ["1S", "2S", "3S", pd.offsets.Second(5)])
def test_time_rolling_cov(window):
    # DataFrame
    prolling = ts.drop("a", 1).rolling(window)
    drolling = dts.drop("a", 1).rolling(window)
    assert_eq(prolling.cov(), drolling.cov())

    # Series
    prolling = ts.b.rolling(window)
    drolling = dts.b.rolling(window)
    assert_eq(prolling.cov(), drolling.cov())

@pytest.mark.parametrize(
    "window,N",
    [("1s", 10), ("2s", 10), ("10s", 10), ("10h", 10), ("10s", 100), ("10h", 100)],
)
def test_time_rolling_large_window_fixed_chunks(window, N):
    df = pd.DataFrame(
        {
            "a": pd.date_range("2016-01-01 00:00:00", periods=N, freq="1s"),
            "b": np.random.randint(100, size=(N,)),
        }
    )
    df = df.set_index("a")
    ddf = dd.from_pandas(df, 5)
    assert_eq(ddf.rolling(window).sum(), df.rolling(window).sum())
    assert_eq(ddf.rolling(window).count(), df.rolling(window).count())
    assert_eq(ddf.rolling(window).mean(), df.rolling(window).mean())

@pytest.mark.parametrize("window", ["2s", "5s", "20s", "10h"])
def test_time_rolling_large_window_variable_chunks(window):
    df = pd.DataFrame(
        {
            "a": pd.date_range("2016-01-01 00:00:00", periods=100, freq="1s"),
            "b": np.random.randint(100, size=(100,)),
        }
    )
    ddf = dd.from_pandas(df, 5)
    ddf = ddf.repartition(divisions=[0, 5, 20, 28, 33, 54, 79, 80, 82, 99])
    df = df.set_index("a")
    ddf = ddf.set_index("a")
    assert_eq(ddf.rolling(window).sum(), df.rolling(window).sum())
    assert_eq(ddf.rolling(window).count(), df.rolling(window).count())
    assert_eq(ddf.rolling(window).mean(), df.rolling(window).mean())

@pytest.mark.parametrize("before, after", [("6s", "6s"), ("2s", "2s"), ("6s", "2s")])
def test_time_rolling(before, after):
    window = before
    before = pd.Timedelta(before)
    after = pd.Timedelta(after)
    result = dts.map_overlap(lambda x: x.rolling(window).count(), before, after)
    expected = dts.compute().rolling(window).count()
    assert_eq(result, expected)

def test_rolling_agg_aggregate():
    df = pd.DataFrame({"A": range(5), "B": range(0, 10, 2)})
    ddf = dd.from_pandas(df, npartitions=3)

    assert_eq(
        df.rolling(window=3).agg([np.mean, np.std]),
        ddf.rolling(window=3).agg([np.mean, np.std]),
    )

    assert_eq(
        df.rolling(window=3).agg({"A": np.sum, "B": lambda x: np.std(x, ddof=1)}),
        ddf.rolling(window=3).agg({"A": np.sum, "B": lambda x: np.std(x, ddof=1)}),
    )

    assert_eq(
        df.rolling(window=3).agg([np.sum, np.mean]),
        ddf.rolling(window=3).agg([np.sum, np.mean]),
    )

    assert_eq(
        df.rolling(window=3).agg({"A": [np.sum, np.mean]}),
        ddf.rolling(window=3).agg({"A": [np.sum, np.mean]}),
    )

    kwargs = {"raw": True}
    assert_eq(
        df.rolling(window=3).apply(lambda x: np.std(x, ddof=1), **kwargs),
        ddf.rolling(window=3).apply(lambda x: np.std(x, ddof=1), **kwargs),
    )

@pytest.mark.skipif(not dd._compat.PANDAS_GT_100, reason="needs pandas>=1.0.0")
def test_rolling_numba_engine():
    pytest.importorskip("numba")
    df = pd.DataFrame({"A": range(5), "B": range(0, 10, 2)})
    ddf = dd.from_pandas(df, npartitions=3)

    def f(x):
        return np.sum(x) + 5

    assert_eq(
        df.rolling(3).apply(f, engine="numba", raw=True),
        ddf.rolling(3).apply(f, engine="numba", raw=True),
    )

@pytest.mark.skipif(dd._compat.PANDAS_GT_100, reason="Requires pandas<1.0.0")
def test_rolling_apply_numba_raises():
    df = pd.DataFrame({"A": range(5), "B": range(0, 10, 2)})
    ddf = dd.from_pandas(df, npartitions=3)
    with pytest.raises(NotImplementedError, match="pandas>=1.0.0"):
        ddf.rolling(3).apply(lambda x: x.sum(), engine="numba", raw=True)

max_stars_repo_name: nre/dask

max_issues_repo_name: nre/dask

max_forks_repo_name: nre/dask


text:
# Lint as: python3
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base language model configurations."""

import math
from typing import Sequence

from jax import numpy as jnp
from lingvo.jax import asserts
from lingvo.jax import base_model_params
from lingvo.jax import layers
from lingvo.jax import model
from lingvo.jax import optimizers
from lingvo.jax import py_utils
from lingvo.jax import schedules
import numpy as np

InstantiableParams = py_utils.InstantiableParams
NestedMap = py_utils.NestedMap
WeightInit = py_utils.WeightInit

def set_sharding_annotations_v1(model_p: InstantiableParams,
                                mesh_shape: Sequence[int]) -> None:
  """Sets the sharding annotations in the model config for the given mesh.

  Args:
    model_p: The model parameters to update with sharding annotations.
    mesh_shape: a 3D sequence representing the mesh shape.
  """
  asserts.eq(len(mesh_shape), 3)
  device_count = np.prod(mesh_shape)
  device_ids_mesh = np.arange(device_count).reshape(mesh_shape)
  model_p.device_mesh = device_ids_mesh
  replica_axis = 'replica'
  data_axis = 'data'
  mdl_axis = 'mdl'
  mesh_axis_names = [replica_axis, data_axis, mdl_axis]
  model_p.train.inputs_split_mapping = NestedMap(
      map_1d=((replica_axis, data_axis),),
      map_2d=((replica_axis, data_axis), None))
  model_p.mesh_axis_names = mesh_axis_names
  model_p.lm = model_p.lm.cls.set_sharding_params_v1(
      model_p.lm,
      replica_axis=replica_axis,
      data_axis=data_axis,
      mdl_axis=mdl_axis,
      device_ids_mesh=device_ids_mesh,
      mesh_axis_names=mesh_axis_names)

def set_default_adam(model_p: InstantiableParams, learning_rate: float,
                     weight_decay: float) -> None:
  """Sets the default Adam optimizer settings in the model config.

  Args:
    model_p: The model parameters to update with optimizer specs.
    learning_rate: The learning rate to set.
    weight_decay: The weight_decay to set.
  """
  lp = model_p.train.learner
  lp.loss_name = 'total_loss'
  lp.optimizer = optimizers.Adam.Params().Set(
      beta1=0.9,
      beta2=0.99,
      weight_decay=weight_decay,
      clip_gradient_norm_to_value=5.0)
  lp.optimizer.learning_rate = learning_rate
  lp.optimizer.lr_schedule = (
      schedules.LinearRampupExponentialDecay.Params().Set(
          warmup=4000,
          decay_start=4001,
          decay_end=300000,
          min_ratio=0.1,
          max=1.0))

def set_default_adafactor(model_p: InstantiableParams, learning_rate: float,
                          weight_decay: float) -> None:
  """Sets the default AdaFactor optimizer settings in the model config.

  Args:
    model_p: The model parameters to update with optimizer specs.
    learning_rate: The learning rate to set.
    weight_decay: The weight_decay to set.
  """
  lp = model_p.train.learner
  lp.loss_name = 'total_loss'
  lp.optimizer = optimizers.ShardedAdafactor.Params().Set(
      decay_method='adam',
      beta1=0.9,
      decay_adam=0.99,
      weight_decay=weight_decay,
      clip_gradient_norm_to_value=5.0)
  lp.optimizer.learning_rate = learning_rate
  lp.optimizer.lr_schedule = (
      schedules.LinearRampupExponentialDecay.Params().Set(
          warmup=4000,
          decay_start=4001,
          decay_end=100000,
          min_ratio=0.1,
          max=1.0))

class TransformerBertPmapAdam(base_model_params.BaseModelParams):
  """Base Pmap Transformer Bert configuration using Adam."""

  NUM_LAYERS = 4
  VOCAB_SIZE = 32000
  NUM_HEADS = 8
  MODEL_DIMS = 128
  HIDDEN_DIMS = MODEL_DIMS * 4
  DROPOUT_PROB = 0.0
  LEARNING_RATE = 1e-3
  WEIGHT_DECAY = 1e-3
  USE_REPEATED_LAYER = False
  ENABLE_WHILE_LOOP = True
  CHECKPOINT_POLICY = layers.AutodiffCheckpointType.SAVE_DOT_WITH_NO_BATCH_DIM
  ACTIVATION_FUNCTION = 'RELU'
  # Save a checkpoint every n steps.
  CHECKPOINT_EVERY_N_STEPS = 5000

  ENABLE_BFLOAT16 = True

  def task(self) -> InstantiableParams:
    """Returns the task parameters."""
    model_p = model.BertModel.Params().Set(name='bert_lm')
    model_p.lm.masked_lm = True
    model_p.lm.packed_input = True
    model_p.lm.model_dims = self.MODEL_DIMS
    model_p.lm.hidden_dims = self.HIDDEN_DIMS
    model_p.lm.num_layers = self.NUM_LAYERS
    model_p.lm.num_heads = self.NUM_HEADS
    model_p.lm.vocab_size = self.VOCAB_SIZE
    model_p.lm.softmax_tpl.scale_sqrt_depth = True
    model_p.lm.softmax_tpl.soft_cap_logits = 30.0
    if self.USE_REPEATED_LAYER:
      model_p.lm.stacked_transformer_tpl = (
          layers.StackedTransformerRepeated.Params())
    else:
      model_p.lm.stacked_transformer_tpl = layers.StackedTransformer.Params()
    model_p.lm.stacked_transformer_tpl.enable_while_loop = (
        self.ENABLE_WHILE_LOOP)
    model_p.lm.stacked_transformer_tpl.checkpoint_policy = (
        self.CHECKPOINT_POLICY)
    model_p.lm.stacked_transformer_tpl.dropout_prob = self.DROPOUT_PROB
    transformer_layer_p = (
        model_p.lm.stacked_transformer_tpl.transformer_layer_params_tpl)
    transformer_layer_p.tr_atten_tpl.atten_logit_cap = 50.0
    transformer_layer_p.tr_atten_tpl.use_bias = False
    transformer_layer_p.tr_fflayer_tpl.activation = self.ACTIVATION_FUNCTION

    softmax_init = WeightInit.Gaussian(1.0 / math.sqrt(self.MODEL_DIMS))
    model_p.lm.softmax_tpl.params_init = softmax_init

    model_p.train.save_interval_steps = self.CHECKPOINT_EVERY_N_STEPS

    if self.ENABLE_BFLOAT16:
      model_p.fprop_dtype = jnp.bfloat16

    set_default_adam(model_p, self.LEARNING_RATE, self.WEIGHT_DECAY)

    return model_p

class TransformerBertSpmdAdafactor(base_model_params.BaseModelParams):
  """Base SPMD Transformer Bert configuration using AdaFactor."""

  NUM_LAYERS = 32
  VOCAB_SIZE = 32000
  NUM_HEADS = 16
  MODEL_DIMS = 1024
  HIDDEN_DIMS = MODEL_DIMS * 4
  DROPOUT_PROB = 0.0
  LEARNING_RATE = 1e-3
  WEIGHT_DECAY = 1e-3
  USE_REPEATED_LAYER = False
  ENABLE_WHILE_LOOP = True
  CHECKPOINT_POLICY = layers.AutodiffCheckpointType.SAVE_DOT_WITH_NO_BATCH_DIM
  ENABLE_BFLOAT16 = False
  MASK_TOKEN_ID = 0

  ACTIVATION_FUNCTION = 'RELU'

  # Sub-class has to specify a mesh.
  MESH_SHAPE = None

  # Save a checkpoint every n steps.
  CHECKPOINT_EVERY_N_STEPS = 500
  CHECKPOINT_SAVE_MAX_TO_KEEP = 10

  def task(self) -> InstantiableParams:
    """Returns the task parameters."""
    model_p = model.BertModel.Params().Set(name='bert_lm')
    model_p.mask_token_id = self.MASK_TOKEN_ID
    model_p.lm.masked_lm = True
    model_p.lm.packed_input = True
    model_p.lm.model_dims = self.MODEL_DIMS
    model_p.lm.hidden_dims = self.HIDDEN_DIMS
    model_p.lm.num_layers = self.NUM_LAYERS
    model_p.lm.num_heads = self.NUM_HEADS
    model_p.lm.vocab_size = self.VOCAB_SIZE
    model_p.lm.softmax_tpl.scale_sqrt_depth = True
    model_p.lm.softmax_tpl.soft_cap_logits = 30.0
    if self.USE_REPEATED_LAYER:
      model_p.lm.stacked_transformer_tpl = (
          layers.StackedTransformerRepeated.Params())
    else:
      model_p.lm.stacked_transformer_tpl = layers.StackedTransformer.Params()
    model_p.lm.stacked_transformer_tpl.enable_while_loop = (
        self.ENABLE_WHILE_LOOP)
    model_p.lm.stacked_transformer_tpl.checkpoint_policy = (
        self.CHECKPOINT_POLICY)
    model_p.lm.stacked_transformer_tpl.dropout_prob = self.DROPOUT_PROB
    transformer_layer_p = (
        model_p.lm.stacked_transformer_tpl.transformer_layer_params_tpl)
    transformer_layer_p.tr_atten_tpl.atten_logit_cap = 50.0
    transformer_layer_p.tr_atten_tpl.use_bias = False
    transformer_layer_p.tr_atten_tpl.combine_qkv = True
    transformer_layer_p.tr_fflayer_tpl.activation = self.ACTIVATION_FUNCTION
    softmax_init = WeightInit.Gaussian(1.0 / math.sqrt(self.MODEL_DIMS))
    model_p.lm.softmax_tpl.params_init = softmax_init

    if self.ENABLE_BFLOAT16:
      model_p.fprop_dtype = jnp.bfloat16

    model_p.train.save_max_to_keep = self.CHECKPOINT_SAVE_MAX_TO_KEEP

    set_default_adafactor(model_p, self.LEARNING_RATE, self.WEIGHT_DECAY)

    model_p.train.save_interval_steps = self.CHECKPOINT_EVERY_N_STEPS

    set_sharding_annotations_v1(model_p, self.MESH_SHAPE)

    return model_p

class TransformerLmPmapAdam(base_model_params.BaseModelParams):
  """Base Pmap Transformer LM configuration using Adam."""

  NUM_LAYERS = 32
  VOCAB_SIZE = 32000
  NUM_HEADS = 16
  MODEL_DIMS = 1024
  HIDDEN_DIMS = MODEL_DIMS * 4
  DROPOUT_PROB = 0.0
  LEARNING_RATE = 1e-3
  WEIGHT_DECAY = 1e-3
  ENABLE_WHILE_LOOP = True

  def task(self) -> InstantiableParams:
    """Returns the task parameters."""
    model_p = model.LanguageModel.Params().Set(name='xformer_lm')
    model_p.lm.packed_input = True
    model_p.lm.model_dims = self.MODEL_DIMS
    model_p.lm.hidden_dims = self.HIDDEN_DIMS
    model_p.lm.num_layers = self.NUM_LAYERS
    model_p.lm.num_heads = self.NUM_HEADS
    model_p.lm.vocab_size = self.VOCAB_SIZE
    model_p.lm.softmax_tpl.scale_sqrt_depth = True
    model_p.lm.stacked_transformer_tpl = layers.StackedTransformer.Params()
    model_p.lm.stacked_transformer_tpl.enable_while_loop = (
        self.ENABLE_WHILE_LOOP)
    model_p.lm.stacked_transformer_tpl.dropout_prob = self.DROPOUT_PROB
    transformer_layer_p = (
        model_p.lm.stacked_transformer_tpl.transformer_layer_params_tpl)
    transformer_layer_p.tr_atten_tpl.atten_logit_cap = 50.0
    transformer_layer_p.tr_atten_tpl.use_bias = False
    softmax_init = WeightInit.Gaussian(1.0 / math.sqrt(self.MODEL_DIMS))
    model_p.lm.softmax_tpl.params_init = softmax_init

    set_default_adam(model_p, self.LEARNING_RATE, self.WEIGHT_DECAY)

    return model_p

class TransformerLmSpmdAdam(base_model_params.BaseModelParams):
  """Base SPMD Transformer LM configuration using Adam."""

  NUM_LAYERS = 10
  VOCAB_SIZE = 32000
  DIMS_PER_HEAD = 128
  MODEL_DIMS = 2048
  HIDDEN_DIMS = MODEL_DIMS * 4
  DROPOUT_PROB = 0.0
  LEARNING_RATE = 2.5e-4
  WEIGHT_DECAY = 1e-3
  ENABLE_WHILE_LOOP = True

  # Autodiff remat.
  CHECKPOINT_POLICY = layers.AutodiffCheckpointType.SAVE_NOTHING

  # Sub-class has to specify a mesh.
  MESH_SHAPE = None

  def task(self) -> InstantiableParams:
    """Returns the task parameters."""
    assert self.MODEL_DIMS % self.DIMS_PER_HEAD == 0
    num_heads = int(self.MODEL_DIMS / self.DIMS_PER_HEAD)
    dropout_prob = self.DROPOUT_PROB

    model_p = model.LanguageModel.Params().Set(name='xformer_lm')
    model_p.lm.packed_input = True
    model_p.lm.model_dims = self.MODEL_DIMS
    model_p.lm.hidden_dims = self.HIDDEN_DIMS
    model_p.lm.num_layers = self.NUM_LAYERS
    model_p.lm.num_heads = num_heads
    model_p.lm.vocab_size = self.VOCAB_SIZE
    model_p.lm.softmax_tpl.scale_sqrt_depth = True
    model_p.lm.softmax_tpl.soft_cap_logits = 30.0

    model_p.lm.stacked_transformer_tpl = layers.StackedTransformer.Params()
    model_p.lm.stacked_transformer_tpl.enable_while_loop = (
        self.ENABLE_WHILE_LOOP)
    model_p.lm.stacked_transformer_tpl.checkpoint_policy = (
        self.CHECKPOINT_POLICY)

    model_p.lm.stacked_transformer_tpl.dropout_prob = dropout_prob
    transformer_layer_p = (
        model_p.lm.stacked_transformer_tpl.transformer_layer_params_tpl)
    transformer_layer_p.tr_atten_tpl.atten_logit_cap = 50.0
    transformer_layer_p.tr_atten_tpl.use_bias = False
    transformer_layer_p.tr_atten_tpl.combine_qkv = True
    transformer_layer_p.tr_fflayer_tpl.activation = 'GELU'
    softmax_init = WeightInit.Gaussian(1.0 / math.sqrt(self.MODEL_DIMS))
    model_p.lm.softmax_tpl.params_init = softmax_init

    # Enable bf16.
    model_p.fprop_dtype = jnp.bfloat16

    set_default_adam(model_p, self.LEARNING_RATE, self.WEIGHT_DECAY)

    model_p.train.save_interval_steps = 5000

    set_sharding_annotations_v1(model_p, self.MESH_SHAPE)

    return model_p

max_stars_repo_name: drpngx/lingvo

max_issues_repo_name: drpngx/lingvo

max_forks_repo_name: drpngx/lingvo

reason: import numpy,from jax


text:
# coding=utf-8
# Copyright 2019 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for valan.r2r.agent."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

from absl import flags
import numpy as np
import tensorflow.compat.v2 as tf
from valan.framework import common
from valan.framework import hparam
from valan.framework import utils

from valan.r2r import agent
from valan.r2r import agent_config
from valan.r2r import constants
from valan.r2r import env
from valan.r2r import env_config

FLAGS = flags.FLAGS

class AgentR2RTest(tf.test.TestCase):

  def setUp(self):
    super(AgentR2RTest, self).setUp()
    self.num_panos = 36
    self.image_feature_size = 64 + 256
    self.num_actions = 14
    self.time_step = 3
    self.batch_size = 1
    done = np.array([[True], [False], [True]])
    self._test_environment = common.EnvOutput(
        reward=None,
        done=done,
        observation={
            constants.HEADING: np.zeros([self.time_step, self.batch_size, 1]),
            constants.PITCH: np.zeros([self.time_step, self.batch_size, 1]),
            constants.PANO_ENC:
                tf.random.normal([
                    self.time_step, self.batch_size, self.num_panos,
                    self.image_feature_size
                ]),
            constants.CONN_ENC:
                tf.random.normal([
                    self.time_step, self.batch_size, self.num_actions,
                    self.image_feature_size
                ]),
            constants.PREV_ACTION_ENC:
                tf.random.normal([
                    self.time_step, self.batch_size, self.image_feature_size
                ]),
            constants.INS_TOKEN_IDS:
                np.array([
                    [[3, 6, 1, 0, 0]],
                    [[3, 6, 1, 0, 0]],
                    [[3, 6, 1, 0, 0]],
                ]),
            constants.VALID_CONN_MASK:
                np.array([
                    [[1.0] * 14],
                    [[1.0] * 5 + [0.0] * 9],
                    [[1.0] * 2 + [0.0] * 12],
                ])
        },
        info='')

  def test_call_r2r(self):
    self._agent = agent.R2RAgent(agent_config.get_r2r_agent_config())
    self.data_dir = FLAGS.test_srcdir + (
        'valan/r2r/testdata')

    self._env_config = hparam.HParams(
        problem='R2R',
        scan_base_dir=self.data_dir,
        data_base_dir=self.data_dir,
        vocab_dir=self.data_dir,
        vocab_file='vocab.txt',
        images_per_pano=36,
        max_conns=14,
        image_encoding_dim=64,
        direction_encoding_dim=256,
        image_features_dir=os.path.join(self.data_dir, 'image_features'),
        instruction_len=50,
        max_agent_actions=6,
        reward_fn=env_config.RewardFunction.get_reward_fn('distance_to_goal'))

    self._runtime_config = common.RuntimeConfig(task_id=0, num_tasks=1)
    self._env = env.R2REnv(
        data_sources=['R2R_small_split'],
        runtime_config=self._runtime_config,
        env_config=self._env_config)

    env_output = self._env.reset()
    observation = tf.nest.map_structure(lambda t: tf.expand_dims(t, 0),
                                        env_output.observation)
    initial_agent_state = self._agent.get_initial_state(
        observation, batch_size=1)
    # Agent always expects time,batch dimensions. First add and then remove.
    env_output = utils.add_time_batch_dim(env_output)
    agent_output, _ = self._agent(env_output, initial_agent_state)

    self.assertEqual(agent_output.policy_logits.shape, [1, 1, 14])
    self.assertEqual(agent_output.baseline.shape, [1, 1])

    initial_agent_state = ([
        (tf.random.normal([self.batch_size,
                           512]), tf.random.normal([self.batch_size, 512])),
        (tf.random.normal([self.batch_size,
                           512]), tf.random.normal([self.batch_size, 512]))
    ], tf.random.normal([self.batch_size, 5, 512]))
    agent_output, _ = self._agent(self._test_environment, initial_agent_state)

    self.assertEqual(agent_output.policy_logits.shape,
                     [self.time_step, self.batch_size, 14])
    self.assertEqual(agent_output.baseline.shape,
                     [self.time_step, self.batch_size])

  def test_call_ndh(self):
    self._agent = agent.R2RAgent(agent_config.get_ndh_agent_config())
    self.data_dir = FLAGS.test_srcdir + (
        'valan/r2r/testdata')

    self._env_config = hparam.HParams(
        problem='NDH',
        history='all',
        path_type='trusted_path',
        max_goal_room_panos=4,
        scan_base_dir=self.data_dir,
        data_base_dir=self.data_dir,
        vocab_dir=self.data_dir,
        problem_path=os.path.join(self.data_dir, 'NDH'),
        vocab_file='vocab.txt',
        images_per_pano=36,
        max_conns=14,
        image_encoding_dim=64,
        direction_encoding_dim=256,
        image_features_dir=os.path.join(self.data_dir, 'image_features'),
        instruction_len=50,
        max_agent_actions=6,
        reward_fn=env_config.RewardFunction.get_reward_fn('distance_to_goal'))

    self._runtime_config = common.RuntimeConfig(task_id=0, num_tasks=1)
    self._env = env.R2REnv(
        data_sources=['R2R_small_split'],
        runtime_config=self._runtime_config,
        env_config=self._env_config)

    env_output = self._env.reset()
    observation = tf.nest.map_structure(lambda t: tf.expand_dims(t, 0),
                                        env_output.observation)
    initial_agent_state = self._agent.get_initial_state(
        observation, batch_size=1)
    # Agent always expects time,batch dimensions. First add and then remove.
    env_output = utils.add_time_batch_dim(env_output)
    agent_output, _ = self._agent(env_output, initial_agent_state)

    self.assertEqual(agent_output.policy_logits.shape, [1, 1, 14])
    self.assertEqual(agent_output.baseline.shape, [1, 1])

    initial_agent_state = ([
        (tf.random.normal([self.batch_size,
                           512]), tf.random.normal([self.batch_size, 512])),
        (tf.random.normal([self.batch_size,
                           512]), tf.random.normal([self.batch_size, 512]))
    ], tf.random.normal([self.batch_size, 5, 512]))
    agent_output, _ = self._agent(self._test_environment, initial_agent_state)

    self.assertEqual(agent_output.policy_logits.shape,
                     [self.time_step, self.batch_size, 14])
    self.assertEqual(agent_output.baseline.shape,
                     [self.time_step, self.batch_size])

if __name__ == '__main__':
  tf.enable_v2_behavior()
  tf.test.main()

max_stars_repo_name: google-research/valan

max_issues_repo_name: google-research/valan

max_forks_repo_name: google-research/valan


text:
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import csv
import numpy as np
import os
import sys

from observations.util import maybe_download_and_extract

def aircondit(path):
  """Failures of Air-conditioning Equipment

  Proschan (1963) reported on the times between failures of the
  air-conditioning equipment in 10 Boeing 720 aircraft. The `aircondit`
  data frame contains the intervals for the ninth aircraft while
  `aircondit7` contains those for the seventh aircraft.

  Both data frames have just one column. Note that the data have been
  sorted into increasing order.

  The data frames contain the following column:

  `hours`
      The time interval in hours between successive failures of the
      air-conditioning equipment

  The data were taken from

  Cox, D.R. and Snell, E.J. (1981) *Applied Statistics: Principles and
  Examples*. Chapman and Hall.

  Args:

    path: str.
      Path to directory which either stores file or otherwise file will
      be downloaded and extracted there.
      Filename is `aircondit.csv`.

  Returns:

    Tuple of np.ndarray `x_train` with 12 rows and 1 columns and
    dictionary `metadata` of column headers (feature names).
  """
  import pandas as pd
  path = os.path.expanduser(path)
  filename = 'aircondit.csv'
  if not os.path.exists(os.path.join(path, filename)):
    url = 'http://dustintran.com/data/r/boot/aircondit.csv'
    maybe_download_and_extract(path, url,
                               save_file_name='aircondit.csv',
                               resume=False)

  data = pd.read_csv(os.path.join(path, filename), index_col=0,
                     parse_dates=True)
  x_train = data.values
  metadata = {'columns': data.columns}
  return x_train, metadata


text:
from numpy import array
from numpy import mean
from numpy import cov
from numpy.linalg import eig
import matplotlib.pyplot as plt
import numpy as np
import cv2
from PIL import Image

image = cv2.imread('C:/Users/varun/images/test.png')
images = cv2.resize(image, (900,900))
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
grays = cv2.resize(gray, (900,900))
img1 = Image.open('C:/Users/varun/images/test.png').convert('L')
img1_as_np = np.asarray(img1)
vec1 = img1_as_np.flatten()

image2 = cv2.imread('C:/Users/varun/images/test2.png')
images2 = cv2.resize(image, (900,900))
gray2 = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
grays2 = cv2.resize(gray, (900,900))
img2 = Image.open('C:/Users/varun/images/test2.png').convert('L')
img2_as_np = np.asarray(img2)
vec2 = img2_as_np.flatten()

image3 = cv2.imread('C:/Users/varun/images/test3.png')
images3 = cv2.resize(image, (900,900))
gray3 = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
grays3 = cv2.resize(gray, (900,900))
img3 = Image.open('C:/Users/varun/images/test3.png').convert('L')
img3_as_np = np.asarray(img3)
vec3 = img3_as_np.flatten()

image4 = cv2.imread('C:/Users/varun/images/test4.png')
images4 = cv2.resize(image, (900,900))
gray4 = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
grays4 = cv2.resize(gray, (900,900))
img4 = Image.open('C:/Users/varun/images/test4.png').convert('L')
img4_as_np = np.asarray(img4)
vec4 = img4_as_np.flatten()

A = np.column_stack((vec1,vec2,vec3,vec4))
print(A)
A = A.T
#cv2.waitKey(0)
#cv2.destroyAllWindows()
# define a matrix
#A = array([[1, 2], [3, 4], [5, 6]])
print(A)
# calculate the mean of each column
M = mean(A, axis=0)
print(M)
#C = A - M
#-----------------------------------------------------------------------#

#print(C)

V = cov(A)
g ,h = np.linalg.eig(V)
print('cov')
#V = np.dot(C,(C.T))
#print(V)
#values, vectors = eig(V)
u = np.sort(g)
u2 = h[:,g.argsort()]
A=A.T
fineg = np.dot(A,u2)
print(fineg)

fin1 = fineg[:,0]
#fin2 = fineg[:,1]
#fin3 = fineg[:,2]
fin1 = np.array(fin1)
fin1 = np.reshape(fin1,(3456,3456))
cv2.namedWindow('Resized Window', cv2.WINDOW_NORMAL)

#resize the window according to the screen resolution
cv2.resizeWindow('Resized Window', 600, 600)
cv2.imshow('Resized Window',fin1.astype("uint8"))

#print(fin1)
#plt.imshow(fin1, cmap="gray")
#plt.show()
#eweig = np.dot((A.T),A)
#print("================")
#print(eweig)
#val ,neweig = eig(eweig)
#print('========================================')
#print(neweig)
#print(type(neweig))
#onedneweig=neweig.flatten()
#print(onedneweig)
#plt.imshow(neweig, cmap="gray")
#plt.show()
#print(vectors)
#neweig = np.dot(vectors,)
#print(values)
#vectors.resize(3456)
#P = vectors.T.dot(C.T)
#print(P.T)
#fin1 = vectors[:,0]
#fin2 = vectors[:,1]
#fin3 = vectors[:,2]
#cv::imshow("test", eigen2cv(fin1));
#im = Image.fromarray(vectors)
#im.show()

reason: import numpy,from numpy


text:
import pytest
import btk
import imp
import os
import sys
import numpy as np

def compare_basic_metric(param, user_config_dict,
                         simulation_config_dict, btk_input):
    # Set seed
    test_metric_summary = np.array(
        [[5, 3, 2, 0, 0, 3, 2, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [1, 1, 0, 0, 0, 1, 0, 0, 0],
         [6, 2, 4, 0, 0, 2, 4, 0, 0]])
    np.random.seed(int(param.seed))
    draw_blend_generator = btk_input.make_draw_generator(
        param, user_config_dict, simulation_config_dict)
    measure_generator = btk_input.make_measure_generator(
        param, user_config_dict, draw_blend_generator)
    metric_param = btk.utils.Basic_metric_params(
        meas_generator=measure_generator, sim_param=param)
    results = btk.compute_metrics.run(metric_param, test_size=1)
    detected_metrics_summary = results['detection'][2]
    np.testing.assert_array_almost_equal(
        detected_metrics_summary, test_metric_summary, decimal=3,
        err_msg="Did not get desired detection metrics summary")
    pass

def run_metrics_basic():
    test_input = imp.load_source("", 'tests/test_input.py')
    args = test_input.Input_Args()
    sys.path.append(os.getcwd())
    btk_input = __import__('btk_input')
    config_dict = btk_input.read_configfile(args.configfile, args.simulation,
                                            args.verbose)
    simulation_config_dict = config_dict['simulation'][args.simulation]
    simulation_config_dict['max_number'] = 6
    simulation_config_dict['batch_size'] = 4
    user_config_dict = config_dict['user_input']
    catalog_name = os.path.join(
        user_config_dict['data_dir'],
        simulation_config_dict['catalog'])
    # Set parameter values in param
    param = btk_input.get_config_class(simulation_config_dict,
                                       catalog_name, args.verbose)
    compare_basic_metric(param, user_config_dict,
                         simulation_config_dict, btk_input)
    pass

def compare_sep_group_metric(param, user_config_dict,
                             simulation_config_dict, btk_input):
    # Set seed
    test_metric_summary = np.array(
        [[3, 1, 2, 0, 0, 1, 2, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [3, 1, 2, 0, 0, 1, 2, 0, 0],
         [2, 2, 0, 0, 0, 2, 0, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [3, 1, 2, 0, 0, 1, 2, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [5, 2, 3, 0, 0, 2, 3, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [3, 1, 2, 0, 0, 1, 2, 0, 0],
         [5, 2, 3, 0, 0, 2, 3, 0, 0],
         [5, 1, 4, 0, 0, 1, 4, 0, 0],
         [6, 3, 3, 0, 0, 3, 3, 0, 0],
         [2, 1, 1, 0, 0, 1, 1, 0, 0],
         [6, 3, 3, 0, 0, 3, 3, 0, 0]])
    np.random.seed(int(param.seed))
    draw_blend_generator = btk_input.make_draw_generator(
        param, user_config_dict, simulation_config_dict)
    measure_generator = btk_input.make_measure_generator(
        param, user_config_dict, draw_blend_generator)
    metric_param = btk.utils.Basic_metric_params(
        meas_generator=measure_generator, sim_param=param)
    results = btk.compute_metrics.run(metric_param, test_size=2)
    detected_metrics_summary = results['detection'][2]
    np.testing.assert_array_almost_equal(
        detected_metrics_summary, test_metric_summary, decimal=3,
        err_msg="Did not get desired detection metrics summary")
    pass

def run_metrics_sep():
    test_input = imp.load_source("", 'tests/test_input.py')
    simulations = ['group', ]
    for simulation in simulations:
        args = test_input.Input_Args(simulation=simulation)
        sys.path.append(os.getcwd())
        btk_input = __import__('btk_input')
        config_dict = btk_input.read_configfile(
            args.configfile, args.simulation, args.verbose)
        simulation_config_dict = config_dict['simulation'][args.simulation]
        user_config_dict = config_dict['user_input']
        user_config_dict['utils_input']['measure_function'] = 'SEP_params'
        catalog_name = os.path.join(
            user_config_dict['data_dir'],
            simulation_config_dict['catalog'])
        # Set parameter values in param
        param = btk_input.get_config_class(simulation_config_dict,
                                           catalog_name, args.verbose)
        compare_sep_group_metric(param, user_config_dict,
                                 simulation_config_dict, btk_input)
    pass

def compare_stack_group_metric(param, user_config_dict,
                               simulation_config_dict, btk_input):
    # Set seed
    test_metric_summary = np.array([[3, 1, 2, 0, 0, 1, 2, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [3, 1, 2, 0, 0, 1, 2, 0, 0],
                                    [2, 2, 0, 0, 0, 2, 0, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [3, 1, 2, 0, 0, 1, 2, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [5, 0, 5, 0, 0, 0, 5, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [3, 1, 2, 0, 0, 1, 2, 0, 0],
                                    [5, 2, 3, 0, 0, 2, 3, 0, 0],
                                    [5, 1, 4, 0, 0, 1, 4, 0, 0],
                                    [6, 3, 3, 0, 0, 3, 3, 0, 0],
                                    [2, 1, 1, 0, 0, 1, 1, 0, 0],
                                    [6, 3, 3, 0, 0, 3, 3, 0, 0]])
    np.random.seed(int(param.seed))
    draw_blend_generator = btk_input.make_draw_generator(
        param, user_config_dict, simulation_config_dict)
    measure_generator = btk_input.make_measure_generator(
        param, user_config_dict, draw_blend_generator)
    metric_param = btk.utils.Stack_metric_params(
        meas_generator=measure_generator, sim_param=param)
    results = btk.compute_metrics.run(metric_param, test_size=2)
    detected_metrics_summary = results['detection'][2]
    np.testing.assert_array_almost_equal(
        detected_metrics_summary, test_metric_summary, decimal=3,
        err_msg="Did not get desired detection metrics summary")
    pass

def run_metrics_stack():
    test_input = imp.load_source("", 'tests/test_input.py')
    simulations = ['group', ]
    for simulation in simulations:
        args = test_input.Input_Args(simulation=simulation)
        sys.path.append(os.getcwd())
        btk_input = __import__('btk_input')
        config_dict = btk_input.read_configfile(
            args.configfile, args.simulation, args.verbose)
        simulation_config_dict = config_dict['simulation'][args.simulation]
        user_config_dict = config_dict['user_input']
        user_config_dict['utils_input']['measure_function'] = 'Stack_params'
        catalog_name = os.path.join(
            user_config_dict['data_dir'],
            simulation_config_dict['catalog'])
        # Set parameter values in param
        param = btk_input.get_config_class(simulation_config_dict,
                                           catalog_name, args.verbose)
        compare_stack_group_metric(param, user_config_dict,
                                   simulation_config_dict, btk_input)
    pass

@pytest.mark.timeout(15)
def test_metrics_all():
    run_metrics_basic()
    try:
        run_metrics_sep()
    except ImportError:
            print("sep not found")
    try:
        run_metrics_stack()
    except ImportError:
            print("stack not found")

@pytest.mark.timeout(3)
def test_detection_eff_matrix():
    """Tests detection efficiency matrix computation in utils by inputting a
    summary table with 4 entries, with number of true sources between 1-4 and
    all detected and expecting matrix with
    secondary diagonal being one"""
    summary = np.array([[1, 1, 0, 0, 0], [2, 2, 0, 0, 0],
                        [3, 3, 0, 0, 0], [4, 4, 0, 0, 0]])
    num = 4
    eff_matrix = btk.utils.get_detection_eff_matrix(summary, num)
    test_eff_matrix = np.eye(num+2)[:, :num+1] * 100
    test_eff_matrix[0, 0] = 0.
    np.testing.assert_array_equal(eff_matrix, test_eff_matrix,
                                  err_msg="Incorrect efficiency matrix")
    pass

max_stars_repo_name: herjy/BlendingToolKit

max_issues_repo_name: herjy/BlendingToolKit

max_forks_repo_name: herjy/BlendingToolKit


text:
# -*- coding: utf-8 -*-

import os
import inspect
import cPickle as pickle
import numpy as np

from os.path import exists
from skimage.io import imread
from skimage.transform import resize, rotate, swirl
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from random import randint
from math import sqrt

TRAIN_PERCENT = 0.7
VALID_PERCENT = 0.1
TEST_PERCENT = 0.2
SAVE = False

CLASS_NAMES = (
    'Protists',
    'Crustaceans',
    'PelagicTunicates',
    'Artifacts',
    'Chaetognaths',
    'Planktons',
    'Copepods',
    'Ctenophores',
    'ShrimpLike',
    'Detritus',
    'Diotoms',
    'Echinoderms',
    'GelatinousZoolankton',
    'Fish',
    'Gastropods',
    'Hydromedusae',
    'InvertebrateLarvae',
    'Siphonophores',
    'Trichodesmium',
    'Unknowns'
)

CLASSES = {
    'Protists': (0, 1, 2, 82, 83, 84, 85, 86, 90, 91),
    'Crustaceans': (3, 27, 106),
    'PelagicTunicates': (4, 5, 6, 7, 113, 114, 115, 116, 117),
    'Artifacts': (8, 9),
    'Chaetognaths': (10, 11, 12),
    'Planktons': (13, 81),
    'Copepods': (14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26),
    'Ctenophores': (28, 29, 30, 31),
    'ShrimpLike': (32, 47, 48, 92, 93, 94, 95),
    'Detritus': (33, 34, 35, 49),
    'Diotoms': (36, 37),
    'Echinoderms': (38, 39, 40, 41, 42, 43, 44, 45),
    'GelatinousZoolankton': (46, 80),
    'Fish': (50, 51, 52, 53, 54, 55),
    'Gastropods': (56, 87, 88, 89),
    'Hydromedusae': (57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77),
    'InvertebrateLarvae': (78, 79, 107, 112),
    'Siphonophores': (96, 97, 98, 99, 100, 101, 102, 103, 104, 105),
    'Trichodesmium': (108, 109, 110, 111),
    'Unknowns': (118, 119, 120),
}

def print_images(images, folder='cropped/'):
    import imageio as imio
    width = sqrt(images.shape[1])
    for i, img in enumerate(images):
        img = img.reshape(width, width)
        imio.imsave(folder + str(i) + '.jpg', img)

class RotationalDDM(DenseDesignMatrix):
    """
        Rot_angle is the rotational angle allowed.
        To allow all degrees, rot_angle=1,
        to allow only 90°, rot_angle=90
    """

    def __init__(self, X, y, rot_angle=90, y_labels=None):
        self.original_X = X
        self.rot_angle = rot_angle
        super(RotationalDDM, self).__init__(X=X, y=y, y_labels=y_labels)

    def rotation(self, x):
        width = sqrt(x.shape[0])
        angle = randint(0, 359) * self.rot_angle
        img = x.reshape(width, width)
        return rotate(img, angle, mode='nearest').ravel()

    def parallel_rotate(self, X):
        return [self.rotation(x) for x in X]

    def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None,
                 data_specs=None, return_tuple=False):
        self.X = self.parallel_rotate(self.original_X)
        self.X = np.array(self.X)
        print 'Rotated'
        return super(RotationalDDM, self).iterator(
            mode=mode,
            batch_size=batch_size,
            num_batches=num_batches,
            rng=rng,
            data_specs=data_specs,
            return_tuple=return_tuple
        )

class Data(object):

    def __init__(self, size=28, train_perc=TRAIN_PERCENT, valid_perc=VALID_PERCENT, test_perc=TEST_PERCENT, augmentation=0):
        print 'loading data'
        self.size = size
        self.train_perc = train_perc
        self.valid_perc = valid_perc
        self.test_perc = test_perc
        self.augmentation = augmentation
        data, targets = self.get_data(size)
        nb_train = int(self.train_perc * len(targets))
        nb_valid = int(self.valid_perc * len(targets))
        nb_test = int(self.test_perc * len(targets))
        total = nb_train + nb_valid + nb_test
        total_perc = self.train_perc + self.valid_perc + self.test_perc
        data = np.around(data, 4)
        data, targets = self.shuffle_data(data[:total], targets[:total])
        self.train_X = data[:nb_train]
        self.train_Y = targets[:nb_train]
        self.valid_X = data[nb_train:nb_train + nb_valid]
        self.valid_Y = targets[nb_train:nb_train + nb_valid]
        self.test_X = data[nb_train + nb_valid:total]
        self.test_Y = targets[nb_train + nb_valid:total]
        name = 'train' + str(size) + '_' + str(total_perc)
        self.save_set(name, data[:total], targets[:total])

    def create_categories(self):
        train_classes_X = dict()
        train_classes_Y = dict()
        test_classes_X = dict()
        test_classes_Y = dict()
        for name in CLASS_NAMES:
            train_classes_X[name] = []
            train_classes_Y[name] = []
            test_classes_X[name] = []
            test_classes_Y[name] = []
        for img_i, img in enumerate(self.train_X):
            label = self.train_Y[img_i]
            for name in CLASS_NAMES:
                if label in CLASSES[name]:
                    train_classes_X[name].append(img)
                    train_classes_Y[name].append(label)
                    break
        for img_i, img in enumerate(self.test_X):
            label = self.test_Y[img_i]
            for name in CLASS_NAMES:
                if label in CLASSES[name]:
                    test_classes_X[name].append(img)
                    test_classes_Y[name].append(label)
                    break
        total = self.train_perc + self.valid_perc + self.test_perc
        for name in CLASS_NAMES:
            filename = name + str(self.size) + '_' + str(total)
            dir = 'classes/'
            X = train_classes_X[name]
            y = train_classes_Y[name]
            self.save_set('train_' + filename, X, y, dir)
            X = test_classes_X[name]
            y = test_classes_Y[name]
            self.save_set('test_' + filename, X, y, dir)
        self.train_cat_X = train_classes_X
        self.train_cat_Y = train_classes_Y
        self.test_cat_X = test_classes_X
        self.test_cat_Y = test_classes_Y

    def create_parent_labels(self):
        new_train_labels = []
        for y in self.train_Y:
            for new_label, name in enumerate(CLASS_NAMES):
                if y in CLASSES[name]:
                    new_train_labels.append(new_label)
                    break
        new_test_labels = []
        for y in self.test_Y:
            for new_label, name in enumerate(CLASS_NAMES):
                if y in CLASSES[name]:
                    new_test_labels.append(new_label)
                    break
        new_valid_labels = []
        for y in self.valid_Y:
            for new_label, name in enumerate(CLASS_NAMES):
                if y in CLASSES[name]:
                    new_valid_labels.append(new_label)
                    break
        self.train_parent_Y = new_train_labels
        self.valid_parent_Y = new_valid_labels
        self.test_parent_Y = new_test_labels

    def save_set(self, name, X, y,  directory=''):
        if SAVE:
            curr_dir = os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe())))
            filename = os.path.join(curr_dir, directory + name + '.pkl')
            f = open(filename, 'wb')
            pickle.dump((X, y), f, protocol=pickle.HIGHEST_PROTOCOL)
            f.close()

    def convertBinaryValues(self, image_set=None, threshold=0.5):
        binary = np.array(image_set) > threshold
        return binary.astype(int)

    def augment_data(self, image, target):
        images = [image.ravel(), ]
        targets = [target, ]
        image_modifiers = (
            lambda x: rotate(x, 90),
            lambda x: rotate(x, 180),
            lambda x: rotate(x, 270),
            lambda x: rotate(x, 45),
            lambda x: swirl(x)
        )
        for i in xrange(self.augmentation):
            img = image_modifiers[i](image)
            images.append(img.ravel())
            targets.append(target)
        return images, targets

    def create_thumbnail(self, size, img=None):
        print 'processing raw images'
        if img:
            return resize(img, (size, size))
        curr_dir = os.path.dirname(
            os.path.abspath(inspect.getfile(inspect.currentframe())))
        folders = os.walk(os.path.join(curr_dir, '../../data/train/'))
        images = []
        classes = []
        targets = []
        for class_id, folder in enumerate(folders):
            classes.append(folder[0][17:])
            for img in folder[2]:
                if img.index('.jpg') == -1:
                    continue
                image = imread(folder[0] + '/' + img)
                image = resize(image, (size, size))
                # Important to put -1, to have it 0-based.
                target = class_id - 1
                new_images, new_targets = self.augment_data(image, target)
                images.extend(new_images)
                targets.extend(new_targets)
        train = (images, targets)
        self.save_set('train' + str(size), images, targets)
        # f = open(curr_dir + '/train' + str(size) + '.pkl', 'wb')
        # pickle.dump(train, f, protocol=pickle.HIGHEST_PROTOCOL)
        # f.close()
        return train

    def shuffle_data(self, X, y):
        shp = np.shape(X)
        shuffle = np.zeros((shp[0], shp[1] + 1))
        shuffle[:, :-1] = X
        shuffle[:, -1] = y
        np.random.shuffle(shuffle)
        return (shuffle[:, :-1], shuffle[:, -1])

    def get_data(self, size):
        curr_dir = os.path.dirname(
            os.path.abspath(inspect.getfile(inspect.currentframe())))
        filename = os.path.join(curr_dir, 'train' + str(size) + '.pkl')
        total = self.train_perc + self.valid_perc + self.test_perc
        previous_file = os.path.join(
            curr_dir, 'train' + str(size) + '_' + str(total) + '.pkl')
        if exists(previous_file):
            print 'loaded from smaller dump'
            f = open(previous_file, 'rb')
            content = pickle.load(f)
            f.close()
            return content
        if not exists(filename):
            return self.create_thumbnail(size)
        return pickle.load(open(filename, 'rb'))

if __name__ == '__main__':
        print 'processing raw images'
        curr_dir = os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe())))
        folders = os.walk(os.path.join(curr_dir, '../../data/train/'))
        for class_id, folder in enumerate(folders):
            print folder[0]


text:
import platform
import unittest

import numpy as np
import pandas as pd

from zipline.data.hdf5_daily_bars import (
    HDF5DailyBarReader,
    HDF5DailyBarWriter,
    MultiCountryDailyBarReader,
)
import zipline.testing.fixtures as zp_fixtures
from zipline.testing.predicates import assert_equal

@unittest.skipIf(platform.system() == 'Windows', "Don't run test on windows")
class H5WriterTestCase(zp_fixtures.WithTmpDir,
                       zp_fixtures.ZiplineTestCase):

    def test_write_empty_country(self):
        """
        Test that we can write an empty country to an HDF5 daily bar writer.

        This is useful functionality for some tests, but it requires a bunch of
        special cased logic in the writer.
        """
        path = self.tmpdir.getpath('empty.h5')
        writer = HDF5DailyBarWriter(path, date_chunk_size=30)
        writer.write_from_sid_df_pairs('US', iter(()))

        reader = HDF5DailyBarReader.from_path(path, 'US')

        assert_equal(reader.sids, np.array([], dtype='int64'))

        empty_dates = np.array([], dtype='datetime64[ns]')
        assert_equal(reader.asset_start_dates, empty_dates)
        assert_equal(reader.asset_end_dates, empty_dates)
        assert_equal(reader.dates, empty_dates)

    def test_multi_country_attributes(self):
        path = self.tmpdir.getpath('multi.h5')
        writer = HDF5DailyBarWriter(path, date_chunk_size=30)

        US = pd.DataFrame(
            data=np.ones((3, 5)),
            index=pd.to_datetime(['2014-01-02', '2014-01-03', '2014-01-06']),
            columns=np.arange(1, 6),
        )

        CA = pd.DataFrame(
            data=np.ones((2, 4)) * 2,
            index=pd.to_datetime(['2014-01-04', '2014-01-07']),
            columns=np.arange(100, 104),
        )

        def ohlcv(frame):
            return {
                'open': frame,
                'high': frame,
                'low': frame,
                'close': frame,
                'volume': frame,
            }

        writer.write('US', ohlcv(US))
        writer.write('CA', ohlcv(CA))

        reader = MultiCountryDailyBarReader.from_path(path)
        assert_equal(reader.countries, {'US', 'CA'})
        assert_equal(
            reader.sessions,
            pd.to_datetime([
                '2014-01-02',
                '2014-01-03',
                '2014-01-04',
                '2014-01-06',
                '2014-01-07',
            ], utc=True)
        )

max_stars_repo_name: TheTradingAngel/zipline-trader

max_issues_repo_name: TheTradingAngel/zipline-trader

max_forks_repo_name: TheTradingAngel/zipline-trader


text:
import numpy

def fft_wavefunction(psi, nx, ny, ns, sin):
    return numpy.fft.fft2(psi.reshape(nx,ny,ns),
                          axes=(0,1)).reshape(sin)

def ifft_wavefunction(psi, nx, ny, ns, sin):
    return numpy.fft.ifft2(psi.reshape(nx,ny,ns),
                                 axes=(0,1)).reshape(sin)

max_stars_repo_name: pauxy-qmc/pauxy

max_issues_repo_name: pauxy-qmc/pauxy

max_forks_repo_name: pauxy-qmc/pauxy


text:
# emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
# ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the NiBabel package for the
#   copyright and license terms.
#
# ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
""" Testing fileutils module
"""

from ..fileutils import read_zt_byte_strings

from numpy.testing import (assert_almost_equal,
                           assert_array_equal)

from nose.tools import (assert_true, assert_false, assert_raises,
                        assert_equal, assert_not_equal)

from ..tmpdirs import InTemporaryDirectory

def test_read_zt_byte_strings():
    # sample binary block
    binary = b'test.fmr\x00test.prt\x00something'
    with InTemporaryDirectory():
        # create a tempfile
        path = 'test.bin'
        fwrite = open(path, 'wb')
        # write the binary block to it
        fwrite.write(binary)
        fwrite.close()
        # open it again
        fread = open(path, 'rb')
        # test readout of one string
        assert_equal(read_zt_byte_strings(fread), [b'test.fmr'])
        # test new file position
        assert_equal(fread.tell(), 9)
        # manually rewind
        fread.seek(0)
        # test readout of two strings
        assert_equal(read_zt_byte_strings(fread, 2),
                     [b'test.fmr', b'test.prt'])
        assert_equal(fread.tell(), 18)
        # test readout of more strings than present
        fread.seek(0)
        assert_raises(ValueError, read_zt_byte_strings, fread, 3)
        fread.seek(9)
        assert_raises(ValueError, read_zt_byte_strings, fread, 2)
        # Try with a small bufsize
        fread.seek(0)
        assert_equal(read_zt_byte_strings(fread, 2, 4),
                     [b'test.fmr', b'test.prt'])
        fread.close()

max_stars_repo_name: Raniac/NEURO-LEARN

max_forks_repo_name: Raniac/NEURO-LEARN

reason: from numpy


text:
import numpy as np
import os
import heapq
import argparse
import threading
import multiprocessing
import sys
import Queue

def read_seed_info(path):
    if os.path.exists(path):
        try:
            f = open(path, 'r')
            txt = f.readlines()
            seeds = list()
            for line in txt:
                seeds.append(int(line))
            return seeds
        except IOError:
            print 'IOError'
    else:
        print 'file can not found'

# read and analyse the data in the file to obtain a graph object
def read_graph_info(path):
    if os.path.exists(path):
        parents = {}
        children = {}
        edges = {}
        nodes = set()

        try:
            f = open(path, 'r')
            txt = f.readlines()
            header = str.split(txt[0])
            node_num = int(header[0])
            edge_num = int(header[1])

            for line in txt[1:]:
                row = str.split(line)

                src = int(row[0])
                des = int(row[1])
                nodes.add(src)
                nodes.add(des)

                if children.get(src) is None:
                    children[src] = []
                if parents.get(des) is None:
                    parents[des] = []

                weight = float(row[2])
                edges[(src, des)] = weight
                children[src].append(des)
                parents[des].append(src)

            return list(nodes), edges, children, parents, node_num, edge_num
        except IOError:
            print 'IOError'
    else:
        print 'file can not found'

def happen_with_prop(rate):
    rand = np.random.ranf()
    if rand <= rate:
        return True
    else:
        return False

def print_seeds(seeds):
    for seed in seeds:
        print(seed)

class Graph:
    nodes = None
    edges = None
    children = None
    parents = None
    node_num = None
    edge_num = None

    def __init__(self, (nodes, edges, children, parents, node_num, edge_num)):
        self.nodes = nodes
        self.edges = edges
        self.children = children
        self.parents = parents
        self.node_num = node_num
        self.edge_num = edge_num

    def get_children(self, node):
        ch = self.children.get(node)
        if ch is None:
            self.children[node] = []
        return self.children[node]

    def get_parents(self, node):
        pa = self.parents.get(node)
        if pa is None:
            self.parents[node] = []
        return self.parents[node]

    def get_weight(self, src, dest):
        weight = self.edges.get((src, dest))
        if weight is None:
            return 0
        else:
            return weight

    # return true if node1 is parent of node 2 , else return false
    def is_parent_of(self, node1, node2):
        if self.get_weight(node1, node2) != 0:
            return True
        else:
            return False

    # return true if node1 is child of node 2 , else return false
    def is_child_of(self, node1, node2):
        return self.is_parent_of(node2, node1)

    def get_out_degree(self, node):
        return len(self.get_children(node))

    def get_in_degree(self, node):
        return len(self.get_parents(node))

# base on heap queue
# The CELF queue is maintained in decreasing order of the marginal
# gains and thus, no other node can have a larger marginal gain.
class CELFQueue:
    # create if not exist
    nodes = None
    q = None
    nodes_gain = None

    def __init__(self):
        self.q = []
        self.nodes_gain = {}

    def put(self, node, marginalgain):
        self.nodes_gain[node] = marginalgain
        heapq.heappush(self.q, (-marginalgain, node))

    def update(self, node, marginalgain):
        self.remove(node)
        self.put(node, marginalgain)

    def remove(self, node):
        self.q.remove((-self.nodes_gain[node], node))
        self.nodes_gain[node] = None
        heapq.heapify(self.q)

    def topn(self, n):
        top = heapq.nsmallest(n, self.q)
        top_ = list()
        for t in top:
            top_.append(t[1])
        return top_

    def get_gain(self, node):
        return self.nodes_gain[node]

def get_sample_graph(graph):
    nodes = graph.nodes
    edges = {}
    children = {}
    parents = {}
    node_num = graph.node_num

    for edge in graph.edges:
        if happen_with_prop(graph.edges[edge]):
            edges[edge] = graph.edges[edge]
            src = edge[0]
            des = edge[1]

            if children.get(src) is None:
                children[src] = []
            if parents.get(des) is None:
                parents[des] = []

            children[src].append(des)
            parents[des].append(src)
    return Graph((nodes, edges, children, parents, node_num, len(edges)))

def BFS(graph, nodes, get_checked_array=False):
    node_list = list()
    node_list.extend(nodes)
    result_list = list()
    checked = np.zeros(graph.node_num)
    for node in node_list:
        checked[node - 1] = 1
    while len(node_list) != 0:
        current_node = node_list.pop(0)
        result_list.append(current_node)
        children = graph.get_children(current_node)
        for child in children:
            if checked[child - 1] == 0:
                checked[child - 1] = 1
                node_list.append(child)
    if get_checked_array:
        return result_list, checked
    return result_list

# graph : Graph
# seed : list
# sample_num : int
# graph : Graph
# seed : list
# sample_num : int
# use multiple thread
def influence_spread_computation_IC(graph, seeds, sample_num=10000):
    influence = 0
    for i in range(sample_num):
        node_list = list()
        node_list.extend(seeds)
        checked = np.zeros(graph.node_num)
        for node in node_list:
            checked[node - 1] = 1
        while len(node_list) != 0:
            current_node = node_list.pop(0)
            influence = influence + 1
            children = graph.get_children(current_node)
            for child in children:
                if checked[child - 1] == 0:
                    if happen_with_prop(graph.get_weight(current_node, child)):
                        checked[child - 1] = 1
                        node_list.append(child)
    return influence

def influence_spread_computation_IC_Mu(graph, seeds, n=multiprocessing.cpu_count()):
    pool = multiprocessing.Pool()
    results = []
    sub = int(10000 / n)
    for i in range(n):
        result = pool.apply_async(influence_spread_computation_IC, args=(graph, seeds, sub))
        results.append(result)
    pool.close()
    pool.join()
    influence = 0
    for result in results:
        influence = influence + result.get()
    return influence / 10000

# k: seed size
def new_greedyIC(graph, k, R=10000):
    seeds = set()
    for i in range(k):
        sv = np.zeros(graph.node_num)
        afficted_nodes, is_afficted = BFS(graph, list(seeds), True)
        # for seed in seeds:
        for i in range(R):
            sample_graph = get_sample_graph(graph)
            for v in range(graph.node_num):
                if is_afficted[v] == 0:
                    sv[v] = sv[v] + len(BFS(sample_graph, [v + 1]))
        for i in range(graph.node_num):
            sv[i] = sv[i] / R
        seeds.add(sv.argmax() + 1)
    return list(seeds)

def sample_IC(graph, is_afficted, seed, R):
    sv = np.zeros(graph.node_num)
    for i in range(R):
        sample_graph = get_sample_graph(graph)
        for v in range(graph.node_num):
            if is_afficted[v] == 0:
                sv[v] = sv[v] + len(BFS(sample_graph, [v + 1]))
    return sv

def new_greedyIC_Mu(graph, k, R=10000, n=multiprocessing.cpu_count()):
    sub = int(R / n)
    seeds = set()
    for i in range(k):
        sv = np.zeros(graph.node_num)
        afficted_nodes, is_afficted = BFS(graph, list(seeds), True)
        pool = multiprocessing.Pool()
        results = []
        for i in range(n):
            result = pool.apply_async(sample_IC, args=(graph, is_afficted, seeds, sub))
            results.append(result)
        pool.close()
        pool.join()
        for result in results:
            for i in range(len(sv)):
                sv[i] = sv[i] + result.get()[i]
        for i in range(len(sv)):
            sv[i] = sv[i] / R
        seeds.add(sv.argmax() + 1)
    return list(seeds)

# Heuristics algorithm for IC model
def degree_discount_ic(k, graph):
    seeds = []
    ddv = np.zeros(graph.node_num)
    tv = np.zeros(graph.node_num)
    for i in range(graph.node_num):
        ddv[i] = graph.get_out_degree(i + 1)
    for i in range(k):
        u = ddv.argmax() + 1
        ddv[u - 1] = -1  # never used
        seeds.append(u)
        children = graph.get_children(u)
        for child in children:
            if child not in seeds:
                tv[child - 1] = tv[child - 1] + 1
                ddv[child - 1] = ddv[child - 1] - 2 * tv[child - 1] - (graph.get_out_degree(child) - tv[
                    child - 1]) * tv[child - 1] * graph.get_weight(u, child)
    return seeds

# BAD Heuristics algorithm for IC model
def degree_discount(k, graph):
    seeds = []
    ddv = np.zeros(graph.node_num)

    for i in range(graph.node_num):
        ddv[i] = graph.get_out_degree(i + 1)
    for i in range(k):
        u = ddv.argmax() + 1
        ddv[u - 1] = -1  # never used
        seeds.append(u)
        children = graph.get_children(u)
        for child in children:
            if child not in seeds:
                ddv[child - 1] = ddv[child - 1] - 1
    return seeds

def init_D(graph):
    D = list()
    for i in range(graph.node_num + 1):
        D.append([])
    return D

def get_vertex_cover(graph):
    # dv[i] out degree of node i+1
    dv = np.zeros(graph.node_num)
    # e[i,j] = 0: edge (i+1,j+1),(j+1,i+1) checked
    check_array = np.zeros((graph.node_num, graph.node_num))
    checked = 0

    for i in range(graph.node_num):
        # for a edge (i,j) and (j,i) may be count twice but the algorithm is to find a vertex cover. it doesn't mater
        dv[i] = graph.get_out_degree(i + 1) + graph.get_in_degree(i + 1)
    # V: Vertex cover
    V = set()
    while checked < graph.edge_num:
        s = dv.argmax() + 1
        V.add(s)
        # make sure that never to select this node again
        children = graph.get_children(s)
        parents = graph.get_parents(s)
        for child in children:
            if check_array[s - 1][child - 1] == 0:
                check_array[s - 1][child - 1] = 1
                checked = checked + 1
        for parent in parents:
            if check_array[parent - 1][s - 1] == 0:
                check_array[parent - 1][s - 1] = 1
                checked = checked + 1
        dv[s - 1] = -1
    return list(V)

# input: Q,D,spd,pp,r,W,U
# Q:list
# D:list(list) D[x]: explored neighbor of x
# spd ,pp,r: float
# W node_set np.array
# U: list
# spdW_
def forward(Q, D, spd, pp, r, W, U, spdW_u, graph):
    x = Q[-1]
    if U is None:
        U = []
    children = graph.get_children(x)
    count = 0
    while True:
        # any suitable chid is ok

        for child in range(count, len(children)):
            if (children[child] in W) and (children[child] not in Q) and (children[child] not in D[x]):
                y = children[child]
                break
            count = count + 1

        # no such child:
        if count == len(children):
            return Q, D, spd, pp

        if pp * graph.get_weight(x, y) < r:
            D[x].append(y)
        else:
            Q.append(y)
            pp = pp * graph.get_weight(x, y)
            spd = spd + pp
            D[x].append(y)
            x = Q[-1]
            for v in U:
                if v not in Q:
                    spdW_u[v] = spdW_u[v] + pp
            children = graph.get_children(x)
            count = 0

def backtrack(u, r, W, U, spdW_, graph):
    Q = [u]
    spd = 1
    pp = 1
    D = init_D(graph)

    while len(Q) != 0:
        Q, D, spd, pp = forward(Q, D, spd, pp, r, W, U, spdW_, graph)
        u = Q.pop()
        D[u] = []
        if len(Q) != 0:
            v = Q[-1]
            pp = pp / graph.get_weight(v, u)
    return spd

def simpath_spread(S, r, U, graph, spdW_=None):
    spread = 0
    # W: V-S
    W = set(graph.nodes).difference(S)
    if U is None or spdW_ is None:
        spdW_ = np.zeros(graph.node_num + 1)
        # print 'U None'
    for u in S:
        W.add(u)
        # print spdW_[u]
        spread = spread + backtrack(u, r, W, U, spdW_[u], graph)
        # print spdW_[u]
        W.remove(u)
    return spread

def simpath(graph, k, r, l):
    C = set(get_vertex_cover(graph))
    V = set(graph.nodes)

    V_C = V.difference(C)
    # spread[x] is spd of S + x
    spread = np.zeros(graph.node_num + 1)
    spdV_ = np.ones((graph.node_num + 1, graph.node_num + 1))
    for u in C:
        U = V_C.intersection(set(graph.get_parents(u)))
        spread[u] = simpath_spread(set([u]), r, U, graph, spdV_)
    for v in V_C:
        v_children = graph.get_children(v)
        for child in v_children:
            spread[v] = spread[v] + spdV_[child][v] * graph.get_weight(v, child)
        spread[v] = spread[v] + 1
    celf = CELFQueue()
    # put all nodes into celf queqe
    # spread[v] is the marginal gain at this time
    for node in range(1, graph.node_num + 1):
        celf.put(node, spread[node])
    S = set()
    W = V
    spd = 0
    # mark the node that checked before during the same Si
    checked = np.zeros(graph.node_num + 1)

    while len(S) < k:
        U = celf.topn(l)
        spdW_ = np.ones((graph.node_num + 1, graph.node_num + 1))
        spdV_x = np.zeros(graph.node_num + 1)
        simpath_spread(S, r, U, graph, spdW_=spdW_)
        for x in U:
            for s in S:
                spdV_x[x] = spdV_x[x] + spdW_[s][x]
        for x in U:
            if checked[x] != 0:
                S.add(x)
                W = W.difference(set([x]))
                spd = spread[x]
                # print spread[x],simpath_spread(S,r,None,None)
                checked = np.zeros(graph.node_num + 1)
                celf.remove(x)
                break
            else:
                spread[x] = backtrack(x, r, W, None, None, graph) + spdV_x[x]
                checked[x] = 1
                celf.update(x, spread[x] - spd)
    return S

def main(args, q):
    graph_path = args.graph_path
    seed_size = args.seed_size
    model = args.model
    random_seed = args.random_seed
    np.random.seed(random_seed)
    graph = Graph(read_graph_info(graph_path))

    if model == 'IC':
        seeds = degree_discount_ic(k=seed_size, graph=graph)
        q.put(seeds)
        seeds = new_greedyIC(graph, k=seed_size, R=10000)
        q.put(seeds)

    elif model == 'LT':
        seeds = degree_discount(seed_size, graph)
        q.put(seeds)
        seeds = simpath(graph, seed_size, 0.001, 7)
        q.put(seeds)
    else:
        raise ValueError('Model type err')

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-i', help='CARP instance file', dest='graph_path')
    parser.add_argument('-k', type=int, help='predefined size of the seed set', dest='seed_size')
    parser.add_argument('-m', help='diffusion model', dest='model')
    parser.add_argument('-b', type=int,
                        help='specifies the termination manner and the value can only be 0 or 1. If it is set to 0, '
                             'the termination condition is as the same defined in your algorithm. Otherwise, '
                             'the maximal time budget specifies the termination condition of your algorithm.',
                        dest='type')
    parser.add_argument('-t', type=float, help='time budget', dest='timeout')
    parser.add_argument('-r', type=int, help='random seed', dest='random_seed')

    args = parser.parse_args()
    graph_path = args.graph_path
    seed_size = args.seed_size
    model = args.model
    termination_type = args.type

    timeout = args.timeout
    random_seed = args.random_seed

    np.random.seed(random_seed)

    if termination_type == 0:
        graph = Graph(read_graph_info(graph_path))
        if model == 'IC':
            if graph.node_num <= 300:
                seeds = new_greedyIC_Mu(graph, k=seed_size, R=10000)
                print_seeds(seeds)
            else:
                seeds = degree_discount_ic(k=seed_size, graph=graph)
                print_seeds(seeds)

        elif model == 'LT':
            seeds = simpath(graph, seed_size, 0.001, 7)
            print_seeds(seeds)
        else:
            print('Model type err')
    elif termination_type == 1:

        q = Queue.Queue()
        t = threading.Thread(target=main, args=(args, q))
        t.setDaemon(True)
        t.start()
        t.join(timeout - 0.5)
        result = list()
        while not q.empty():
            result.append(q.get())
        if len(result) != 0:
            seeds = result[-1]
            print_seeds(seeds)
        else:
            print("Given time was too short")

max_stars_repo_name: Wang-GY/IMP

max_issues_repo_name: Wang-GY/IMP

max_forks_repo_name: Wang-GY/IMP