text:
# -*- coding: utf-8 -*-

"""Dataset-loading utilities module."""

import os.path

import numpy as np
import scipy.sparse as sps

import pandas as pd
import networkx as nx
import pickle as pkl

from functools import partial

from sklearn.preprocessing import LabelBinarizer
from sklearn.datasets import load_iris as _load_iris
from scipy.sparse import coo_matrix

from sklearn.preprocessing import normalize

from variational_gcn.utils.base import (
    recursive_stratified_shuffle_split,
    indices_to_mask,
    mask_values,
)

import tensorflow as tf

def load_pickle(name, ext, data_home="datasets", encoding="latin1"):

    path = os.path.join(data_home, name, "ind.{0}.{1}".format(name, ext))

    with open(path, "rb") as f:

        return pkl.load(f, encoding=encoding)

def load_test_indices(name, data_home="datasets"):

    indices_df = pd.read_csv(
        os.path.join(data_home, name, "ind.{0}.test.index".format(name)), header=None
    )
    indices = indices_df.values.squeeze()

    return indices

def load_dataset(name, data_home="datasets"):

    exts = ["tx", "ty", "allx", "ally", "graph"]

    (X_test, y_test, X_rest, y_rest, G_dict) = map(
        partial(load_pickle, name, data_home=data_home), exts
    )

    _, D = X_test.shape
    _, K = y_test.shape

    ind_test_perm = load_test_indices(name, data_home)
    ind_test = np.sort(ind_test_perm)

    num_test = len(ind_test)
    num_test_full = ind_test[-1] - ind_test[0] + 1

    # TODO: Issue warning if `num_isolated` is non-zero.
    num_isolated = num_test_full - num_test

    # normalized zero-based indices
    ind_test_norm = ind_test - np.min(ind_test)

    # features
    X_test_full = sps.lil_matrix((num_test_full, D))
    X_test_full[ind_test_norm] = X_test

    X_all = sps.vstack((X_rest, X_test_full)).toarray()
    X_all[ind_test_perm] = X_all[ind_test]

    # targets
    y_test_full = np.zeros((num_test_full, K))
    y_test_full[ind_test_norm] = y_test

    y_all = np.vstack((y_rest, y_test_full))
    y_all[ind_test_perm] = y_all[ind_test]

    # graph
    G = nx.from_dict_of_lists(G_dict)
    A = nx.to_scipy_sparse_matrix(G, format="coo")

    return (X_all, y_all, A)

def load_split(name, val_size=500, data_home="datasets"):

    y = load_pickle(name, ext="y", data_home=data_home)

    train_size = len(y)

    ind_train = range(train_size)
    ind_val = range(train_size, train_size + val_size)

    ind_test_perm = load_test_indices(name, data_home)
    ind_test = np.sort(ind_test_perm)

    return ind_train, ind_val, ind_test

def load_iris():

    X_all, y = _load_iris(return_X_y=True)
    y_all = LabelBinarizer().fit_transform(y)

    n, d = X_all.shape

    # no graph structure
    A = coo_matrix((n, n))

    return (X_all, y_all, A)

def load_karate_club():

    G = nx.karate_club_graph()

    A = nx.to_scipy_sparse_matrix(G, format="coo")

    # no features
    X_all = np.eye(G.number_of_nodes())

    labels = [G.node[n]["club"] for n in G.nodes()]
    y_all = LabelBinarizer().fit_transform(labels)

    return (X_all, y_all, A)

def load_twitter(data_home="datasets/twitter"):

    # Load the node features. The index is node IDs
    df = pd.read_csv(
        os.path.join(data_home, "twitter_node_features.csv"), header=0, index_col=0
    )

    print("Loaded node features {}".format(df.shape))
    # Extract the target values

    df_y = pd.DataFrame({"hate": df["hate"], "not-hate": 1 - df["hate"]})

    y_all = df_y.values
    # y_all = LabelBinarizer().fit_transform(y_all)
    # Drop the column with the target values
    df.drop(columns=["hate"], inplace=True)

    X_all = df.values

    print("y_all shape {}".format(y_all.shape))
    print("X_all shape {}".format(X_all.shape))

    # Load the graph
    print("Loading graph")
    g_nx = nx.read_adjlist(os.path.join(data_home, "twitter.edges"), nodetype=int)
    print("...Done")
    print(
        "Graph num nodes {} and edges {}".format(
            g_nx.number_of_nodes(), g_nx.number_of_edges()
        )
    )
    # Extract the adjacency matrix.
    # A should be scipy sparse matrix
    print("Extracting A from networkx graph")
    A = nx.adjacency_matrix(g_nx, nodelist=df.index, weight=None)
    # A = nx.adjacency_matrix(g_nx)
    print("...Done")
    return (X_all, y_all, A)

def load_polblogs(data_home="datasets/polblogs"):

    # load the graph
    g_nx = nx.read_gpickle(os.path.join(data_home, "polblogs_graph.gpickle"))

    # Load the labels
    y_all = np.load(os.path.join(data_home, "polblogs_labels.npy"))

    df_y = pd.DataFrame({"a": y_all, "b": 1 - y_all})
    y_all = df_y.values

    X_all = np.eye(g_nx.number_of_nodes())

    print("y_all shape {}".format(y_all.shape))
    print("X_all shape {}".format(X_all.shape))

    # Extract the adjacency matrix.
    # A should be scipy sparse matrix
    print("Extracting A from networkx graph")
    A = nx.to_scipy_sparse_matrix(g_nx, format="coo", dtype=np.int)
    # Remove self loops and set maximum value to 1
    adj = A.todense()
    adj[adj>1] = 1
    np.fill_diagonal(adj, 0)
    # convert back to scipy sparse matrix
    A = coo_matrix(adj)
    print("...Done")

    return (X_all, y_all, A)

def load_cora(data_home="datasets/feature_based_datasets/cora", cites_filename="cora.cites"):

    df = pd.read_csv(
        os.path.join(data_home, "cora.content"), sep=r"\s+", header=None, index_col=0
    )
    features_df = df.iloc[:, :-1]
    labels_df = df.iloc[:, -1]

    X_all = features_df.values

    y_all = LabelBinarizer().fit_transform(labels_df.values)

    fname_ext = cites_filename.split(".")[-1]

    if fname_ext == "adjlist":
        g_ = nx.read_adjlist(os.path.join(data_home, cites_filename))
        A = nx.to_scipy_sparse_matrix(g_, nodelist=sorted(df.index), format="coo")
    elif fname_ext == "gpickle":
        g_ = nx.read_gpickle(os.path.join(data_home, cites_filename))
        A = nx.to_scipy_sparse_matrix(g_, format="coo")
    else:
        edge_list_df = pd.read_csv(
            os.path.join(data_home, cites_filename), sep=r"\s+", header=None
        )

        idx_map = {j: i for i, j in enumerate(df.index)}

        H = nx.from_pandas_edgelist(edge_list_df, 0, 1)
        G = nx.relabel.relabel_nodes(H, idx_map)

        A = nx.to_scipy_sparse_matrix(G, nodelist=sorted(G.nodes()), format="coo")

    return (X_all, y_all, A)

def load_citeseer(data_home="datasets/feature_based_datasets/citeseer"):

    df = pd.read_csv(
        os.path.join(data_home, "citeseer.content"),
        sep=r"\s+",
        header=None,
        index_col=0,
    )
    df.index = df.index.map(str)

    features_df = df.iloc[:, :-1]
    labels_df = df.iloc[:, -1]

    X_all = features_df.values

    y_all = LabelBinarizer().fit_transform(labels_df.values)

    edge_list_df = pd.read_csv(
        os.path.join(data_home, "citeseer.cites"), sep=r"\s+", dtype=str, header=None
    )

    idx_map = {j: i for i, j in enumerate(df.index)}

    H = nx.from_pandas_edgelist(edge_list_df, 0, 1)
    G = nx.relabel.relabel_nodes(H, idx_map)

    # This dataset has about 15 nodes in the edge list that don't have corresponding entries
    # in citeseer.content, that is don't have features. We need to identify them and then remove
    # them from the graph along with all the edges to/from them.
    nodes_to_remove = [n for n in G.nodes() if type(n) == str]
    G.remove_nodes_from(nodes_to_remove)

    A = nx.to_scipy_sparse_matrix(G, nodelist=sorted(G.nodes()), format="coo")

    return (X_all, y_all, A)

def load_telco(data_home="datasets/telco"):

    df = pd.read_csv(
        os.path.join(data_home, "Telco-Customer-Churn-prepared.csv"), header=0
    )

    df.index = df["customerID"]
    df.drop(columns=["customerID"], inplace=True)

    y_all = df["Churn"].values
    y_all = LabelBinarizer().fit_transform(y_all)

    df.drop(columns=["Churn"], inplace=True)

    X_all = df.values

    n, d = X_all.shape
    # create the adjacency matrix
    A = coo_matrix((n, n))

    return (X_all, y_all, A)

def read_r_data(name, path):

    # importing here is naughty but need to isolate the side-effects
    # of importing, which in this case creates an interactive shell session...
    from rpy2.robjects import r

    load = r["load"]
    load(path)

    H = nx.from_numpy_array(np.asarray(r[name]))
    G = nx.relabel_nodes(H, dict(enumerate(r[name].colnames)))

    return G

def load_protein(data_home="datasets"):

    G = read_r_data(name="Y_Pro", path=os.path.join(data_home, "protein/Y_Pro.rda"))
    A = nx.to_scipy_sparse_matrix(G, format="coo")

    X_all = np.eye(G.number_of_nodes())
    y_all = np.zeros(shape=(G.number_of_nodes(), 2))

    return (X_all, y_all, A)

def load_genesis(data_home="datasets"):

    G = read_r_data(name="Y_Gen", path=os.path.join(data_home, "genesis/Y_Gen.rda"))

    A = nx.to_scipy_sparse_matrix(G, format="coo")

    X_all = np.eye(G.number_of_nodes())
    y_all = np.zeros(shape=(G.number_of_nodes(), 2))

    return (X_all, y_all, A)

def get_data(
    dataset_name,
    random_split,
    split_sizes,
    random_state,
    balanced_split=False,
    samples_per_class=20,
    edgelist=None,
    get_y_values=False
):

    tf.logging.info("Loading '{}' dataset...".format(dataset_name))
    loader = DATASET_LOADERS[dataset_name]
    X, y, A = loader()
    if dataset_name == "cora_edgelist":
        tf.logging.info("Loading with edgelist '{}'".format(edgelist))
        g_ = nx.read_gpickle(os.path.join("attacked_datasets/cora", edgelist))
        A = nx.adjacency_matrix(g_)
    elif dataset_name == "citeseer_edgelist":
        tf.logging.info("Loading with edgelist '{}'".format(edgelist))
        g_ = nx.read_gpickle(os.path.join("attacked_datasets/citeseer", edgelist))
        A = nx.adjacency_matrix(g_)
    elif dataset_name == "pubmed_edgelist":
        tf.logging.info("Loading with edgelist '{}'".format(edgelist))
        g_ = nx.read_gpickle(os.path.join("attacked_datasets/pubmed", edgelist))
        A = nx.adjacency_matrix(g_)
    elif dataset_name == "polblogs_edgelist":
        tf.logging.info("Loading with edgelist '{}'".format(edgelist))
        g_ = nx.read_gpickle(os.path.join("attacked_datasets/polblogs", edgelist))
        A = nx.adjacency_matrix(g_).astype(np.int)
        # Remove self loops and set maximum value to 1
        adj = A.todense()
        adj[adj > 1] = 1
        adj[adj < 0] = 0
        np.fill_diagonal(adj, 0)
        # convert back to scipy sparse matrix
        A = coo_matrix(adj)
    X = normalize(X, norm="l1", axis=1)

    n, d = X.shape
    _, k = y.shape

    tf.logging.info("Dataset has {} samples, dimensionality {}".format(n, d))
    tf.logging.info("Targets belong to {} classes".format(k))

    if random_split:
        split = recursive_stratified_shuffle_split(
            sizes=split_sizes, random_state=random_state
        )
        indices = list(split(X, y))
    elif balanced_split:
        indices = balanced_data_split(X, y, samples_per_class, random_state=random_state)
    else:
        if dataset_name in ("cora_knn", "cora_supervised", "cora_jaccard", "cora_tsne", "cora_cosine","cora_edgelist"):
            indices = load_split(name="cora")
        elif dataset_name in ('citeseer_edgelist'):
            indices = load_split("citeseer")
        elif dataset_name in ('pubmed_edgelist'):
            indices = load_split(name='pubmed')
        else:
            indices = load_split(dataset_name)

    tf.logging.info(
        "Split resulted in "
        "{} training, "
        "{} validation, "
        "{} test samples.".format(*map(len, indices))
    )

    [mask_train, mask_val, mask_test] = masks = list(
        map(partial(indices_to_mask, size=n), indices)
    )

    y_train, y_val, y_test = map(partial(mask_values, y), masks)

    A = A.toarray()
    if get_y_values:
        return X, y, A, mask_train, mask_val, mask_test, y_train, y_val, y_test, n, d, k

    return X, y, A, mask_train, mask_val, mask_test, n, d, k

def balanced_data_split(X, y, samples_per_class, random_state):
    indices = []

    num_classes = y.shape[1]
    train_indices = []

    for c in range(num_classes):
        ind = np.where(y[:, c] > 0.5)[0]
        #train_indices.extend(list(np.random.choice(ind, samples_per_class, replace=False)))
        train_indices.extend(list(random_state.choice(ind, samples_per_class, replace=False)))
        #print(ind)

    indices.append(train_indices)

    all_indices = set(range(y.shape[0]))
    # remove the indices of the data in the training set.
    unused_indices = all_indices - set(train_indices)

    # Now split the remaining data into validation and test sets using
    # uniform sampling such that validation data are 25% of the remaining data
    # and test data is 75% of the remaining data.
    val_size = int(len(unused_indices)*0.30)
    #val_indices = list(np.random.choice(list(unused_indices), val_size, replace=False))
    val_indices = list(random_state.choice(list(unused_indices), val_size, replace=False))

    test_indices = list(unused_indices-set(val_indices))

    # some sanity checks
    assert (len(train_indices)+len(val_indices)+len(test_indices)) == y.shape[0]
    assert len(set(train_indices).intersection(set(val_indices))) == 0
    assert len(set(val_indices).intersection(set(test_indices))) == 0

    indices.append(val_indices)
    indices.append(test_indices)

    return indices

DATASET_LOADERS = dict(
    karate_club=load_karate_club,
    iris=load_iris,
    cora_legacy=partial(load_cora, "datasets/feature_based_datasets/cora", "cora.cites"),
    cora_jaccard=partial(
        load_cora, "datasets/feature_based_datasets/cora", "cora_graph_jaccard.edgelist"
    ),
    cora_knn=partial(
        load_cora, "datasets/feature_based_datasets/cora", "cora_graph_supervised_knn.edgelist"
    ),
    cora_supervised=partial(
        load_cora, "datasets/feature_based_datasets/cora", "cora_graph_supervised.edgelist"
    ),
    cora_tsne=partial(load_cora, "datasets/feature_based_datasets/cora", "cora_graph_tsne.gpickle"),
    cora_cosine=partial(load_cora, "datasets/feature_based_datasets/cora", "cora_graph_cosine_knn_16.gpickle"),
    cora=partial(load_dataset, "cora"),
    cora_edgelist=partial(load_dataset,"cora"),
    citeseer_edgelist=partial(load_dataset,"citeseer"),
    citeseer=partial(load_dataset, "citeseer"),
    polblogs=partial(load_polblogs, "datasets/polblogs"),
    polblogs_edgelist=partial(load_polblogs, "datasets/polblogs"),
    pubmed=partial(load_dataset, "pubmed"),
    pubmed_edgelist=partial(load_dataset, "pubmed"),
    twitter=load_twitter,
    telco=load_telco,
    protein=load_protein,
    genesis=load_genesis,
)

max_stars_repo_name: ebonilla/VGCN

max_issues_repo_name: ebonilla/VGCN

max_forks_repo_name: ebonilla/VGCN

reason: import numpy,import scipy,from scipy,import networkx,from networkx


text:
import numpy, time, sys, inspect
import logging

log_and_raise_error = lambda logger, message: log(logger, message, lvl="ERROR", exception=RuntimeError, rollback=2)
log_warning = lambda logger, message: log(logger, message, lvl="WARNING", exception=None, rollback=2)
log_info = lambda logger, message: log(logger, message, lvl="INFO", exception=None, rollback=2)
log_debug = lambda logger, message: log(logger, message, lvl="DEBUG", exception=None, rollback=2)

def log(logger, message, lvl, exception=None, rollback=1):
    logcalls = {"ERROR": logger.error,
                "WARNING": logger.warning,
                "DEBUG": logger.debug,
                "INFO": logger.info}
    if lvl not in logcalls:
        print("%s is an invalid logger level." % lvl)
        sys.exit(1)
    logcall = logcalls[lvl]
    # This should maybe go into a handler
    if (logger.getEffectiveLevel() >= logging.INFO) or rollback is None:
        # Short output
        msg = "%s" % message
    else:
        # Detailed output only in debug mode
        func = inspect.currentframe()
        for r in range(rollback):
            # Rolling back in the stack, otherwise it would be this function
            func = func.f_back
        code = func.f_code
        msg = "%s\n\t=> in \'%s\' function \'%s\' [%s:%i]" % (message,
                                                              func.f_globals["__name__"],
                                                              code.co_name,
                                                              code.co_filename,
                                                              code.co_firstlineno)
    logcall("\t" + msg)
    if exception is not None:
        raise exception(message)

def log_execution_time(logger):
    def st_time(func):
        def st_func(*args, **keyArgs):
            t1 = time.time()
            r = func(*args, **keyArgs)
            t2 = time.time()
            try:
                filename = inspect.getsourcefile(func)
                module = inspect.getmodule(func)
                line = inspect.getsourcelines(func)[1]
                loc = "\'%s\' [%s:%i]" % (func.__name__,
                                         filename,
                                         line)
            except TypeError:
                loc = "\'%s\'" % func.__name__
            msg = "Execution time = %.4f sec\n\t=> in function %s" % (t2 - t1,loc)
            log(logger, msg, "DEBUG", exception=None, rollback=None)
            return r
        return st_func
    return st_time

ext: py

max_stars_repo_name: FilipeMaia/spts

max_issues_repo_name: FilipeMaia/spts

max_forks_repo_name: FilipeMaia/spts

reason: import numpy


text:
import justpy as jp
import pandas as pd
import numpy as np

# https://worldhappiness.report/ed/2019/
df = pd.read_csv('http://elimintz.github.io/happiness_report_2019.csv').round(3)

def grid_change(self, msg):
    msg.page.df = jp.read_csv_from_string(msg.data)
    c = msg.page.df.jp.plot('Country', msg.page.cols_to_plot, kind='column', title='World Happiness Ranking',
                    subtitle='Click and drag in the plot area to zoom in. Shift + drag to pan',
                    stacking=msg.page.stacking, temp=True)
    msg.page.c.options = c.options

def stack_change(self, msg):
    msg.page.c.options.plotOptions.series.stacking = self.value
    msg.page.stacking = self.value

def series_change(self, msg):
    msg.page.cols_to_plot = []
    for i, toggle in enumerate(msg.page.toggle_list):
        if toggle.value:
            msg.page.cols_to_plot.append(i+3)
    c = msg.page.df.jp.plot('Country', msg.page.cols_to_plot, kind='column', title='World Happiness Ranking',
                           subtitle='Click and drag in the plot area to zoom in. Shift + drag to pan',
                           stacking=msg.page.stacking, temp=True)
    msg.page.c.options = c.options

async def corr_button_click(self, msg):
    msg.page.open = '/corr'
    await msg.page.update()
    msg.page.open = ''
    return True

def happiness_plot(request):
    # ['Country', 'Rank', 'Score', 'Unexplained', 'GDP', 'Social_support', 'Health', 'Freedom', 'Generosity', Corruption']
    wp = jp.QuasarPage()
    chart_theme = request.query_params.get('theme', 8)  # Default is 'grid'
    themes = ['high-contrast-dark', 'high-contrast-light','avocado', 'dark-blue', 'dark-green', 'dark-unica', 'gray', 'grid-light', 'grid', 'sand-signika', 'skies', 'sunset']
    wp.highcharts_theme = themes[int(chart_theme)]
    wp.stacking = ''
    wp.cols_to_plot = [3,4,5,6,7,8,9]
    wp.df = df
    d = jp.Div(classes='q-ma-lg', a=wp)
    bg = jp.QBtnToggle(push=True, glossy=True, toggle_color='primary', value='', a=d, input=stack_change, style='margin-right: 30px')
    bg.options = [
          {'label': 'No Stacking', 'value': ''},
          {'label': 'Normal', 'value': 'normal'},
          {'label': 'percent', 'value': 'percent'}
        ]
    wp.toggle_list = []
    corr_button = jp.QBtn(label='Show pairwise correlation', a=d, click=corr_button_click)
    for i in df.columns[3:]:
        wp.toggle_list.append(jp.QToggle(checked_icon='check', color='green', unchecked_icon='clear', value=True,
                                         label=f'{i}', a=d,  input=series_change)) # ,style='margin-right: 10px',
    chart = df.jp.plot('Country', wp.cols_to_plot, kind='column', a=wp, title='World Happiness Ranking',
                   subtitle='Click and drag in the plot area to zoom in. Shift + drag to pan',
                   stacking='', classes='border m-2 p-2 q-ma-lg p-ma-lg')
    grid = df.jp.ag_grid(a=wp)
    grid.options.columnDefs[0].rowDrag = True
    for event_name in ['sortChanged', 'filterChanged', 'columnMoved', 'rowDragEnd']:
        grid.on(event_name, grid_change)
    wp.c = chart
    grid.c = chart
    bg.c = chart
    return wp

class ScatterWithRegression(jp.Scatter):

    def __init__(self, x, y, **kwargs):
        super().__init__(x, y, **kwargs)
        x = np.asarray(x)
        y = np.asarray(y)
        m = (len(x) * np.sum(x * y) - np.sum(x) * np.sum(y)) / (len(x) * np.sum(x * x) - np.sum(x) ** 2)
        b = (np.sum(y) - m * np.sum(x)) / len(x)
        s = jp.Dict()   # The new series
        s.type = 'line'
        s.marker.enabled = False
        s.enableMouseTracking = False
        min = float(x.min())
        max = float(x.max())
        s.data = [[min, m * min + b], [max, m * max + b]]
        s.name = f'Regression, m: {round(m, 3)}, b: {round(b, 3)}'
        self.options.series.append(s)

def create_corr_page():
    # creates a page showing correlation between factors
    score_factors = ['Unexplained', 'GDP', 'Social_support', 'Health', 'Freedom', 'Generosity', 'Corruption']
    wp = jp.WebPage(delete_flag=False)
    d = jp.Div(a=wp, classes='flex flex-wrap')
    for x_col in score_factors:
        for y_col in score_factors[score_factors.index(x_col)+1:]:
            x = df[x_col]
            y = df[y_col]
            chart = ScatterWithRegression(x, y, a=d, style='width: 400px; height: 400px; margin: 10px;', classes='border')
            o = chart.options
            o.title.text = f'{x_col} vs. {y_col}'
            o.xAxis.title.text = x_col
            o.yAxis.title.text = y_col
            o.tooltip.pointFormat = f'{{point.country}}
{x_col}: {{point.x}}
{y_col}: {{point.y}}
'
            o.tooltip.useHtml = True
            o.series[0].name = f'{x_col} vs. {y_col}'
            data = []
            # Make the first series data a dictionary so that the country name  will be available to the tooltip
            for i, point in enumerate(o.series[0].data):
                data.append({'x': point[0], 'y': point[1], 'country': df['Country'][i]})
            o.series[0].data = data
    corr_def = df[['Unexplained', 'GDP', 'Social_support', 'Health', 'Freedom', 'Generosity', 'Corruption']].corr().round(3)
    corr_def.insert(loc=0, column='Factors/Factors', value=corr_def.index)
    g = corr_def.jp.ag_grid(a=wp, style='width: 90%; height: 250px', classes='m-2')
    return wp

corr_page = create_corr_page()

@jp.SetRoute('/corr')
def corr_test():
    return(corr_page)

jp.justpy(happiness_plot)

max_stars_repo_name: AgarwalPragy/justpy

max_issues_repo_name: AgarwalPragy/justpy

max_forks_repo_name: AgarwalPragy/justpy


text:
"""
Module providing additional functionality for dealing with hierarchically nested tables, i.e.,
tables containing DynamicTableRegion references.
"""
import pandas as pd
import numpy as np
from hdmf.common.table import DynamicTable, DynamicTableRegion, VectorIndex
from hdmf.common.alignedtable import AlignedDynamicTable
from hdmf.utils import docval, getargs

@docval({'name': 'dynamic_table', 'type': DynamicTable,
         'doc': 'DynamicTable object to be converted to a hierarchical pandas.Dataframe'},
        returns="Hierarchical pandas.DataFrame with usually a pandas.MultiIndex on both the index and columns.",
        rtype='pandas.DataFrame',
        is_method=False)
def to_hierarchical_dataframe(dynamic_table):
    """
    Create a hierarchical pandas.DataFrame that represents all data from a collection of linked DynamicTables.

    **LIMITATIONS:** Currently this function only supports DynamicTables with a single DynamicTableRegion column.
    If a table has more than one DynamicTableRegion column then the function will expand only the
    first DynamicTableRegion column found for each table. Any additional DynamicTableRegion columns will remain
    nested.

    **NOTE:** Some useful functions for further processing of the generated
    DataFrame include:

    * pandas.DataFrame.reset_index to turn the data from the pandas.MultiIndex into columns
    * :py:meth:`~hdmf.common.hierarchicaltable.drop_id_columns` to remove all 'id' columns
    * :py:meth:`~hdmf.common.hierarchicaltable.flatten_column_index` to flatten the column index
    """
    # TODO: Need to deal with the case where we have more than one DynamicTableRegion column in a given table
    # Get the references column
    foreign_columns = dynamic_table.get_foreign_columns()
    # if table does not contain any DynamicTableRegion columns then we can just convert it to a dataframe
    if len(foreign_columns) == 0:
        return dynamic_table.to_dataframe()
    hcol_name = foreign_columns[0]   # We only denormalize the first foreign column for now
    hcol = dynamic_table[hcol_name]  # Either a VectorIndex pointing to a DynamicTableRegion or a DynamicTableRegion
    # Get the target DynamicTable that hcol is pointing to. If hcol is a VectorIndex then we first need
    # to get the target of it before we look up the table.
    hcol_target = hcol.table if isinstance(hcol, DynamicTableRegion) else hcol.target.table

    # Create the data variables we need to collect the data for our output dataframe and associated index
    index = []
    data = []
    columns = None
    index_names = None

    #  First we here get a list of DataFrames, one for each row of the column we need to process.
    #  If  hcol is a VectorIndex (i.e., our column is a ragged array of row indices), then simply loading
    #  the data from the VectorIndex will do the trick. If we have a regular DynamicTableRegion column,
    #  then we need to load the elements ourselves (using slice syntax to make sure we get DataFrames)
    #  one-row-at-a-time
    if isinstance(hcol, VectorIndex):
        rows = hcol.get(slice(None), index=False, df=True)
    else:
        rows = [hcol[i:(i+1)] for i in range(len(hcol))]
    # Retrieve the columns we need to iterate over from our input table. For AlignedDynamicTable we need to
    # use the get_colnames function instead of the colnames property to ensure we get all columns not just
    # the columns from the main table
    dynamic_table_colnames = (dynamic_table.get_colnames(include_category_tables=True, ignore_category_ids=False)
                              if isinstance(dynamic_table, AlignedDynamicTable)
                              else dynamic_table.colnames)

    # Case 1:  Our DynamicTableRegion column points to a DynamicTable that itself does not contain
    #          any DynamicTableRegion references (i.e., we have reached the end of our table hierarchy).
    #          If this is the case than we need to de-normalize the data and flatten the hierarchy
    if not hcol_target.has_foreign_columns():
        # Iterate over all rows, where each row is described by a DataFrame with one-or-more rows
        for row_index, row_df in enumerate(rows):
            # Since each row contains a pandas.DataFrame (with possible multiple rows), we
            # next need to iterate over all rows in that table to denormalize our data
            for row in row_df.itertuples(index=True):
                # Determine the column data for our row. Each selected row from our target table
                # becomes a row in our flattened table
                data.append(row)
                #  Determine the multi-index tuple for our row, consisting of: i) id of the row in this
                #  table, ii) all columns (except the hierarchical column we are flattening), and
                #  iii) the index (i.e., id) from our target row
                index_data = ([dynamic_table.id[row_index], ] +
                              [dynamic_table[row_index, colname]
                               for colname in dynamic_table_colnames if colname != hcol_name])
                index.append(tuple(index_data))

        # Determine the names for our index and columns of our output table
        # We need to do this even if our table was empty (i.e. even is len(rows)==0)
        # NOTE: While for a regular DynamicTable the "colnames" property will give us the full list of column names,
        #       for AlignedDynamicTable we need to use the get_colnames() function instead to make sure we include
        #       the category table columns as well.
        index_names = ([(dynamic_table.name, 'id')] +
                       [(dynamic_table.name, colname)
                        for colname in dynamic_table_colnames if colname != hcol_name])
        # Determine the name of our columns
        hcol_iter_columns = (hcol_target.get_colnames(include_category_tables=True, ignore_category_ids=False)
                             if isinstance(hcol_target, AlignedDynamicTable)
                             else hcol_target.colnames)
        columns = pd.MultiIndex.from_tuples([(hcol_target.name, 'id'), ] +
                                            [(hcol_target.name, c) for c in hcol_iter_columns],
                                            names=('source_table', 'label'))

    # Case 2:  Our DynamicTableRegion columns points to another table with a DynamicTableRegion, i.e.,
    #          we need to recursively resolve more levels of the table hierarchy
    else:
        # First we need to recursively flatten the hierarchy by calling 'to_hierarchical_dataframe()'
        # (i.e., this function) on the target of our hierarchical column
        hcol_hdf = to_hierarchical_dataframe(hcol_target)
        # Iterate over all rows, where each row is described by a DataFrame with one-or-more rows
        for row_index, row_df_level1 in enumerate(rows):
            # Since each row contains a pandas.DataFrame (with possible multiple rows), we
            # next need to iterate over all rows in that table to denormalize our data
            for row_df_level2 in row_df_level1.itertuples(index=True):
                # Since our target is itself a a DynamicTable with a DynamicTableRegion columns,
                # each target row itself may expand into multiple rows in the flattened hcol_hdf.
                # So we now need to look up the rows in hcol_hdf that correspond to the rows in
                # row_df_level2.
                # NOTE: In this look-up we assume that the ids (and hence the index) of
                #       each row in the table are in fact unique.
                for row_tuple_level3 in hcol_hdf.loc[[row_df_level2[0]]].itertuples(index=True):
                    # Determine the column data for our row.
                    data.append(row_tuple_level3[1:])
                    # Determine the multi-index tuple for our row,
                    index_data = ([dynamic_table.id[row_index], ] +
                                  [dynamic_table[row_index, colname]
                                   for colname in dynamic_table_colnames if colname != hcol_name] +
                                  list(row_tuple_level3[0]))
                    index.append(tuple(index_data))
        # Determine the names for our index and columns of our output table
        # We need to do this even if our table was empty (i.e. even is len(rows)==0)
        index_names = ([(dynamic_table.name, "id")] +
                       [(dynamic_table.name, colname)
                        for colname in dynamic_table_colnames if colname != hcol_name] +
                       hcol_hdf.index.names)
        columns = hcol_hdf.columns

    # Check if the index contains any unhashable types. If a table contains a VectorIndex column
    # (other than the DynamicTableRegion column) then "TypeError: unhashable type: 'list'" will
    # occur when converting the index to pd.MultiIndex. To avoid this error, we next check if any
    # of the columns in our index are of type list or np.ndarray
    unhashable_index_cols = []
    if len(index) > 0:
        unhashable_index_cols = [i for i, v in enumerate(index[0]) if isinstance(v, (list, np.ndarray))]

    # If we have any unhashable list or np.array objects in the index then update them to tuples.
    # Ideally we would detect this case when constructing the index, but it is easier to do this
    # here and it should not be much more expensive, but it requires iterating over all rows again
    if len(unhashable_index_cols) > 0:
        for i, v in enumerate(index):
            temp = list(v)
            for ci in unhashable_index_cols:
                temp[ci] = tuple(temp[ci])
            index[i] = tuple(temp)

    # Construct the pandas dataframe with the hierarchical multi-index
    multi_index = pd.MultiIndex.from_tuples(index, names=index_names)
    out_df = pd.DataFrame(data=data, index=multi_index, columns=columns)
    return out_df

def __get_col_name(col):
    """
    Internal helper function to get the actual name of a pandas DataFrame column from a
    column name that may consists of an arbitrary sequence of tuples. The function
    will return the last value of the innermost tuple.
    """
    curr_val = col
    while isinstance(curr_val, tuple):
        curr_val = curr_val[-1]
    return curr_val

def __flatten_column_name(col):
    """
    Internal helper function used to iteratively flatten a nested tuple

    :param col: Column name to flatten
    :type col: Tuple or String

    :returns: If col is a tuple then the result is a flat tuple otherwise col is returned as is
    """
    if isinstance(col, tuple):
        re = col
        while np.any([isinstance(v, tuple) for v in re]):
            temp = []
            for v in re:
                if isinstance(v, tuple):
                    temp += list(v)
                else:
                    temp += [v, ]
            re = temp
        return tuple(re)
    else:
        return col

@docval({'name': 'dataframe', 'type': pd.DataFrame,
         'doc': 'Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)'},
        {'name': 'inplace', 'type': 'bool', 'doc': 'Update the dataframe inplace or return a modified copy',
         'default': False},
        returns="pandas.DataFrame with the id columns removed",
        rtype='pandas.DataFrame',
        is_method=False)
def drop_id_columns(**kwargs):
    """
    Drop all columns named 'id' from the table.

    In case a column name is a tuple the function will drop any column for which
    the inner-most name is 'id'. The 'id' columns of DynamicTable is in many cases
    not necessary for analysis or display. This function allow us to easily filter
    all those columns.

    :raises TypeError: In case that dataframe parameter is not a pandas.Dataframe.
    """
    dataframe, inplace = getargs('dataframe', 'inplace', kwargs)
    col_name = 'id'
    drop_labels = []
    for col in dataframe.columns:
        if __get_col_name(col) == col_name:
            drop_labels.append(col)
    re = dataframe.drop(labels=drop_labels, axis=1, inplace=inplace)
    return dataframe if inplace else re

@docval({'name': 'dataframe', 'type': pd.DataFrame,
         'doc': 'Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)'},
        {'name': 'max_levels', 'type': (int, np.integer),
         'doc': 'Maximum number of levels to use in the resulting column Index. NOTE:  When '
                'limiting the number of levels the function simply removes levels from the '
                'beginning. As such, removing levels may result in columns with duplicate names.'
                'Value must be >0.',
         'default': None},
        {'name': 'inplace', 'type': 'bool', 'doc': 'Update the dataframe inplace or return a modified copy',
         'default': False},
        returns="pandas.DataFrame with a regular pandas.Index columns rather and a pandas.MultiIndex",
        rtype='pandas.DataFrame',
        is_method=False)
def flatten_column_index(**kwargs):
    """
    Flatten the column index of a pandas DataFrame.

    The functions changes the dataframe.columns from a pandas.MultiIndex to a normal Index,
    with each column usually being identified by a tuple of strings. This function is
    typically used in conjunction with DataFrames generated
    by :py:meth:`~hdmf.common.hierarchicaltable.to_hierarchical_dataframe`

    :raises ValueError: In case the num_levels is not >0
    :raises TypeError: In case that dataframe parameter is not a pandas.Dataframe.
    """
    dataframe, max_levels, inplace = getargs('dataframe', 'max_levels', 'inplace', kwargs)
    if max_levels is not None and max_levels <= 0:
        raise ValueError('max_levels must be greater than 0')
    # Compute the new column names
    col_names = [__flatten_column_name(col) for col in dataframe.columns.values]
    # Apply the max_levels filter. Make sure to do this only for columns that are actually tuples
    # in order not to accidentally shorten the actual string name of columns
    if max_levels is None:
        select_levels = slice(None)
    elif max_levels == 1:
        select_levels = -1
    else:  # max_levels > 1
        select_levels = slice(-max_levels, None)
    col_names = [col[select_levels] if isinstance(col, tuple) else col for col in col_names]
    re = dataframe if inplace else dataframe.copy()
    re.columns = col_names
    return re

max_stars_repo_name: kangdh/hdmf

max_issues_repo_name: kangdh/hdmf

max_forks_repo_name: kangdh/hdmf


text:
from pkg_resources import resource_filename

import numpy as np
import pandas

import pytest
import numpy.testing as nptest

from wqio import Location

import pynsqd
import pynsqd.dataAccess as da

class Test_NSQData:
    def setup(self):
        self.testfile = resource_filename('pynsqd.data', 'testdata.csv')

        self.data = da.NSQData(datapath=self.testfile)
        self.known_landuses = np.array([
            'Commercial', 'Freeway', 'Industrial', 'Institutional',
            'Open Space', 'Residential', 'Unknown'
        ])

        self.known_columns = [
            'epa_rain_zone', 'state', 'location_code', 'station_name',
            'jurisdiction_county', 'jurisdiction_city', 'primary_landuse',
            'secondary_landuse', 'percent_impervious', 'start_date',
            'days since last rain', 'precipitation_depth_(in)', 'season',
            'parameter', 'fraction', 'units', 'res', 'qual',
            'drainage_area_acres', 'latitude', 'longitude', 'station'
        ]

        self.known_commerical_copper_shape = (329, 22)

    def test_data(self):
        assert (hasattr(self.data, 'data'))
        assert isinstance(self.data.data, pandas.DataFrame)
        assert (self.data.data.columns.tolist() == self.known_columns)

    def test_data_season(self):
        known_seasons = ['autumn', 'spring', 'summer', 'winter']
        seasons = sorted(self.data.data['season'].unique().tolist())
        assert (seasons == known_seasons)

    def test_landuses(self):
        assert (hasattr(self.data, 'landuses'))
        assert isinstance(self.data.landuses, list)
        nptest.assert_array_equal(
            sorted(self.data.landuses),
            sorted(self.known_landuses)
        )

    def test_parameters(self):
        '''Doesn't test values -- too many to list out'''
        assert (hasattr(self.data, 'parameters'))
        assert isinstance(self.data.parameters, list)

    def test__check_col_bad(self):
        with pytest.raises(ValueError):
            self.data._check_col('junk')

    def test_check_col_good(self):
        self.data._check_col('res')

    def test_getColumnValues(self):
        plu = self.data.getColumnValues('primary_landuse')
        assert isinstance(self.data.landuses, list)
        nptest.assert_array_equal(
            sorted(self.data.landuses),
            sorted(self.known_landuses)
        )

    def test_getData(self):
        assert (hasattr(self.data, 'getData'))
        subset = self.data.getData(
            primary_landuse=self.known_landuses[0],
            parameter='Copper',
            fraction='Total'
        )
        assert isinstance(subset, pandas.DataFrame)
        assert (subset.columns.tolist() == self.known_columns)
        assert (subset.shape == self.known_commerical_copper_shape)

    def test_getData_as_Location(self):
        subset = self.data.getData(
            primary_landuse=self.known_landuses[0],
            parameter='Copper',
            fraction='Total',
            as_location=True
        )
        assert isinstance(subset, Location)

max_stars_repo_name: phobson/pynsqd

max_issues_repo_name: phobson/pynsqd

max_forks_repo_name: phobson/pynsqd


text:
#loads
import scipy.stats as stats
import numpy as np
import pandas as pd
import csv

class ContTabs():

    def __init__(self, df, q_cols, dem_cols):
        self.df = df
        self.q_cols = q_cols
        self.dem_cols = dem_cols

#First I want to create a contingency table for each pairing of demographic facet and question/answer
    def cont_tables(self):
        tables = []
        facets = []
        answers = []

        for facet in self.dem_cols:
            for answer in self.q_cols:
                table = pd.crosstab(self.df[facet], [answer], normalize = 'index')
                tables.append(table)
                facets.append(facet)
                answers.append(answer)

        return tables, facets, answers

#Then I want to apply Cramer's chi square test to each of the dataframes output and write the results to a text file
    def cram_chi(self):
        tables, facets, answers = self.cont_tables()
        chi_results = open('../data/chiresults.txt', 'w')
        for data in tables:
            #data = pandas.DataFrame(data).tonumpy()
            num = 0

            '''
            chi2 = stats.chi2_contingency(data)[0]
            n = data.sum().sum()
            phi2 = chi2/n
            r,k = data.shape
            phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))
            rcorr = r - ((r-1)**2)/(n-1)
            kcorr = k - ((k-1)**2)/(n-1)
            newline =  np.sqrt(phi2corr / min( (kcorr-1), (rcorr-1)))
            '''

            chi2, p, dof, _ = stats.chi2_contingency([data.iloc[0].values,data.iloc[1].values])

            addition = f"The Pearson's Chi-Squared test result for {facets[num]} and {answers[num]}:  \n chi2 : {chi2} \n p-value: {p} \n Degrees of Freedom: {dof}"
            chi_results.write("\n")
            chi_results.write(addition)
            num +=1

        print("Your results are ready")
        chi_results.close()

if __name__ == "__main__":
    ContTabs()

reason: import numpy,import scipy


text:
import numpy as np
import scipy.stats as sp
import pandas as pd
from .util import *
from sklearn.neural_network import MLPRegressor
from sklearn import linear_model
from sklearn import tree
from sklearn import preprocessing
from sklearn import utils
from sklearn.datasets import load_iris
from .write_json import modify
import json
import matplotlib.pyplot as plt
import pickle

# import matplotlib.pyplot as plt
# calc WYT and 8RI. add columns to datafile from cdec_scraper.
# confirm against http://cdec.water.ca.gov/cgi-progs/iodir/WSIHIST
cfsd_mafd = 2.29568411*10**-5 * 86400 / 10 ** 6
cfs_tafd = 2.29568411*10**-5 * 86400 / 1000
pd.options.mode.chained_assignment = None  # default='warn'

water_year = lambda d: d.year+1 if d.dayofyear >= 274 else d.year
# Sacramento Water Year Index (historical)
get_SR_WYI = lambda x,p: 0.3*x[winter(x)].sum() + 0.4*x[summer(x)].sum() + 0.3*p

# def water_year_day(d):  #obtain day of water year, which begins on October 1st
#   if d.is_leap_year:
#     if d.dayofyear >= 275:
#       return d.dayofyear - 274
#     elif d.dayofyear <= 274 and d.dayofyear >= 59:
#       return d.dayofyear + 92
#     else:
#       return d.dayofyear + 92
#   elif not d.is_leap_year:
#     if d.dayofyear >= 274:
#       return d.dayofyear - 273
#     else:
#       return d.dayofyear + 92

winter = lambda y: (y.index.month >= 10) | (y.index.month <= 3)
summer = lambda y: (y.index.month >= 4) & (y.index.month <= 7)

def water_month(m): #obtain month of water year
  return m - 9 if m >= 9 else m + 3

def WYI_to_WYT(WYI, thresholds, values): #converts water year index to water year type
  for t,v in zip(thresholds,values):
    if WYI > t:
      return v

def rolling_fci(inflow, k, start): #used to obtain reservoir flood control index based on inflow
  T = len(inflow)
  x = np.zeros(T)
  for i,t in enumerate(inflow.index):
    if t.month==10 and t.day==1:
      x[i] = start # cfs, start each WY here
    else:
      x[i] = inflow[t] + k*x[i-1]

  return pd.Series(x, index=inflow.index)

def process(df,evap_regr,gains_regr,inf_regr): #used for historical data processing
  SR_pts = ['BND_fnf', 'ORO_fnf', 'YRS_fnf', 'FOL_fnf'] # flows to calculate Sacramento Valley WYI
  SJR_pts = ['NML_fnf', 'TLG_fnf', 'MRC_fnf', 'MIL_fnf'] #flows to calculate San-Joaquin Valley WYI

  df['WY'] = pd.Series([water_year(d) for d in df.index], index=df.index)
  df['DOWY'] = pd.Series([water_year_day(d) for d in df.index], index=df.index) #day of water year

  #historical net Deltagains, with perfect hindsight in pumping
  df['DeltaIn'] = df['DeltaOut'] + df['HRO_pump'] + df['TRP_pump']
  df['netgains'] = (df.DeltaIn -
                    df.SHA_out.shift(5) -
                    df.ORO_out.shift(3) -
                    df.FOL_out.shift(1))
  df.netgains.fillna(method='bfill', inplace=True)

  df['SR_WYI'] = pd.Series(index=df.index)

  prev_year = 12.89 # WY 1999 was 9.8
  for y,g in df.groupby('WY'):
    flow = (g[SR_pts] * cfsd_mafd).sum(axis=1)
    WYI = get_SR_WYI(flow, prev_year)
    df.loc[df.WY==y, 'SR_WYI'] = WYI
    prev_year = np.min((10.0,WYI))

  df['SR_WYT'] = df.SR_WYI.apply(WYI_to_WYT,
                                 thresholds=[9.2, 7.8, 6.5, 5.4, 0.0],
                                 values=['W', 'AN', 'BN', 'D', 'C'])

  df['SR_WYT_rolling'] = (df.SR_WYI #updating WYT throughout the year, thus limiting perfect foresight
                            .rolling(120).mean()
                            .apply(WYI_to_WYT,
                                 thresholds=[9.2, 7.8, 6.5, 5.4, 0.0],
                                 values=['W', 'AN', 'BN', 'D', 'C']))

  df['SR_WYT_rolling'].fillna(method='bfill', inplace=True)

  # San Joaquin Water Year Type
  thresholds = [3.8, 3.1, 2.5, 2.1, 0.0]
  values = ['W', 'AN', 'BN', 'D', 'C']
  prev_year = 4.12 # WY 1996, 3.59 in 1999
  get_SJR_WYI = lambda x,p: 0.2*x[winter(x)].sum() + 0.6*x[summer(x)].sum() + 0.2*p

  df['SJR_WYI'] = pd.Series(index=df.index)
  df['SJR_WYT'] = pd.Series(index=df.index)

  for y,g in df.groupby('WY'):
    flow = (g[SJR_pts] * cfsd_mafd).sum(axis=1)
    WYI = get_SJR_WYI(flow, prev_year)
    prev_year = np.min((4.5,WYI))
    for t,v in zip(thresholds,values):
      if WYI > t:
        df.loc[df.WY==y, 'SJR_WYT'] = v
        break

  df['8RI'] = ((df[SR_pts + SJR_pts] * cfsd_mafd) #8 station river index
               .sum(axis=1)
               .resample('M')
               .sum())

  df['8RI'].fillna(method='bfill', inplace=True)

  ###transformed inflows
  res_ids = ['SHA','ORO','FOL']
  for r in res_ids:
    coeffs = []
    intercepts = []
    df['%s_in_tr'%r] = pd.Series()
    for m in range(1,13):
        dfm = df[(df.index.month == m)]
        fnf = dfm['%s_fnf'%r].values
        inf = dfm['%s_in'%r].values
        fit = np.polyfit(fnf,inf,1)

        in_tr = fit[1] + fnf*fit[0]
        dfm['%s_in_tr'%r] = in_tr
        df['%s_in_tr'%r] = df['%s_in_tr'%r].fillna(dfm['%s_in_tr'%r])
        coeffs.append(fit[0])
        intercepts.append(fit[1])
    modify(inf_regr,"%s_inf_coeffs" %r, coeffs)
    modify(inf_regr,"%s_inf_int"%r, intercepts)
  df['ORO_precip'] = df[['SVL_pr','FRD_pr','DAV_pr','SBY_pr','CHS_pr','BRS_pr','QCY_pr','DES_pr']].mean(axis=1)
  df['FOL_precip'] = df[['BLC_pr','GTW_pr','PFH_pr']].mean(axis=1)
  df = df.drop(df[['SVL_pr','FRD_pr','DAV_pr','SBY_pr','CHS_pr','BRS_pr','QCY_pr','DES_pr','BLC_pr','GTW_pr','PFH_pr']],axis = 1)

  # flood control indices
  df['SHA_fci'] = rolling_fci(df['SHA_in_tr'], k=0.95, start=100000)
  df.SHA_fci.fillna(method='bfill', inplace=True)

  df['ORO_fci'] = rolling_fci(df['ORO_precip'], k=0.97, start=0)
  df.ORO_fci.fillna(method='bfill', inplace=True)

  df['FOL_fci'] = rolling_fci(df['FOL_precip'], k=0.97, start=0)
  df.FOL_fci.fillna(method='bfill', inplace=True)
  df.loc['2014-01-14','ORO_tas'] = df.loc['2014-01-13','ORO_tas']
  df.loc['2012-06-21','ORO_tas'] = df.loc['2012-06-20','ORO_tas']
  ### evaporation  regression
  for r in res_ids:
    dfe = df[['%s_storage'%r,'%s_evap'%r,'%s_tas'%r]] #evaporation datafile
    dfe[['storage','evap','tas']] = dfe[['%s_storage'%r,'%s_evap'%r,'%s_tas'%r]]
    dfe = dfe.dropna(axis = 0)

    storage = dfe.storage.values
    evap = dfe.evap.values
    temp = dfe.tas.values
    storage2 = storage**2
    temp2 = temp**2
    X2Y2 = (temp**2)*(storage**2)
    XY = temp*storage
    reg = linear_model.LinearRegression()
    X = np.vstack([temp, storage,XY,temp2, storage2])
    reg.fit(X.T, evap)
    # R2 = reg.score(X.T, evap)
    # print('R2: %f' %(R2))
    coeffs = reg.coef_
    intercept = reg.intercept_
    modify(evap_regr,"%s_evap_coeffs" %r, coeffs.tolist())
    modify(evap_regr,"%s_evap_int"%r, intercept)

  ##clean up snowpack data and resample monthly
  snow_ids = ['GOL_swe','CSL_swe','HYS_swe','SCN_swe','RBB_swe','CAP_swe','RBP_swe','KTL_swe',
  				'HMB_swe','FOR_swe','RTL_swe','GRZ_swe','SDF_swe','SNM_swe','SLT_swe','MED_swe']
  dfs = df[snow_ids] #working with only snow for these calculations
  num = dfs._get_numeric_data()
  num[num < 0 ] = np.NaN
  #num[num > 150 ] = np.NaN#oroville,folsom,shast,new bullards
  num[num > 150 ] = np.NaN
  dfs = dfs.interpolate(method = 'linear')
  dfs = dfs.resample('M').mean()
  df = df.drop(df[snow_ids],axis = 1)

  df = df.join(dfs).fillna(method = 'ffill') #snow stations now cleaned up and back in main datafile

  df = df[(df.index > '1996-09-30')]#start at 1997 water year
  #sum of stations for each basin
  df['YRS_swe'] = df[['GOL_swe','CSL_swe']].mean(axis=1)
  df['FOL_swe'] = df[['HYS_swe', 'SCN_swe', 'RBB_swe', 'CAP_swe']].mean(axis = 1) #taking out RBP (for this time), also test taking out RBB later
  df['ORO_swe'] = df[['KTL_swe', 'HMB_swe', 'FOR_swe', 'RTL_swe', 'GRZ_swe']].mean(axis = 1)
  df['BND_swe'] = df[['SDF_swe', 'SNM_swe', 'SLT_swe']].mean(axis = 1)

  BND = (df['BND_fnf'].to_frame(name='inf'))
  ORO = (df['ORO_fnf'].to_frame(name='inf'))
  YRS = (df['YRS_fnf'].to_frame(name='inf'))
  FOL = (df['FOL_fnf'].to_frame(name='inf'))

  #gains regression with perfect foresight WYI
  # ### delta gains calculations
  dfg = df[['MIL_fnf','NML_fnf','YRS_fnf','TLG_fnf','MRC_fnf','MKM_fnf','NHG_fnf','netgains','SR_WYI']] #gains datafile
  stations = ['SHA','ORO','YRS','FOL','MIL','NML','YRS','TLG','MRC','MKM','NHG']
  stations_WYI = ['SHA','ORO','YRS','FOL','MIL','NML','YRS','TLG','MRC','MKM','NHG','WYI']

  dfg = dfg[dfg.index != '1998-10-09']
  dfg = dfg[dfg.index != '1997-01-03']
  dfg = dfg[dfg.index != '1997-01-04']
  dfg = dfg[dfg.index != '1997-01-05']
  dfg = dfg[dfg.index != '2012-07-01']

  gain_errors = ['1998-10-09','1997-01-03','1997-01-04','1997-01-05','2012-07-01']
  for g in gain_errors:
    df.loc[g, 'netgains'] = np.NaN
  for station in stations:
    dfg['%s_fnf' %station] = df['%s_fnf' %station].shift(2)
    dfg['%s_rol10' %station] = df['%s_fnf' %station].rolling(10).mean()
    dfg['%s_rol20' %station] = df['%s_fnf' %station].rolling(20).sum()
    dfg['%s_rol30' %station] = df['%s_fnf' %station].rolling(30).sum()

  dfg = dfg.dropna()

  month_arr = np.arange(1,13)
  dfgs = pd.DataFrame(index = df.index) #gains_sim df
  dfgs['gains_sim'] = pd.Series()
  for mth in month_arr:
    dfm = dfg[dfg.index.month == mth] #for each month #hist
    gains = dfm.netgains.values#.rolling(3, center = True, min_periods=1).mean().values
    WYI = dfm.SR_WYI.values#rolling(3, center = True, min_periods=1).mean().values
    m = dfm.index.month.values
    d = dfm.index.day.values

    X = np.vstack([WYI])

    X = np.vstack([X,[m]])

    for station in stations:
      V = np.vstack([dfm['%s_fnf' %station]])
      V = np.vstack([dfm['%s_rol10' %station]])
      V = np.vstack([dfm['%s_rol20' %station]])
      V = np.vstack([dfm['%s_rol30' %station]])
      X = np.vstack([X,V])
    X = X.T
    if mth in [10,11,12,1]:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 1000, hidden_layer_sizes=(50,), random_state=3)
    elif mth == 2:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 100000, hidden_layer_sizes=(1000,), random_state=3)
    elif mth == 3:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 100000, hidden_layer_sizes=(1000,), random_state=3)
    elif mth == 4:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(75,), random_state=3)
    elif mth == 5:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(600,), random_state=3)
    elif mth == 6:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(100,), random_state=3)
    elif mth == 7:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(400,), random_state=3)
    elif mth == 8:
      clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(200,), random_state=3)
    elif mth == 9:
       clf = MLPRegressor(solver='adam', alpha=1e-5,max_iter = 10000, hidden_layer_sizes=(85,), random_state=3)
    ann = clf.fit(X, gains.T)
    pickle.dump(ann, open("orca/data/gains-neural-nets/ann_%s.pkl"%mth, "wb" ))
    month_gains = clf.predict(X)
    dfm['gains_sim'] = month_gains
    dfgs['gains_sim_%s'%mth] = dfm['gains_sim']
    dfgs = dfgs.fillna(0)
    dfgs['gains_sim'] = dfgs['gains_sim'] + dfgs['gains_sim_%s'%mth]
    #to test:
    # if mth == 7:
    #   plt.plot(gains)
    #   plt.plot(month_gains)
    #   plt.show()
  df['gains_sim'] = dfgs.gains_sim.fillna(method = 'bfill') * cfs_tafd #fill in missing beggining values (because of rolling)
  df['netgains'] = df.netgains.fillna(method = 'bfill') * cfs_tafd #fill in missing beggining values (because of rolling)
  # plt.plot(df.netgains)
  # plt.plot(df.gains_sim)
  # plt.show()

  #if looping gains with historical. probably no longer needed with neural net
  df_g = pd.DataFrame() #gains datafile
  for WYT in ['C','D','BN','AN','W']:
    dfw = df[(df.SR_WYT == WYT)]
    means = dfw.netgains.groupby([dfw.index.strftime('%m-%d')]).mean()
    df_g[WYT] = means
    days = [dfw.index.strftime('%Y-%m-%d')]
    days = days[0]

    for d in days:
      df.loc[df.index == d,'newgains'] = means[d[5:]]

  df['newgains'] = df.newgains.rolling(5).mean()
  df['newgains']=df.newgains.fillna(method='bfill')
  # df['gains_sim'] = (df['newgains']*0.75+df['gains_sim']*0.25)

  df['OMR'] = df.OMR + df.HRO_pump + df.TRP_pump

  df2 = df[(df.index > '2008-12-01')]#start at 1997 water year
  df['OMR_sim'] = pd.Series()
  df_OM = pd.DataFrame() #for means for OMR loops
  for WYT in ['C','D','BN','AN','W']:
    if WYT =='AN':
      dfWEY = df2[(df2.SR_WYT == 'W')]
      means_wet = dfWEY.OMR.groupby([dfWEY.index.strftime('%m-%d')]).mean()

      dfBN = df2[(df2.SR_WYT == 'BN')]
      means_BN = dfBN.OMR.groupby([dfBN.index.strftime('%m-%d')]).mean()
      means = means_wet.add(means_BN)/2
    else:
      dfw = df2[(df2.SR_WYT == WYT)]
      means = dfw.OMR.groupby([dfw.index.strftime('%m-%d')]).mean()
    df_OM[WYT] = means
    dfh = df[(df.SR_WYT == WYT)]

    days = [dfh.index.strftime('%Y-%m-%d')]
    days = days[0]
    for i,d in enumerate(days):
      if d[5:] != '02-29':
        df.loc[df.index == d,'OMR_sim'] = means[d[5:]]
      elif d[5:] == '02-29':
        df.loc[df.index == d,'OMR_sim'] = means[days[i-1][5:]]
  df['OMR_sim']=df.OMR_sim.fillna(method='ffill')
  return df, df_g, df_OM

def process_projection(df,df_g,df_OMR,gains_regr,inf_regr,window): #used to process climate projection data
  SR_pts = ['BND_fnf', 'ORO_fnf', 'YRS_fnf', 'FOL_fnf']
  SJR_pts = ['NML_fnf', 'TLG_fnf', 'MRC_fnf', 'MIL_fnf']
  df = df[(df.index > '1951-09-30')]
  gains_reg = json.load(open(gains_regr))
  inf_reg = json.load(open(inf_regr))

  df['WY'] = pd.Series([water_year(d) for d in df.index], index=df.index)
  df['DOWY'] = pd.Series([water_year_day(d) for d in df.index], index=df.index)
  #bias corrections for fnfs
  df['SHA_fnf'] = df['SHA_fnf'] * 0.95 #0.95
  df['ORO_fnf'] = df['ORO_fnf'] * 1.1
  df['FOL_fnf'] = df['FOL_fnf'] * 0.8 #0.8
  df['MRC_fnf'] = df['MRC_fnf'] * 0.9
  df['MKM_fnf'] = df['MKM_fnf'] * 0.9
  df['NHG_fnf'] = df['NHG_fnf'] * 0.25
  df['YRS_fnf'] = df['YRS_fnf'] * 0.85

  snow_ids = ['MED_swe','SDF_swe','SLT_swe','BKL_swe','HMB_swe','FOR_swe','RTL_swe',
                  'GRZ_swe','GOL_swe','CSL_swe','HYS_swe','SCN_swe','RBB_swe','RBP_swe','CAP_swe']
  # convert snow to inches

  #temp conversion and bias correction
  df['SHA_tas'] = (df['SHA_tas'] * 9/5 + 32) * 1.06
  df['ORO_tas'] = (df['ORO_tas'] * 9/5 + 32) * 1.01
  df['FOL_tas'] = (df['FOL_tas'] * 9/5 + 32) * 0.98

  df['SR_WYI'] = pd.Series(index=df.index)

  prev_year = 9.8 # WY 1996, 1999 was 9.8
  for y,g in df.groupby('WY'):
    flow = (g[SR_pts] * cfsd_mafd).sum(axis=1)
    # plt.plot(flow.cumsum().values)
    WYI = get_SR_WYI(flow, prev_year)
    df.loc[df.WY==y, 'SR_WYI'] = WYI
    prev_year = np.min((10.0,WYI))

  df['SR_WYT'] = df.SR_WYI.apply(WYI_to_WYT,
                                 thresholds=[9.2, 7.8, 6.5, 5.4, 0.0],
                                 values=['W', 'AN', 'BN', 'D', 'C'])

  df['SR_WYT_rolling'] = (df.SR_WYI
                            .rolling(120).mean()
                            .apply(WYI_to_WYT,
                                 thresholds=[9.2, 7.8, 6.5, 5.4, 0.0],
                                 values=['W', 'AN', 'BN', 'D', 'C']))

  df['SR_WYT_rolling'].fillna(method='bfill', inplace=True)

  # San Joaquin Water Year Type #only using historical now--- may end up predicting to help with delta regressions
  thresholds = [3.8, 3.1, 2.5, 2.1, 0.0]
  values = ['W', 'AN', 'BN', 'D', 'C']
  prev_year = 4.12 # WY 1996, 3.59 in 1999
  get_SJR_WYI = lambda x,p: 0.2*x[winter(x)].sum() + 0.6*x[summer(x)].sum() + 0.2*p

  df['SJR_WYI'] = pd.Series(index=df.index)
  df['SJR_WYT'] = pd.Series(index=df.index)

  for y,g in df.groupby('WY'):
    flow = (g[SJR_pts] * cfsd_mafd).sum(axis=1)
    WYI = get_SJR_WYI(flow, prev_year)
    prev_year = np.min((4.5,WYI))
    for t,v in zip(thresholds,values):
      if WYI > t:
        df.loc[df.WY==y, 'SJR_WYT'] = v
        break

  df['8RI'] = ((df[SR_pts + SJR_pts] * cfsd_mafd)
               .sum(axis=1)
               .resample('M')
               .sum())

  df['8RI'].fillna(method='bfill', inplace=True)

  ###transfolrmed inflows
  res_ids = ['SHA','ORO','FOL']
  for r in res_ids:
    coeffs = inf_reg["%s_inf_coeffs" %r]
    intercepts = inf_reg["%s_inf_int"%r]
    df['%s_in_tr'%r] = pd.Series()

    for m in range(1,13):
      # for WYT in ['C','D','BN','AN','W']:
        dfm = df[(df.index.month == m)]
        # dfm = dfm[(dfm.SR_WYT == WYT)]
        inter = intercepts[m-1]
        coeff = coeffs[m-1]
        fnf = dfm['%s_fnf'%r].values
        in_tr = inter + fnf*coeff
        dfm['%s_in_tr'%r] = in_tr
        df['%s_in_tr'%r] = df['%s_in_tr'%r].fillna(dfm['%s_in_tr'%r])

########flood incides
  df['SHA_fci'] = rolling_fci(df['SHA_in_tr'], k=0.95, start=100000)
  df.SHA_fci.fillna(method='bfill', inplace=True)

  df['ORO_fci'] = rolling_fci(df['ORO_pr'], k=0.97, start=0)
  df.ORO_fci.fillna(method='bfill', inplace=True)

  df['FOL_fci'] = rolling_fci(df['FOL_pr'], k=0.97, start=0)
  df.ORO_fci.fillna(method='bfill', inplace=True)

  snow_ids = ['GOL_swe','CSL_swe','HYS_swe','SCN_swe','RBB_swe','CAP_swe','RBP_swe',
            'HMB_swe','FOR_swe','RTL_swe','GRZ_swe','SDF_swe','SLT_swe','MED_swe']

  if window == 'historical':
    for sn in snow_ids:
      df[sn] = df[sn]/25.4
    #snow bias correction (none for RBP and CAP)
    df['MED_swe'] = df['MED_swe'] * 8.0
    df['SDF_swe'] = df['SDF_swe'] * 0.6
    df['SNM_swe'] = df['SDF_swe'] * 0.9
    df['SLT_swe'] = df['SLT_swe'] * 2.7
    df['KTL_swe'] = df['BKL_swe'] * 0.6*0.65
    df['HMB_swe'] = df['HMB_swe'] * 3.2
    df['FOR_swe'] = df['FOR_swe'] * 4.8
    df['RTL_swe'] = df['RTL_swe'] * 2.3
    df['GRZ_swe'] = df['GRZ_swe'] * 1.8
    df['GOL_swe'] = df['GOL_swe'] * 2.0
    df['CSL_swe'] = df['CSL_swe'] * 1.2
    df['HYS_swe'] = df['HYS_swe'] * 0.85
    df['SCN_swe'] = df['SCN_swe'] * 1.7
    df['RBB_swe'] = df['RBB_swe'] * 1.7

    ##clean up snowpack data and resample monthly
    dfs = df[snow_ids] #working with only snow for these calculations
    num = dfs._get_numeric_data()
    num[num < 0 ] = np.NaN
    #num[num > 150 ] = np.NaN#oroville,folsom,shast,new bullards
    num[num > 150 ] = np.NaN
    dfs = dfs.interpolate(method = 'linear')
    dfs = dfs.resample('M').mean()
    df = df.drop(df[snow_ids],axis = 1)
    df = df.join(dfs).fillna(method = 'ffill') #snow stations now cleaned up and back in main datafile

    df = df[(df.index > '1951-09-30')]#start at 2000 water year

    #sum of stations for each basins
    df['YRS_swe'] = df[['GOL_swe','CSL_swe']].mean(axis=1)
    df['FOL_swe'] = df[['HYS_swe', 'SCN_swe', 'RBB_swe', 'CAP_swe']].mean(axis = 1) #taking out RBP (for this time), also test taking out RBB later
    df['ORO_swe'] = df[['KTL_swe', 'HMB_swe', 'FOR_swe', 'RTL_swe', 'GRZ_swe']].mean(axis = 1)
    df['BND_swe'] = df[['SDF_swe', 'SNM_swe', 'SLT_swe']].mean(axis = 1)

  elif (window == 'rolling') | (window =='expanding') | (window == 'stationary'):
    for sn in snow_ids:
      df[sn] = df[sn]/25.4
############below here for basin averaged.
    snow_basins = ['BND_swe','FOL_swe','ORO_swe','YRS_swe']
    for sn in snow_basins:
      df[sn] = df[sn]/25.4

    dfs = df[snow_basins] #working with only snow for these calculations
    num = dfs._get_numeric_data()
    num[num < 0 ] = np.NaN
    #num[num > 150 ] = np.NaN#oroville,folsom,shast,new bullards
    num[num > 150 ] = np.NaN
    dfs = dfs.interpolate(method = 'linear')
    dfs = dfs.resample('M').mean()
    df = df.drop(df[snow_basins],axis = 1)
    df = df.join(dfs).fillna(method = 'ffill') #snow stations now cleaned up and back in main datafile

    df = df[(df.index > '1951-09-30')]#start at 2000 water year

  df = df.drop(df[snow_ids],axis = 1)

  BND = (df['BND_fnf'].to_frame(name='inf'))
  ORO = (df['ORO_fnf'].to_frame(name='inf'))
  YRS = (df['YRS_fnf'].to_frame(name='inf'))
  FOL = (df['FOL_fnf'].to_frame(name='inf'))

  ## delta gains calculations
  dfg = df[['MIL_fnf','NML_fnf','YRS_fnf','TLG_fnf','MRC_fnf','MKM_fnf','NHG_fnf','SR_WYI']] #gains dataframe
  stations = ['SHA','ORO','YRS','FOL','MIL','NML','YRS','TLG','MRC','MKM','NHG']
  stations_WYI = ['SHA','ORO','YRS','FOL','MIL','NML','YRS','TLG','MRC','MKM','NHG','WYI']
  # dfg = df[['MIL_fnf','NML_fnf','YRS_fnf','TLG_fnf','MRC_fnf','MKM_fnf','NHG_fnf','netgains','WYI_sim']] #gains datafile
  # stations = ['MIL','NML','YRS','TLG','MRC','MKM','NHG']

  for station in stations:
    dfg['%s_fnf' %station] = df['%s_fnf' %station].shift(2)
    dfg['%s_rol10' %station] = df['%s_fnf' %station].rolling(10).sum()
    dfg['%s_rol20' %station] = df['%s_fnf' %station].rolling(20).sum()
    dfg['%s_rol30' %station] = df['%s_fnf' %station].rolling(30).sum()
  dfg = dfg.replace([np.inf, -np.inf], np.nan)
  dfg = dfg.dropna()

  # dfg = dfg.dropna()
  dfgs = pd.DataFrame(index = df.index) #gains_sim df
  dfgs['gains_sim'] = pd.Series()
  month_arr = np.arange(1,13)

  for mth in month_arr:
    dfm = dfg[dfg.index.month == mth] #for each month #hist
    WYI = dfm.SR_WYI.values#rolling(3, center = True, min_periods=1).mean().values
    m = dfm.index.month.values
    d = dfm.index.day.values

    X = np.vstack([WYI])

    X = np.vstack([X,[m]])

    for station in stations:
      V = np.vstack([dfm['%s_fnf' %station]])
      V = np.vstack([dfm['%s_rol10' %station]])
      V = np.vstack([dfm['%s_rol20' %station]])
      V = np.vstack([dfm['%s_rol30' %station]])
      X = np.vstack([X,V])
    X = X.T
    ann = pickle.load( open( "orca/data/gains-neural-nets/ann_%s.pkl"%mth, "rb" ))
    month_gains = ann.predict(X)
    if mth == 10:
      month_gains = month_gains - 500
    elif mth == 11:
      month_gains = month_gains - 200
    elif mth == 12:
      month_gains = month_gains *1.5
    elif mth == 1:
      month_gains = month_gains *1.4
    elif mth == 2:
      month_gains = month_gains *0.75
    elif mth == 4:
      month_gains = month_gains *1.5
    elif mth == 5:
      month_gains = month_gains
    elif mth == 7:
      month_gains = month_gains - 20
    elif mth == 8:
      month_gains = (month_gains - 9000)*0.6
    elif mth == 9:
      month_gains = month_gains - 300
    #to test
    # if mth == 7:
    #   plt.plot(month_gains)

    dfm['gains_sim'] = month_gains
    dfgs['gains_sim_%s'%mth] = dfm['gains_sim']
    dfgs = dfgs.fillna(0)
    dfgs['gains_sim'] = dfgs['gains_sim'] + dfgs['gains_sim_%s'%mth]
  df['gains_sim'] = dfgs.gains_sim.fillna(method = 'bfill') * cfs_tafd #fill in missing beggining values (because of rolling)
  df['gains_sim'] = df.gains_sim.rolling(7, center = True, min_periods=4).mean()

  # plt.plot(df.gains_sim)
  # plt.show()

  #get daily gains from regression
  #using coefficients obtained from process()

  # df['newgains'] = pd.Series()
  # for WYT in ['C','D','BN','AN','W']:
  #   dfw = df[(df.SR_WYT == WYT)]
  #   means = df_g[WYT]
  #   days = [dfw.index.strftime('%Y-%m-%d')]
  #   days = days[0]

  #   for d in days:
  #     df.loc[df.index == d,'newgains'] = means[d[5:]]
  # df['newgains'] = df.newgains.rolling(5).mean()
  # df['newgains']=df.newgains.fillna(method='bfill')
  # df['gains_sim'] = (df['newgains']*0.75+df['gains_sim']*0.25)

  df['OMR_sim'] = pd.Series()
  for WYT in ['C','D','BN','AN','W']:
    dfh = df[(df.SR_WYT == WYT)]
    days = [dfh.index.strftime('%Y-%m-%d')]
    days = days[0]
    for i,d in enumerate(days):
      if d[5:] != '02-29':
        df.loc[df.index == d,'OMR_sim'] = df_OMR[WYT][d[5:]]
      elif d[5:] == '02-29':
        df.loc[df.index == d,'OMR_sim'] = df_OMR[WYT][days[i-1][5:]]
  df['OMR_sim'] = df.OMR_sim.rolling(5).mean()
  df['OMR_sim']=df.OMR_sim.fillna(method='bfill')
  return df


text:
""" Implements class for reading / writing experiences to the "replay dataset."

We assume the following:

(1) Actions and rewards for the full history fit comfortably in memory.
(2) The game state representation for the full history does not.
(3) A single sample of states fits comfortable in memory

For example, if the replay dataset stores the last 1 million experiences,
then the history of actions is 1 byte x 1 M = 1 MB. The same holds for the
history of rewards. However, a modest game state representation might be
four frames of a 64 x 64 pixel grayscale image. In which case the full history
of 1 million states would be (64 x 64 x 4 x 1 bytes x 1 M = 17 GB)
"""
import random
import h5py
import numpy as np

class ReplayDataset(object):
    """ A wrapper around a replay dataset residing on disk as HDF5. """
    def __init__(self, filename, state_shape, dset_size=1000, overwrite=False):
        if overwrite:
            self.fp = h5py.File(filename, 'w')
        else:
            self.fp = h5py.File(filename, 'a')

        if all(x in self.fp for x in ("state", "action", "reward", "non_terminal")):
            self.state = self.fp['state']
            self.dset_size = self.state.shape[0]

            self.action = np.empty(self.dset_size, dtype=np.uint8)
            self.fp['action'].read_direct(self.action)

            self.reward = np.empty(self.dset_size, dtype=np.int16)
            self.fp['reward'].read_direct(self.reward)

            self.non_terminal = np.empty(self.dset_size, dtype=bool)
            self.fp['non_terminal'].read_direct(self.non_terminal)

            if self.dset_size != dset_size:
                print ("Warning: dataset loaded from %s is of size %d, "
                       "not %d as requested. Using existing size."
                       % (filename, self.dset_size, dset_size))

        else:
            self.state = self.fp.create_dataset("state",
                                                (dset_size,) + state_shape,
                                                dtype='uint8')
            self.fp.create_dataset("action", (dset_size,), dtype='uint8')
            self.fp.create_dataset("reward", (dset_size,), dtype='int16')
            self.fp.create_dataset("non_terminal", (dset_size,), dtype=bool)

            self.action = np.empty(dset_size, dtype=np.uint8)
            self.reward = np.empty(dset_size, dtype=np.int16)
            self.non_terminal = np.empty(dset_size, dtype=bool)

            self.state.attrs['head'] = 0
            self.state.attrs['valid'] = 0
            self.dset_size = dset_size

        # Index of current 'write' location
        self.head = self.state.attrs["head"]

        # Greatest index of locations with valid (i.e. initialized)
        # experiences. Indices in the range [0, self.valid) are OK.
        self.valid = self.state.attrs["valid"]

    def add_experience(self, action, reward, state):
        """ Add the next step in a game sequence, i.e. a triple
        (action, reward, state) indicating that the agent took 'action',
        received 'reward' and *then* ended up in 'state.' The original state
        is presumed to be the state at index (head - 1)

        Args:
            action:  index of the action chosen
            reward:  integer value of reward, positive or negative
            state:   a numpy array of shape NUM_FRAMES x WIDTH x HEIGHT
                     or None if this action ended the game.
        """
        self.action[self.head] = action
        self.reward[self.head] = reward
        if state is not None:
            self.state[self.head] = state
            self.non_terminal[self.head] = True
        else:
            self.non_terminal[self.head] = False

        # Update head pointer and valid pointer
        self.head = (self.head + 1) % self.dset_size
        self.valid = min(self.dset_size, self.valid + 1)

    def sample(self, sample_size):
        """ Uniformly sample (s,a,r,s) experiences from the replay dataset.

        Args:
            sample_size: (self explanatory)

        Returns:
            A tuple of numpy arrays for the |sample_size| experiences.

                (state, action, reward, next_state)

            The zeroth dimension of each array corresponds to the experience
            index. The i_th experience is given by:

                (state[i], action[i], reward[i], next_state[i])
        """
        if sample_size >= self.valid:
            raise ValueError(
                  "Can't draw sample of size %d from replay dataset of size %d"
                  % (sample_size, self.valid))

        idx = random.sample(xrange(0, self.valid), sample_size)

        # We can't include head - 1 in sample because we don't know the next
        # state, so simply resample (very rare if dataset is large)
        while (self.head - 1) in idx:
            idx = random.sample(xrange(0, self.valid), sample_size)

        idx.sort()  # Slicing for hdf5 must be in increasing order
        next_idx = [x + 1 for x in idx]

        # next_state might wrap around end of dataset
        if next_idx[-1] == self.dset_size:
            next_idx[-1] = 0
            shape = (sample_size,)+self.state[0].shape
            next_states = np.empty(shape, dtype=np.uint8)
            next_states[0:-1] = self.state[next_idx[0:-1]]
            next_states[-1] = self.state[0]
        else:
            next_states = self.state[next_idx]

        is_non_terminal = np.arrays([not self.is_terminal(idx)
                                     for idx in next_idx], dtype=bool)

        return (self.state[idx],
                self.action[next_idx],
                self.reward[next_idx],
                next_states,
                self.non_terminal[next_idx])

    def sample_direct(self, state, action, reward, next_state,
                      non_terminal, sample_size):
        """ Same as sample() but writes data directly to argument arrays. """
        if sample_size >= self.valid:
            raise ValueError(
                  "Can't draw sample of size %d from replay dataset of size %d"
                  % (sample_size, self.valid))

        idx = random.sample(xrange(0, self.valid), sample_size)

        # We can't include head - 1 in sample because we don't know the next
        # state, so simply resample (very rare if dataset is large)
        while (self.head - 1) in idx:
            idx = random.sample(xrange(0, self.valid), sample_size)

        idx.sort()  # Slicing for hdf5 must be in increasing order
        next_idx = [x + 1 for x in idx]

        # next_state might wrap around end of dataset
        if next_idx[-1] == self.dset_size:
            next_idx[-1] = 0
            self.state.read_direct(next_state, np.s_[next_idx[0:-1]], np.s_[0:-1])
            self.state.read_direct(next_state, np.s_[0], np.s_[-1])
        else:
            self.state.read_direct(next_state, np.s_[next_idx])

        self.state.read_direct(state, np.s_[idx])

        # Reward
        reward.flat[:] = self.reward[next_idx]

        # Action
        if action.shape[1] > 1:  # Using a one-hot representation
            action[:] = 0
            action[xrange(sample_size), self.action[next_idx]] = 1
        else:
            action[:] = self.action[next_idx]

        # Check if any of our states are TERMINAL states
        non_terminal.flat[:] = self.non_terminal[next_idx]

    def __del__(self):
        self.fp['non_terminal'][:] = self.non_terminal
        self.fp['action'][:] = self.action
        self.fp['reward'][:] = self.reward
        self.state.attrs['head'] = self.head
        self.state.attrs['valid'] = self.valid

        self.fp.close()

max_stars_repo_name: kjchavez/distributed-deep-q

max_issues_repo_name: kjchavez/distributed-deep-q

max_forks_repo_name: kjchavez/distributed-deep-q


text:
from __future__ import print_function
'''
Help functions for io.
'''
try:
    from builtins import range
except:
    pass

import numpy as np
import h5py, glob

def read_slice(lines):
    n = len(lines) / 2
    U = []
    for i in range(n):
        V = []
        line_real = lines[i]
        line_real = line_real.split()
        line_imag = lines[i + n]
        line_imag = line_imag.split()
        for j in range(n):
            V.append(float(line_real[j]) + float(line_imag[j]) * 1.j)
        U.append(V)
    U = np.array(U).T  # Fortran convension to C convension
    return U

def read_TRANS(fileName):
    '''
    Read transformation matrix.
    '''
    with open(fileName, 'r') as f:
        lines = f.readlines()
    line = lines[0]
    line = line.split()
    numOrbitals = int(line[0])
    numIons = int(line[1])
    u_list = []
    iBase = 1
    for ion in range(numIons):
        u_list.append(read_slice(lines[iBase: iBase + numOrbitals * 2]))
        iBase += numOrbitals * 2
    return u_list

def read_Hs(fileName):
    '''
    Read Hermitian matrix basis.
    '''
    with open(fileName, 'r') as f:
        lines = f.readlines()
    line = lines[0]
    line = line.split()
    numIons = int(line[0])
    iLine = 1
    HsAll = []
    for ion in range(numIons):
        Hs = []
        line = lines[iLine]
        line = line.split()
        numOrbitals = int(line[0])
        numHs = int(line[1])
        iLine += 1
        for iHs in range(numHs):
            Hs.append(read_slice(lines[iLine: iLine + numOrbitals * 2]))
            iLine += numOrbitals * 2
        HsAll.append(Hs)
    return HsAll

def write_slice(f, U):
    for V in U.T:
        f.write("".join("%20.16f" % (x.real) for x in V) + '\n')
    for V in U.T:
        f.write("".join("%20.16f" % (x.imag) for x in V) + '\n')

def write_TRANS(fileName, u_list):
    '''
    Write transformation matrix.
    '''
    numIons = len(u_list)
    numOrbitals = len(u_list[0])
    f = open(fileName, 'w')
    f.write(" %5i %5i" % (numOrbitals, numIons) + '\n')
    for u in u_list:
        write_slice(f, u)
    f.close()

def write_RLNEF(fileName, RAll, LA1All, NKSAll, EFermi):
    '''
    Write new solution under the current basis transformation.
    '''
    numIons = len(RAll)
    numOrbitals = len(RAll[0])
    with open(fileName, 'w') as f:
        f.write(" %5i %5i" % (numOrbitals, numIons) + '\n')
        for ion in range(numIons):
            write_slice(f, RAll[ion])
            write_slice(f, LA1All[ion])
            write_slice(f, NKSAll[ion])
        f.write(" %20.12f" % (EFermi))

def write_Hs(fileName, HsAll):
    '''
    Write new solution under the current basis transformation.
    '''
    numIons = len(HsAll)
    with open(fileName, 'w') as f:
        f.write(" %5i" % (numIons) + '\n')
        for ion in range(numIons):
            numHs = len(HsAll[ion])
            f.write(" %5i %5i" % (len(HsAll[ion][0]), numHs) + '\n')
            for iHs in range(numHs):
                write_slice(f, HsAll[ion][iHs])

def write_sigma_struct(sigma_list, file_name="WH_SIGMA_STRUCT.INP"):
    '''
    Write WH_SIGMA_STRUCT.INP file.
    '''
    with open(file_name, 'w') as f:
        for ni, sigma in enumerate(sigma_list):
            f.write(" %5i %5i" % (ni + 1, len(sigma)) + '\n')
            for col in np.array(sigma).T:
                f.write(' '.join('%4d' % (x) for x in col) + '\n')

def write_frozen(ne_mott_list, orb_mott_list):
    '''
    Write FROZEN.INP file.
    '''
    with open("FROZEN.INP", 'w') as f:
        for ni, orbs in enumerate(orb_mott_list):
            f.write(" %5i ! NI" % (ni + 1) + '\n')
            f.write(" %5i %5i ! N_ORB, N_ELECTRON_MOTT_LOCALIZED" % (
                    len(orbs), ne_mott_list[ni]) + '\n')
            if len(orbs) > 0:
                f.write(''.join(" %5i " % (orb + 1) for orb in orbs) + '\n')

def write_WH_SL_VEC(s_vec_list, l_vec_list):
    '''
    Write WH_SL_VEC.INP file.
    '''
    s_vec_list = np.swapaxes(s_vec_list, 0, 1)
    l_vec_list = np.swapaxes(l_vec_list, 0, 1)
    with open("WH_SL_VEC.INP", 'w') as f:
        for _s_list in s_vec_list:
            for _s in _s_list:
                write_slice(f, _s)
        for _l_list in l_vec_list:
            for _l in _l_list:
                write_slice(f, _l)

def special_reorder(a_list):
    if len(a_list) != 1 or a_list[0].shape[0] != 6:
        print('Dimension not correct, special reorder cancelled!')
        return a_list

    umat = np.array([
        [0, 1, 0],
        [1, 0, 0],
        [0, 0, 1]], dtype=complex)

    mat2 = np.zeros((6, 6), dtype=complex)
    mat2[0:3, 0:3] = umat.dot(a_list[0].T[::2, ::2]).dot(umat.T.conj()).T
    mat2[3:6, 3:6] = mat2[0:3, 0:3]
    return [mat2]

def h52txt_rln(fh5_name, ftxt_name, specialt=False):
    lma_list = []
    nks_list = []
    r_list = []
    with h5py.File(fh5_name, 'r') as f:
        for imp in range(1000):
            path = '/IMPURITY_{}'.format(imp+1)
            if path in f:
                lma_list.append(f[path+'/LAMBDA'][()].T)
                nks_list.append(f[path+'/NKS'][()].T)
                r_list.append(f[path+'/R'][()].T)
            else:
                break
    if specialt:
        lma_list = special_reorder(lma_list)
        nks_list = special_reorder(nks_list)
        r_list = special_reorder(r_list)

    write_RLNEF(ftxt_name, r_list, lma_list, nks_list, 0.0)

def h52txt_rln_list(h5p='WH_RL_INIT.h5', txtp='WH_RLNEF.INP', specialt=False):
    for fname in glob.glob(h5p+'*'):
        h52txt_rln(fname, txtp+fname[len(h5p):], specialt=specialt)

if __name__=='__main__':
    h52txt_rln_list(specialt=True)

max_stars_repo_name: comscope/comsuite

max_issues_repo_name: comscope/Comsuite

max_forks_repo_name: comscope/Comsuite


text:
from mbi import Dataset, FactoredInference, Domain
import numpy as np
from scipy import sparse

"""
This file is essentially the same as adult_example with two key differences:

    (1) it is run with the torch backend
    (2) scipy.sparse.eye(n) is replaced with Identity(n), which improves speed by a factor of 2

Note that for this example, running on a GPU does not speed up inference, likely because
the total clique size is small (6516).  With more complicated measurements/models torch may help.
"""

class Identity(sparse.linalg.LinearOperator):
    def __init__(self, n):
        self.shape = (n,n)
        self.dtype = np.float64
    def _matmat(self, X):
        return X
    def __matmul__(self, X):
        return X
    def _transpose(self):
        return self
    def _adjoint(self):
        return self

# load adult dataset

data = Dataset.load('../data/adult.csv', '../data/adult-domain.json')
domain = data.domain
total = data.df.shape[0]

print(domain)

# spend half of privacy budget to measure all 1 way marginals
np.random.seed(0)

epsilon = 1.0
sigma = 1.0 / len(data.domain) / 2.0

measurements = []
for col in data.domain:
    x = data.project(col).datavector()
    y = x + np.random.laplace(loc=0, scale=sigma, size=x.size)
    I = Identity(x.size)
    measurements.append( (I, y, sigma, (col,)) )

# spend half of privacy budget to measure some more 2 and 3 way marginals

cliques = [('age', 'education-num'),
            ('marital-status', 'race'),
            ('sex', 'hours-per-week'),
            ('hours-per-week', 'income>50K'),
            ('native-country', 'marital-status', 'occupation')]

sigma = 1.0 / len(cliques) / 2.0

for cl in cliques:
    x = data.project(cl).datavector()
    y = x + np.random.laplace(loc=0, scale=sigma, size=x.size)
    I = Identity(x.size)
    measurements.append( (I, y, sigma, cl) )

# now perform inference to estimate the data distribution

engine = FactoredInference(domain, backend='torch', log=True, iters=10000)
model = engine.estimate(measurements, total=total, engine='RDA')

# now answer new queries

y1 = model.project(('sex', 'income>50K')).datavector()
y2 = model.project(('race', 'occupation')).datavector()

reason: import numpy,from scipy


text:
# this program will populate the core_data.csv file with results of object detection on respective image

import cv2
import os
import sys
import pandas as pd
import numpy as np

# Root directory of the project
MAIN_ROOT_DIR = os.path.abspath(".")
print(MAIN_ROOT_DIR)
#Path for root directory Mask RCNN folder
ROOT_DIR = os.path.join(MAIN_ROOT_DIR, "Mask_RCNN")
print(ROOT_DIR)

# Import Mask RCNN local libraries
sys.path.append(ROOT_DIR)  # To find local version of the library
from mrcnn import utils
import mrcnn.model as modellib

#This is a python file created for training the MaskRCNN model for sepcific application
# it contains details/configuration specific to application here pothole
import pot_hole

# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "logs")

# Local path to trained weights file
POTHOLE_MODEL_PATH = os.path.join(MAIN_ROOT_DIR, "mask_rcnn_pot_hole_0007.h5")
print(POTHOLE_MODEL_PATH)

# Directory of images to run detection on
#Note main_dir is same as MAIN_ROOT_DIR
main_dir = os.path.abspath(".")
IMAGE_DIR =  os.path.join(main_dir, "road_images")# r"C:\Users\manis\PycharmProjects\pothole_heatmap\road_images"
print("img_dir", IMAGE_DIR)

"""
## Configurations

We'll be using a model trained on the pothole dataset. The configurations of this model are in the ```Pot_holeConfig``` class in ```pot_hole.py```.

For inferencing, modify the configurations a bit to fit the task. To do so, sub-class the ```Pot_holeConfig``` class and override the attributes you need to change.
"""
config = pot_hole.Pot_holeConfig()

class InferenceConfig(config.__class__):
    # Set batch size to 1 since we'll be running inference on
    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

config = InferenceConfig()
config.display()

"""
## Create Model and Load Trained Weights
"""
# Create model object in inference mode.
model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config)

# Load weights of our model
model.load_weights(POTHOLE_MODEL_PATH, by_name=True)

"""
## Class Names

The model classifies objects and returns class IDs, which are integer value that identify each class. Some datasets assign integer values to their classes and some don't. For example, in the MS-COCO dataset, the 'person' class is 1 and 'teddy bear' is 88. The IDs are often sequential, but not always. The COCO dataset, for example, has classes associated with class IDs 70 and 72, but not 71.

To improve consistency, and to support training on data from multiple sources at the same time, our ```Dataset``` class assigns it's own sequential integer IDs to each class. For example, if you load the COCO dataset using our ```Dataset``` class, the 'person' class would get class ID = 1 (just like COCO) and the 'teddy bear' class is 78 (different from COCO). Keep that in mind when mapping class IDs to class names.

To get the list of class names, you'd load the dataset and then use the ```class_names``` property like this.

"""

# Here we are declearing the names of our classes
class_names = ['BG', 'Pot_hole']

#This function defines differentcolur for each type of object during object detection
def random_colors(N):
    np.random.seed(1)
    colors = [tuple(255 * np.random.rand(3)) for _ in range(N)]
    return colors

#Function places coloured tanslucent mask on the pixels where object is identified
def apply_mask(image, mask, color, alpha=0.5):
    """apply mask to image"""
    for n, c in enumerate(color):
        image[:, :, n] = np.where(
            mask == 1,
            image[:, :, n] * (1 - alpha) + alpha * c,
            image[:, :, n]
        )
    return image

#This function add boxes to object, calls the apply mask function, and return the final image
def display_instances(image, boxes, masks, ids, names, scores):
    """
        take the image and results and apply the mask, box, and Label
    """
    n_instances = boxes.shape[0]
    colors = random_colors(n_instances)

    if not n_instances:
        print('NO INSTANCES TO DISPLAY')
    else:
        assert boxes.shape[0] == masks.shape[-1] == ids.shape[0]

    for i, color in enumerate(colors):
        if not np.any(boxes[i]):
            continue

        y1, x1, y2, x2 = boxes[i]
        label = names[ids[i]]
        score = scores[i] if scores is not None else None
        caption = '{} {:.2f}'.format(label, score) if score else label
        mask = masks[:, :, i]

        image = apply_mask(image, mask, color)
        image = cv2.rectangle(image, (x1, y1), (x2, y2), color, 2)
        image = cv2.putText(
            image, caption, (x1, y1), cv2.FONT_HERSHEY_COMPLEX, 0.7, color, 2
        )

    return image
#This function store the object detection values in the columnsof new dataframe against each image
def save_results(df, img, boxes, masks,classid, names, scores):
    print(img)
    df["boxes"][img] = boxes
    print(df["boxes"][img])
    #Below we have commented out the the string of mask value in csv,
    #this is because, it is of dimention 1024*1024 hence it is not stored in usable form
    #In future wi will convert such that it becomes usable
    #df["masks"][img] = masks
    #print(df["masks"][img])
    df["class_ids"][img] = classid
    print(df["class_ids"][img])
    df["class_names"][img] = names
    #print(df["class_names"][img])
    df["scores"][img] = scores
    print(df["scores"][img])
    return df
"""
## Run Object Detection
"""

#creating list of al the image for inference
images = os.listdir(IMAGE_DIR)
print(images)

#loading core _data csv file to populate it furthe with object detection results, according added new Columns
core_data_loc = os.path.join(main_dir, "core_data.csv")
df= pd.read_csv(core_data_loc)
df = df.set_index("Image")
df["boxes"] = 1
df["masks"] = 1
df["class_ids"] = 1
df["class_names"] = 1
df["scores"] = 1
print(df)

#The Results of object detection are multidimentional hence defininf the dtaatype for each columens
convert_dict = {"boxes": object, "masks": object, "class_ids": object, "class_names": object,"scores":object,}
df = df.astype(convert_dict)
print(df)

for img in images:
    print(img)

    #reading each image
    image = cv2.imread(os.path.join(IMAGE_DIR, img))

    #Converting it to a size that is compatible to our model
    image= cv2.resize(image, (1024, 1024))

    #Runnig Inference
    results = model.detect([image], verbose=1)
    r = results[0]

    #Printing the object detection results just for reference
    print(type(r),type(r['rois']),type(r['masks']),type(r['class_ids']),type(class_names), type(r['scores']))
    masked_image= display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
    loc = os.path.join(main_dir, "pot_holes_detected",  img)
    print(loc)

    #Saving the masked image into pothole_detected folder
    cv2.imwrite(loc,masked_image)

    #Saving the results of object detection into the dataframe
    all_data = save_results(df,img, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
    print(all_data)

    #Coverting dataframe into CSV
    super_core_data_loc = os.path.join(main_dir, "super_core_data.csv")
    print(super_core_data_loc)
    all_data.to_csv(super_core_data_loc)

# closing all open windows
cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_name: manish-sin/BeterRoads

max_issues_repo_name: manish-sin/BeterRoads

max_forks_repo_name: manish-sin/BeterRoads


text:
import random
import time
import numpy as np
from hps.algorithms.HPOptimizationAbstract import HPOptimizationAbstract

class SimulatedAnnealing(HPOptimizationAbstract):

    def __init__(self, **kwargs):
        # inheritance init
        super(SimulatedAnnealing, self).__init__(**kwargs)
        self._check_hpo_params()

    def _check_hpo_params(self):
        self._n_pop = self._n_params
        self._T0 = self._hpo_params["T0"]
        self._alpha = self._hpo_params["alpha"]
        self._k = self._hpo_params["k"]
        self._n_steps = self._hpo_params["n_steps"]

    def _generate(self, param_list, score_list, iter):
        result_param_list = list()

        # random init population
        best_param_list = self._population(param_list)
        # population 값과 비교할 후보군
        neighbor_selection = self._neighbor(best_param_list)
        # population & 후보군 중 선택하는 방법
        accept_criteria = self._accept(neighbor_selection, best_param_list)

        result_param_list = accept_criteria
        result_param_list = self._remove_duplicate_params(result_param_list)
        return result_param_list

    # population 초기화
    def _population(self, param_list):
        if len(param_list) == 0:
            return self._generate_param_dict_list(self._n_pop)
        else :
            return param_list

    # neighbor selection로 candidate 생성
    def _neighbor(self, param_dict_list):
        return self._generate_param_dict_list(self._n_params)

    # accept criteria
    def _accept(self, param_dict_list, best_params):
        best_params_list = list()

        _, of_final = self._learn(self._n_steps, param_dict_list)
        _, of_new = self._learn(self._n_steps, best_params)

        # best값과 neighbor값의 비교
        if of_new <= of_final :
            best_params_list = param_dict_list
        else :
            random_value = np.random.rand()
            form = 1 / (np.exp((of_new[1] - of_final[1]) / self._T0))
            if random_value <= form :
                best_params_list = best_params
            else :
                best_params_list = param_dict_list

        # temperature 조절
        self._T0 = self._alpha * self._T0

        return best_params_list

if __name__ == '__main__':
    hprs_info = {
        "hpo_params" : {
            "T0" : 0.40,
            "alpha" : 0.85,
            "n_pop" : 1,
            "k" : 0.1,
            "n_params": 10,
            "k_val": 1,
            "eval_key": "accuracy"
        },
        "ml_params":{
            "model_param":{
                "input_units" : "100",
                "output_units" : "1",
                "global_step" : "10",
                "early_type" : "2",
                "min_step" : "10",
                "early_key" : "accuracy",
                "early_value" : "0.98",
                "method_type" : "Basic",
                "global_sn" : "0",
                "alg_sn" : "0",
                "algorithm_type" : "classifier",
                "job_type" : "learn"
            },
            "pbounds":{
                "dropout_prob": [0, 0.5],
                "optimizer_fn": "Adam",
                "learning_rate": 0.8,
                "act_fn": "Sigmoid",
                "hidden_units" : 50
            }
        }
    }
    sa = SimulatedAnnealing(hps_info = hprs_info)
    best_params = sa._generate([], [])
    print(best_params)

max_stars_repo_name: chunghyunhee/GoldMine

max_issues_repo_name: chunghyunhee/GoldMine

max_forks_repo_name: chunghyunhee/GoldMine


text:
import os.path as path
import json
import numpy as np
import tensorflow as tf
from .tool_generate_data import GenerateData
from .hmm import HMM

thisdir = path.dirname(path.realpath(__file__))

max_iter = 100
tolerance = 0.0001
generator = GenerateData(num_time=2, seed=2)

# Make data
states, emissions = generator.data()

# Check reference
hmm = HMM(generator.num_states, generator.num_dims,
          obs=generator.num_obs, time=generator.num_time)
with tf.Session(graph=hmm._graph) as sess:
    sess.run(tf.global_variables_initializer())
    alpha, c, beta, gamma, xi = sess.run(
        [hmm._alpha, hmm._c, hmm._betta, hmm._gamma, hmm._xi],
        {
            hmm._dataset_tf: np.transpose(emissions, [1, 0, 2]),
            hmm._p0_tf: generator.pi[np.newaxis, :],
            hmm._tp_tf: generator.A,
            hmm._mu_tf: generator.mu,
            hmm._sigma_tf: generator.Sigma
        }
    )

# Save input and output
with open(path.join(thisdir, 'expectation.json'), 'w') as fp:
    json.dump({
        'config': {
            **generator.config,
            'maxIterations': max_iter,
            'tolerance': tolerance
        },
        'input': {
            'emissions': emissions.tolist(),
            'pi': generator.pi.tolist(),
            'A': generator.A.tolist(),
            'mu': generator.mu.tolist(),
            'Sigma': generator.Sigma.tolist()
        },
        'output': {
            'alpha': alpha.tolist(),
            'c': c.tolist(),
            'beta': beta.tolist(),
            'gamma': gamma.tolist(),
            'xi': xi.tolist()
        }
    }, fp)

max_stars_repo_name: ben-hayes/node-hidden-markov-model-tf

max_issues_repo_name: ben-hayes/node-hidden-markov-model-tf

max_forks_repo_name: ben-hayes/node-hidden-markov-model-tf