text:
import sympy
import collections
from IPython.display import display
from sympy import *
from sympy.tensor import IndexedBase, Idx, Indexed
from sympy.physics.secondquant import KroneckerDelta as delta
from sympy.core.relational import Relational

init_printing()

class Equals(Relational):
    """
    Non-functional equals for display purpose only
    """
    rel_op = '=='

def apply_recursive(f, expr):
    """
    Applys a function to each sub-tree of the expression tree,
    returns the new expression.
    """
    if expr.args:
        new_args = [apply_recursive(f,e) for e in expr.args]
        return f(expr.func(*new_args))
    else: return f(expr)

def expand_sums(expr, indices):
    """
    Expands sums on multiplicative terms on RHS where
    indices appear exactly twice.
    """

    if not indices: return expr
    if not isinstance(indices, collections.Iterable): indices = [indices]

    contracted = set()

    def sum_mul_term(expr, index):
        if not isinstance(expr, sympy.Mul): return expr

        count = expr.count(index)
        if (count == 2):
            contracted.add(index)
            terms = [expr.subs(index,val) for val in range(index.lower, index.upper+1)]
            expr = sympy.Add(*terms)
        return expr

    # Expand sum for every index
    rhs = expr.rhs
    for index in indices:
        rhs = apply_recursive(lambda e: sum_mul_term(e, index), rhs)

    return Equals(expr.lhs, rhs)

def enumerate_indices(expr, indices):
    """
    Generates a list of expressions from a single
    expression by substitutng values into indices
    based on their range.
    """

    # Make sure indices is a list
    if not indices: return expr
    if not isinstance(indices, collections.Iterable): indices = [indices]

    all_expr = [expr]
    indices = list(indices)        # To allow for popping below

    # For every index
    while indices:
        index = indices.pop()

        # For every previously added expression
        num_expr = len(all_expr)
        for i in range(0,num_expr):
            expr = all_expr.pop(0)
            # Substitute
            new_exprs = [expr.subs(index, val) for val in range(index.lower, index.upper+1)]
            # Add only if doesnt exist yet
            [all_expr.append(new) for new in new_exprs if new not in all_expr]

    return all_expr

# Pair -> Parent value
voigt_map = {1:{(0,0):0},

             2:{(0,0):0,  (0,1):2,
                (1,0):2,  (1,1):1},

             3:{(0,0):0, (0,1):5, (0,2):4,
                (1,0):5, (1,1):1, (1,2):3,
                (2,0):4, (2,1):3, (2,2):2}}

def apply_rules(tensor, full_exprs, voigt_pairs, kinematic_pairs, vector_pairs):
    """
    Cycles through index pairs and applies voigt/vector rules if required
    """

    # Cycle through index pairs
    indices = tensor.indices
    for pos in range(len(indices)-1,-1,-1):
        pair = tuple(indices[pos:pos+2])

        # Determine what rule to apply
        rule = None
        rhs_transform = lambda num_pair, x: x
        if pair in voigt_pairs:
            # Get corrent dimension
            dim = pair[0].upper+1

            # Get factor to multiply rhs by
            if pair in kinematic_pairs:
                rhs_transform = lambda num_pair, x: 2*x if num_pair[0] != num_pair[1] else x

            # Pair mapping rule
            rule = lambda num_pair: voigt_map[dim][num_pair]

        elif pair in vector_pairs:
            # Flip pair
            if (pair[0].label.name.islower()):
                dim = pair[0].upper+1
                rule = lambda num_pair: num_pair[1]*dim + num_pair[0]
            else:
                dim = pair[1].upper+1
                rule = lambda num_pair: num_pair[0]*dim + num_pair[1]

        # Dont transform this rule if nothing found
        if not rule: continue

        results = []
        for expr in full_exprs:
            lhs = expr.lhs
            num_pair = tuple(lhs.indices[pos:pos+2])
            new_val = rule(num_pair)
            new_indices = lhs.indices[0:pos] + (new_val,) + lhs.indices[pos+2:]
            lhs = lhs.base[new_indices]
            rhs = rhs_transform(num_pair, expr.rhs)
            results.append(Equals(lhs, rhs))
        full_exprs = results

    return full_exprs

def generate_matrix(exprs):
    """
    Converts a set of expressions to a matrix
    """

    # Count required indicies
    nindices = min([len(expr.lhs.indices) for expr in exprs])
    nrows = max([expr.lhs.indices[0] for expr in exprs]) + 1
    if nindices == 1:
        ncols = 1
    elif nindices == 2:
        ncols = max([expr.lhs.indices[1] for expr in exprs]) + 1
    else:
        return exprs    # Cannot generate matrix

    #lhs_mat = zeros(nrows, ncols)
    rhs_mat = zeros(nrows, ncols)

    for expr in exprs:
        row = expr.lhs.indices[0]
        if nindices == 2: col = expr.lhs.indices[1]
        else: col = 0
        #lhs_mat[row,col] = expr.lhs
        if not rhs_mat[row,col]: rhs_mat[row,col] = expr.rhs

    return Equals(expr.lhs.base, rhs_mat)

def generate_subs_map(exprs):
    return {expr.rhs : expr.lhs for expr in exprs}

def transform(expr, indices, voigt=[], kinematic=[], vector=[]):
    """
    Converts a tensor expression to a set of expanded expressions with
    voigt and vector rules applied
    """

    # Convert voigt, kinematic, vector to lists of pairs, if required
    def ensure_list_of_pairs(x): return [x] if (x and isinstance(x[0], Idx)) else x
    voigt     = ensure_list_of_pairs(voigt)
    kinematic = ensure_list_of_pairs(kinematic)
    vector    = ensure_list_of_pairs(vector)

    # This is the lhs of the original tensor expression, which is used to determine
    # where the original indices were
    tensor = expr.lhs
    if not isinstance(tensor, Indexed): raise TypeError("Require an indexed expression on lhs.")

    # Expand sums, generate expressions
    exprs = expand_sums(expr, indices)
    exprs = enumerate_indices(exprs, indices)
    exprs = apply_rules(tensor, exprs, voigt, kinematic, vector)
    return exprs

A = IndexedBase("A")
B = IndexedBase("B")
C = IndexedBase("\mathbb{C}")
N = IndexedBase("N")
dNdX = IndexedBase("\\frac{dN}{dX}")
IoI = IndexedBase("I \otimes I")
II = IndexedBase("\mathbb{I}")
P = IndexedBase("\mathbb{P}")
dRdu = IndexedBase("dRdu")

strain = IndexedBase("varepsilon")

i = Idx("i",2)
j = Idx("j",2)
k = Idx("k",2)
l = Idx("l",2)
r = Idx("r",2)
m = Idx("m",2)
n = Idx("n",2)

I = Idx("I",4)
J = Idx("J",4)
M = Idx("M",4)

# Strain tensor
expr = Equals(strain[i,j], strain[i,j])
exprs = transform(expr, (i,j), (i,j), (i,j))
mat = generate_matrix(exprs)
display(mat)

# B Matrix
expr = Equals(B[i,j,r,I], Rational(1,2)*(N[I,j]*delta(r,i) + N[I,i]*delta(r,j)))
exprs = transform(expr, (i,j,r,I), voigt=(i,j), kinematic=(i,j), vector=(r,I))
B = generate_matrix(exprs)
display(B)
B = B.rhs

# Trace operatpr
expr = Equals(IoI[i,j,k,l], delta(i,j)*delta(k,l))
exprs = transform(expr, (i,j,k,l), voigt=[(i,j), (k,l)])
mat = generate_matrix(exprs)
display(mat)

# 4th order identity
expr = Equals(II[i,j,k,l], 0.5*delta(i,k)*delta(j,l) + 0.5*delta(i,l)*delta(j,k))
exprs = transform(expr, (i,j,k,l), voigt=[(i,j), (k,l)])
mat = generate_matrix(exprs)
display(mat)

# Projection
expr = Equals(P[i,j,k,l], Rational(1,2)*(delta(i, k)*delta(j, l)+delta(i, l)*delta(j, k))-Rational(1,3)*delta(i, j)*delta(k, l))
exprs = transform(expr, (i,j,k,l), voigt=[(i,j), (k,l)])
mat = generate_matrix(exprs)
display(mat)

# Tangent matrix
expr = Equals(C[i,j,k,l], C[i,j,k,l])
exprs = transform(expr, (i,j,k,l), voigt=[(i,j), (k,l)])
sub_map = generate_subs_map(exprs)
CC = generate_matrix(exprs)
CC = CC.rhs

# Derivative of residual  w.r.t. displacement
expr = Equals(dRdu[I, i, J, j], N[I,k]*C[i, k, j, n]*N[J,n])
exprs = transform(expr, (i,I,j,J,k,n), vector=[(I,i), (J,j)])
exprs = [e.subs(sub_map) for e in exprs]
mat = generate_matrix(exprs)
display(mat)

# Multiplication
A = IndexedBase("A")
B = IndexedBase("B")
C = IndexedBase("C")

i = Idx("i", 2)
j = Idx("j", 2)
k = Idx("k", 2)

expr = Equals(C[i,j], A[i,k]*B[k,j])

exprs = transform(expr, (i,j,k))
mat = generate_matrix(exprs)

reason: import sympy,from sympy


text:
import os

import numpy as np
import pandas as pd

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import KFold
from .predict_missing_value import data_dropna
from .predict import predict

def cross_validate_grid_search(values, X_df, y_df):

    """
        This function uses 10-fold cross-validation to choose the best
        combination of max_depth and n_estimators for random forest classifier.

        Arguments:
        values: [list1, list2], where list1 is a list of values for max_depth
        and list2 is a list of values for n_estimators

        X_df: dataframe, where each column represents a predictor
        y_df: series, representing the response variable

        Return:
        best_for_total: tuple, parameter combination to achieve the highest
                        total accuracy
        best_for_solution: tuple, parameter combination to achive the highest
                        accuracy for classifing "solution"
        best_for_capsules: tuple, parameter combination to achive the highest
                        accuracy for classifing "capsules"
        best_for_tablets: tuple, parameter combination to achive the highest
                        accuracy for classifing "tablets"

    """
    assert len(values) == 2
    assert isinstance(X_df, pd.DataFrame)
    assert isinstance(y_df, pd.DataFrame)

    X = X_df.values
    y = y_df.values.flatten()

    assert X.shape[0] == y.shape[0]

    best_accuracy = 0.
    best_depth = 0
    best_ntrees = 0
    accuracies_k_fold = []

    solution_k_fold = []
    capsules_k_fold = []
    tablets_k_fold = []

    grid = []
    gridvalues = []

    for max_depth in values[0]:
        for ntrees in values[1]:
            grid.append([max_depth, ntrees])

            accuracies = []
            solution = 0.
            capsules = 0.
            tablets = 0.

            kf = KFold(n_splits=10, shuffle=True, random_state=123)
            for train_index, test_index in kf.split(X):
                X_trn, X_tst = X[train_index], X[test_index]
                y_trn, y_tst = y[train_index], y[test_index]

                clf = RandomForestClassifier(
                                    n_estimators=ntrees,
                                    max_depth=max_depth,
                                    random_state=0)
                clf.fit(X_trn, y_trn)

                y_pred = clf.predict(X_tst)
                m = y_pred.shape[0]
                accuracy = [1 for i in range(m) if y_pred[i] == y_tst[i]]
                accuracy = np.sum(accuracy)/m
                accuracies.append(accuracy)

                report = classification_report(
                    y_tst, y_pred, output_dict=True, zero_division=1)
                solution += report['solution']['f1-score']
                capsules += report['capsules']['f1-score']
                tablets += report['tablets']['f1-score']

            acc_k_fold = np.mean(accuracies)
            accuracies_k_fold.append(acc_k_fold)

            solution_k_fold.append(solution/10)
            capsules_k_fold.append(capsules/10)
            tablets_k_fold.append(tablets/10)

            if acc_k_fold > best_accuracy:
                best_accuracy = acc_k_fold
                best_depth = max_depth
                best_ntrees = ntrees

            print('max depth: {:d}, n_estimators: {:d}, accuracy: {:f}'.format(
                max_depth, ntrees, acc_k_fold))
            gridvalues.append([max_depth, ntrees, acc_k_fold])

    # End search
    # Save grid points and values for plotting figures
    os.makedirs('./images', exist_ok=True)
    np.save('./images/gridvalues.npy', gridvalues)

    best_for_total = (best_depth, best_ntrees)

    best_for_solution = grid[np.argmax(solution_k_fold)]
    best_for_capsules = grid[np.argmax(capsules_k_fold)]
    best_for_tablets = grid[np.argmax(tablets_k_fold)]

    best_solution_accuracy = np.max(solution_k_fold)
    best_capsules_accuracy = np.max(capsules_k_fold)
    best_tablets_accuracy = np.max(tablets_k_fold)

    print('Best accuracy for solution: {:f}'.format(best_solution_accuracy))
    print('Best accuracy for capsules: {:f}'.format(best_capsules_accuracy))
    print('Best accuracy for tablets: {:f}'.format(best_tablets_accuracy))

    print('Best accuracy for total: {:f}'.format(best_accuracy))

    results = [
            best_for_total,
            best_for_solution,
            best_for_capsules,
            best_for_tablets]

    return results

def cross_validate_n_predictors(X_df, y_df, max_depth, n_estimators):

    """
        This function uses 10-fold cross-validation to choose the best
        number of predictors to be included in the training set.

        Arguments:
        X_df: dataframe, where each column represents a predictor
        y_df: series, representing the response variable

        max_depth: the maximum depth of the tree
        n_estimators: the number of trees in the forest.

        Return:
        best_p: number of predictors to be included to achieve
        the highest total accuracy
    """
    assert isinstance(X_df, pd.DataFrame)
    assert isinstance(y_df, pd.DataFrame)
    assert isinstance(max_depth, int)
    assert isinstance(n_estimators, int)

    X = X_df.values
    y = y_df.values.flatten()

    assert X.shape[0] == y.shape[0]

    p = X.shape[1]
    p_values = range(1, p+1)

    best_accuracy = 0.
    best_p = p
    accuracies_k_fold = []

    for p_val in p_values:

        accuracies = []

        kf = KFold(n_splits=10, shuffle=True, random_state=123)
        for train_index, test_index in kf.split(X):
            X_trn, X_tst = X[train_index, :p_val], X[test_index, :p_val]
            y_trn, y_tst = y[train_index], y[test_index]

            clf = RandomForestClassifier(
                            n_estimators=n_estimators,
                            max_depth=max_depth,
                            random_state=0)
            clf.fit(X_trn, y_trn)

            # y_pred = clf.predict(X_tst)
            y_pred = predict(clf, X_tst)
            m = y_pred.shape[0]
            accuracy = [1 for i in range(m) if y_pred[i] == y_tst[i]]
            accuracy = np.sum(accuracy)/m
            accuracies.append(accuracy)

        acc_k_fold = np.mean(accuracies)
        accuracies_k_fold.append(acc_k_fold)
        if acc_k_fold > best_accuracy:
            best_accuracy = acc_k_fold
            best_p = p_val

        print('p: {:d}, accuracy: {:f}'.format(p_val, accuracy))

    print('best p: {:d}, best accuracy: {:f}'.format(best_p, best_accuracy))

    return best_p

if __name__ == '__main__':

    data_path = "./formulation/data/"
    data_fname = 'FDA_APPROVED.csv'

    # Read csv file
    df = pd.read_csv(data_path+data_fname)
    print(df.tail(3))

    # Dropnan
    columns = ['CLogP', 'HBA', 'HBD', 'PSDA', 'Formulation']
    df = data_dropna(df, needed_cols=columns, subset=columns)
    print(df.tail(10))

    # Print count of each category
    print(df.groupby('Formulation').count())

    # Prepare predictors and response variable
    features = ['CLogP', 'HBA', 'HBD', 'PSDA']
    X = df[features]

    target = ['Formulation']
    y = df[target]

    max_depth = range(1, 5)
    ntrees = range(1, 200, 50)

    results = cross_validate_grid_search([max_depth, ntrees], X, y)
    best_for_total = results[0]
    best_for_solution = results[1]
    best_for_capsules = results[2]
    best_for_tablets = results[3]
    print('Best max_depth: {:d}, best n_estimators: {:d}'.format(
                        best_for_total[0], best_for_total[1]))
    print('Best parameter for solution catogory:', best_for_solution)
    print('Best parameter for capsules catogory:', best_for_capsules)
    print('Best parameter for tablets catogory:', best_for_tablets)

    best_p = cross_validate_n_predictors(
                X, y, 2, 100)

ext: py

max_stars_repo_name: Chenyi-Mao/formulation

max_issues_repo_name: Chenyi-Mao/formulation

max_forks_repo_name: Chenyi-Mao/formulation

reason: import numpy


text:
#!/usr/bin/env python
'''
Plot calibrated hits from VDS files
Author: Jonas Sellberg
'''

import h5py
import numpy as np
import argparse
from tqdm import tqdm
import os
from matplotlib import pyplot as plt
from matplotlib import colors

if __name__ == '__main__':

    parser = argparse.ArgumentParser(
        description='Create calibrated AGIPD VDS files')
    parser.add_argument(
        'run',
        help=
        'Get specific run, can also be a set of runs, for example: 1,3,5-10,20,22'
    )
    parser.add_argument('-P',
                        '--plot',
                        help='Plot the first X number of hits (default=all)',
                        type=int,
                        default=0)
    parser.add_argument(
        '-i',
        '--in_folder',
        help=
        'Path of input folder (default=./)',
        default='./')
    parser.add_argument('-o',
                        '--out_folder',
                        help='Path of output folder (default is rxxxx_hits.h5, xxxx is the run number',
                        default=None)
    parser.add_argument('-m',
                        '--module',
                        help='Module of litpixels (default is read from hits file)',
                        default=None)
    parser.add_argument('-v',
                        '--verbose',
                        help='Output additional information (default=False)',
                        default=False,
                        action='store_true')
    args = parser.parse_args()

    if args.run is not None:
        runs = []
        tmp = args.run
        for s in tmp.split(','):
            if "-" in s:
                rmin, rmax = s.split('-')
                runs += list(range(int(rmin), int(rmax) + 1))
            else:
                runs += [int(s)]
    out_folder = args.out_folder
    module = args.module
    for r in runs:
        fname = '%s/r%04d_hits.h5' % (args.in_folder, r)

        if out_folder is None:
            out_folder = 'r%04d_hits' % (r)
        if not os.path.isdir(out_folder):
            os.mkdir(out_folder)
            if args.verbose:
                print('Created folder: %s' % out_folder)

        if not os.path.exists(fname):
            print('WARNING: hits file does not exist, skipping %s' % fname)
            continue
        with h5py.File(fname, 'r') as f:
            hits = f['hits/assembled']
            frameID = f['hits/frameID']
            if module is None:
                module = f['hits/litpixelModule'][()]
            lp = f[f'litpixels_{module}']
            nhits = hits.shape[0]
            if args.verbose:
                print('Found %d hits from run %d in: %s' % (nhits, r, fname))
            if args.plot < 1 or args.plot > nhits:
                nhits_to_plot = nhits
            else:
                nhits_to_plot = args.plot
            print('Plotting %d hits from run %d in: %s'
                  % (nhits_to_plot, r, fname))
            # plot shots in photons [gain = 45 ADU]
            for n in tqdm(range(nhits_to_plot)):
                fig = plt.figure(n)
                #plt.imshow(hits[n]/45., norm=colors.LogNorm(vmin=0.1, vmax=100), interpolation='nearest', cmap='jet')
                #plt.imshow(hits[n], norm=colors.LogNorm(vmin=0.1, vmax=1000))
                plt.imshow(hits[n] / 45.,
                           norm=colors.LogNorm(vmin=0.1, vmax=100))
                plt.xlim(420, 620)
                plt.ylim(560, 720)
                ax = fig.gca()
                circ = plt.Circle(
                    ((hits[n].shape[1] - 1) / 2., (hits[n].shape[0] - 1) / 2.),
                    radius=1,
                    linewidth=1,
                    color='k')
                circ.set_fill(True)
                ax.add_patch(circ)
                plt.title('run %d - shot %d - %d lit pixels'
                          % (r, frameID[n], lp[frameID[n]]))
                plt.colorbar()
                plt.savefig('%s/run%d_shot%d_zoom.png'
                            % (out_folder, r, frameID[n]))
                if args.verbose:
                    print('\tSaved figure: run%d_shot%d_zoom.png'
                          % (r, frameID[n]))
                plt.close()

max_stars_repo_name: JunCEEE/cxiapi

max_issues_repo_name: JunCEEE/cxiapi

max_forks_repo_name: JunCEEE/cxiapi


text:
import copy
import numpy as np

from nilabels.tools.aux_methods.sanity_checks import is_valid_permutation

def relabeller(in_data, list_old_labels, list_new_labels, verbose=True):
    """
    :param in_data: array corresponding to an image segmentation.
    :param list_old_labels: list or tuple of labels
    :param list_new_labels: list or tuple of labels of the same len as list_old_labels
    :param verbose:
    :return: array where all the labels in list_new_labels are substituted with list_new_label in the same order.
    """

    if isinstance(list_new_labels, int):
        list_new_labels = [list_new_labels, ]
    if isinstance(list_old_labels, int):
        list_old_labels = [list_old_labels, ]

    # sanity check: old and new must have the same number of elements
    if not len(list_old_labels) == len(list_new_labels):
        raise IOError('Labels lists old and new do not have the same length.')

    new_data = copy.deepcopy(in_data)

    for k in range(len(list_new_labels)):
        places = in_data == list_old_labels[k]
        if np.any(places):
            np.place(new_data, places, list_new_labels[k])
            if verbose:
                print('Label {0} substituted with label {1}'.format(list_old_labels[k], list_new_labels[k]))
        else:
            if verbose:
                print('Label {0} not present in the array'.format(list_old_labels[k]))

    return new_data

def permute_labels(in_data, permutation):
    """
    Permute the values of the labels in an int image.
    :param in_data: array corresponding to an image segmentation.
    :param permutation:
    :return:
    """
    if not is_valid_permutation(permutation):
        raise IOError('Input permutation not valid.')
    return relabeller(in_data, permutation[0], permutation[1])

def erase_labels(in_data, labels_to_erase):
    """
    :param in_data: array corresponding to an image segmentation.
    :param labels_to_erase: list or tuple of labels
    :return: all the labels in the list labels_to_erase will be assigned to zero.
    """
    if isinstance(labels_to_erase, int):
        labels_to_erase = [labels_to_erase, ]
    return relabeller(in_data, list_old_labels=labels_to_erase,
                      list_new_labels=[0, ] * len(labels_to_erase))

def assign_all_other_labels_the_same_value(in_data, labels_to_keep, same_value_label=255):
    """
    All the labels that are not in the list labels_to_keep will be given the value same_value_label
    :param in_data: array corresponding to an image segmentation.
    :param labels_to_keep: list or tuple of value in in_data
    :param same_value_label: a single label value.
    :return: segmentation of the same size where all the labels not in the list label_to_keep will be assigned to the
             value same_value_label.
    """
    list_labels = sorted(list(set(in_data.flat)))
    if isinstance(labels_to_keep, int):
        labels_to_keep = [labels_to_keep, ]

    labels_that_will_have_the_same_value = list(set(list_labels) - set(labels_to_keep) - {0})

    return relabeller(in_data, list_old_labels=labels_that_will_have_the_same_value,
                      list_new_labels=[same_value_label, ] * len(labels_that_will_have_the_same_value))

def keep_only_one_label(in_data, label_to_keep):
    """
    From a segmentation keeps only the values in the list labels_to_keep.
    :param in_data: a segmentation (only positive labels allowed).
    :param label_to_keep: the single label that will be kept.
    :return:
    """

    list_labels = sorted(list(set(in_data.flat)))

    if label_to_keep not in list_labels:
        print('the label {} you want to keep is not present in the segmentation'.format(label_to_keep))
        return in_data

    labels_not_to_keep = list(set(list_labels) - {label_to_keep})
    return relabeller(in_data, list_old_labels=labels_not_to_keep, list_new_labels=[0, ]*len(labels_not_to_keep),
                      verbose=False)

def relabel_half_side_one_label(in_data, label_old, label_new, side_to_modify, axis, plane_intercept):
    """
    :param in_data: input data array (must be 3d)
    :param label_old: single label to be replaced
    :param label_new: single label to replace with
    :param side_to_modify: can be the string 'above' or 'below'
    :param axis: can be 'x', 'y', 'z'
    :param plane_intercept: voxel along the selected direction plane where to consider the symmetry.
    :return:
    """
    if not in_data.ndim == 3:
        msg = 'Input array must be 3-dimensional.'
        raise IOError(msg)

    if side_to_modify not in ['below', 'above']:
        msg = 'side_to_copy must be one of the two {}.'.format(['below', 'above'])
        raise IOError(msg)

    if axis not in ['x', 'y', 'z']:
        msg = 'axis variable must be one of the following: {}.'.format(['x', 'y', 'z'])
        raise IOError(msg)

    positions = in_data == label_old
    halfed_positions = np.zeros_like(positions)
    if axis == 'x':
        if side_to_modify == 'above':
            halfed_positions[plane_intercept:, :, :] = positions[plane_intercept:, :, :]
        if side_to_modify == 'below':
            halfed_positions[:plane_intercept, :, :] = positions[:plane_intercept, :, :]
    if axis == 'y':
        if side_to_modify == 'above':
            halfed_positions[:, plane_intercept:, :] = positions[:, plane_intercept:, :]
        if side_to_modify == 'below':
            halfed_positions[:, plane_intercept, :] = positions[:, plane_intercept, :]
    if axis == 'z':
        if side_to_modify == 'above':
            halfed_positions[:, :, plane_intercept:] = positions[ :, :, plane_intercept:]
        if side_to_modify == 'below':
            halfed_positions[:, :, :plane_intercept] = positions[:, :, :plane_intercept]

    new_data = in_data * np.invert(halfed_positions) + label_new * halfed_positions.astype(np.int)
    return new_data

max_stars_repo_name: nipy/nilabels

max_issues_repo_name: SebastianoF/LabelsManager

max_forks_repo_name: nipy/nilabels


text:
import torch
from torch import nn
import math
import os
import argparse
import cv2
import numpy as np
from net.predictor import resfcn256
from loader.Dataset import TrainData

class PRNet(object):

    def __init__(self, opt):
        self.batch_size = opt.batch_size
        self.epochs = opt.epochs
        self.learning_rate = opt.learning_rate
        self.net = resfcn256()
        self.optimizer = torch.optim.Adam(self.net.parameters(), lr=self.learning_rate)
        self.criterion = nn.MSELoss(reduction='none')

        #self.weights = cv2.imread("Data/uv-data/weight_mask_final.jpg")  # [256, 256, 3]
        self.weights = cv2.imread("Data/uv-data/map_xgtu.jpg")  # [256, 256, 3]
        self.weights_data = np.zeros([1, 256, 256, 3], dtype=np.float32)

    def setup(self,PATH):
        # load state_dict
        if os.path.exists(PATH):
            self.net.load_state_dict(torch.load(PATH))
        # load weight data
        weights = self.weights
        for i in range(13):
            for j in range(27):
                weights[i + 59, j + 78, :] = 250
        for i in range(13):
            for j in range(27):
                weights[i + 59, j + 153, :] = 250
        # cv2.imshow('weights',weights)
        # cv2.waitKey(0)
        weights_data = self.weights_data
        weights_data[0, :, :, :] = weights
        # [b, h, w, c] to [b, c, h, w]
        weights_data = np.swapaxes(weights_data, 1, 2)
        weights_data = np.swapaxes(weights_data, 1, 3)

        self.weights_data = torch.from_numpy(weights_data)

    def train(self):
        self.optimizer.zero_grad()

    def eval(self):
        self.net.eval()

    def optimize_parameters(self, x, label):
        x = np.array(x)
        x = np.swapaxes(x, 1, 2)
        x = np.swapaxes(x, 1, 3)
        x = torch.from_numpy(x)
        label = np.swapaxes(label, 1, 2)
        label = np.swapaxes(label, 1, 3)
        label = torch.from_numpy(label)
        x_op = self.net.forward(x)
        loss = self.criterion(x_op, label)
        loss = loss.mul(self.weights_data)
        loss = loss.mean()
        loss.backward()
        self.optimizer.step()
        return loss

    def forward(self, x, label):
        x = np.array(x)
        x = np.swapaxes(x, 1, 2)
        x = np.swapaxes(x, 1, 3)
        x = torch.from_numpy(x)
        x_op = self.net.forward(x)
        label = np.swapaxes(label, 1, 2)
        label = np.swapaxes(label, 1, 3)
        label = torch.from_numpy(label)
        error = self.criterion(x_op, label)
        error = error.mean()
        loss = error.mul(self.weights_data)
        loss = loss.mean()
        x_op = x_op.numpy()
        x_op = np.swapaxes(x_op, 1, 3)
        x_op = np.swapaxes(x_op, 1, 2)
        return x_op, error, loss

    def generate(self, x):
        x = np.array(x)
        # forward without error
        x = np.swapaxes(x, 1, 2)
        x = np.swapaxes(x, 1, 3)
        x = torch.from_numpy(x)
        x = torch.tensor(x, dtype=torch.float32)
        x_op = self.net.forward(x)
        x_op = x_op.detach().numpy()
        x_op = np.swapaxes(x_op, 1, 3)
        x_op = np.swapaxes(x_op, 1, 2)
        return x_op

    def save(self,save_path):
        torch.save(self.net.state_dict(), save_path)

max_stars_repo_name: weepiess/PRnet-train

max_issues_repo_name: weepiess/PRnet-train

max_forks_repo_name: weepiess/PRnet-train


text:
# Fix paths for imports to work in unit tests ----------------

if __name__ == "__main__":

    from _fix_paths import fix_paths
    fix_paths()

# ------------------------------------------------------------

# Load libraries ---------------------------------------------

from typing import Dict

from collections import namedtuple

import numpy as np

from ssa_sim_v2.simulator.modules.simulator_module import SimulatorModule

# ------------------------------------------------------------

class MultiStateSimulatorModule(object):
    """
    Base class for all multi state simulator modules.

    :ivar int seed: Seed for the random number generator.
    :ivar np.random.RandomState rng: Random number generator.
    :ivar pd.DataFrame priors: DataFrame with columns for the state
        (e.g. date, hour_of_week) and the prior column. The last column
        defines priors (in the form of dictionaries) for every state.
    :ivar pd.DataFrame base_classes: DataFrame with three columns: date, hour_of_week,
            base_class. The last column defines class for for given dates and hours of week.
    :ivar Dict[str, SimulatorModule] models: Dictionary of single state modules for every valid pair
        of date and hour of week.
    """

    Params = namedtuple('Params', [])

    def __init__(self, priors=None, base_classes=None, seed=9):
        """
        :param pd.DataFrame priors: DataFrame with columns for the state
            (e.g. date, hour_of_week) and the prior column. The last column
            defines priors (in the form of dictionaries) for every state.
        :param pd.DataFrame base_classes: DataFrame with three columns: date, hour_of_week,
            base_class. The last column defines class for given dates and hours of week.
        :param int seed: Seed for the random number generator.
        """

        self.seed = seed
        self.rng = np.random.RandomState(seed)
        self.priors = priors
        self.base_classes = base_classes

        seed_min = 100000
        seed_max = 999999
        seeds = self.rng.randint(low=seed_min, high=seed_max, size=len(self.priors))

        base_df = priors.copy()
        base_df.loc[:, "base_class"] = base_classes["base_class"]
        base_df.loc[:, "seed"] = seeds

        self.models = {}  # type: Dict[str, SimulatorModule]

        for index, row in base_df.iterrows():
            self.models["{}.{}".format(row["date"], row["hour_of_week"])] = row["base_class"](row["prior"], row["seed"])

    def get_models(self):
        """
        Returns a dictionary of underlying click probability models.

        :return: Dictionary of models.
        :rtype: Dict[str, SimulatorModule]
        """
        return self.models


text:
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

"""
data convert to mindrecord file.
"""

import os
import argparse
import numpy as np
import dataset as data
from tokenizer import FullTokenizer
from data_util import Tuple, Pad, Stack
from mindspore.mindrecord import FileWriter
TASK_CLASSES = {
    'udc': data.UDCv1,
    'dstc2': data.DSTC2,
    'atis_slot': data.ATIS_DSF,
    'atis_intent': data.ATIS_DID,
    'mrda': data.MRDA,
    'swda': data.SwDA,
}

def data_save_to_file(data_file_path=None, vocab_file_path='bert-base-uncased-vocab.txt', \
        output_path=None, task_name=None, mode="train", max_seq_length=128):
    """data save to mindrecord file."""
    MINDRECORD_FILE_PATH = output_path + task_name+"/" + task_name + "_" + mode + ".mindrecord"
    if not os.path.exists(output_path + task_name):
        os.makedirs(output_path + task_name)
    if os.path.exists(MINDRECORD_FILE_PATH):
        os.remove(MINDRECORD_FILE_PATH)
        os.remove(MINDRECORD_FILE_PATH + ".db")
    dataset_class = TASK_CLASSES[task_name]
    tokenizer = FullTokenizer(vocab_file=vocab_file_path, do_lower_case=True)
    dataset = dataset_class(data_file_path+task_name, mode=mode)
    applid_data = []
    datalist = []
    print(task_name + " " + mode + " data process begin")
    dataset_len = len(dataset)
    if args.task_name == 'atis_slot':
        batchify_fn = lambda samples, fn=Tuple(
            Pad(axis=0, pad_val=0),  # input
            Pad(axis=0, pad_val=0),  # mask
            Pad(axis=0, pad_val=0),  # segment
            Pad(axis=0, pad_val=0, dtype='int64')  # label
        ): fn(samples)
    else:
        batchify_fn = lambda samples, fn=Tuple(
            Pad(axis=0, pad_val=0),  # input
            Pad(axis=0, pad_val=0),  # mask
            Pad(axis=0, pad_val=0),  # segment
            Stack(dtype='int64')  # label
        ): fn(samples)
    for idx, example in enumerate(dataset):
        if idx % 1000 == 0:
            print("Reading example %d of %d" % (idx, dataset_len))
        data_example = dataset_class.convert_example(example=example, \
                tokenizer=tokenizer, max_seq_length=max_seq_length)
        applid_data.append(data_example)

    applid_data = batchify_fn(applid_data)
    input_ids, input_mask, segment_ids, label_ids = applid_data

    for idx in range(dataset_len):
        if idx % 1000 == 0:
            print("Processing example %d of %d" % (idx, dataset_len))
        sample = {
            "input_ids": np.array(input_ids[idx], dtype=np.int64),
            "input_mask": np.array(input_mask[idx], dtype=np.int64),
            "segment_ids": np.array(segment_ids[idx], dtype=np.int64),
            "label_ids": np.array([label_ids[idx]], dtype=np.int64),
        }
        datalist.append(sample)

    print(task_name + " " + mode + " data process end")
    writer = FileWriter(file_name=MINDRECORD_FILE_PATH, shard_num=1)
    nlp_schema = {
        "input_ids": {"type": "int64", "shape": [-1]},
        "input_mask": {"type": "int64", "shape": [-1]},
        "segment_ids": {"type": "int64", "shape": [-1]},
        "label_ids": {"type": "int64", "shape": [-1]},
    }
    writer.add_schema(nlp_schema, "proprocessed classification dataset")
    writer.write_raw_data(datalist)
    writer.commit()
    print("write success")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="run classifier")
    parser.add_argument(
        "--task_name",
        default=None,
        type=str,
        required=True,
        help="The name of the task to train.")
    parser.add_argument(
        "--data_dir",
        default=None,
        type=str,
        help="The directory where the dataset will be load.")
    parser.add_argument(
        "--vocab_file_dir",
        default=None,
        type=str,
        help="The directory where the vocab will be load.")
    parser.add_argument(
        "--output_dir",
        default=None,
        type=str,
        help="The directory where the mindrecord dataset file will be save.")
    parser.add_argument(
        "--max_seq_len",
        default=128,
        type=int,
        help="The maximum total input sequence length after tokenization for trainng. ")
    parser.add_argument(
        "--eval_max_seq_len",
        default=None,
        type=int,
        help="The maximum total input sequence length after tokenization for trainng. ")

    args = parser.parse_args()
    if args.eval_max_seq_len is None:
        args.eval_max_seq_len = args.max_seq_len
    data_save_to_file(data_file_path=args.data_dir, vocab_file_path=args.vocab_file_dir, output_path=args.output_dir, \
            task_name=args.task_name, mode="train", max_seq_length=args.max_seq_len)
    data_save_to_file(data_file_path=args.data_dir, vocab_file_path=args.vocab_file_dir, output_path=args.output_dir, \
            task_name=args.task_name, mode="dev", max_seq_length=args.eval_max_seq_len)
    data_save_to_file(data_file_path=args.data_dir, vocab_file_path=args.vocab_file_dir, output_path=args.output_dir, \
            task_name=args.task_name, mode="test", max_seq_length=args.eval_max_seq_len)

max_stars_repo_name: leelige/mindspore

max_issues_repo_name: leelige/mindspore

max_forks_repo_name: leelige/mindspore


text:
import numpy as np
import pandas as pd

import spanda.sql.functions as F
from spanda.core.typing import *

class _SpandaWindowType:
    NONE            = 0
    PARTITION_BY    = 1
    ORDER_BY        = 2
    ROWS_BETWEEN    = 3
    RANGE_BETWEEN   = 4

class SpandaWindowSpec:
    """
    A window specification that defines the partitioning, ordering,
    and frame boundaries.

    Use the static methods in :class:`Window` to create a :class:`WindowSpec`.
    """

    def __init__(self, name: str, op: Callable, window_type: Union[Tuple[_SpandaWindowType], _SpandaWindowType],
                 _orderby_col: Optional[F.Column] = None):
        self._name = name
        self._op = op
        self._orderby_col = _orderby_col
        if isinstance(window_type, tuple):
            self._window_types = window_type
        else:
            self._window_types = (window_type,) if window_type != _SpandaWindowType.NONE else tuple()

    def _get_group_data(self, df: pd.DataFrame) -> Tuple[dict]:
        assert len(self._window_types) != 0, "cannot apply window function over empty Window"
        row2grp, grp2rows = self._op(df)
        return row2grp, grp2rows

    def _get_orderby_col(self):
        assert self._orderby_col is not None, "this window operation requires to use .orderBy()"
        return self._orderby_col

    def partitionBy(self, *cols: str) -> 'SpandaWindowSpec':
        """
         Defines the partitioning columns in a :class:`WindowSpec`.
         """

        assert len(self._window_types) == 0, ".partitionBy() must be used only after Window"

        def f(df):
            grp2rows = df.groupby(*cols).groups
            row2grp = {}
            for grp in grp2rows:
                for row in grp2rows[grp]:
                    row2grp[row] = grp
            return row2grp, grp2rows

        window_types = self._window_types + (_SpandaWindowType.PARTITION_BY,)
        return SpandaWindowSpec(self._name + f" PARTITION BY {', '.join([F.Column.getName(col) for col in cols])}",
                                lambda df: f(self._op(df)), window_types)

    def rowsBetween(self, start: int, end: int) -> 'SpandaWindowSpec':
        """
        Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).

        Both `start` and `end` are relative positions from the current row.
        For example, "0" means "current row", while "-1" means the row before
        the current row, and "5" means the fifth row after the current row.

        We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
        and ``Window.currentRow`` to specify special boundary values, rather than using integral
        values directly.

        :param start: boundary start, inclusive.
                      The frame is unbounded if this is ``Window.unboundedPreceding``.
        :param end: boundary end, inclusive.
                    The frame is unbounded if this is ``Window.unboundedFollowing``.
        """
        assert len(self._window_types) == 0, ".rowsBetween() must be used only after Window"

        def f(df):
            row2grp = {idx: idx for idx in df.index}
            grp2rows = {}
            for i in range(len(df)):
                l = max(0, i+start)
                r = min(len(df), i+end+1)
                rows = df.iloc[l:r].index
                idx = df.iloc[i].name
                grp2rows[idx] = rows
            return row2grp, grp2rows

        window_types = self._window_types + (_SpandaWindowType.ROWS_BETWEEN,)
        return SpandaWindowSpec(self._name + f" ROWS BETWEEN ({start}, {end})", lambda df: f(self._op(df)),
                                window_types)

    def orderBy(self, *cols: str, ascending: bool = True) -> 'SpandaWindowSpec':
        """
        Defines the ordering columns in a :class:`WindowSpec`.
        """

        assert set(self._window_types).issubset({_SpandaWindowType.PARTITION_BY,
                                                 _SpandaWindowType.ROWS_BETWEEN}), \
            ".orderBy() can be used only after .partitionBy() or .rowsBetween()"
        if len(self._window_types) > 0:
            def f(group_data, df):
                row2grp, grp2rows = group_data

                def _idx_to_key(row_idx):
                    row = df.loc[row_idx]
                    return tuple([row[c] for c in cols])
                grp2rows = {grp_name: sorted(rows, key=_idx_to_key, reverse=not ascending)
                            for grp_name, rows in grp2rows.items()}
                return row2grp, grp2rows

            window_types = self._window_types + (_SpandaWindowType.ORDER_BY,)
            return SpandaWindowSpec(self._name + f" ORDER BY ({', '.join([F.Column.getName(col) for col in cols])})",
                                    lambda df: f(self._op(df), df),
                                    window_types, _orderby_col=F.struct(*map(F.col, cols)))
        else:
            raise NotImplementedError

    def rangeBetween(self, start: int, end: int) -> 'SpandaWindowSpec':
        """
        Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).

        Both `start` and `end` are relative from the current row. For example,
        "0" means "current row", while "-1" means one off before the current row,
        and "5" means the five off after the current row.

        We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
        and ``Window.currentRow`` to specify special boundary values, rather than using integral
        values directly.

        :param start: boundary start, inclusive.
                      The frame is unbounded if this is ``Window.unboundedPreceding``.
        :param end: boundary end, inclusive.
                    The frame is unbounded if this is ``Window.unboundedFollowing``.
        """
        raise NotImplementedError

class Window:
    unboundedPreceding = -np.inf
    currentRow = 0
    unboundedFollowing = np.inf

    @staticmethod
    def _init():
        return SpandaWindowSpec(name="", op=lambda df: df, window_type=_SpandaWindowType.NONE)

    @staticmethod
    def partitionBy(*cols: str) -> SpandaWindowSpec:
        """
        Creates a `WindowSpec` with the partitioning defined.
        """
        return Window._init().partitionBy(*cols)

    @staticmethod
    def orderBy(*cols: str) -> SpandaWindowSpec:
        """
        Creates a `WindowSpec` with the ordering defined.
        """
        return Window._init().orderBy(*cols)

    @staticmethod
    def rangeBetween(start: int, end: int) -> SpandaWindowSpec:
        """
        Creates a `WindowSpec` with the frame boundaries defined,
        from `start` (inclusive) to `end` (inclusive).

        Both `start` and `end` are relative from the current row. For example,
        "0" means "current row", while "-1" means one off before the current row,
        and "5" means the five off after the current row.

        We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
        and ``Window.currentRow`` to specify special boundary values, rather than using integral
        values directly.

        :param start: boundary start, inclusive.
                      The frame is unbounded if this is ``Window.unboundedPreceding``.
        :param end: boundary end, inclusive.
                    The frame is unbounded if this is ``Window.unboundedFollowing``.
        """
        return Window._init().rangeBetween(start, end)

    @staticmethod
    def rowsBetween(start: int, end: int) -> SpandaWindowSpec:
        """
        Creates a `WindowSpec` with the frame boundaries defined,
        from `start` (inclusive) to `end` (inclusive).

        Both `start` and `end` are relative positions from the current row.
        For example, "0" means "current row", while "-1" means the row before
        the current row, and "5" means the fifth row after the current row.

        We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
        and ``Window.currentRow`` to specify special boundary values, rather than using integral
        values directly.

        :param start: boundary start, inclusive.
                      The frame is unbounded if this is ``Window.unboundedPreceding``
        :param end: boundary end, inclusive.
                    The frame is unbounded if this is ``Window.unboundedFollowing``
        """
        return Window._init().rowsBetween(start, end)


text:
# -*- coding: utf-8 -*-
# https://raw.githubusercontent.com/Jack-guo-xy/Python-IW-SSIM/master
# /IW_SSIM_PyTorch.py
import numpy as np
import pyrtools as pt
import torch
import torch.nn.functional as F
from PIL import Image

class IW_SSIM:
    def __init__(
        self, iw_flag=True, Nsc=5, blSzX=3, blSzY=3, parent=True, sigma_nsq=0.4, use_cuda=True, use_double=False
    ):
        # MS-SSIM parameters
        self.K = [0.01, 0.03]
        self.L = 255
        self.weight = [0.0448, 0.2856, 0.3001, 0.2363, 0.1333]
        self.winsize = 11
        self.sigma = 1.5

        # IW-SSIM parameters
        self.iw_flag = iw_flag
        self.Nsc = Nsc  # scales
        self.blSzX = blSzX  # Neighbor size
        self.blSzY = blSzY
        self.parent = parent
        self.sigma_nsq = sigma_nsq

        self.bound = np.ceil((self.winsize - 1) / 2)
        self.bound1 = self.bound - np.floor((self.blSzX - 1) / 2)
        self.use_cuda = use_cuda
        self.use_double = use_double

        self.samplet = torch.tensor([1.0])
        if self.use_cuda:
            self.samplet = self.samplet.cuda()
        if self.use_double:
            self.samplet = self.samplet.double()
        self.samplen = np.array([1.0])
        if not self.use_double:
            self.samplen = self.samplen.astype("float32")

    def fspecial(self, fltr, ws, **kwargs):
        if fltr == "uniform":
            return np.ones((ws, ws)) / ws ** 2

        elif fltr == "gaussian":
            x, y = np.mgrid[-ws // 2 + 1 : ws // 2 + 1, -ws // 2 + 1 : ws // 2 + 1]
            g = np.exp(-((x ** 2 + y ** 2) / (2.0 * kwargs["sigma"] ** 2)))
            g[g < np.finfo(g.dtype).eps * g.max()] = 0
            assert g.shape == (ws, ws)
            den = g.sum()
            if den != 0:
                g /= den
            return g

        return None

    def get_pyrd(self, imgo, imgd):
        imgopr = {}
        imgdpr = {}
        lpo = pt.pyramids.LaplacianPyramid(imgo, height=5)
        lpd = pt.pyramids.LaplacianPyramid(imgd, height=5)
        for scale in range(1, self.Nsc + 1):
            imgopr[scale] = (
                torch.from_numpy(lpo.pyr_coeffs[(scale - 1, 0)]).unsqueeze(0).unsqueeze(0).type(self.samplet.type())
            )
            imgdpr[scale] = (
                torch.from_numpy(lpd.pyr_coeffs[(scale - 1, 0)]).unsqueeze(0).unsqueeze(0).type(self.samplet.type())
            )

        return imgopr, imgdpr

    def scale_qualty_maps(self, imgopr, imgdpr):

        ms_win = self.fspecial("gaussian", ws=self.winsize, sigma=self.sigma)
        ms_win = torch.from_numpy(ms_win).unsqueeze(0).unsqueeze(0).type(self.samplet.type())
        C1 = (self.K[0] * self.L) ** 2
        C2 = (self.K[1] * self.L) ** 2
        cs_map = {}
        for i in range(1, self.Nsc + 1):
            imgo = imgopr[i]
            imgd = imgdpr[i]
            mu1 = F.conv2d(imgo, ms_win)
            mu2 = F.conv2d(imgd, ms_win)
            sigma12 = F.conv2d(imgo * imgd, ms_win) - mu1 * mu2
            sigma1_sq = F.conv2d(imgo ** 2, ms_win) - mu1 * mu1
            sigma2_sq = F.conv2d(imgd ** 2, ms_win) - mu2 * mu2
            sigma1_sq = torch.max(torch.zeros(sigma1_sq.shape).type(self.samplet.type()), sigma1_sq)
            sigma2_sq = torch.max(torch.zeros(sigma2_sq.shape).type(self.samplet.type()), sigma2_sq)
            cs_map[i] = (2 * sigma12 + C2) / (sigma1_sq + sigma2_sq + C2)
            if i == self.Nsc:
                l_map = (2 * mu1 * mu2 + C1) / (mu1 ** 2 + mu2 ** 2 + C1)

        return l_map, cs_map

    def roll(self, x, shift, dim):
        if dim == 0:
            return torch.cat((x[-shift:, :], x[:-shift, :]), dim)
        else:
            return torch.cat((x[:, -shift:], x[:, :-shift]), dim)

    def imenlarge2(self, im):
        _, _, M, N = im.shape
        t1 = F.interpolate(im, size=(int(4 * M - 3), int(4 * N - 3)), mode="bilinear", align_corners=False)
        t2 = torch.zeros([1, 1, 4 * M - 1, 4 * N - 1]).type(self.samplet.type())
        t2[:, :, 1:-1, 1:-1] = t1
        t2[:, :, 0, :] = 2 * t2[:, :, 1, :] - t2[:, :, 2, :]
        t2[:, :, -1, :] = 2 * t2[:, :, -2, :] - t2[:, :, -3, :]
        t2[:, :, :, 0] = 2 * t2[:, :, :, 1] - t2[:, :, :, 2]
        t2[:, :, :, -1] = 2 * t2[:, :, :, -2] - t2[:, :, :, -3]
        imu = t2[:, :, ::2, ::2]

        return imu

    def info_content_weight_map(self, imgopr, imgdpr):

        tol = 1e-15
        iw_map = {}
        for scale in range(1, self.Nsc):

            imgo = imgopr[scale]
            imgd = imgdpr[scale]
            win = np.ones([self.blSzX, self.blSzY])
            win = win / np.sum(win)
            win = torch.from_numpy(win).unsqueeze(0).unsqueeze(0).type(self.samplet.type())
            padding = int((self.blSzX - 1) / 2)

            # Prepare for estimating IW-SSIM parameters
            mean_x = F.conv2d(imgo, win, padding=padding)
            mean_y = F.conv2d(imgd, win, padding=padding)
            cov_xy = F.conv2d(imgo * imgd, win, padding=padding) - mean_x * mean_y
            ss_x = F.conv2d(imgo ** 2, win, padding=padding) - mean_x ** 2
            ss_y = F.conv2d(imgd ** 2, win, padding=padding) - mean_y ** 2

            ss_x[ss_x < 0] = 0
            ss_y[ss_y < 0] = 0

            # Estimate gain factor and error
            g = cov_xy / (ss_x + tol)
            vv = ss_y - g * cov_xy
            g[ss_x < tol] = 0
            vv[ss_x < tol] = ss_y[ss_x < tol]
            ss_x[ss_x < tol] = 0
            g[ss_y < tol] = 0
            vv[ss_y < tol] = 0

            # Prepare parent band
            aux = imgo
            _, _, Nsy, Nsx = aux.shape
            prnt = self.parent and scale < self.Nsc - 1
            BL = torch.zeros([1, 1, aux.shape[2], aux.shape[3], 1 + prnt])
            if self.use_cuda:
                BL = BL.cuda()
            if self.use_double:
                BL = BL.double()

            BL[:, :, :, :, 0] = aux
            if prnt:
                auxp = imgopr[scale + 1]
                auxp = self.imenlarge2(auxp)
                BL[:, :, :, :, 1] = auxp[:, :, 0:Nsy, 0:Nsx]
            imgo = BL
            _, _, nv, nh, nb = imgo.shape

            block = torch.tensor([win.shape[2], win.shape[3]])
            if self.use_cuda:
                block = block.cuda()

            # Group neighboring pixels
            nblv = nv - block[0] + 1
            nblh = nh - block[1] + 1
            nexp = nblv * nblh
            N = torch.prod(block) + prnt
            Ly = (block[0] - 1) // 2
            Lx = (block[1] - 1) // 2
            Y = torch.zeros([nexp, N]).type(self.samplet.type())

            n = -1
            for ny in range(-Ly, Ly + 1):
                for nx in range(-Lx, Lx + 1):
                    n = n + 1
                    temp = imgo[0, 0, :, :, 0]
                    foo1 = self.roll(temp, ny, 0)
                    foo = self.roll(foo1, nx, 1)
                    foo = foo[Ly : Ly + nblv, Lx : Lx + nblh]
                    Y[:, n] = foo.flatten()
            if prnt:
                n = n + 1
                temp = imgo[0, 0, :, :, 1]
                foo = temp
                foo = foo[Ly : Ly + nblv, Lx : Lx + nblh]
                Y[:, n] = foo.flatten()

            C_u = torch.mm(torch.transpose(Y, 0, 1), Y) / nexp.type(self.samplet.type())
            eig_values, H = torch.eig(C_u, eigenvectors=True)
            eig_values = eig_values.type(self.samplet.type())
            H = H.type(self.samplet.type())
            if self.use_double:
                L = (
                    torch.diag(eig_values[:, 0] * (eig_values[:, 0] > 0).double())
                    * torch.sum(eig_values)
                    / (
                        (torch.sum(eig_values[:, 0] * (eig_values[:, 0] > 0).double()))
                        + (torch.sum(eig_values[:, 0] * (eig_values[:, 0] > 0).double()) == 0)
                    )
                )
            else:
                L = (
                    torch.diag(eig_values[:, 0] * (eig_values[:, 0] > 0).float())
                    * torch.sum(eig_values)
                    / (
                        (torch.sum(eig_values[:, 0] * (eig_values[:, 0] > 0).float()))
                        + (torch.sum(eig_values[:, 0] * (eig_values[:, 0] > 0).float()) == 0)
                    )
                )
            C_u = torch.mm(torch.mm(H, L), torch.transpose(H, 0, 1))
            C_u_inv = torch.inverse(C_u)
            ss = (torch.mm(Y, C_u_inv)) * Y / N.type(self.samplet.type())
            ss = torch.sum(ss, 1)
            ss = ss.view(nblv, nblh)
            ss = ss.unsqueeze(0).unsqueeze(0)
            g = g[:, :, Ly : Ly + nblv, Lx : Lx + nblh]
            vv = vv[:, :, Ly : Ly + nblv, Lx : Lx + nblh]

            # Calculate mutual information
            infow = torch.zeros(g.shape).type(self.samplet.type())
            for j in range(len(eig_values)):
                infow = infow + torch.log2(
                    1
                    + ((vv + (1 + g * g) * self.sigma_nsq) * ss * eig_values[j, 0] + self.sigma_nsq * vv)
                    / (self.sigma_nsq * self.sigma_nsq)
                )
            infow[infow < tol] = 0
            iw_map[scale] = infow

        return iw_map

    def test_iw_ssim(self, gray_imgo, gray_imgd):
        gray_imgo = gray_imgo.astype(self.samplen.dtype)
        gray_imgd = gray_imgd.astype(self.samplen.dtype)
        imgopr, imgdpr = self.get_pyrd(gray_imgo, gray_imgd)
        l_map, cs_map = self.scale_qualty_maps(imgopr, imgdpr)
        if self.iw_flag:
            iw_map = self.info_content_weight_map(imgopr, imgdpr)

        wmcs = []
        for s in range(1, self.Nsc + 1):
            cs = cs_map[s]
            if s == self.Nsc:
                cs = cs_map[s] * l_map

            if self.iw_flag:
                if s < self.Nsc:
                    iw = iw_map[s]
                    if self.bound1 != 0:
                        iw = iw[:, :, int(self.bound1) : -int(self.bound1), int(self.bound1) : -int(self.bound1)]
                    else:
                        iw = iw[:, :, int(self.bound1) :, int(self.bound1) :]
                else:
                    iw = torch.ones(cs.shape).type(self.samplet.type())
                Image.fromarray((iw.squeeze().float().cpu().numpy() * 255).astype(np.uint8)).show()
                wmcs.append(torch.sum(cs * iw) / torch.sum(iw))
            else:
                wmcs.append(torch.mean(cs))

        wmcs = torch.tensor(wmcs).type(self.samplet.type())
        self.weight = torch.tensor(self.weight).type(self.samplet.type())
        score = torch.prod((torch.abs(wmcs)) ** (self.weight))

        return score

    def test_iw_sparse(self, gray_imgo, gray_imgd):
        gray_imgo = gray_imgo.astype(self.samplen.dtype)
        gray_imgd = gray_imgd.astype(self.samplen.dtype)
        imgopr, imgdpr = self.get_pyrd(gray_imgo, gray_imgd)
        l_map, cs_map = self.scale_qualty_maps(imgopr, imgdpr)
        if self.iw_flag:
            iw_map = self.info_content_weight_map(imgopr, imgdpr)

        wmcs = []
        for s in range(1, self.Nsc + 1):
            cs = cs_map[s]
            if s == self.Nsc:
                cs = cs_map[s] * l_map

            if self.iw_flag:
                if s < self.Nsc:
                    iw = iw_map[s]
                    if self.bound1 != 0:
                        iw = iw[:, :, int(self.bound1) : -int(self.bound1), int(self.bound1) : -int(self.bound1)]
                    else:
                        iw = iw[:, :, int(self.bound1) :, int(self.bound1) :]
                else:
                    iw = torch.ones(cs.shape).type(self.samplet.type())
                Image.fromarray((iw.squeeze().float().cpu().numpy() * 255).astype(np.uint8)).show()
                wmcs.append(torch.sum(cs * iw) / torch.sum(iw))
            else:
                wmcs.append(torch.mean(cs))

        wmcs = torch.tensor(wmcs).type(self.samplet.type())
        self.weight = torch.tensor(self.weight).type(self.samplet.type())
        score = torch.prod((torch.abs(wmcs)) ** (self.weight))

        return score

if __name__ == "__main__":
    pred = np.asarray(Image.open(""))
    gt = np.asarray(Image.open(""))
    iw_ssim = IW_SSIM()
    iw_score = iw_ssim.test(pred, gt)  # test one image
    print(iw_score)

max_stars_repo_name: lartpang/CoSaliencyProj

max_issues_repo_name: lartpang/CoSaliencyProj

max_forks_repo_name: lartpang/CoSaliencyProj


text:
import numpy as np
import scipy as sp

def quadratic_search(f, a, b, c, n):
    ya, yb, yc = f(a), f(b), f(c)
    for i in range(1, n - 3):
        x = 0.5 * (ya * (b ** 2 - c ** 2) + yb * (c ** 2 - a ** 2) + yc * (a ** 2 - b ** 2)) / (
                    ya * (b - c) + yb * (c - a)
                    + yc * (a - b))
        yx = f(x)
        if x > b:
            if yx > yb:
                c, yc = x, yx
            else:
                a, ya, b, yb = b, yb, x, yx
        elif x < b:
            if yx > yb:
                a, ya = x, yx
            else:
                c, yc, b, yb = b, yb, x, yx

    return a, b, c

if __name__ == '__main__':
    """
    Bessel Function is an oscillation curve with decay.

    :param np.cos(x) / x**0.5 -> Bessel Half Function
    :param sp.jv -> Bessel Function
    """
    funcs = {
        1: lambda x: (-x ** 2 / (x ** 3 + np.cos(x))),
        2: lambda x: (x**4 - x**3 - x**2) * np.exp(-x**2),
        3: lambda x: np.sin(x) / x**0.5,
        4: lambda x: np.cos(x) / x**0.5,
        5: lambda x: (x + 1)*((x - 3)**2)*((x + 7)**(-2)),
        6: lambda x: (x - 2)*((x - 3)**2)*((x - 5)**2),
        7: lambda x: 1 - np.exp(-(x - 3)**2),
    }

    a1, a2, a3 = quadratic_search(lambda x: (-x ** 2 / (x ** 3 + np.cos(x))), 0.5, 2, 4, 10)
    print(a1, a2, a3)

max_stars_repo_path: Work Sheet 3/script.py

max_issues_repo_path: Work Sheet 3/script.py

max_forks_repo_path: Work Sheet 3/script.py

reason: import numpy,import scipy


text:
# imports
import tensorflow as tf
import pandas as pd
import numpy as np

class Pipeline:
    """
    Inputs: dataframe of COCO-format annotations, including mandatory columns
    ['image_path', 'width', 'height'] and optional columns
    ['bbox', 'category', 'supercategory', 'attribute'].

    Outputs:
    - TF dataset object with labelled values ['image_id', 'image', 'bbox',
        'category', 'supercategory', 'attribute'].
    - Labels are provided as ragged tensors to allow for variable numbers of objects per image.
    - Bounding boxes are normalized as floats in [0,1] in same format as they arrived from dataframe
    - Image tensors can be either decoded into tensor or left serialized as jpg/png/gif
    - Image tensors have float values in [0,1] and are resized to uniform shape (user defined)
    - Image tensors can (optionally) have random augmentations dynamically applied
    - If any of ['bbox', 'category', 'supercategory', 'attribute'] are missing from the
        dataframe, dummy values are output for the missing info.
    """

    def __init__(self, image_size, **kwargs):

        self.target_size = image_size
        self.TRANSLATION_TABLE = str.maketrans({'[':None, ']':None, ',':None, "'":None, '"':None, })

        # key labels
        self.image_path_key = 'image_path'
        self.image_key = 'image'
        self.bbox_key = 'bbox'
        self.category_key= 'category'
        self.attribute_key = 'attribute'
        self.num_objects_key = 'num_objects'
        self.image_id_key = 'image_id'
        self.width = 'width'
        self.height = 'height'

    # Helper functions
    # Key names
    def remove_keys(self, val):
        outputs = val[self.image_key], val[self.category_key], val[self.attribute_key], \
                    val[self.bbox_key], val[self.num_objects_key], val[self.image_id_key], \
                    val[self.width], val[self.height]
        return outputs

    def without_keys(self, ds):
        return ds.map(self.remove_keys)

    # ## image loaders
    def load_image(self, image_path):
        image_path = tf.squeeze(image_path)
        image = tf.keras.layers.Lambda(lambda x: tf.io.read_file(x))(image_path)
        return image

    def decode_one_image(self, image):
        target_size=self.target_size

        image = tf.keras.layers.Lambda(lambda x: tf.io.decode_image(x, channels=3, expand_animations=False))(image)
        image = tf.keras.layers.experimental.preprocessing.Resizing(*target_size)(image)
        image = tf.cast(image, tf.float32) / 255.0
        return image

    def COCO_to_ymin_xmin_ymax_xmax(self, boxes):
        return np.concatenate([boxes[:, 1:2],
                               boxes[:, 0:1],
                               boxes[:, 1:2] + boxes[:, 3:4],
                               boxes[:, 0:1] + boxes[:, 2:3]], axis=-1)

    def ymin_xmin_ymax_xmax_to_COCO(self, boxes):
        return np.concatenate([boxes[:, 0:1],
                               boxes[:, 1:2],
                               boxes[:, 2:3] - boxes[:, 0:1],
                               boxes[:, 3:4] - boxes[:, 1:2]], axis=-1)

    def COCO_to_VOC(self, boxes):
        return np.concatenate([boxes[:, 0:1],
                               boxes[:, 1:2],
                               boxes[:, 0:1] + boxes[:, 2:3],
                               boxes[:, 1:2] + boxes[:, 3:4]], axis=-1)

    def VOC_to_COCO(self, boxes):
        return np.concatenate([boxes[:, 0:1],
                               boxes[:, 1:2],
                               boxes[:, 2:3] - boxes[:, 0:1],
                               boxes[:, 3:4] - boxes[:, 1:2]], axis=-1)

    # TF Dataset Creation
    def data_generator(self, labels_df, decode_images=True, stream_from_directory=False,
                       box_pad_val=-10.0, string_pad_val='<PAD>'):

        """
        Converts image data into a Tensorflow Dataset.
        Inputs:
        parameters: instance of hyper_parameters.Parameters() class
        'stream_from_directory' argument has no effect when labels_df is passed.

        Training datasets creation:
        'labels_df' is required and must include columns 'image_path', and 'y_true'.

        Inference datasets creation:
        Pass either a label_df ('image_path' column required) or provide the
        image directory path string in 'stream_from_directory=image_directory'
        """

        # get params
        target_size = self.target_size

        # Initial ds creation
        # Option 1: get info from dataframe
        if labels_df is not None:

            # shuffle
            df = labels_df.sample(frac=1)
            num_samples = len(df)

            # Create image path ds
            ds_image_path = tf.data.Dataset.from_tensor_slices(df['image_path'])

            # Create image id ds
            ds_image_id = tf.data.Dataset.from_tensor_slices(df['id_num'])

            # Create image shape ds
            ds_height = tf.data.Dataset.from_tensor_slices(df['height'])
            ds_width = tf.data.Dataset.from_tensor_slices(df['width'])

            # Create num_boxes ds
            if 'num_boxes' in df.columns:
                max_num_obj = df['num_boxes'].max()
                num_boxes = df['num_boxes']
            else:
                max_num_obj = 1
                num_boxes = df['image_path'].apply(lambda x: 0)
            ds_num_objects = tf.data.Dataset.from_tensor_slices(num_boxes)

            # Create categories ds
            if 'category' in df.columns:
                category = df['category']
            else:
                category = df['image_path'].apply(lambda x: [[string_pad_val]])

            # create uniform shape tensors
            category = tf.ragged.constant(category.to_list())
            category = category.to_tensor(shape=[num_samples, max_num_obj, 1],
                                            default_value=string_pad_val)

            ds_category = tf.data.Dataset.from_tensor_slices(category)

            # Create attributes ds
            if 'attribute' in df.columns:
                attribute = df['attribute']
            else:
                attribute = df['image_path'].apply(lambda x: [[string_pad_val]])

            # create uniform shape tensors
            attribute = tf.ragged.constant(attribute.to_list())
            max_attributes = attribute.bounding_shape()[-1]
            attribute = attribute.to_tensor(shape=[num_samples, max_num_obj, max_attributes],
                                            default_value=string_pad_val)
            ds_attribute = tf.data.Dataset.from_tensor_slices(attribute)

            # Create bbox ds
            if 'bbox' in df.columns:
                bbox = df['bbox']
            else:
                bbox = df['image_path'].apply(lambda x: [[box_pad_val, box_pad_val, box_pad_val, box_pad_val]])

            # ##create uniform shape tensors
            bbox = tf.ragged.constant(df['bbox'].to_list())
            bbox = bbox.to_tensor(shape=[num_samples, max_num_obj, 4],
                                            default_value=box_pad_val)

            ds_bbox = tf.data.Dataset.from_tensor_slices(bbox)

            # Merge into a single dataset
            ds = tf.data.Dataset.zip((ds_image_id, ds_image_path, ds_num_objects,
                                ds_category, ds_attribute, ds_bbox, ds_width, ds_height))

            def apply_names(image_id, image_path, num_objects, category, attribute, bbox, width, height):
                return {'image_id':image_id, 'image_path':image_path,
                        'num_objects':num_objects, 'category':category,
                        'attribute':attribute, 'bbox':bbox, 'width':width, 'height':height}

            ds = ds.map(apply_names)

            # shuffle
            ds = ds.shuffle(50000)

            # decode images into array. (Note: uses tf.io, which requires unbatched images)
            def load(val):
                image_path = val[self.image_path_key]
                val[self.image_key] = self.load_image(image_path)
                del val[self.image_path_key]
                return val

            ds = ds.map(load, num_parallel_calls=tf.data.AUTOTUNE)

            if decode_images:
                def decoder(val):
                    image = val[self.image_key]
                    val[self.image_key] = self.decode_one_image(image)
                    return val

                ds = ds.map(decoder, num_parallel_calls=tf.data.AUTOTUNE)

        # Option 2: get info from image file directory
        else:
            image_ds = tf.keras.preprocessing.image_dataset_from_directory(
                directory=stream_from_directory, labels='inferred', label_mode=None,
                class_names=None, color_mode='rgb', batch_size=1,
                image_size=target_size, shuffle=False, seed=None, validation_split=None,
                subset=None, follow_links=False)

            # unbatch (needed for later ops)
            image_ds = image_ds.unbatch()

            # set filenames as image id and batch results
            image_id_ds = tf.data.Dataset.from_tensor_slices(image_ds.file_paths)
            image_id_ds = image_id_ds.map(lambda x: tf.strings.split(x, os.path.sep)[-1],
                                            num_parallel_calls=tf.data.AUTOTUNE)

            if decode_images is False:
                # convert image to raw byte string. Note: cannot have batch dim for encoding
                image_ds = image_ds.unbatch()
                image_ds = image_ds.map(lambda x: tf.cast(x, dtype=tf.uint16))
                image_ds = image_ds.map(lambda image: tf.io.encode_png(image))
                image_ds = image_ds.map(lambda image: tf.io.serialize_tensor(image))

            # create  placeholder for (unknown) labels
            placeholder_num_obj_ds = image_id_ds.map(lambda x: -1, num_parallel_calls=tf.data.AUTOTUNE)
            placeholder_box_ds = image_id_ds.map(lambda x: [[box_pad_val, box_pad_val, box_pad_val, box_pad_val]], num_parallel_calls=tf.data.AUTOTUNE)
            placeholder_string_ds = image_id_ds.map(lambda x: [[string_pad_val]], num_parallel_calls=tf.data.AUTOTUNE)

            # merge into single dataset
            ds = tf.data.Dataset.zip((image_ds, image_id_ds, placeholder_string_ds,
                    placeholder_string_ds, placeholder_box_ds, placeholder_num_obj_ds))

            # name the elements for interpretability
            def map_names(image, image_id, a, b, c, d):
                return  {self.image_key: image,
                         self.image_id_key: image_id,
                         self.category_key: a,
                         self.attribute_key: b,
                         self.bbox_key: c,
                         self.num_objects_key: d}

            ds = ds.map(map_names, num_parallel_calls=tf.data.AUTOTUNE)

        # display summary
        print(ds.element_spec, '\n')

        return ds

# Image Augmentations for Training Data
class Augmentations(Pipeline):
    def __init__(self):
        super().__init__(image_size=None)

        """  inherited values
        self.image_path_key = 'image_path'
        self.image_key = 'image'
        self.bbox_key = 'bbox'
        self.category_key= 'category'
        self.attribute_key = 'attribute'
        self.num_objects_key = 'num_objects'
        self.image_id_key = 'image_id'
        """

    def random_downsizer_with_pad(self, image, bbox):
        """ Randomly shrinks image (doesn't preserve aspect ratio), shifts it
         and pads back to original size with whitespace. Bounding box values
         are adjusted accordingly. """

        # choose random downsizing factors between 1 and ~3
        # choice is heavily weighted towards 1 (i.e. no change in size)
        rand_val = tf.math.maximum(1.0, tf.random.truncated_normal(
                        shape=([2]), mean=0.5, stddev=0.7))

        # get shapes
        orig_shape = tf.shape(image)[-3:-1]
        orig_shape_float = tf.cast(orig_shape, dtype=rand_val.dtype)

        # downsize, shift and pad image
        # ## resize
        new_shape = orig_shape_float / rand_val
        new_shape = tf.cast(new_shape, dtype=orig_shape.dtype)
        image = tf.image.resize(image, size=new_shape)

        # ## random shift values
        offset_height = tf.random.uniform(minval=0, maxval= orig_shape[0] - new_shape[0] + 1,
                                          shape=[], dtype=orig_shape.dtype)
        offset_width = tf.random.uniform(minval=0, maxval= orig_shape[1] - new_shape[1] + 1,
                                         shape=[], dtype=orig_shape.dtype)

        # ## pad back to orig size with whitespace
        image = tf.image.pad_to_bounding_box(image, offset_height, offset_width,
                                            target_height=orig_shape[0], target_width=orig_shape[1])

        # downsize and shift bbox
        # ## downscale box
        denom =  [tf.concat([rand_val, rand_val], axis=-1)]
        bbox = bbox / denom

        # ## normalize shift values
        offset_height = tf.cast(offset_height / orig_shape[0], dtype=bbox.dtype)
        offset_width = tf.cast(offset_width / orig_shape[1], dtype=bbox.dtype)

        # ## shift boxes
        shift = tf.concat([[[offset_height]], [[offset_width]], [[offset_height]], [[offset_width]]], axis=-1)
        bbox = bbox + shift

        return image, bbox

    ## random contrast
    def random_contrast(self, image, lower=.8, upper=1.2):
        image = tf.image.random_contrast(image, lower=lower, upper=upper)
        return image

    ## random brightness
    def random_brightness(self, image, max_delta=.1):
        image = tf.image.random_brightness(image, max_delta=max_delta)
        return image

    ## random image quality
    def random_quality(self, image, min_quality=70, max_quality=100):
        image_shape = tf.shape(image)
        image = tf.image.random_jpeg_quality(image, min_jpeg_quality=min_quality,
                                             max_jpeg_quality=max_quality)
        image = tf.reshape(image, [image_shape[0], image_shape[1], image_shape[2]])
        return image

    ## random saturation
    def random_saturation(self, image, min_saturation=.8, max_saturation=1.2):
        image = tf.image.random_saturation(image, lower=min_saturation, upper=max_saturation)
        return image

    # Combined Image Augmentation path
    def apply_image_augmentations(self, dataset):
        # NOTE: dataset elements arrive nbatched as dict with 'image' in keys

        # random_downsizer_with_pad.
        def temp_mapper(val):
            image = val[self.image_key]
            bbox = val[self.bbox_key]
            val[self.image_key], val[self.bbox_key] = self.random_downsizer_with_pad(image, bbox)
            return val

        dataset = dataset.map(temp_mapper, num_parallel_calls=tf.data.AUTOTUNE)

        # random_contrast
        def temp_mapper(val):
            image = val[self.image_key]
            val[self.image_key] = self.random_contrast(image)
            return val
        dataset = dataset.map(temp_mapper, num_parallel_calls=tf.data.AUTOTUNE)

        # random_brightness
        def temp_mapper(val):
            image = val[self.image_key]
            val[self.image_key] = self.random_brightness(image)
            return val
        dataset = dataset.map(temp_mapper, num_parallel_calls=tf.data.AUTOTUNE)

        # random_quality
        def temp_mapper(val):
            image = val[self.image_key]
            val[self.image_key] = self.random_quality(image)
            return val
        dataset = dataset.map(temp_mapper, num_parallel_calls=tf.data.AUTOTUNE)

        # random_saturation
        def temp_mapper(val):
            image = val[self.image_key]
            val[self.image_key] = self.random_saturation(image)
            return val
        dataset = dataset.map(temp_mapper, num_parallel_calls=tf.data.AUTOTUNE)

        return dataset

class TFRecordsConversions(Pipeline):
    def __init__(self, image_size, **kwargs):
        super().__init__(image_size)
        self.image_size = image_size

        """  inherited values
        self.image_path_key = 'image_path'
        self.image_key = 'image'
        self.bbox_key = 'bbox'
        self.category_key= 'category'
        self.attribute_key = 'attribute'
        self.num_objects_key = 'num_objects'
        self.image_id_key = 'image_id'
        """

    # TF Records Conversion
    # NOTE: Requires unbatched ds with raw images (i.e. images have not been decoded)
    def num_shards_needed(self, num_files, files_per_shard):
        num_shards = num_files // files_per_shard
        if  num_files % files_per_shard > 0:
            num_shards = num_shards + 1

        return num_shards

    # ## Conversion: Tensor - > Example
    def _bytes_feature(self, x):
        """Returns a bytes_list from a flat string / byte."""
        if isinstance(x, type(tf.constant(0))):
            x = tf.train.BytesList(value=[x.numpy()])
        return tf.train.Feature(bytes_list=x)

    def _float_feature(self, x):
        """ Returns a float_list from a flat float / double."""
        return tf.train.Feature(float_list=tf.train.FloatList(value=[x]))

    def _int64_feature(self, x):
        """ Returns an int64_list from a flat bool / enum / int / uint."""
        return tf.train.Feature(int64_list=tf.train.Int64List(value=[x]))

    def _non_flat_feature(self, x):
        """ Returns a bytes_list from a multidim tensor."""
        x = tf.train.BytesList(value=[tf.io.serialize_tensor(x).numpy()])
        return tf.train.Feature(bytes_list=x)

    # converts single ds element to TF Example
    def serialize_example(self, bbox, attribute, category, num_objects, image_id, image):

        feature = {self.bbox_key: self._non_flat_feature(bbox),
                   self.attribute_key: self._non_flat_feature(attribute),
                   self.category_key: self._non_flat_feature(category),
                   self.num_objects_key: self._int64_feature(num_objects),
                   self.image_id_key: self._int64_feature(image_id),
                   self.image_key: self._bytes_feature(image)
            }

        feature = tf.train.Features(feature=feature)
        example_proto = tf.train.Example(features=feature)

        return example_proto.SerializeToString()

    def remove_labels(self, val):
        bbox = val[self.bbox_key]
        attribute = val[self.attribute_key]
        category = val[self.category_key]
        num_objects = val[self.num_objects_key]
        image_id = val[self.image_id_key]
        image = val[self.image_key]

        return bbox, attribute, category, num_objects, image_id, image

    def tf_serialize_example(self, *x):
        tf_string = tf.py_function(self.serialize_example, [*x], tf.string)
        return tf.reshape(tf_string, ()) # The result is a scalar.

    # converts tensor Dataset -> TF Record Files
    def write_TFRecords(self, ds, num_files, files_per_shard, name_qualifier=None):

        ds = ds.map(self.remove_labels)
        ds = ds.map(self.tf_serialize_example)

        elements_remaining = True
        file_number = 0

        while elements_remaining:

            # create dataset segment for next record
            try:
                ds_i = ds.skip(file_number*files_per_shard).take(files_per_shard)
            except:
                elements_remaining = False
                break

            # write next record
            filename = f'{name_qualifier}_data_{file_number}.tfrecord'
            writer = tf.data.experimental.TFRecordWriter(filename)
            writer.write(ds_i)

            file_number = file_number + 1

            if tf.math.greater_equal(file_number, num_files // files_per_shard):
                elements_remaining = False
                break

        print(f'Completed. {file_number} files written.')
        return None

    # ## Conversion: Example -> Tensor
    def decode_one_image(self, image):
        target_size = self.image_size

        image = tf.keras.layers.Lambda(lambda x: tf.io.decode_image(x, channels=3,
                                                 expand_animations=False))(image)
        image = tf.keras.layers.experimental.preprocessing.Resizing(*target_size)(image)
        image = tf.cast(image, tf.float32) / 255.0
        return image

    def decode_one_float(self, x):
        x = tf.io.parse_tensor(x, out_type=tf.float32)
        return x

    def decode_one_string(self, x):
        x = tf.io.parse_tensor(x, out_type=tf.string)
        return x

    def parse_one_example(self, example):
        # convert to feature
        feature_description = {
            self.bbox_key: tf.io.FixedLenFeature([], tf.string, default_value=''),
            self.attribute_key: tf.io.FixedLenFeature([], tf.string, default_value=''),
            self.category_key: tf.io.FixedLenFeature([], tf.string, default_value=''),
            self.image_id_key: tf.io.FixedLenFeature([], tf.int64, default_value=0),
            self.num_objects_key: tf.io.FixedLenFeature([], tf.int64, default_value=0),
            self.image_key: tf.io.FixedLenFeature([], tf.string, default_value=''),
        }

        feature = tf.io.parse_single_example(example, feature_description)

        # convert bytestrings back to ragged tensors
        feature[self.bbox_key] = self.decode_one_float(feature[self.bbox_key])
        feature[self.attribute_key] = self.decode_one_string(feature['attribute'])
        feature[self.category_key] = self.decode_one_string(feature[self.category_key])
        feature[self.image_key] = self.decode_one_image(feature[self.image_key])

        return feature